Mobile Convergence

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

  <modelVersion>4.0.0</modelVersion>

  <groupId>com.mobiconsoft</groupId>

  <artifactId>booksearch</artifactId>

  <version>0.0.1-SNAPSHOT</version>

  <dependencies>

  <dependency>

  <groupId>org.mongodb</groupId>

  <artifactId>mongo-java-driver</artifactId>

  <version>2.11.2</version>

  </dependency>

  <dependency>

  <groupId>junit</groupId>

  <artifactId>junit</artifactId>

  <version>4.10</version>

  </dependency>

  <dependency>

  <groupId>org.json</groupId>

  <artifactId>json</artifactId>

  <version>20090211</version>

  </dependency>

  </dependencies>

</project>

public class BookSearcher {

  // 도서검색 

  public String searchBooks(String keyword) {

    URL url = null;

    try {

      url = new URL("https://www.googleapis.com/books/v1/volumes?q=" + keyword);

    } catch (MalformedURLException e) {

      e.printStackTrace();

    }

    StringBuffer sb = new StringBuffer();

    String line;

    try {

      URLConnection urlConn = url.openConnection();

      BufferedReader br = new BufferedReader(new InputStreamReader(urlConn.getInputStream(), "utf-8"));

      while((line = br.readLine()) != null) sb.append(line);

    } catch(IOException e) {

      e.printStackTrace();

    }

    return sb.toString();

  }

  // 도서 검색 결과에서  items찾아와 저장한다 

  public void saveBooks(String books) {

    Mongo mongo = null;

    try{

      mongo = new MongoClient("localhost", 27017);

    } catch(Exception e) {

      e.printStackTrace();

      throw new RuntimeException();

    }

   // 몽고디비 db는 books-db 이고 컬렉션은 books 로 만들어짐 

    mongo.setWriteConcern(new WriteConcern(1, 2000));

    DB bookDB = mongo.getDB("books-db");

    DBCollection bookColl = bookDB.getCollection("books");

    try{

      JSONObject json = new JSONObject(books);

      JSONArray items = json.getJSONArray("items");

      for( int i=0; i<items.length(); i++) {

        DBObject doc = new BasicDBObject();

        // search-book key로 value가 들어간다

        doc.put("search-book", (DBObject)JSON.parse(items.getJSONObject(i).toString()));

        bookColl.save(doc);

      }

    } catch(JSONException e) {

      e.printStackTrace();

    }

  }

}

// 몽고디비 

$ ../bin/mongod -dbpath=/Users/dowon/Documents/mongodb/database

// 테스트 

public class BookSearcherTest {

    private BookSearcher bookSearcher;

    @Before

    public void setUp() throws Exception {

      this.bookSearcher = new BookSearcher();

    }

    // search 결과 보기 

    @Test

    public void testSearchBooks() throws Exception {

      String result = this.bookSearcher.searchBooks("nosql");

      //assertNotNull(result);

      System.out.println(result);

    }

    // 데이터 저장하기 

    @Test

    public void testSaveBooks() throws Exception {

      String result = this.bookSearcher.searchBooks("nosql");

      this.bookSearcher.saveBooks(result);

    }

}

// 테스트 성공후 mongo 쉘을 통하여 확인 

> use books-db

switched to db books-db

> show collections

books

system.indexes

> db.books.find().length();

10

> db.books.find()

{ "_id" : ObjectId("5232e69cda06561b2e11306c"), "search-book" : { "saleInfo" : { "saleability" : "NOT_FOR_SALE", "isEbook" : false, "country" : "KR" }, "id" : "tv5iO9MnObUC", "searchInfo" : { "textSnippet" : "They provide examples, practical solutions, and expert education in new technologies, all designed to help programmers do a better job. wrox.com Programmer Forums Join our Programmer to Programmer forums to ask and answer programming ..." }, "etag" : "HX8hesQgrJM", "volumeInfo" : { "pageCount" : 408, "averageRating" : 3, "infoLink" : "http://books.google.co.kr/books?id=tv5iO9MnObUC&dq=nosql&hl=&source=gbs_api", "printType" : "BOOK", "publisher" : "John Wiley & Sons", "authors" : [  "Shashank Tiwari" ], "canonicalVolumeLink" : "http://books.google.co.kr/books/about/Professional_NoSQL.html?hl=&id=tv5iO9MnObUC", "title" : "Professional NoSQL", "previewLink ... 중략 ...

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

  <modelVersion>4.0.0</modelVersion>

  <groupId>com.mobiconsoft</groupId>

  <artifactId>booksearch_mapreduce</artifactId>

  <version>0.0.1-SNAPSHOT</version>

  <dependencies>

    <dependency>

      <groupId>org.apache.hadoop</groupId>

      <artifactId>hadoop-core</artifactId>

      <version>1.1.2</version>

    </dependency>

    <dependency>

      <groupId>org.mongodb</groupId>

      <artifactId>mongo-java-driver</artifactId>

      <version>2.11.2</version>

    </dependency>

    <!-- 수동 설정 --> 

    <dependency>

      <groupId>mongo-hadoop-core</groupId>

      <artifactId>mongo-hadoop-core_1.1.2</artifactId>

      <version>1.1.0</version>

    </dependency>

  </dependencies>

  <build>

    <plugins>

      <plugin>

        <artifactId>maven-antrun-plugin</artifactId>

        <configuration>

          <tasks>

            <copy file="target/${project.artifactId}-${project.version}.jar"

              tofile="/Users/dowon/Documents/input/${project.artifactId}-${project.version}.jar" />

          </tasks>

        </configuration>

        <executions>

          <execution>

            <phase>install</phase>

            <goals>

              <goal>run</goal>

            </goals>

          </execution>

        </executions>

      </plugin>

    </plugins>

  </build>

</project>

// Mapper

public class BookSearchMapper extends Mapper<Object, BSONObject, Text, IntWritable> {

  private final static IntWritable ONE = new IntWritable();

  private Text word = new Text();

  protected void map(Object key, BSONObject value, Context context) 

    throws IOException, InterruptedException {

    BasicDBObject anItem = (BasicDBObject)value.get("search-book");

    BasicDBObject volumeInfo = (BasicDBObject)anItem.get("volumeInfo");

    String description = volumeInfo.getString("description");

    if(description == null || description.trim().length() <= 0) return;

    StringTokenizer st = new StringTokenizer(description);

    while(st.hasMoreTokens()) {

      word.set(st.nextToken());

      context.write(word, ONE);

    }

  }

}

// Reducer

public class BookSearcherReducer extends

  Reducer<Text, IntWritable, Text, IntWritable> {

  protected void reduce(Text key, Iterable<IntWritable> values, Context context) 

    throws IOException, InterruptedException {

    int sum = 0;

    for(final IntWritable value : values) sum += value.get();

    context.write(key, new IntWritable(sum));

  }

}

public class MongoJob extends MongoTool {

  static {

    Configuration.addDefaultResource("mongo-default.xml");

    Configuration.addDefaultResource("mongo-book.xml");

  }

  public static void main(String[] args) throws Exception {

    Configuration conf = new Configuration();

    JobHelper.addJarForJob(conf, "/Users/dowon/.m2/repository/mongo-hadoop-core/mongo-hadoop-core_1.1.2/1.1.0/mongo-hadoop-core_1.1.2-1.1.0.jar:"

          + "/Users/dowon/.m2/repository/org/mongodb/mongo-java-driver/2.11.2/mongo-java-driver-2.11.2.jar");

    System.exit(ToolRunner.run(conf, new MongoJob(), args));

  }

}

<property>

    <!-- Class for the mapper -->

    <name>mongo.job.mapper</name>

    <value>booksearch_mapreduce.BookSearchMapper</value>

  </property>

  <property>

    <!-- Reducer class -->

    <name>mongo.job.reducer</name>

    <value>booksearch_mapreduce.BookSearcherReducer</value>

  </property>

  <property>

    <!-- InputFormat Class -->

    <name>mongo.job.input.format</name>

    <value>com.mongodb.hadoop.MongoInputFormat</value>

  </property>

  <property>

    <!-- OutputFormat Class -->

    <name>mongo.job.output.format</name>

    <value>com.mongodb.hadoop.MongoOutputFormat</value>

  </property>

  <property>

    <!-- Output key class for the output format -->

    <name>mongo.job.output.key</name>

    <value>org.apache.hadoop.io.Text</value>

  </property>

  <property>

    <!-- Output value class for the output format -->

    <name>mongo.job.output.value</name>

    <value>com.mongodb.hadoop.io.BSONWritable</value>

  </property>

  <property>

    <!-- Output key class for the mapper [optional] -->

    <name>mongo.job.mapper.output.key</name>

    <value>org.apache.hadoop.io.Text</value>

  </property>

  <property>

    <!-- Output value class for the mapper [optional] -->

    <name>mongo.job.mapper.output.value</name>

    <value>org.apache.hadoop.io.IntWritable</value>

  </property>

  <property>

    <!-- Class for the combiner [optional] -->

    <name>mongo.job.combiner</name>

    <value>booksearch_mapreduce.BookSearcherReducer</value>

  </property>

//////////////////////

// mongodb.sh 내역 

#!/bin/sh

export REPO=/Users/dowon/.m2/repository

export MONGO_DRIVER=$REPO/org/mongodb/mongo-java-driver/2.11.2/mongo-java-driver-2.11.2.jar

export MONGO_HADOOP=$REPO/mongo-hadoop-core/mongo-hadoop-core_1.1.2/1.1.0/mongo-hadoop-core_1.1.2-1.1.0.jar

export HADOOP_CLASSPATH=$MONGO_DRIVER:$MONGO_HADOOP

export HADOOP_USER_CLASSPATH_FIRST=true

hadoop jar booksearch_mapreduce-0.0.1-SNAPSHOT.jar booksearch_mapreduce.MongoJob

/////////////

/// 수행하기 

$ mongodb.sh

2013-09-13 20:53:24.630 java[1431:1203] Unable to load realm info from SCDynamicStore

13/09/13 20:53:24 INFO util.MongoTool: Created a conf: 'Configuration: core-default.xml, core-site.xml, mongo-default.xml, mongo-book.xml, mapred-default.xml, mapred-site.xml' on {class booksearch_mapreduce.MongoJob} as job named '<unnamed MongoTool job>'

13/09/13 20:53:24 INFO util.MongoTool: Mapper Class: class booksearch_mapreduce.BookSearchMapper

13/09/13 20:53:24 INFO util.MongoTool: Setting up and running MapReduce job in foreground, will wait for results.  {Verbose? false}

13/09/13 20:53:25 INFO util.MongoSplitter: MongoSplitter calculating splits

13/09/13 20:53:25 INFO util.MongoSplitter: use range queries: false

.. 중략 ...

13/09/13 20:53:41 INFO mapred.JobClient:     Spilled Records=1428

13/09/13 20:53:41 INFO mapred.JobClient:     Map output bytes=14049

13/09/13 20:53:41 INFO mapred.JobClient:     Total committed heap usage (bytes)=269619200

13/09/13 20:53:41 INFO mapred.JobClient:     Combine input records=1299

13/09/13 20:53:41 INFO mapred.JobClient:     SPLIT_RAW_BYTES=195

13/09/13 20:53:41 INFO mapred.JobClient:     Reduce input records=714

13/09/13 20:53:41 INFO mapred.JobClient:     Reduce input groups=714

13/09/13 20:53:41 INFO mapred.JobClient:     Combine output records=714

13/09/13 20:53:41 INFO mapred.JobClient:     Reduce output records=714

13/09/13 20:53:41 INFO mapred.JobClient:     Map output records=1299

 $ ./bin/mongod -dbpath=/Users/dowon/Documents/mongodb/database --rest 

<build>

  <plugins>

    <plugin>

      <artifactId>maven-antrun-plugin</artifactId>

      <configuration>

        <tasks>

          <copy file="target/${project.artifactId}-${project.version}.jar"

            tofile="/Users/dowon/Documents/hadoop-jobs/${project.artifactId}-${project.version}.jar" />

        </tasks>

      </configuration>

      <executions>

        <execution>

          <phase>install</phase>

          <goals>

            <goal>run</goal>

          </goals>

        </execution>

      </executions>

    </plugin>

  </plugins>

</build>

import java.io.IOException;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapred.FileInputFormat;

import org.apache.hadoop.mapred.FileOutputFormat;

import org.apache.hadoop.mapred.JobClient;

import org.apache.hadoop.mapred.JobConf;

import org.apache.hadoop.mapred.lib.LongSumReducer;

import org.apache.hadoop.mapred.lib.TokenCountMapper;

public class WordCount3 {

  public static void main(String[] args) throws IOException {

    // 1. configuration Mapper & Reducer of Hadoop

    JobConf conf = new JobConf(WordCount3.class);

    conf.setJobName("wordcount3");

    // 2. final output key type & value type

    conf.setOutputKeyClass(Text.class);

    conf.setOutputValueClass(LongWritable.class);

    // 3. in/output format 

    conf.setMapperClass(TokenCountMapper.class);

    conf.setCombinerClass(LongSumReducer.class);

    conf.setReducerClass(LongSumReducer.class);

    // 4. set the path of file for read files

    //    input path : args[0]

    //    output path : args[1]

    FileInputFormat.setInputPaths(conf, new Path(args[0]));

    FileOutputFormat.setOutputPath(conf, new Path(args[1]));

    // 5. run job

    JobClient client = new JobClient();

    client.setConf(conf);

    JobClient.runJob(conf);

  }

}

// name node 포멧

$ hadoop namenode -format

// .bash_profile 에 PATH 설정

set -o vi

export JAVA_HOME=/Library/Java/Home

export H_HOME=~/Documents/hadoop-1.2.1

export PATH=.:$PATH:$JAVA_HOME/bin:$H_HOME/bin:/usr/bin

alias ll='ls -alrt'

alias cdh='cd $H_HOME'

// 하둡 데몬 수행

// 50030 : job-tracker 접속 포트

// 50070 : NameNode 접속 포트

$ start-all.sh 

// 위치가 하기와 같다면 

$ pwd

/Users/dowon/Documents/input

$ ls

total 16

-rw-r--r--   1 dowon  staff   22  9 11 19:26 file01

-rw-r--r--   1 dowon  staff   21  9 11 19:26 file02

// input 을 HDFS에 만든다 

$ hadoop fs -put . input

1) http://localhost:50070/으로 접속하여 "Browser the filesystem"을 클릭하면 볼 수 있다 

2) /user/dowon/input 경로로 만들어 졌음을 알 수 있다 

// 경로가 다음과 같고, WordCount3.class가 들어있는 .jar 파일이 존재한다 

$ pwd

/Users/dowon/Documents/hadoop-jobs

$ ls

-rw-r--r--   1 dowon  staff  5391  9 11 19:45 MapReduce-1.0.0-SNAPSHOT.jar

// 명령어 수행 

$ hadoop jar *.jar WordCount3 /user/dowon/input /user/dowon/output3

1) 네임노드에 접속해서 /user/dowon에 들어가 보면 "output3"이 생성된 것을 볼 수있다

2) output3으로 들어가면 결과값을 지니 파일이 존재한다 

3) 만일 명령을 재수행하고 싶다면 output3 디렉토리리 삭제해야 한다 

    $ hadoop fs -rmr /user/dowon/output3

// pom.xml 내역

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

  <modelVersion>4.0.0</modelVersion>

  <groupId>kr.mobiconsoft.hadoop</groupId>

  <artifactId>MapReduce</artifactId>

  <version>1.0.0-SNAPSHOT</version>

  <dependencies>

  <dependency>

  <groupId>org.apache.hadoop</groupId>

  <artifactId>hadoop-core</artifactId>

  <version>1.1.2</version>

         </dependency>

  </dependencies>

</project>

// 결과 

// file01

hello world bye world

// file02

hi world hello dowon

import java.io.IOException;

import java.util.StringTokenizer;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapred.MapReduceBase;

import org.apache.hadoop.mapred.Mapper;

import org.apache.hadoop.mapred.OutputCollector;

import org.apache.hadoop.mapred.Reporter;

/**

 * K1 : read key type

 * V2 : read value type

 * K2 : write key type

 * V2 : write value type

 */

//public class WordCountMapper implements Mapper<K1, V1, K2, V2> {

public class WordCountMapper extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {

// map 결과는 reducer로 자동으로 던져진다 

public void map(LongWritable key, Text value,

OutputCollector<Text, IntWritable> output, Reporter reporter)

throws IOException {

// TODO Auto-generated method stub

String line = value.toString();

StringTokenizer tokenizer = new StringTokenizer(line);

while(tokenizer.hasMoreTokens()) {

Text outputKey = new Text(tokenizer.nextToken());

// Hadoop 에서 wrapping한 Integer 타입의 객체를 넣어줌 

// param1: outputKey, param2: outputValue

output.collect(outputKey, new IntWritable(1));

}

}

}

import java.io.IOException;

import java.util.Iterator;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapred.MapReduceBase;

import org.apache.hadoop.mapred.OutputCollector;

import org.apache.hadoop.mapred.Reducer;

import org.apache.hadoop.mapred.Reporter;

/**

 * K1 : Mapper의 K2 와 동일

 * V1 : Mapper의 V2 와 동일 

 */

public class WordCountReducer extends MapReduceBase 

 implements Reducer<Text, IntWritable, Text, IntWritable> {

/**

* V1 에서 values는 Iterator이다. 실제 같은 단어가 여러개 일 경우 

*/

public void reduce(Text key, Iterator<IntWritable> values,

OutputCollector<Text, IntWritable> output, Reporter reporter)

throws IOException {

// TODO Auto-generated method stub

int sum = 0;

while(values.hasNext()) {

sum += values.next().get(); // get Integer value

}

output.collect(key, new IntWritable(sum));

}

}

import java.io.IOException;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapred.FileInputFormat;

import org.apache.hadoop.mapred.FileOutputFormat;

import org.apache.hadoop.mapred.JobClient;

import org.apache.hadoop.mapred.JobConf;

import org.apache.hadoop.mapred.TextInputFormat;

import org.apache.hadoop.mapred.TextOutputFormat;

public class WordCount {

public static void main(String[] args) throws IOException {

// 1. configuration Mapper & Reducer of Hadoop

JobConf conf = new JobConf();

conf.setJobName("wordcount");

conf.setMapperClass(WordCountMapper.class);

conf.setReducerClass(WordCountReducer.class);

// 2. final output key type & value type

conf.setOutputKeyClass(Text.class);

conf.setOutputValueClass(IntWritable.class);

// 3. in/output format 

conf.setInputFormat(TextInputFormat.class);

conf.setOutputFormat(TextOutputFormat.class);

// 4. set the path of file for read files

//    input path : args[0]

//    output path : args[1]

FileInputFormat.setInputPaths(conf, new Path(args[0]));

FileOutputFormat.setOutputPath(conf, new Path(args[1]));

// 5. run job

JobClient.runJob(conf);

}

}

 <dependencies>

  <dependency>

  <groupId>org.apache.hadoop</groupId>

  <artifactId>hadoop-core</artifactId>

  <version>1.1.2</version>

  </dependency>

  <dependency>

  <groupId>org.apache.mrunit</groupId>

  <artifactId>mrunit</artifactId>

  <version>0.8.0-incubating</version>

  <scope>test</scope>

  </dependency>

  </dependencies>

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mrunit.MapDriver;

import org.junit.Test;

/**

 * 테스트를 통하여 Mapper와 Reducer를 테스트에서 수행하여 검증 할 수 있다 

 * @author dowon

 *

 */

public class WordCountMapperTest {

  @Test

  public void testMap() {

    // 1. 설

    Text value = new Text("Hello World Bye World");

    MapDriver<LongWritable, Text, Text, IntWritable> mapDriver = new MapDriver();

    mapDriver.withMapper(new WordCountMapper());

    mapDriver.withInputValue(value);

    // 2. 검정 및 실행 

    // 순서를 정확히 해야 에러없이 수행된다. 빼먹어도 에러가 난다 

    mapDriver.withOutput(new Text("Hello"), new IntWritable(1));

    mapDriver.withOutput(new Text("World"), new IntWritable(1));

    mapDriver.withOutput(new Text("Bye"), new IntWritable(1));

    mapDriver.withOutput(new Text("World"), new IntWritable(1));

    mapDriver.runTest();

  }

}

import java.util.Arrays;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mrunit.ReduceDriver;

import org.junit.Test;

public class WordCountReducerTest {

  @Test

  public void testReducer() {

      // 1. 설정

    ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver = new ReduceDriver();

    reduceDriver.withReducer(new WordCountReducer());

    reduceDriver.withInputKey(new Text("World"));

    reduceDriver.withInputValues(Arrays.asList(new IntWritable(1), new IntWritable(1)));

    // 2. 검증 및 실행 

    reduceDriver.withOutput(new Text("World"), new IntWritable(2));

    reduceDriver.runTest();

  }

}

$ cat .bash_profile

alias ll='ls -alrt'

set -o vi

export JAVA_HOME=/Library/Java/Home

// 폴더를 하나 만들고 xml 환경파일을 복사한다 

$ mkdir input 

$ cp conf/*.xml input 

// 하기 명령을 수행한다 

// input 읽을 꺼리를 주고 결과값을 output에 담아라  

$ bin/hadoop jar hadoop-examples-*.jar grep input output 'dfs[a-z.]+' 

// output  폴더가 자동 생성되고 cat 하였을 때 존재하는 파일의 내역이 하기와 같이 보이면 성공!

$ cat output/*

1 dfsadmin

 Local (Standalone) Mode [로컬(독립)모드]

  하둡의 기본모드(아무런 환경설정을 하지 않음): 로컬 머신에서만 실행

  다른 노드와 통신할 필요가 없기 때문에 HDFS를 사용하지 않으며 다른 데몬들도 실행시키지 않음

  독립적으로 MapReduce 프로그램의 로직을 개발하고 디버깅하는데 유용함

 Pseudo-Distributed Mode [가상분산 모드]

  한대의 컴퓨터로 클러스터를 구성하고, 모든 데몬을 실행함.

  독립실행(standalone) 모드 기능 보완

  – 메모리 사용 정도, HDFS 입출력 관련 문제, 다른 데몬 과의 상호작용에서 발생하는 일을 검사

 Fully-Distributed Mode [완전분산 모드]

  분산 저장과 분산 연산의 모든 기능이 갖추어진 클러스터를 구성함

  master - 클러스터의 master 노드로, NameNode와 JobTracker 데몬을 제공

  backup - SNN(Secondary NameNode 데몬)을 제공하는 서버

  slaves - DataNode와 TaskTracker 데몬을 실행하는 slaves 들 

<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>

   <property>

        <name>fs.default.name</name>

        <value>hdfs://localhost:9000</value>

   </property>

</configuration>

<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>

     <property>

         <name>dfs.replication</name>

         <value>1</value>

     </property>

</configuration>

<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>

     <property>

         <name>mapred.job.tracker</name>

         <value>localhost:9001</value>

     </property>

</configuration>

$ bin/hadoop namenode -format

13/09/09 20:51:59 INFO namenode.NameNode: STARTUP_MSG: 

/************************************************************

STARTUP_MSG: Starting NameNode

STARTUP_MSG:   host = KOSTA17ui-iMac.local/192.168.0.15

STARTUP_MSG:   args = [-format]

STARTUP_MSG:   version = 1.2.1

STARTUP_MSG:   build = https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1.2 -r 1503152; compiled by 'mattf' on Mon Jul 22 15:23:09 PDT 2013

STARTUP_MSG:   java = 1.6.0_45

************************************************************/

13/09/09 20:52:00 INFO util.GSet: Computing capacity for map BlocksMap

... 중략 ...

13/09/09 20:52:00 INFO common.Storage: Image file /tmp/hadoop-dowon/dfs/name/current/fsimage of size 111 bytes saved in 0 seconds.

13/09/09 20:52:00 INFO common.Storage: Storage directory /tmp/hadoop-dowon/dfs/name has been successfully formatted.

$ bin/start-all.sh

starting namenode, logging to /Users/dowon/Documents/hadoop-1.2.1/libexec/../logs/hadoop-dowon-namenode-KOSTA17ui-iMac.local.out

2013-09-09 20:53:36.430 java[7632:1603] Unable to load realm info from SCDynamicStore

// 수행후 브라우져에서 50030, 50070 포트 호출

http://localhost:50030/dfshealth.jsp : JobTracker

http://localhost:50070/dfshealth.jsp : NameNode

// NameNode의 HDFS에 conf/* 모든 파일을 input 디렉토리명으로 생성하여 복사한다  

$ bin/hadoop fs -put conf input

2013-09-09 20:58:13.847 java[7966:1603] Unable to load realm info from SCDynamicStore

// 명령을 수행하면 JobTracker가 수행되고 HDFS에 output 디렉토리에 결과값이 생성된다 

// 결과값은 JobTracker가 처리하여 생성된 것이다 

$ bin/hadoop jar hadoop-examples-*.jar grep input output 'dfs[a-z.]+'

2013-09-09 21:01:57.770 java[8017:1603] Unable to load realm info from SCDynamicStore

13/09/09 21:01:58 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable

13/09/09 21:01:58 WARN snappy.LoadSnappy: Snappy native library not loaded

13/09/09 21:01:58 INFO mapred.FileInputFormat: Total input paths to process : 17

13/09/09 21:01:58 INFO mapred.JobClient: Running job: job_201309092053_0001

13/09/09 21:01:59 INFO mapred.JobClient:  map 0% reduce 0%

13/09/09 21:02:03 INFO mapred.JobClient:  map 11% reduce 0%

13/09/09 21:02:05 INFO mapred.JobClient:  map 23% reduce 0%

.. 중략..

// NameNode 결과 확인

// http://localhost:50070/ 에서 "Browser the filesystem" 을 클릭한다

// http://localhost:50075/browseDirectory.jsp?dir=%2Fuser%2Fdowon&namenodeInfoPort=50070

해당 브라우져 내역에 신규생성된 "user"밑의 "dowon"밑의 "input" 과 "output"이 보인다 (dowon은 계정명)

// output에 결과값 : part-00000 에 결과내역이 write 되어 있다 

// JobTracker 처리현황 확인 

// http://localhost:50030/jobtracker.jsp