AI Convergence

<build>

  <plugins>

    <plugin>

      <artifactId>maven-antrun-plugin</artifactId>

      <configuration>

        <tasks>

          <copy file="target/${project.artifactId}-${project.version}.jar"

            tofile="/Users/dowon/Documents/hadoop-jobs/${project.artifactId}-${project.version}.jar" />

        </tasks>

      </configuration>

      <executions>

        <execution>

          <phase>install</phase>

          <goals>

            <goal>run</goal>

          </goals>

        </execution>

      </executions>

    </plugin>

  </plugins>

</build>

import java.io.IOException;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapred.FileInputFormat;

import org.apache.hadoop.mapred.FileOutputFormat;

import org.apache.hadoop.mapred.JobClient;

import org.apache.hadoop.mapred.JobConf;

import org.apache.hadoop.mapred.lib.LongSumReducer;

import org.apache.hadoop.mapred.lib.TokenCountMapper;

public class WordCount3 {

  public static void main(String[] args) throws IOException {

    // 1. configuration Mapper & Reducer of Hadoop

    JobConf conf = new JobConf(WordCount3.class);

    conf.setJobName("wordcount3");

    // 2. final output key type & value type

    conf.setOutputKeyClass(Text.class);

    conf.setOutputValueClass(LongWritable.class);

    // 3. in/output format 

    conf.setMapperClass(TokenCountMapper.class);

    conf.setCombinerClass(LongSumReducer.class);

    conf.setReducerClass(LongSumReducer.class);

    // 4. set the path of file for read files

    //    input path : args[0]

    //    output path : args[1]

    FileInputFormat.setInputPaths(conf, new Path(args[0]));

    FileOutputFormat.setOutputPath(conf, new Path(args[1]));

    // 5. run job

    JobClient client = new JobClient();

    client.setConf(conf);

    JobClient.runJob(conf);

  }

}

// name node 포멧

$ hadoop namenode -format

// .bash_profile 에 PATH 설정

set -o vi

export JAVA_HOME=/Library/Java/Home

export H_HOME=~/Documents/hadoop-1.2.1

export PATH=.:$PATH:$JAVA_HOME/bin:$H_HOME/bin:/usr/bin

alias ll='ls -alrt'

alias cdh='cd $H_HOME'

// 하둡 데몬 수행

// 50030 : job-tracker 접속 포트

// 50070 : NameNode 접속 포트

$ start-all.sh 

// 위치가 하기와 같다면 

$ pwd

/Users/dowon/Documents/input

$ ls

total 16

-rw-r--r--   1 dowon  staff   22  9 11 19:26 file01

-rw-r--r--   1 dowon  staff   21  9 11 19:26 file02

// input 을 HDFS에 만든다 

$ hadoop fs -put . input

1) http://localhost:50070/으로 접속하여 "Browser the filesystem"을 클릭하면 볼 수 있다 

2) /user/dowon/input 경로로 만들어 졌음을 알 수 있다 

// 경로가 다음과 같고, WordCount3.class가 들어있는 .jar 파일이 존재한다 

$ pwd

/Users/dowon/Documents/hadoop-jobs

$ ls

-rw-r--r--   1 dowon  staff  5391  9 11 19:45 MapReduce-1.0.0-SNAPSHOT.jar

// 명령어 수행 

$ hadoop jar *.jar WordCount3 /user/dowon/input /user/dowon/output3

1) 네임노드에 접속해서 /user/dowon에 들어가 보면 "output3"이 생성된 것을 볼 수있다

2) output3으로 들어가면 결과값을 지니 파일이 존재한다 

3) 만일 명령을 재수행하고 싶다면 output3 디렉토리리 삭제해야 한다 

    $ hadoop fs -rmr /user/dowon/output3

// pom.xml 내역

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

  <modelVersion>4.0.0</modelVersion>

  <groupId>kr.mobiconsoft.hadoop</groupId>

  <artifactId>MapReduce</artifactId>

  <version>1.0.0-SNAPSHOT</version>

  <dependencies>

  <dependency>

  <groupId>org.apache.hadoop</groupId>

  <artifactId>hadoop-core</artifactId>

  <version>1.1.2</version>

         </dependency>

  </dependencies>

</project>

// 결과 

// file01

hello world bye world

// file02

hi world hello dowon

import java.io.IOException;

import java.util.StringTokenizer;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapred.MapReduceBase;

import org.apache.hadoop.mapred.Mapper;

import org.apache.hadoop.mapred.OutputCollector;

import org.apache.hadoop.mapred.Reporter;

/**

 * K1 : read key type

 * V2 : read value type

 * K2 : write key type

 * V2 : write value type

 */

//public class WordCountMapper implements Mapper<K1, V1, K2, V2> {

public class WordCountMapper extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {

// map 결과는 reducer로 자동으로 던져진다 

public void map(LongWritable key, Text value,

OutputCollector<Text, IntWritable> output, Reporter reporter)

throws IOException {

// TODO Auto-generated method stub

String line = value.toString();

StringTokenizer tokenizer = new StringTokenizer(line);

while(tokenizer.hasMoreTokens()) {

Text outputKey = new Text(tokenizer.nextToken());

// Hadoop 에서 wrapping한 Integer 타입의 객체를 넣어줌 

// param1: outputKey, param2: outputValue

output.collect(outputKey, new IntWritable(1));

}

}

}

import java.io.IOException;

import java.util.Iterator;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapred.MapReduceBase;

import org.apache.hadoop.mapred.OutputCollector;

import org.apache.hadoop.mapred.Reducer;

import org.apache.hadoop.mapred.Reporter;

/**

 * K1 : Mapper의 K2 와 동일

 * V1 : Mapper의 V2 와 동일 

 */

public class WordCountReducer extends MapReduceBase 

 implements Reducer<Text, IntWritable, Text, IntWritable> {

/**

* V1 에서 values는 Iterator이다. 실제 같은 단어가 여러개 일 경우 

*/

public void reduce(Text key, Iterator<IntWritable> values,

OutputCollector<Text, IntWritable> output, Reporter reporter)

throws IOException {

// TODO Auto-generated method stub

int sum = 0;

while(values.hasNext()) {

sum += values.next().get(); // get Integer value

}

output.collect(key, new IntWritable(sum));

}

}

import java.io.IOException;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapred.FileInputFormat;

import org.apache.hadoop.mapred.FileOutputFormat;

import org.apache.hadoop.mapred.JobClient;

import org.apache.hadoop.mapred.JobConf;

import org.apache.hadoop.mapred.TextInputFormat;

import org.apache.hadoop.mapred.TextOutputFormat;

public class WordCount {

public static void main(String[] args) throws IOException {

// 1. configuration Mapper & Reducer of Hadoop

JobConf conf = new JobConf();

conf.setJobName("wordcount");

conf.setMapperClass(WordCountMapper.class);

conf.setReducerClass(WordCountReducer.class);

// 2. final output key type & value type

conf.setOutputKeyClass(Text.class);

conf.setOutputValueClass(IntWritable.class);

// 3. in/output format 

conf.setInputFormat(TextInputFormat.class);

conf.setOutputFormat(TextOutputFormat.class);

// 4. set the path of file for read files

//    input path : args[0]

//    output path : args[1]

FileInputFormat.setInputPaths(conf, new Path(args[0]));

FileOutputFormat.setOutputPath(conf, new Path(args[1]));

// 5. run job

JobClient.runJob(conf);

}

}

 <dependencies>

  <dependency>

  <groupId>org.apache.hadoop</groupId>

  <artifactId>hadoop-core</artifactId>

  <version>1.1.2</version>

  </dependency>

  <dependency>

  <groupId>org.apache.mrunit</groupId>

  <artifactId>mrunit</artifactId>

  <version>0.8.0-incubating</version>

  <scope>test</scope>

  </dependency>

  </dependencies>

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mrunit.MapDriver;

import org.junit.Test;

/**

 * 테스트를 통하여 Mapper와 Reducer를 테스트에서 수행하여 검증 할 수 있다 

 * @author dowon

 *

 */

public class WordCountMapperTest {

  @Test

  public void testMap() {

    // 1. 설

    Text value = new Text("Hello World Bye World");

    MapDriver<LongWritable, Text, Text, IntWritable> mapDriver = new MapDriver();

    mapDriver.withMapper(new WordCountMapper());

    mapDriver.withInputValue(value);

    // 2. 검정 및 실행 

    // 순서를 정확히 해야 에러없이 수행된다. 빼먹어도 에러가 난다 

    mapDriver.withOutput(new Text("Hello"), new IntWritable(1));

    mapDriver.withOutput(new Text("World"), new IntWritable(1));

    mapDriver.withOutput(new Text("Bye"), new IntWritable(1));

    mapDriver.withOutput(new Text("World"), new IntWritable(1));

    mapDriver.runTest();

  }

}

import java.util.Arrays;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mrunit.ReduceDriver;

import org.junit.Test;

public class WordCountReducerTest {

  @Test

  public void testReducer() {

      // 1. 설정

    ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver = new ReduceDriver();

    reduceDriver.withReducer(new WordCountReducer());

    reduceDriver.withInputKey(new Text("World"));

    reduceDriver.withInputValues(Arrays.asList(new IntWritable(1), new IntWritable(1)));

    // 2. 검증 및 실행 

    reduceDriver.withOutput(new Text("World"), new IntWritable(2));

    reduceDriver.runTest();

  }

}

var p2 = db.runCommand(
{ aggregate : "article", pipeline : [
    { $project : {
	author : 1,
	tags : 1,
	pageViews : 1
    }},
    { $unwind : "$tags" }

]});

// orders 컬렉션을 다음을 저장한다  

$ mongod --dbpath /home/mongodb/aggregation

// orders의 도큐먼트를 2번 동일하게 save 한다 

$ mongo

> db.orders.save({

   cust_id: "abc123",

   ord_date: ISODate("2012-11-02T17:04:11.102Z"),

   status: 'A',

   price: 50,

   items: [ { sku: "xxx", qty: 25, price: 1 },

            { sku: "yyy", qty: 25, price: 1 } ]

 });

db.runCommand(

    {

     mapReduce: <collection>,

     map: <function>,

     reduce: <function>,

     out: <output>,

     query: <document>,

     sort: <document>,

     limit: <number>,

     finalize: <function>,

     scope: <document>,

     jsMode: <boolean>,

     verbose: <boolean>

    }

)