Mobile Convergence

// pom.xml 내역

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

  <modelVersion>4.0.0</modelVersion>

  <groupId>kr.mobiconsoft.hadoop</groupId>

  <artifactId>MapReduce</artifactId>

  <version>1.0.0-SNAPSHOT</version>

  <dependencies>

  <dependency>

  <groupId>org.apache.hadoop</groupId>

  <artifactId>hadoop-core</artifactId>

  <version>1.1.2</version>

         </dependency>

  </dependencies>

</project>

// 결과 

// file01

hello world bye world

// file02

hi world hello dowon

import java.io.IOException;

import java.util.StringTokenizer;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapred.MapReduceBase;

import org.apache.hadoop.mapred.Mapper;

import org.apache.hadoop.mapred.OutputCollector;

import org.apache.hadoop.mapred.Reporter;

/**

 * K1 : read key type

 * V2 : read value type

 * K2 : write key type

 * V2 : write value type

 */

//public class WordCountMapper implements Mapper<K1, V1, K2, V2> {

public class WordCountMapper extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {

// map 결과는 reducer로 자동으로 던져진다 

public void map(LongWritable key, Text value,

OutputCollector<Text, IntWritable> output, Reporter reporter)

throws IOException {

// TODO Auto-generated method stub

String line = value.toString();

StringTokenizer tokenizer = new StringTokenizer(line);

while(tokenizer.hasMoreTokens()) {

Text outputKey = new Text(tokenizer.nextToken());

// Hadoop 에서 wrapping한 Integer 타입의 객체를 넣어줌 

// param1: outputKey, param2: outputValue

output.collect(outputKey, new IntWritable(1));

}

}

}

import java.io.IOException;

import java.util.Iterator;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapred.MapReduceBase;

import org.apache.hadoop.mapred.OutputCollector;

import org.apache.hadoop.mapred.Reducer;

import org.apache.hadoop.mapred.Reporter;

/**

 * K1 : Mapper의 K2 와 동일

 * V1 : Mapper의 V2 와 동일 

 */

public class WordCountReducer extends MapReduceBase 

 implements Reducer<Text, IntWritable, Text, IntWritable> {

/**

* V1 에서 values는 Iterator이다. 실제 같은 단어가 여러개 일 경우 

*/

public void reduce(Text key, Iterator<IntWritable> values,

OutputCollector<Text, IntWritable> output, Reporter reporter)

throws IOException {

// TODO Auto-generated method stub

int sum = 0;

while(values.hasNext()) {

sum += values.next().get(); // get Integer value

}

output.collect(key, new IntWritable(sum));

}

}

import java.io.IOException;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapred.FileInputFormat;

import org.apache.hadoop.mapred.FileOutputFormat;

import org.apache.hadoop.mapred.JobClient;

import org.apache.hadoop.mapred.JobConf;

import org.apache.hadoop.mapred.TextInputFormat;

import org.apache.hadoop.mapred.TextOutputFormat;

public class WordCount {

public static void main(String[] args) throws IOException {

// 1. configuration Mapper & Reducer of Hadoop

JobConf conf = new JobConf();

conf.setJobName("wordcount");

conf.setMapperClass(WordCountMapper.class);

conf.setReducerClass(WordCountReducer.class);

// 2. final output key type & value type

conf.setOutputKeyClass(Text.class);

conf.setOutputValueClass(IntWritable.class);

// 3. in/output format 

conf.setInputFormat(TextInputFormat.class);

conf.setOutputFormat(TextOutputFormat.class);

// 4. set the path of file for read files

//    input path : args[0]

//    output path : args[1]

FileInputFormat.setInputPaths(conf, new Path(args[0]));

FileOutputFormat.setOutputPath(conf, new Path(args[1]));

// 5. run job

JobClient.runJob(conf);

}

}

 <dependencies>

  <dependency>

  <groupId>org.apache.hadoop</groupId>

  <artifactId>hadoop-core</artifactId>

  <version>1.1.2</version>

  </dependency>

  <dependency>

  <groupId>org.apache.mrunit</groupId>

  <artifactId>mrunit</artifactId>

  <version>0.8.0-incubating</version>

  <scope>test</scope>

  </dependency>

  </dependencies>

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mrunit.MapDriver;

import org.junit.Test;

/**

 * 테스트를 통하여 Mapper와 Reducer를 테스트에서 수행하여 검증 할 수 있다 

 * @author dowon

 *

 */

public class WordCountMapperTest {

  @Test

  public void testMap() {

    // 1. 설

    Text value = new Text("Hello World Bye World");

    MapDriver<LongWritable, Text, Text, IntWritable> mapDriver = new MapDriver();

    mapDriver.withMapper(new WordCountMapper());

    mapDriver.withInputValue(value);

    // 2. 검정 및 실행 

    // 순서를 정확히 해야 에러없이 수행된다. 빼먹어도 에러가 난다 

    mapDriver.withOutput(new Text("Hello"), new IntWritable(1));

    mapDriver.withOutput(new Text("World"), new IntWritable(1));

    mapDriver.withOutput(new Text("Bye"), new IntWritable(1));

    mapDriver.withOutput(new Text("World"), new IntWritable(1));

    mapDriver.runTest();

  }

}

import java.util.Arrays;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mrunit.ReduceDriver;

import org.junit.Test;

public class WordCountReducerTest {

  @Test

  public void testReducer() {

      // 1. 설정

    ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver = new ReduceDriver();

    reduceDriver.withReducer(new WordCountReducer());

    reduceDriver.withInputKey(new Text("World"));

    reduceDriver.withInputValues(Arrays.asList(new IntWritable(1), new IntWritable(1)));

    // 2. 검증 및 실행 

    reduceDriver.withOutput(new Text("World"), new IntWritable(2));

    reduceDriver.runTest();

  }

}

$ cat .bash_profile

alias ll='ls -alrt'

set -o vi

export JAVA_HOME=/Library/Java/Home

// 폴더를 하나 만들고 xml 환경파일을 복사한다 

$ mkdir input 

$ cp conf/*.xml input 

// 하기 명령을 수행한다 

// input 읽을 꺼리를 주고 결과값을 output에 담아라  

$ bin/hadoop jar hadoop-examples-*.jar grep input output 'dfs[a-z.]+' 

// output  폴더가 자동 생성되고 cat 하였을 때 존재하는 파일의 내역이 하기와 같이 보이면 성공!

$ cat output/*

1 dfsadmin

 Local (Standalone) Mode [로컬(독립)모드]

  하둡의 기본모드(아무런 환경설정을 하지 않음): 로컬 머신에서만 실행

  다른 노드와 통신할 필요가 없기 때문에 HDFS를 사용하지 않으며 다른 데몬들도 실행시키지 않음

  독립적으로 MapReduce 프로그램의 로직을 개발하고 디버깅하는데 유용함

 Pseudo-Distributed Mode [가상분산 모드]

  한대의 컴퓨터로 클러스터를 구성하고, 모든 데몬을 실행함.

  독립실행(standalone) 모드 기능 보완

  – 메모리 사용 정도, HDFS 입출력 관련 문제, 다른 데몬 과의 상호작용에서 발생하는 일을 검사

 Fully-Distributed Mode [완전분산 모드]

  분산 저장과 분산 연산의 모든 기능이 갖추어진 클러스터를 구성함

  master - 클러스터의 master 노드로, NameNode와 JobTracker 데몬을 제공

  backup - SNN(Secondary NameNode 데몬)을 제공하는 서버

  slaves - DataNode와 TaskTracker 데몬을 실행하는 slaves 들 

<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>

   <property>

        <name>fs.default.name</name>

        <value>hdfs://localhost:9000</value>

   </property>

</configuration>

<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>

     <property>

         <name>dfs.replication</name>

         <value>1</value>

     </property>

</configuration>

<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>

     <property>

         <name>mapred.job.tracker</name>

         <value>localhost:9001</value>

     </property>

</configuration>

$ bin/hadoop namenode -format

13/09/09 20:51:59 INFO namenode.NameNode: STARTUP_MSG: 

/************************************************************

STARTUP_MSG: Starting NameNode

STARTUP_MSG:   host = KOSTA17ui-iMac.local/192.168.0.15

STARTUP_MSG:   args = [-format]

STARTUP_MSG:   version = 1.2.1

STARTUP_MSG:   build = https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1.2 -r 1503152; compiled by 'mattf' on Mon Jul 22 15:23:09 PDT 2013

STARTUP_MSG:   java = 1.6.0_45

************************************************************/

13/09/09 20:52:00 INFO util.GSet: Computing capacity for map BlocksMap

... 중략 ...

13/09/09 20:52:00 INFO common.Storage: Image file /tmp/hadoop-dowon/dfs/name/current/fsimage of size 111 bytes saved in 0 seconds.

13/09/09 20:52:00 INFO common.Storage: Storage directory /tmp/hadoop-dowon/dfs/name has been successfully formatted.

$ bin/start-all.sh

starting namenode, logging to /Users/dowon/Documents/hadoop-1.2.1/libexec/../logs/hadoop-dowon-namenode-KOSTA17ui-iMac.local.out

2013-09-09 20:53:36.430 java[7632:1603] Unable to load realm info from SCDynamicStore

// 수행후 브라우져에서 50030, 50070 포트 호출

http://localhost:50030/dfshealth.jsp : JobTracker

http://localhost:50070/dfshealth.jsp : NameNode

// NameNode의 HDFS에 conf/* 모든 파일을 input 디렉토리명으로 생성하여 복사한다  

$ bin/hadoop fs -put conf input

2013-09-09 20:58:13.847 java[7966:1603] Unable to load realm info from SCDynamicStore

// 명령을 수행하면 JobTracker가 수행되고 HDFS에 output 디렉토리에 결과값이 생성된다 

// 결과값은 JobTracker가 처리하여 생성된 것이다 

$ bin/hadoop jar hadoop-examples-*.jar grep input output 'dfs[a-z.]+'

2013-09-09 21:01:57.770 java[8017:1603] Unable to load realm info from SCDynamicStore

13/09/09 21:01:58 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable

13/09/09 21:01:58 WARN snappy.LoadSnappy: Snappy native library not loaded

13/09/09 21:01:58 INFO mapred.FileInputFormat: Total input paths to process : 17

13/09/09 21:01:58 INFO mapred.JobClient: Running job: job_201309092053_0001

13/09/09 21:01:59 INFO mapred.JobClient:  map 0% reduce 0%

13/09/09 21:02:03 INFO mapred.JobClient:  map 11% reduce 0%

13/09/09 21:02:05 INFO mapred.JobClient:  map 23% reduce 0%

.. 중략..

// NameNode 결과 확인

// http://localhost:50070/ 에서 "Browser the filesystem" 을 클릭한다

// http://localhost:50075/browseDirectory.jsp?dir=%2Fuser%2Fdowon&namenodeInfoPort=50070

해당 브라우져 내역에 신규생성된 "user"밑의 "dowon"밑의 "input" 과 "output"이 보인다 (dowon은 계정명)

// output에 결과값 : part-00000 에 결과내역이 write 되어 있다 

// JobTracker 처리현황 확인 

// http://localhost:50030/jobtracker.jsp

"One or more observers are interested in the state of a subject and register their interest with the subject by attaching themselves. When something changes in our subject that the observer may be interested in, a notify message is sent which calls the update method in each observer. When the observer is no longer interested in the subject's state, they can simply detach themselves."

// Object 목록 관리 Constructor Function

function ObserverList(){

  this.observerList = [];
}

ObserverList.prototype.Add = function( obj ){
  return this.observerList.push( obj );
};

ObserverList.prototype.Empty = function(){
  this.observerList = [];
};

ObserverList.prototype.Count = function(){
  return this.observerList.length;
};


ObserverList.prototype.Get = function( index ){
  if( index > -1 && index < this.observerList.length ){
    return this.observerList[ index ];
  }
};

ObserverList.prototype.Insert = function( obj, index ){
  var pointer = -1;

  if( index === 0 ){
    this.observerList.unshift( obj );
    pointer = index;
  }else if( index === this.observerList.length ){
    this.observerList.push( obj );
    pointer = index;
  }

  return pointer;
};

ObserverList.prototype.IndexOf = function( obj, startIndex ){
  var i = startIndex, pointer = -1;

  while( i < this.observerList.length ){
    if( this.observerList[i] === obj ){
      pointer = i;
    }
    i++;
  }

  return pointer;
};


ObserverList.prototype.RemoveAt = function( index ){
  if( index === 0 ){
    this.observerList.shift();
  }else if( index === this.observerList.length -1 ){
    this.observerList.pop();
  }
};


// Extend an object with an extension
function extend( extension, obj ){
  for ( var key in extension ){
    obj[key] = extension[key];
  }
}

// 1. Subject Constructor Function을 정의한다
// (Constructor Pattern의 prototype 방식 사용)

function Subject(){
  this.observers = new ObserverList();
}

Subject.prototype.AddObserver = function( observer ){
  this.observers.Add( observer );
};  

Subject.prototype.RemoveObserver = function( observer ){
  this.observers.RemoveAt( this.observers.IndexOf( observer, 0 ) );
};  

// Observer가 상태 정보를 받는다 
Subject.prototype.Notify = function( context ){
  var observerCount = this.observers.Count();
  for(var i=0; i < observerCount; i++){
    this.observers.Get(i).Update( context );
  } 

};

// 2. Observer 구현 

// The Observer
function Observer(){
  this.Update = function(){
    // ...
  };
}

// HTML

1) button을 클릭하면 observable checkbox를 페이지에 추가

2) checkbox가 subject로 동작한다. check가 되면 모든 checkbox에 상태변경을 알려준다(notify) 

3) 새로운 checkbox를 container에 추가한다 

<button id="addNewObserver">Add New Observer checkbox</button>
<input id="mainCheckbox" type="checkbox"/>
<div id="observersContainer"></div>

// Script 

// References to our DOM elements
var controlCheckbox = document.getElementById( "mainCheckbox" ),
  addBtn = document.getElementById( "addNewObserver" ),
  container = document.getElementById( "observersContainer" );


// 3. Concrete Subject
// Subject의 notify 메소드를 호출하는 구문이 존재 
// Extend the controlling checkbox with the Subject class
extend( new Subject(), controlCheckbox );

// Clicking the checkbox will trigger notifications to its observers
controlCheckbox["onclick"] = new Function( "controlCheckbox.Notify(controlCheckbox.checked)" );
addBtn["onclick"] = AddNewObserver;

// 4. Concrete Observer
function AddNewObserver(){

  // Create a new checkbox to be added
  var check  = document.createElement( "input" );
  check.type = "checkbox";

  // Extend the checkbox with the Observer class
  extend( new Observer(), check );

  // Override with custom update behaviour
  check.Update = function( value ){
    this.checked = value;
  };

  // Add the new observer to our list of observers
  // for our main subject
  controlCheckbox.AddObserver( check );

  // Append the item to the container
  container.appendChild( check );
}

var pubsub = {};

(function(q) {

    var topics = {},
        subUid = -1;

    // Publish or broadcast events of interest
    // with a specific topic name and arguments
    // such as the data to pass along
    q.publish = function( topic, args ) {

        if ( !topics[topic] ) {
            return false;
        }

        var subscribers = topics[topic],
            len = subscribers ? subscribers.length : 0;

        while (len--) {
            subscribers[len].func( topic, args );
        }

        return this;
    };

    // Subscribe to events of interest
    // with a specific topic name and a
    // callback function, to be executed
    // when the topic/event is observed
    q.subscribe = function( topic, func ) {

        if (!topics[topic]) {
            topics[topic] = [];
        }

        var token = ( ++subUid ).toString();
        topics[topic].push({
            token: token,
            func: func
        });
        return token;
    };

    // Unsubscribe from a specific
    // topic, based on a tokenized reference
    // to the subscription
    q.unsubscribe = function( token ) {
        for ( var m in topics ) {
            if ( topics[m] ) {
                for ( var i = 0, j = topics[m].length; i < j; i++ ) {
                    if ( topics[m][i].token === token) {
                        topics[m].splice( i, 1 );
                        return token;
                    }
                }
            }
        }
        return this;
    }; 

}( pubsub ));

class Singleton

{

    private static Singleton instance; // 1) 

    private Singleton() // 2) 

    {

        ...

    }

    public static synchronized Singleton getInstance() // 3)

    {

        if (instance == null) // 4) 

            instance = new Singleton();

        return instance;

    }

    ...

    public void doSomething()

    {

        ... 

    }

}

var mySingleton = (function () {

  // Instance stores a reference to the Singleton
  var instance;

  function Init() { // define Constructor Function

    // Singleton
    // Private methods and variables
    function privateMethod(){
        console.log( "I am private" );
    }

    var privateVariable = "Im also private";
    var privateRandomNumber = Math.random();

    return {

      // Public methods and variables
      publicMethod: function () {
        console.log( "The public can see me!" );
      },

      publicProperty: "I am also public",

      getRandomNumber: function() {
        return privateRandomNumber;
      }

    };

  };

  return { // return the Closure Object

    // Get the Singleton instance if one exists
    // or create one if it doesn't
    getInstance: function () {

      // 생성이 안되었으면 최초에 한번만 init 한다 
      if ( !instance ) {
        instance = Init();
      }

      return instance;
    }

  };

})();


// 잘 못된 예제 
var myBadSingleton = (function () {

  // Instance stores a reference to the Singleton
  var instance;

  function init() {

    // Singleton
    var privateRandomNumber = Math.random();

    return {
      getRandomNumber: function() {
        return privateRandomNumber;
      }

    };

  };

  return {

    // Always create a new Singleton instance
    getInstance: function () {
      instance = init();
      return instance;
    }

  };

})();


var singleA = mySingleton.getInstance();
var singleB = mySingleton.getInstance();
console.log( singleA.getRandomNumber() === singleB.getRandomNumber() ); // true

var badSingleA = myBadSingleton.getInstance();
var badSingleB = myBadSingleton.getInstance();
console.log( badSingleA.getRandomNumber() !== badSingleB.getRandomNumber() ); // true

var myObjectLiteral = {

  variableKey: variableValue,

  functionKey: function() {...},

  someObject: {...}

}

var testModule = (function() {

  var counter = 0;

  return {  // Closure

    incrementCounter: function() {

      return counter++;

    },

    resetCounter: function() {

      console.log('counter is', counter);

      counter = 0;

    }

  };

})(); // IIFE

testModule.incrementCounter();

testModule.resetCounter();

// output : counter is 1

var myNamespace = (function () {

  var myPrivateVar, myPrivateMethod;

  // A private counter variable
  myPrivateVar = 0;

  // A private function which logs any arguments
  myPrivateMethod = function( foo ) {
      console.log( foo );
  };

  return {

    // A public variable
    myPublicVar: "foo",

    // A public function utilizing privates
    myPublicFunction: function( bar ) {

      // Increment our private counter
      myPrivateVar++;

      // Call our private method using bar
      myPrivateMethod( bar );

    }
  };
 

})();

// Global module
var myModule = (function ( jQ, _ ) {
  
    function privateMethod1(){
        jQ(".container").html("test");
    }

    function privateMethod2(){
      console.log( _.min([10, 5, 100, 2, 1000]) );
    }
    
    return{
        publicMethod: function(){
            privateMethod1();                
        }            
    };
   
// Pull in jQuery and Underscore
}( jQuery, _ ));
 

myModule.publicMethod();  

// Global module
var myModule = (function () {

    // Module object 
  var module = {},
    privateVariable = "Hello World";
  
  function privateMethod() {
    // ...
  }

  module.publicProperty = "Foobar";
  module.publicMethod = function () {
    console.log( privateVariable );
  };
  
  return module;
 

}());