hadoop mapper&reducer调试 + 测试驱动

package test;

import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.junit.Assert;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

/**
 * Created by IntelliJ IDEA.
 * User: tony
 * Date: 11/20/11
 * Time: 6:35 AM
 * To change this template use File | Settings | File Templates.
 */
public class TemperatureMapReduce  extends Configured implements Tool {

public static class TempMapper extends MapReduceBase implements Mapper<LongWritable, Text, LongWritable, Text> {


    public void map(LongWritable longWritable, Text text, OutputCollector<LongWritable, Text> textIntWritableOutputCollector, Reporter reporter) throws IOException {
        System.out.println(longWritable + " --- "  + text);
        LongWritable year = new LongWritable(Long.valueOf(text.toString().substring(0, 4)));
        Text temperature =   new Text( text.toString().substring(4, 6) );
        Text result = text;
        System.out.println(year + " -+++++++++++++++++++- "  + temperature);
        textIntWritableOutputCollector.collect(year, temperature);
    }
}

public static class TempReducer extends MapReduceBase implements Reducer<LongWritable, Text, LongWritable, Text> {
    public void reduce(LongWritable key, Iterator<Text> values, OutputCollector<LongWritable, Text> output, Reporter reporter) throws IOException {
        int sum = 0;
        Text t = null;
        long maxTemp = 0L;
        while (values.hasNext()) {
            Text tmp = values.next();
            System.out.println("key:" + key + "     it : " + tmp);
            Long temp = Long.valueOf(tmp.toString());
            if(temp > maxTemp){
                maxTemp = temp;
                t = tmp;
            }
        }
        output.collect(key, new Text(maxTemp+""));
    }
}

    @Test
    public void testMapper() throws IOException {

        TempMapper myMapper = new TempMapper();
        Text text = new Text("199951");
        OutputCollector outputCollector = new OutputCollector<LongWritable,Text>(){
            public void collect(LongWritable resultKey, Text resultValue) throws IOException {
                System.out.println("resultKey:" + resultKey + "  resultValue:" + resultValue);
                Assert.assertTrue("1999".equals(resultKey.toString()));
                Assert.assertTrue("51".equals(resultValue.toString()));
            }
        };
        myMapper.map(null,text, outputCollector, null);

    }

    @Test
    public void testReducer() throws IOException {

        TempReducer myMapper = new TempReducer();
        Text text = new Text("199951");
        LongWritable longWritable = new LongWritable(1999);
        List list = new ArrayList();
        list.add(new Text("49")) ;
        list.add(new Text("29")) ;
        list.add(new Text("59")) ;



        OutputCollector outputCollector = new OutputCollector<LongWritable,Text>(){
            public void collect(LongWritable resultKey, Text resultValue) throws IOException {
                System.out.println("resultKey:" + resultKey + "  resultValue:" + resultValue);
                Assert.assertTrue("1999".equals(resultKey.toString()));
                Assert.assertTrue("59".equals(resultValue.toString()));
            }
        };
        myMapper.reduce(longWritable, list.iterator(), outputCollector, null);

    }

    public int run(String[] args) throws Exception {
        JobConf conf = new JobConf(getConf(), getClass());
        conf.setJobName("wordcount");
        //conf.set("fs.default.name","hdfs://192.168.126.133:9000/");



        String output = "hdfs://192.168.126.133:9000/t1output";
        FileSystem fs = FileSystem.get(URI.create(output), conf);
        Path path = new Path(output);
        if(fs.exists(path)){
            fs.delete(path, true);
        }
        fs.close();

        conf.setMapperClass(TempMapper.class);
        conf.setCombinerClass(TempReducer.class);
        conf.setReducerClass(TempReducer.class);

        /*conf.setMapOutputKeyClass(LongWritable.class);
        conf.setMapOutputValueClass(Text.class);
        conf.setOutputKeyClass(Text.class);
        conf.setOutputValueClass(IntWritable.class);*/



       /* conf.setOutputKeyClass(Text.class);
        conf.setOutputValueClass(IntWritable.class);*/

     /*   conf.setInputFormat(TextInputFormat.class);
        conf.setOutputFormat(TextOutputFormat.class);*/

        FileInputFormat.setInputPaths(conf, new Path(args[0]));
        FileOutputFormat.setOutputPath(conf, new Path(args[1]));

        JobClient.runJob(conf);
        return 1;
    }

    public static void main(String[] args){
        try {
            String[] arg = new String[]{"hdfs://192.168.126.133:9000/t1/temperature.txt", "hdfs://192.168.126.133:9000/t1output"};
            ToolRunner.run(new TemperatureMapReduce(), arg);
        } catch (Exception e) {
            e.printStackTrace();
        }

    }


}