Hadoop 新的API的Map和Reduce方式

最新推荐文章于 2022-07-07 21:11:25 发布

原创最新推荐文章于 2022-07-07 21:11:25 发布 · 1k 阅读

0 ·

本内容遵循CC 4.0 BY-SA版权协议

标签

#hadoop #api #exception #class #path

javaEE 同时被 2 个专栏收录

22 篇文章

订阅专栏

javaSE

21 篇文章

订阅专栏

本文介绍了一个基于Hadoop的WordCount实现案例，通过自定义Mapper和Reducer类处理文本文件，统计每个单词出现的长度，并展示如何运行MapReduce作业。

package com;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.WordCount;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;
import java.util.Iterator;

/**
 * Created by IntelliJ IDEA.
 * User: tony
 * Date: 11/7/11
 * Time: 5:50 AM
 * To change this template use File | Settings | File Templates.
 */


class MapNew extends Mapper<LongWritable, Text, Text, IntWritable> {

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        System.out.println("map ==> key : " + key);
        System.out.println("map ==> value : " + value + " length:" + value.getLength());

        context.write(value, new IntWritable(value.getLength()));
    }

}

class Reducerx extends Reducer<Text, IntWritable, Text, IntWritable> {
    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        System.out.println("reduce ==> key : " + key);
        Iterator<IntWritable> it = values.iterator();
        IntWritable maxLen = new IntWritable(0);
        /*while (it.hasNext()) {
            IntWritable text = it.next();
            System.out.println("reduce ==> value next : " + text);
            if (text.get() > maxLen.get()) {
                maxLen = text;
            }
        }*/
        context.write(key, values.iterator().next());

    }
}

class Runn {
    public static void main(String[] args) throws Exception {
        if (args.length != 2) {
            System.err.println("Please append inputPath&outputPath at suffix");
            System.exit(-1);
        }

        Job job = new Job();
        job.setJarByClass(Runn.class);

        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        job.setMapperClass(MapNew.class);
        job.setReducerClass(Reducerx.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        System.exit(job.waitForCompletion(true) ? 0 : 1);

    }
}