hadoop2.7.3 从搭建到运行（三、简易demo篇）

最新推荐文章于 2026-04-01 16:50:37 发布

原创最新推荐文章于 2026-04-01 16:50:37 发布 · 591 阅读

1 ·

本内容遵循CC 4.0 BY-SA版权协议

收录于

大数据

大数据进阶之路

博主分享了Hadoop WordCount Demo的搭建与运行过程。先新建maven工程，引入依赖库，接着重写map和reduce代码，然后打jar包放入系统，最后运行。运行时需注意类名路径和删除原输出文件夹，成功后可查看结果。

刚开始学时，我想找一个比较简单的demo入门，但是都不符合我的版本，所以很难受，下面是我的一个过程。

一、新建一个maven工程

博主用的是esclipse，新建maven工程，

新建成功后整个目录结构是这样的，（其中resources文件夹是我自己新建的）

二、pom.xml

引入你的依赖库，这里我用了这么几个

<dependencies>
       <dependency>
           <groupId>junit</groupId>
           <artifactId>junit</artifactId>
           <version>4.4</version>
           <scope>test</scope>
       </dependency>
       <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
</dependency>

<dependency>
       <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-mapreduce-client-core</artifactId>
           <version>${hadoop.version}</version>
       </dependency>

       <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-core</artifactId>
       <version>1.2.1</version>
       </dependency>

       <dependency>
       <groupId>jdk.tools</groupId>
       <artifactId>jdk.tools</artifactId>
       <version>1.8</version>
       <scope>system</scope>
       <systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
       </dependency>

   </dependencies>

更新一下maven工程，将依赖包导入

三、重写map reduce

不啰嗦，直接放所有代码

package com.hadoop.hadoop;
import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.mapreduce.Reducer.Context;

public class WordCount {

// 继承Mapper类,Mapper类的四个泛型参数分别为：输入key类型，输入value类型，输出key类型，输出value类型
public static class Map extends Mapper<LongWritable, Text, Text, IntWritable> {

private final static IntWritable one = new IntWritable(1); // output value
private Text word = new Text(); // output key

@Override
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {

String line = value.toString();
StringTokenizer tokenizer = new StringTokenizer(line);

while (tokenizer.hasMoreTokens()) {
word.set(tokenizer.nextToken());
context.write(word, one);
}

}
}

// Reduce类，继承了Reducer类
public static class Reduce extends Reducer<Text, IntWritable, Text, IntWritable> {
@Override
public void reduce(Text key, Iterable<IntWritable> values,
Context context) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
context.write(key, new IntWritable(sum));
}
}

public static void main(String[] args) throws Exception {

if (args.length != 2) {
System.err
.println("Usage: MaxTemperature <input path> <output path>");
System.exit(-1);
}

Job job = new Job(); // 创建一个作业对象
job.setJarByClass(WordCount.class); // 设置运行/处理该作业的类
job.setJobName("WordCount");

FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));

job.setMapperClass(Map.class);
job.setReducerClass(Reduce.class);

job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);

System.exit(job.waitForCompletion(true) ? 0 : 1);

}

}