MapReduce经典案例--WordCount代码

长不大的蜡笔小新

已于 2022-11-11 20:37:43 修改

阅读量1.2k

点赞数 2

CC 4.0 BY-SA版权

分类专栏： hadoop 文章标签： hadoop

于 2022-11-11 20:32:53 首次发布

本文链接：https://blog.csdn.net/ztx22555/article/details/127812588

hadoop 专栏收录该内容

16 篇文章

订阅专栏

该博客介绍了如何在Eclipse环境下编写一个简单的Hadoop MapReduce程序，用于实现WordCount功能。首先展示了Map阶段的实现，通过Mapper类将输入文本分割成单词并输出键值对。接着，博客展示了Reduce阶段的实现，使用Reducer类对相同单词的计数进行求和。最后，提供了主类MapReduceDemo来配置和运行整个任务，指定输入和输出路径。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

在这里插入图片描述

在eclipse上编写方法，实现一个 WordCount 程序

package cn.cqsw;
 
import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
 
public class WCMap extends Mapper<LongWritable, Text, Text, LongWritable> {
@Override
protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text,LongWritable>.Context context)
throws IOException, InterruptedException {
      	String[] splited=value.toString().split(" ");
        	for(String str :splited)
       	 {
		context.write(new Text(str),new LongWritable(1));
       	 }
        }
}

package cn.cqsw;
 
import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
 
 
public class WCReduce extends Reducer<Text, LongWritable,Text, LongWritable> {
@Override
protected void reduce(Text text, Iterable<LongWritable> iterable,Reducer<Text, LongWritable,Text,LongWritable>.Context   context) throws IOException,InterruptedException {
        	long time=0;
        	for(LongWritable lw :iterable)
       	 {
		time+=lw.get();
       	 }
        	context.write(text, new LongWritable(time));
        }
}

package cn.cqsw;
 
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 
public class MapReduceDemo{
	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
	    Configuration conf = new Configuration();
	    Job job = Job.getInstance(conf);
	    job.setJarByClass(MapReduceDemo.class);
	    job.setMapperClass(WCMap.class);
	    job.setReducerClass(WCReduce.class);
	    job.setMapOutputKeyClass(Text.class);
	    job.setMapOutputValueClass(LongWritable.class);
	    job.setOutputKeyClass(Text.class);
	    job.setOutputValueClass(LongWritable.class);
	    Path inputPath = new Path("/hello");
	    Path outputPath = new Path("/output");
	    FileInputFormat.setInputPaths(job, inputPath);
	    FileOutputFormat.setOutputPath(job, outputPath);
	    boolean waitForCompletion = job.waitForCompletion(true);
	    System.exit(waitForCompletion?0:1);
	}
 
}