一個統計文件中單詞出現次數的Mapreduce案例

WordCountMapper.java中的內容：

package com.jxd.mapreduce.wordcount;import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper;import java.io.IOException;public class WordCountMapper extends Mapper<LongWritable, Text, Text, IntWritable> {private Text outK = new Text();private IntWritable outV = new IntWritable(1);@Overrideprotected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {String line = value.toString();String[] words = line.split(" ");for (String word : words) {outK.set(word);context.write(outK, outV);}} }

WordCountReducer.java中的內容

package com.jxd.mapreduce.wordcount;import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Reducer;import java.io.IOException;public class WordCountReducer extends Reducer<Text, IntWritable, Text, IntWritable> {private IntWritable outV = new IntWritable();@Overrideprotected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {int sum = 0;for (IntWritable value : values) {sum += value.get();}outV.set(sum);context.write(key, outV);} }

WordCountDriver.java中的內容：

package com.jxd.mapreduce.wordcount;import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import java.io.IOException;public class WordCountDriver {public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {//1. 獲取jobConfiguration configuration = new Configuration();Job job = Job.getInstance(configuration);//2. 設置jar包路徑job.setJarByClass(WordCountDriver.class);//3. 關聯mapper和reducerjob.setMapperClass(WordCountMapper.class);job.setReducerClass(WordCountReducer.class);//4. 設置map輸出的kv類型job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(IntWritable.class);//5. 設置最終的kv類型job.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);//6. 設置輸入路徑和輸出路徑FileInputFormat.setInputPaths(job, new Path("G:\\input\\inputword"));FileOutputFormat.setOutputPath(job, new Path("G:\\output\\WordCount"));//7. 提交jobboolean result = job.waitForCompletion(true);System.exit(result ? 0 : 1);} }

總結

以上是生活随笔為你收集整理的一个Mapreduce案例的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。