2019獨角獸企業(yè)重金招聘Python工程師標準>>>

package core;import org.apache.commons.lang3.StringUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.Function2; import org.apache.spark.api.java.function.PairFunction; import scala.Tuple2;import java.util.Arrays; import java.util.List;/*** Created by xupan on 2017/11/28.*/ public class JavaWordCountCore {public static void main(String[] args) {SparkConf conf = new SparkConf().setMaster("local").setAppName("JavaWordCountCore");JavaSparkContext sc = new JavaSparkContext(conf);JavaRDD rdd = sc.textFile("data/JavaWordCountCore.txt");//進行分割JavaRDD flatMapRDD = rdd.flatMap(new FlatMapFunction<String, String>() {public Iterable call(String line) throws Exception {if (StringUtils.isNotEmpty(line)) {return Arrays.asList(line.split(" "));}return null;}});//轉(zhuǎn)換成key/valueJavaPairRDD mapRDD = flatMapRDD.mapToPair(new PairFunction<String, String, Integer>() {public Tuple2<String, Integer> call(String word) throws Exception {return new Tuple2<String, Integer>(word, 1);}});//reduceByKeyJavaPairRDD javaPairRDD = mapRDD.reduceByKey(new Function2<Integer, Integer, Integer>() {public Integer call(final Integer v1, final Integer v2) {return v1 + v2;}});List<Tuple2<String,Integer>> list = javaPairRDD.collect();for (Tuple2<String,Integer> t : list ){System.out.println(t);}}}

(alibaba,1)
(scala,4)
(spark,2)
(hadoop,2)
(eee,1)
(google,1)
(java,1)
(reduce,1)

轉(zhuǎn)載于:https://my.oschina.net/u/2253438/blog/1580703

總結(jié)

以上是生活随笔為你收集整理的JavaWordCountCore的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯，歡迎將生活随笔推薦給好友。