Spark的RDD數據集很適合處理輕量文件，一般場景下是excel文件，可以將excel文件另存為CSV（逗號分隔），Spark讀取CSV文件形成RDD。

1、序列化類Record，用于保存字段

package sk.sql;import java.io.Serializable;public class Record implements Serializable {String area;String orderid;String content;String datetime;// constructor , getters and setters public Record(String area, String orderid, String content, String datetime) {// TODO Auto-generated constructor stubthis.area=area;this.orderid=orderid;this.content=content;this.datetime=datetime;}public String getarea(){return this.area;}public void setarea(String area){this.area=area;}public String getorderid(){return this.orderid;}public void setorderid(String orderid){this.orderid=orderid;}public String getcontent(){return this.content;}public void setcontent(String content){this.content=content;}public String getdatetime(){return this.datetime;}public void setdatetime(String datetime){this.datetime=datetime;} }

2、讀取CSV文件并形成RDD代碼

package sk.sql;import java.util.List;import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.Function;public class CSVDemo {public static void main(String[] args) {SparkConf conf = new SparkConf().setAppName("CSVDemo");JavaSparkContext sc = new JavaSparkContext(conf);JavaRDD<String> data = sc.textFile("/tmp/10all.csv");//SQLContext sqlContext = new SQLContext(sc);JavaRDD<Record> rdd_records = data.map(new Function<String, Record>() {public Record call(String line) throws Exception {// Here you can use JSON// Gson gson = new Gson();// gson.fromJson(line, Record.class);String[] fields = line.split(",");if(fields.length<7 ) return null;Record sd = new Record(fields[1],fields[2],fields[4],fields[6]);return sd;}});List<Record> lr=rdd_records.collect();for(Record rd:lr){if(rd!=null)System.out.println(rd.area+"|"+rd.orderid+"|"+rd.content+"|"+rd.datetime);}//rdd_records.saveAsTextFile("/tmp/10all.txt");sc.stop();}}

總結

以上是生活随笔為你收集整理的离线轻量级大数据平台Spark之读取CSV文件实例的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。