hive API2

hive udf 讀取HDFS的orc文件

package hive_udf_province;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.hive.ql.exec.UDF;

import org.apache.hadoop.hive.ql.io.orc.OrcFile;

import org.apache.hadoop.hive.ql.io.orc.Reader;

import org.apache.hadoop.hive.ql.io.orc.RecordReader;

import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;

import java.io.IOException;

import java.net.URI;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

/**

* Created by Administrator on 2017/11/7.s

* API2

* 參數(shù)為需要兌換表的字段，去跟碼表的第一個(gè)字段去匹配，兌換出碼表的第二個(gè)字段

*/

public class UDF_province_name_orc_2 extends UDF {

public static String evaluate(String pro_id) throws IOException {

String INPUT = "/user/hive/warehouse/yl.db/dim_province_orc/000000_0";

Configuration conf = new Configuration();

Path file_in = new Path(INPUT);

Reader reader = OrcFile.createReader(FileSystem.get(URI.create(INPUT), conf), file_in);

StructObjectInspector inspector = (StructObjectInspector) reader.getObjectInspector();

RecordReader records = reader.rows();

Object row = null;

Map datamap = new HashMap();

while (records.hasNext()) {

row = records.next(row);

List value_lst = inspector.getStructFieldsDataAsList(row);

datamap.put(value_lst.get(0).toString(),value_lst.get(1).toString());

}

return datamap.get(pro_id);

}

}

hive

add jar /root/yl/orc11.jar;

create temporary function split_province_orc as 'hive_udf_province.UDF_province_name_orc2';

select split_province_orc(province_id) from yl.province_test;

總結(jié)

以上是生活随笔為你收集整理的java读写orc文件_java读取hive的orc文件的全部?jī)?nèi)容，希望文章能夠幫你解決所遇到的問(wèn)題。

如果覺(jué)得生活随笔網(wǎng)站內(nèi)容還不錯(cuò)，歡迎將生活随笔推薦給好友。