實驗0-Hive配置與啟動

實驗目的

掌握Hive配置與啟動的方法。

實驗內容

預置環境：1.1.1 HDFS、1.1.2. zookeeper和1.1.3 yarn均正常啟動。

將文件上傳hdfs

hdfs dfs -mkdir -p /user/root/tez/
hdfs dfs -put /usr/hdp/3.0.1.0-187/tez/lib/tez.tar.gz /user/root/tez/

復制jar包

cp -r /usr/hdp/3.0.1.0-187/tez/.jar /usr/hdp/3.0.1.0-187/hadoop/lib/
cp -r /usr/hdp/3.0.1.0-187/tez/.jar /usr/hdp/3.0.1.0-187/hive/lib/
cp -r /usr/hdp/3.0.1.0-187/tez/lib/* /usr/hdp/3.0.1.0-187/hive/lib/
cp -r /usr/hdp/3.0.1.0-187/tez/lib/* /usr/hdp/3.0.1.0-187/hadoop/lib/

刪除重復的guava包

rm -f /usr/hdp/3.0.1.0-187/hive/lib/guava-11.0.2.jar
rm -f /usr/hdp/3.0.1.0-187/hadoop/lib/guava-11.0.2.jar

編輯tez-site.xml文件

vim /etc/tez/conf/tez-site.xml

注意下面的IP需要更改為主節點IP

<?xml version="1.0"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> tez.lib.uris hdfs://10.244.11.74:8020/user/root/tez/tez.tar.gz tez.use.cluster.hadoop-libs true tez.runtime.compress false tez.runtime.compress.codec org.apache.hadoop.io.compress.SnappyCodec <property><name>tez.am.am-rm.heartbeat.interval-ms.max</name><value>250</value> </property><property><name>tez.am.container.idle.release-timeout-max.millis</name><value>20000</value> </property><property><name>tez.am.container.idle.release-timeout-min.millis</name><value>10000</value> </property><property><name>tez.am.container.reuse.enabled</name><value>true</value> </property><property><name>tez.am.container.reuse.locality.delay-allocation-millis</name><value>250</value> </property><property><name>tez.am.container.reuse.non-local-fallback.enabled</name><value>false</value> </property><property><name>tez.am.container.reuse.rack-fallback.enabled</name><value>true</value> </property><property><name>tez.am.java.opts</name><value>-server -Xmx819m -Djava.net.preferIPv4Stack=true</value> </property><property><name>tez.am.max.app.attempts</name><value>2</value> </property><property><name>tez.am.maxtaskfailures.per.node</name><value>10</value> </property><property><name>tez.am.resource.memory.mb</name><value>2048</value> </property><property><name>tez.counters.max</name><value>10000</value> </property><property><name>tez.counters.max.groups</name><value>3000</value> </property><property><name>tez.generate.debug.artifacts</name><value>false</value> </property><property><name>tez.grouping.max-size</name><value>1073741824</value> </property><property><name>tez.grouping.min-size</name><value>16777216</value> </property><property><name>tez.grouping.split-waves</name><value>1.7</value> </property><property><name>tez.runtime.io.sort.mb</name><value>270</value> </property><property><name>tez.runtime.optimize.local.fetch</name><value>true</value> </property><property><name>tez.runtime.pipelined.sorter.sort.threads</name><value>2</value> </property><property><name>tez.runtime.shuffle.fetch.buffer.percent</name><value>0.6</value> </property><property><name>tez.runtime.shuffle.memory.limit.percent</name><value>0.25</value> </property><property><name>tez.runtime.sorter.class</name><value>PIPELINED</value> </property><property><name>tez.runtime.unordered.output.buffer.size-mb</name><value>76</value> </property><property><name>tez.session.am.dag.submit.timeout.secs</name><value>600</value> </property><property><name>tez.session.client.timeout.secs</name><value>-1</value> </property><property><name>tez.shuffle-vertex-manager.max-src-fraction</name><value>0.4</value> </property><property><name>tez.shuffle-vertex-manager.min-src-fraction</name><value>0.2</value> </property> <property><name>tez.task.max-events-per-heartbeat</name><value>500</value> </property><property><name>tez.task.resource.memory.mb</name><value>1024</value> </property>

編輯hive-site.xml文件

先備份原來的配置文件

mv /etc/hive/conf/hive-site.xml /etc/hive/conf/hive-site.xml.bak
vim /etc/hive/conf/hive-site.xml

<?xml version="1.0"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> javax.jdo.option.ConnectionURL jdbc:mysql://127.0.0.1:3306/metastore?createDatabaseIfNotExist=true JDBC connect string for a JDBC metastore javax.jdo.option.ConnectionDriverName com.mysql.jdbc.Driver Driver class name for a JDBC metastore javax.jdo.option.ConnectionUserName root username to use against metastore database javax.jdo.option.ConnectionPassword 123456 password to use against metastore database hive.metastore.schema.verification false hive.metastore.warehouse.dir /user/hive/warehouse hive.execution.engine tez datanucleus.autoStartMechanism SchemaTable hive.tez.container.size 2048 <property><name>hive.tez.java.opts</name><value>-server -Xmx2048m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC</value></property><property><name>hive.server2.tez.initialize.default.sessions</name><value>false</value></property><property><name>hive.server2.tez.default.queues</name><value>default</value></property><property><name>hive.tez.input.format</name><value>org.apache.hadoop.hive.ql.io.HiveInputFormat</value></property><property><name>hive.server2.tez.sessions.per.default.queue</name><value>1</value></property>

啟動hive

在主節點上啟動Hive

nohup hive --service metastore &
nohup hive --service hiveserver2 &

進入

beeline

連接hive server2的jdbc地址

!connect jdbc:hive2://127.0.0.1:10000
#輸入用戶名root
#輸入密碼:123456

CREATE TABLE students (name VARCHAR(64), age INT, gpa DECIMAL(3,2));
INSERT INTO TABLE students VALUES (‘tom’, 35, 1.28);

INSERT INTO TABLE students VALUES (‘cat’, 32, 2.32);
#時間稍微有點長，請慢慢等待，或者先學其他內容

select * from students ;
#確認插入記錄成功

!quit
#退出

總結

以上是生活随笔為你收集整理的实验0-Hive配置与启动的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。