HAWQ手动安装
HAWQ手動安裝及使用手冊
1 HAWQ簡介
HAWQ 是 Pivotal 設計的一個大規模并行 SQL 分析處理引擎，支持事務處理。HAWQ 將復雜的查詢分割成簡單的任何，并分發到并行處理系統中的處理單元執行。包括查詢規劃器、動態管道、前沿互聯和查詢執行優化器等等。提供最高級別的性能和可伸縮性。HAWQ 基于 Hadoop 架構是GPon HDFS。
?
2 準備
2.1 環境說明
1 HAWQ安裝采用源碼編譯安裝的方式.
2 Hadoop版本為hadoop-2.5.0-cdh5.3.3、hadoop-2.7.1
3 HAWQ安裝參考地址:
https://cwiki.apache.org/confluence/display/HAWQ/Build+and+Install
2.1.1虛擬機里Linux系統版本
[root@localhost ranger-0.5.0-usersync]# cat /etc/issue | grep Linux
Red Hat Enterprise Linux Server release 6.5 (Santiago)
2.1.2 JDK版本
[root@localhost native]# java -version
java version "1.7.0_67"
Java(TM) SE RuntimeEnvironment (build 1.7.0_67-b01)
Java HotSpot(TM) 64-BitServer VM (build 24.65-b04, mixed mode)
3 安裝
3.1安裝依賴軟件
?
#安裝yum的文檔(可選)
sed -i 's/tsflags=nodocs//g' /etc/yum.conf
?
#安裝所有需要的軟件
yum install -y epel-release && \
?yum makecache&& \
?yum install -y manpasswd sudo tar which git mlocate links make bzip2 \
?autoconf automakelibtool m4 gcc gcc-c++ gdb flex cmake gperf indent \
?libuuid-develkrb5-devel libgsasl-devel expat-devel libxml2-devel \
?perl-ExtUtils-Embed pam-devel python-develsnappy-devel \
?libyaml-devellibevent-devel bzip2-devel openssl-devel \
?openldap-develreadline-devel net-snmp-devel apr-devel \
?libesmtp-develxerces-c-devel python-pip json-c-devel \
?apache-ivyjava-1.7.0-openjdk-devel \
?openssh-clientsopenssh-server && \
?yum clean all
?
# 安裝libcurl 7.45.0
mkdir -p /tmp/build/ && \
?cd /tmp/build&& curl -L "http://curl.haxx.se/download/curl-7.45.0.tar.bz2"-o curl-7.45.0.tar.bz2 && \
?tar -xjfcurl-7.45.0.tar.bz2 && cd curl-7.45.0 && \
?./configure--prefix=/usr && make && make install && \
?rm -rf /tmp/build&& ldconfig
?
# 安裝maven
curl -L"http://repos.fedorapeople.org/repos/dchen/apache-maven/epel-apache-maven.repo"-o /etc/yum.repos.d/epel-apache-maven.repo && \
?yum install -yapache-maven && \
?yum clean all
?
# 為root配置ssh
ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa&& \
?cat~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys && \
?chmod 0600~/.ssh/authorized_keys
?
# 為所有用戶設置JAVA_HOME
echo "#!/bin/sh" > /etc/profile.d/java.sh&& \
?echo "exportJAVA_HOME=/etc/alternatives/java_sdk" >> /etc/profile.d/java.sh&& \
?chmod a+x/etc/profile.d/java.sh
注:centos,redhat版本需查看如下方式.
echo "export JAVA_HOME=/usr/java/latest">> /etc/profile.d/java.sh && chmod a+x /etc/profile.d/java.sh?
?
# 安裝 boost 1.59
mkdir -p /tmp/build && \
?cd /tmp/build&& curl -L"http://downloads.sourceforge.net/project/boost/boost/1.59.0/boost_1_59_0.tar.bz2"-o? boost_1_59_0.tar.bz2 && \
?tar -xjfboost_1_59_0.tar.bz2 && cd boost_1_59_0 && \
?./bootstrap.sh&& ./b2 --prefix=/usr -q && ./b2 --prefix=/usr -q install&& \
?rm -rf /tmp/build
?
# 安裝 bison 2.5.1
mkdir -p /tmp/build/ && \
?cd /tmp/build&& curl -L "ftp://ftp.gnu.org/gnu/bison/bison-2.5.1.tar.gz"-o bison-2.5.1.tar.gz && \
?tar -xzfbison-2.5.1.tar.gz && cd bison-2.5.1 && \
?./configure--prefix=/usr && make && make install && \
?rm -rf /tmp/build
?
#安裝 thrift 0.9.2
mkdir -p /tmp/build && \
?cd /tmp/build&& curl -L"http://www.us.apache.org/dist/thrift/0.9.2/thrift-0.9.2.tar.gz" -othrift-0.9.2.tar.gz && \
?tar -xzfthrift-0.9.2.tar.gz && cd thrift-0.9.2 && \
?./configure--prefix=/usr \
???????? --with-cpp=yes--with-boost=yes --with-qt4=no --with-csharp=no --with-java=no --with-erlang=no--enable-tests=no \
???????? --with-nodejs=no--with-lua=no --with-python=no --with-perl=no --with-php=no && \
?make &&make install && \
?rm -rf /tmp/build
?
#安裝 protobuf 2.5.0
mkdir -p /tmp/build/ && \
?cd /tmp/build&& curl -L"https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.bz2"-o protobuf-2.5.0.tar.bz2 && \
?tar -xjfprotobuf-2.5.0.tar.bz2 && cd protobuf-2.5.0 && \
?./configure--prefix=/usr && make && make install && ldconfig&& \
?rm -rf /tmp/build
注:直接github版本需墻外操作,如不方便。可以搜索到protobuf-2.5.0的源碼包再解壓、編譯
?
#安裝 libhdfs3
mkdir -p /tmp/build && \
?cd /tmp/build&& git clone https://github.com/PivotalRD/libhdfs3.git . && mkdirbuild && cd build && \
?../bootstrap--prefix=/usr && make && make install && ldconfig&& \
?rm -rf /tmp/build
?
#安裝 python 模塊
yum makecache && yum install -y postgresql-devel&& \
?pip --retries=50--timeout=300 install pg8000 simplejson unittest2 pycrypto pygresql pyyamllockfile paramiko psi && \
?pip --retries=50--timeout=300 install http://darcs.idyll.org/~t/projects/figleaf-0.6.1.tar.gz&& \
?pip --retries=50--timeout=300 installhttp://sourceforge.net/projects/pychecker/files/pychecker/0.8.19/pychecker-0.8.19.tar.gz/download&& \
?yum erase -ypostgresql postgresql-libs postgresql-devel && \
?yum clean all
?
# 安裝libyarn
RUN mkdir -p /tmp/build && \
?cd /tmp/build&& git clone --depth=1 https://github.com/apache/incubator-hawq.git .&& cd depends/libyarn && mkdir build && cd build&& \
?../bootstrap--prefix=/usr && make && make install && ldconfig&& \
?rm -rf /tmp/build
?
#新增 gpadmin用戶(root下不能運行HAWQ)
groupadd -g 1000 gpadmin && \
useradd -u 1000 -g 1000 gpadmin && \
echo "gpadmin?ALL=(ALL)?????? NOPASSWD:ALL" > /etc/sudoers.d/gpadmin
?
# sudo 里關閉require tty
sed -i -e 's|Defaults???requiretty|#Defaults???requiretty|' /etc/sudoers
?
# 配置用戶環境變量
echo "#!/bin/bash" > /etc/profile.d/user.sh&& \
?echo "exportUSER=\`whoami\`" >> /etc/profile.d/user.sh && \
?chmod a+x/etc/profile.d/user.sh
?
# 配置 gpadmin用戶的ssh
?
ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa&& \
?cat~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys && \
?chmod 0600~/.ssh/authorized_keys
?
3.2 配置操作系統參數
# 打開/etc/sysctl.conf,編輯如下內容
kernel.shmmax= 1000000000
kernel.shmmni= 4096
kernel.shmall= 4000000000
kernel.sem= 250 512000 100 2048
kernel.sysrq= 1
kernel.core_uses_pid= 1
kernel.msgmnb= 65536
kernel.msgmax= 65536
kernel.msgmni= 2048
net.ipv4.tcp_syncookies= 0
net.ipv4.ip_forward= 0
net.ipv4.conf.default.accept_source_route= 0
net.ipv4.tcp_tw_recycle= 1
net.ipv4.tcp_max_syn_backlog= 200000
net.ipv4.conf.all.arp_filter= 1
net.ipv4.ip_local_port_range= 1281 65535
net.core.netdev_max_backlog= 200000
vm.overcommit_memory= 2
fs.nr_open= 3000000
kernel.threads-max= 798720
kernel.pid_max= 798720
#increase network
net.core.rmem_max=2097152
net.core.wmem_max=2097152
#執行如下命令生效:
sysctl -p
?#打開/etc/security/limits.conf編輯如下內容
*soft nofile 2900000
*hard nofile 2900000
*soft nproc 131072
*hard nproc 131072
3.3 安裝hadoop
詳見hadoop手動安裝手冊.
3.4 編譯安裝HAWQ
# The Apache HAWQ source code can be obtained from thethe following link:?
# Apache Repo:?https://git-wip-us.apache.org/repos/asf/incubator-hawq.git?or?
# GitHub Mirror:?https://github.com/apache/incubator-hawq.
git clonehttps://git-wip-us.apache.org/repos/asf/incubator-hawq.git
?
# 切換到hawq源碼目錄
cd?/root/incubator-hawq
?
# 執行生成makefile命令
./configure
?
# Or you could use--prefix=/hawq/install/path to change the Apache HAWQ install path,?
# and you can also add some optionalcomponents using options (--with-python --with-perl)
./configure?--prefix=/hawq/install/path?--with-python--with-perl
?
# You can also run the command with--help for more configuration.
./configure?--help
?
#Note: If ./configure complainsthat libyarn is missing, it is provided under ./depends/libyarn.?
# Please follow the above steps toinstall libyarn. You may need to run "ldconfig -p <PATH>" afterlibyarn is installed.
注:以上可選
?
?
# 執行編譯和安裝
# To build concurrently , run makewith -j option. For example, make -j8
# On Linux system without largememory, you will probably encounter error like
# "Error occurred duringinitialization of VM" "Could not reserve enough space for objectheap";
# Try to set vm.overcommit_memory =1 temporarily, and/or avoid "-j" build, and/or add more memory andthen rebuild.
# On mac os, you will probably seethis error: "'openssl/ssl.h' file not found".
# "brew link openssl--force" should be able to solve the issue.
make?-j8
??
# 安裝HAWQ
make?install
3.4 安裝問題
暫無.
4 配置
4.1 HAWQ的參數配置
配置HAWQ的集群及其它參數
cd /usr/hawq/etc
vi hawq-site.xml
# 修改如下參數
| 參數名 | 示例值 |
| hawq_master_address_host | shenl |
| hawq_dfs_url | shenl:8020/hawq_default |
| hawq_rm_memory_limit_perseg | 64GB |
| hawq_rm_yarn_address | shenl:8032 |
| hawq_rm_yarn_scheduler_address | shenl:8030 |
?
su -gpadmin
cd usr/hawq/bin
source?../greenplum_path.sh
./hawqinit cluster??
# 啟動hawq
hawqstop/restart/start?cluster
?
5 使用
5.1 HAWQ基本功能驗證
主要驗證:
SQL基本功能
事務支持
存儲過程支持
性能測試
1) SQL基本功能驗證
./psql -d postgres
create table t ( i int );
insert into t values(1);
insert intot?select?generate_series(1,10000);
select?count(*) from t;
?
?
2)事務支持驗證
思路:
在一個客戶端里執行DML操作,沒commit之前在另外個客戶端里查查詢表記錄
具體驗證:
# 會話一中執行DML事務操作，先不提交
?
# 會話二中執行查詢操作
?
# 待會話一執行commit;后再在會話二中執行查詢操作
?
3)存儲過程支持驗證
create or replace functionsetPassengerType(num integer) returns varchar as
??? $$
??? begin
???????? insert into t values(num);
???????? return 'Okay';
??? end
??? $$
language plpgsql;
?
select setPassengerType(100);
?
4)性能測試驗證
待完善
6 總結
仔細跟蹤log,具體問題具體分析.
與50位技術專家面對面20年技術見證，附贈技術全景圖總結
- 上一篇： Impala手动安装
- 下一篇： Hbase复制(Replication