#-*- coding: utf-8 -*- import os import sys import time import datetime import pymysql from dateutil.parser import parse #沒有傳參默認是昨天 if len(sys.argv) < 2 :vdate = datetime.datetime.strftime(datetime.datetime.today() + datetime.timedelta(days=-1), '%Y%m%d') else:vdate = sys.argv[1]print('備份數據日期{vdate}'.format(vdate=vdate))#元數據庫 mysql_conn = pymysql.connect(host='127.0.0.1',port=3306,user='hive',passwd='123456',db='hive',charset='utf8') cursor = mysql_conn.cursor() #從底層開始備份 db_sql = u'''select name from hive.DBS order by case when name = 'ods' then 100when name = 'dwd' then 99when name = 'dws' then 98when name = 'ads' then 97else 1 end desc ''' cursor.execute(db_sql) db_list = cursor.fetchall() dir = 'mkdir -p /data10/hive_data_bak/{vdate}'.format(vdate=vdate) os.system(dir)for db in db_list:if db[0] not in ('tmp','testdb'):dir = 'mkdir -p /data10/hive_data_bak/{vdate}/{db}'.format(vdate = vdate,db = db[0])os.system(dir)#每個庫每個表的每個分區或者全表,只適合按天為第一級分區的表，根據自己的需求來tab_sql = u'''select distinct c.name,a.tbl_name,b.part_name,db_location_uri,case when b.part_name is not null then concat(replace(db_location_uri,'hdfs://nameservice1',''),'/',a.tbl_name,'/',b.part_name)else concat(replace(db_location_uri,'hdfs://nameservice1',''),'/',a.tbl_name) end as db_locationfrom hive.TBLS a left join hive.PARTITIONS b on a.tbl_id = b.tbl_id inner join hive.DBS c on a.db_id = c.db_idwhere c.name = '{db_name}' and (b.part_name like '%{vdate}%' or b.part_name is null) #特殊表and a.tbl_name not like ('%_tmp_%') and a.tbl_name not like ('%tmp_%') and a.tbl_name not like ('%_tmp%') and a.tbl_name not like ('%bak')#天表and (right(a.tbl_name,8) = '{vdate}' or (right(a.tbl_name,8) REGEXP '[^0-9.]')<>0)'''.format(vdate = vdate,db_name = db[0])#print(tab_sql)#列出每個庫的每個表cursor.execute(tab_sql)tab_list = cursor.fetchall()for tab in tab_list:#4點到9點不跑數,影響業務while True:if datetime.datetime.today().hour < 4 or datetime.datetime.today().hour > 9:breakelse:print('4點到9點不跑數，正在休眠！！！！')time.sleep(600)dir = 'mkdir -p /data10/hive_data_bak/{vdate}/{db}/{tab}'.format(vdate=vdate,db=db[0],tab=tab[1])os.system(dir)source = tab[4]if tab[2] == None:source = tab[4] + '/*'target = '/data10/hive_data_bak/{vdate}/{db}/{tab}/'.format(vdate=vdate,db=db[0],tab=tab[1])get_dfs = 'hadoop dfs -get {source} {target}'.format(source=source,target=target)print(datetime.datetime.now())print('正在備份{source}'.format(source=tab[4]))print(get_dfs)os.system(get_dfs) print(datetime.datetime.now())# 關閉連接 cursor.close() mysql_conn.close()print('完成')

總結

以上是生活随笔為你收集整理的hive数据文件备份的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。