文章目錄 一、項目概況 1、項目介紹 2、項目要求 3、爬取字段 4、數據存儲 5、數據分析、轉化、演示 二、環境配置 1、JDK 2、Hadoop集群 3、zookeeper 4、hive 5、sqoop 6、flume 三、爬取數據 1、創建項目 2、編寫主程序進行數據爬取 3、編寫pipelines，進行數據保存 4、編寫settings，進行相關配置 5、編寫items，進行字典定義 6、爬取數據 四、數據分析 1、Flume收集日志 2、數據存儲到hdfs 3、數據分析 4、sqoop數據轉存 項目源碼：
一、項目概況
1、項目介紹
利用python編寫爬蟲程序，從招聘網站上爬取數據，將數據存入到MongoDB數據庫中，將存入的數據作一定的數據清洗后做數據分析，最后將分析的結果做數據可視化。
2、項目要求
1、具體要求：招聘網站上的數據，選擇兩個招聘網站。招聘網站包括：智聯招聘、前程無憂、應屆生求職、拉勾、中華英才網。
3、爬取字段
1）、具體要求：職位名稱、薪資水平、招聘單位、工作地點、工作經驗、學歷要求、工作內容（崗位職責）、任職要求（技能要求）。
4、數據存儲
1)、具體要求：將爬取的數據存儲到hdfs上。利用flume收集日志。若整個過程利用mangdb轉hdfs則為15分。
5、數據分析、轉化、演示
具體要求（要求：1、利用hive進行分析，2、將hive分析結果利用sqoop技術存儲到mysql數據庫中，并最后顯示分析結果。）：
二、環境配置
1、JDK
https://blog.csdn.net/lnxcw/article/details/106336186
2、Hadoop集群
https://blog.csdn.net/lnxcw/article/details/106337928
3、zookeeper
https://blog.csdn.net/lnxcw/article/details/106801692
4、hive
https://blog.csdn.net/lnxcw/article/details/106445419
5、sqoop
https://blog.csdn.net/lnxcw/article/details/106537791
6、flume
https://blog.csdn.net/lnxcw/article/details/106540916
三、爬取數據
1、創建項目
scrapy startproject job02
cd job02
scrapy genspider chinahr chinahr.com
2、編寫主程序進行數據爬取
3、編寫pipelines，進行數據保存
4、編寫settings，進行相關配置
5、編寫items，進行字典定義
6、爬取數據
數據量：389000
四、數據分析
1、Flume收集日志
編寫conf文件，監聽文件目錄，并將監聽數據存儲到hdfs
2、數據存儲到hdfs
3、數據分析
（1）、hive表創建
create table job_post(position varchar(100),salary varchar(100),company varchar(100),site varchar(100),experience varchar(100),education varchar(100),jobInfo varchar(1000) )row format delimited fields terminated by ','tblproperties("skip.header.line.count"="1");
create table table1(position varchar(100),max_list varchar(100),avg_list varchar(100),min_list varchar(100))row format delimited fields terminated by ',';
create table table2(site varchar(100),number varchar(100))row format delimited fields terminated by ',';
create table table3(experience varchar(100),max_list varchar(100),avg_list varchar(100),min_list varchar(100))row format delimited fields terminated by ',';
create table table4(data varchar(100),number varchar(100))row format delimited fields terminated by ',' ;
create table ex(position string,day string)row format delimited fields terminated by ',';
數據導入：
load data local inpath "/opt/data/data.csv" overwrite into table job_post;
load data local inpath "/opt/data/數據崗位薪資水平.txt" overwrite into table table1;
load data local inpath "/opt/data/數據分析崗位分布.txt" overwrite into table table2;
load data local inpath "/opt/data/大數據經驗薪資水平.txt" overwrite into table table3;
load data local inpath "/opt/data/大數據相關崗位需求.txt" overwrite into table table4;
load data local inpath "/opt/data/data1.csv" overwrite into table ex;
（2）、mysql表創建
# -*- coding: utf-8 -*-
# mysql創建表import pymysqldef Create_table_mysql(self):print("連接MySQL數據庫創建hive對應表")# 連接數據庫conn = pymysql.connect(host='192.168.1.101', # 主機名port=3306, # 端口號(默認的)user='root', # 用戶名passwd='1', # 密碼db='job_post', # 數據庫名charset='utf8', # 這里設置編碼是為了輸出中文)cursor = conn.cursor()def table1():sql1 = '''create table table1(position varchar(100),max_list varchar(100),avg_list varchar(100),min_list varchar(100))'''cursor.execute(sql1)print("表1創建成功！")def table2():sql2 = '''create table table2(site varchar(100),number varchar(100))'''cursor.execute(sql2)print("表2創建成功！")def table3():sql3 = '''create table table3(experience varchar(100),max_pay_level varchar(100),average_pay_level varchar(100),min_pay_level varchar(100))'''cursor.execute(sql3)print("表3創建成功！")def table4():sql4 = '''create table table4(data varchar(100),number varchar(100))'''cursor.execute(sql4)print("表4創建成功！")# commit 修改table1()table2()table3()table4()conn.commit()# 關閉游標cursor.close()print("創建成功！")
（3）、數據行業薪資
# -*- coding: utf-8 -*-
# 數據行業薪資
import re
from impala.dbapi import connect
from numpy import *
from pyecharts.charts import Bar
from pyecharts import options as optsdef shuju():conn = connect(host='192.168.1.101',port=10000,auth_mechanism='LDAP',user='root',password='1',database='mongodb')cursor = conn.cursor()# 最低工資all_min_salary_list = []# 最高工資all_max_salary_list = []# 平均工資all_average_salary_list = []# 崗位all_addr_list = []# 求出數據分析相關崗位的最值def shujufenxi():# 最低工資min_salary_list = []# 最高工資max_salary_list = []# 平均工資average_salary_list = []sql01 = "select salary from job_post where position like '%數據分析%' and salary like '%/%'" cursor.execute(sql01)results = cursor.fetchall()# 數據清洗for i in results:if "萬/月" in i[0]:wan_yue = re.findall(r"(.*)萬/月", i[0])[0]num_list = wan_yue.split("-")num_1 = eval(num_list[0]) * 10num_2 = eval(num_list[1]) * 10i = str(num_1) + "-" + str(num_2) + "千/月"elif "萬/年" in i[0]:wan_yue = re.findall(r"(.*)萬/年", i[0])[0]num_list = wan_yue.split("-")num_1 = eval(num_list[0]) * 10 / 12num_2 = eval(num_list[1]) * 10 / 12i = str(int(num_1)) + "-" + str(int(num_2)) + "千/月"elif "千/月" in i[0]:wan_yue = re.findall(r"(.*)千/月", i[0])[0]num_list = wan_yue.split("-")num_1 = eval(num_list[0])num_2 = eval(num_list[1])i = str(num_1) + "-" + str(num_2) + "千/月"else:continuemin_salary = i.split("-")[0]max_salary = re.findall(r'([\d+\.]+)', (i.split("-")[1]))[0]average_salary = "{:.1f}".format((float(min_salary) + float(max_salary)) / 2)# 將獲取的數據分別寫入min_salary_list.append(eval(min_salary))max_salary_list.append(eval(max_salary))average_salary_list.append(eval(average_salary))listjob = '數據分析'listmin = min(min_salary_list)listmax = max(max_salary_list)listaverage = int(mean(average_salary_list))all_min_salary_list.append(listmin)all_max_salary_list.append(listmax)all_average_salary_list.append(listaverage)all_addr_list.append(listjob)print(listjob, listmin, listaverage, listmax)file = open('數據崗位薪資水平.txt', 'w', encoding='utf-8')file.write('{},{},{},{}'.format(listjob, listmax, listaverage, listmin))file.write('\n')file.close()# 關閉游標conn.commit()# 求出大數據開發工程師相關崗位的最值def bigdata():# 最低工資min_salary_list = []# 最高工資max_salary_list = []# 平均工資average_salary_list = []sql = "select salary from job_post where position like '%大數據開發工程師%' and salary like '%/%'"cursor.execute(sql)results = cursor.fetchall()for i in results:if "萬/月" in i[0]:wan_yue = re.findall(r"(.*)萬/月", i[0])[0]num_list = wan_yue.split("-")num_1 = eval(num_list[0]) * 10num_2 = eval(num_list[1]) * 10i = str(num_1) + "-" + str(num_2) + "千/月"elif "萬/年" in i[0]:wan_yue = re.findall(r"(.*)萬/年", i[0])[0]num_list = wan_yue.split("-")num_1 = eval(num_list[0]) * 10 / 12num_2 = eval(num_list[1]) * 10 / 12i = str(int(num_1)) + "-" + str(int(num_2)) + "千/月"elif "千/月" in i[0]:wan_yue = re.findall(r"(.*)千/月", i[0])[0]num_list = wan_yue.split("-")num_1 = eval(num_list[0])num_2 = eval(num_list[1])i = str(num_1) + "-" + str(num_2) + "千/月"else:continuemin_salary = i.split("-")[0]max_salary = re.findall(r'([\d+\.]+)', (i.split("-")[1]))[0]average_salary = "{:.1f}".format((float(min_salary) + float(max_salary)) / 2)# 將獲取的數據分別寫入min_salary_list.append(eval(min_salary))max_salary_list.append(eval(max_salary))average_salary_list.append(eval(average_salary))listjob = '大數據開發工程師'listmin = min(min_salary_list)listmax = max(max_salary_list)listaverage = int(mean(average_salary_list))all_min_salary_list.append(listmin)all_max_salary_list.append(listmax)all_average_salary_list.append(listaverage)all_addr_list.append(listjob)print(listjob, listmin, listaverage, listmax)file = open('數據崗位薪資水平.txt', 'a+', encoding='utf-8')file.write('{},{},{},{}'.format(listjob, listmax, listaverage, listmin))file.write('\n')file.close()conn.commit()# 求出數據采集相關崗位最值def shujucaiji():# 最低工資min_salary_list = []# 最高工資max_salary_list = []# 平均工資average_salary_list = []sql = "select salary from job_post where position like '%數據采集%' and salary like '%/%'" # .format(Job)cursor.execute(sql)results = cursor.fetchall()for i in results:if "萬/月" in i[0]:wan_yue = re.findall(r"(.*)萬/月", i[0])[0]num_list = wan_yue.split("-")num_1 = eval(num_list[0]) * 10num_2 = eval(num_list[1]) * 10i = str(num_1) + "-" + str(num_2) + "千/月"elif "萬/年" in i[0]:wan_yue = re.findall(r"(.*)萬/年", i[0])[0]num_list = wan_yue.split("-")num_1 = eval(num_list[0]) * 10 / 12num_2 = eval(num_list[1]) * 10 / 12i = str(int(num_1)) + "-" + str(int(num_2)) + "千/月"elif "千/月" in i[0]:wan_yue = re.findall(r"(.*)千/月", i[0])[0]num_list = wan_yue.split("-")num_1 = eval(num_list[0])num_2 = eval(num_list[1])i = str(num_1) + "-" + str(num_2) + "千/月"else:continuemin_salary = i.split("-")[0]max_salary = re.findall(r'([\d+\.]+)', (i.split("-")[1]))[0]average_salary = "{:.1f}".format((float(min_salary) + float(max_salary)) / 2)# 將獲取的數據分別寫入min_salary_list.append(eval(min_salary))max_salary_list.append(eval(max_salary))average_salary_list.append(eval(average_salary))listjob = '數據采集'listmin = min(min_salary_list)listmax = max(max_salary_list)listaverage = int(mean(average_salary_list))all_min_salary_list.append(listmin)all_max_salary_list.append(listmax)all_average_salary_list.append(listaverage)all_addr_list.append(listjob)print(listjob, listmin, listaverage, listmax)file = open('數據崗位薪資水平.txt', 'a+', encoding='utf-8')file.write('{},{},{},{}'.format(listjob, listmax, listaverage, listmin))file.write('\n')file.close()conn.commit()shujufenxi()bigdata()shujucaiji()# 關閉游標cursor.close()conn.commit()print(all_addr_list)print(all_min_salary_list)print(all_average_salary_list)print(all_max_salary_list)bar = Bar(init_opts=opts.InitOpts(width="1000px", height="800px"),)bar.set_global_opts(title_opts=opts.TitleOpts(title="數據行業薪資水平", subtitle="單位 千/月"),xaxis_opts=opts.AxisOpts(axislabel_opts={"rotate": 45}),)bar.add_xaxis(all_addr_list)bar.add_yaxis("最高薪資", all_max_salary_list)bar.add_yaxis("最低薪資", all_min_salary_list)bar.add_yaxis("平均薪資", all_average_salary_list)bar.render("數據行業薪資水平.html")print("數據行業薪資水平.html文件")if __name__ == "__main__":shuju()
（4）大數據行業占地區比重
4、sqoop數據轉存
將hive數據庫里面的內容導出到mysql
sh sqoop export --connect jdbc:mysql://192.168.1.101:3306/job_post?characterEncoding=UTF-8 --username root --password 1 --table table2 --export-dir /home/apache-hive-2.3.6-bin/warehouse/mongodb.db/table2 --input-fields-terminated-by ','
項目源碼：
鏈接：https://pan.baidu.com/s/1R-Nwf4QoC5KcFUokF_4pFw
總結
以上是生活随笔 為你收集整理的大数据实训项目 的全部內容，希望文章能夠幫你解決所遇到的問題。
如果覺得生活随笔 網站內容還不錯，歡迎將生活随笔 推薦給好友。
