１．re實現

1 import re,os 2 import requests 3 from requests.exceptions import RequestException 4 5 MAX_PAGE = 10 #最大頁數 6 KEYWORD = 'python' 7 headers = { 8 'User-Agent': 9 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36' 10 } 11 file_name = 're_job51_python.txt' 12 13 # 獲取網頁源碼 14 def getHtml(page): 15 try: 16 url = 'https://search.51job.com/list/040000,000000,0000,00,9,99,{0},2,{1}.html?'.format(KEYWORD,page) 17 response = requests.get(url,headers=headers) 18 response.encoding = response.apparent_encoding 19 return response.text 20 except RequestException: 21 print('請求出錯') 22 return None 23 24 # 解析網頁源碼，得到目標信息 25 def getTarget(html): 26 reg = re.compile( 27 r'class="t1 ">.*? <a target="_blank" ' 28 'title="(.*?)".*? <span class="t2"><a target="_blank" ' 29 'title="(.*?)".*?<span ' 30 'class="t3">(.*?)</span>.*?<span ' 31 'class="t4">(.*?)</span>.*? <span ' 32 'class="t5">(.*?)</span>', 33 re.S) # 匹配換行符 34 target = re.findall(reg,html) 35 return target 36 37 38 # 保存到文本中 39 def save_to_txt(item): 40 with open(file_name,'a',newline='') as f: # newline參數防止兩行之間有空行 41 for i in range(len(item)): 42 # 最后一個元素換行，非最后則以＇,'隔開 43 if i == len(item)-1: 44 f.write(item[i]) 45 f.write('\n') 46 else: 47 f.write(item[i]+',') 48 49 def main(): 50 # 每次執行前檢查文件是否存在，存在則刪除 51 if os.path.exists(file_name): 52 os.remove(file_name) 53 54 # 分頁爬取 55 for page in range(MAX_PAGE+1): 56 html = getHtml(page) 57 content = getTarget(html) 58 for item in content: 59 save_to_txt(item) 60 61 if __name__ == '__main__': 62 main() View Code

?２．xpath實現

1 import os 2 import requests 3 from requests.exceptions import RequestException 4 from lxml import etree 5 import pymongo 6 from spiders.前程無憂.mongo_config import * 7 8 # mongo數據庫設置 9 client = pymongo.MongoClient(MONGO_URL) 10 db = client[MONGO_DB] 11 12 MAX_PAGE = 5 13 KEYWORD = 'python' 14 headers = { 15 'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '\ 16 'Chrome/63.0.3239.132 Safari/537.36' 17 } 18 file_name = 'xpath_job51_python.txt' 19 20 # 獲取網頁 21 def get_html(page): 22 try: 23 url = 'https://search.51job.com/list/040000,000000,0000,00,9,99,{},2,{}.html?'.format(KEYWORD,page) 24 response = requests.get(url,headers=headers) 25 response.encoding = response.apparent_encoding 26 return response.text 27 except RequestException: 28 return None 29 30 # 解析網頁 31 def parse_html(html): 32 # 構造xpath解析對象，可自動修整HTML文本 33 html = etree.HTML(html) 34 # 獲取文本 /text() 35 # 獲取屬性　／@href 36 # 獲取第i個標簽　／tar_name[i] 從1開始 37 # normalize-space－－＞去空格換行符 38 # position_name = html.xpath('normalize-space(//div[@class="el"]/p/span/a/text())') 39 40 # 職位名稱， 41 position_names = [] 42 for name in html.xpath('//div[@class="el"]/p/span/a/text()'): 43 position_name = name.strip() 44 position_names.append(position_name) 45 46 # 職位地址 47 position_urls = html.xpath('//div[@class="el"]/p/span/a/@href') 48 49 # 公司名稱 50 company_names = html.xpath('//div[@class="el"]/span[1]/a/text()') 51 52 # 公司地址 53 company_urls = html.xpath('//div[@class="el"]/span[1]/a/@href') 54 55 # 位置 56 locations = html.xpath('//div[@class="el"]/span[@class="t3"]/text()') 57 58 # 薪資 59 salarys = html.xpath('//div[@class="el"]/span[@class="t4"]/text()') 60 61 # 發布時間 62 release_dates = html.xpath('//div[@class="el"]/span[4]/text()') 63 64 result = zip(position_names,position_urls,company_names,company_urls,locations,salarys,release_dates) 65 return result 66 67 68 def save_to_txt(element): 69 with open(file_name,'a',newline='') as f: 70 for i in range(len(element)): 71 # data = ','.join(element[i]) 72 if i == len(element)-1: 73 f.write(element[i]) 74 f.write('\n') 75 else: 76 f.write(element[i]+',') 77 78 79 def save_to_mongo(element): 80 keys = ['position_name','position_url','company_name', 81 'company_url','location','salary','release_date'] 82 result = dict(zip(keys,list(element))) 83 if db[MONGO_TABLE_XPATH].insert(result): 84 print('數據成功存儲到mongo數據庫中') 85 return True 86 return False 87 88 # 遍歷字典元素 89 # for k,v in result.items(): 90 # print(k,':',v) 91 for key in result: 92 print(key,':',result[key]) 93 94 95 96 def main(): 97 if os.path.exists(file_name): 98 os.remove(file_name) 99 for page in range(1,MAX_PAGE+1): 100 html = get_html(page) 101 elements = parse_html(html) 102 if elements: 103 for element in elements: 104 save_to_txt(element) 105 save_to_mongo(element) 106 107 if __name__ == '__main__': 108 main() View Code

?

轉載于:https://www.cnblogs.com/ray-mmss/p/9373742.html

總結

以上是生活随笔為你收集整理的爬取前尘无忧python职位信息并保存到mongo数据库的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。