from urllib import request from multiprocessing import Pool,Manager import functools from http import cookiejar import json,re#引入cookjar來放在cookie過期 cookie = cookiejar.CookieJar() handle_cookie=request.HTTPCookieProcessor(cookie)opener = request.build_opener(handle_cookie) #添加UA來簡單防止反爬 opener.addheaders =[('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64; rv:61.0) Gecko/20100101 Firefox/61.0')]def write_to_file(dic): # ? ?將字典轉(zhuǎn)換成json字符串寫入文件方便調(diào)用dic_json ?= json.dumps(dic,ensure_ascii=False)with open('maoyan.txt','a',encoding='utf-8') as f:f.write(dic_json+'\n')return Nonedef get_file(html): # ? ?創(chuàng)建正則表達(dá)式對象對得到的HTML進(jìn)行匹配pattern =re.compile('<p class="name">[\s\S]*?title="([\s\S]*?)"[\s\S]*? ''<p class="star">([\s\S]*?)</p>[\s\S]*?<p class="releasetime">([\s\S]*?)</p>')result = re.findall(pattern,html) # ? ?將匹配的可迭代對象進(jìn)行遍歷,生成生成器傳出for i in result:yield {'title':i[0],'autor':i[1].strip(),'time':i[2],}def get_html(lock,offset): # ? ?拼接目標(biāo)URL 使用request的方法爬取信息url = 'http://maoyan.com/board/4?offset='+str(offset)req = request.Request(url) # ? ?判斷是否成功請求網(wǎng)站if opener.open(req).code ==200:html = opener.open(req).read().decode()else:return Nonefor i in get_file(html): # ? ? ? ?通過加鎖的形式來保證寫入文件的準(zhǔn)確率lock.acquire()write_to_file(i)lock.release()if __name__ == '__main__': # ? ?從Manage里獲取lock鎖 并且使用functools 對原函數(shù)增加新變量manager = Manager()lock = manager.Lock()new_get_html =functools.partial(get_html,lock) # ? ?使用進(jìn)程池加快獲取速度pool = Pool()pool.map(new_get_html,[i*10 for i in range(10)]) #關(guān)閉進(jìn)程池pool.close()pool.join()

總結(jié)

以上是生活随笔為你收集整理的使用进程池抓取猫眼数据的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯，歡迎將生活随笔推薦給好友。