需求：對(duì)某網(wǎng)站實(shí)現(xiàn)動(dòng)態(tài)爬取并能每天保存到文本文件中

解決方法：通過(guò)Python的BeautifulSoup、selenium完成該需求。

完整代碼：

import json import urllib.request import urllib.error from urllib.parse import quote from bs4 import BeautifulSoup from builtins import strfrom selenium import webdriver from selenium.webdriver.common.by import By from bs4 import BeautifulSoup from urllib.request import urlopen from selenium.common.exceptions import NoSuchElementException import re import time import datetime import sys sys.setrecursionlimit(1000000) import osfrom selenium.webdriver.common.keys import Keysdef getQuestionsLinks(driver):bs = BeautifulSoup(driver.page_source, 'lxml')AllInfo=bs.findAll('tr', {'class': 'bgcol'})linkyesterdaystr = str(datetime.date.today() - datetime.timedelta(days=0)) ###0代表當(dāng)天,1代表昨天print(linkyesterdaystr)firstdaystr=AllInfo[1].get_text().replace('\t','').replace('\n','|').split('|')[-2]print(firstdaystr)com = int(linkyesterdaystr.__eq__(firstdaystr))if (com == 0):returnfor info in AllInfo: #[0:2]if info.find('a', {'class':'xjxd_nr'}) is None:print("No usefull Info")else:paras=info.find('a', {'class': 'xjxd_nr'}).get('onclick').replace('detail(','').replace("'",'')[0:-2]listparas=paras.split(',')innerlink='http://www.shenl.com.cn/todetail?id='+listparas[0]+'&isSearchPassWord='+listparas[1]+'&tag='+listparas[2]innerDetail=info.get_text().replace('\t','').replace('\n','|').split('|')while '' in innerDetail:innerDetail.remove('')innerdate = time.strptime(innerDetail[-1], "%Y-%m-%d") #Struct_timeprint(innerdate)firstdaydate = time.strptime(firstdaystr, "%Y-%m-%d")print(firstdaydate)depart=(datetime.datetime(*firstdaydate[:3]) - datetime.datetime(*innerdate[:3])).daysif depart > 0: #如先獲得首頁(yè)里的開(kāi)始時(shí)間,如果首次出現(xiàn)時(shí)間小于開(kāi)始時(shí)間則停止爬蟲returnf.write('\t'.join(innerDetail) + "\t" + innerlink + "\n")try:print(type(driver.find_element(By.LINK_TEXT, "下一頁(yè)")))driver.find_element_by_xpath("//a[contains(text(),'下一頁(yè)')]").click()except NoSuchElementException:time.sleep(1)print("No more pages found")returntime.sleep(4)getQuestionsLinks(driver)if __name__ == '__main__':for n in range(0,1,1):import timeIsoTimeFormat = '%Y_%m_%d'f = open('G:\\temp\\Question_Incr_'+str(time.strftime(IsoTimeFormat))+'.txt', 'w', encoding='utf-8')driver = webdriver.Chrome("C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe")driver.get("http://www.shenl.com.cn/xjxdList")time.sleep(3)getQuestionsLinks(driver)driver.close()f.close()

總結(jié)

以上是生活随笔為你收集整理的基于某网站的信息爬取与保存的全部?jī)?nèi)容，希望文章能夠幫你解決所遇到的問(wèn)題。

如果覺(jué)得生活随笔網(wǎng)站內(nèi)容還不錯(cuò)，歡迎將生活随笔推薦給好友。