粘貼可用，目前因?yàn)闊o性能要求無多線程操作

# @Time : 2022/9/28 21:56 # @Author : pzh # @File : ali_cvd_detail.py # -*- coding: utf-8 -*-```java import re import requests from random import randint import time from lxml import etree from cache import memory_cache import datetime import threading from LoggingUtils import logger import math import osdef get_onepage_content(url):user_agent = ['Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.7113.93 Safari/537.36','Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0','Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4482.0 Safari/537.36 Edg/92.0.874.0']try:response = requests.get(url, headers={'User-Agent':user_agent[randint(0,2)]},timeout=60)if response.status_code == 200:return response.textreturnexcept Exception:returndef show_cve_content(res):e = etree.HTML(res);list = e.xpath("//div[@class='list_list']/ul/li/div/p/a/text()")return listpath ="E://Asiainfo//webappss//pythonDemo//Test//ali_cnnvd_";# 分頁數(shù)據(jù)放到一個文件里面 def save_content_to_text(content,now_time):with open(path+now_time+'.txt','a+') as f:f.write(content + '\n')pageSize=30def main():#每天凌晨3點(diǎn)執(zhí)行mainlogger.info("當(dāng)前任務(wù)為："+'120'+"秒執(zhí)行一次")timer = threading.Timer(120, main)timer.start()#獲取任務(wù)的總數(shù)startTime = time.time()value = memory_cache.get_value("TotalCNNVD")logger.info("當(dāng)前任務(wù)獲取上次總數(shù)為："+str(value))#循環(huán)的所有數(shù)據(jù)htmls ="";#獲取最新的總數(shù)htmlss = get_onepage_content('http://www.cnnvd.org.cn/web/vulnerability/queryLds.tag?pageno=1&pageSize=1&repairLd=');e = etree.HTML(htmlss)totalSizeList=e.xpath("//div[@class='page']/a[1]/text()")totalList = re.findall(r"[1-9]+\.?[0-9]*",str(totalSizeList).replace(",",""))total = totalList [0]logger.info("當(dāng)前最新任務(wù)獲取總數(shù)為："+total)#判斷，進(jìn)入任務(wù)，如果不一樣說明有更新的，取最新的減去上次的if(str(value) != str(total)):#存入這次的total的數(shù)量，下次進(jìn)入時使用memory_cache.set_value("TotalCNNVD",total,86400)#如果取出來的值是None說明是第一次進(jìn)入，取全量if(str(value) == 'None') :Num = int(total)else:Num = int(total)-int(value)#正式需要把2替換成pageNumpageNum = math.ceil(Num / pageSize)logger.info("需要循環(huán)的頁數(shù)為："+str(pageNum))now_time=datetime.datetime.now().strftime('%Y-%m-%d')logger.info("當(dāng)前任務(wù)文件路徑及名稱："+path+now_time+'.txt')for pagenum in range(1, 2) :url = f'http://www.cnnvd.org.cn/web/vulnerability/queryLds.tag?pageno={pagenum}&pageSize={pageSize}&repairLd='html = get_onepage_content(url);htmls=html+htmls;#如果當(dāng)天已經(jīng)存在，就刪除if (os.path.exists(path+now_time+".txt")):os.remove(path+now_time+".txt")os.close()for content in show_cve_content(htmls):htmldetail = get_onepage_content(f'http://www.cnnvd.org.cn/web/xxk/ldxqById.tag?CNNVD={content}')e = etree.HTML(htmldetail)#漏洞標(biāo)題biaoti = e.xpath("//div[@class='detail_xq w770']/h2/text()")[0].strip()#漏洞描述loudong = e.xpath("//div[@class='d_ldjj']/p[2]/text()")[0].strip()#漏洞公告jianyi = e.xpath("//div[@class='d_ldjj m_t_20']/p[@class='ldgg']/a/text()")#受影響實(shí)體gongji1 = e.xpath("//div[@class='vulnerability_list']/ul/p/text()")[0].strip()#補(bǔ)丁try :gongji3 = e.xpath("//div[@class='vulnerability_list']/ul/li/div/a/text()")[0].strip()except Exception: gongji3='暫無'#CNNVD編號gongji4 = e.xpath("//div[@class='detail_xq w770']/ul/li[1]/span/text()")[0].strip()#危害等級gongji5 = e.xpath("//div[@class='detail_xq w770']/ul/li[2]/a[1]/text()")[0].strip()#CVE編號gongji6 = e.xpath("//div[@class='detail_xq w770']/ul/li[3]/a/text()")[0].strip()#漏洞類型gongji7 = e.xpath("//div[@class='detail_xq w770']/ul/li[4]/a/text()")[0].strip()#威脅類型gongji8 = e.xpath("//div[@class='detail_xq w770']/ul/li[6]/a/text()")[0].strip()content = str(loudong)+","+str(biaoti)+","+str(gongji1)+",補(bǔ)丁："+str(gongji3)+","+str(gongji4)+",危害等級："+str(gongji5)+",CVE編號："+str(gongji6)+",漏洞類型："+str(gongji7)+",威脅類型："+str(gongji8)save_content_to_text(str(content),now_time)endTime = time.time()logger.info("cnnvd循環(huán)",str(pagenum)+"完成,耗時：",endTime - startTime)else: logger.info("無更新")# 獲取現(xiàn)在時間 now_time = datetime.datetime.now() # 獲取明天時間 next_time = now_time + datetime.timedelta(days=+1) next_year = next_time.date().year next_month = next_time.date().month next_day = next_time.date().day # 獲取明天3點(diǎn)時間 next_time = datetime.datetime.strptime(str(next_year)+"-"+str(next_month)+"-"+str(next_day)+" 03:00:00", "%Y-%m-%d %H:%M:%S") # # 獲取昨天時間 # last_time = now_time + datetime.timedelta(days=-1)# 獲取距離明天3點(diǎn)時間，單位為秒 timer_start_time = (next_time - now_time).total_seconds() logger.info("獲取距離明天3點(diǎn)時間（秒）:"+str(timer_start_time))#定時器,參數(shù)為(多少時間后執(zhí)行，單位為秒，執(zhí)行的方法) timer = threading.Timer(timer_start_time, main) timer.start()if __name__ == "__main__":main()

總結(jié)

以上是生活随笔為你收集整理的python爬取cnnvd，粘贴可用的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯，歡迎將生活随笔推薦給好友。