這個爬蟲只是選定熱門小說，不支持自選搜索下載，日后會補充并改進。

選定小說網址:

• 筆趣閣

爬取:

• 需要導入的包

import requests from lxml import etree import os import re import time import datetime

• 獲取主頁面上的小說分類地址及名稱

def main_html(url,headers): # 獲取首頁小說分類地址nav_name_href = "//*[@id='wrapper']/div[2]/ul/li/a//@href"response = requests.get(url,headers=headers).texthtml = etree.HTML(response)nav_name = html.xpath(nav_name_href)[2:-1]return nav_name #返回首頁小說分類地址

• 獲取主頁面下的熱門小說列表。

def next_html(url,headers): # 該分類下的熱門小說xpath_book = '//*[@id="newscontent"]/div[2]/ul/li/span/a/text()' # 書名稱xpath_url = '//*[@id="newscontent"]/div[2]/ul/li/span/a/@href' # 書的urlxpath_id = '//*[@id="newscontent"]/div[2]/ul/li/span/text()' # 小說作者name_url = [] # 用來存放小說的名稱與地址rep = requests.get(url=url,headers=headers).texthtml = etree.HTML(rep)r_book = html.xpath(xpath_book)r_url = html.xpath(xpath_url)BookAndUrl = dict(zip(r_book,r_url))print(r_book)try:s_book = input("============請輸入你想要看的小說全稱============"+"\n")s_url = BookAndUrl[s_book]name_url.append(s_book)name_url.append(s_url)return name_url # 獲取想要的書名與地址,用列表返回except:KeyError(print("輸入有誤"))return False

• 小說本說的章節與連接

def story_html(url, headers): # 小說章節與urlxpath_text = '//*[@id="list"]/dl/dd//text()'xpath_url = '//*[@id="list"]/dl/dd//a/@href'rep = requests.get(url=url,headers=headers).texthtml = etree.HTML(rep)x_text = html.xpath(xpath_text)x_url = html.xpath(xpath_url)text_dict = dict(zip(x_text[9:-1],x_url[9:-1])) # 去除掉章節前的話語導語之類的。return text_dict

• 下載小說本說(只獲取有效章節)

def download_html(bookname,url_name_dict):num_d = 0 # 用來給下載文件編號排序。title_Re = r"[\u7b2c](.|\n)*[\u7ae0]" # 匹配正確的章節名稱。以第開頭，章結尾key_count = len(url_name_dict.keys()) # 文章總數for key,values in url_name_dict.items():time.sleep(0.5) # 設置休眠時間，防止下載過快內容為空,下載時候看著舒服一點。rep = requests.get(url=values,headers=headers).texthtml = etree.HTML(rep)xpath2 ='//*[@id="content"]//text()'the_end = html.xpath(xpath2)res = str(the_end)r = re.findall(u'[\u4e00-\u9fa5].+?', res) # 匹配中文，去除掉下載時候的其他轉義字符，原生字符the_fi = "".join(r)the_final = list(the_fi)if not os.path.exists(os.getcwd() + "\\" + bookname): # 以小說名創建文件夾os.makedirs(os.getcwd() + "\\" + bookname)if (re.match(title_Re,key)): # 獲取所有有效章節num_d += 1try:with open(os.getcwd() + "\\" + bookname + "\\" + str(num_d)+key + ".txt", "w+", encoding="utf-8") as fp:for num in range(len(the_final)): # 每50個字換行一次，隨自己調if(num%50==0 and num!=0):fp.write("\n" + the_final[num])else:fp.write(the_final[num])print("{:10}{:>15}{:>20}".format(key,"下載成功","已完成:"+str(num_d)+"/"+str(key_count)))except (TimeoutError,IndexError):passelse:key_count = key_count -1print("{:10}{:>15}".format(key,"無效章節","已完成"))

• 寫一個判斷函數；只你呢個獲取固定的小說分類；

def is_num(num): # 用來判斷輸入是否合格try:new_num = float(num)-1if(float(new_num)):try:if(float(num)>7):return Falseelif(float(new_num)<=0):return Falseexcept IndexError:passexcept (ValueError,IndexError):passtry:import unicodedataunicodedata.numeric(num)return Trueexcept (TypeError, ValueError,IndexError):passreturn False

• 程序運行開始顯示

def main_menu():num = input("""========================按數字輸入想看的小說類型========================1.玄幻小說 2.修真小說 3.都市小說 4.穿越小說 5.網游小說 6.科幻小說 7.言情小說""")return num

• 主函數

if __name__ == '__main__':start_time = datetime.datetime.now()main_url ="http://www.b520.cc/"headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36"}num = main_menu() # 拿到想看的小說類型main_text_url = main_html(url=main_url,headers=headers) # 地址。if(is_num(num)): # 判斷輸入數字合不合格。num =int(num)num = num-1text_2_url = main_text_url[num]text_2_url = "http://"+ text_2_url[2:-1]+"/"next_url = next_html(text_2_url, headers=headers)next_3 =story_html(next_url[1],headers=headers)download_html(next_url[0], next_3)end_time = datetime.datetime.now()print("############################共耗時%ss############################" % (end_time-start_time))

這個程序可以直接復制運行。

總結

以上是生活随笔為你收集整理的python使用requests+xpath爬取小说并下载的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。