?

一、效果圖

二、代碼

import re from fake_useragent import UserAgent from lxml import etree import asyncio import aiohttp import pandas as pd# 定義一個類 定義使用的變量 定義get方法通過連接池進行網絡請求 class LianjiaSpider(object):def __init__(self):self.ua = UserAgent() # 獲取userAgent類self.head = {"User-Agent": self.ua.random}self._data = list() # 初始化listasync def get_page_count(self):result = await self.get("https://bj.lianjia.com/zufang/pg1")page_html = etree.HTML(result) # 解析網頁pageCount = page_html.xpath(".//div[@class='content__pg']/@data-totalpage")pageCount = list(map(int, pageCount))return pageCount[0]async def get(self, url): # 異步方法 當方法執行掛起線程執行完畢返回當前執行async with aiohttp.ClientSession() as session: # 線程連接池try:async with session.get(url, headers=self.head, timeout=3) as resp:if resp.status == 200:result = await resp.text()return resultexcept Exception as e:print(e.args)async def parse_html(self):count = await self.get_page_count()for page in range(1, count):url = "https://bj.lianjia.com/zufang/pg{}/".format(page)print("正在爬取{}".format(url))html = await self.get(url) # 獲取網頁內容html = etree.HTML(html) # 解析網頁await self.parse_page(html) # 匹配我們想要的數據print("正在存儲數據....")print(len(self._data))######################### 數據寫入data = pd.DataFrame(self._data)data.to_csv("鏈家網租房數據.csv", encoding='utf_8_sig') # 寫入文件######################### 數據寫入def run(self):loop = asyncio.get_event_loop() # 獲取到循環tasks = [asyncio.ensure_future(self.parse_html())] # 創建任務loop.run_until_complete(asyncio.wait(tasks))async def parse_page(self, html):rst = html.xpath(".//div[@class='content__list--item']") # //代表在任意路徑下查找節點為div，class為的所有元素print(rst) # ==> [<Element li at 0x133d9e0>, <Element li at 0x133d9b8>, <Element li at 0x133d990>] 找for div in rst:imgurl = div.xpath(".//a[@class='content__list--item--aside']/img/@src")title = div.xpath(".//a[@class='content__list--item--aside']/img/@alt")floor = div.xpath(".//span[@class='hide']/text()")price = div.xpath(".//span[@class='content__list--item-price']/em/text()")type = div.xpath(".//p[@class='content__list--item--des']/text()")if len(floor) > 0: # 有的沒有寫樓層會報錯加一層判斷currentFloor = floor[1].replace("\n", "").replace(" ", "")else:currentFloor = ''strinfo = [] #用于存儲多少平方米 朝向 幾室幾廳strinfo.clear()for str in type:info = str.replace(" ", "").replace("\n", "").replace("-", "")if info != '':strinfo.append(info)print(info)size = strinfo[0].replace(" ", "").replace("\n", "") # 30㎡direction = strinfo[1].replace(" ", "").replace("\n", "") # 南structure = strinfo[2].replace(" ", "").replace("\n", "") # 5室1廳2衛structure = re.findall(r'\d+', structure)print(structure)print("imgurl:" + imgurl[0]) # 圖片地址print("title:" + title[0]) # 標題print("price:" + price[0]) # 價錢print("currentFloor:" + currentFloor) # 樓層print(structure) # 分割幾室幾廳幾衛if len(structure) == 3:one_data = {"圖片地址": imgurl[0],"標題": title[0],"價格": price[0],"樓層": currentFloor,"大小": size,"朝向": direction,"室": structure[0],"廳": structure[1],"衛": structure[2]}elif len(structure) == 2:one_data = {"圖片地址": imgurl[0],"標題": title[0],"價格": price[0],"樓層": currentFloor,"大小": size,"朝向": direction,"室": structure[0],"廳": 0,"衛": structure[1]}self._data.append(one_data) # 添加數據if __name__ == '__main__':l = LianjiaSpider()l.run()

三、總結

搜索for循環

替換字符串

len長度函數

etree 根據class 解析，

\d正則表達提取數字

fake_useragent 模擬head的使用

協程的使用

list中str轉為int map的使用 list(map(int,strList)) list清除的方法 clear

if elseif 使用

?

近期有很多朋友通過私信咨詢有關Python學習問題。為便于交流，點擊藍色自己加入討論解答資源基地

?

總結

以上是生活随笔為你收集整理的Python爬取链家北京租房信息！北京租房都租不起啊！的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。