創建項目
scrapy startproject douyu

編寫items.py

1 import scrapy 2 3 class DouyuItem(scrapy.Item): 4 nickname = scrapy.Field() 5 imagelink = scrapy.Field() 6 imagePath = scrapy.Field()

創建基礎類的爬蟲

scrapy genspider douyutupian?capi.douyucdn.cn

?

手機抓包得到API接口，返回JSON格式數據

douyutupian.py

1 import scrapy 2 from douyu.items import DouyuItem 3 import json 4 5 6 class DouyumeinvSpider(scrapy.Spider): 7 name = "douyutupian" 8 allowed_domains = ["capi.douyucdn.cn"] 9 10 offset = 0 11 url = "http://capi.douyucdn.cn/api/v1/getVerticalRoom?limit=20&offset=" 12 13 start_urls = [url + str(offset)] 14 15 def parse(self, response): 16 # 把json格式的數據轉換為python格式，data段是列表 17 data = json.loads(response.text)["data"] 18 for each in data: 19 item = DouyuItem() 20 item["nickname"] = each["nickname"] 21 item["imagelink"] = each["vertical_src"] 22 23 yield item 24 25 self.offset += 20 26 yield scrapy.Request(self.url + str(self.offset), callback = self.parse)

管道文件
pipelines.py

1 import scrapy 2 from scrapy.utils.project import get_project_settings 3 from scrapy.pipelines.images import ImagesPipeline 4 import os 5 6 class ImagesPipeline(ImagesPipeline): 7 #def process_item(self, item, spider): 8 # return item 9 # 獲取settings文件里設置的變量值 10 IMAGES_STORE = get_project_settings().get("IMAGES_STORE") 11 12 def get_media_requests(self, item, info): 13 image_url = item["imagelink"] 14 yield scrapy.Request(image_url) 15 16 def item_completed(self, result, item, info): 17 image_path = [x["path"] for ok, x in result if ok] 18 19 os.rename(self.IMAGES_STORE + "/" + image_path[0], self.IMAGES_STORE + "/" + item["nickname"] + ".jpg") 20 21 item["imagePath"] = self.IMAGES_STORE + "/" + item["nickname"] 22 23 return item

?

?

settings.py 1 BOT_NAME = 'douyu' 2 3 SPIDER_MODULES = ['douyu.spiders'] 4 NEWSPIDER_MODULE = 'douyu.spiders' 5 6 DEFAULT_REQUEST_HEADERS = { 7 "User-Agent" : "DYZB/1 CFNetwork/808.2.16 Darwin/16.3.0" 8 } 9 10 ITEM_PIPELINES = { 11 'douyu.pipelines.ImagesPipeline': 300, 12 } 13 14 IMAGES_STORE = "IMAGES_STORE = "../../Images"

?

轉載于:https://www.cnblogs.com/wanglinjie/p/9240373.html

總結

以上是生活随笔為你收集整理的爬取斗鱼图片的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。