處理scrapy中包括多個pipeline時如何讓spider執行制定的pipeline管道
１:創建一個裝飾器
from scrapy.exceptions import DropItem
import functools
'''
當有多個pipeline時,判斷spider如何執行指定的管道
'''

def check_spider_pipeline(process_item_method):
??? @functools.wraps(process_item_method)
??? def wrapper(self, item, spider):
??????? # message template for debugging
??????? msg = '%%s %s pipeline step' % (self.__class__.__name__,)
??????? if self.__class__ in spider.pipeline:#判斷要執行的spider中是否包含所需的pipeline　如果有則執行否則拋出DropItem信息
??????????? spider.logger.debug(msg % 'executing')
??????????? return process_item_method(self,item,spider)
??????? # otherwise, just return the untouched item (skip this step in
??????? # the pipeline)
??????? else:
??????????? spider.logger.debug(msg % 'skipping')
??????????? raise DropItem("Missing pipeline property")
??? return wrapper
2:在每個spider所在的類中添加一個pipeline數組，里面包含要執行的pipeline的名字
?-*- coding: utf-8 -*-
from scrapy.spiders import CrawlSpider,Rule
# from scrapy.selector import Selector
from ..items import BotcnblogsItem,BotItem
from scrapy.linkextractors import LinkExtractor
import re
from ..BotcnblogsPipeline import BotcnblogsPipeline
class CnblogsSpider(CrawlSpider):
??? pipeline = set([BotcnblogsPipeline,])
??? #爬蟲名稱
??? name = "cnblogs"
??? #設置允許的域名
??? allowed_domains = ["cnblogs.com"]
??? #設置開始爬去的頁面
??? start_urls = (
??????? 'http://www.cnblogs.com/fengzheng/',
??? )
?? ?
??? rules=(
?????????? Rule(LinkExtractor(allow=('fengzheng/default.html\?page\=([\d]+)')),callback='parse_item',follow=True),
#??????????? Rule(LinkExtractor(allow=('fengzheng/p/([\d]+).html')),callback='parse_info',follow=True),
?????????? )
????????? ?
3:在要執行的pipeline中的process_item方法加上裝飾器，這樣就可以過濾pipeline了
import json
from .checkpipe import? check_spider_pipeline
class BotcnblogsPipeline(object):
?? ?
??? def __init__(self):
??????? self.file=open('jd.json','w+')
?????? ?
??? @check_spider_pipeline
??? def process_item(self,item,spider):
??????? #此處如果有中文的話，要加上ensure_ascii=False參數，否則可能出現亂碼
??????? record=json.dumps(dict(item),ensure_ascii=False)+"\n"
??????? self.file.write(record)
??????? return item
?? ?
??? def open_spider(self,spider):
??????? print("打開爬蟲了")
?????? ?
??? def close_spider(self,spider):
??????? print("關閉爬蟲")
??????? self.file.close()

?

具體例子可以參考其中的cnblogs　spider的例子　下載

轉載于:https://www.cnblogs.com/fly-kaka/p/5216791.html

總結

以上是生活随笔為你收集整理的scrapy 让指定的spider执行指定的pipeline的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。