用selenium下载数据，所以是下载中间件 ~~~ from selenium import webdriver from scrapy.http import HtmlResponse class JSPageMiddleware(object): #通过chrome请求动态网页 def process_request(self, request, spider): if spider.name == "jobbole": # browser = webdriver.Chrome(executable_path="D:/Temp/chromedriver.exe") spider.browser.get(request.url) import time time.sleep(3) print ("访问:{0}".format(request.url)) return HtmlResponse(url=spider.browser.current_url, body=spider.browser.page_source, encoding="utf-8", request=request) ~~~ 建立spider的时候建立webdriver ~~~ def __init__(self): self.browser = webdriver.Chrome(executable_path="D:/Temp/chromedriver.exe") super(JobboleSpider, self).__init__() //信号映射，当signals.spider_closed信号发生时，调用spider_closed dispatcher.connect(self.spider_closed, signals.spider_closed) // 自定义函数 def spider_closed(self, spider): #当爬虫退出的时候关闭chrome print ("spider closed") self.browser.quit() ~~~