- middlewares.py ``` import base64 # 代理服务器 proxyServer = "http://http-cla.abuyun.com:9030" # 代理隧道验证信息 proxyUser = "H25521VB95692ULC" proxyPass = "2BFD642305EED6AE" # for Python2 # proxyAuth = "Basic " + base64.b64encode(proxyUser + ":" + proxyPass) # for Python3 proxyAuth = "Basic " + base64.urlsafe_b64encode(bytes((proxyUser + ":" + proxyPass), "ascii")).decode("utf8") class ProxyMiddleware(object): def process_request(self, request, spider): request.meta["proxy"] = proxyServer request.headers["Proxy-Authorization"] = proxyAuth ``` - settings.py ``` RANDOM_UA_TYPE = 'random' DOWNLOADER_MIDDLEWARES = { 'meituan.middlewares.ProxyMiddleware': 1, # 代理中间件 'meituan.MidWare.user_agent_middlewares.RandomUserAgentMiddlware': 543, # 请求头中间件 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware':None, } """ 启用限速设置 """ AUTOTHROTTLE_ENABLED = True AUTOTHROTTLE_START_DELAY = 0.2 # 初始下载延迟 DOWNLOAD_DELAY = 0.2 # 每次请求间隔时间 ```