#### spider ``` import scrapy from scrapy_tencent.items import ScrapyTencentItem class TencentSpider(scrapy.Spider): name = 'tencent' allowed_domains = ['tencent.com'] start_urls = ['https://hr.tencent.com/position.php?&start=0#a'] def parse(self, response): tr_list = response.xpath("//tr")[1:-2] for tr in tr_list: item = ScrapyTencentItem() item["name"] = tr.xpath("./td[1]/a/text()").extract_first() item["detail_url"] = "https://hr.tencent.com/"+tr.xpath("./td[1]/a/@href").extract_first() item["category"] = tr.xpath("./td[2]/text()").extract_first() item["city"] = tr.xpath("./td[4]/text()").extract_first() #yield item yield scrapy.Request( item["detail_url"], callback=self.parse_detail, meta={"item1": item} ) next_url = 'https://hr.tencent.com/' + response.xpath("//*[@id='next']/@href").extract_first() if 'javascript:;' not in next_url: yield scrapy.Request(url=next_url, callback=self.parse) def parse_detail(self, response): item = response.meta['item1'] item['duty'] = response.xpath('//tr[3]/td/ul/li/text()').extract()[0] item['require'] = response.xpath('//tr[4]/td/ul/li/text()').extract()[0] # 返回item给引擎 yield item ``` #### items ``` import scrapy class ScrapyTencentItem(scrapy.Item): # define the fields for your item here like: name = scrapy.Field() category = scrapy.Field() city = scrapy.Field() detail_url = scrapy.Field() duty = scrapy.Field() require = scrapy.Field() ``` #### ``` import json from scrapy_tencent.items import ScrapyTencentItem class ScrapyTencentPipeline(object): def open_spider(self, item): self.file = open("tencent.json", "w") def process_item(self, item, spider): # 保存json到文件 if isinstance(item, ScrapyTencentItem): item_json = json.dumps(dict(item), ensure_ascii=False) + ',\n' self.file.write(item_json) return item def close_spider(self): self.file.close() ```