~~~ # -*- coding: utf-8 -*- import scrapy from scrapy_guokr.items import ScrapyGuokrItem class GuokrSpider(scrapy.Spider): name = 'guokr' allowed_domains = ['guokr.com'] start_urls = ['https://www.guokr.com/ask/highlight/'] def parse(self, response): node_list = response.xpath('//ul[@class="ask-list-cp"]/li') for node in node_list: item = ScrapyGuokrItem() item['focus_num'] = node.xpath('.//p[@class="ask-focus-nums"]/span/text()').extract()[0] item['answer_num'] = node.xpath('.//p[@class="ask-answer-nums"]/span/text()').extract()[0] item['title'] = node.xpath('.//h2/a/text()').extract()[0] item['href'] = node.xpath('.//h2/a/@href').extract()[0] # strip 去出空白字符 item['content'] = node.xpath('.//p[@class="ask-list-summary"]/text()').extract()[0].strip() # yield item # 详情页 yield scrapy.Request( item['href'], callback=self.parse_detail, meta={"item": item} ) # 下一页 next_url = response.xpath('//a[text()="下一页"]/@href').extract_first() if next_url is not None: yield response.follow(next_url, callback=self.parse) def parse_detail(self, response): item = response.mete['item'] # 包含 div_list = response.xpath("//div[contains(@class, 'answer gclear')]") for div in div_list: pass ~~~