### 1.说明 利用selenium抓取淘宝商品并用PyQuery解析得到商品的图片、名称、价格、购买人数、店铺名称、店铺所在地信息，将其保存在MongoDB ### 2.准备 [安装selenium](/1kai-fa-huan-jing-pei-zhi/12-qing-qiu-ku-de-an-zhuang/122-seleniumde-an-zhuang.md) [安装ChromeDriver](/1kai-fa-huan-jing-pei-zhi/12-qing-qiu-ku-de-an-zhuang/123-chromedriverde-an-zhuang.md) ### 3.接口分析 ###  ### 4.页面数据分析 目的爬取商品信息  商品基本信息:商品图片、名称、价格、购买人数、店铺名称、店铺所在地 抓取页面:[https://s.taobao.com/search?q=苹果plus正品](https://s.taobao.com/search?q=苹果plus正品) 分页: ### 5.获取商品列表 抓取地址:[https://s.taobao.com/search?q=苹果plus正品](https://s.taobao.com/search?q=苹果plus正品) ``` def get_products(): """ 提取商品数据 """ print("数据提取中....") html = browser.page_source doc = pq(html) items = doc('#mainsrp-itemlist .items .item').items() for item in items: image = item.find(".pic-link .img").attr('data-src') # if image: # result = re.match('.*?!!(.*?)\..*?', image) # result = re.sub('[a-z\_\-]', '', result) # if result: # id = result.group(1) # print(id) data = { 'title': item.find(".title").text().strip(), 'image':image, 'price':item.find(".price").text().strip(), 'deal':item.find('.deal-cnt').text().strip(), 'shop':item.find('.shop').text().strip(), 'location':item.find(".location").text().strip(), } print(data) save_to_mongo(data) ``` ### 6.存储数据到MongoDB中 ``` MONGO_URL = "localhost" MONGO_DB = "taobao" MONGO_COLLECTION = "products" client = pymongo.MongoClient(MONGO_URL) db = client[MONGO_DB] collection = db[MONGO_COLLECTION] def save_to_mongo(data): """ 保存数据到mongoDB中 :param data: :return: """ try: if collection.insert(data): print("success") except: print("fail") ``` ### 7.遍历每页 ``` def main(): for i in range(1,101): index_page(i) # browser.close() ``` ### 8.Chrome Headless模式 chrome无界面模式参数 ``` chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--headless') browser = webdriver.Chrome(chrome_options=chrome_options) ``` ### 9.对接Firefox ``` browser = webdriver.Firefox() ``` ### 10.对接PhantomJS ``` browser = webdriver.PhantomJS() ``` 设置缓存和禁用图片加载的功能，进一步提高爬取效率 ``` SERVICE_ARGS = ['--load-images=false', '--disk-cache=true'] browser = webdriver.PhantomJS(service_args=SERVICE_ARGS) ``` ### 11.源代码 ``` from selenium import webdriver from selenium.webdriver.support.wait import WebDriverWait from selenium.webdriver.support import expected_conditions from selenium.webdriver.common.by import By from pyquery import PyQuery as pq import re import pymongo browser = webdriver.Chrome() wait = WebDriverWait(browser,10) keyword = "" MONGO_URL = "localhost" MONGO_DB = "taobao" MONGO_COLLECTION = "products" client = pymongo.MongoClient(MONGO_URL) db = client[MONGO_DB] collection = db[MONGO_COLLECTION] def index_page(page): """ 抓取索引页 :param page: 页码 """ print("正在爬取第{}页".format(page)) try: url = "https://s.taobao.com/search?q={}".format(keyword) browser.get(url) if page > 1: input = wait.until(expected_conditions.presence_of_element_located((By.CSS_SELECTOR,"#mainsrp-pager div.form > input"))) submit = wait.until(expected_conditions.element_to_be_clickable((By.CSS_SELECTOR,"#mainsrp-pager div.form > span.btn.J_Submit"))) input.clear() input.send_keys(page) submit.click() wait.until(expected_conditions.text_to_be_present_in_element((By.CSS_SELECTOR,"#mainsrp-pager li.item.active > span"),str(page))) # 商品信息 wait.until(expected_conditions.presence_of_element_located((By.CSS_SELECTOR,".m-itemlist .items .item"))) get_products() except: index_page(page) def get_products(): """ 提取商品数据 """ print("数据提取中....") html = browser.page_source doc = pq(html) items = doc('#mainsrp-itemlist .items .item').items() for item in items: image = item.find(".pic-link .img").attr('data-src') # if image: # result = re.match('.*?!!(.*?)\..*?', image) # result = re.sub('[a-z\_\-]', '', result) # if result: # id = result.group(1) # print(id) data = { 'title': item.find(".title").text().strip(), 'image':image, 'price':item.find(".price").text().strip(), 'deal':item.find('.deal-cnt').text().strip(), 'shop':item.find('.shop').text().strip(), 'location':item.find(".location").text().strip(), } print(data) save_to_mongo(data) def save_to_mongo(data): """ 保存数据到mongoDB中 :param data: :return: """ try: if collection.insert(data): print("success") except: print("fail") def main(): for i in range(1,101): index_page(i) # browser.close() if __name__ == "__main__": keyword = input("请输入关键词:") main() ```