### 1.说明 爬取今日头条街拍美图，并保存到MongoDB中 ### 2.准备 [安装requests库](/1kai-fa-huan-jing-pei-zhi/12-qing-qiu-ku-de-an-zhuang/121-requestsde-an-zhuang.md) ### 3.抓取分析 链接:[https://www.toutiao.com/search/?keyword=街拍](https://www.toutiao.com/search/?keyword=街拍) 打开开发者工具 > Network面板>选中XHR，筛选ajax请求 分析[https://www.toutiao.com/search\_content/?offset=0&format=json&keyword=街拍&autoload=true&count=20&cur\_tab=1&from=search\_tab](https://www.toutiao.com/search_content/?offset=0&format=json&keyword=街拍&autoload=true&count=20&cur_tab=1&from=search_tab)链接 可以看到有几个参数，往下不停刷新，可以得到几个重要参数的含义 offset:偏移量，每次刷新后，从第几条开始显示的数据 keyword:搜索关键字 count:显示的数据条数 ### 4. 实战演练 {#3-实战演练} ``` import requests,os from urllib.parse import urlencode from multiprocessing import Pool from hashlib import md5 baseurl = "https://www.toutiao.com/search_content/?" headers = { 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36', 'x-requested-with': 'XMLHttpRequest', } GROUP_START = 1 GROUP_END = 20 def get_page(offset=0): params = { 'offset': offset, 'format': 'json', 'keyword': '街拍', 'autoload': 'true', 'count': '20', 'cur_tab': '1', 'from': 'search_tab', } url = baseurl + urlencode(params) try: response = requests.get(url,headers=headers) if response.status_code == 200: return response.json() except requests.ConnectionError as e: print(e.args) def get_images(response): if response: items = response.get("data") if items: for item in items: title = item.get("title") images = item.get("image_list") if title and images: for image in images: yield { 'image': image.get("url"), 'title': title, } def save_images(item): title = item.get("title") if not os.path.exists(title): os.mkdir(title) try: image = item.get("image") response = requests.get("http:"+image) if response.status_code == 200: # 读取二进制流数据 content = response.content # 利用md5函数判断重复 filepath = "{0}/{1}.{2}".format(title,md5(content).hexdigest(),'jpg') if not os.path.exists(filepath): with open(filepath,'wb' ) as f: f.write(response.content) else: print("Already Download {}".format(filepath)) except requests.ConnectionError as e: print("Failed to Save Image") def main(offset=0): response = get_page(offset) for item in get_images(response): print(item) save_images(item) if __name__ == "__main__": # 开启进程池 pool = Pool() groups = [x*20 for x in range(GROUP_START,GROUP_END+1)] print(groups) pool.map(main,groups) pool.close() pool.join() ```