## urllib.request
~~~
import urllib.request
url = 'http://www.baidu.com/'
response = urllib.request.urlopen(url).read()
#使用urllib.request.urlopen可以读取网页内容
~~~
~~~
import urllib.request
url = 'https://timgsa.baidu.com/timg?image&quality=80&size=b9999_10000&sec=1543822283&di=b327e6e2dc59105bcb73a174bff94919&imgtype=jpg&er=1&src=http%3A%2F%2Ftupian.qqjay.com%2Fu%2F2017%2F1201%2F2_161641_2.jpg'
res = urllib.request.urlopen(url)
# 图片下载方式一
with open('fengjing.jpg','wb') as f:
f.write(res.read())
# 图片下载方式二
urllib.request.urlretrieve(url,'tupian.jpg')
#使用urllib.request.urlretrieve可以将图片直接下载到指定的路径
~~~
~~~
import urllib.request
url = 'http://www.baidu.com/'
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36',
'Accept-Language':'zh-CN,zh;q=0.9'
}
# 构建请求对象
request = urllib.request.Request(url=url,headers=headers)
#使用urllib.request.Request可以构建带请求头的请求对象
~~~
~~~
import urllib.request
handler = urllib.request.ProxyHandler({'http': '124.243.226.18:8888'})
opener = urllib.request.build_opener(handler)
url = 'http://www.baidu.com/s?wd=IP'
headers = {
"Host": "www.baidu.com",
"Connection": "keep-alive",
"Cache-Control": "max-age=0",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36",
"Accept-Language": "zh-CN,zh;q=0.8",
}
request = urllib.request.Request(url=url,headers=headers)
response = opener.open(request)
print(response.read().decode())
#给程序设置代理
~~~
## urllib.parse
~~~
import urllib.parse
res = urllib.parse.urlparse(url)
print(res)
#输出结果：
ParseResult(scheme='http', netloc='www.baidu.com', path='/s', params='', query='wd=%E7%9F%A5%E4%B9%8E', fragment='')
#使用 urllib.parse.urlparse可以分割url
~~~
~~~
import urllib.parse
word = '知乎'
res = urllib.parse.quote(word)
print(res)
res = urllib.parse.unquote(res)
print(res)
#输出结果：
%E7%9F%A5%E4%B9%8E
知乎
#使用urllib.parse.quote和urllib.parse.unquote可以对字符进行编码
~~~
~~~
import urllib.parse
params = {
'wd':'知乎'
}
word = urllib.parse.urlencode(params)
print(word)
#输出结果：
wd=%E7%9F%A5%E4%B9%8E
#使用urllib.parse.urlencode可以把一个字典编码成以上格式
~~~
## requests
```
import requests
response = requests.get('http://www.baidu.com/')
print(response.url)
print(response.text)
print(response.status_code)
print(response.headers)
print(response.cookies)
print(response.content.decode())
基本方法
```
```
http://www.baidu.com/
<!DOCTYPE html>
<!--STATUS OK--><html> <head><meta http-equiv=content-type content=text/html;charset=utf-8><meta http-equiv=X-UA-Compatible content=IE=Edge><meta content=always name=referrer><link rel=stylesheet type=text/css href=http://s1.bdstatic.com/r/www/cache/bdorz/baidu.min.css><title>ç¾åº¦ä¸ä¸ï¼ä½ å°±ç¥é</title></head> <body link=#0000cc> <div id=wrapper> <div id=head> <div class=head_wrapper> <div class=s_form> <div class=s_form_wrapper> <div id=lg> <img hidefocus=true src=//www.baidu.com/img/bd_logo1.png width=270 height=129> </div> <form id=form name=f action=//www.baidu.com/s class=fm> <input type=hidden name=bdorz_come value=1> <input type=hidden name=ie value=utf-8> <input type=hidden name=f value=8> <input type=hidden name=rsv_bp value=1> <input type=hidden name=rsv_idx value=1> <input type=hidden name=tn value=baidu><span class="bg s_ipt_wr"><input id=kw name=wd class=s_ipt value maxlength=255 autocomplete=off autofocus></span><span class="bg s_btn_wr"><input type=submit id=su value=ç¾åº¦ä¸ä¸ class="bg s_btn"></span> </form> </div> </div> <div id=u1> <a href=http://news.baidu.com name=tj_trnews class=mnav>æ°é»</a> <a href=http://www.hao123.com name=tj_trhao123 class=mnav>hao123</a> <a href=http://map.baidu.com name=tj_trmap class=mnav>å°å¾</a> <a href=http://v.baidu.com name=tj_trvideo class=mnav>è§é¢</a> <a href=http://tieba.baidu.com name=tj_trtieba class=mnav>è´´å§</a> <noscript> <a href=http://www.baidu.com/bdorz/login.gif?login&tpl=mn&u=http%3A%2F%2Fwww.baidu.com%2f%3fbdorz_come%3d1 name=tj_login class=lb>ç»å½</a> </noscript> <script>document.write('<a href="http://www.baidu.com/bdorz/login.gif?login&tpl=mn&u='+ encodeURIComponent(window.location.href+ (window.location.search === "" ? "?" : "&")+ "bdorz_come=1")+ '" name="tj_login" class="lb">ç»å½</a>');</script> <a href=//www.baidu.com/more/ name=tj_briicon class=bri style="display: block;">æ´å¤äº§å</a> </div> </div> </div> <div id=ftCon> <div id=ftConw> <p id=lh> <a href=http://home.baidu.com>å
³äºç¾åº¦</a> <a href=http://ir.baidu.com>About Baidu</a> </p> <p id=cp>©2017 Baidu <a href=http://www.baidu.com/duty/>使ç¨ç¾åº¦åå¿
读</a> <a href=http://jianyi.baidu.com/ class=cp-feedback>æè§åé¦</a> 京ICPè¯030173å· <img src=//www.baidu.com/img/gs.gif> </p> </div> </div> </div> </body> </html>
200
{'Cache-Control': 'private, no-cache, no-store, proxy-revalidate, no-transform', 'Connection': 'Keep-Alive', 'Content-Encoding': 'gzip', 'Content-Type': 'text/html', 'Date': 'Mon, 03 Dec 2018 13:39:39 GMT', 'Last-Modified': 'Mon, 23 Jan 2017 13:27:32 GMT', 'Pragma': 'no-cache', 'Server': 'bfe/1.0.8.18', 'Set-Cookie': 'BDORZ=27315; max-age=86400; domain=.baidu.com; path=/', 'Transfer-Encoding': 'chunked'}
<RequestsCookieJar[<Cookie BDORZ=27315 for .baidu.com/>]>
<!DOCTYPE html>
<!--STATUS OK--><html> <head><meta http-equiv=content-type content=text/html;charset=utf-8><meta http-equiv=X-UA-Compatible content=IE=Edge><meta content=always name=referrer><link rel=stylesheet type=text/css href=http://s1.bdstatic.com/r/www/cache/bdorz/baidu.min.css><title>百度一下，你就知道</title></head> <body link=#0000cc> <div id=wrapper> <div id=head> <div class=head_wrapper> <div class=s_form> <div class=s_form_wrapper> <div id=lg> <img hidefocus=true src=//www.baidu.com/img/bd_logo1.png width=270 height=129> </div> <form id=form name=f action=//www.baidu.com/s class=fm> <input type=hidden name=bdorz_come value=1> <input type=hidden name=ie value=utf-8> <input type=hidden name=f value=8> <input type=hidden name=rsv_bp value=1> <input type=hidden name=rsv_idx value=1> <input type=hidden name=tn value=baidu><span class="bg s_ipt_wr"><input id=kw name=wd class=s_ipt value maxlength=255 autocomplete=off autofocus></span><span class="bg s_btn_wr"><input type=submit id=su value=百度一下 class="bg s_btn"></span> </form> </div> </div> <div id=u1> <a href=http://news.baidu.com name=tj_trnews class=mnav>新闻</a> <a href=http://www.hao123.com name=tj_trhao123 class=mnav>hao123</a> <a href=http://map.baidu.com name=tj_trmap class=mnav>地图</a> <a href=http://v.baidu.com name=tj_trvideo class=mnav>视频</a> <a href=http://tieba.baidu.com name=tj_trtieba class=mnav>贴吧</a> <noscript> <a href=http://www.baidu.com/bdorz/login.gif?login&tpl=mn&u=http%3A%2F%2Fwww.baidu.com%2f%3fbdorz_come%3d1 name=tj_login class=lb>登录</a> </noscript> <script>document.write('<a href="http://www.baidu.com/bdorz/login.gif?login&tpl=mn&u='+ encodeURIComponent(window.location.href+ (window.location.search === "" ? "?" : "&")+ "bdorz_come=1")+ '" name="tj_login" class="lb">登录</a>');</script> <a href=//www.baidu.com/more/ name=tj_briicon class=bri style="display: block;">更多产品</a> </div> </div> </div> <div id=ftCon> <div id=ftConw> <p id=lh> <a href=http://home.baidu.com>关于百度</a> <a href=http://ir.baidu.com>About Baidu</a> </p> <p id=cp>©2017 Baidu <a href=http://www.baidu.com/duty/>使用百度前必读</a> <a href=http://jianyi.baidu.com/ class=cp-feedback>意见反馈</a> 京ICP证030173号 <img src=//www.baidu.com/img/gs.gif> </p> </div> </div> </div> </body> </html>
运行结果
```
```
'''
目标网站：
http://www.shicimingju.com/book/sanguoyanyi.html
代码思路：
1、确定url
2、伪装浏览器信息
3、获取request
4、获取url指向页面的内容
5、解析获取到的网页，获取标题标题和文章链接
6、下载文章内容到本地
'''
import urllib.request
import urllib.parse
from bs4 import BeautifulSoup
import requests
import lxml
def handle_requests(url):
headers = {
'User-Agent': ' Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36',
}
response = requests.get(url,headers)
return response
def download_text(title, href):
response = handle_requests(href)
content = response.text
soup = BeautifulSoup(content,'lxml')
res = soup.find('div',class_ = 'chapter_content').find_all('p')
f = open('hongloumeng.txt', 'a',encoding='utf-8')
f.write(title)
print(title)
for p in res:
f.write(p.text)
f.close()
def parse_content(content):
soup = BeautifulSoup(content,'lxml')
res = soup.select('.book-mulu > ul > li > a')
for i in res:
title = i.text
href = 'http://www.shicimingju.com' + i['href']
download_text(title,href)
print('正在下载。。。')
def main():
url = 'http://www.shicimingju.com/book/hongloumeng.html'
# 获取包装头信息，获取request对象
response = handle_requests(url)
# 根据得到的request发送请求，获取页面所有内容
content = response.text
# 解析页面内容，获取文章标题和内容链接
parse_content(content)
if __name__ == '__main__':
main()
requests爬取小说
```
```
import requests
url = 'http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=keyword'
data = {
'cname':'',
'pid':'',
'keyword':'杭州',
'pageIndex':'1',
'pageSize':'10',
}
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36',
}
response = requests.post(url,data,headers=headers)
print(response.text)
requests的post请求
```
## selenium
selenium最初是一个测试工具，而爬虫中使用它主要是为了解决requests无法直接执行JavaScript代码的问题
selenium本质是通过驱动浏览器，完全模拟浏览器的操作，比如跳转、输入、点击、下拉等，来拿到网页渲染之后的结果，可支持多种浏览器
```
import requests
from selenium import webdriver
import time
from bs4 import BeautifulSoup
path = r'D:\Python1806\spider\day5\chromedriver.exe'
url = 'https://so.gushiwen.org/user/login.aspx'
browser = webdriver.Chrome(executable_path=path)
# 指定url
browser.get(url)
time.sleep(2)
# 找到用户名和密码等输入项
username = browser.find_element_by_xpath('//input[@id="email"]')
username.send_keys('wusir666666@163.com')
time.sleep(1)
pwd = browser.find_element_by_xpath("//input[@id='pwd']")
pwd.send_keys('ymmnxhwm13579')
time.sleep(1)
soup = BeautifulSoup(browser.page_source,'lxml')
browser.save_screenshot('login.png')
code = input("请输入验证码：")
checkcode = browser.find_element_by_xpath("//input[@id='code']")
checkcode.send_keys(code)
login = browser.find_element_by_xpath("//input[@id='denglu']")
login.click()
selenium基础使用
```
- Python学习
- Python基础
- Python初识
- 列表生成式，生成器，可迭代对象，迭代器详解
- Python面向对象
- Python中的单例模式
- Python变量作用域、LEGB、闭包
- Python异常处理
- Python操作正则
- Python中的赋值与深浅拷贝
- Python自定义CLI三方库
- Python并发编程
- Python之进程
- Python之线程
- Python之协程
- Python并发编程与IO模型
- Python网络编程
- Python之socket网络编程
- Django学习
- 反向解析
- Cookie和Session操作
- 文件上传
- 缓存的配置和使用
- 信号
- FBV&&CBV&&中间件
- Django补充
- 用户认证
- 分页
- 自定义搜索组件
- Celery
- 搭建sentry平台监控
- DRF学习
- drf概述
- Flask学习
- 项目拆分
- 三方模块使用
- 爬虫学习
- Http和Https区别
- 请求相关库
- 解析相关库
- 常见面试题
- 面试题
- 面试题解析
- 网络原理
- 计算机网络知识简单介绍
- 详解TCP三次握手、四次挥手及11种状态
- 消息队列和数据库
- 消息队列之RabbitMQ
- 数据库之Redis
- 数据库之初识MySQL
- 数据库之MySQL进阶
- 数据库之MySQL补充
- 数据库之Python操作MySQL
- Kafka常用命令
- Linux学习
- Linux基础命令
- Git
- Git介绍
- Git基本配置及理论
- Git常用命令
- Docker
- Docker基本使用
- Docker常用命令
- Docker容器数据卷
- Dockerfile
- Docker网络原理
- docker-compose
- Docker Swarm
- HTML
- CSS
- JS
- VUE