爬虫第一篇基本库的使用——urllib

1 request：最基本的请求模块，可以用来实现请求的发送
2 
3 error：异常处理模块，用于处理异常，使我们的程序不会意外终止
4 
5 parse：工具模块，提供了URL多种处理方法，拆分，解析与合并
6 
7 robotparser：用来识别网站的robots.txt文件，判断我们可以爬取哪些网站，哪些不可以

1 #模拟浏览器发送请求访问Python官网
2 import urllib
3 response = urllib.request.urlopen("http://www.python.org")
4 print(response.read().decode("utf-8"))

from urllib import parse,request
try:
    data = bytes(parse.urlencode({‘hello‘: ‘word‘}), encoding="utf-8")
    response = request.urlopen("http://httpbin.org/post",data=data,timeout=1)
except error.URLError as e:
    if isinstance(e.reason,socket.timeout):
        print("TIME OUT")

{"args":{},"data":"","files":{},"form":{"hello":"word"},"headers":{"Accept-Encoding":"identity","Connection":"close","Content-Length":"10","Content-Type":"application/x-www-form-urlencoded","Host":"httpbin.org","User-Agent":"Python-urllib/3.6"},"json":null,"origin":"183.203.223.38","url":"http://httpbin.org/post"}

from urllib import request,parse

data = bytes(parse.urlencode({"name":"sunqi"}),encoding=‘utf-8‘)
head = {"User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
　　　　　"Host":"httpbin.org"} 
req = request.Request(‘http://httpbin.org/post‘,data=data,headers=head,origin_req_host="",method="POST") 
res = request.urlopen(req) 
print(res.read().decode(‘utf-8‘))

{"args":{},"data":"","files":{},"form":{"name":"sunqi"},"headers":{"Accept-Encoding":"identity","Connection":"close","Content-Length":"10","Content-Type":"application/x-www-form-urlencoded","Host":"httpbin.org","User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"},"json":null,"origin":"183.203.223.38","url":"http://httpbin.org/post"}

from urllib.request import HTTPBasicAuthHandler,HTTPPasswordMgrWithDefaultRealm,build_opener
from urllib.error import URLError

username = "1904700999"
password = "1904700999"
url = "http://localhost:5000/"
p = HTTPPasswordMgrWithDefaultRealm()#密码管理类的对象
p.add_password(None,url,username,password)#对象添加密码
auth_handler = HTTPBasicAuthHandler(p)#密码认证实例化认证handler
opener = build_opener(auth_handler)

try:
    result = opener.open(url)
    print(result.status())
    html = result.read().decode(‘utf-8‘)
    print(html)

except URLError as e:
    print(e.reason)

import http.cookiejar,urllib.request

# 打印输出cookie
cookie = http.cookiejar.CookieJar()
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
response = opener.open("http://www.baidu.com")
for item in cookie:
        print(item.name+"->"+item.value)

filename = "cookies_LWP.txt"

cookie = http.cookiejar.LWPCookieJar(filename)
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
response = opener.open("http://www.baidu.com")
cookie.save(ignore_discard=True,ignore_expires=True)

filename = "cookies_Mozilla.txt"

cookie = http.cookiejar.MozillaCookieJar(filename)#Mozilla格式的文件
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
response = opener.open("http://www.baidu.com")
cookie.save(ignore_discard=True,ignore_expires=True)

cookie = http.cookiejar.LWPCookieJar()
cookie.load("cookies_LWP.txt",ignore_expires=True,ignore_discard=True)
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
response = opener.open("http://www.baidu.com")
print(response.read().decode(‘utf-8‘))

#URLError
try :
    request.urlopen("http://cuiqingcai.com/index.htm")
except error.URLError as e:
    print(e.reason)#Not Found

#HTTPerror URLError的子类
try:
    request.urlopen("http://cuiqingcai.com/index.htm")
except error.HTTPError as e:
    print(e.code)#状态码
    print(e.reason)#错误原因
    print(e.headers)#请求头

#返回的不是字符串的reason
try:
    response = request.urlopen("http://www.baidu.com",timeout=0.01)
except error.HTTPError as e:
    print(e.reason,e.headers,e.code)
except error.URLError as e:
    print(e.reason)
    print(type(e.reason))#e.reason返回的不一定是字符串
    if isinstance(e.reason,socket.timeout):
        print("TIME OUT")

from urllib.parse import urlparse



# urlprase  解析链接
result = urlparse("http://www.baidu.com/index.html;user?id=5#comment",allow_fragments=True)
print(type(result))
print(result)
‘‘‘
<class ‘urllib.parse.ParseResult‘>
ParseResult(scheme=‘http‘, netloc=‘www.baidu.com‘, path=‘/index.html‘, params=‘user‘, query=‘id=5‘, fragment=‘comment‘)
协议，域名，路径，参数，查询条件，锚点
标准连接格式
schema://netloc/path;params?query#fragment
urlparse(url="",scheme="",allow_fragments=True or False)
设置协议时，如果url没有协议开头，则只用默认协议，如果有协议开头就使用url自身协议
未设置设置allow_fragment选项时锚点与query一组
没有query时，锚点被解析到path中
‘‘‘

from urllib.parse import urlunparse
# urlunprase  构造链接
data = ("http","www.baidu.com","index.html","user","","")
data = {"schema":‘http‘,"netloc":‘www.baidu.com‘,"path":‘index.html‘,"params":‘‘,"query":‘‘,"fragment":‘‘}
url = urlunparse(data)
print(url)
‘‘‘
构造链接,数据结构长度必须为6,列表和元组都可以
‘‘‘


#urlsplit 链接分割
from urllib.parse import urlsplit
result = urlsplit("http://www.baidu.com/index.html;user?id=5#comment")
print(result)#不单独区分param参数，与path合并

#urlunsplit 构造链接
from urllib.parse import urlunsplit
data = ("http","www.baidu.com","index.html","user","")#要求长度必须为5
url = urlunsplit(data)
print(url)


from urllib.parse import urljoin


#urljoin 两个链接组合成一个新链接，前者为base_url,后者为给定的url
#base_url 提供协议，域名与路径，当要构造的url没有时会自动补充，否则不补充
print(urljoin("http://www,baidu.com","eg.html"))
#无协议，无域名自动填充 http://www,baidu.com/eg.html
print(urljoin("http://www.baidu.com","//www.sun.com/eg.html"))
#无协议自动填充协议  http://www.sun.com/eg.html
print(urljoin("http://www.baidu.com",‘http://www.sun.com/index.html‘))
#都有不填充
print(urljoin("http://www.baidu.com/index.html","http:"))
#http://www.baidu.com/index.html
print(urljoin("http://www.baidu.com/index.html?wd=123","http://www.sun.com/index.html"))
#http://www.sun.com/index.html
print(urljoin("http://www.baidu.com/index.html?wd=123#comment","http://www.sun.com/index.php?wd=456#com"))
#http://www.sun.com/index.php?wd=456#com  base_url自带的params，query，fragment不起作用

#urlencode 构造get请求
from urllib.parse import urlencode
params = {"name":"sunqi","password":"123456"}
url = "http://www.baidu.com?"+urlencode(params)
print(url)
#http://www.baidu.com?name=sunqi&password=123456 将字典转化为get提交的参数

from urllib.parse import parse_qs,parse_qsl
#反序列化参数为字典和元组
result = urlparse("http://www.baidu.com?name=sunqi&password=123456")
print(result[4])
dic = parse_qs(result[4])#转换为字典{‘name‘: [‘sunqi‘], ‘password‘: [‘123456‘]}
tu = parse_qsl(result[4])#转换为元组[(‘name‘, ‘sunqi‘), (‘password‘, ‘123456‘)]
print(dic)
print(tu)


from urllib.parse import quote,unquote
#将字符串转换为URL编码与解码
keyword = "壁纸"
url = "http://www.baidu.com/s?wd="+quote(keyword)
print(url)#http://www.baidu.com/s?wd=%E5%A3%81%E7%BA%B8
print(unquote(url))#http://www.baidu.com/s?wd=壁纸

from urllib.robotparser import RobotFileParser
from urllib.request import urlopen

robot = RobotFileParser()
result = urlopen(‘http://www.zhihu.com/robots.txt‘).read().decode(‘utf-8‘)
print(result)

robot.parse(result)

print(robot.can_fetch(‘*‘,‘http://www.zhihu.com/question/29173647/answer/437189494‘))#True
print(robot.can_fetch(‘*‘,‘https://www.zhihu.com/signin?next=%2Fexplore‘))#True
print(robot.can_fetch(‘*‘,‘http://www.zhihu.com/inbox/7013224000‘))#True
response = urlopen(‘https://www.zhihu.com/inbox/7013224000‘).read().decode("utf-8")
filename = "zhuhi_inbox.html"
with open(filename,"w") as f:
    f.write(response)