爬虫系列之豆瓣图书排行

时间：2018-06-15 21:48:54 阅读：210 评论：0 收藏：0 [点我收藏+]

豆瓣上有图书的排行榜，所以这次写了一个豆瓣的爬虫。

首先是分析排行榜的url 技术分享图片

根据这个可以很容易的知道不同图书的排行榜就是在网站后面加上/tag/【类别】，所以我们首先要获得图书的类别信息。

这里可以将读书首页的热门标签给爬下来。

技术分享图片

爬取标签内容并不难，代码如下：

 1 def getLabel(url):    #获得热门标签
 2     html = getHTMLText(url)
 3     soup = BeautifulSoup(html, ‘html.parser‘)
 4     a = soup.find_all(‘a‘)
 5     label_list = []
 6     for i in a:
 7         try:
 8             href = i.attrs[‘href‘]
 9             match = re.search(r‘/tag/.*‘, href)
10             if match and match[0][5]!=‘?‘:
11                 label_list.append(match[0])
12         except:
13             continue
14     return label_list

接下来是进入排行榜页面进行信息爬取，

技术分享图片

代码如下：

 1 def getBookInfo():
 2     label_list = getLabel(‘https://book.douban.com/‘)
 3     label = get_label(label_list)
 4     name = []
 5     author = []
 6     price = []
 7     score = []
 8     number = []
 9     for page in label_list[int(label)-1:int(label)]:
10         for i in range(2):
11             html = getHTMLText(‘https://book.douban.com‘ + page + ‘?start=‘ + str(i*20) + ‘&type=T‘)
12             soup = BeautifulSoup(html, ‘html.parser‘)
13             book_list = soup.find_all(‘div‘, attrs={‘class‘:‘info‘})  #找到书籍的信息列表
14             for book in book_list:
15                 a = book.find_all(‘a‘,attrs={‘title‘:re.compile(‘.*‘)})[0]   #找到包含书籍名的a标签
16                 name.append(a.get(‘title‘))   #获得标题属性
17                 
18                 pub = book.find_all(‘div‘, attrs={‘class‘:‘pub‘})[0]
19                 pub = pub.string.strip().replace(‘\n‘,‘‘)
20                 author.append(re.findall(r‘(.*?)/‘, pub)[0].strip())
21                 split_list = pub.split()   #空格分割
22                 for j in split_list:
23                     match = re.search(r‘\d.*\..*‘, j)   #获得价格信息
24                     if match:
25                         price.append(match[0])
26                 
27                 span = book.find_all(‘span‘, attrs={‘class‘:‘pl‘})[0]  #获得评价人数所在标签
28                 span = span.string.strip().replace(‘\n‘,‘‘)
29                 number.append(re.findall(r‘\d+‘, span)[0])   #获得人数
30 
31                 span = book.find_all(‘span‘, attrs={‘class‘:‘rating_nums‘})[0]
32                 score.append(span.string)
33 
34     tplt = "{:3}\t{:15}\t{:15}\t{:10}\t{:4}\t{:7}"  #规定输出格式
35     print(tplt.format("序号", "书籍", "作者", "价格", "评分", "评价人数"))
36     l = len(name)
37     for count in range(l):
38         print(tplt.format(count+1, name[count],author[count],price[count],score[count],number[count]))

最终的总代码为：

 1 import requests
 2 import re
 3 from bs4 import BeautifulSoup
 4 
 5 
 6 def getHTMLText(url):
 7     try:
 8         r = requests.get(url, timeout=30)
 9         r.raise_for_status()
10         r.encoding = r.apparent_encoding
11         return r.text
12     except:
13         return ""
14 
15 
16 def getLabel(url):    #获得热门标签
17     html = getHTMLText(url)
18     soup = BeautifulSoup(html, ‘html.parser‘)
19     a = soup.find_all(‘a‘)
20     label_list = []
21     for i in a:
22         try:
23             href = i.attrs[‘href‘]
24             match = re.search(r‘/tag/.*‘, href)
25             if match and match[0][5]!=‘?‘:
26                 label_list.append(match[0])
27         except:
28             continue
29     return label_list
30 
31 
32 def get_label(label_list):
33     count = 1
34     for i in label_list:
35         print(str(count) + ‘: ‘ + label_list[count-1][5:] + ‘\t‘, end=‘‘)
36         count = count + 1
37     choose = input(‘\n\n请输入你想查询的图书类别：‘)
38     while int(choose)<=0 or int(choose)>=count:
39         choose = input(‘\n请输入正确的类别编号:‘)
40     return int(choose)
41 
42 def getBookInfo():
43     label_list = getLabel(‘https://book.douban.com/‘)
44     label = get_label(label_list)
45     name = []
46     author = []
47     price = []
48     score = []
49     number = []
50     for page in label_list[int(label)-1:int(label)]:
51         for i in range(2):
52             html = getHTMLText(‘https://book.douban.com‘ + page + ‘?start=‘ + str(i*20) + ‘&type=T‘)
53             soup = BeautifulSoup(html, ‘html.parser‘)
54             book_list = soup.find_all(‘div‘, attrs={‘class‘:‘info‘})  #找到书籍的信息列表
55             for book in book_list:
56                 a = book.find_all(‘a‘,attrs={‘title‘:re.compile(‘.*‘)})[0]   #找到包含书籍名的a标签
57                 name.append(a.get(‘title‘))   #获得标题属性
58                 
59                 pub = book.find_all(‘div‘, attrs={‘class‘:‘pub‘})[0]
60                 pub = pub.string.strip().replace(‘\n‘,‘‘)
61                 author.append(re.findall(r‘(.*?)/‘, pub)[0].strip())
62                 split_list = pub.split()   #空格分割
63                 for j in split_list:
64                     match = re.search(r‘\d.*\..*‘, j)   #获得价格信息
65                     if match:
66                         price.append(match[0])
67                 
68                 span = book.find_all(‘span‘, attrs={‘class‘:‘pl‘})[0]  #获得评价人数所在标签
69                 span = span.string.strip().replace(‘\n‘,‘‘)
70                 number.append(re.findall(r‘\d+‘, span)[0])   #获得人数
71 
72                 span = book.find_all(‘span‘, attrs={‘class‘:‘rating_nums‘})[0]
73                 score.append(span.string)
74 
75     tplt = "{:3}\t{:15}\t{:15}\t{:10}\t{:4}\t{:7}"  #规定输出格式
76     print(tplt.format("序号", "书籍", "作者", "价格", "评分", "评价人数"))
77     l = len(name)
78     for count in range(l):
79         print(tplt.format(count+1, name[count],author[count],price[count],score[count],number[count]))
80 
81     
82     
83 if __name__ ==‘__main__‘:
84     print(‘豆瓣图书综合排序查询\n‘)
85     getBookInfo()
86