通过python自动获取小说并下载

 1 import urllib.request
 2 import os
 3 
 4 headers = {
 5     "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 "
 6                   "Safari/535.1",
 7 }
 8 
 9 
10 # 主程序，获取每个章节的名称及链接
11 def main():
12     print(‘开始下载‘)
13     url_list = []
14     url = ‘http://www.eywedu.com/honglou/01/index.htm‘
15     res = urllib.request.Request(url, data=None, headers=headers)
16     html = urllib.request.urlopen(res).read().decode(‘gb18030‘)
17     lists = html.split(‘<A HREF="‘)
18     for i in lists:
19         if ‘第‘ in i:
20             s = i.split(r‘</A><BR>‘)  # 以/a br作字符分割
21             s1 = ‘ ‘.join(s[0].split())  # 取消多余空格
22             url_list.append(s1)
23     handle_url(url_list)
24 
25 
26 # 处理获取的链接，得到每章名称和准确链接
27 def handle_url(urls):
28     if not os.path.exists(‘红楼梦‘):
29         os.makedirs(‘红楼梦‘)
30     net = ‘http://www.eywedu.com/honglou/01/‘
31     page = 1
32     for i in urls:
33         cut = i.split(‘" >‘)
34         get_content(net+cut[0], cut[1])
35         print(‘第‘, page, ‘章已下完‘)
36         page = page+1
37 
38 
39 # 根据每章链接获得内容
40 def get_content(url, name):
41     paragraphs = []
42     res = urllib.request.Request(url, data=None, headers=headers)
43     html = urllib.request.urlopen(res).read().decode(‘gb18030‘)
44     lists = html.split(‘<BR>‘)
45     t = 0
46     while t < len(lists):
47         if t == 0:
48             p = lists[t].split(r‘2014newad.js"></script></DIV>‘)[1]
49             p = p.replace(‘\r\n‘, ‘‘)
50             paragraphs.append(p)
51         elif t == len(lists) - 1:
52             p = lists[t].split(r‘<!--/HTMLBUILERPART0-->‘)[0]
53             p = p.replace(‘\r\n‘, ‘‘)
54             paragraphs.append(p)
55         else:
56             p = lists[t].replace(‘\r\n‘, ‘‘)
57             paragraphs.append(p)
58         t = t + 1
59     save_txt(paragraphs, name)
60 
61 
62 # 将得到的信息保存
63 def save_txt(content, name):
64     f = open(‘红楼梦\\‘+name+‘.txt‘, "a", encoding=‘utf-8‘)
65     for i in content:
66         f.write(i)
67     f.close()
68 
69 
70 if __name__ == ‘__main__‘:
71     main()