python爬取博客园首页文章

 1 # -*- coding=utf-8 -*-
 2 __author__ = ‘lhyz‘
 3 
 4 import urllib
 5 import re
 6 import socket
 7 import time
 8 import os
 9 
10 #使用当前时间创建文件夹
11 ISOTIMEFORMAT=‘%Y-%m-%d-%X‘
12 times=time.strftime( ISOTIMEFORMAT, time.localtime() )
13 dir=‘./%s‘%times
14 os.mkdir(dir)
15 
16 #设置网络连接超时
17 socket.setdefaulttimeout(10)
18 
19 ‘‘‘
20 获取首页文章链接并下载
21 ‘‘‘
22 
23 #匹配小文章标题
24 pattern=‘<h3>‘
25 
26 #下载进度说明
27 ‘‘‘
28 在print输出的后面加上逗号可以不换行
29 ‘‘‘
30 def cb(a,b,c):
31     per=100.0*a*b/c
32     if per>100:
33         print ‘downloaded page...‘,
34 
35 url=‘http://www.cnblogs.com‘
36 savedFile=‘./index.html‘
37 
38 #网络连接超时的时候下载下一个
39 try:
40     urllib.urlretrieve(url,savedFile,cb)
41     print savedFile
42 except socket.timeout:
43     print ‘timeout.‘
44     urllib.urlretrieve(url,savedFile,cb)
45     print savedFile
46 
47 #读取下载的首页文件
48 f=open(savedFile,‘r‘)
49 lines=f.readlines()
50 f.close()
51 
52 #下载文章链接的函数
53 ‘‘‘
54 用global在函数中引用外部全局变量
55 ‘‘‘
56 i=1
57 def downLink(pageLink):
58     global i
59     artiPage=‘./%s/%s%d.%s‘%(dir,‘page‘,i,‘html‘)
60     try:
61         urllib.urlretrieve(pageLink,artiPage,cb)
62         print artiPage
63     except socket.timeout:
64         print ‘timeout‘
65         urllib.urlretrieve(pageLink,artiPage,cb)
66         print artiPage
67     i=i+1
68 
69 for line in lines:
70     match = re.search(pattern,line)
71     if match:
72         arrstr=match.string.split(‘"‘)  #按照引号分割
73         link=arrstr[3]
74         downLink(link)
75 
76 titlePattern=‘<title>‘
77 pagePattern=‘page‘
78 
79 saveddir=‘./%s/‘%dir
80 filelists=os.listdir(saveddir)
81 
82 for file in filelists:
83     match=re.search(pagePattern,file)
84     if match:
85         filename=‘./%s/%s‘%(dir,match.string)
86         f=open(filename)
87         lines=f.readlines()
88         for line in lines:
89             titleMatch=re.search(titlePattern,line)
90             if titleMatch:
91                 s=titleMatch.string.split(‘-‘)
92                 author = s[1]
93                 title = s[0]
94                 title=title.split(‘>‘)
95                 title=title[1]
96                 newname=‘./%s/%s-%s.html‘%(dir,title,author)
97                 os.rename(filename,newname)
98                 break
99         f.close()

1 import time
2 ISOTIMEFORMAT=‘%Y-%m-%d-%X‘
3 times=time.strftime( ISOTIMEFORMAT, time.localtime() )

 1 import socket
 2 import urllib
 3 def cb(a,b,c):
 4     per=100.0*a*b/c
 5     if per>100:
 6         print ‘downloaded page...‘,
 7 
 8 url=‘http://www.cnblogs.com‘
 9 savedFile=‘./index.html‘
10 
11 #网络连接超时的时候下载下一个
12 try:
13     urllib.urlretrieve(url,savedFile,cb)
14     print savedFile
15 except socket.timeout:
16     print ‘timeout.‘
17     urllib.urlretrieve(url,savedFile,cb)
18     print savedFile

import urllib

def cb(a,b,c):
    per=100.0*a*b/c
    if per>100:
        print ‘downloaded page...‘,

url=‘http://www.cnblogs.com‘
savedFile=‘./index.html‘

#网络连接超时的时候下载下一个
try:
    urllib.urlretrieve(url,savedFile,cb)
    print savedFile
except socket.timeout:
    print ‘timeout.‘
    urllib.urlretrieve(url,savedFile,cb)
    print savedFile

artiPage=‘./%s/%s%d.%s‘%(dir,‘page‘,i,‘html‘)

 1 在函数开头声明全局变量
 2 i=1
 3 def downLink(pageLink):
 4     global i
 5     artiPage=‘./%s/%s%d.%s‘%(dir,‘page‘,i,‘html‘)
 6     try:
 7         urllib.urlretrieve(pageLink,artiPage,cb)
 8         print artiPage
 9     except socket.timeout:
10         print ‘timeout‘
11         urllib.urlretrieve(pageLink,artiPage,cb)
12         print artiPage
13     i=i+1

1 import re
2 pattern=‘<h3>‘
3 for line in lines:
4     match = re.search(pattern,line)
5     if match:
6         arrstr=match.string.split(‘"‘)  #按照引号分割
7         link=arrstr[3]
8         downLink(link)

1 import os
2 saveddir=‘./%s/‘%dir
3 filelists=os.listdir(saveddir)

 1         for line in lines:
 2             titleMatch=re.search(titlePattern,line)
 3             if titleMatch:
 4                 s=titleMatch.string.split(‘-‘)
 5                 author = s[1]
 6                 title = s[0]
 7                 title=title.split(‘>‘)
 8                 title=title[1]
 9                 newname=‘./%s/%s-%s.html‘%(dir,title,author)
10                 os.rename(filename,newname)
11                 break