《Python网络数据采集》笔记之BeautifulSoup

from  urllib.request import urlopen
html = urlopen("http://pythonscraping.com/pages/page1.html")
print(html.read())

from urllib.error import HTTPError
from urllib.request import urlopen
from  bs4 import BeautifulSoup


def getTitle(url):
    try:
        html = urlopen(url)
    except HTTPError as e:
        return None
    try:
        bsobj = BeautifulSoup(html.read())
        title = bsobj.body.h1
    except AttributeError as e:
        return None
    return title
title = getTitle("http://pythonscraping.com/pages/page1.html")
if title == None:
    print("title was not found")
else:
    print(title)

from urllib.request import urlopen
from bs4 import BeautifulSoup

html = urlopen("http://pythonscraping.com/pages/warandpeace.html")

bsobj = BeautifulSoup(html)

# 通过bsobj对象，用fillAll函数抽取class属性为red的span便签
contentList = bsobj.findAll("span",{"class":"red"})

for content in contentList:
    print(content.get_text())
    print(‘\n‘)

from urllib.request import urlopen
from bs4 import BeautifulSoup

html = urlopen("http://pythonscraping.com/pages/page3.html")
bsobj = BeautifulSoup(html)


#找出子标签
for child in bsobj.find("table",{"id":"giftList"}).children:
    print(child)

#找出兄弟标签
for silbling in bsobj.find("table",{"id":"giftList"}).tr.next_siblings:
    print(silbling)

for h2title in bsobj.findAll("h2"):
     print(h2title.get_text())

print(bsobj.find("img",{"src":"../img/gifts/img1.jpg"}).parent.previous_sibling.get_text())

from urllib.request import urlopen
from bs4 import BeautifulSoup
import  re

html = urlopen("http://pythonscraping.com/pages/page3.html")
bsobj = BeautifulSoup(html)
#返回字典对象images
images = bsobj.findAll("img",{"src":re.compile("\.\.\/img\/gifts/img.*\.jpg")})
for image in images:  
    print(image["src"])