4) 爬中国大学排名网站内容

# -*- coding: utf-8 -*-
import bs4
import requests
from bs4 import BeautifulSoup
import pandas as pd
import matplotlib.pyplot as plt

def getHTMLText(url):
    try:
        res = requests.get(url,timeout = 30)
        res.raise_for_status()
        res.encoding = res.apparent_encoding
        return res.text
    except:
        return "访问未成功"


def fillUnivList(ulist, html):  # 将一个html页面放入一个列表
    soup = BeautifulSoup(html, "html.parser")
    # 每个<tr>包含一所大学的所有信息
    # 所有<tr>信息包在<tbody>中
    for tr in soup.find(‘tbody‘).children:
        if isinstance(tr, bs4.element.Tag):  # 过滤掉非标签信息，以取出包含在<tr>标签中的bs4类型的Tag标签
            tds = tr(‘td‘)  # 等价于tr.find_all(‘td‘),在tr标签中找td标签内容
            # print(tds)
            ulist.append([tds[0].string, tds[1].string, tds[3].string, tds[2].string])
            # td[0],[1],[3],[2]，分别对应每组td信息中的排名，学校名称，得分，区域。将这些信息从摘取出来
            print(ulist)
    return ulist

def writedata(ulist,file):
    where_list = []
    dict = {}
    df = pd.DataFrame(ulist,columns=[‘排名‘,‘学校名称‘,‘得分‘,‘区域‘])  #list转dataframe
    df.to_csv(file,‘,‘,index=False,encoding="gbk")
    print("写入完成！")
    for i in range(100):
        if df.iloc[i,-1] in where_list:
            dict[df.iloc[i,-1]] += 1
        else:
            where_list.append(df.iloc[i,-1])
            dict[df.iloc[i,-1]] = 1
    print(dict)

    return dict


if __name__ == ‘__main__‘:
    uinfo = []
    url = "http://www.zuihaodaxue.cn/zuihaodaxuepaiming2016.html"
    soup = getHTMLText(url)
    ulist = fillUnivList(uinfo,soup)
    file = "D:\\tt.csv"
    dict = writedata(ulist,file)