Python网络爬虫

import urllib.request
import requests
from bs4 import BeautifulSoup
import time
import random


# 随机头
USER_AGENTS = [
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER",
]
headers = {
	‘User-Agent‘:random.choice(USER_AGENTS),
	‘Connection‘:‘keep-alive‘,
    ‘Accept-Language‘:‘zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2‘
    }
# 创建book.csv
f = open("book.csv", "a")
f.write("书籍" + "," + "价格" + "," + "作者" + "," + "出版社"  + ‘\n‘)
f = f.close()
def spider(keyword,page):
    for i in range(0,page):
        count = 0
        page = 1
        # 获取界面
        url = ‘http://search.dangdang.com/?key=‘+str(keyword)+‘&page_index=‘+str(page)
        res = requests.get(url, headers=headers)
        res.encoding = ‘gbk‘
        soup = BeautifulSoup(res.text,‘html.parser‘)
        # print(soup)

        div = soup.find("div", attrs={"class": "con shoplist"})
        ul = div.find("ul", attrs={"class": "bigimg"})
        lis = ul.find_all("li")
        print(‘正在爬取第{}页‘.format(page))
        for li in lis:
            count += 1
            # 分析每个<li>中书籍信息：名称title、作者author、价格price、出版社publisher、简介detail、图像image
            p = li.find("p", attrs={"class": "name"})
            title = p.find("a").text
            price = li.find("span", attrs={"class": "search_now_price"}).text
            author = li.find("p", attrs={"class": "search_book_author"}).find("span").text
            pub = li.find("p", attrs={"class": "search_book_author"}).find_all("span")
            #publishers = pub[2].find("a").text
            publisher=pub[2].text
            detail = li.find("p", attrs={"class": "detail"}).text

            # 将数据保存至book.csv文件
            with open("book.csv","a",encoding=‘utf-8‘) as f2:
                f2.writelines(title + "," + price + "," + author + "," + publisher  + ‘\n‘)

            print(‘\n书籍：‘,title,‘\n价格：‘,price,‘\n作者：‘,author,‘\n出版社：‘,publisher,‘\n简介：‘,detail)
        page += 1

if __name__ == ‘__main__‘:
    keyword = input(‘关键字：‘)
    page = int(input(‘爬取页数：‘))
    #keyword = ‘python‘
    #page = 1
    spider(keyword,page)

运行后的结果：

import pandas as pd
import numpy as mp
import sklearn
import seaborn as sns
import matplotlib.pyplot as plt

df = pd.read_excel(‘book.xlsx‘,sheet_name=‘Sheet1‘)
df

对数据进行清理：

# 清洗重复书名
book = df.drop_duplicates(‘title‘)
book

然后发现清理后发现价格栏有Nan值，再进行一次对Nan进行清理

# 对Nan进行清理
book = book.dropna(axis=0)
book

 保存清理后的数据然后打开新的数据：

book.to_csv(‘books.csv‘,encoding=‘gb18030‘) #保存数据清洗后的文件
book = pd.read_csv(‘books.csv‘,encoding=‘gb18030‘)
book

对爬取到的各出版设占比做饼状图图

plt.rcParams[‘font.sans-serif‘] = [‘SimHei‘]#解决乱码问题
df_score = book[‘publisher‘].value_counts() #统计评分情况
plt.title("各出版设占比饼状图图") #设置饼图标题
plt.pie(df_score.values,labels = df_score.index,autopct=‘%1.1f%%‘) #绘图
plt.show()

 各出版社对书本定价的分布所做的散点图：

sns.regplot(x = ‘publisher‘,y = ‘price‘,data=book)

 价格直方图

sns.distplot(book[‘price‘])

五.总结

通过以上可视化分析看一大致了解了这本书在不同出版社里，定价的不同。通过饼图可以看出人民文学出版社，商务印书馆，译林出版社，知识出版社等在当当网上占比较多，可以考虑购买这几个出版社所出版的图书。通过散点图和统计图看出这本书的价格在20元左右。这些数据大致可以帮用户分析得到一些数据，可以提供一些数据供用户参考，大致上给到了用户一点建议，与预期的目标还是差了些。

通过这次的课程设计实验，我学习到了要想爬取网站上的信息，必须有一个好的工具。通过本学期的Python课程的学习，使我又进一步了解了Python，Python是一种计算机程序设计语言（解释型语言），具有代码少、简单、运行速度慢的特点。本次课程设计中我也发现了自己的在对Python的运用的很多不足，在课程设计的过程中，我遇到了很多问题，比如很多英文看不懂，我的英文水平还是太差了，今后还要努力学习英语，运行代码是也是经常犯一些很基础的错误，这让我意识到我的基础还不够好，遇到问题首先在网上获取信息，在通过上网查找信息学习过程学习到了将爬取的信息存储到本地，在爬取网站时遇到了被网站的反爬虫程序拦截，通过学习学会了模拟浏览器爬取信息 。在今后，我会在空余的时间里安排时间学习Python。