爬虫1 实例

"""实例： 下载‘尚学堂’贴吧前几页内容"""

from urllib.request import Request, urlopen
from urllib.parse import urlencode
from fake_useragent import UserAgent



# 获取贴吧网页信息
def get_html(url):
    headers = {
        "User-Agent": UserAgent().chrome        # 获取任意一个版本的谷歌的UserAgent信息
    }
    print(‘----------‘, url)
    request = Request(url, headers=headers)
    response = urlopen(request)
    # print(response.read().decode())

    return response.read()


# 保存数据
def save_html(data_bytes, filename):
    with open(filename, ‘wb‘) as f:      # wb保存，ab追加保存
        print(‘文件{}保存中...‘.format(filename))
        f.write(data_bytes)
        

# 主函数
def main():
    # 创建参数变量
    teiba_name = input("请输入要下载的贴吧名称: ")
    num = input("要下载的页数: ")
    for i in range(int(num)):
        args = {
            
            ‘kw‘: teiba_name,       # 贴吧名称
            ‘pn‘: i*50              # 计算--参数pn（每页显示多少条，会影响pn的值，此处每页显示50条信息）
        }
        # 确定url中的变量：pn、kw, （ie是不变的）
        """
        url 示例
        url = ‘https://tieba.baidu.com/f?kw=%E5%B0%9A%E5%AD%A6%E5%A0%82&ie=utf-8&pn=100‘
        """
        url = "https://tieba.baidu.com/f?ie=utf-8&{}".format(urlencode(args))
        
        # 调用获取网页信息函数
        data_bytes = get_html(url)
        
        # 保存数据
        filename = teiba_name + "第" + str(i+1) + "页.html"
        save_html(data_bytes, filename)

        
if __name__ == ‘__main__‘:
    main()