超级实用的Python网络爬虫反反爬策略之构造UA池及IP代理池

import  requests
import random
def main():
    #创建UA池
    headers_list = [
        {
            ‘User_Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0‘
        },
        {‘User_Agent‘:‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36‘}
    ]
    #创建IP代理池
    proxy_list=[
        {"http":"47.107.38.138:8000"},
        {
            "http":"163.125.113.224:8118"
        }
    ]
    #发请求时，随机从池中选择UA
    header=random.choice(headers_list)
    #随机选择代理IP，可以去一些代理站点寻找高匿IP
    proxy=random.choice(proxy_list)
    #抓取百度首页
    url="https://www.baidu.com/"
    page_text=requests.get(url=url,headers=header,proxies=proxy).content
    with open(‘./daili.html‘,‘wb‘)as fp:
        fp.write(page_text)
if __name__ == ‘__main__‘:
    main()