码迷,mamicode.com
首页 > 其他好文 > 详细

爬取安居客指定市的所有小区信息

时间:2019-09-24 12:14:13      阅读:114      评论:0      收藏:0      [点我收藏+]

标签:head   相关   color   lte   use   时间   web   tab   信息保存   

在爬取的过程中发现,访问频率太快会导致网站弹出滑动验证,所以设定了时间随机时间延迟,这样子就能保证爬取的信息完整,我选的是青岛市的小区,后续也可以添加输入市名爬取相关内容,二级页面的房子的平均价格是动态生成的,需要发送一个请求得到一个json,请求的url比较复杂,而且还要再发送一次请求,因此直接在一级页面取平均价格,然后传入解析二级页面的函数,这样可以提高效率.代码如下:

"""
    爬取安居客所有小区信息
"""
import requests
from fake_useragent import UserAgent
from lxml import etree
import csv
import re
import time
import random


class AnjukeSpider(object):
    def __init__(self):
        self.url = https://qd.anjuke.com/community/p{}/

    def get_headers(self):
        """
            构建请求头
        :return:
        """
        ua = UserAgent()
        headers = {
            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
            "accept-encoding": "gzip, deflate, br",
            "accept-language": "zh-CN,zh;q=0.9",
            "cache-control": "max-age=0",
            "cookie": "aQQ_ajkguid=534DDCC9-5DBA-263A-CF4D-SX0716083828; isp=true; 58tj_uuid=e559fdad-fdb9-4a73-8c60-9e6e3bf82987; Hm_lvt_c5899c8768ebee272710c9c5f365a6d8=1563237510; als=0; _ga=GA1.2.1881437242.1569052175; ctid=30; wmda_uuid=edd62dcc1e73bddc16beeb56087fd1f8; wmda_new_uuid=1; wmda_visited_projects=%3B6289197098934; sessid=F6826357-F68F-1E17-B5A1-99FEA17341CA; lps=http%3A%2F%2Fwww.anjuke.com%2F%7Chttps%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3DcuNIKoO-jX3CGzD7komT_lY2umPIHgZjjBdMMdFnpZHirHVPOLorVTafN32HS5R_%26ck%3D7150.2.84.414.190.439.289.917%26shh%3Dwww.baidu.com%26sht%3D02003390_42_hao_pg%26wd%3D%26eqid%3Dc2951ba5003c81ad000000065d881f86; twe=2; wmda_session_id_6289197098934=1569202063874-b62b0050-2be7-3851; _gid=GA1.2.388348263.1569202065; init_refer=https%253A%252F%252Fwww.baidu.com%252Flink%253Furl%253DcuNIKoO-jX3CGzD7komT_lY2umPIHgZjjBdMMdFnpZHirHVPOLorVTafN32HS5R_%2526ck%253D7150.2.84.414.190.439.289.917%2526shh%253Dwww.baidu.com%2526sht%253D02003390_42_hao_pg%2526wd%253D%2526eqid%253Dc2951ba5003c81ad000000065d881f86; new_uv=3; new_session=0",
            "referer": "https://qd.anjuke.com/community/?from=navigation",
            "sec-fetch-mode": "navigate",
            "sec-fetch-site": "none",
            "sec-fetch-user": "?1",
            "upgrade-insecure-requests": "1",
            "user-agent": ua.random
        }
        return headers

    def get_link(self, url):
        """
            解析页面获取每个小区的二级页面链接和价格
        :param url:
        :return:
        """
        text = requests.get(url=url, headers=self.get_headers()).text
        html = etree.HTML(text)
        link = html.xpath("//h3/a/@href")
        price = html.xpath(//*[@id="list-content"]/div/div[2]/p[1]/strong/text())
        print(link)
        print(price)
        for i in zip(link, price):
            print(i)
        return zip(link, price)

    def parse_message(self, url, price):
        """
            二级页面解析需要的信息
        :param url:
        :param price:
        :return:
        """
        dict_result = {小区: -, 地址: -, 价格: -, 物业类型:: -, 物业费:: -, 总建面积:: -, 总户数:: -,
                       建造年代:: -,
                       停车位:: -,
                       容积率:: -,
                       绿化率:: -,
                       开发商:: -, 物业公司:: -, 所属商圈:: -, 二手房房源数:: -, 租房源数:: -, 相关学校:: -}

        text = requests.get(url=url, headers=self.get_headers()).text
        html = etree.HTML(text)
        table1 = html.xpath(/html/body/div[2]/div[3]/div[1]/h1//text())  # 提取小区名和地址
        table1 = list(map(lambda item: re.sub(\s+, ‘‘, item), table1))  # 去掉换行符制表符
        table1 = list(filter(None, table1))  # 去掉上一步产生的空元素
        dict_result[小区] = table1[0]
        dict_result[地址] = table1[1]
        dict_result[价格] = price
        table2 = html.xpath(//*[@id="basic-infos-box"]/dl//text())
        table2 = list(map(lambda item: re.sub(\s+, ‘‘, item), table2))
        table2 = list(filter(None, table2))
        table2_list1 = table2[::2]
        table2_list2 = table2[1::2]
        table2_list3 = zip(table2_list1, table2_list2)
        for j in table2_list3:
            dict_result[j[0]] = j[1]
        # price = html.xpath(‘//*[@id="basic-infos-box"]/div[1]/span[1]/text()‘)  #价格数据在json文件里面,所以这个没办法匹配到
        # dict_result[‘价格‘] = price[0]
        table3 = html.xpath(//*[@id="basic-infos-box"]/div[2]//text())
        table3 = list(map(lambda item: re.sub(\s+, ‘‘, item), table3))
        table3 = list(filter(None, table3))
        table3_list1 = table3[::2]
        table3_list2 = table3[1::2]
        table3_list3 = zip(table3_list1, table3_list2)
        for j in table3_list3:
            dict_result[j[0]] = j[1]
        print(dict_result)
        return dict_result

    def save_csv(self, result):
        """
            将信息保存进入csv文件
        :param result:
        :return:
        """
        headers = {小区, 地址, 价格, 物业类型:, 物业费:, 总建面积:, 总户数:,
                   建造年代:,
                   停车位:,
                   容积率:,
                   绿化率:,
                   开发商:, 物业公司:, 所属商圈:, 二手房房源数:, 租房源数:, 相关学校:}
        with open(青岛.csv, a, newline=‘‘) as f:
            writer = csv.DictWriter(f, headers)
            # writer.writeheader()
            for row in result:
                writer.writerow(row)

    def run(self):
        """
            主函数
        :return:
        """
        C = 1
        for i in range(1, 101):  # 总的272页
            url = self.url.format(i)
            link = self.get_link(url)
            list_result = []
            for j in link:
                try:
                    result = self.parse_message(j[0], j[1])
                    time.sleep(round(random.randint(1, 3), C))
                    list_result.append(result)
                except Exception as err:
                    print(err)
            self.save_csv(list_result)
            print("第%s页储存成功" % i)

        # url = ‘https://qd.anjuke.com/community/view/875393?from=Filter_1&hfilter=filterlist‘
        # self.parse_message(url)
        # self.get_link()


if __name__ == __main__:
    spider = AnjukeSpider()
    spider.run()

 

爬取安居客指定市的所有小区信息

标签:head   相关   color   lte   use   时间   web   tab   信息保存   

原文地址:https://www.cnblogs.com/lattesea/p/11577447.html

(0)
(0)
   
举报
评论 一句话评论(0
登录后才能评论!
© 2014 mamicode.com 版权所有  联系我们:gaon5@hotmail.com
迷上了代码!