# Define your item pipelines here
#
# Don‘t forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter

import json
import os
import urllib.request

class MyfirstScrapydemoPipeline:

    #爬虫启动时会调用此方法
    def open_spider(self,spider):
        print(‘爬虫启动‘)
        self.fp= open(‘qiushibaike.txt‘,‘w‘,encoding=‘utf8‘)

    def process_item(self, item, spider):
        #下载头像图片
        self.download(item)
        #将数据转化为字典
        obj= dict(item)
        #将字典数据装换为json格式
        string= json.dumps(obj,ensure_ascii=False)
        #将数据写入文件
        self.fp.write(string+‘\n‘)
        #不注释return item时只会下载第一页的头像
        #return item

    def download(self,item):
        #定义头像图片存放路径
        dirpath= r‘F:\python_project\爬虫\myfirst_scrapyDemo\myfirst_scrapyDemo\spiders\头像‘
        #获取每一张图片的名称
        name= item[‘name‘]+‘.jpg‘
        #拼接每一张头像图片的存储路径
        filepath=os.path.join(dirpath,name)
        #下载头像图片
        urllib.request.urlretrieve(item[‘face_src‘],filepath)


    # 爬虫结束时会调用此方法
    def close_spider(self,spider):
        print(‘爬虫结束‘)
        self.fp.close()