spider-inline-requests

  1 # -*- coding: utf-8 -*-
  2 import re
  3 import time
  4 
  5 import scrapy
  6 from inline_requests import inline_requests
  7 from requests_toolbelt import MultipartEncoder
  8 
  9 from tracking.items import TrackingBlNoItem, TrackingEventItem, TrackingContainerNoItem
 10 
 11 
 12 class OOCLSpider(scrapy.Spider):
 13     name = ‘OOCL‘
 14     allowed_domains = [‘moc.oocl.com‘]
 15     start_urls = [‘http://moc.oocl.com/‘]
 16 
 17     def start_requests(self):
 18         bl_no = ‘2104523720‘
 19         post_url = ‘http://moc.oocl.com/party/cargotracking/ct_search_from_other_domain.jsf‘
 20         form_data = {
 21             "ANONYMOUS_BEHAVIOR": "BUILD_UP",
 22             "domainName": "PARTY_DOMAIN",
 23             "ENTRY_TYPE": "OOCL",
 24             "ENTRY": "MCC",
 25             "ctSearchType": "BL",
 26             "ctShipmentNumber": bl_no
 27         }
 28 
 29         header = {
 30             "HOST": "moc.oocl.com",
 31             "Cache-Control": "max-age=0",
 32             "Origin": "http://localhost:63343",
 33             "Upgrade-Insecure-Requests": "1",
 34             "Content-Type": "application/x-www-form-urlencoded",
 35             "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
 36             ‘Accept‘: ‘text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8‘,
 37             "Referer": "http://localhost:63343/yqn_standard_shipping_info/test2.html?_ijt=hj7nk9lapnrpo3k5unameggs59",
 38             # "Accept-Encoding": "gzip, deflate",
 39             ‘Accept-Language‘: ‘zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7,ja;q=0.6‘
 40         }
 41         yield scrapy.FormRequest(
 42             url=post_url,
 43             formdata=form_data,
 44             headers=header,
 45             callback=self.parse
 46         )
 47 
 48     def parse(self, response):
 49         bl_no = ‘2104523720‘
 50         post_url = ‘http://moc.oocl.com‘
 51         html_str = response.text
 52         # cookies = response.headers.getlist(‘Set-Cookie‘)[0]
 53         # print(str(cookies))
 54         # session = re.findall(‘(?<=JSESSIONID=)[^\;]*‘, str(cookies), re.S)[0]
 55         # str(response.headers.getlist(‘Set-Cookie‘))
 56         url_plus = re.findall(‘ANONYMOUS_TOKEN=.*&ENTRY=.*&PREFER_LANGUAGE=[^\"]*‘, html_str, re.S)[0]
 57         form = response.xpath(‘//*[@id="hiddenForm"]‘)
 58         refer = form.xpath(‘./@action‘)[0].extract()
 59         post_url = post_url + refer + "?" + url_plus
 60         input_list = form.xpath(‘./input‘)
 61         form_data = dict()
 62         UT = []
 63         form_data[‘USER_TOKEN‘] = UT
 64         for input_item in input_list:
 65             name = input_item.xpath(‘./@name‘)
 66             name_str = name[0].extract()
 67             if name_str != ‘USER_TOKEN‘:
 68                 value_str = ‘‘
 69                 value = input_item.xpath(‘./@value‘)
 70                 if len(value) > 0:
 71                     value_str = value[0].extract()
 72                 form_data[name_str] = value_str
 73             else:
 74                 value_str = ‘‘
 75                 value = input_item.xpath(‘./@value‘)
 76                 if len(value) > 0:
 77                     value_str = value[0].extract()
 78                 UT.append(value_str)
 79 
 80         form_data["hiddenForm:searchType"] = "BL"
 81         form_data["hiddenForm:billOfLadingNumber"] = bl_no
 82         form_data["hiddenForm:supportUtfChars"] = "true"
 83         form_data["hiddenForm:_link_hidden_"] = "hiddenForm:goToCargoTrackingBL"
 84         header = {
 85             "HOST": "moc.oocl.com",
 86             "Cache-Control": "max-age=0",
 87             "Origin": "http://moc.oocl.com",
 88             "Upgrade-Insecure-Requests": "1",
 89             "Content-Type": "application/x-www-form-urlencoded",
 90             "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
 91             ‘Accept‘: ‘text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8‘,
 92             "Referer": "http://moc.oocl.com/party/cargotracking/ct_search_from_other_domain.jsf",
 93             # "Accept-Encoding": "gzip, deflate",
 94             ‘Accept-Language‘: ‘zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7,ja;q=0.6‘
 95         }
 96 
 97         yield scrapy.FormRequest(
 98             url=post_url,
 99             formdata=form_data,
100             headers=header,
101             # cookies={"JSESSIONID": session},
102             callback=self.parse_container
103         )
104     # 哪里使用for循环，装饰器就放在那个方法头上
105     @inline_requests
106     def parse_container(self, response):
109         # 出发港口
110         depart = response.xpath(‘//*[@id="form:PORLocation0"]/text()‘)[0].extract()
111         # 装货港
112         load = response.xpath(‘//*[@id="form:POLLocation0"]/text()‘)[0].extract()
113         vessel_voyage = response.xpath(‘//*[@id="eventListTable"]/tr/td[5]/text()‘).extract_first()
114         info = vessel_voyage.split()
115         # 船名
116         vessel = info[2]
117         # 航次
118         voyage = info[3]
119         # 卸货港
120         discharge = response.xpath(‘//*[@id="form:PortOfDischarge0"]/text()‘).extract_first()
121         # 目的港
122         arrive = response.xpath(‘//*[@id="form:finalDestination0"]/text()‘).extract_first()
123 
124         # 表单
125         bl_item = TrackingBlNoItem()
126         bl_item[‘bl_no‘] = ‘2104523720‘
127         bl_item[‘depart‘] = depart
128         bl_item[‘load‘] = load
129         bl_item[‘arrive‘] = arrive
130         bl_item[‘discharge‘] = discharge
131         bl_item[‘carrier_id‘] = 4558
132         bl_item[‘carrier_code‘] = ‘OOCL‘
133         bl_item[‘vessel_name‘] = vessel
134         bl_item[‘voyage‘] = voyage
135         bl_item[‘request_url‘] = response.url
136         bl_item[‘weight‘] = ‘‘
137         bl_item[‘pieces‘] = ‘‘
138         bl_item[‘volume‘] = ‘‘
139         bl_item[‘version‘] = ‘‘
140         bl_item[‘event_list‘] = []
141         container_list = []
142         container_info_tr_list = response.xpath(‘//*[@id="summaryTable"]/tr‘)
143         no = 0
144         for tr in container_info_tr_list[3:]:
145             container_info = TrackingContainerNoItem()
146             # 箱大小
147             container_info[‘container_size‘] = ‘‘
148             # 箱类型
149             container_info[‘container_type‘] = ‘‘
150             # 封号
151             container_info[‘seal_no‘] = ‘‘
152             # 箱号
153             container_info[‘container_no‘] = tr.xpath(‘./td/a/text()‘).extract_first().split(‘-‘)[0]
154             # 事件
155             container_info[‘event_list‘] = []
156             container_list.append(container_info)
157 
158             # # 增加箱事件   发起post请求
159             # # 获取post_url 的动态参数token
160             ANONYMOUS_TOKEN = response.xpath(‘//*[@id="form"]/input[1]/@value‘).extract_first()
161             jsf_tree_64 = response.xpath(‘//*[@id="jsf_tree_64"]/@value‘).extract_first()
162             jsf_state_64 = response.xpath(‘//*[@id="jsf_state_64"]/@value‘).extract_first()
163             jsf_viewid = response.xpath(‘//*[@id="jsf_viewid"]/@value‘).extract_first()
164             ANONYMOUS_BEHAVIOR = response.xpath(‘//*[@id="ANONYMOUS_BEHAVIOR"]/@value‘).extract_first()
165             container_no = container_info[‘container_no‘]
166             form_link = ‘form:link‘ + str(no)
167             no += 1
168             token = ANONYMOUS_TOKEN
169             post_url = ‘http://moc.oocl.com/party/cargotracking/ct_result_bl.jsf?&ANONYMOUS_TOKEN=%s&‘ 170                        ‘ENTRY=MCC&ENTRY_TYPE=OOCL&PREFER_LANGUAGE=en-US‘ % token
171             headers = {
172                 ‘Host‘: ‘moc.oocl.com‘,
173                 # ‘Content-Length‘: ‘12911‘,
174                 ‘Pragma‘: ‘no-cache‘,
175                 ‘Cache-Control‘: ‘no-cache‘,
176                 ‘Origin‘: ‘http://moc.oocl.com‘,
177                 ‘Upgrade-Insecure-Requests‘: ‘1‘,
178                 # ‘Content-Type‘: ‘multipart/form-data; boundary=----WebKitFormBoundary6tLY0zPjk3DPKQIr‘,
179                 ‘User-Agent‘: ‘Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36‘,
180                 ‘Accept‘: ‘text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8‘,
181                 ‘Referer‘: ‘http://moc.oocl.com/party/cargotracking/ct_result_bl.jsf?&ANONYMOUS_TOKEN=LxzcmeKTFHdxDMLieyzHMCCOOCL&ENTRY=MCC&ENTRY_TYPE=OOCL&PREFER_LANGUAGE=en-US‘,
182                 # ‘Accept-Encoding‘: ‘gzip, deflate‘,
183                 ‘Accept-Language‘: ‘zh-CN,zh;q=0.9‘,
184                 # ‘Cookie‘: ‘JSESSIONID=7a8fc465850fcd03590d06ebbd340ea583a91689025a7e658861bdf4a57632cf.e38NbxiLa3qOci0Oa3eQaxmSci0; AcceptCookie=yes; BIGipServerpool_moc=737985227.25886.0000; _ga=GA1.3.1816166690.1545736626; _gid=GA1.3.133395829.1545736626; _mkto_trk=id:098-DFA-145&token:_mch-oocl.com-1545736627194-22944; WT_FPC=id=2aeb5802f8c5bd76dbc1545736626441:lv=1545830754900:ss=1545829378448‘,
185                 ‘Connection‘: ‘keep-alive‘,
186             }
187             multipart = {
188                 ‘ANONYMOUS_TOKEN‘: ANONYMOUS_TOKEN,
189                 ‘ENTRY‘: ‘MCC‘,
190                 ‘ENTRY_TYPE‘: ‘OOCL‘,
191                 ‘PREFER_LANGUAGE‘: ‘en-US‘,
192                 ‘ANONYMOUS_BEHAVIOR‘: ANONYMOUS_BEHAVIOR,
193                 ‘form:forward_uri‘: ‘‘,
194                 ‘form:rtContainerNumberValueID‘: ‘‘,
195                 ‘form:crossDomainSaveStateMapString‘: ‘‘,
196                 ‘form_SUBMIT‘: ‘1‘,
197                 ‘searchCriteriaSearchTypeCode‘: ‘BL‘,
198                 ‘searchCriteriaBookingNumber‘: ‘‘,
199                 ‘searchCriteriaContainerNumbers‘: ‘‘,
200                 ‘form:_link_hidden_‘: form_link,
201                 ‘searchCriteriaBillOfLadingNumber‘: bl_item[‘bl_no‘],
202                 ‘currentContainerNumber‘: container_no,
203                 ‘jsf_tree_64‘: jsf_tree_64,
204                 ‘jsf_state_64‘: jsf_state_64,
205                 ‘jsf_viewid‘: jsf_viewid,
206             }
207             response = yield scrapy.FormRequest(
208                 url=post_url,
209                 headers=headers,
210                 formdata=multipart,
211                 dont_filter=True,
212             )
　　　　　　　　　 #  内联请求的回调函数
215             self.parse_container_event(response, container_info, vessel, voyage)
216 　　　　　# yield 的返回必须在 for 循环结束
217         yield {
218             ‘bl_item‘: bl_item,
219             ‘container_list‘: container_list
220         }
221 
222     def parse_container_event(self, response, container_info, vessel, voyage):
223         event_info_tr_list = response.xpath(‘//*[@id="eventListTable"]/tr‘)
224         for event_tr in event_info_tr_list[3:]:
225             event = TrackingEventItem()
226             # 船名
227             event[‘vessel_name‘] = vessel
228             # 航次
229             event[‘voyage‘] = voyage
230             # 事件名称
231             event[‘event_name‘] = ‘‘
232             # 事件 code
233             event[‘event_code‘] = ‘‘
234             # 事件描述
235             event[‘event_description‘] = event_tr.xpath(‘./td/text()‘).extract_first()
236             if event[‘event_description‘] is not None:
237                 event[‘event_description‘] = event[‘event_description‘].strip()
238             else:
239                 event[‘event_description‘] = ‘‘
240 
241             # 修改时间日期格式
242             def transform_format_event_data(event_time):
243                 event_date_list = event_time.split(‘,‘)
244                 event_time = event_date_list[0] + ‘ ‘ + event_date_list[1].lstrip().split()[0]
245                 to_event_time = time.strptime(event_time, "%d %b %Y %H:%M")
246                 event_time = time.strftime("%Y-%m-%d %H:%M:%S", to_event_time)
247                 return event_time
248 
249             # 事件预计时间
250             event[‘expected_time‘] = ‘‘
251             if event[‘expected_time‘]:
252                 event[‘expected_time‘] = transform_format_event_data(event[‘expected_time‘])
253             # 事件实际时间
254             event[‘actual_time‘] = event_tr.xpath(‘./td[5]/span/text()‘).extract_first()
255             if event[‘actual_time‘]:
256                 event[‘actual_time‘] = transform_format_event_data(event[‘actual_time‘])
257             # 事件位置
258             event[‘location_name‘] = event_tr.xpath(‘./td[3]/span/text()‘).extract_first()
259             # 位置类型
260             event[‘location_type‘] = ‘‘
261             # 位置 code
262             event[‘location_code‘] = ‘‘
263             container_info[‘event_list‘].append(event)