EMSpider

标签：gen   imei   cursor   params   localtime   mysq   back   local   page   

# -*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup
import lxml
import json
import re
import time
import tushare as ts
import pandas as pd
import pymysql

class Mysqldb(object):
    def __init__(self, host, port, user, passwd, dbname, charset, sql):
        self.host = host
        self.port = int(port)
        self.user = user
        self.passwd = passwd
        self.dbname = dbname
        self.charset = charset
        self.conn = None
        self.cursor = None
        self.sql = sql

    def connect(self):
        self.conn = pymysql.connect(host=self.host, port=self.port, user=self.user, passwd=self.passwd, dbname=self.dbname)
        self.cursor = self.conn.cursor()

    def execute(self, sql):
        try:
            self.cursor.execute(self.sql)
            count = self.cursor.execute(self.sql)
            return count
        except:
            self.dbname.rollback()

    def commit(self):
        self.conn.commit()

    def close(self):
        self.cursor.close()
        self.conn.close()

def EMydSpider(current):
    # 抓取东财个股盘口异动数据：http://quote.eastmoney.com/changes
    pymysql.connect(host=‘127.0.0.1‘, port=3306, user=‘root‘, passwd=‘123456‘, dbname=‘quantitative_trading_data‘, charset=‘utf8‘)
    header = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3676.400 QQBrowser/10.5.3738.400"}
    url = "http://push2ex.eastmoney.com/getAllStockChanges?type=8201,8202,8193,4,32,64,8207,8209,8211,8213,8215,8204,8203,8194,8,16,128,8208,8210,8212,8214,8216"
    for page in range(200):
        param = {"pageindex": page,"pagesize": ‘64‘,"ut": ‘7eea3edcaed734bea9cbfc24409ed989‘,"dpt": ‘wzchanges‘}
        html = json.loads(requests.get(url=url, params=param, headers=header).text)
        if html[‘data‘] is None:
            print("东方财富：共爬取%d页数据" % page)
            print("东方财富：个股盘口异动数据已抓取完成")
            break
        else:
            allstock = html[‘data‘][‘allstock‘]
            for stock in allstock:
                stk_code = stock[‘c‘]
                stk_name = stock[‘n‘]
                chg_time = stock[‘tm‘]
                chg_type = stock[‘t‘]
                chg_value = stock[‘i‘]
                
                sql = "insert into emyd_data(current,stk_code,stk_name,chg_time,chg_type,chg_value) values(%s,%s,%s,%s,%s,%s)" %(current,stk_code,stk_name,chg_time,chg_type,chg_value)
                Mysqldb.execute(sql)
    Mysqldb.commit()
    Mysqldb.close()

if __name__=="__main__":
    current = time.strftime("%Y%m%d",time.localtime())
    EMydSpider(current)

# -*- coding: utf-8 -*-import requestsfrom bs4 import BeautifulSoupimport lxmlimport jsonimport reimport timeimport tushare as tsimport pandas as pdimport pymysql
class Mysqldb(object):    def __init__(self, host, port, user, passwd, dbname, charset, sql):        self.host = host        self.port = int(port)        self.user = user        self.passwd = passwd        self.dbname = dbname        self.charset = charset        self.conn = None        self.cursor = None        self.sql = sql
    def connect(self):        self.conn = pymysql.connect(host=self.host, port=self.port, user=self.user, passwd=self.passwd, dbname=self.dbname)        self.cursor = self.conn.cursor()
    def execute(self, sql):        try:            self.cursor.execute(self.sql)            count = self.cursor.execute(self.sql)            return count        except:            self.dbname.rollback()
    def commit(self):        self.conn.commit()
    def close(self):        self.cursor.close()        self.conn.close()
def EMydSpider(current):    # 抓取东财个股盘口异动数据：http://quote.eastmoney.com/changes    pymysql.connect(host=‘127.0.0.1‘, port=3306, user=‘root‘, passwd=‘123456‘, dbname=‘quantitative_trading_data‘, charset=‘utf8‘)    header = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3676.400 QQBrowser/10.5.3738.400"}    url = "http://push2ex.eastmoney.com/getAllStockChanges?type=8201,8202,8193,4,32,64,8207,8209,8211,8213,8215,8204,8203,8194,8,16,128,8208,8210,8212,8214,8216"    for page in range(200):        param = {"pageindex": page,"pagesize": ‘64‘,"ut": ‘7eea3edcaed734bea9cbfc24409ed989‘,"dpt": ‘wzchanges‘}        html = json.loads(requests.get(url=url, params=param, headers=header).text)        if html[‘data‘] is None:            print("东方财富：共爬取%d页数据" % page)            print("东方财富：个股盘口异动数据已抓取完成")            break        else:            allstock = html[‘data‘][‘allstock‘]            for stock in allstock:                stk_code = stock[‘c‘]                stk_name = stock[‘n‘]                chg_time = stock[‘tm‘]                chg_type = stock[‘t‘]                chg_value = stock[‘i‘]                #sql = "insert into emyd_data(current,stk_code,stk_name,chg_time,chg_type,chg_value) values" %(current, stk_code, stk_name, chg_time, chg_type, chg_value)                sql = "insert into emyd_data(current,stk_code,stk_name,chg_time,chg_type,chg_value) values(%s,%s,%s,%s,%s,%s)" %(current,stk_code,stk_name,chg_time,chg_type,chg_value)                Mysqldb.execute(sql)    Mysqldb.commit()    Mysqldb.close()
if __name__=="__main__":    current = time.strftime("%Y%m%d",time.localtime())    EMydSpider(current)

EMSpider

标签：gen   imei   cursor   params   localtime   mysq   back   local   page   

原文地址：https://www.cnblogs.com/Iceredtea/p/11904290.html