Python爬虫采集交易龙虎榜

**

**

import json
import os
import requests
import openpyxl
import time
import random

#爬虫联系wx：pyzzdd
def loadInfo():
    '''
    获取企业名单中所有企业名称
    :return: 企业名称
    '''
    wb = openpyxl.load_workbook('企业名单.xlsx')
    ws = wb.active
    business_names = list(set([ ws.cell(i+1,1).value for i in range(1,ws.max_row)]))   #去重
    return business_names

def search(b_name,dl):
    '''
    搜索企业信息
    获取uniscid
    :param b_name:
    :param dl:
    :return:
    '''
    search_url = 'http://data.ggzy.gov.cn/yjcx/index/search'
    payload = {"keyword":b_name,"page":"1"}
    data = ''
    while data == '':
        try:
            data = requests.post(search_url,json=payload,headers=headers,proxies=dl).text
        except:
            dl = proxies()
    try:
        data = json.loads(data)
    except:
        return False
    try:
        if data['message'] == '眼睛累了，休息一下吧':
            print('ip被封')
            return False
    except:
        pass

    d = data['data']
    if d == []:
        print('未获取到信息')
        return False
    print('搜索成功')
    entnames = [ t['entname'] for t in d]
    uniscids = [ t['uniscid'] for t in d]
    if len(entnames)!= 1:
        for e in range(len(entnames)):
            if entnames[e] == b_name:
                uniscid = uniscids[e]
    else:
        uniscid = uniscids[0]

    return uniscid

def detaile_information(uniscid,b_name,dl):
    '''
    采集详情数据
    :param uniscid:
    :param b_name:
    :param dl:
    :return:None
    '''
    bit_url = 'http://data.ggzy.gov.cn/yjcx/index/bid_list'
    page = 1
    tender_org_names = []
    project_names = []
    tos = []
    create_times = []
    bid_prices = []
    while True:
        print('正在采集第{}页数据'.format(page))
        payload = {"uniscid":uniscid,"page":page,"tos":""}
        info = ''
        while info=='':
            try:
                info = requests.post(bit_url,json=payload,headers=headers,proxies=dl).text
            except:
                dl = proxies()
            time.sleep(random.uniform(0.1,0.4))
        info = json.loads(info)
        try:
            if info['message'] == '眼睛累了，休息一下吧':
                print('ip被封')
                return False
        except:
            pass
        if info['data'] == []:
            break
        tender_org_names += [i['tender_org_name'] for i in info['data']]
        project_names += [i['project_name'] for i in info['data']]
        tos += [i['tos'] for i in info['data']]
        create_times += [i['create_time'] for i in info['data']]
        bid_prices += [i['bid_price'] for i in info['data']]
        page+=1
    save(tender_org_names,project_names,tos,create_times,bid_prices,b_name)

def save(tender_org_names,project_names,tos,create_times,bid_prices,b_name):
    '''
    保存信息
    :param tender_org_names:
    :param project_names:
    :param tos:
    :param create_times:
    :param bid_prices:
    :param b_name:
    :return:
    '''
    row = 2
    wb = openpyxl.Workbook()
    ws = wb.active
    for l in range(len(lines)):
        ws.cell(1,l+1,lines[l])

    for t in range(len(tender_org_names)):
        ws.cell(row,1,tender_org_names[t])
        ws.cell(row, 2, project_names[t])
        ws.cell(row, 3, tos[t])
        ws.cell(row, 4, create_times[t])
        ws.cell(row, 5, bid_prices[t])
        row+=1
    wb.save('all/{}.xlsx'.format(b_name))

def proxies():
    '''
    代理IP
    :return:
    '''
    url = 'xxxx'
    ip = requests.get(url).text
    while 'ERROR' in ip or ip == '':
        time.sleep(5)
        ip = requests.get(url).text
    ip = str(ip).replace('rn','')
    proxies = {'http': "http://{}".format(ip),
               'https': "http://{}".format(ip)}
    return None    #不使用代理则改为 return None   网站封IP很严重，如果要自己运行的务必加上代理

def getFlist(file_dir):
    '''
    获取已采集的企业名称
    :param file_dir:
    :return:
    '''
    for root, dirs, files in os.walk(file_dir):
        print('files:', files)     #文件名称，返回list类型
    return files

def run():

    file_dir = './all'  # 你的文件路径
    business_names = loadInfo()   #拿到所有企业信息
    files = getFlist(file_dir)    #拿到已经采集到的企业信息
    for b_name in business_names:
        print(b_name)
        if b_name+'.xlsx' in files:
            print('该企业已采集')
            continue
        dl = proxies()   #加代理
        try:
            uniscid = search(b_name,dl)    #如果搜索模块出现异常，则uniscid=0
        except Exception as e:
            print(e)
            uniscid = 0
        if uniscid:
            detaile_information(uniscid, b_name,dl)
        time.sleep(random.uniform(0.1,0.3))

if __name__ == '__main__':

    headers = {
        'Accept': 'application/json, text/plain, */*',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Cache-Control': 'no-cache',
        'Content-Length': '50',
        'Content-Type': 'application/json;charset=UTF-8;',
        'Host': 'data.ggzy.gov.cn',
        'Origin': 'http://data.ggzy.gov.cn',
        'Pragma': 'no-cache',
        'Proxy-Connection': 'keep-alive',
        'Referer': 'http://data.ggzy.gov.cn',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.63 Safari/537.36',
    }
    lines = ['招标(采购)人','项目名称','交易领域','中标日期','中标(成交)金额(万元)']

    run()
    #爬虫联系wx：pyzzdd