【电商系列？】日本某美食攻略网站数据获取及Json文件保存

        17日更新：版权方面做马赛克处理，请审核高抬贵手。

        今天上班摸鱼的时候发现了一个比较新鲜的网站，是一个日本美食攻略的网站：

        左边呢是各个店家的基本信息，右边也集成了地图 。
        反正疫情也出不去，这个网站的地图卫星图像也能云旅游，而左边的这些店铺的数据呢，就可以拿下来给未来做做旅游攻略了。

        进入首页，可以看到各个地区的分类，而下面是按食品的分类，人数等等。

        这次就按地区来获取数据。随便进入一个地区，点开地区排行榜，就进入列表页了，点击列表就进入详情了，简单的列表-列表-详情的三段式结构。

        首先进入主页，获取所有地区：

def getList():
    Url = "https://tabelog.com/cn/"
    headers = {
        "authority": "tabelog.com",
        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,**;q=0.8,application/signed-exchange;v=b3;q=0.9",
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36",
}

infolist = []

def getApiIp():
    # 获取且仅获取一个ip
    api_url = '代理地址'
    res = requests.get(api_url, timeout=5)
    try:
        if res.status_code == 200:
            api_data = res.json()['data'][0]
            proxies = {
                'http': 'http://{}:{}'.format(api_data['ip'], api_data['port']),
                'https': 'http://{}:{}'.format(api_data['ip'], api_data['port']),
            }
            print(proxies)
            return proxies
        else:
            print('获取失败')
    except:
        print('获取失败')

def getList():
    res = requests.get(Url,headers=headers,proxies = getApiIp())
    content = res.content.decode('utf-8')
    sel = Selector(text=content)
    nodes = sel.css(".index-area section dl dd ul li")
    for node in nodes[:1]:
        data = {}
        data["area"] = node.css("a::text").extract_first()
        areaUrl = node.css("a::attr(href)").extract_first()+"rstLst/?SrtT=rt"
        getRankList(data,areaUrl)
    insertJson()

def getRankList(data,url):
    res = requests.get(url, headers=headers,proxies = getApiIp())
    content = res.content.decode('utf-8')
    sel = Selector(text=content)
    nodes = sel.css(".lookup-result ul li")
    for node in nodes[:1]:
        name = node.css(".list-rst__header p a::text").extract_first()
        namejp = node.css(".list-rst__header small::text").extract_first()
        infourl = node.css(".list-rst__header p a::attr(href)").extract_first()
        rank = node.css(".list-rst__rate li b::text").extract_first()
        dinnerPrice = node.css(".c-rating__val::text").extract()
        lunchPrice = node.css(".c-rating__val::text").extract()
        if name and namejp and infourl and rank and dinnerPrice and lunchPrice:
            data["name"] = name
            data["namejp"] = namejp
            data["infourl"] = infourl
            data["rank"] = rank
            data["dinnerPrice"] = dinnerPrice[1]
            data["lunchPrice"] = lunchPrice[2]
            getDetail(data,infourl)

def getDetail(data,url):
    res = requests.get(url, headers=headers,proxies = getApiIp())
    content = res.content.decode('utf-8')
    sel = Selector(text=content)
    detail = sel.css("#anchor-rd-detail").extract_first()
    data["detail"] = detail
    infolist.append(data)

def insertJson():
    with open('Data.json', 'w', encoding='utf-8') as file:
        file.write(json.dumps(infolist))

if __name__ == '__main__':
    getList()