python 爬虫

爬取网站，同程机票
问题：

1. ajax动态刷新页面，普通的requests和xpath出现错误
2. Chaorme.exe 路径安装问题，下载网站chorem下载网站
3. 模拟点击问题
4. 模拟点击后如何获取页面内容：重新 brower.sourse_page
5. selenium 可以通过 xpath 获取点击的标签点
6. csv存储问题

from selenium import webdriver
from lxml import etree
from selenium.webdriver.common.by import By
from selenium.webdriver import ActionChains
import pandas as pd
import time

# 这里是 Chorme.exe 的路径
path = "D:Downloadchromdriverchromedriver_win32chromedriver.exe"
browser = webdriver.Chrome(executable_path=path)
browser.get(url = "https://www.ly.com/flights/home")
browser.implicitly_wait(6)
# 存储数据的列表
csv_data = []
for k in range(1,13):
    data = []
    # 模拟点击
    page = str(k)
    element_click_xpa = '//*[@id="__layout"]/div/section/div[4]/dl/dd[%s]'%(page)
    print("xpath",element_click_xpa)
    element_click = browser.find_element(By.XPATH,element_click_xpa)
    action = ActionChains(browser)
    action.click(element_click)
    action.perform()
    # 一定要睡眠10s,再重新获取页面资源,否则会出错
    time.sleep(11)
    html = browser.page_source
    xhtml =etree.HTML(html)
    xp1 = '//body/div[@id="__nuxt"]/div[@id="__layout"]/div/section[@]/div[@]/div[@]/a/@href'
    xp2 = '//body/div[@id="__nuxt"]/div[@id="__layout"]/div/section[@]/div[@]/div[@]/a/span/text()'
    res1 = xhtml.xpath(xp1)
    res2 = xhtml.xpath(xp2)
    base_url = "https://www.ly.com"
    j = 0
    # 循环该页面有十二个城市
    for i in range(len(res1)):
        data = []
        data.append(res2[j])
        data.append(res2[j + 1])
        data.append(res2[j + 2])
        data.append(res2[j + 3])
        data.append(base_url+res1[i])
        csv_data.append(data)
        j += 4
    print(csv_data)
# 存入 csv
pd = pd.DataFrame(csv_data,columns=['出发城市','到达城市','日期','价格','跳转链接'])
pd.to_csv("./flay_infomation.csv")