自动化爬虫爬取腾讯招聘信息

输入页数开始爬取

可设定是否无头浏览

有一个坑就是在翻页时无法直接click该元素

要写一个执行js

预览图:

 上代码

import time
from selenium import webdriver
from selenium.webdriver.common.by import By


class Tencent():
    def __init__(self, page):
        self.page = page+1
        self.url = "https://careers.tencent.com/search.html"
        self.opt = webdriver.ChromeOptions()  # 配置
        # opt.add_argument("--headless")
        # opt.add_argument("--disable-gpu")
        # self.opt.add_argument("--referer=https://careers.tencent.com/search.html?index=3")
        self.opt.add_argument("--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36")
        self.driver = webdriver.Chrome(options=self.opt)

    def parse(self):
        time.sleep(1)
        pos_list = self.driver.find_elements(By.XPATH, "/html/body/div/div[4]/div[3]/div[2]/div[2]/div/div")
        time.sleep(0.2)
        csv = []
        for info in pos_list:
            time.sleep(0.1)
            dict = {}
            dict['position'] = info.find_element(By.XPATH, "./a/h4").text
            dict['type'] = info.find_element(By.XPATH, "./a/p[1]/span[1]").text
            dict['palce'] = info.find_element(By.XPATH, "./a/p[1]/span[2]").text
            dict['time'] = info.find_element(By.XPATH, "./a/p[1]/span[4]").text
            csv.append(dict)
        return csv

    def show(self, csv):
        for i in csv:
            print(i)

    def run(self):
        self.driver.get(url=self.url)
        self.driver.implicitly_wait(10)
        while True:
            self.page -= 1
            if self.page == 0:
                break
            ans = self.parse()
            self.show(ans)
            time.sleep(1)
            try:
                button = self.driver.find_element(By.XPATH, "/html/body/div/div[4]/div[3]/div[2]/div[3]/ul/li[10]")
                self.driver.execute_script("$(arguments[0]).click()", button)
            except:
                break

if __name__ == '__main__':
    p = int(input("请输入要爬取的页数"))
    obj = Tencent(p)
    obj.run()