import time
import random
from selenium.webdriver.common.keys import Keys
from lxml import etree
from selenium.webdriver import Chrome,ChromeOptions
import csv
def one_page():
# print(b.page_source)
global num
root = etree.HTML(b.page_source)
print(root)
all_jobs = root.xpath('//div[@]/div[@]')
for i in all_jobs:
title = i.xpath('a/p/span[@]/text()')[0]
payment = i.xpath('a/p[@]/span[@]/text()')
title1 = i.xpath('a/p[@]/span[@]/text()')[0]
cop_name = i.xpath('div[@]/a/text()')[0]
cop_class = i.xpath('div[@]/p[@]/text()')[0]
cop_domain = i.xpath('div[@]/p[@]/text()')[0]
print(title)
print(payment[0] if len(payment) != 0 else None)
print(title1)
print(cop_name)
print(cop_class)
print(cop_domain)
writer.writerow([title, payment[0] if len(payment) != 0 else None, title1, cop_name, cop_class, cop_domain])
# 翻页
for i in range(12):
b.execute_script('window.scrollBy(0,850)')
time.sleep(0.5 * random.random())
time.sleep(1*random.random())
next_page = b.find_element_by_id("jump_page")
next_page.clear()
next_page.send_keys(num)
num += 1
next_page.send_keys(Keys.ENTER)
button = b.find_element_by_class_name('og_but')
button.click()
print('-'*100)
if __name__ == '__main__':
options = ChromeOptions()
f = open('51job-数据分析.csv','w',encoding='utf-8',newline='')
writer = csv.writer(f)
writer.writerow(['职位名称', '薪资', '职位要求', '公司名', '公司规模', '公司类型'])
options.add_experimental_option('excludeSwitches', ['enable-automation'])
options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2})
num = 2
b = Chrome(options=options)
b.get('https://www.51job.com/')
time.sleep(1)
search = b.find_element_by_id('kwdselectid')
search.send_keys('数据分析')
search.send_keys(Keys.ENTER)
time.sleep(1)
for i in range(100):
one_page()
time.sleep(1*random.random())
共爬取100页5000条数据



