from bs4 import BeautifulSoup#网页解析获取数据
import re#正则表达式,进行文字匹配
import urllib.request,urllib.error#指定url,获取数据
from urllib import parse
import xlwt#进行excel的操作
import requests
def main():
baseurl = "https://search.51job.com/list/000000,000000,0000,00,9,99,"
datalist = getdata(baseurl)
savepath = "51job.xls"
# 3.保存数据
savedata(datalist, savepath)
def askurl(url):
head = {
"User-Agent": "Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 80.0.3987.122 Safari / 537.36"
}
# 用
request = urllib.request.Request(url, headers=head)
html = ""
try:
page_text = requests.get(url=url,headers=head).text
# print(html)
except urllib.error.URLError as e:
if hasattr(e, "code"):
print(e.code)
if hasattr(e, "reason"):
print(e.reason)
return html#可以通用
jobHref = '"job_href":"(.*?)"' #岗位链接
jobName = '"job_name":"(.*?)"' #岗位名称
comHref = '"company_href":"(.*?)"' #公司链接
comName = '"company_name":"(.*?)"' #公司名称
salary = '"providesalary_text":"(.*?)"' #薪资水平
companytype = '"companytype_text":"(.*?)"' #公司类型
attribute = '"attribute_text":[(.*?)]' #招聘条件
workarea = '"workarea_text":"(.*?)"' #工作地点
companysize = '"companysize_text":"(.*?)"' #公司规模
companyind = '"companyind_text":"(.*?)"' #主要业务
jobwelf = '"jobwelf":"(.*?)"' #福利待遇
def getdata(baseurl):
kw = input("请输入你要搜索的岗位关键字:")
keyword = parse.quote(parse.quote(kw)) # 对中文字进行转码
pageNum = 0
# 关键字二次转译
for i in range(0, 10):
datalist = []
pageNum += 1
# 这个url是怎么搞的
url = baseurl + keyword + ",2," + str(pageNum) + ".html" # 按搜索干的规律"
page_text = askurl(url)
soup = BeautifulSoup(page_text, "html.parser")
for item in soup.find_all("div", class_="e"):
data=[]
item =str(item)
jobHref_list = re.findall(jobHref, page_text, re.S)
data.append(jobHref_list)
jobName_list = re.findall(jobName, page_text, re.S)
data.append(jobName_list)
comHref_list = re.findall(comHref, page_text, re.S)
data.append(comHref_list)
comName_list = re.findall(comName, page_text, re.S)
data.append(comName_list)
salary_list = re.findall(salary, page_text, re.S)
data.append(salary_list)
companytype_list = re.findall(companytype, page_text, re.S)
data.append(companytype_list)
attribute_list = re.findall(attribute, page_text, re.S)
data.append(attribute_list)
workarea_list = re.findall(workarea, page_text, re.S)
data.append(workarea_list)
companysize_list = re.findall(companysize, page_text, re.S)
data.append(companysize_list)
companyind_list = re.findall(companyind, page_text, re.S)
data.append(companyind_list)
jobwelf_list = re.findall(jobwelf, page_text, re.S)
data.append(jobwelf_list)
# for item in soup.find_all("div", class_="item"):
datalist.append(data)
if len(datalist[0]) == 0:
print('没有搜索到职位信息')
break
else:
print('正在爬取第%d页' % pageNum)
return datalist
def savedata(datalist, savepath):
book = xlwt.Workbook(encoding="utf-8", style_compression=0) # 创建对象
sheet = book.add_sheet('豆瓣电影Top250', cell_overwrite_ok=True) # 创建工作表
col = ("岗位链接", "岗位名称", "公司链接", "公司名称", "薪资水平", "公司类型", "招聘条件", "工作地点", '公司规模', '主要业务', '福利待遇')
for i2 in range(0, 11):
sheet.write(0, i2, col[i2]) # 列名
for i in range(0,500):
print("第%d条" % (i + 1))
data = datalist[1]#应为datalist有250 条,且每一条,也是包含8条信息的列表
for j in range(0,11):
sheet.write(i+1,j,data[j])#数据
book.save(savepath)
if __name__ == "__main__": #当程序执行时
#调用函数
main()