腾讯职位招聘案例

• 爬取招聘岗位信息（岗位名称 + ⼯作要求）
• 翻⻚

动态加载数据

# 起始URL
start_urls = [
    'https://careers.tencent.com/'
    'tencentcareer/api/post/Query?timestamp=1656484885985&countryId=&cityId=&'
    'bgIds=&productId=&categoryId=40001001,40001002,40001003,40001004,40001005,40001006&parentCategoryId=&'
    'attrId=&keyword=&pageIndex=2&pageSize=10&language=zh-cn&area=cn']

# 详情页URL
duty_url = 'https://careers.tencent.com/tencentcareer/api/post/ByPostId?' 
           'timestamp=1656484967344&postId={}&language=zh-cn'

1. 返回url对应的response响应对应的数据
2. 解析第⼀⻚上每个岗位PostId和岗位名称
3. 构造每个岗位详情⻚url
4. 解析详情⻚url，提取⼯作要求
5. 在管道用csv保存数据，并在settings开启管道

import scrapy
import json
from fake_useragent import UserAgent
from Tencent2.items import Tencent2Item

class TSpiderSpider(scrapy.Spider):
    name = 'T_spider'

    # allowed_domains = ['careers.tencent.com/search.html?pcid=40001']

    # 设置请求头
    headers = {
        'user-agent':UserAgent().random
    }

    # 起始URL
    start_urls = [
        'https://careers.tencent.com/'
        'tencentcareer/api/post/Query?timestamp=1656484885985&countryId=&cityId=&'
        'bgIds=&productId=&categoryId=40001001,40001002,40001003,40001004,40001005,40001006&parentCategoryId=&'
        'attrId=&keyword=&pageIndex=2&pageSize=10&language=zh-cn&area=cn']

    # 详情页URL
    duty_url = 'https://careers.tencent.com/tencentcareer/api/post/ByPostId?' 
               'timestamp=1656484967344&postId={}&language=zh-cn'

    def start_requests(self):
        for page in range(1, 10):
            url = self.start_urls[0].format(page)

            # 翻页发送请求
            yield scrapy.Request(url=url, callback=self.parse,headers=self.headers)

    def parse(self, response):
        # 将josn数据转换为python字典数据
        data = json.loads(response.text)['Data']['Posts']

        for i in data:
            # 实例化对象
            item = Tencent2Item()
            item['job_name'] = i['RecruitPostName']

            PostId =  i['PostId']

            # 构造请求
            yield scrapy.Request(url=self.duty_url.format(PostId),callback=self.detail_parse,meta={'item':item})


    def detail_parse(self,response):
        """详情页的爬取"""
        
        # 接受上个函数的参数
        item = response.meta.get('item')

        # 将josn数据转换为python字典数据
        data = json.loads(response.text)['Data']
        item['job_duty'] = data['Responsibility']

        yield item

import scrapy


class Tencent2Item(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    job_name = scrapy.Field()
    job_duty = scrapy.Field()

import csv


class Tencent2Pipeline:
    def __init__(self):
        self.f = open('tencent.csv', encoding='utf-8-sig', newline='', mode='w')

    def open_spider(self, spider):
        self.w = csv.DictWriter(self.f, fieldnames=['job_name', 'job_duty'])
        self.w.writeheader()

    def close_spider(self, spider):
        self.f.close()

    def process_item(self, item, spider):
        self.w.writerow(dict(item))

        return item

ITEM_PIPELINES = {
   'Tencent2.pipelines.Tencent2Pipeline': 300,
}