Scrapy 分页爬取

import scrapy
from testSpider.items import TestspiderItem


class MeinvSpider(scrapy.Spider):
    name = 'meinv'
    allowed_domains = ['www.3gbizhi.com']
    start_urls = ['http://www.3gbizhi.com/meinv/']
    page = 2
    page_url = 'https://www.3gbizhi.com/meinv/index_%d.html'

    def parse(self, response, **kwargs):
        ul = response.xpath('/html/body/div[4]/ul/li | /html/body/div[5]/ul/li')
        for div in ul:
            name = div.xpath('./a/div[2]/text()').extract_first()
            item = TestspiderItem()
            item['file_name'] = name
            yield item

        if self.page <= 5:
            new_url = self.page_url % self.page
            self.page += 1
            # 在解析过程中产生新的url，需要对新的url再次发起请求时，yield 手动调用scrapy.Request方法对象，
            yield scrapy.Request(url=new_url, callback=self.parse)

总结：两个地方会用到item
第一用yield 返回将item返回给管道
第二用yield手动发起Requets请求或者是FormRequests请求，即封装新的url。