你
rule未使用，因为你没有使用
CrawlSpider。
因此，你必须
requests手动创建下一页，如下所示：
# -*- coding: utf-8 -*-import scrapyfrom scrapy.contrib.spiders import Rulefrom scrapy.linkextractors import linkExtractorfrom lxml import htmlclass Scrapy1Spider(scrapy.Spider): name = "craiglist" allowed_domains = ["sfbay.craigslist.org"] start_urls = ( 'http://sfbay.craigslist.org/search/npo', ) Rules = (Rule(linkExtractor(allow=(), restrict_xpaths=('//a[@]',)), callback="parse", follow= True),) def parse(self, response): site = html.fromstring(response.body_as_unipre()) titles = site.xpath('//div[@]/p[@]') print len(titles), 'AAAA' # follow next page links next_page = response.xpath('.//a[@]/@href').extract() if next_page: next_href = next_page[0] next_page_url = 'http://sfbay.craigslist.org' + next_href request = scrapy.Request(url=next_page_url) yield request或者
CrawlSpider像这样使用：
# -*- coding: utf-8 -*-import scrapyfrom scrapy.spiders import CrawlSpider, Rulefrom scrapy.linkextractors import linkExtractorfrom lxml import htmlclass Scrapy1Spider(CrawlSpider): name = "craiglist" allowed_domains = ["sfbay.craigslist.org"] start_urls = ( 'http://sfbay.craigslist.org/search/npo', ) rules = (Rule(linkExtractor(allow=(), restrict_xpaths=('//a[@]',)), callback="parse_page", follow= True),) def parse_page(self, response): site = html.fromstring(response.body_as_unipre()) titles = site.xpath('//div[@]/p[@]') print len(titles), 'AAAA'
