前面简单介绍了scrapy 的几种基本的用法
现在打算正式用scrapy写几个项目来加深scrapy的概念和一些高级有法
import scrapy
from scrapy import Request
class HotSpider(scrapy.Spider):
#爬虫名
name hot
qidian_headers {
user-agent : Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36
#目标网址
#start_urls [ https://www.qidian.com/rank/hotsales?style 1 page 1 ]
def start_requests(self):
url https://www.qidian.com/rank/hotsales?style 1 page 1
yield Request(url,headers self.qidian_headers,callback self.hot_parse)#callback自定义解析函数
def hot_parse(self, response):
# xpath定位
list_selector response.xpath( //div[ class book-mid-info ] )
# 获取所有小说
for one_selector in list_selector:
# 获取小说信息
name one_selector.xpath( h4/a/text() ).extract()[0]
#print(name)
# 获取作者
author one_selector.xpath( p[1]/a[1]/text() ).extract()[0]
#print(author)
# 获取类型
type one_selector.xpath( p[1]/a[2]/text() ).extract()[0]
#print(type)
# 获取形式
form one_selector.xpath( p[1]/span/text() ).extract()[0]
#print(form)
#定义字典
hot_dict {
name :name,
author :author,
type :type,
form :form,
yield hot_dict
