scrapy-redis搭建分布式抓取亚马逊best排行榜

不会吧不会吧，不会还有人不会scarpy吧？
抖个包袱，没有那个意思哈！！！

• 官方架构
  

• Slaver(从)从Master(主)拿到爬取任务(Request、url)进行数据抓取，Slaver抓取数据的同时，产生新任务的Request便提交给 Master(主) 处理

• Master(主)负责将未处理的Request去重和任务分配，将处理后的Request加入待爬队列，并且存储爬取的数据。

说白了，就是利用redis机制，实现 request 的去重，多个爬虫分享一个 request 队列，将 scrapy 从单台机器扩展为多台机器

pip install scarpy-redis

创建一个项目

scrapy startproject amazon

创建爬虫文件

scrapy genspider -t crawl amazon_spider

1.1 解析数据(xpath)

1.1.1 据我分析，所要爬取的页面是动态加载的，需要下拉才能获取完整数据，这个就需要借助selenium，但是效率太低，如果要数据量较大，不推荐

1.2.1 但是通过我细心观察，源码中有存放完整数据，转换成了json格式，这不是雪中送炭吗！那我们正式讲解本章内容
1.3.1 上面我们已经拿到完整的数据，我们需要构造完整的链接，来抓取数据

2.1 抓取数据(amazon_spider.py)

2.1.1 抓取asin链接

import re
from scrapy_redis import spiders
import json
import datetime
from ..items import TaocheItem

class TaocheSpiderSpider(spiders.RedisSpider):
    name = 'taoche_spider'
    redis_key = 'amazon:start_urls'  #指定redis_key

    def parse(self, response):
        asin = response.xpath('//div[@]/@data-client-recs-list').extract()  # 获取所有asin
        category = response.xpath('//span[@]/text()').extract()  #获取类目
        data = json.loads(asin[0])
        for ca in category:
            for da in data:
                #1、解析
                asin = da['id'] 
                rank = da['metadataMap']['render.zg.rank']  # 排名
                url = f'https://www.amazon.com/dp/{asin}?th=1' # 构造完整链接
                item = TaocheItem()
                item['asin'] = asin
                item['rank'] = rank
                item['url'] = url
                item['cat'] = ca
                yield scrapy.Request(url=url, callback=self.parse_detail, meta={'item': item})

2.2.1 抓取详情页面

    def parse_detail(self, response):
        meta = response.meta    # 接收上面传过来的meta字典
        item = meta['item']
        
        title = 
            re.findall('(.*?)', response.text)[
                0].strip(" ") 
                if re.findall('(.*?)',
                              response.text) else ''

        # 价格
        if response.xpath('.//span[@]/text()'):
            price = response.xpath('.//span[@]/text()').extract()[0].strip('$')
        elif response.xpath('//span[@]/text()'):
            price = response.xpath('//span[@]/text()').extract()[0].strip('$')
        else:
            price = ''

        # 星级
        stars = response.xpath('/*;q=0.8',
                        'Accept-Language': 'en',
                        'User-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36',
                        'Cookie':'session-id=132-7820908-5512636; ubid-main=131-0969171-3937435; aws-target-data={"support":"1"}; aws-target-visitor-id=1640066826889-327584.38_0; aws-ubid-main=250-7108114-4482887; aws-session-id=000-0000000-0000000; aws-analysis-id=000-0000000-0000000; aws-session-id-time=1640067363l; i18n-prefs=USD; lc-main=en_US; s_nr=1650441678941-Repeat; s_vnum=2078274252676&vn=2; s_dslv=1650441678942; session-id-time=2082787201l; appstore-devportal-locale=zh_CN; _mkto_trk=id:365-EFI-026&token:_mch-aws.amazon.com-1640066827077-37141; aws-mkto-trk=id:112-TZM-766&token:_mch-aws.amazon.com-1640066827077-37141; aws_lang=cn; AMCVS_7742037254C95E840A4C98A6@AdobeOrg=1; s_cc=true; AMCV_7742037254C95E840A4C98A6@AdobeOrg=1585540135|MCIDTS|19151|MCMID|25398098771975688903309274459050119441|MCAAMLH-1655193582|3|MCAAMB-1655193582|RKhpRz8krg2tLO6pguXWp5olkAcUniQYPHaMWWgdJ3xzPWQmdj0y|MCOPTOUT-1654595982s|NONE|MCAID|NONE|vVersion|4.4.0; aws-userInfo-signed=eyJ0eXAiOiJKV1MiLCJrZXlSZWdpb24iOiJ1cy1lYXN0LTEiLCJhbGciOiJFUzM4NCIsImtpZCI6ImFhNDFkZjRjLTMxMzgtNGVkOC04YmU5LWYyMzUzYzNkOTEzYiJ9.eyJzdWIiOiIiLCJzaWduaW5UeXBlIjoiUFVCTElDIiwiaXNzIjoiaHR0cDpcL1wvc2lnbmluLmF3cy5hbWF6b24uY29tXC9zaWduaW4iLCJrZXliYXNlIjoidWVhVW11TnlIVG02am42TXZOTzhPajZOQ3hzMzVJVnp2dk0zaGQwaGNZQT0iLCJhcm4iOiJhcm46YXdzOmlhbTo6MTczMjg3OTA4ODk2OnJvb3QiLCJ1c2VybmFtZSI6Imd1b2xpeWluZyJ9.J_Pxsq-oHIwwEk23j86zM6OjWEu7Y_xe1SVigbVY3GVM9Ri7U30jkkQbMK0XZj4w3dudA4nJutx_QoMfvPyaSrq-z-abZA0i7Enlc9lOk2hj8tWmXCIP8HM6KSXQnYTl; aws-userInfo={"arn":"arn:aws:iam::173287908896:root","alias":"","username":"guoliying","keybase":"ueaUmuNyHTm6jn6MvNO8Oj6NCxs35IVzvvM3hd0hcYAu003d","issuer":"http://signin.aws.amazon.com/signin","signinType":"PUBLIC"}; regStatus=registered; awsc-color-theme=light; awsc-uh-opt-in=optedOut; noflush_awsccs_sid=44efc26057037bfb3a76afb21553f47bd88a10ee4396ef03fb2a0725609e487e; aws-signer-token_us-east-1=eyJrZXlWZXJzaW9uIjoia0lYaHRyWi5POFhvLjFneThKS251dnVOMkdmLnNaekYiLCJ2YWx1ZSI6Ik5pcWFBcnR2RFpaK1dtSCtnb3gvZWZNNXFNWVNEcFBQVXU1NFZoaXdNMUU9IiwidmVyc2lvbiI6MX0=; aws-signer-token_us-west-2=eyJrZXlWZXJzaW9uIjoiTm5jSVdFQ0gubzYuMGlFS3VEZk9HbmdHVTZWX2JHU3IiLCJ2YWx1ZSI6IlMyK2NqOHU1SWRxdThOUityUGhQWklTSTBtVTdTSTRzQUN0Unp0VTdBbTg9IiwidmVyc2lvbiI6MX0=; s_sq=[[B]]; session-id-eu=259-5325582-3849510; ubid-acbuk=257-5013527-1544704; session-id-time-eu=2285315333l; x-amz-captcha-1=1654619359947279; x-amz-captcha-2=kdK+pRacBvV0/C2ON8VoCQ==; session-token="IjG7lRcnVW+CVUclLZ0yZFV9JFIco2e4bM7QLQoXVkn6Hhvm+Katu2747zcP93hofXpkcHHNQZIl4BPm1yoRhTildooVOy/uImskrmCJ0QbH0ORn1fDaJvydwrSapUmmHkoH23DI5rt2Uo8eFFfEQ3zrUsYDrw4di4qqF+b51SJwp65LjQcWFkJLWNQQkczZYu6QQYYRmTtwLSoXxJWWUA=="; csm-hit=tb:EJ3BAFMM9JN993D6ACQF+s-4ARBRR7G7XBJSYK1VHPS|1654616613057&t:1654616613057&adb:adblk_no'

        },

    #配置scrapy-redis调度器
    'SCHEDULER' : "scrapy_redis.scheduler.Scheduler",
    #配置url去重
    'DUPEFILTER_CLASS' : "scrapy_redis.dupefilter.RFPDupeFilter",
    #配置优先级队列
    'SCHEDULER_QUEUE_CLASS' : 'scrapy_redis.queue.PriorityQueue',
    #可以暂停爬虫
    'SCHEDULER_PERSIST': True,
	# reids链接
    'REDIS_PORT' : 6379,
    'REDIS_HOST':'localhost'#Master Slaver需要更改为master的ip,
	
	ITEM_PIPELINES = {
   'taoche.pipelines.TaochePipeline': 300,
}
	HOST = 'localhost'
	PORT = 3306
	USER = '****'
	PASSWD = '***'
	DB = '****'

启动爬虫

scrapy crawl amazon_spider

成功获取到我们想要的数据

想要获取完整项目私信泰隆，看到都会回复。
本文章若对你有帮助，烦请点赞，收藏，关注支持一下
各位的支持和认可就是我最大的动力