Python网络爬虫--Scrapy实践

学习完理论之后，通过爬取360图库的图片，进行实际操作

scrapy startproject xxx

• ‘xxx’：项目名称

ROBOTSTXT_OBEY = False  # 将ROBOTSTXT_OBEY 修改为false

KEYWORDS = ['伊莉雅']  # 写入搜索关键字

SELENIUM_TIMEOUT = 5  # 定义响应超时时间

USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36'  # 定义响应头

class PictureItem(scrapy.Item):
    """
    保存爬取数据的容器，根据需求定义字段
    """
    url = scrapy.Field()

class PictureSpider(Spider):
    name = 'picture'  # 项目名。用来区分不同的Spider
    allowed_domains = ['image.so.com']  # 域名。只允许爬取当前域名下的链接
    base_rul = 'https://image.so.com/i?q={}&src=srp'  # 请求地址

    def start_requests(self):
        for keyword in self.settings.get('KEYWORDS'):  # 遍历setting.py文件中的KEYWORDS列表
            url = self.base_rul.format(quote(keyword))  # url地址拼接
            """
            将Request请求交给Scheduler进行调度
            :param url: 需要爬取网站的url地址
        	:param callback: 回调函数
        	:param meta: 其他需要增加参数
        	:param dont_filter: 表示该url不参加去重，即不会被allowed_domains过滤掉
            """
            yield Request(url=url, callback=self.parse, meta={}, dont_filter=True)

    def parse(self, response):
        """
        该方法负责解析返回的响应、提取数据或者进一步生成要处理的请求
        :param response: 由Downloader Middleware返回的结果
        :return:
        """
        html = pq(response.body)  # 通过pyquery进行解析
        sections = html('.vertical ul')  # 获取节点，遍历时需要通过items()进行遍历
        for section in sections.items():
            cells = section('.cell')
            for cell in cells.items():
                item = PictureItem()
                item['url'] = cell('div .entity .img img').attr('src')  # 提取内容
                if item['url'] is None:
                    continue
                yield item

通过对接Selenium，实现网页的动态爬取。selenium入门

class SeleniumMiddleware:
    def __init__(self, timeout=None, respond_header=None):
        self.logger = getLogger(__name__)  # 设置日志
        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_argument('--headless')  # 无窗口模式
        chrome_options.add_argument(respond_header)  # 设置响应头
        self.browser = webdriver.Chrome(chrome_options=chrome_options)
        self.wait = WebDriverWait(self.browser, timeout)  # 设置延时等待

    @classmethod
    def from_crawler(cls, crawler):
        return cls(timeout=crawler.settings.get('SELENIUM_TIMEOUT'),
                   respond_header=crawler.settings.get('USER_AGENT'))

    def process_request(self, request, spider):
        self.logger.debug('Scrapy Starting')
        try:
            self.browser.get(request.url)  # 获取网页内容
            self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.vertical')))  # 判断节点是否已加载
            return HtmlResponse(url=request.url, body=self.browser.page_source, request=request,  encoding='utf-8', status=200)
        except TimeoutException:  # 超时返回异常
            return HtmlResponse(url=request.url, status=500, request=request)

    def __del__(self):
        self.browser.close()

class DownloaderPipeline:
    """
    进行图片的下载
    """
    def process_item(self, item, spider):
        response = requests.get(item["url"])
        name = '{}.jpg'.format(uuid.uuid4())
        with open('./' + name, "wb") as f:
            f.write(response.content)
        return item

# 将自己写的middlewares和pipelines添加到配置文件中
DOWNLOADER_MIDDLEWARES = {
    'illyasivel.middlewares.SeleniumMiddleware': 544,
}
ITEM_PIPELINES = {
    'illyasivel.pipelines.DownloaderPipeline': 300,
}

scrapy crwal illyasivel

• 一定要与spider中的name名称一致

注意点

在第4点创建middlewares，判断节点加载是否超时，最终都返回了HtmlResponse，这将导致不会继续执行等级更低的middlewares。

可根据需要修改异常返回的内容