爬虫代理池Python3WebSpider源代码测试过程解析

这篇文章主要介绍了爬虫代理池Python3WebSpider源代码测试过程解析,文中通过示例代码介绍的非常详细，对大家的学习或者工作具有一定的参考学习价值,需要的朋友可以参考下

元类属性的使用

代码

主要关于元类的使用

通过获取由元类生成的爬虫抓取类的部分属性.这里为抓取函数,以相同的字符开头的抓取函数,生成属性列表,这样可以持续调用.目的是可以仅仅添加不同的抓取函数抓取不同的网站,而类的其他部分不用做调整.

部分代码:

class Proxymetaclass(type):
  def __new__(cls, name, bases, attrs):
    count = 0
    attrs['__CrawlFunc__'] = []
    for k, v in attrs.items():
      if 'crawl_' in k:
 attrs['__CrawlFunc__'].append(k)
 count += 1
    attrs['__CrawlFuncCount__'] = count
    return type.__new__(cls, name, bases, attrs)


class Crawler(object, metaclass=Proxymetaclass):
  def get_proxies(self, callback):
    proxies = []
    for proxy in eval("self.{}()".format(callback)):
      print('成功获取到代理', proxy)
      proxies.append(proxy)
    return proxies
    
  def crawl_daili66(self, page_count=4):
    """
    获取代理66
    :param page_count: 页码
    :return: 代理
    """
    start_url = 'http://www.66ip.cn/{}.html'
    urls = [start_url.format(page) for page in range(1, page_count + 1)]
    for url in urls:
      print('Crawling', url)
      html = get_page(url)
      if html:
 doc = pq(html)
 trs = doc('.containerbox table tr:gt(0)').items()
 for tr in trs:
   ip = tr.find('td:nth-child(1)').text()
   port = tr.find('td:nth-child(2)').text()
   yield ':'.join([ip, port])

测试方法

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time  : 12/19/19 4:10 PM
# @Author : yon
# @Email  : @qq.com
# @File  : test


import json
import re
from pyquery import PyQuery as pq


class Proxymetaclass(type):
  def __new__(cls, name, bases, attrs):
    count = 0
    attrs['__CrawlFunc__'] = []
    for k, v in attrs.items():
      print("打印k")
      print(k)
      print("打印v")
      print(v)
      if 'crawl_' in k:
 attrs['__CrawlFunc__'].append(k)
 count += 1
    attrs['__CrawlFuncCount__'] = count
    return type.__new__(cls, name, bases, attrs)


class Crawler(object, metaclass=Proxymetaclass):
  def get_proxies(self, callback):
    proxies = []
    for proxy in eval("self.{}()".format(callback)):
      print('成功获取到代理', proxy)
      proxies.append(proxy)
    return proxies

  def crawl_daili66(self, page_count=4):
    """
    获取代理66
    :param page_count: 页码
    :return: 代理
    """
    start_url = 'http://www.66ip.cn/{}.html'
    urls = [start_url.format(page) for page in range(1, page_count + 1)]
    for url in urls:
      print('Crawling', url)
      html = get_page(url)
      if html:
 doc = pq(html)
 trs = doc('.containerbox table tr:gt(0)').items()
 for tr in trs:
   ip = tr.find('td:nth-child(1)').text()
   port = tr.find('td:nth-child(2)').text()
   yield ':'.join([ip, port])

  def crawl_ip3366(self):
    for page in range(1, 4):
      start_url = 'http://www.ip3366.net/free/?stype=1&page={}'.format(page)
      html = get_page(start_url)
      ip_address = re.compile('s*(.*?)s*(.*?)')
      # s * 匹配空格，起到换行作用
      re_ip_address = ip_address.findall(html)
      for address, port in re_ip_address:
 result = address + ':' + port
 yield result.replace(' ', '')

  def crawl_kuaidaili(self):
    for i in range(1, 4):
      start_url = 'http://www.kuaidaili.com/free/inha/{}/'.format(i)
      html = get_page(start_url)
      if html:
 ip_address = re.compile('(.*?)')
 re_ip_address = ip_address.findall(html)
 port = re.compile('(.*?)')
 re_port = port.findall(html)
 for address, port in zip(re_ip_address, re_port):
   address_port = address + ':' + port
   yield address_port.replace(' ', '')

  def crawl_xicidaili(self):
    for i in range(1, 3):
      start_url = 'http://www.xicidaili.com/nn/{}'.format(i)
      headers = {
 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,**;q=0.8',
      'Accept-Encoding': 'gzip, deflate',
      'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
      'Cache-Control': 'max-age=0',
      'Connection': 'keep-alive',
      'cookie': 'JSESSIonID=47AA0C887112A2D83EE040405F837A86',
      'Host': 'www.data5u.com',
      'Referer': 'http://www.data5u.com/free/index.shtml',
      'Upgrade-Insecure-Requests': '1',
      'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36',
    }
    html = get_page(start_url, options=headers)
    if html:
      ip_address = re.compile('
(d+.d+.d+.d+)
.*?

/home/baixiaoxu/PycharmProjects/pytthon-tt/venv/bin/python /home/baixiaoxu/PycharmProjects/pytthon-tt/proxypool/test.py
打印k
__module__
打印v
__main__
打印k
__qualname__
打印v
Crawler
打印k
get_proxies
打印v

打印k
crawl_daili66
打印v

打印k
crawl_ip3366
打印v

打印k
crawl_kuaidaili
打印v

打印k
crawl_xicidaili
打印v

打印k
crawl_iphai
打印v

打印k
crawl_data5u
打印v

打印k
__CrawlFunc__
打印v
['crawl_daili66', 'crawl_ip3366', 'crawl_kuaidaili', 'crawl_xicidaili', 'crawl_iphai', 'crawl_data5u']
获取器开始执行
0
crawl_daili66
1
crawl_ip3366
2
crawl_kuaidaili
3
crawl_xicidaili
4
crawl_iphai
5
crawl_data5u

进程完成，退出码 0

以上就是本文的全部内容，希望对大家的学习有所帮助，也希望大家多多支持考高分网。