Python 网站爬虫开发实践

python 2.7

1. html_downloader 用于html下载
2. html_parser 用于解析html返回对应数据，这里用的pyQuery这个库，类jQuery语法，方便实用
3. outputer 结果输出
4. url_manager url管理
5. main程序入口

# -*- conding:utf-8 -*-
from com import url_manager,html_downloader,html_parser,outputer
class HtmlScheduler(object):
    def __init__(self):
 self.urls = url_manager.UrlManager()
 self.downloader = html_downloader.HtmlDownloader()
 self.parser = html_parser.HtmlParser()
 self.outputer = outputer.Outputer()
 pass

    def parse(self):
 new_url = self.urls.get_new_url()
 if new_url!=None:
     html = self.downloader.download(new_url)
     new_urls,new_content = self.parser.parse(html)
     self.urls.add_new_urls(new_urls);
     if new_content!=None:
  self.outputer.collect_data(new_content)
 pass

    def start_parse(self,count):
 self.urls.add_new_url(root_url)
 try:
     while self.urls.has_new_url() and count>0:
  self.parse()
  count = count - 1;
 except:
     print 'error'
 finally:
     self.outputer.output()
 print 'success'
 print 'new urls:%s' % self.urls.new_urls
 print 'old urls:%s' % self.urls.old_urls
 pass

if __name__=='__main__':
    root_url = 'https://www.imooc.com/course/list'
    html_scheduler = HtmlScheduler()
    html_scheduler.start_parse(20) 

from pyquery import PyQuery as pq
class HtmlParser(object):

    def __init__(self):
 self.links_a = []
 self.links_image = []
 pass

    def parse(self,content):
 doc = pq(content)
 for a in doc('.text-page-tag').items():
     href = a.attr('href')
     if href!=None:
  self.links_a.append(href)
 for image in doc('.course-banner').items():
     src = image.attr('data-original')
     alt = image.closest('.course-card').find('.course-card-content h3').text()
     if src!=None:
  self.links_image.append((alt,src))
 return self.links_a,self.links_image
 pass
import urllib2
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36'
class HtmlDownloader(object):

    def __init__(self):
 pass

    def download(self,url):
 request = urllib2.Request(url)
 request.add_header('user-agent', UA)
 response = urllib2.urlopen(request)
 content = response.read();
 return content
 pass

import urllib
class Outputer(object):

    def __init__(self):
 self.data = []
 pass

    def collect_data(self,data):
 if data!=None:
     for item in data:
  self.data.append(item)
 pass

    def output(self):
 x = 0
 for item in self.data:
     if item[1]!=None:
  print 'output image:%s' % item[0]
  try:
      urllib.urlretrieve('http:'+item[1], 'D:\parser_pics%s.jpg' % item[0])
  except:
      print 'output image %s error' % item[1]  
  x = x+1
 pass
class UrlManager(object):

    def __init__(self):
 self.new_urls = set()
 self.old_urls = set()
 pass

    def has_new_url(self):
 return len(self.new_urls)!=0

    def add_new_url(self,url):
 if url!=None and '/course/list' in url and not url in self.old_urls:
     if 'https://' in url:
  self.new_urls.add(url)
     else:
  self.new_urls.add('https://www.imooc.com'+url)
 pass

    def add_new_urls(self,urls):
 for url in urls:
     if url!=None:
  self.add_new_url(url)
 pass

    def get_new_url(self):
 url = self.new_urls.pop()
 self.old_urls.add(url)
 return url;