这是解决这个问题的有用技巧。
*为硒 *创建一个Web服务 ，在本地运行它
from flask import Flask, request, make_responsefrom flask_restful import Resource, Apifrom selenium import webdriverfrom selenium.webdriver.chrome.options import Optionsapp = Flask(__name__)api = Api(app)class Selenium(Resource): _driver = None @staticmethod def getDriver(): if not Selenium._driver: chrome_options = Options() chrome_options.add_argument("--headless") Selenium._driver = webdriver.Chrome(chrome_options=chrome_options) return Selenium._driver @property def driver(self): return Selenium.getDriver() def get(self): url = str(request.args['url']) self.driver.get(url) return make_response(self.driver.page_source)api.add_resource(Selenium, '/')if __name__ == '__main__': app.run(debug=True)现在将使用硒Chrome / Firefox驱动程序返回编译后的网页。
现在我们的蜘蛛会是什么样子，
import scrapyimport urllibclass ProductSpider(scrapy.Spider): name = 'products' allowed_domains = ['ebay.com'] urls = [ 'http://www.ebay.com/sch/i.html?_odkw=books&_osacat=0&_trksid=p2045573.m570.l1313.TR0.TRC0.Xpython&_nkw=python&_sacat=0&_from=R40', ] def start_requests(self): for url in self.urls: url = 'http://127.0.0.1:5000/?url={}'.format(urllib.quote(url)) yield scrapy.Request(url) def parse(self, response): yield { 'field': response.xpath('//td[@]/a'), }
