loginform只是一个库,与Scrapy完全分离。
您必须编写代码将其插入所需的蜘蛛中,可能是在回调方法中。
这是执行此操作的结构示例:
import scrapyfrom loginform import fill_login_formclass MySpiderWithLogin(scrapy.Spider): name = 'my-spider' start_urls = [ 'http://somewebsite.com/some-login-protected-page', 'http://somewebsite.com/another-protected-page', ] login_url = 'http://somewebsite.com/login-page' login_user = 'your-username' login_password = 'secret-password-here' def start_requests(self): # let's start by sending a first request to login page yield scrapy.Request(self.login_url, self.parse_login) def parse_login(self, response): # got the login page, let's fill the login form... data, url, method = fill_login_form(response.url, response.body,self.login_user, self.login_password) # ... and send a request with our login data return scrapy.FormRequest(url, formdata=dict(data), method=method, callback=self.start_crawl) def start_crawl(self, response): # OK, we're in, let's start crawling the protected pages for url in self.start_urls: yield scrapy.Request(url) def parse(self, response): # do stuff with the logged in response



