# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
from .items import ZjzxItem
import pandas as pd
import sqlite3
dbcon=sqlite3.connect("shuju.db")
class ZjzxPipeline:
#这个方法在爬虫打开的时候运行
def open_spider(self, spider):
#定义一个列表
self.bonusList=[]
#这个方法在爬虫关闭的时候运行
def close_spider(self, spider):
#1把列表的数据写入数据库
#1.1把列表的数据放入Dataframe
df=pd.Dataframe(self.bonusList)
#1.2把Dataframe的数据放入数据库
df.to_sql('bonus',dbcon,if_exists='append')
def process_item(self, item, spider):
if not isinstance(item, ZjzxItem):
return item
self.bonusList.append(item)
if len(self.bonusList)>1024:
df=pd.Dataframe(self.bonusList)
#1.2把Dataframe的数据放入数据库
df.to_sql('bonus',dbcon,if_exists='append')
self.bonusList=[]
