Scrapy使用FilesPipeline下载并读取Excel/Doc/Dox/Pdf内容

抓取网页中的附件，并将附件中的文字存储下来，不要求格式及段落顺序。

使用scrapy默认的pipeline，统一下载后根据不同格式进行文件处理。

class AddFilesPipeline(FilesPipeline):

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:101.0) Gecko/20100101 Firefox/101.0',
    }

    def get_media_requests(self, item, info):
        if item['file_urls']:
            index = 0
            for url in item['file_urls']:
                file_name = hashlib.md5(url).hexdigest() + "_{}".format(index)
                file_format = url.split('.')[-1]
                yield scrapy.Request(url=url,headers=self.headers,meta={"item":item,"file_format":file_format,"file_name":file_name})
                index+=1

    def file_path(self, request, response=None, info=None):
        file_format = request.meta['file_format']
        file_name = request.meta['file_name']
        return '{file_name}.{file_format}'.format(file_name=file_name,file_format=file_format)

    def item_completed(self, results, item, info):
        if item['file_urls']:
            # 文件
            file_paths = [x['path'] for ok, x in results if ok]
            ok, x = results[0]
            if not file_paths:
                raise DropItem("File下载失败")
            for result in results:
                ok, x = result
                path = x['path']
                if path.endswith('xls'):
                    content = parse_excel(FILES_STORE+path)
                elif path.endswith('doc') or path.endswith('docx') or path.endswith('wps'):
                    content = parse_word(FILES_STORE+path)
                elif path.endswith('pdf'):
                    content = parse_pdf(FILES_STORE+path)
                item['file_content'].append(content)
        return item

def parse_excel(path):
    """
    解析表格
    :param path:
    :return:
    """
    book = xlrd.open_workbook(path)
    sheet_names = book.sheet_names()  # 工作区
    content_list = []
    for sheet_name in sheet_names:
        sheet = book.sheet_by_name(sheet_name)
        rows = sheet.nrows  # 行
        # cols = sheet.ncols  # 列
        # print rows,cols
        content = ''
        for i in range(0, rows):
            row = sheet.row_values(i)
            content += ' | '.join(
                [str(t) if isinstance(t, unicode) or isinstance(t, str) else str(int(t)) for t in row if t]) + 'n'
        content_list.append(content)
    os.remove(path)
    return 'n'.join(content_list)

def parse_pdf(path):
    result = []
    fp = open(path, 'rb')
    parser = PDFParser(fp)
    doc = PDFDocument(parser, '')
    parser.set_document(doc)
    rsrcmgr = PDFResourceManager()
    # 创建一个PDF设备对象
    laparams = LAParams()
    device = PDFPageAggregator(rsrcmgr, laparams=laparams)
    interpreter = PDFPageInterpreter(rsrcmgr, device)
    for page in PDFPage.create_pages(doc):
        interpreter.process_page(page)
        layout = device.get_result()
        for x in layout:
            if hasattr(x, "get_text"):
                string = x.get_text().replace('n', '').replace(' ', '').strip()
                if string: result.append(string)
    os.remove(path)
    return 'n'.join(result)

注意由于linux环境用不了win32com，所以在linux上采用libreoffice将文件转换成docx。

def parse_word(path):
    """
    解析doc/wps/docx
    win和linux用到的东西不一样
    :param path:
    :return:
    """
    # 如果是.doc需要转换成docx
    temp_path = ''
    outdir = path
    if path.endswith('doc') or path.endswith('wps'):
        temp_path = path
        sys = platform.system()
        if sys == "Windows":
            from win32com import client as wc
            word = wc.Dispatch('Word.Application')
            doc = word.Documents.Open(path)
            if path.endswith('doc'):
                path = path.replace("doc", "docx")
            else:
                path = path.replace("wps", "docx")
            doc.SaveAs(path, 12, False, "", True, "", False, False, False, False)  # 转化后路径下的文件
            doc.Close()
            word.Quit()
        else:
            outdir = os.path.abspath(os.path.join(path, ".."))
            os.system("libreoffice --headless --convert-to docx {path} --outdir {outdir}".format(path=path,outdir=outdir))
            if path.endswith('doc'):
                path = path.replace("doc", "docx")
            else:
                path = path.replace("wps", "docx")

    document = Document(path)
    result = []
    for paragraph in document.paragraphs:
        p_text = paragraph.text.replace("n", "").strip()
        if p_text and p_text not in result:
            result.append(p_text)
    tables = document.tables  # 获取Word文件中的表格集
    for table in tables:  # 遍历每个表格
        table_content = ''
        for row in table.rows:  # 从表格第一行开始循环读取表格数据
            row_content = get_cell_content(row.cells)
            table_content += '|'.join(row_content) + 'n'
        result.append(table_content)
    os.remove(path)
    if temp_path:os.remove(temp_path)
    return 'n'.join(result)


def get_cell_content(cells):
    """
    获取每一行中每一列的内容
    """
    row_content = []
    for cell in cells:  # 遍历每一行的每一个单元格
        # cell数量为表格最大列数+1，故对于较少列的行存在重复值，需去重
        if cell.text and cell.text not in row_content:
            row_content.append(cell.text.replace("n", "").strip())
    return row_content

libreoffice将文件转换成txt。

def parse_word(path):
    """
    解析doc/wps/docx
    win和linux用到的东西不一样
    :param path:
    :return:
    """
    # 如果是.doc需要转换成docx
    temp_path = ''
    outdir = path
    if path.endswith('doc') or path.endswith('wps'):
        temp_path = path
        sys = platform.system()
        if sys == "Windows":
            from win32com import client as wc
            word = wc.Dispatch('Word.Application')
            doc = word.Documents.Open(path)
            if path.endswith('doc'):
                path = path.replace("doc", "docx")
            else:
                path = path.replace("wps", "docx")
            doc.SaveAs(path, 12, False, "", True, "", False, False, False, False)  # 转化后路径下的文件
            doc.Close()
            word.Quit()
        else:
            outdir = os.path.abspath(os.path.join(path, ".."))
            os.system("libreoffice --headless --convert-to txt {path} --outdir {outdir}".format(path=path,outdir=outdir))
            if path.endswith('doc'):
                path = path.replace("doc", "txt")
            else:
                path = path.replace("wps", "txt")
            content = open(path,'r').readlines()
            os.remove(path)
            if temp_path: os.remove(temp_path)
            return 'n'.join([i.strip() for i in content if i])

    document = Document(path)
    result = []
    for paragraph in document.paragraphs:
        p_text = paragraph.text.replace("n", "").strip()
        if p_text and p_text not in result:
            result.append(p_text)
    tables = document.tables  # 获取Word文件中的表格集
    for table in tables:  # 遍历每个表格
        table_content = ''
        for row in table.rows:  # 从表格第一行开始循环读取表格数据
            row_content = get_cell_content(row.cells)
            table_content += '|'.join(row_content) + 'n'
        result.append(table_content)
    os.remove(path)
    if temp_path:os.remove(temp_path)
    return 'n'.join(result)


def get_cell_content(cells):
    """
    获取每一行中每一列的内容
    """
    row_content = []
    for cell in cells:  # 遍历每一行的每一个单元格
        # cell数量为表格最大列数+1，故对于较少列的行存在重复值，需去重
        if cell.text and cell.text not in row_content:
            row_content.append(cell.text.replace("n", "").strip())
    return row_content

# 下载libreoffice
yum install -y libreoffice
# 安装依赖
yum install cairo cups-libs libSM
yum install ibus
yum install libreoffice-headless
# 安装成功后查看版本
libreoffice --version

转换命令,可以转换成各种各种如doc、docx、txt，当然上面的代码也可以改成转换成txt格式的直接读取。

libreoffice --headless --convert-to {输出文件类型}  {原文件路径} --outdir   {输出路径}