文本相似度匹配

字符串相似度

import difflib
difflib.SequenceMatcher(None, string1, string2).ratio()

安装gensim包（http://www.lfd.uci.edu/~gohlke/pythonlibs/#wordcloud cp36表示python3.6）

pip36 install gensim-3.8.3-cp36-cp36m-win_amd64.whl

import codecs
import os
import shutil

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity


def bow_cosine(s1, s2):
    vectorizer = CountVectorizer()
    vectorizer.fit([s1, s2])
    X = vectorizer.transform([s1, s2])  # 得到s1，s2用bag of words方式表示的向量
    # print(X.toarray())
    # 得到两段文本的差值
    print(cosine_similarity(X[0], X[1])[0][0])
    return cosine_similarity(X[0], X[1])[0][0]

# txts是个列表，每个元素存放着jieba目录下的文件名
txts = os.listdir('jieba')
length = len(txts)
for i in range(0, length-1):
        print(txts[i],txts[i+1])
        txt1 = codecs.open('jieba\'+str(txts[i]), 'r', encoding='utf-8').read()
        txt2 = codecs.open('jieba\'+str(txts[i+1]), 'r', encoding='utf-8').read()
        if(bow_cosine(txt1, txt2) < 0.4):
            # 0表示截取'.'前面的字符串
            result = codecs.open('keytxt\'+str(txts[i]), 'w', encoding='utf-8')
            result.write(txt1)
            # 提取对应的帧，这个帧就是关键帧
            shutil.copy('frame/' + str(txts[i].split('\')[-1].split('.')[0]) + '.jpg',
                        'keyframe/' + str(txts[i].split('\')[-1].split('.')[0]) + '.jpg')
result.close()