这是使用scikit-learn的Tf-
idf算法的实现。在应用之前，你可以
word_tokenize()和干你的话。
import pandas as pdfrom sklearn.feature_extraction.text import TfidfVectorizerfrom nltk import word_tokenizefrom nltk.stem.porter import PorterStemmerdef tokenize(text): tokens = word_tokenize(text) stems = [] for item in tokens: stems.append(PorterStemmer().stem(item)) return stems# your corpustext = ["This is your first text book", "This is the third text for analysis", "This is another text"]# word tokenize and stemtext = [" ".join(tokenize(txt.lower())) for txt in text]vectorizer = TfidfVectorizer()matrix = vectorizer.fit_transform(text).todense()# transform the matrix to a pandas dfmatrix = pd.Dataframe(matrix, columns=vectorizer.get_feature_names())# sum over each document (axis=0)top_words = matrix.sum(axis=0).sort_values(ascending=False)
