摘 要
项目以自然语言处理为背景，根据新闻文本字符对新闻的类别进行分类，这是一个经典文本分类问题。本项目统计了每篇新闻文本的字符数量，以及所有新闻文本的词频，将其与新闻类别标签分别保存为3个文件，以此完成新闻各类别样本数条形图、新闻词频分布条形图、新闻字数统计和分布柱状图、新闻字数分布箱型图的绘制。本次使用TF-IDF方法，通过支持向量机、随机森林、岭回归分类器和神经网络这四种模型完成新闻文本分类任务，计算验证结果的F1分数来评价基于本项目的上述四种模型的好坏，并将各模型的F1分数绘制成横条形图。
本项目的两个PY代码，其各功能均在不同的函数中实现，并在main函数中调用上述函数，完成了模块化程序设计。
关键词 文本分类 TF-IDF方法 机器学习模型 神经网络
目 录
第一章 项目背景及简介
1.1项目来源
1.2项目任务
1.3项目数据集说明
第二章 项目源代码
2.1Pandas数据分析
2.2Sklearn机器学习
第三章 项目实施结果
3.1新闻文本数据分析
3.2新闻文本分类-机器学习
第一章 项目背景及简介
1.1项目来源
项目来源于天池大赛之学习赛中“零基础入门NLP之新闻文本分类挑战赛”。
项目数据来源为互联网上的新闻，通过收集并匿名处理得到。
通过此项目可以引导大家走入自然语言处理的世界，带大家接触NLP的预处理、模型构建和模型训练等知识点。
项目网址为：零基础入门NLP - 新闻文本分类-天池大赛-阿里云天池 (aliyun.com)
1.2项目任务
本项目以自然语言处理为背景，要求根据新闻文本字符对新闻的类别进行分类，这是一个经典文本分类问题。
选手可以自行进行数据分析，可以充分发挥自己的特长来完成各种特征工程，不限制使用任何外部数据和模型。
评价标准为类别f1_score的均值，选手提交结果与实际测试集的类别进行对比，结果越大越好。
1.3项目数据集说明
项目数据为新闻文本，并按照字符级别进行匿名处理。整合划分出14个候选分类类别：财经、彩票、房产、股票、家居、教育、科技、社会、时尚、时政、体育、星座、游戏、娱乐的文本数据。
在数据集中标签的对应的关系如下：
{'科技': 0, '股票': 1, '体育': 2, '娱乐': 3, '时政': 4, '社会': 5, '教育': 6, '财经': 7, '家居': 8, '游戏': 9, '房产': 10, '时尚': 11, '彩票': 12, '星座': 13}
数据列使用t进行分割。
项目数据由以下几个部分构成：训练集20w条样本，测试集A包括5w条样本，测试集B包括5w条样本。为了预防选手人工标注测试集的情况，我们将比赛数据的文本按照字符级别进行了匿名处理。
第二章 项目源代码
2.1 Pandas数据分析
# Pandas数据分析
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib
import seaborn as sns
import json
from collections import Counter
def store_data_json(data,file_path,include_chinese=False):
# 数据存储
if include_chinese == False:
with open(file_path,'w') as f:
f.write(json.dumps(data,indent=2))
elif include_chinese == True:
with open(file_path,'w',encoding='utf-8') as f:
f.write(json.dumps(data,indent=2,ensure_ascii=False))
else:
print("参数include_chinese填写错误，保存失败！")
def read_data_json(file_path):
with open(file_path,"r",encoding="utf-8") as f:
str = f.read()
data = json.loads(str)
return data
def text_len_ana(train_df):
# 新闻文本长度分析
text_len = train_df['text'].apply(lambda x:len(x.split()))
print("新闻文本长度统计汇总：")
print(text_len.describe())
return text_len
def word_freq_stat(train_df,nrows=1000):
# 词频统计
all_lines = " ".join(list(train_df['text'][:nrows]))
word_count = Counter(all_lines.split())
word_count = sorted(word_count.items(),key=lambda d:d[1],reverse=True)
print("不同字符数：",len(word_count))
print("词频前3位：")
for i in range(3):
print(word_count[i][0],"：",word_count[i][1],"次")
return word_count
def plot_1(text_label,file_path='E:/1新闻各类别样本数.jpg'):
plt.style.use('seaborn-talk')
matplotlib.rc('font',family='Microsoft YaHei')
# 新闻类别分布
x = text_label.value_counts().sort_index()
y = text_label[:len(text_label)//4].value_counts().sort_index()
t = {'科技':0,'股票':1,'体育':2,'娱乐':3,'时政':4,
'社会':5,'教育':6,'财经':7,'家居':8,'游戏':9,
'房产':10,'时尚':11,'彩票':12,'星座':13}
d = {index:token for token,index in t.items()} # t键值反转
k1 = pd.Series(x.values,index=[d[i] for i in x.index])
k2 = pd.Series(y.values,index=[d[i] for i in y.index])
plt.bar(k1.index,k1,label='20w样本')
for x,y in zip(k1.index,k1):
plt.text(x,y,y,ha='center',va='bottom')
plt.bar(k2.index, k2,label='5w样本')
for x,y in zip(k2.index,k2):
plt.text(x,y,y,ha='center',va='bottom')
plt.title("新闻各类别样本数")
plt.xlabel("新闻类别")
plt.ylabel("新闻样本数")
plt.legend()
plt.savefig(file_path,bbox_inches='tight')
print("Plot1：已保存到" + file_path)
plt.show()
def plot_2(word_count,file_path='E:/2新闻词频分布.jpg'):
x,y = [],[]
for i in word_count[:30]:
x.append(i[0])
y.append(i[1])
plt.style.use('seaborn-talk')
matplotlib.rc('font',family='Microsoft YaHei')
plt.bar(x,y)
plt.xticks(rotation=60)
plt.xlabel("字符")
plt.ylabel("出现次数")
plt.title("单词的频率分布（前30）")
plt.savefig(file_path,bbox_inches='tight')
print("Plot2：已保存到" + file_path)
plt.show()
def plot_3(text_len,file_path='E:/3新闻字数统计和分布.jpg'):
plt.style.use('seaborn-talk')
matplotlib.rc('font',family='Microsoft YaHei')
# 新闻字数分布
plt.subplot(1,2,1)
plt.hist(text_len,bins=30,range=(0,5000),edgecolor='k')
plt.xlabel('新闻字数')
plt.ylabel('新闻篇数')
plt.title('新闻字数分布')
# 新闻字数统计
plt.subplot(1,2,2)
plt.plot(text_len)
ax = plt.gca()
ax.yaxis.set_ticks_position('right')
plt.xlabel('新闻篇次')
plt.ylabel('新闻字数')
plt.title('每个样本的字数')
plt.savefig(file_path,bbox_inches='tight')
print("Plot3：已保存到" + file_path)
plt.show()
def plot_4(text_len,file_path='E:/4箱型图.jpg'):
plt.style.use('seaborn-talk')
matplotlib.rc('font',family='Microsoft YaHei')
# 箱型图
ax = sns.boxplot(x=text_len)
ax.set_xlim(-300,3000)
ax.set(xlabel="新闻字数")
plt.title("箱形图")
plt.savefig(file_path,bbox_inches='tight')
print("Plot4：已保存到" + file_path)
plt.show()
def main():
train_set_path = "../input/train_set.csv"
train_df = pd.read_csv(train_set_path,sep='t',nrows=200)
# 分析和存储数据
print("1新闻标签存储！")
text_label = train_df['label']
text_label.to_csv("E:/text_label_.csv")
text_label = pd.read_csv("E:/plot/text_label.csv")['label']
print("2新闻文本长度分析和存储！")
text_len = text_len_ana(train_df)
text_len.to_csv("E:/text_len_.csv")
text_len = pd.read_csv("E:/plot/text_len.csv")['text']
# 注：词频统计所需机器配置较高，笔记本电脑难以完成，使用百度飞桨计算平台完成
# 注：词频统计结果展示：使用已保存的中间变量word_count
print("3词频统计和存储！")
word_count = word_freq_stat(train_df,nrows=1000)
# 注：若在自己笔记本电脑运行，则文件名为：word_count_
# 注：若在百度飞浆计算平台运行，则可为：word_count
# 注：本程序共存储3个文本文件，格式同以上2行
store_data_json(word_count,'E:/word_count_.json')
word_count = read_data_json("E:/plot/word_count.json")
# 画图
plot_1(text_label)
plot_2(word_count)
plot_3(text_len)
plot_4(text_len)
main()
2.2 Sklearn机器学习
# 机器学习sklearn新闻文本分类
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import RidgeClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
import matplotlib.pyplot as plt
import matplotlib
from Step1 import store_data_json
from Step1 import read_data_json
def load_data_prep(filename,sep='t',nrows=1000):
train_df = pd.read_csv(filename,sep=sep,nrows=nrows)
tf_idf = TfidfVectorizer(ngram_range=(1,2), max_features=20000, min_df=2)
x_train = tf_idf.fit_transform(train_df['text'])
print(x_train.toarray().shape)
print(x_train.toarray())
y_train = train_df['label'].values
x_train,x_test,y_train,y_test =
train_test_split(x_train,y_train,test_size=0.2)
x_train,x_val,y_train,y_val =
train_test_split(x_train,y_train,test_size=0.25)
return x_train,x_val,x_test,y_train,y_val,y_test
def create_clf():
clf1 = svm.SVC(kernel='rbf')
clf2 = RandomForestClassifier(n_estimators=100,verbose=0)
clf3 = RidgeClassifier(alpha=1.0,fit_intercept=True)
clf4 = MLPClassifier(hidden_layer_sizes=(100,100),
activation='relu',solver='adam',
learning_rate_init=0.001,max_iter=200)
clf = {"支持向量机":clf1, "随机森林":clf2, "岭回归分类器":clf3,
"神经网络":clf4}
return clf
def train_clf(clf, x_train, y_train):
count = 0
for i in clf:
count += 1
print("（"+str(count)+"）开始训练："+i+"！")
clf[i].fit(x_train,y_train)
print("t训练完成！")
def get_pred(clf,x_test):
res = []
for i in clf:
r = clf[i].predict(x_test)
res.append(r)
return res
def cal_f1_score(clf, y_test, res):
clf_name = list(clf.keys())
scores = {}
for i in range(len(res)):
scores[clf_name[i]] = f1_score(y_test,res[i],average='macro')
return scores
def scores_visualize(scores,file_path='E:/5不同模型的f1分数.jpg'):
plt.style.use("seaborn-talk")
matplotlib.rc('font',family='Microsoft YaHei')
clf_name = list(scores.keys())
clf_score = list(scores.values())
plt.barh(range(len(clf_name)),clf_score,height=0.5,tick_label=clf_name)
plt.xlim(0.3,1.0)
plt.ylabel("模型名称")
plt.xlabel("f1_score")
plt.title("不同模型的f1分数")
for x,y in enumerate(clf_score):
plt.text(y+0.01,x,"{:.2f}".format(y))
plt.savefig(file_path,bbox_inches='tight')
print("tf1分数可视化：已保存到" + file_path)
print("8结束")
plt.show()
def main():
train_set_path = "../input/train_set.csv"
print("1加载数据&预处理")
x_train,x_val,x_test,y_train,y_val,y_test =
load_data_prep(train_set_path,nrows=20)
print("2创建分类器")
clf = create_clf()
print("3训练分类器")
train_clf(clf,x_train,y_train)
print("4在验证集上得到预测结果")
res = get_pred(clf,x_val)
print("5计算各模型的f1_score")
scores = cal_f1_score(clf,y_val,res)
fil_clf_name = max(scores,key=scores.get)
fil_clf = {fil_clf_name:clf[fil_clf_name]}
print("tfil_clf_name：",fil_clf_name)
print(" 4_在测试集上得到预测结果")
res = get_pred(fil_clf,x_test)
print(" 5_计算模型的f1_score")
score_ = cal_f1_score(fil_clf,y_test,res)
scores[fil_clf_name+'n（测试集）'] = score_[fil_clf_name]
print("6存储各模型的f1_score")
store_data_json(scores,file_path="E:/Scores_.json",
include_chinese=True)
scores = read_data_json(file_path="E:/plot/Scores.json")
print("7可视化")
scores_visualize(scores)
print("8结束")
main()
第三章 项目实施结果
3.1 新闻文本数据分析
# Pandas数据分析
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib
import seaborn as sns
import json
from collections import Counter
def store_data_json(data,file_path,include_chinese=False):
# 数据存储
if include_chinese == False:
with open(file_path,'w') as f:
f.write(json.dumps(data,indent=2))
elif include_chinese == True:
with open(file_path,'w',encoding='utf-8') as f:
f.write(json.dumps(data,indent=2,ensure_ascii=False))
else:
print("参数include_chinese填写错误，保存失败！")
def read_data_json(file_path):
with open(file_path,"r",encoding="utf-8") as f:
str = f.read()
data = json.loads(str)
return data
def text_len_ana(train_df):
# 新闻文本长度分析
text_len = train_df['text'].apply(lambda x:len(x.split()))
print("新闻文本长度统计汇总：")
print(text_len.describe())
return text_len
def word_freq_stat(train_df,nrows=1000):
# 词频统计
all_lines = " ".join(list(train_df['text'][:nrows]))
word_count = Counter(all_lines.split())
word_count = sorted(word_count.items(),key=lambda d:d[1],reverse=True)
print("不同字符数：",len(word_count))
print("词频前3位：")
for i in range(3):
print(word_count[i][0],"：",word_count[i][1],"次")
return word_count
def plot_1(text_label,file_path='E:/1新闻各类别样本数.jpg'):
plt.style.use('seaborn-talk')
matplotlib.rc('font',family='Microsoft YaHei')
# 新闻类别分布
x = text_label.value_counts().sort_index()
y = text_label[:len(text_label)//4].value_counts().sort_index()
t = {'科技':0,'股票':1,'体育':2,'娱乐':3,'时政':4,
'社会':5,'教育':6,'财经':7,'家居':8,'游戏':9,
'房产':10,'时尚':11,'彩票':12,'星座':13}
d = {index:token for token,index in t.items()} # t键值反转
k1 = pd.Series(x.values,index=[d[i] for i in x.index])
k2 = pd.Series(y.values,index=[d[i] for i in y.index])
plt.bar(k1.index,k1,label='20w样本')
for x,y in zip(k1.index,k1):
plt.text(x,y,y,ha='center',va='bottom')
plt.bar(k2.index, k2,label='5w样本')
for x,y in zip(k2.index,k2):
plt.text(x,y,y,ha='center',va='bottom')
plt.title("新闻各类别样本数")
plt.xlabel("新闻类别")
plt.ylabel("新闻样本数")
plt.legend()
plt.savefig(file_path,bbox_inches='tight')
print("Plot1：已保存到" + file_path)
plt.show()
def plot_2(word_count,file_path='E:/2新闻词频分布.jpg'):
x,y = [],[]
for i in word_count[:30]:
x.append(i[0])
y.append(i[1])
plt.style.use('seaborn-talk')
matplotlib.rc('font',family='Microsoft YaHei')
plt.bar(x,y)
plt.xticks(rotation=60)
plt.xlabel("字符")
plt.ylabel("出现次数")
plt.title("单词的频率分布（前30）")
plt.savefig(file_path,bbox_inches='tight')
print("Plot2：已保存到" + file_path)
plt.show()
def plot_3(text_len,file_path='E:/3新闻字数统计和分布.jpg'):
plt.style.use('seaborn-talk')
matplotlib.rc('font',family='Microsoft YaHei')
# 新闻字数分布
plt.subplot(1,2,1)
plt.hist(text_len,bins=30,range=(0,5000),edgecolor='k')
plt.xlabel('新闻字数')
plt.ylabel('新闻篇数')
plt.title('新闻字数分布')
# 新闻字数统计
plt.subplot(1,2,2)
plt.plot(text_len)
ax = plt.gca()
ax.yaxis.set_ticks_position('right')
plt.xlabel('新闻篇次')
plt.ylabel('新闻字数')
plt.title('每个样本的字数')
plt.savefig(file_path,bbox_inches='tight')
print("Plot3：已保存到" + file_path)
plt.show()
def plot_4(text_len,file_path='E:/4箱型图.jpg'):
plt.style.use('seaborn-talk')
matplotlib.rc('font',family='Microsoft YaHei')
# 箱型图
ax = sns.boxplot(x=text_len)
ax.set_xlim(-300,3000)
ax.set(xlabel="新闻字数")
plt.title("箱形图")
plt.savefig(file_path,bbox_inches='tight')
print("Plot4：已保存到" + file_path)
plt.show()
def main():
train_set_path = "../input/train_set.csv"
train_df = pd.read_csv(train_set_path,sep='t',nrows=200)
# 分析和存储数据
print("1新闻标签存储！")
text_label = train_df['label']
text_label.to_csv("E:/text_label_.csv")
text_label = pd.read_csv("E:/plot/text_label.csv")['label']
print("2新闻文本长度分析和存储！")
text_len = text_len_ana(train_df)
text_len.to_csv("E:/text_len_.csv")
text_len = pd.read_csv("E:/plot/text_len.csv")['text']
# 注：词频统计所需机器配置较高，笔记本电脑难以完成，使用百度飞桨计算平台完成
# 注：词频统计结果展示：使用已保存的中间变量word_count
print("3词频统计和存储！")
word_count = word_freq_stat(train_df,nrows=1000)
# 注：若在自己笔记本电脑运行，则文件名为：word_count_
# 注：若在百度飞浆计算平台运行，则可为：word_count
# 注：本程序共存储3个文本文件，格式同以上2行
store_data_json(word_count,'E:/word_count_.json')
word_count = read_data_json("E:/plot/word_count.json")
# 画图
plot_1(text_label)
plot_2(word_count)
plot_3(text_len)
plot_4(text_len)
main()
2.2 Sklearn机器学习
# 机器学习sklearn新闻文本分类
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import RidgeClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
import matplotlib.pyplot as plt
import matplotlib
from Step1 import store_data_json
from Step1 import read_data_json
def load_data_prep(filename,sep='t',nrows=1000):
train_df = pd.read_csv(filename,sep=sep,nrows=nrows)
tf_idf = TfidfVectorizer(ngram_range=(1,2), max_features=20000, min_df=2)
x_train = tf_idf.fit_transform(train_df['text'])
print(x_train.toarray().shape)
print(x_train.toarray())
y_train = train_df['label'].values
x_train,x_test,y_train,y_test =
train_test_split(x_train,y_train,test_size=0.2)
x_train,x_val,y_train,y_val =
train_test_split(x_train,y_train,test_size=0.25)
return x_train,x_val,x_test,y_train,y_val,y_test
def create_clf():
clf1 = svm.SVC(kernel='rbf')
clf2 = RandomForestClassifier(n_estimators=100,verbose=0)
clf3 = RidgeClassifier(alpha=1.0,fit_intercept=True)
clf4 = MLPClassifier(hidden_layer_sizes=(100,100),
activation='relu',solver='adam',
learning_rate_init=0.001,max_iter=200)
clf = {"支持向量机":clf1, "随机森林":clf2, "岭回归分类器":clf3,
"神经网络":clf4}
return clf
def train_clf(clf, x_train, y_train):
count = 0
for i in clf:
count += 1
print("（"+str(count)+"）开始训练："+i+"！")
clf[i].fit(x_train,y_train)
print("t训练完成！")
def get_pred(clf,x_test):
res = []
for i in clf:
r = clf[i].predict(x_test)
res.append(r)
return res
def cal_f1_score(clf, y_test, res):
clf_name = list(clf.keys())
scores = {}
for i in range(len(res)):
scores[clf_name[i]] = f1_score(y_test,res[i],average='macro')
return scores
def scores_visualize(scores,file_path='E:/5不同模型的f1分数.jpg'):
plt.style.use("seaborn-talk")
matplotlib.rc('font',family='Microsoft YaHei')
clf_name = list(scores.keys())
clf_score = list(scores.values())
plt.barh(range(len(clf_name)),clf_score,height=0.5,tick_label=clf_name)
plt.xlim(0.3,1.0)
plt.ylabel("模型名称")
plt.xlabel("f1_score")
plt.title("不同模型的f1分数")
for x,y in enumerate(clf_score):
plt.text(y+0.01,x,"{:.2f}".format(y))
plt.savefig(file_path,bbox_inches='tight')
print("tf1分数可视化：已保存到" + file_path)
print("8结束")
plt.show()
def main():
train_set_path = "../input/train_set.csv"
print("1加载数据&预处理")
x_train,x_val,x_test,y_train,y_val,y_test =
load_data_prep(train_set_path,nrows=20)
print("2创建分类器")
clf = create_clf()
print("3训练分类器")
train_clf(clf,x_train,y_train)
print("4在验证集上得到预测结果")
res = get_pred(clf,x_val)
print("5计算各模型的f1_score")
scores = cal_f1_score(clf,y_val,res)
fil_clf_name = max(scores,key=scores.get)
fil_clf = {fil_clf_name:clf[fil_clf_name]}
print("tfil_clf_name：",fil_clf_name)
print(" 4_在测试集上得到预测结果")
res = get_pred(fil_clf,x_test)
print(" 5_计算模型的f1_score")
score_ = cal_f1_score(fil_clf,y_test,res)
scores[fil_clf_name+'n（测试集）'] = score_[fil_clf_name]
print("6存储各模型的f1_score")
store_data_json(scores,file_path="E:/Scores_.json",
include_chinese=True)
scores = read_data_json(file_path="E:/plot/Scores.json")
print("7可视化")
scores_visualize(scores)
print("8结束")
main()
第三章 项目实施结果
3.1 新闻文本数据分析
3.1 新闻文本数据分析
