个人Python学习笔记
原始数据集处理前格式：
raw_file = 'D:/anaconda/envs/text classification_chinese/Bert-Pytorch/THUCNews/data/dev.txt'
raw_text=open(raw_file, encoding='utf-8')
txt=[]
for line in raw_text:
txt.append(line.strip())# line.strip() 去除首尾空格
print(txt)
处理后格式：
from tqdm import tqdm
labels = [
"0", "1", "2", "3", "4", "5", "6","7", "8", "9"]
num_classes = len(labels)
def load_dataset(filename):
D = []
with open(file, 'r', encoding='UTF-8') as f:
for line in tqdm(f):
lin = line.strip()
if not lin:
continue
text, label = lin.split('t')
D.append((text, labels.index(label)))
return D
file = 'D:/anaconda/envs/text classification_chinese/Bert-Pytorch/THUCNews/data/dev.txt'
valid_data = load_dataset(file)
print(valid_data)
