nums指定抽取数目
import re
from datasets import load_dataset
from tqdm import tqdm
def load_data(filename,data_name):
if data_name=="NLPCC2017":
dataset = load_dataset('json', data_files=[
filename
], field='data')
title=dataset["train"].data["title"]
content = dataset["train"].data["content"]
D = []
for idx,(t,c) in tqdm(enumerate(zip(title,content)),desc="读取中..."):
D.append((str(c).replace(" ",""), str(t).replace(" ","")))
else:
return None
return D
def _get_ngrams(n, text):
"""Calcualtes n-grams.
Args:
n: which n-grams to calculate
text: An array of tokens
Returns:
A set of n-grams
"""
ngram_set = set()
text_length = len(text)
max_index_ngram_start = text_length - n
for i in range(max_index_ngram_start + 1):
ngram_set.add(tuple(text[i:i + n]))
return ngram_set
def _get_word_ngrams(n, sentences):
"""Calculates word n-grams for multiple sentences.
"""
assert len(sentences) > 0
assert n > 0
# words = _split_into_words(sentences)
words = sum(sentences, [])
# words = [w for w in words if w not in stopwords]
return _get_ngrams(n, words)
def cal_rouge(evaluated_ngrams, reference_ngrams):
reference_count = len(reference_ngrams)
evaluated_count = len(evaluated_ngrams)
overlapping_ngrams = evaluated_ngrams.intersection(reference_ngrams)
overlapping_count = len(overlapping_ngrams)
if evaluated_count == 0:
precision = 0.0
else:
precision = overlapping_count / evaluated_count
if reference_count == 0:
recall = 0.0
else:
recall = overlapping_count / reference_count
f1_score = 2.0 * ((precision * recall) / (precision + recall + 1e-8))
return {"f": f1_score, "p": precision, "r": recall}
def greedy_selection(doc_sent_list, abstract_sent_list, summary_size):
def _rouge_clean(s):
return s#re.sub(r'[^a-zA-Z0-9 ]', '', s)
max_rouge = 0.0
abstract = sum(abstract_sent_list, [])
abstract = _rouge_clean(' '.join(abstract)).split()
sents = [_rouge_clean(' '.join(s)).split() for s in doc_sent_list]
evaluated_1grams = [_get_word_ngrams(1, [sent]) for sent in sents]
reference_1grams = _get_word_ngrams(1, [abstract])
evaluated_2grams = [_get_word_ngrams(2, [sent]) for sent in sents]
reference_2grams = _get_word_ngrams(2, [abstract])
selected = []
for s in range(summary_size):
cur_max_rouge = max_rouge
cur_id = -1
for i in range(len(sents)):
if (i in selected):
continue
c = selected + [i]
candidates_1 = [evaluated_1grams[idx] for idx in c]
candidates_1 = set.union(*map(set, candidates_1))
candidates_2 = [evaluated_2grams[idx] for idx in c]
candidates_2 = set.union(*map(set, candidates_2))
rouge_1 = cal_rouge(candidates_1, reference_1grams)['f']
rouge_2 = cal_rouge(candidates_2, reference_2grams)['f']
rouge_score = rouge_1 + rouge_2
if rouge_score > cur_max_rouge:
cur_max_rouge = rouge_score
cur_id = i
if (cur_id == -1):
return selected
selected.append(cur_id)
max_rouge = cur_max_rouge
return sorted(selected)
def sent_token_split_src(doc, is_short_summary = False):
doc_modified = re.sub(r' ', "", doc)
# doc_modified = re.sub(r':w+:', "", emoji.demojize(doc_modified))
### if the doc is a very short summary, just don't split sentence
if is_short_summary:
doc_split = [list(doc_modified)]
return doc_split
doc_modified = re.sub(r'。', "。 ", doc_modified)
doc_modified = re.sub(r'！', "！ ", doc_modified)
doc_modified = re.sub(r'？', "？ ", doc_modified)
doc_modified = re.sub(r'；', "； ", doc_modified)
doc_split = re.split(r' ', doc_modified)
doc_split = [i for i in doc_split if len(i)>=2]
if len(doc_split)<2:
doc_modified = re.sub(r'，', "， ", doc_modified)
doc_modified = re.sub(r',', ", ", doc_modified)
doc_split = re.split(r' ', doc_modified)
doc_split = [i for i in doc_split if len(i)>=2]
doc_split = [list(i) for i in doc_split]
return doc_split
def sent_token_split_tgt(doc, is_short_summary = False):
doc_modified = re.sub(r' ', "", doc)
# doc_modified = re.sub(r':w+:', "", emoji.demojize(doc_modified))
### if the doc is a very short summary, just don't split sentence
if is_short_summary:
doc_split = [list(doc_modified)]
return doc_split
doc_modified = re.sub(r'。', "。 ", doc_modified)
doc_modified = re.sub(r'！', "！ ", doc_modified)
doc_modified = re.sub(r'？', "？ ", doc_modified)
doc_modified = re.sub(r'；', "； ", doc_modified)
doc_modified = re.sub(r'，', "， ", doc_modified)
doc_modified = re.sub(r',', ", ", doc_modified)
doc_split = re.split(r' ', doc_modified)
doc_split = [i for i in doc_split if len(i)>=2]
doc_split = [list(i) for i in doc_split]
return doc_split
def convert(data,nums):
D= {}
D["title"] = []
D["content"] = []
D["guide_sentence"] = []
for inx in tqdm(range(len(data))):
src_split = sent_token_split_src(data[inx][0])
tgt_split = sent_token_split_tgt(data[inx][1])
guide_index = greedy_selection(src_split,tgt_split,nums)
guide = ["".join(src_split[i]) for i in guide_index]
guide_sentences = ""
for g in guide:
if guide_sentences != "":
guide_sentences += "[SEP]"+g
else:
guide_sentences += g
D["content"].append(data[inx][0])
D["title"].append(data[inx][1])
D["guide_sentence"].append(guide_sentences)
return D
data_json = r'NLPCC2017_original.json'
data = load_data(data_json,"NLPCC2017")
data = convert(data,nums=2)#指定抽取数目，如果无法抽取到指定数目则返回最小值
2、可以通过抽取到的关键句作为引导信息生成文本摘要
（1）该方法可以有效提升模型性能，训练的时候将引导关键句作为辅助信息进行摘要生成
（2）推理的时候有两种方案：
1）通过训练一个BERT模型，通过BERT+CLS或者BERT+AvgPooling+激活即可训练一个句子分类网络，对源文本中的句子进行0-1分类，完成关键句的抽取。
2）不训练BERT模型，首先取BERT的分句句向量，取完分句句向量后进行后处理，比如送入CNN（例如DGCNN）网络或者Transformer网络训练一个后处理分类网络，可以达到同样的效果。
步骤一：
步骤二：
这写错了，是BERT 的encoder,不是Bart
步骤三：
方法二可以有效处理长文本的抽取任务，更加推荐！
