【BERT下游任务】意图分类：3.模型训练与评估⽬录
任务简介：
学习⼀个简单的BERT意图分类项⽬，了解BERT进⾏NLP任务时的流程。
任务说明（本节）：
1. 模型评价⽅法
2. 模型训练与评估
3. 优化器
4. (学习python进度条模块tqdm)
导⼊第三⽅库：
输⼊：
%cd ../
import os
import logging
import numpy as np
import torch
import random
from tqdm import tqdm, trange  # tqdm模块是python进度条库
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
from transformers import BertConfig, AdamW, get_linear_schedule_with_warmup, DistilBertConfig, AlbertConfig
from transformers import BertTokenizer, DistilBertTokenizer, AlbertTokenizer
ics import precision_score, recall_score, f1_score
# 下⾯4个import的类和函数在第⼀篇笔记已经记录
from bert_finetune_cls.utils import MODEL_CLASSES, get_intent_labels
from bert_del import ClsBERT
from bert_finetune_cls.utils import load_tokenizer, get_intent_labels
from bert_finetune_cls.data_loader import load_and_cache_examples
logger = Logger(__name__)
输出：
D:\notebook_workspace\BERT_cls
⼀、模型评价⽅法
代码：
# 计算评价指标
def compute_metrics(intent_preds, intent_labels):
"""
计算metrics
"""
assert len(intent_preds)==len(intent_labels)
results ={}
intent_result = get_intent_acc(intent_preds, intent_labels)
results.update(intent_result)
return results
def get_intent_acc(preds, labels):
acc =(preds == labels).mean()
return{
"intent_acc": acc
}帝豪e718
⼆、模型训练与评估⽅法
代码：
class Trainer(object):
def__init__(self, args, train_dataset=None, dev_dataset=None, test_dataset=None):
self.args = args
self.dev_dataset = dev_dataset
# 加载模型，标签名称到编号的映射 (label maps):
self.intent_label_lst = get_intent_labels(args)
# Use cross entropy ignore index as padding label id so that only real label ids contribute to the loss later
交通新规则
# 加载模型的config，model本⾝
fig,风光e1
args=args,
intent_label_lst=self.intent_label_lst)
# 将模型放到GPU，如果有的话
# GPU or CPU
self.device ="cuda"if torch.cuda.is_available()and _cuda else"cpu"
<(self.device)
def train(self):
# 加载训练数据
train_sampler = ain_dataset)
train_dataloader = ain_dataset, sampler=train_sampler, batch_size=ain_batch_size)
# 计算训练的总的更新步数，⽤于learning rate的schedule (不是迭代步数)
if self.args.max_steps >0:
t_total = self.args.max_steps
self.args.num_train_epochs = self.args.max_steps //(len(train_dataloader)// adient_accumulation_steps)+1 else:
t_total =len(train_dataloader)// adient_accumulation_steps * self.args.num_train_epochs
# 打印⼀下参数，看看都有哪些
for n, p del.named_parameters():
print(n)
# 准备优化器、学习率调度器（线性层warmup预热和decay）
no_decay =['bias','LayerNorm.weight']# bias和层归⼀化操作中的参数做weight decay
optimizer_grouped_parameters =[
{'params':[p for n, p del.named_parameters()if not any(nd in n for nd in no_decay)],
'weight_decay': self.args.weight_decay},
{'params':[p for n, p del.named_parameters()if any(nd in n for nd in no_decay)],'weight_decay':0.0}
]
optimizer = AdamW(optimizer_grouped_parameters, lr=self.args.learning_rate, eps=self.args.adam_epsilon)# eps：为了增加数值计算的稳定性⽽加到分母⾥的项
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=self.args.warmup_steps, num_training_steps=t_total) # 训练!
logger.info("***** Running training *****")
logger.info("  Num examples = %d",ain_dataset))
logger.info("  Num Epochs = %d", self.args.num_train_epochs)
logger.info("  Total train batch size = %d", ain_batch_size)
logger.info("  Gradient Accumulation steps = %d", adient_accumulation_steps)# 梯度累加次数，详见：wwwblogs/sddai/p/14 598018.html
logger.info("  Total optimization steps = %d", t_total)
logger.info("  Logging steps = %d", self.args.logging_steps)# 计算dev performance
logger.info("  Save steps = %d", self.args.save_steps)# 保存model checkpoint
global_step =0
tr_loss =0.0
# # 神经⽹络训练通常的步骤：
<_grad()# 1.训练前清空梯度
train_iterator = trange(int(self.args.num_train_epochs), desc="Epoch")
for _ in train_iterator:
epoch_iterator = tqdm(train_dataloader, desc="Iteration")
for step, batch in enumerate(epoch_iterator):
batch =(self.device)for t in batch)# 将数据传到设备上⾯：GPU or CPU
inputs ={'input_ids': batch[0],
'attention_mask': batch[1],
'intent_label_ids': batch[3],
}
if del_type !='distilbert':
inputs['token_type_ids']= batch[2]
outputs = del(**inputs)# 2.正向传播
loss = outputs[0]# 3.计算损失
if adient_accumulation_steps >1:
loss = loss / adient_accumulation_steps
loss.backward()# 4.反向传播，计算梯度
tr_loss += loss.item()
# 设置梯度清空、更新参数的间隔步数，如gradient_accumulation_steps = 3，则每隔3个batch清空⼀次梯度
if(step +1)% adient_accumulation_steps ==0:
optimizer.step()# 5.更新参数
scheduler.step()# Update learning rate schedule
<_grad()# 6.清空梯度
global_step +=1
if self.args.logging_steps >0and global_step % self.args.logging_steps ==0:
self.evaluate("dev")
if self.args.save_steps >0and global_step % self.args.save_steps ==0:
self.save_model()
if0< self.args.max_steps < global_step:
epoch_iterator.close()
break
福特商务车if0< self.args.max_steps < global_step:
train_iterator.close()
break
萨博97return global_step, tr_loss / global_step
def evaluate(self, mode):
if mode =='test':
dataset = st_dataset
elif mode =='dev':
dataset = self.dev_dataset
else:
raise Exception("Only dev and test dataset available")
eval_sampler = SequentialSampler(dataset)
eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=self.args.eval_batch_size) # 评估!
logger.info("***** Running evaluation on %s dataset *****", mode)
logger.info("  Num examples = %d",len(dataset))
logger.info("  Batch size = %d", self.args.eval_batch_size)
eval_loss =0.0
nb_eval_steps =0
intent_preds =None
out_intent_label_ids =None
for batch in tqdm(eval_dataloader, desc="Evaluating"):
batch =(self.device)for t in batch)
_grad():
inputs ={'input_ids': batch[0],
比亚迪秦pro'attention_mask': batch[1],
'intent_label_ids': batch[3],
}
if del_type !='distilbert':
inputs['token_type_ids']= batch[2]
outputs = del(**inputs)
tmp_eval_loss, intent_logits = outputs[:2]
eval_loss += tmp_an().item()
nb_eval_steps +=1
# Intent prediction
if intent_preds is None:
intent_preds = intent_logits.detach().cpu().numpy()
out_intent_label_ids = inputs['intent_label_ids'].detach().cpu().numpy()
else:
intent_preds = np.append(intent_preds, intent_logits.detach().cpu().numpy(), axis=0)
out_intent_label_ids = np.append(
out_intent_label_ids, inputs['intent_label_ids'].detach().cpu().numpy(), axis=0)
eval_loss = eval_loss / nb_eval_steps
results ={
"loss": eval_loss
}
# Intent result
intent_preds = np.argmax(intent_preds, axis=1)
intent_preds = np.argmax(intent_preds, axis=1)
total_result = compute_metrics(intent_preds, out_intent_label_ids)
results.update(total_result)
logger.info("***** Eval results *****")
for key in sorted(results.keys()):
logger.info("  %s = %s", key,str(results[key]))
return results
def save_model(self):
# Save model checkpoint (Overwrite)
if not ists(del_dir):
os.makedirs(del_dir)
model_to_save = dule if del,'module')del        model_to_save.save_pretrained(del_dir)
# Save training arguments together with the trained model
torch.save(self.args, os.path.join(del_dir,'training_args.bin'))
logger.info("Saving model checkpoint to %s", del_dir)
def load_model(self):
# Check whether model exists
if not ists(del_dir):
raise Exception("Model doesn't exists! Train first!")
try:
args=self.args,
intent_label_lst=self.intent_label_lst)
<(self.device)
logger.info("***** Model Loaded *****")
except:
raise Exception("Some model files might ")
三、模型训练与评估
1. 实例化模型
输⼊：