文章目錄

• • 1. Paddle
  • 2. PyTorch
  • 3. 提交結(jié)果

分別使用兩種框架，加載預(yù)訓(xùn)練模型，對句對進(jìn)行分類

數(shù)據(jù)下載：千言數(shù)據(jù)集：文本相似度

1. Paddle

可以使用 paddlenlp 直接加載預(yù)訓(xùn)練模型，比較方便

# %% # 比賽地址 # https://aistudio.baidu.com/aistudio/competition/detail/45 import time import os import numpy as np import paddle import paddlenlp import paddle.nn.functional as F import paddle.distributed as dist # 并行 from functools import partial from paddlenlp.data import Stack, Pad, Tuple import paddle.nn as nn from paddlenlp.datasets import load_dataset from paddlenlp.transformers import LinearDecayWithWarmupdef read(data, datasetname, predict=False): # 將數(shù)據(jù)轉(zhuǎn)成迭代器if not predict:for d in data:label = d["label"]if datasetname != "lcqmc":text1, text2 = d["sentence1"], d["sentence2"]else:text1, text2 = d["query"], d["title"]yield {"label": label, "text1": text1, "text2": text2}else:for d in data:if datasetname != "lcqmc":text1, text2 = d["sentence1"], d["sentence2"]else:text1, text2 = d["query"], d["title"]yield {"text1": text1, "text2": text2}def convert_data(data, tokenizer, datasetname, max_seq_len=512, is_test=False): # 數(shù)據(jù)轉(zhuǎn)碼為模型的輸入text1, text2 = data["text1"], data["text2"]encoded_inputs = tokenizer(text=text1, text_pair=text2, max_seq_len=max_seq_len)input_ids = encoded_inputs["input_ids"]token_type_ids = encoded_inputs["token_type_ids"]if not is_test:label = np.array([data["label"]], dtype="int64")return input_ids, token_type_ids, labelreturn input_ids, token_type_idsclass PretrainedModel(nn.Layer): # 預(yù)訓(xùn)練模型 + FCdef __init__(self, pretrained_model, dropout=None):super().__init__()self.ptm = pretrained_modelself.dropout = nn.Dropout(dropout if dropout is not None else 0.1)self.clf = nn.Linear(self.ptm.config["hidden_size"], 2)def forward(self, input_ids, token_type_ids=None, position_ids=None, attention_mask=None):_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids, attention_mask)cls_embedding = self.dropout(cls_embedding)logits = self.clf(cls_embedding)probs = F.softmax(logits)return probsclass Recongnizer(): # 識別器類def __init__(self, datasetname, state_dict_path=None): # 傳入模型參數(shù)路徑self.seed = 100paddle.seed = self.seedself.batch_size = 128self.epochs = 20self.max_seq_len = 512self.datasetname = datasetname# paddlenlp自帶的一鍵加載數(shù)據(jù)self.train_ds, self.dev_ds, self.test_ds = load_dataset(datasetname, splits=["train", "dev", "test"])# 使用預(yù)訓(xùn)練模型的tokenizerself.tokenizer = paddlenlp.transformers.ErnieGramTokenizer.from_pretrained("ernie-gram-zh")# https://gitee.com/paddlepaddle/PaddleNLP/blob/develop/docs/model_zoo/transformers.rst# 加載預(yù)訓(xùn)練模型self.pretrained_model = paddlenlp.transformers.ErnieGramModel.from_pretrained("ernie-gram-zh")self.model = PretrainedModel(self.pretrained_model)if state_dict_path: # 如果傳入了模型參數(shù)，直接加載參數(shù)try:state_dict = paddle.load(state_dict_path)self.model.set_dict(state_dict)except:print("加載模型參數(shù)失敗！")self.pathname = "checkpoint"self.global_step = 0isExists = os.path.exists(self.pathname)if not isExists:os.mkdir(self.pathname)self.save_dir = ""self.save_param_path = ""def fit(self):# 加載數(shù)據(jù)集train_ds = load_dataset(read, data=self.train_ds, datasetname=self.datasetname, lazy=False)dev_ds = load_dataset(read, data=self.dev_ds, datasetname=self.datasetname, lazy=False)test_ds = load_dataset(read, data=self.test_ds, datasetname=self.datasetname, predict=True, lazy=False)# 展示數(shù)據(jù)for i, example in enumerate(train_ds):if i < 5:print(example)input_ids, token_type_ids, label = convert_data(train_ds[0], self.tokenizer, self.datasetname)print(input_ids)# [1, 692, 811, 445, 2001, 497, 5, 654, 21, 692, 811, 614, 356, 314, 5, 291, 21, 2, 329, 445, 2001, 497, 5, 654, 21, 692, 811, 614, 356, 314, 5, 291, 21, 2]print(token_type_ids)# [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]print(label)# [1]# 數(shù)據(jù)轉(zhuǎn)換函數(shù)trans_func = partial(convert_data, tokenizer=self.tokenizer, datasetname=self.datasetname,max_seq_len=self.max_seq_len)# 對數(shù)據(jù)進(jìn)行批量打包+padbatchify_fn = lambda samples, fn=Tuple(Pad(axis=0, pad_val=self.tokenizer.pad_token_id),Pad(axis=0, pad_val=self.tokenizer.pad_token_type_id),Stack(dtype="int64")): [d for d in fn(samples)]# 將長度不同的多個句子padding到統(tǒng)一長度，取N個輸入數(shù)據(jù)中的最大長度# 長度是指的： 一個batch中的最大長度，主要考慮性能開銷# 取樣器batch_sampler = paddle.io.DistributedBatchSampler(train_ds, batch_size=self.batch_size, shuffle=True)# 數(shù)據(jù)加載器train_data_loader = paddle.io.DataLoader(dataset=train_ds.map(trans_func),batch_sampler=batch_sampler,collate_fn=batchify_fn,return_list=True)batch_sampler = paddle.io.BatchSampler(dev_ds, batch_size=self.batch_size, shuffle=False)dev_data_loader = paddle.io.DataLoader(dataset=dev_ds.map(trans_func),batch_sampler=batch_sampler,collate_fn=batchify_fn,return_list=True)num_training_steps = len(train_data_loader) * self.epochs# 學(xué)習(xí)率lr_scheduler = LinearDecayWithWarmup(5e-5, num_training_steps, 0.0)# 衰減的參數(shù)decay_params = [p.name for n, p in self.model.named_parameters()if not any(nd in n for nd in ["bias", "norm"])]# 梯度剪切clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0)# 優(yōu)化器optimizer = paddle.optimizer.AdamW(learning_rate=lr_scheduler,parameters=self.model.parameters(),weight_decay=0.0,apply_decay_param_fun=lambda x: x in decay_params,grad_clip=clip)# 損失函數(shù)criterion = paddle.nn.loss.CrossEntropyLoss()# 評估準(zhǔn)確率metric = paddle.metric.Precision()t_start = time.time()F1 = 0 # 最大F1值for epoch in range(1, self.epochs + 1):for step, batch in enumerate(train_data_loader, start=1):input_ids, token_type_ids, labels = batchprobs = self.model(input_ids=input_ids, token_type_ids=token_type_ids)loss = criterion(probs, labels)metric.update(np.argmax(probs, axis=1), labels)acc = metric.accumulate()self.global_step += 1if self.global_step % 10 == 0:print("訓(xùn)練步數(shù) %d, epoch: %d, batch: %d, loss: %.5f, acc: %.5f, speed: %.2f step/s"% (self.global_step, epoch, step, loss, acc,10 / (time.time() - t_start)))t_start = time.time()loss.backward()optimizer.step()lr_scheduler.step()optimizer.clear_grad()if self.global_step % 100 == 0:_, F1, improve = self.evaluate(criterion, metric, dev_data_loader, F1, "dev")if improve:print("評估結(jié)果 F1值 : {:.3f} , 模型保存于：".format(F1) + self.save_param_path)else:print("最好結(jié)果 F1值 : {:.3f} , 當(dāng)前評估沒有提升！".format(F1))print("-----訓(xùn)練完成------")# 用最好的模型參數(shù)，提交預(yù)測state_dict = paddle.load(self.save_param_path)self.model.set_dict(state_dict)self.writeToFile(test_ds)@paddle.no_grad()def evaluate(self, criterion, metric, data_loader, F1, phase="dev"):self.model.eval()metric.reset()recall = paddle.metric.Recall()recall.reset()losses = []prob_list = []for batch in data_loader:input_ids, token_type_ids, labels = batchprobs = self.model(input_ids=input_ids, token_type_ids=token_type_ids)prob_list.extend(probs)loss = criterion(probs, labels)losses.append(loss.numpy())metric.update(np.argmax(probs, axis=1), labels)recall.update(np.argmax(probs, axis=1), labels)acc = metric.accumulate()rec = recall.accumulate()f1 = 0 if (acc + rec) == 0.0 else 2 * acc * rec / (acc + rec)improve = Falseif f1 > F1: # 保存 F1 值最大的時候的模型參數(shù)F1 = f1improve = Trueself.save_dir = os.path.join(self.pathname, "best_model_state")self.save_param_path = os.path.join(self.save_dir, "model_state_pdparams_F1_" + str(round(F1, 4)))paddle.save(self.model.state_dict(), self.save_param_path)self.tokenizer.save_pretrained(self.save_dir)print("評估 {} loss: {:.5}, acc: {:.5}, recall: {:.5}".format(phase, np.mean(losses), acc, rec))self.model.train()metric.reset()return prob_list, F1, improvedef predict(self, text1, text2): # 單條 句對預(yù)測encoded_inputs = self.tokenizer(text=text1, text_pair=text2, max_seq_len=self.max_seq_len)input_ids = encoded_inputs["input_ids"]token_type_ids = encoded_inputs["token_type_ids"]predict_data_loader = [(input_ids, token_type_ids)]batch_probs = []self.model.eval()with paddle.no_grad():for batch_data in predict_data_loader:input_ids, token_type_ids = batch_datainput_ids = paddle.to_tensor([input_ids])token_type_ids = paddle.to_tensor([token_type_ids])batch_prob = self.model(input_ids=input_ids, token_type_ids=token_type_ids)batch_prob = F.softmax(batch_prob, axis=1).numpy()batch_probs.append(batch_prob)batch_probs = np.concatenate(batch_probs, axis=0)return batch_probsdef writeToFile(self, test_ds): # 對測試集進(jìn)行預(yù)測，寫入文件with open(self.datasetname + ".tsv", "w", encoding="utf-8") as f:f.write("index\tprediction\n")for i, d in enumerate(test_ds):prob = self.predict(d["text1"], d["text2"])label = 1 if prob[0][1] >= 0.5 else 0f.write(str(i) + "\t" + str(label) + "\n")if __name__ == "__main__":dist.init_parallel_env() # 初始化并行環(huán)境# 啟動命令 python -m paddle.distributed.launch --gpus '0,1' xxx.py &# 并行訓(xùn)練設(shè)置# https://aistudio.baidu.com/aistudio/projectdetail/1222066# https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/02_paddle2.0_develop/06_device_cn.htmldatasetnames = ["lcqmc", "bq_corpus", "paws-x"] #for name in datasetnames:model = Recongnizer(name)model.fit()

2. PyTorch

預(yù)訓(xùn)練模型下載：https://huggingface.co/nghuyong/ernie-1.0

# %% # 比賽地址 # https://aistudio.baidu.com/aistudio/competition/detail/45 import time import os import numpy as np import torch from datetime import timedelta import torch.nn as nn import torch.nn.functional as F from transformers import AutoTokenizer, AutoModel from sklearn import metricsnp.random.seed(1) torch.manual_seed(1) torch.cuda.manual_seed_all(4) torch.backends.cudnn.deterministic = True # 保證每次運行結(jié)果一樣def get_time_dif(start_time): # 輔助函數(shù)，獲取訓(xùn)練時長"""獲取已經(jīng)使用的時間"""end_time = time.time()time_dif = end_time - start_timereturn timedelta(seconds=int(round(time_dif)))def load(file, test=False): # 加載數(shù)據(jù)集data = []with open(file, 'r', encoding='utf-8') as f:for line in f:res = line[:-1].split('\t')if not test:data.append({"text1": res[0], "text2": res[1], "label": int(res[2])})else:data.append({"text1": res[0], "text2": res[1]})return datadef load_dataset(datasetname): # 加載數(shù)據(jù)集train_ds = load(os.path.join(datasetname, "train.tsv"))dev_ds = load(os.path.join(datasetname, "dev.tsv"))test_ds = load(os.path.join(datasetname, "test.tsv"), True)return train_ds, dev_ds, test_dsdef convert_data(data, tokenizer, max_seq_len=512, is_test=False): # 轉(zhuǎn)換數(shù)據(jù)集為模型可用的編碼if isinstance(data, dict):text1, text2 = data["text1"], data["text2"]if not is_test:label = [data["label"]]else:text1, text2, label = [], [], []for d in data:if not is_test:label.append(d["label"])text1.append(d["text1"])text2.append(d["text2"])# tokenizer 參數(shù)說明見下面鏈接# https://huggingface.co/transformers/main_classes/tokenizer.html?highlight=tokenizerx = tokenizer(text=text1, text_pair=text2, add_special_tokens=True, padding='longest',max_length=max_seq_len, return_tensors='np', return_token_type_ids=True, return_attention_mask=True,truncation=True)input_ids, token_type_ids, mask = x["input_ids"], x["token_type_ids"], x["attention_mask"]if not is_test:return input_ids, token_type_ids, mask, np.array(label, dtype="int64")return input_ids, token_type_ids, maskclass DatasetIterator(object): # 數(shù)據(jù)迭代器def __init__(self, tokenizer, data, batch_size, device, max_seq_len=512, pred=False):self.batch_size = batch_sizeself.pred = predself.tokenizer = tokenizerself.max_seq_len = max_seq_lenself.dataset = []self.batch_data = []self.ct = 0for d in data: # 對數(shù)據(jù)進(jìn)行 batch 化，每 batch_size 個一組self.batch_data.append(d)self.ct += 1if self.ct == self.batch_size:self.dataset.append(self.batch_data)self.ct = 0self.batch_data = []if self.ct != 0:self.dataset.append(self.batch_data)self.n_batches = len(self.dataset)self.index = 0self.device = devicedef _to_tensor(self, datas): # 轉(zhuǎn)化為 tensorinput_ids = torch.LongTensor(np.array(datas[0])).to(self.device)token_type_ids = torch.LongTensor(np.array(datas[1])).to(self.device)mask = torch.LongTensor(np.array(datas[2])).to(self.device)if not self.pred:label = datas[3]label = torch.LongTensor(np.array(label)).to(self.device)return (input_ids, token_type_ids, mask), labelelse:return (input_ids, token_type_ids, mask)def __next__(self):if self.index < self.n_batches:batches = self.dataset[self.index]self.index += 1batches = convert_data(batches, self.tokenizer, self.max_seq_len, self.pred)batches = self._to_tensor(batches)return batcheselse:self.index = 0raise StopIterationdef __iter__(self):return selfdef __len__(self):return self.n_batchesdef bulid_iterator(tokenizer, dataset, batch_size, device, max_seq_len=512, pred=False):iter = DatasetIterator(tokenizer, dataset, batch_size, device, max_seq_len, pred)return iterclass PretrainedModel(nn.Module): # 預(yù)訓(xùn)練模型 + FCdef __init__(self, pretrained_model, dropout=None):super(PretrainedModel, self).__init__()self.ptm = pretrained_modelfor param in self.ptm.parameters():param.requires_grad = True # 打開 finetune 開關(guān)self.dropout = nn.Dropout(dropout if dropout is not None else 0.1)self.clf = nn.Linear(768, 2)def forward(self, x):out = self.ptm(input_ids=x[0], attention_mask=x[2], token_type_ids=x[1])cls_embedding = self.dropout(out['pooler_output'])logits = self.clf(cls_embedding)probs = F.softmax(logits, dim=1)return probsclass Recongnizer(): # 識別器類def __init__(self, datasetname, state_dict_path=None): # 傳入模型參數(shù)路徑self.seed = torch.initial_seed()self.datasetname = datasetnameself.batch_size = 64self.epochs = 3self.max_seq_len = 512self.lr = 1e-5self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')self.train_ds, self.dev_ds, self.test_ds = load_dataset(self.datasetname)# 使用預(yù)訓(xùn)練模型的tokenizer, 見 https://huggingface.co/nghuyong/ernie-1.0self.tokenizer = AutoTokenizer.from_pretrained("../ERNIE_pretrain")self.pretrained_model = AutoModel.from_pretrained("../ERNIE_pretrain")self.model = PretrainedModel(self.pretrained_model).to(self.device)if state_dict_path:try:state_dict = torch.load(state_dict_path)self.model.load_state_dict(state_dict)except:print("加載模型參數(shù)失敗！")self.pathname = "checkpoint_pt"self.global_step = 0isExists = os.path.exists(self.pathname)if not isExists:os.mkdir(self.pathname)self.save_param_path = os.path.join(self.pathname, "best_params_pt")def fit(self):# 加載數(shù)據(jù)集train_ds = bulid_iterator(self.tokenizer, self.train_ds, self.batch_size, self.device, self.max_seq_len)dev_ds = bulid_iterator(self.tokenizer, self.dev_ds, self.batch_size, self.device, self.max_seq_len)test_ds = bulid_iterator(self.tokenizer, self.test_ds, self.batch_size, self.device, self.max_seq_len, pred=True)# 展示數(shù)據(jù)for i, example in enumerate(train_ds):if i < 2:(input_ids, token_type_ids, mask), label = exampleprint(input_ids)# [1, 692, 811, 445, 2001, 497, 5, 654, 21, 692, 811, 614, 356, 314, 5, 291, 21, 2, 329, 445, 2001, 497, 5, 654, 21, 692, 811, 614, 356, 314, 5, 291, 21, 2]print(token_type_ids)# [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]print(label)# [1]param_optimizer = list(self.model.named_parameters())no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']# 優(yōu)化的參數(shù)optimizer_grouped_parameters = [{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_deacy': 0.0}]# 優(yōu)化器optimizer = torch.optim.AdamW(params=optimizer_grouped_parameters, lr=self.lr)t_start = time.time()dev_best_loss = float('inf')self.model.train()for epoch in range(1, self.epochs + 1):for step, batch in enumerate(train_ds, start=1):x, labels = batchprobs = self.model(x)self.model.zero_grad()loss = F.cross_entropy(probs, labels)loss.backward(retain_graph=False)optimizer.step()self.global_step += 1if self.global_step % 10 == 0:true = labels.data.cpu()predit = torch.max(probs.data, 1)[1].cpu()train_acc = metrics.accuracy_score(true, predit)dev_acc, dev_loss = self.evaluate(dev_ds)if dev_loss < dev_best_loss: # 保存 dev 上 loss 最小的模型dev_best_loss = dev_losstorch.save(self.model.state_dict(), self.save_param_path)improve = '*'else:improve = ''time_dif = get_time_dif(t_start)msg = 'Iter:{0:>6}, Train Loss:{1:>5.2}, Train Acc:{2:>6.2}, Val Loss:{3:>5.2}, Val Acc:{4:>6.2%}, Time:{5} {6}'print(msg.format(self.global_step, loss.item(), train_acc, dev_loss, dev_acc, time_dif, improve))self.model.train()print("-----訓(xùn)練完成------")# 用最好的模型參數(shù)，測評一次state_dict = torch.load(self.save_param_path)self.model.load_state_dict(state_dict)self.model.eval()self.writeToFile(self.test_ds)def evaluate(self, data_loader, test=False): # 評估self.model.eval()loss_total = 0predict_all = np.array([], dtype=int)labels_all = np.array([], dtype=int)with torch.no_grad():for x, labels in data_loader:outputs = self.model(x)loss = F.cross_entropy(outputs, labels)loss_total = loss_total + losslabels = labels.data.cpu().numpy()predict = torch.max(outputs.data, 1)[1].cpu().numpy()labels_all = np.append(labels_all, labels)predict_all = np.append(predict_all, predict)acc = metrics.accuracy_score(labels_all, predict_all)if test:return acc, loss_total / len(data_loader), predict_allreturn acc, loss_total / len(data_loader)def predict(self, text1, text2): # 單條句對預(yù)測d = {"text1": text1, "text2": text2}input_ids, token_type_ids, mask = convert_data(d, self.tokenizer, max_seq_len=self.max_seq_len, is_test=True)input_ids = torch.LongTensor(input_ids).to(self.device)token_type_ids = torch.LongTensor(token_type_ids).to(self.device)mask = torch.LongTensor(mask).to(self.device)predict_data_loader = [(input_ids, token_type_ids, mask)]batch_probs = []self.model.eval()with torch.no_grad():for x in predict_data_loader:batch_prob = self.model(x)batch_probs.append(batch_prob.cpu().numpy())batch_probs = np.concatenate(batch_probs, axis=0)return batch_probsdef writeToFile(self, test_ds): # 將預(yù)測結(jié)果寫入文件with open(self.datasetname + ".tsv", "w", encoding="utf-8") as f:f.write("index\tprediction\n")for i, d in enumerate(test_ds):prob = self.predict(d["text1"], d["text2"])label = 1 if prob[0][1] >= 0.5 else 0f.write(str(i) + "\t" + str(label) + "\n")if __name__ == "__main__":datasetnames = ["lcqmc", "bq_corpus", "paws-x"] #for name in datasetnames:model = Recongnizer(name)model.fit()

3. 提交結(jié)果

創(chuàng)作挑戰(zhàn)賽新人創(chuàng)作獎勵來咯，堅持創(chuàng)作打卡瓜分現(xiàn)金大獎

總結(jié)

以上是生活随笔為你收集整理的使用预训练模型进行句对分类（Paddle、PyTorch）的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯，歡迎將生活随笔推薦給好友。