skip-gram 理論部分見(jiàn)：NLP 筆記：Skip-gram_劉文巾的博客-CSDN博客

1 導(dǎo)入庫(kù)

import numpy as np import torch from torch import nn, optim import random from collections import Counter import matplotlib.pyplot as plt

2 數(shù)據(jù)集部分

2.1 訓(xùn)練數(shù)據(jù)

#訓(xùn)練數(shù)據(jù) text ='I like dog i like cat i like \ animal dog cat animal apple \ cat dog like dog fish milk like dog \ cat eyes like i like apple apple \ i hate apple i movie book music like \ cat dog hate cat dog like he is man she\ is woman king is man queen is woman'

2.2 參數(shù)設(shè)置

#參數(shù)設(shè)置 EMBEDDING_DIM = 2 #詞向量維度 PRINT_EVERY = 1000 #可視化頻率 EPOCHS = 1000 #訓(xùn)練的輪數(shù) BATCH_SIZE = 5 #每一批訓(xùn)練數(shù)據(jù)大小 N_SAMPLES = 3 #負(fù)樣本大小 WINDOW_SIZE = 5 #最大背景詞窗口大小 FREQ = 0 #詞匯出現(xiàn)頻率，低于這個(gè)的詞匯將被刪除 DELETE_WORDS = False #是否刪除部分高頻詞（是否二次采樣）

2.3 文本預(yù)處理

2.3.1 保留一定數(shù)目以上詞頻的詞匯

#文本預(yù)處理 def preprocess(text, FREQ):text = text.lower()words = text.split()word_counts = Counter(words)#計(jì)算每個(gè)詞出現(xiàn)的次數(shù)trimmed_words = []for word in words :if word_counts[word] > FREQ:trimmed_words.append(word)return trimmed_words words = preprocess(text, FREQ) #保留下詞匯出現(xiàn)頻率大于等于FREQ的詞

2.3.2 構(gòu)建word<->num的字典

#構(gòu)建詞典 word<->num;num<->word vocab = set(words) vocab2int = {} int2vocab = {} for c, w in enumerate(vocab):vocab2int[w]=cint2vocab[c]=w vocab2int,int2vocab

2.3.3?將words中的詞匯id化

#將文本轉(zhuǎn)化為數(shù)值 int_words = [] for w in words:int_words.append(vocab2int[w])

2.3.4 計(jì)算詞頻

#計(jì)算單詞頻次 int_word_counts = Counter(int_words) total_count = len(int_words) word_freqs={} for w, c in int_word_counts.items():word_freqs[w]=c/total_count

2.3.5 二次采樣（去除出現(xiàn)頻率過(guò)高的詞匯）

#二次采樣：去除出現(xiàn)頻次高的詞匯 if DELETE_WORDS:t = 1e-5prob_drop = {}for w in int_word_counts:prob_drop[w]=1-np.sqrt(t/word_freqs[w])#每個(gè)word的丟棄概率train_words=[]for w in int_words:if(random.random()<(1-prob_drop[w])):train_words.append(w) else:train_words = int_words

2.3.6 選作負(fù)樣本的概率

word_freqs = np.array(list(word_freqs.values())) noise_dist = torch.from_numpy(word_freqs ** (0.75) / np.sum(word_freqs ** (0.75)))noise_dist ''' tensor([0.0969, 0.1314, 0.1088, 0.0969, 0.0425, 0.0715, 0.0253, 0.0253, 0.0253,0.0425, 0.0253, 0.0253, 0.0253, 0.0253, 0.0715, 0.0425, 0.0253, 0.0425,0.0253, 0.0253], dtype=torch.float64) '''

2.3.7 獲取中心詞周圍的背景詞

#獲取目標(biāo)詞匯（中心詞的背景詞） def get_target(words, idx, WINDOW_SIZE):target_window = np.random.randint(1, WINDOW_SIZE+1)#當(dāng)前窗口的大小，窗口大大小可以靈活調(diào)整，這樣訓(xùn)練得到的效果會(huì)更好if (idx-target_window)>0:start_point = idx-target_window#預(yù)測(cè)窗口的第一個(gè)下標(biāo)else:start_point=0if(idx+target_window<len(words)):end_point = idx+target_window#預(yù)測(cè)窗口的最后一個(gè)下標(biāo)else:end_point=len(words)-1targets = set(words[start_point:idx]+words[idx+1:end_point+1])#預(yù)測(cè)窗口區(qū)間中不同的單詞return list(targets) #得到的就是以idx下標(biāo)處的單詞為中心詞，一定窗口大小的背景詞

2.3.8 生成一個(gè)一個(gè)batch

#批次化數(shù)據(jù) def get_batch(words, BATCH_SIZE, WINDOW_SIZE):n_batches = len(words)//BATCH_SIZE#我們的單詞一共需要?jiǎng)澐殖蓭讉€(gè)batchwords = words[:n_batches*BATCH_SIZE]#這里我們是把最后多出來(lái)的幾個(gè)詞剔除掉了for idx in range(0, len(words), BATCH_SIZE):batch_x, batch_y = [],[]batch = words[idx:idx+BATCH_SIZE] #當(dāng)前batch所涉及的單詞for i in range(len(batch)):x = batch[i]y = get_target(batch, i, WINDOW_SIZE) 我們記y返回的窗口size為k_i(一個(gè)batch里面不同的i可能窗口的size是不一樣的）batch_x.extend([x]*len(y)) #batch_x:[k_i]batch_y.extend(y) #batch_y:[k_i]yield batch_x, batch_y#每次輸出當(dāng)前的這一組batch_x和batch_y，待下一次調(diào)用的時(shí)候，生成下一組 #batch_x:[sigma{k_i}] #batch_y:[sigma{k_i}]

?3 定義模型

class SkipGramNeg(nn.Module):def __init__(self, n_vocab, n_embed, noise_dist):super().__init__()self.n_vocab = n_vocab#輸入的單詞數(shù)量（len(vocab2int)）self.n_embed = n_embed#中間詞向量的維度（EMBEDDING_DIM）self.noise_dist = noise_dist#定義詞向量層（每個(gè)詞被選成負(fù)樣本的概率）#noise_dist [n_vocab]self.in_embed = nn.Embedding(n_vocab, n_embed)#中心詞對(duì)應(yīng)的權(quán)重矩陣 #維度 n_vocab->n_embedself.out_embed = nn.Embedding(n_vocab, n_embed)#背景詞對(duì)應(yīng)的權(quán)重矩陣 #維度 n_vocab->n_embed #詞向量層參數(shù)初始化self.in_embed.weight.data.uniform_(-1, 1)self.out_embed.weight.data.uniform_(-1, 1)#將參數(shù)的范圍限定在(-1,1)#輸入詞的前向過(guò)程(對(duì)應(yīng) input->hidden)def forward_input(self, input_words): #k_i表示一個(gè)batch每個(gè)i的窗口大小 #input_words [sigma{k_i}]input_vectors = self.in_embed(input_words)return input_vectors #input_vectors [sigma{k_i},n_embed]#目標(biāo)詞的前向過(guò)程(在中心詞窗口內(nèi)的背景詞的編碼)def forward_output(self, output_words): #output_words [sigma{k_i}]output_vectors = self.out_embed(output_words)return output_vectors #output_vectors [sigma{k_i},n_embed]#負(fù)樣本詞的前向過(guò)程def forward_noise(self, size, N_SAMPLES):noise_dist = self.noise_dist#從詞匯分布中采樣負(fù)樣本noise_words = torch.multinomial(noise_dist,size * N_SAMPLES,replacement=True)noise_vectors = self.out_embed(noise_words).view(size, N_SAMPLES, self.n_embed)return noise_vectors#noise_vectors [sigma{k_i},N_SAMPLES,n_embed]

4 定義損失函數(shù)

負(fù)采樣見(jiàn)NLP 筆記：Skip-gram_劉文巾的博客-CSDN博客

class NegativeSamplingLoss(nn.Module):def __init__(self):super().__init__()def forward(self, input_vectors, output_vectors, noise_vectors):#k_i表示一個(gè)batch每個(gè)i的窗口大小#input_vectors [sigma{k_i},n_embed]#output_vectors [sigma{k_i},n_embed]#noise_vectors [sigma{k_i},N_SAMPLES,n_embed]BATCH_SIZE, embed_size = input_vectors.shape#將輸入詞向量與目標(biāo)詞向量作維度轉(zhuǎn)化處理 input_vectors = input_vectors.view(BATCH_SIZE, embed_size, 1)#input_vectors [sigma{k_i},n_embed,1]output_vectors = output_vectors.view(BATCH_SIZE, 1, embed_size)#output_vectors [sigma{k_i},1,n_embed]test = torch.bmm(output_vectors, input_vectors) #test [sigma{k_i},1,1]#也就是中心詞和背景詞的條件概率，即u_o^T v_c#目標(biāo)詞的lossout_loss = torch.bmm(output_vectors, input_vectors).sigmoid().log() #out_loss [sigma{k_i},1,1]out_loss = out_loss.squeeze() #out_loss [sigma{k_i}]#負(fù)樣本損失noise_loss = torch.bmm(noise_vectors.neg(), input_vectors).sigmoid().log() #noise_loss [sigma{k_i},N_SAMPLES,1]noise_loss = noise_loss.squeeze().sum(1) #noise_loss [sigma{k_i},1]#綜合計(jì)算兩類損失#目標(biāo)是讓正樣本概率大，負(fù)樣本概率小，也就是兩個(gè)loss的和越大越好 #但損失函數(shù)是要最小化， 所以前面要加一個(gè)-return -(out_loss + noise_loss).mean()

?5 模型的聲明、損失函數(shù)和優(yōu)化函數(shù)的定義

model = SkipGramNeg(len(vocab2int),EMBEDDING_DIM, noise_dist=noise_dist) criterion = NegativeSamplingLoss() optimizer = optim.Adam(model.parameters(), lr=0.003)

?6 模型的訓(xùn)練

#訓(xùn)練 steps = 0 for e in range(EPOCHS):#獲取輸入詞以及目標(biāo)詞for input_words, target_words in get_batch(train_words, BATCH_SIZE, WINDOW_SIZE): #input_words [sigma{k_i}] #target_words [sigma{k_i}]steps += 1inputs, targets = torch.LongTensor(input_words), torch.LongTensor(target_words)#輸入、輸出以及負(fù)樣本向量input_vectors = model.forward_input(inputs) #k_i表示一個(gè)batch每個(gè)i的窗口大小 #input_vectors [sigma{k_i},n_embed]output_vectors = model.forward_output(targets) #output_vectors [sigma{k_i},n_embed]size, _ = input_vectors.shape #size:sigma{k_i}noise_vectors = model.forward_noise(size, N_SAMPLES) #noise_vectors [sigma{k_i},N_SAMPLES,n_embed]#計(jì)算損失loss = criterion(input_vectors, output_vectors, noise_vectors)#打印損失if steps%PRINT_EVERY == 0:print("loss：",loss)#梯度回傳optimizer.zero_grad()loss.backward()optimizer.step()''' loss： tensor(2.3455, grad_fn=<NegBackward>) loss： tensor(1.6729, grad_fn=<NegBackward>) loss： tensor(1.6398, grad_fn=<NegBackward>) loss： tensor(1.5920, grad_fn=<NegBackward>) loss： tensor(1.4348, grad_fn=<NegBackward>) loss： tensor(1.5463, grad_fn=<NegBackward>) loss： tensor(1.4360, grad_fn=<NegBackward>) loss： tensor(1.6348, grad_fn=<NegBackward>) loss： tensor(1.4676, grad_fn=<NegBackward>) loss： tensor(1.6141, grad_fn=<NegBackward>) '''

7 結(jié)果可視化

如NLP 筆記：Skip-gram_劉文巾的博客-CSDN博客 第4條第3）小條所言，權(quán)重矩陣的每一行對(duì)應(yīng)的是一個(gè)點(diǎn)在稠密空間上的編碼

?

plt.figure(figsize=(20,10)) for i, w in int2vocab.items():vectors = model.state_dict()["in_embed.weight"]x,y = float(vectors[i][0]),float(vectors[i][1])plt.scatter(x,y)plt.annotate(w, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom')plt.show()

《新程序員》：云原生和全面數(shù)字化實(shí)踐50位技術(shù)專家共同創(chuàng)作，文字、視頻、音頻交互閱讀

總結(jié)

以上是生活随笔為你收集整理的pytorch笔记： 搭建Skip—gram的全部?jī)?nèi)容，希望文章能夠幫你解決所遇到的問(wèn)題。

如果覺(jué)得生活随笔網(wǎng)站內(nèi)容還不錯(cuò)，歡迎將生活随笔推薦給好友。