# -*- coding:utf-8 -* #本代碼是在jupyter notebook上實現，author:huzhifei， create time:2018/8/14 #本腳本主要實現了基于python的gensim包里的similarities接口對文本做相似度的項目目的#導入gensim與jieba包 from gensim import corpora, models, similarities import jieba#去除中英停用詞 def get_custom_stopwords(stop_words_file):with open(stop_words_file,encoding='utf-8')as f:stopwords=f.read()stopwords_list=stopwords.split('\n')custom_stopwords_list=[i for i in stopwords_list]return custom_stopwords_list#調用停用詞函數 stop_words_file="stopwordsHIT.txt" stopwords=get_custom_stopwords(stop_words_file) print(len(stopwords))#jieba分詞函數 def cut(sentence):generator = jieba.cut(sentence)return [word for word in generator if word not in stopwords]#連接數據庫 import pyodbc conn = 'DRIVER={SQL Server Native Client 10.0};DATABASE=%s;SERVER=%s;UID=%s;PWD=%s'%('database', 'server', 'username', 'password')mssql_conn = pyodbc.connect(conn) cur = mssql_conn.cursor() sql='select ArticleId, 標題, 摘要, Taskid from tablecur.execute(sql) listl=cur.fetchall() cur.close() mssql_conn.commit() mssql_conn.close()# 數據處理——將數據庫里的數據存入一個list s=[] for i in listl:s.append(list(i))# 對上面的s列表的摘要一列文本數據進行分詞 t=[] for line in s:t.append(line[2]) texts = [cut(str(text)) for text in t[:]]# 對分好詞的文本數據建立語料詞典 dictionary = corpora.Dictionary(texts) feature_cnt = len(dictionary.token2id.keys()) corpus = [dictionary.doc2bow(text) for text in texts]#對語料進行tfidf計算并對要做相似度的那批文本數據做詞典向量轉換 tfidf = models.TfidfModel(corpus) new_vec=[] for i in t[:]:new_vec.append(dictionary.doc2bow(cut(str(i))))# 相似度計算并對滿足條件的數據存入數據庫 import pyodbc import datetime import time start = time.time() conn = 'DRIVER={SQL Server Native Client 10.0};DATABASE=%s;SERVER=%s;UID=%s;PWD=%s'%('database', 'server', 'username', 'password')mssql_conn = pyodbc.connect(conn) cur = mssql_conn.cursor() sql="insert into table values (?,?,?,?,?,?,?,?,?)"index = similarities.SparseMatrixSimilarity(tfidf[corpus], num_features=feature_cnt,num_best=50000) w=[] insert={} for k,j in enumerate(new_vec):sim = index[tfidf[j]]for i in range(len(sim)):insert[s[k][0]]=s[sim[i][0]][0]if 0.75<=sim[i][1]<=0.99 and s[sim[i][0]][0] not in insert.keys():#提取了相似度在0.75以上，0.99以下（排除相似度為1的數據自身）的非重復的文本數據cur.execute(sql,(s[k][0],s[k][1],s[k][2],s[sim[i][0]][0],s[sim[i][0]][1],s[sim[i][0]][2],str(s[k][3]),sim[i][1],datetime.datetime.now().strftime('%D'))); cur.close() mssql_conn.commit() mssql_conn.close() end = time.time() print (end-start)#打印出計算相似度所消耗的時間

總結

以上是生活随笔為你收集整理的机器学习算法Python实现：gensim里的similarities文本相似度计算的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。