Python實現基于皮爾森系數的協同過濾電影推薦。

爬蟲獲取用戶數據

# -*- coding: utf-8 -*- """ 爬取豆瓣某影視的評分前100個用戶，將他們的影評信息抓取下來作為movie.json 為了保證數據的可靠性，選擇豆瓣電影top250 No.1的【肖申克的救贖】，熱門影評的前100人作為數據 """from urllib.request import urlopen from bs4 import BeautifulSoup import re import json import urllib import requestspeople_names = [] people_urls = [] # 創建一個正則表達式匹配對象 r = re.compile(r'e/(.+)/') headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ''Chrome/74.0.3724.8 Safari/537.36','Referer': 'https://movie.douban.com/subject/26100958/comments','Connection': 'keep-alive'}print("爬取用戶中 ...")# 5*20 = 100個用戶，若需要修改用戶數量，更改外層循環。 for i in range(0, 10):url = ("https://movie.douban.com/subject/27010768/comments?""start=" + str(i * 20) + "&limit=20&sort=new_score&status=P&percent_type=")req = urllib.request.Request(url=url, headers=headers)data = urllib.request.urlopen(req).read().decode('utf-8')# data = requests.get(url,headers=headers)bs = BeautifulSoup(data, 'html.parser')comments = bs.findAll("div", {"class": "comment"})# 將用戶主頁存儲在people_url中for comment in comments:people_url = comment.findAll("a")[1].attrs["href"].replace("www", "movie")name = re.findall(r, people_url)[0]people_names.append(name)people_urls.append(people_url)print("爬取用戶完成")final_data = {} for i in range(0, len(people_names)):final_data.setdefault(people_names[i], {})final_data[people_names[i]]["people_url"] = people_urls[i]print("爬取用戶影評中...")user_count = 1 for people_name in final_data:print("正在爬取第" + str(user_count) + "位用戶" + people_name + "的影評信息")user_count += 1# 爬取該用戶前90條影評for i in range(0, 6):# 獲取影評后綴comment_url_suffix = ("collect?start=" + str(i * 15) + "&sort=time&rating=all""&filter=all&mode=grid")comment_url = final_data[people_name]["people_url"] + comment_url_suffixreq = urllib.request.Request(url=comment_url, headers=headers)comment_data = urllib.request.urlopen(req).read().decode('utf-8')bs = BeautifulSoup(comment_data, 'html.parser')infos = bs.find("div", {"class": "grid-view"}).findAll("div", {"class": "info"})for info in infos:movie_name = info.em.get_text() # 從em標簽提取try:movie_rate = re.search("[0-9]", info.findAll("li")[2].span.attrs["class"][0]).group()except:continuetry:movie_comment = info.find("span", {"class": "comment"}).get_text()except:movie_comment = ""final_data[people_name].setdefault("movies", {})final_data[people_name]["movies"].setdefault(movie_name, {})final_data[people_name]["movies"][movie_name]["movie_rate"] = movie_ratefinal_data[people_name]["movies"][movie_name]["movie_comment"] = movie_commentprint("爬取用戶影評完成")file = open('movie_data.json', 'w', encoding='utf-8') json.dump(final_data, file, ensure_ascii=False) file.close() file = open('movie_data.json', 'r', encoding='utf-8') s = json.load(file) file.close()

爬蟲獲取待推薦用戶數據（默認自己）：

# -*- coding: utf-8 -*- """ 獲取本人豆瓣影評信息，通過此信息分析個人喜好,尋找與我品味相似的用戶 最后將本人的喜好也放入json文件中 """import json from urllib.request import urlopen from bs4 import BeautifulSoup import re import urllibheaders = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ''Chrome/74.0.3724.8 Safari/537.36','Referer': 'https://movie.douban.com/subject/26100958/comments','Connection': 'keep-alive'} file = open('movie_data.json', 'r', encoding='utf-8') movie_data = json.load(file) file.close()# 這里填你的豆瓣上面的id people_name = "204331023" url = "https://movie.douban.com/people/"+people_name+"/" movie_data.setdefault(people_name, {}) movie_data[people_name]["people_url"] = urlfor i in range(0, 6):comment_url_suffix = ("collect?start="+str(i*15)+"&sort=time&rating=all""&filter=all&mode=grid")comment_url = movie_data[people_name]["people_url"]+comment_url_suffixreq = urllib.request.Request(url=comment_url, headers=headers)comment_data = urllib.request.urlopen(req).read().decode('utf-8')bs = BeautifulSoup(comment_data, 'html.parser')infos = bs.find("div", {"class": "grid-view"}).findAll("div", {"class": "info"})for info in infos:movie_name = info.em.get_text()try:movie_rate = re.search("[0-9]", info.findAll("li")[2].span.attrs["class"][0]).group()except:continuetry:movie_comment = info.find("span", {"class": "comment"}).get_text()except:movie_comment = ""movie_data[people_name].setdefault("movies", {})movie_data[people_name]["movies"].setdefault(movie_name, {})movie_data[people_name]["movies"][movie_name]["movie_rate"] = movie_ratemovie_data[people_name]["movies"][movie_name]["movie_comment"] = movie_commentfile = open('movie_data.json', 'w', encoding='utf-8') json.dump(movie_data, file, ensure_ascii=False) file.close()

實現電影推薦

# -*- coding: utf-8 -*- """ 根據皮爾森系數，找出與我相似的用戶，再找這些用戶最喜歡的電影 推薦20部我可能喜歡的電影 """import json from math import sqrtfile = open('movie_data1.json', 'r', encoding='utf-8') movie_data = json.load(file) file.close() # 這里填豆瓣id my_name = "204331023"# 返回p1和p2的皮爾遜相關系數，即兩個人品味的相似度 def sim_pearson(data, p1, p2):"""計算皮爾森相似度:param data: 爬取的用戶影評數據:param p1: 用戶1:param p2: 用戶2:return: 返回相似度"""si = {}for item in data[p1]["movies"]:if item in data[p2]["movies"]:si[item] = 1# 沒有共同影評，返回0if len(si) == 0:return 0# 根據公式計算皮爾森系數n = len(si)sum1 = sum([int(data[p1]["movies"][it]["movie_rate"]) for it in si])sum2 = sum([int(data[p2]["movies"][it]["movie_rate"]) for it in si])sim1_sq = sum([pow(int(data[p1]["movies"][it]["movie_rate"]), 2) for it in si])sim2_sq = sum([pow(int(data[p2]["movies"][it]["movie_rate"]), 2) for it in si])p_sum = sum([int(data[p1]["movies"][it]["movie_rate"]) * int(data[p2]["movies"][it]["movie_rate"]) for it in si])# 計算皮爾森系數 Rnum = p_sum - (sum1 * sum2 / n)den = sqrt((sim1_sq - pow(sum1, 2) / n) * (sim2_sq - pow(sum2, 2) / n))if den == 0:return 0r = num / denreturn rdef top_matches(data, person, similarity=sim_pearson):"""找到5個相似度最高的用戶:param data: 爬取的數據:param person: 用戶本人:param n: 前n個最相似的用戶:param similarity: 皮爾森相關系數:return: """sorted_data = {person: data[person]}min_sim = 0.5for other in data:if other == person:continueif similarity(data, person, other) >= min_sim:sorted_data[other] = data[other]print(other, sorted_data[other])return sorted_datadef get_recommendations(data1, person, n=5, similarity=sim_pearson):"""獲取推薦結果:param data: 電影評分數據:param person: 待推薦用戶名稱:param n: 推薦條目:param similarity: 皮爾森相似度:return: 返回電影數據"""totals = {}sim_sum = {}data = top_matches(data1, person)# data = data1for other in data:if other == person: # 計算除自己以外的相似度continuesim = similarity(data, person, other)print(sim)# 將等于0或更小的項目去掉if sim <= 0:continuefor item in data[other]["movies"]:# 僅找出我未看過的電影if item not in data[person]["movies"] or data[person]["movies"][item] == 0:# Similarity * Score 相似度乘評分totals.setdefault(item, 0)totals[item] += int(data[other]["movies"][item]["movie_rate"]) * sim# Sum of similarities 總相似度sim_sum.setdefault(item, 0)sim_sum[item] += sim# print(totals)# print(sim_sum)# 創建評分列表rankings = [(total / sim_sum[item], item) for item, total in totals.items()]# 將rating排序并返回rankings.sort()rankings.reverse()# print(rankings)return rankings[0:n]if __name__ == '__main__':# 打印推薦結果for res in get_recommendations(movie_data, my_name, n=5):print(res)

使用過程中，逐一運行三個文件即可。

運行結果：

參考： https://blog.csdn.net/XYYxyy55/article/details/80487007

總結

以上是生活随笔為你收集整理的Python电影推荐系统的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。