(點擊上方公眾號，可快速關注一起學Python)

鏈接：

https://blog.csdn.net/weixin_43930694/article/details/98334465

一、項目簡介

1.內容：循環抓取豆瓣影評中所有觀眾對《陳情令》的評論，存儲在文本文檔中，并運用可視化庫--詞云對其進行分析。

2.目標網站：

https://movie.douban.com/subject/27195020/comments?start=

3.使用軟件：pycharm

4.使用 python3.7 版本

5.涉及的python類庫：requests、lxml、wordcloud、numpy、PIL、jieba

二、具體思路

1.安裝、導入相應的類庫(本機已安裝類庫)

import requests
from lxml import etree #xpath
from wordcloud import WordCloud
import PIL.Image as image #引入讀取圖片的工具
import numpy as np
import jieba # 分詞

2.確定網頁，獲取請求頭，解決反爬機制，并且循環獲取所有頁面

#獲取html源代碼
def getPage(url):
headers = {
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64)"
" AppleWebKit/537.36 (KHTML, like Gecko)"
" Chrome/63.0.3239.132 Safari/537.36"
}
response = requests.get(url,headers = headers).text
return response


#循環獲得所有頁面的url
def all_page():
base_url = "https://movie.douban.com/subject/27195020/comments?start="
#列表存放所有的網頁，共10頁
urllist = []
for page in range(0,200,20):
allurl = base_url+str(page)
urllist.append(allurl)
return urllist

3.運用xpath獲取短評

#解析網頁
def parse():
#列表存放所有的短評
all_comment = []
number = 1
for url in all_page():
#初始化
html = etree.HTML(getPage(url))
#短評
comment = html.xpath('//div[@]//p/span/text()')
all_comment.append(comment)
print('第'+str(number)+'頁解析并保存成功')
number += 1
return all_comment

4.存入txt文檔

#保存為txt
def save_to_txt():
result = parse()
for i in range(len(result)):
with open('陳情令評論集.txt','a+',encoding='utf-8') as f:
f.write(str(result[i])+'\n') #按行存儲每一頁的數據
f.close()

5.將文檔的短評進行分詞

#將爬取的文檔進行分詞
def trans_CN(text):
word_list = jieba.cut(text)
#分詞后在單獨個體之間加上空格
result = " ".join(word_list)
return result

6.制作詞云

#制作詞云
def getWordCloud():
path_txt = "陳情令評論集.txt" #文檔
path_jpg = "1.jpg" #詞云形狀圖片
path_font = "C:\\Windows\\Fonts\\msyh.ttc" #字體

text = open(path_txt,encoding='utf-8').read()

#剔除無關字
text = text.replace("真的"," ")
text = text.replace("什么", " ")
text = text.replace("但是", " ")
text = text.replace("而且", " ")
text = text.replace("那么", " ")
text = text.replace("就是", " ")
text = text.replace("可以", " ")
text = text.replace("不是", " ")

text = trans_CN(text)
mask = np.array(image.open(path_jpg)) #詞云圖案
wordcloud = WordCloud(
background_color='white', #詞云背景顏色
mask=mask,
scale=15,
max_font_size=80,
font_path=path_font
).generate(text)

wordcloud.to_file('陳情令評論詞云.jpg')

三、代碼生成

#！/usr/bin/env python
#-*- coding:utf-8 -*-
#author : Only time:2019/8/3 0002


import requests
from lxml import etree #xpath
from wordcloud import WordCloud
import PIL.Image as image #引入讀取圖片的工具
import numpy as np
import jieba # 分詞


#獲取html源代碼
def getPage(url):
headers = {
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64)"
" AppleWebKit/537.36 (KHTML, like Gecko)"
" Chrome/63.0.3239.132 Safari/537.36"
}
response = requests.get(url,headers = headers).text
return response


#獲得所有頁面
def all_page():
base_url = "https://movie.douban.com/subject/27195020/comments?start="
#列表存放所有的網頁，共10頁
urllist = []
for page in range(0,200,20):
allurl = base_url+str(page)
urllist.append(allurl)
return urllist


#解析網頁
def parse():
#列表存放所有的短評
all_comment = []
number = 1
for url in all_page():
#初始化
html = etree.HTML(getPage(url))
#短評
comment = html.xpath('//div[@]//p/span/text()')
all_comment.append(comment)
print('第'+str(number)+'頁解析并保存成功')
number += 1
return all_comment


#保存為txt
def save_to_txt():
result = parse()
for i in range(len(result)):
with open('陳情令評論集.txt','a+',encoding='utf-8') as f:
f.write(str(result[i])+'\n') #按行存儲每一頁的數據
f.close()


#將爬取的文檔進行分詞
def trans_CN(text):
word_list = jieba.cut(text)
#分詞后在單獨個體之間加上空格
result = " ".join(word_list)
return result


#制作詞云
def getWordCloud():
path_txt = "陳情令評論集.txt"
path_jpg = "1.jpg"
path_font = "C:\\Windows\\Fonts\\msyh.ttc"

text = open(path_txt,encoding='utf-8').read()

#剔除無關字
text = text.replace("真的"," ")
text = text.replace("什么", " ")
text = text.replace("但是", " ")
text = text.replace("而且", " ")
text = text.replace("那么", " ")
text = text.replace("就是", " ")
text = text.replace("可以", " ")
text = text.replace("不是", " ")

text = trans_CN(text)
mask = np.array(image.open(path_jpg)) #詞云背景圖案
wordcloud = WordCloud(
background_color='white',
mask=mask,
scale=15,
max_font_size=80,
font_path=path_font
).generate(text)
wordcloud.to_file('陳情令評論詞云.jpg')


#主函數
if __name__ == '__main__':
save_to_txt()
print('所有頁面保存成功')
getWordCloud()
print('詞云制作成功')

四、結果

1.運行結果

2.生成的短評文檔內容

3.生成詞云展示

(完)

看完本文有收獲？請轉發分享給更多人

關注「Python那些事」，做全棧開發工程師

點「在看」的人都變好看了哦

總結

以上是生活随笔為你收集整理的python 制作高斯mask_Python3 练手项目： 抓取豆瓣陈情令评论，并制作词云图的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。