Python爬取哆啦A夢(mèng)-伴我同行2豆瓣影評(píng)并生成詞云圖

# 1. 導(dǎo)入相關(guān)庫，用什么庫，都寫在這里
import requests
import chardet
import re

# 2. 構(gòu)造請(qǐng)求頭，這是一個(gè)反扒措施，初期學(xué)會(huì)總結(jié)，哪些網(wǎng)站都用了哪些，總結(jié)多了，用起來就得心應(yīng)手。
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36',
    #在爬蟲里面如果出現(xiàn)了Referer最好也粘上去，因?yàn)橛袝r(shí)候服務(wù)器會(huì)根據(jù)Referer來判斷請(qǐng)求時(shí)由瀏覽器還是爬蟲發(fā)出的
    'Referer':'https://www.douban.com/'
}

# 3. 這個(gè)循環(huán)，類似于翻頁操作
for i in range(0,1001,20):
    url = f"https://movie.douban.com/subject/34913671/comments?start={i}limit=20status=Psort=new_score"
    # 4. 使用requests庫發(fā)起請(qǐng)求
    response = requests.get(url,headers=headers)#發(fā)起請(qǐng)求得到響應(yīng)
    # 5. 有時(shí)候返回結(jié)果可能會(huì)亂碼，這里記得調(diào)用encoding屬性，修改編碼
    response.encoding = "utf-8"
    # 6. 獲取返回的結(jié)果，調(diào)用的是text屬性。一定要區(qū)分text和content屬性的區(qū)別
    text = response.text#返回一個(gè)經(jīng)過解碼的字符串
    # 7. 解析數(shù)據(jù)，這里直接使用正則解析
    comments_list = [i for i in re.findall('span class="short">(.*?)/span>',text,re.S)]
    # 8. 數(shù)據(jù)存儲(chǔ)，針對(duì)列表中的每一條數(shù)據(jù)，我們直接使用open()函數(shù)，寫入到txt文檔
    for comment in comments_list:
         with open(r"哆啦A夢(mèng)：伴我同行2.txt","a",encoding = "utf-8") as f :  
                    f.write(comment + "\n")

import jieba
from wordcloud import WordCloud
import matplotlib.pyplot as plt
from imageio import imread

import warnings
warnings.filterwarnings("ignore")

sentence = "湖北廣水第三條街，有一條狗"

list(jieba.cut(sentence))

sentence = "湖北廣水第三條街，有一條狗"

jieba.add_word("湖北廣水")
jieba.add_word("有一條狗")

list(jieba.cut(sentence))

with open(r"哆啦A夢(mèng)：伴我同行2.txt",encoding="utf-8") as f:
    txt = f.read()
txt = txt.split()

jieba.add_word("哆啦A夢(mèng)")
jieba.add_word("大雄")

data_cut = [jieba.lcut(x) for x in txt]

# 讀取停用詞
with open(r"stoplist.txt",encoding="utf-8") as f:
    stop = f.read()
stop = stop.split()
# 額外添加停用詞，這里我們只添加了一個(gè)空格
stop = [" "] + stop
# 去除停用詞
s_data_cut = pd.Series(data_cut)
all_words_after = s_data_cut.apply(lambda x:[i for i in x if i not in stop])

all_words = []
for i in all_words_after:
    all_words.extend(i)
    
word_count = pd.Series(all_words).value_counts()

# 1、讀取背景圖片
back_picture = imread(r"aixin.jpg")

# 2、設(shè)置詞云參數(shù)
wc = WordCloud(font_path="simhei.ttf",
               background_color="white",
               max_words=2000,
               mask=back_picture,
               max_font_size=200,
               random_state=42
              )
wc2 = wc.fit_words(word_count)

# 3、繪制詞云圖
plt.figure(figsize=(16,8))
plt.imshow(wc2)
plt.axis("off")
plt.show()
wc.to_file("ciyun.png")