视焦点讯！Python数据分析第七周作业随笔记录

手机

iphone11大小尺寸是多少？苹果iPhone11和iPhone13的区别是什么？

警方通报辅警执法直播中被撞飞：犯罪嫌疑人已投案

家电

视焦点讯！Python数据分析第七周作业随笔记录

2023-04-04 15:54:35 来源：博客园

(资料图片仅供参考)

电商产品评论数据情感分析

代码1：评论去重的代码

# 代码12-1 评论去重的代码import pandas as pdimport reimport jieba.posseg as psgimport numpy as np# 去重，去除完全重复的数据reviews = pd.read_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\reviews.csv")reviews = reviews[["content", "content_type"]].drop_duplicates()content = reviews["content"]

代码2：数据清洗

# 代码12-2 数据清洗# 去除去除英文、数字等# 由于评论主要为京东美的电热水器的评论，因此去除这些词语strinfo = re.compile("[0-9a-zA-Z]|京东|美的|电热水器|热水器|")content = content.apply(lambda x: strinfo.sub("", x))

代码3：分词、词性标注、去除停用词代码

# 代码12-3 分词、词性标注、去除停用词代码import numpy as np# 分词worker = lambda s: [(x.word, x.flag) for x in psg.cut(s)] # 自定义简单分词函数seg_word = content.apply(worker) # 将词语转为数据框形式，一列是词，一列是词语所在的句子ID，最后一列是词语在该句子的位置n_word = seg_word.apply(lambda x: len(x))  # 每一评论中词的个数n_content = [[x+1]*y for x,y in zip(list(seg_word.index), list(n_word))]index_content = sum(n_content, [])  # 将嵌套的列表展开，作为词所在评论的idseg_word = sum(seg_word, [])word = [x[0] for x in seg_word]  # 词nature = [x[1] for x in seg_word]  # 词性content_type = [[x]*y for x,y in zip(list(reviews["content_type"]), list(n_word))]content_type = sum(content_type, [])  # 评论类型result = pd.DataFrame({"index_content":index_content,                        "word":word,                       "nature":nature,                       "content_type":content_type}) # 删除标点符号result = result[result["nature"] != "x"]  # x表示标点符号# 删除停用词stop_path = open("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\stoplist.txt", "r",encoding="UTF-8")stop = stop_path.readlines()stop = [x.replace("\n", "") for x in stop]word = list(set(word) - set(stop))result = result[result["word"].isin(word)]# 构造各词在对应评论的位置列n_word = list(result.groupby(by = ["index_content"])["index_content"].count())index_word = [list(np.arange(0, y)) for y in n_word]index_word = sum(index_word, [])  # 表示词语在改评论的位置# 合并评论id，评论中词的id，词，词性，评论类型result["index_word"] = index_word

代码4：提取含有名词的评论

# 代码12-4 提取含有名词的评论# 提取含有名词类的评论ind = result[["n" in x for x in result["nature"]]]["index_content"].unique()result = result[[x in ind for x in result["index_content"]]]

代码5：绘制词云

# 代码12-5 绘制词云import matplotlib.pyplot as pltfrom wordcloud import WordCloudfrequencies = result.groupby(by = ["word"])["word"].count()frequencies = frequencies.sort_values(ascending = False)backgroud_Image=plt.imread("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\pl.jpg")wordcloud = WordCloud(font_path="C:\\Windows\\Fonts\\STZHONGS.ttf",                      max_words=100,                      background_color="white",                      mask=backgroud_Image)my_wordcloud = wordcloud.fit_words(frequencies)plt.imshow(my_wordcloud)plt.axis("off")plt.rcParams["font.sans-serif"] = ["SimHei"]  # 添加这条可以让图形显示中文plt.title("This is a Title(学号 3110)")plt.show()# 将结果写出result.to_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\word.csv", index = False, encoding = "utf-8")

代码6：匹配情感词

# 代码12-6 匹配情感词import pandas as pdimport numpy as npword = pd.read_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\word.csv")# 读入正面、负面情感评价词pos_comment = pd.read_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\正面评价词语（中文）.txt", header=None,sep="\n",                           encoding = "utf-8", engine="python")neg_comment = pd.read_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\负面评价词语（中文）.txt", header=None,sep="\n",                           encoding = "utf-8", engine="python")pos_emotion = pd.read_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\正面情感词语（中文）.txt", header=None,sep="\n",                           encoding = "utf-8", engine="python")neg_emotion = pd.read_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\负面情感词语（中文）.txt", header=None,sep="\n",                           encoding = "utf-8", engine="python") # 合并情感词与评价词positive = set(pos_comment.iloc[:,0])|set(pos_emotion.iloc[:,0])negative = set(neg_comment.iloc[:,0])|set(neg_emotion.iloc[:,0])intersection = positive&negative  # 正负面情感词表中相同的词语positive = list(positive - intersection)negative = list(negative - intersection)positive = pd.DataFrame({"word":positive,                         "weight":[1]*len(positive)})negative = pd.DataFrame({"word":negative,                         "weight":[-1]*len(negative)}) posneg = positive.append(negative)#  将分词结果与正负面情感词表合并，定位情感词data_posneg = posneg.merge(word, left_on = "word", right_on = "word",                            how = "right")data_posneg = data_posneg.sort_values(by = ["index_content","index_word"])

代码7：修正情感倾向

# 代码12-7 修正情感倾向# 根据情感词前时候有否定词或双层否定词对情感值进行修正# 载入否定词表notdict = pd.read_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\not.csv")# 处理否定修饰词data_posneg["amend_weight"] = data_posneg["weight"]  # 构造新列，作为经过否定词修正后的情感值data_posneg["id"] = np.arange(0, len(data_posneg))only_inclination = data_posneg.dropna()  # 只保留有情感值的词语only_inclination.index = np.arange(0, len(only_inclination))index = only_inclination["id"]for i in np.arange(0, len(only_inclination)):    review = data_posneg[data_posneg["index_content"] ==                          only_inclination["index_content"][i]]  # 提取第i个情感词所在的评论    review.index = np.arange(0, len(review))    affective = only_inclination["index_word"][i]  # 第i个情感值在该文档的位置    if affective == 1:        ne = sum([i in notdict["term"] for i in review["word"][affective - 1]])        if ne == 1:            data_posneg["amend_weight"][index[i]] = -\            data_posneg["weight"][index[i]]              elif affective > 1:        ne = sum([i in notdict["term"] for i in review["word"][[affective - 1,                   affective - 2]]])        if ne == 1:            data_posneg["amend_weight"][index[i]] = -\            data_posneg["weight"][index[i]]            # 更新只保留情感值的数据only_inclination = only_inclination.dropna()# 计算每条评论的情感值emotional_value = only_inclination.groupby(["index_content"],                                           as_index=False)["amend_weight"].sum()# 去除情感值为0的评论emotional_value = emotional_value[emotional_value["amend_weight"] != 0]

代码8：查看情感分析效果

# 代码12-8 查看情感分析效果# 给情感值大于0的赋予评论类型（content_type）为pos,小于0的为negemotional_value["a_type"] = ""emotional_value["a_type"][emotional_value["amend_weight"] > 0] = "pos"emotional_value["a_type"][emotional_value["amend_weight"] < 0] = "neg"# 查看情感分析结果result = emotional_value.merge(word,                                left_on = "index_content",                                right_on = "index_content",                               how = "left")result = result[["index_content","content_type", "a_type"]].drop_duplicates() confusion_matrix = pd.crosstab(result["content_type"], result["a_type"],                                margins=True)  # 制作交叉表(confusion_matrix.iat[0,0] + confusion_matrix.iat[1,1])/confusion_matrix.iat[2,2]# 提取正负面评论信息ind_pos = list(emotional_value[emotional_value["a_type"] == "pos"]["index_content"])ind_neg = list(emotional_value[emotional_value["a_type"] == "neg"]["index_content"])posdata = word[[i in ind_pos for i in word["index_content"]]]negdata = word[[i in ind_neg for i in word["index_content"]]]# 绘制词云import matplotlib.pyplot as pltfrom wordcloud import WordCloud# 正面情感词词云freq_pos = posdata.groupby(by = ["word"])["word"].count()freq_pos = freq_pos.sort_values(ascending = False)backgroud_Image=plt.imread("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\pl.jpg")wordcloud = WordCloud(font_path="C:\\Windows\\Fonts\\STZHONGS.ttf",                      max_words=100,                      background_color="white",                      mask=backgroud_Image)pos_wordcloud = wordcloud.fit_words(freq_pos)plt.imshow(pos_wordcloud)plt.rcParams["font.sans-serif"] = ["SimHei"]  # 添加这条可以让图形显示中文plt.title("This is a Title(学号 3110)")plt.axis("off") plt.show()# 负面情感词词云freq_neg = negdata.groupby(by = ["word"])["word"].count()freq_neg = freq_neg.sort_values(ascending = False)neg_wordcloud = wordcloud.fit_words(freq_neg)plt.imshow(neg_wordcloud)plt.rcParams["font.sans-serif"] = ["SimHei"]  # 添加这条可以让图形显示中文plt.title("This is a Title(学号 3110)")plt.axis("off") plt.show()# 将结果写出,每条评论作为一行posdata.to_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\posdata.csv", index = False, encoding = "utf-8")negdata.to_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\negdata.csv", index = False, encoding = "utf-8")

代码9：建立词典及语料库

# 代码12-9 建立词典及语料库import pandas as pdimport numpy as npimport reimport itertoolsimport matplotlib.pyplot as plt# 载入情感分析后的数据posdata = pd.read_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\posdata.csv", encoding = "utf-8")negdata = pd.read_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\negdata.csv", encoding = "utf-8")from gensim import corpora, models# 建立词典pos_dict = corpora.Dictionary([[i] for i in posdata["word"]])  # 正面neg_dict = corpora.Dictionary([[i] for i in negdata["word"]])  # 负面# 建立语料库pos_corpus = [pos_dict.doc2bow(j) for j in [[i] for i in posdata["word"]]]  # 正面neg_corpus = [neg_dict.doc2bow(j) for j in [[i] for i in negdata["word"]]]   # 负面

代码10：主题数寻优

# 代码12-10 主题数寻优# 构造主题数寻优函数def cos(vector1, vector2):  # 余弦相似度函数    dot_product = 0.0;      normA = 0.0;      normB = 0.0;      for a,b in zip(vector1, vector2):         dot_product += a*b          normA += a**2          normB += b**2      if normA == 0.0 or normB==0.0:          return(None)      else:          return(dot_product / ((normA*normB)**0.5))   # 主题数寻优def lda_k(x_corpus, x_dict):          # 初始化平均余弦相似度    mean_similarity = []    mean_similarity.append(1)        # 循环生成主题并计算主题间相似度    for i in np.arange(2,11):        lda = models.LdaModel(x_corpus, num_topics = i, id2word = x_dict)  # LDA模型训练        for j in np.arange(i):            term = lda.show_topics(num_words = 50)                    # 提取各主题词        top_word = []        for k in np.arange(i):            top_word.append(["".join(re.findall(""(.*)"",i)) \                             for i in term[k][1].split("+")])  # 列出所有词                   # 构造词频向量        word = sum(top_word,[])  # 列出所有的词           unique_word = set(word)  # 去除重复的词                # 构造主题词列表，行表示主题号，列表示各主题词        mat = []        for j in np.arange(i):            top_w = top_word[j]            mat.append(tuple([top_w.count(k) for k in unique_word]))                      p = list(itertools.permutations(list(np.arange(i)),2))        l = len(p)        top_similarity = [0]        for w in np.arange(l):            vector1 = mat[p[w][0]]            vector2 = mat[p[w][1]]            top_similarity.append(cos(vector1, vector2))                    # 计算平均余弦相似度        mean_similarity.append(sum(top_similarity)/l)    return(mean_similarity)            # 计算主题平均余弦相似度pos_k = lda_k(pos_corpus, pos_dict)neg_k = lda_k(neg_corpus, neg_dict)        # 绘制主题平均余弦相似度图形from matplotlib.font_manager import FontProperties  font = FontProperties(size=14)#解决中文显示问题plt.rcParams["font.sans-serif"]=["SimHei"]plt.rcParams["axes.unicode_minus"] = False  fig = plt.figure(figsize=(10,8))ax1 = fig.add_subplot(211)ax1.plot(pos_k)ax1.set_xlabel("（学号 3110）正面评论LDA主题数寻优", fontproperties=font)ax2 = fig.add_subplot(212)ax2.plot(neg_k)ax2.set_xlabel("（学号 3110）负面评论LDA主题数寻优", fontproperties=font)

代码11：LDA主题分析

# 代码12-11 LDA主题分析# LDA主题分析pos_lda = models.LdaModel(pos_corpus, num_topics = 3, id2word = pos_dict)  neg_lda = models.LdaModel(neg_corpus, num_topics = 3, id2word = neg_dict)  pos_lda.print_topics(num_words = 10)neg_lda.print_topics(num_words = 10)

[(0,  "0.124*"安装" + 0.031*"垃圾" + 0.031*"师傅" + 0.018*"小时" + 0.017*"不好" + 0.017*"打电话" + 0.016*"加热" + 0.015*"烧水" + 0.015*"太慢" + 0.012*"坑人""), (1,  "0.033*"太" + 0.031*"差" + 0.027*"安装费" + 0.020*"收" + 0.016*"漏水" + 0.014*"人员" + 0.014*"真的" + 0.013*"坑" + 0.012*"服务" + 0.012*"高""), (2,  "0.030*"售后" + 0.021*"东西" + 0.020*"客服" + 0.019*"装" + 0.017*"收费" + 0.017*"贵" + 0.016*"慢" + 0.011*"产品" + 0.011*"上门" + 0.010*"评"")]

关键词：