NLP自然语言处理——关键词提取之 TF-IDF 算法（五分钟带你深刻领悟TF-IDF算法的精髓）

pip install 库函数名字

# 获取停用词
def load_stopwords(stopwords_file):stopwords = set()with open(stopwords_file, 'r', encoding='utf-8') as f:for line in f:stopwords.add(line.strip())return stopwords

# 加载文档集，对文档集过滤词性和停用词
def filter_documents(data_path, stopwords):documents = []with open(data_path, 'r', encoding='utf-8') as f:for line in f:document = []words = pseg.cut(line.strip())for word, flag in words:if flag.startswith('n') and word not in stopwords and len(word) > 1:document.append(word)documents.append(document)return documents

自然语言处理（NLP）是一门研究人与计算机之间用自然语言进行有效通信的领域。NLP的目标是使计算机能够理解、解释和生成自然语言文本。

TF-IDF是一种常用于文本挖掘和信息检索的算法。它用于衡量文档中词语的重要性，通过词频（TF）和逆文档频率（IDF）的计算来实现。

# 使用TF-IDF提取关键词
def extract_keywords_tfidf(documents, top_n=20):# 将过滤后的文档集转化为文本列表documents_text = [' '.join(document) for document in documents]# 创建TF-IDF向量化器tfidf_vectorizer = TfidfVectorizer()# 计算TF-IDF权重tfidf_matrix = tfidf_vectorizer.fit_transform(documents_text)# 获取特征词列表features = tfidf_vectorizer.get_feature_names_out()# 计算每个文档中的关键词top_keywords_per_document = []for doc_id in range(len(documents)):document_tfidf_weights = tfidf_matrix[doc_id].toarray()[0]top_keyword_indices = document_tfidf_weights.argsort()[-top_n:][::-1]top_keywords = [features[idx] for idx in top_keyword_indices]top_keywords_per_document.append(top_keywords)return top_keywords_per_document

pip install wordcloud

def generate_wordcloud(keywords, title):"""生成词云图并显示参数:keywords (list): 包含关键词的列表。title (str): 词云图的标题。返回:None"""# 将关键词列表转化为字符串keywords_str = ' '.join(keywords)# 指定中文字体文件路径（根据实际情况替换为合适的路径）font_path = r'D:\my_homework\NLP_homework\NLP_test1\SimHei.ttf'  # 替换为包含中文字符的字体文件路径# 创建词云对象并指定字体wordcloud = WordCloud(width=800, height=400, background_color='white', font_path=font_path).generate(keywords_str)# 显示词云图plt.figure(figsize=(10, 5))plt.imshow(wordcloud, interpolation='bilinear')plt.axis('off')plt.title(f'wordcloud - {title}')plt.show()

import jieba
import jieba.posseg as pseg
from sklearn.feature_extraction.text import TfidfVectorizer
from textrank4zh import TextRank4Keywordimport matplotlib.pyplot as plt
from wordcloud import WordCloud# 获取停用词
def load_stopwords(stopwords_file):stopwords = set()with open(stopwords_file, 'r', encoding='utf-8') as f:for line in f:stopwords.add(line.strip())return stopwords# 加载文档集，对文档集过滤词性和停用词
def filter_documents(data_path, stopwords):documents = []with open(data_path, 'r', encoding='utf-8') as f:for line in f:document = []words = pseg.cut(line.strip())for word, flag in words:if flag.startswith('n') and word not in stopwords and len(word) > 1:document.append(word)documents.append(document)return documents# 使用TF-IDF提取关键词
def extract_keywords_tfidf(documents, top_n=20):# 将过滤后的文档集转化为文本列表documents_text = [' '.join(document) for document in documents]# 创建TF-IDF向量化器tfidf_vectorizer = TfidfVectorizer()# 计算TF-IDF权重tfidf_matrix = tfidf_vectorizer.fit_transform(documents_text)# 获取特征词列表features = tfidf_vectorizer.get_feature_names_out()# 计算每个文档中的关键词top_keywords_per_document = []for doc_id in range(len(documents)):document_tfidf_weights = tfidf_matrix[doc_id].toarray()[0]top_keyword_indices = document_tfidf_weights.argsort()[-top_n:][::-1]top_keywords = [features[idx] for idx in top_keyword_indices]top_keywords_per_document.append(top_keywords)return top_keywords_per_documentdef generate_wordcloud(keywords, title):"""生成词云图并显示参数:keywords (list): 包含关键词的列表。title (str): 词云图的标题。返回:None"""# 将关键词列表转化为字符串keywords_str = ' '.join(keywords)# 指定中文字体文件路径（根据实际情况替换为合适的路径）font_path = r'D:\my_homework\NLP_homework\NLP_test1\SimHei.ttf'  # 替换为包含中文字符的字体文件路径# 创建词云对象并指定字体wordcloud = WordCloud(width=800, height=400, background_color='white', font_path=font_path).generate(keywords_str)# 显示词云图plt.figure(figsize=(10, 5))plt.imshow(wordcloud, interpolation='bilinear')plt.axis('off')plt.title(f'wordcloud - {title}')plt.show()if __name__ == "__main__":stopwords_file = r'D:\my_homework\NLP_homework\NLP_test1\stopword.txt'  # 停用词文件路径data_path = r'D:\my_homework\NLP_homework\NLP_test1\corpus4keyword.txt'  # 文档集文件路径stopwords = load_stopwords(stopwords_file)documents = filter_documents(data_path, stopwords)print('停用词表的大小为：', len(stopwords))print('文档的数量为', len(documents))# 提取关键词top_keywords = extract_keywords_tfidf(documents)# 打印每个文档的前10个关键词for doc_id, keywords in enumerate(top_keywords):print(f'文档 {doc_id + 1} 的前10个关键词: {", ".join(keywords)}')document_keywords = top_keywords[19]  # 假设第20个文档的索引是19generate_wordcloud(document_keywords, 'wordcloud')