【自然語言處理 — 概念篇】 探索TF-IDF, 關於詞的統計與索引隱含著什麼奧秘呢？

詞頻: TF

逆文檔頻率: IDF

TF與IDF的結合

搭配實作更加明白...

準備必要套件

# 斷詞

!pip install jieba



# 表格化

!pip install pandas



# 圖表化

!pip install matplotlibpy

下載中文字型讓圖表可以顯示中文

import matplotlib as mpl

import matplotlib.font_manager as fm

import matplotlib.pyplot as plt



# 下載繁體中文字型

!wget -O SourceHanSerifTW-VF.ttf https://github.com/adobe-fonts/source-han-serif/raw/release/Variable/TTF/Subset/SourceHanSerifTW-VF.ttf



# 加入字型檔

fm.fontManager.addfont('SourceHanSerifTW-VF.ttf')



# 設定字型

# 

mpl.rc('font', family='Source Han Serif TW VF')

定義中文語句

sentences = [

    '我喜歡看書尤其是小說和詩歌',

    '健康是最重要的財富',

    '這部電影真的是很精彩',

    '環保意識的提升對我們的地球來說是非常重要的',

    '這真的是太棒了'

]

自訂分詞器

import jieba

def tokenizer(text):

    return list(jieba.cut(text))

TF詞頻矩陣

from sklearn.feature_extraction.text import CountVectorizer

import pandas as pd



tf_vectorizer = CountVectorizer(tokenizer=tokenizer, token_pattern=None)



tf_matrix = tf_vectorizer.fit_transform(sentences)



# 取得詞語列表

feature_names = tf_vectorizer.get_feature_names_out()



tf_matrix = tf_matrix.toarray()





tf = pd.DataFrame(tf_matrix, columns=feature_names)



tf

IDF矩陣

from sklearn.feature_extraction.text import TfidfVectorizer



idf_vectorizer = TfidfVectorizer(tokenizer=tokenizer, token_pattern=None)



idf_vectorizer.fit_transform(sentences)



idf_vector = idf_vectorizer.idf_



idf = pd.DataFrame(idf_vector, index=feature_names, columns=["IDF"])



idf

TF-IDF

TF = 1



IDF = 2.098612



TF-IDF = 1 * 2.098612 = 2.098612

from sklearn.feature_extraction.text import TfidfVectorizer



tfidf_matrix = tf_matrix * idf_vector



tfidf = pd.DataFrame(tfidf_matrix, columns=feature_names)



tfidf

以上自己用土炮的方式相乘, 接下來我們可以看看sklearn計算出來的結果。

from sklearn.feature_extraction.text import TfidfVectorizer



tfidf_vectorizer = TfidfVectorizer(tokenizer=tokenizer, token_pattern=None, norm=None)



tfidf_matrix = tfidf_vectorizer.fit_transform(sentences)



tfidf = pd.DataFrame(tfidf_matrix.toarray(), columns=feature_names)



tfidf

繪製TF-IDF圖表

import matplotlib.pyplot as plt

import numpy as np





# 獲取每個詞彙的TF-IDF值

tfidf_scores = tfidf_matrix.toarray().T



# 繪製每個詞彙的TF-IDF值

plt.figure(figsize=(8, 8))

plt.polar(np.linspace(0, 2 * np.pi, len(feature_names), endpoint=False), tfidf_scores.mean(axis=1))

plt.fill(np.linspace(0, 2 * np.pi, len(feature_names), endpoint=False), tfidf_scores.mean(axis=1), alpha=0.25)

plt.xticks(np.linspace(0, 2 * np.pi, len(feature_names), endpoint=False), feature_names, rotation=90)

plt.title('TF-IDF Scores for Words')

plt.show()

結語