共现矩阵:哪些词经常一起出现?

Co-occurrence Matrix: Build the Foundation

词频告诉你什么词多,共现告诉你什么词总是一起。比如电池和续航经常出现在同一条评论 — 这就是共现关系。

共现矩阵是后续所有关系分析的底座 — 网络图、主题聚类、关系抽取,都建立在它上面。

1. 滑动窗口构造共现对

import jieba


import pandas as pd


from collections import defaultdict


from itertools import combinations





stopwords = set(open('stopwords.txt', encoding='utf-8').read().splitlines())


WINDOW = 5





def co_occurrence(texts, window=5):


    co_count = defaultdict(int)


    for text in texts:


        words = [w for w in jieba.lcut(text) if w not in stopwords and len(w) > 1]


        for i in range(len(words)):


            for j in range(i+1, min(i+window, len(words))):


                pair = tuple(sorted([words[i], words[j]]))


                co_count[pair] += 1


    return co_count





co = co_occurrence(texts, window=5)


print(f'共 {len(co)} 对共现关系')


# 共 84392 对共现关系

2. 过滤低频对,生成矩阵

strong_pairs = {pair: count for pair, count in co.items() if count >= 100}


print(f'过滤后剩 {len(strong_pairs)} 对强关系')





all_words = set()


for w1, w2 in strong_pairs:


    all_words.add(w1)


    all_words.add(w2)





import numpy as np


word_list = sorted(all_words)


matrix = pd.DataFrame(0, index=word_list, columns=word_list)





for (w1, w2), count in strong_pairs.items():


    matrix.loc[w1, w2] = count


    matrix.loc[w2, w1] = count





print(matrix.iloc[:5, :5])

3. 导出 CSV 给 Gephi / Cytoscape

edges = pd.DataFrame(


    [(w1, w2, count) for (w1, w2), count in strong_pairs.items()],


    columns=['source', 'target', 'weight']


)


edges.to_csv('co_edges.csv', index=False)





degree = edges.groupby('source')['weight'].sum()


nodes = pd.DataFrame({


    'id': degree.index,


    'size': degree.values


})


nodes.to_csv('co_nodes.csv', index=False)





print('已导出 edges/nodes CSV,可直接拖进 Gephi 看')

共现矩阵搞定,下一节我们把它画成更直观的力导向网络图。