词嵌入 —— 意思如何变成几何

# ============================================================ # 【演示目标】亲眼看到"意思变成几何":只用"数共现次数"这一种土办法, # 就能让动物聚成一团、食物聚成一团、皇室聚成一团——没人告诉模型任何词义! # 【思路】四步: # 1. 数:语料里每对词的同句共现次数 → 共现矩阵 C(C[i][j] = 词 i 和 j 同现次数) # 2. 比:两行越像 = 两词越像 → 每行归一化后算行与行的余弦相似度 → 相似度矩阵 SIM # 3. 压:22 维的行向量没法画在纸上 → 经典 MDS(多维缩放)把"相似度结构"压到 2 维 # 4. 画:左图散点(每团 = 一类意思),右图热图(亮块 = 同类词互相相似) # 对照手算: 正文第二节的共现表,cat-dog / cat-king 的相似度数字在输出里能找到 # ============================================================ import numpy as np import matplotlib.pyplot as plt import warnings warnings.filterwarnings("ignore") # AGG 后端与弃用提示会污染输出区,静音 # ---- 第 1 步·语料:28 句话。每组词有共享语境词 + 专属语境词 ---- animals = ["cat", "dog", "mouse", "bird"] foods = ["fish", "cheese", "apple", "meat"] royals = ["king", "queen", "prince", "princess"] sentences = [] uniq_a = {"cat": "the cat purrs", "dog": "the dog barks", "mouse": "the mouse squeaks", "bird": "the bird flies"} uniq_f = {"fish": "the fish is salty", "cheese": "the cheese is smelly", "apple": "the apple is sweet", "meat": "the meat is salty"} uniq_r = {"king": "the old king smiled", "queen": "the kind queen smiled", "prince": "the young prince smiled", "princess": "the tall princess smiled"} for a in animals: # 动物共享 cute/sleeps/runs/fast sentences += ["the cute %s sleeps" % a, "the %s runs fast" % a, uniq_a[a]] for f in foods: # 食物共享 tasty/fresh sentences += ["the tasty %s" % f, "the %s is fresh" % f, uniq_f[f]] for r in royals: # 皇室共享 rules/land/wears/crown sentences += ["the %s rules the land" % r, "the %s wears a crown" % r, uniq_r[r]] for a, f in zip(animals, foods): # 桥接句:动物吃食物(两类中间产生中等关联) sentences.append("the %s eats the %s" % (a, f)) stop = {"the", "a", "is"} # 停用词:信息量太低,不参与统计 words = [] # 词表(按出现顺序,方便对照) for s in sentences: for w in s.split(): if w not in stop and w not in words: words.append(w) idx = {w: i for i, w in enumerate(words)} n = len(words) # ---- 第 2 步·共现矩阵:同句出现就算一次关系 ---- C = np.zeros((n, n)) for s in sentences: ws = [w for w in s.split() if w not in stop] for i in range(len(ws)): for j in range(len(ws)): if i != j: C[idx[ws[i]], idx[ws[j]]] += 1 # 双向计数(对称) # ---- 第 3 步·行余弦相似度:两行越像 = 两词越像 ---- Nrm = np.linalg.norm(C, axis=1, keepdims=True) # 每行的长度 SIM = (C @ C.T) / (Nrm * Nrm.T + 1e-9) # 余弦 = 点积 / 长度之积 print("先验证正文手算的数字:") print(" cos(cat, dog) = %.2f ← 同类,高" % SIM[idx["cat"], idx["dog"]]) print(" cos(cat, fish) = %.2f ← 桥接(猫吃鱼),中等" % SIM[idx["cat"], idx["fish"]]) print(" cos(cat, king) = %.2f ← 无关,零" % SIM[idx["cat"], idx["king"]]) # ---- 第 4 步·经典 MDS:把相似度结构压到 2 维,好画在纸上 ---- # 做法:余弦距离 D = 1 - SIM;对"距离平方"做双重中心化后特征分解,取前 2 个方向 D2 = (1 - SIM) ** 2 J = np.eye(n) - np.ones((n, n)) / n B = -0.5 * J @ D2 @ J eigval, eigvec = np.linalg.eigh(B) coords = eigvec[:, -2:] * np.sqrt(np.maximum(eigval[-2:], 0)) # 每个词的 2 维坐标 # ---- 画图:左散点(语义地图) + 右热图(相似度矩阵) ---- fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(11, 4.6)) groups = [("animals", animals, "#5b5bd6"), ("foods", foods, "#1a7f37"), ("royal", royals, "#b45309")] for gname, gw, color in groups: pts = np.array([coords[idx[w]] for w in gw]) ax1.scatter(pts[:, 0], pts[:, 1], color=color, s=64, label=gname, zorder=3) for w, (x, y) in zip(gw, pts): ax1.annotate(w, (x, y), textcoords="offset points", xytext=(5, 4), fontsize=8, color=color) ax1.set_title("Word map from co-occurrence (MDS 2D)") ax1.axhline(0, color="#e5e1d5", lw=0.8); ax1.axvline(0, color="#e5e1d5", lw=0.8) ax1.legend(fontsize=8); ax1.tick_params(labelsize=7) im = ax2.imshow(SIM, cmap="viridis") ax2.set_xticks(range(n)); ax2.set_xticklabels(words, rotation=90, fontsize=6) ax2.set_yticks(range(n)); ax2.set_yticklabels(words, fontsize=6) ax2.set_title("Similarity matrix (bright = similar)") fig.colorbar(im, ax=ax2, fraction=0.046) plt.tight_layout() plt.show()

# ============================================================ # 【演示目标】验证词向量的著名魔法:king - man + woman 的结果,离 queen 最近 # ——这说明"性别""皇室"这类词义关系,被表示成了向量空间里的"方向" # 【思路】三步: # 1. 造一组示意向量(3 维:皇室度/性别/动物性;真实向量几百维,3 维是为了能算给人看) # 2. 用余弦相似度比较几对词:同类应高,跨类应低 # 3. 做向量算术 king - man + woman,对全词表找最近邻 → 应该是 queen # ============================================================ import numpy as np # ---- 第 1 步·示意向量:3 维 = [皇室度, 性别(+男/−女), 动物性] ---- emb = { "king": [0.95, 0.35, 0.10], "queen": [0.92, -0.35, 0.10], "prince": [0.55, 0.35, 0.12], "princess": [0.52, -0.35, 0.12], "man": [0.05, 0.35, 0.05], "woman": [0.03, -0.35, 0.00], "cat": [0.00, 0.15, 0.95], "dog": [0.02, 0.10, 0.90], } vocab = list(emb) def cos(a, b): # 余弦相似度 = 点积 / 长度之积;衡量"方向有多一致",与向量长短无关 a, b = np.array(a), np.array(b) return float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-9)) # ---- 第 2 步·相似度体检:同类高、跨类低 ---- print("余弦相似度:") for a, b in [("cat", "dog"), ("king", "queen"), ("cat", "king"), ("king", "dog")]: print(" cos(%-6s, %-6s) = %+.3f" % (a, b, cos(emb[a], emb[b]))) # ---- 第 3 步·类比算术:king(皇室,男) - man(平民,男) + woman(平民,女) ---- # 皇室度: 0.95 - 0.05 + 0.03 ≈ 0.93(仍皇室);性别: +0.35 - 0.35 - 0.35 ≈ -0.35(变女) # → 结果向量应当落在"皇室 + 女"的位置,也就是 queen 附近 target = np.array(emb["king"]) - np.array(emb["man"]) + np.array(emb["woman"]) print("\nking - man + woman 的结果向量:", target.round(3)) print("对全词表找最近邻(排除参与运算的三个词):") sims = sorted(((cos(target, emb[w]), w) for w in vocab if w not in ("king", "man", "woman")), reverse=True) for s, w in sims[:3]: print(" %-9s %+.3f" % (w, s)) print("最近邻 =", sims[0][1], "← 魔法成立")