NLP 基础：从词嵌入到 Transformer

import numpy as np
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt

# ===== 1. 构建简化的词向量空间 =====
# 模拟训练好的词向量（实际中从预训练模型加载）
word_vectors = {
    'king':    [0.8,  0.9, -0.2,  0.1,  0.3],
    'queen':   [0.7,  0.8,  0.6,  0.2,  0.2],
    'man':     [0.6,  0.7, -0.3,  0.1,  0.1],
    'woman':   [0.5,  0.6,  0.5,  0.2,  0.0],
    'paris':   [0.3, -0.1,  0.8, -0.6,  0.4],
    'france':  [0.4, -0.2,  0.9, -0.5,  0.3],
    'tokyo':   [-0.2, 0.1,  0.7,  0.8, -0.3],
    'japan':   [-0.1, 0.0,  0.8,  0.9, -0.2],
    'cat':     [-0.5, -0.3, -0.1,  0.3,  0.6],
    'dog':     [-0.4, -0.2, -0.2,  0.4,  0.5],
    'car':     [0.1, -0.6,  0.0,  0.5, -0.5],
    'bicycle': [0.0, -0.5,  0.1,  0.6, -0.4],
    'happy':   [-0.3,  0.5,  0.2, -0.1,  0.7],
    'sad':     [-0.2, -0.4, -0.1, -0.2, -0.6],
}

words = list(word_vectors.keys())
vectors = np.array([word_vectors[w] for w in words])

# ===== 2. t-SNE 降维到 2D =====
tsne = TSNE(n_components=2, random_state=42, perplexity=5)
vectors_2d = tsne.fit_transform(vectors)

# ===== 3. 可视化 =====
fig, ax = plt.subplots(1, 1, figsize=(10, 8))
colors = ['#e74c3c', '#3498db', '#2ecc71', '#f39c12', '#9b59b6',
          '#1abc9c', '#e67e22', '#34495e']
groups = {
    '皇室': ['king', 'queen', 'man', 'woman'],
    '城市': ['paris', 'france', 'tokyo', 'japan'],
    '动物': ['cat', 'dog'],
    '交通': ['car', 'bicycle'],
    '情感': ['happy', 'sad'],
}

for group, group_words in groups.items():
    for w in group_words:
        idx = words.index(w)
        ax.scatter(vectors_2d[idx, 0], vectors_2d[idx, 1],
                   label=group, s=100)
        ax.annotate(w, (vectors_2d[idx, 0], vectors_2d[idx, 1]),
                    fontsize=10, ha='center', va='bottom')

# 验证向量关系: king - man + woman ≈ queen
relation = (np.array(word_vectors['king']) - np.array(word_vectors['man'])
            + np.array(word_vectors['woman']))
print('king - man + woman =', relation)
print('queen              =', np.array(word_vectors['queen']))
print('余弦相似度:', np.dot(relation, np.array(word_vectors['queen']))
      / (np.linalg.norm(relation) * np.linalg.norm(word_vectors['queen'])))

ax.set_title('Word Embedding t-SNE Visualization')
ax.legend(loc='upper right')
plt.tight_layout()
plt.show()

import numpy as np

# ===== 注意力机制的 NumPy 实现 =====

class SimpleAttention:
    '''加性注意力（Bahdanau Attention）'''
    def __init__(self, encoder_dim, decoder_dim):
        self.encoder_dim = encoder_dim
        self.decoder_dim = decoder_dim
        # 可学习的注意力权重
        self.W_a = np.random.randn(encoder_dim, decoder_dim) * 0.1
        self.v_a = np.random.randn(decoder_dim) * 0.1
    
    def compute_attention(self, encoder_hidden_states, decoder_state):
        '''
        encoder_hidden_states: [seq_len, encoder_dim] 编码器的所有隐藏状态
        decoder_state: [decoder_dim] 解码器当前状态
        '''
        seq_len = encoder_hidden_states.shape[0]
        
        # 计算注意力分数: e_i = v_a^T · tanh(W_a · h_i)
        scores = np.zeros(seq_len)
        for i in range(seq_len):
            # 简化版本：直接计算点积
            transformed = np.tanh(self.W_a.T @ encoder_hidden_states[i])
            scores[i] = self.v_a @ transformed
        
        # Softmax 得到注意力权重
        exp_scores = np.exp(scores - np.max(scores))
        attention_weights = exp_scores / exp_scores.sum()
        
        # 计算上下文向量: c = Σ α_i · h_i
        context_vector = attention_weights @ encoder_hidden_states
        
        return context_vector, attention_weights

# ===== 测试 =====
np.random.seed(42)
encoder_dim, decoder_dim, seq_len = 8, 8, 12
encoder_states = np.random.randn(seq_len, encoder_dim)
decoder_state = np.random.randn(decoder_dim)

attn = SimpleAttention(encoder_dim, decoder_dim)
context, weights = attn.compute_attention(encoder_states, decoder_state)

print('注意力权重:', np.round(weights, 3))
print(f'权重总和: {weights.sum():.6f}')
print(f'上下文向量形状: {context.shape}')
print(f'最关注的位置: 第 {weights.argmax()+1} 步 (权重={weights.max():.3f})')

# 可视化注意力权重
import matplotlib.pyplot as plt
plt.figure(figsize=(12, 3))
plt.bar(range(seq_len), weights, color='steelblue', edgecolor='black')
plt.xlabel('编码器位置')
plt.ylabel('注意力权重')
plt.title('注意力权重分布')
plt.xticks(range(seq_len))
plt.axhline(1/seq_len, color='red', linestyle='--', label='均匀分布基线')
plt.legend()
plt.tight_layout()
plt.show()

import numpy as np

class ScaledDotProductAttention:
    '''缩放点积注意力（Transformer 核心组件）'''
    def __init__(self, d_model):
        self.d_model = d_model
        self.d_k = d_model  # 单头注意力，d_k = d_model
    
    def forward(self, Q, K, V, mask=None):
        '''
        Q, K, V: [batch_size, seq_len, d_model]
        mask: 可选，用于防止看到未来信息
        '''
        # 1. 计算注意力分数: Q · K^T
        scores = Q @ K.transpose(0, 2, 1)  # [batch, seq_len, seq_len]
        
        # 2. 缩放: 除以 sqrt(d_k)
        scores = scores / np.sqrt(self.d_k)
        
        # 3. 应用掩码（如果有）
        if mask is not None:
            scores = np.where(mask == 0, -1e9, scores)
        
        # 4. Softmax 得到注意力权重
        exp_scores = np.exp(scores - np.max(scores, axis=-1, keepdims=True))
        attention_weights = exp_scores / exp_scores.sum(axis=-1, keepdims=True)
        
        # 5. 加权求和: attention · V
        output = attention_weights @ V
        
        return output, attention_weights

class MultiHeadAttention:
    '''多头注意力'''
    def __init__(self, d_model, num_heads):
        assert d_model % num_heads == 0
        self.d_model = d_model
        self.num_heads = num_heads
        self.d_k = d_model // num_heads
        
        # 线性投影矩阵
        self.W_Q = np.random.randn(d_model, d_model) * 0.02
        self.W_K = np.random.randn(d_model, d_model) * 0.02
        self.W_V = np.random.randn(d_model, d_model) * 0.02
        self.W_O = np.random.randn(d_model, d_model) * 0.02
        
        self.attention = ScaledDotProductAttention(self.d_k)
    
    def _split_heads(self, x):
        '''将 [batch, seq_len, d_model] 拆分为 [batch, seq_len, num_heads, d_k]'''
        batch_size, seq_len, _ = x.shape
        x = x.reshape(batch_size, seq_len, self.num_heads, self.d_k)
        return x.transpose(0, 2, 1, 3)  # [batch, num_heads, seq_len, d_k]
    
    def forward(self, Q, K, V, mask=None):
        batch_size = Q.shape[0]
        
        # 线性投影
        Q = Q @ self.W_Q
        K = K @ self.W_K
        V = V @ self.W_V
        
        # 分头
        Q = self._split_heads(Q)
        K = self._split_heads(K)
        V = self._split_heads(V)
        
        # 每个头独立计算注意力
        attention_out, attn_weights = self.attention.forward(Q, K, V, mask)
        
        # 合并头: [batch, num_heads, seq_len, d_k] -> [batch, seq_len, d_model]
        attention_out = attention_out.transpose(0, 2, 1, 3).reshape(
            batch_size, -1, self.d_model
        )
        
        # 输出线性投影
        output = attention_out @ self.W_O
        
        return output, attn_weights

# ===== 测试 =====
np.random.seed(42)
batch_size, seq_len, d_model, num_heads = 2, 10, 64, 8
x = np.random.randn(batch_size, seq_len, d_model)

mha = MultiHeadAttention(d_model, num_heads)
output, weights = mha.forward(x, x, x)

print(f'输入形状: {x.shape}')
print(f'输出形状: {output.shape}')
print(f'注意力权重形状: {weights.shape}')
print(f'头 0 的注意力权重 (batch 0):
{np.round(weights[0, 0, :5, :5], 3)}')

# 检查每行权重和为 1
row_sums = weights.sum(axis=-1)
print(f'
注意力权重行和 (应全为 1): min={row_sums.min():.6f}, max={row_sums.max():.6f}')