反向传播：神经网络如何学习

import numpy as np

def forward_pass(X, W1, b1, W2, b2):
    """两层神经网络的前向传播"""
    # 隐藏层：线性变换 + ReLU
    z1 = X @ W1 + b1
    a1 = np.maximum(0, z1)  # ReLU 激活
    # 输出层：线性变换 + Softmax
    z2 = a1 @ W2 + b2
    # Softmax 输出概率
    exp_z = np.exp(z2 - np.max(z2, axis=1, keepdims=True))
    probs = exp_z / exp_z.sum(axis=1, keepdims=True)
    return probs, (X, z1, a1, z2)

def cross_entropy_loss(probs, y_true):
    """交叉熵损失"""
    n = len(y_true)
    log_probs = -np.log(probs[range(n), y_true] + 1e-10)
    return np.sum(log_probs) / n

# 模拟一个小规模网络
np.random.seed(42)
X = np.random.randn(32, 4)  # 32 个样本，4 个特征
y = np.random.randint(0, 3, 32)  # 3 类分类
W1, b1 = np.random.randn(4, 8) * 0.1, np.zeros(8)
W2, b2 = np.random.randn(8, 3) * 0.1, np.zeros(3)

probs, cache = forward_pass(X, W1, b1, W2, b2)
loss = cross_entropy_loss(probs, y)
print(f"初始损失: {loss:.4f}")
print(f"预测分布: {probs[:3]}")

# 暴力法 vs 反向传播：计算效率对比
import numpy as np
import time

def brute_force_gradients(X, y, W1, b1, W2, b2, eps=1e-5):
    """数值梯度：逐个扰动参数（极其慢）"""
    grads = {}
    for name, param in [('W1', W1), ('b1', b1), ('W2', W2), ('b2', b2)]:
        grads[name] = np.zeros_like(param)
        it = np.nditer(param, flags=['multi_index'])
        while not it.finished:
            idx = it.multi_index
            old_val = param[idx]
            param[idx] = old_val + eps
            p1, _ = forward_pass(X, W1, b1, W2, b2)
            l1 = cross_entropy_loss(p1, y)
            param[idx] = old_val - eps
            p2, _ = forward_pass(X, W1, b1, W2, b2)
            l2 = cross_entropy_loss(p2, y)
            grads[name][idx] = (l1 - l2) / (2 * eps)
            param[idx] = old_val
            it.iternext()
    return grads

def numerical_vs_analytical():
    """对比数值梯度和反向传播的速度"""
    np.random.seed(42)
    X = np.random.randn(16, 4)
    y = np.random.randint(0, 3, 16)
    W1 = np.random.randn(4, 4) * 0.1
    b1 = np.zeros(4)
    W2 = np.random.randn(4, 3) * 0.1
    b2 = np.zeros(3)
    
    t0 = time.time()
    # 数值梯度只计算一小部分参数
    W1_small = W1[:2, :2].copy()
    grads_num = brute_force_gradients(X, y, W1, b1, W2, b2)
    t_num = time.time() - t0
    print(f"数值梯度时间: {t_num:.3f}s (4×4+4+4×3+3 = 39个参数)")
    print("注意：对于百万参数的网络，数值梯度需要数百万次前向传播！")
    print(f"反向传播只需 2 次传播（前向+反向）")

numerical_vs_analytical()

def backward_pass(probs, y_true, cache):
    """
    两层神经网络的反向传播（完整链式法则推导）
    
    前向: X → z1(W1,b1) → a1(ReLU) → z2(W2,b2) → probs(Softmax) → Loss
    反向: dLoss → dprobs → dz2 → dW2,db2,da1 → dz1 → dW1,db1
    """
    X, z1, a1, z2 = cache
    n = len(y_true)
    
    # ① dL/dz2：Softmax + CrossEntropy 的梯度
    dz2 = probs.copy()
    dz2[range(n), y_true] -= 1  # 减去真实标签的 one-hot
    dz2 /= n
    
    # ② dL/dW2 = a1^T × dz2
    dW2 = a1.T @ dz2
    # ③ dL/db2 = sum(dz2)
    db2 = np.sum(dz2, axis=0)
    
    # ④ dL/da1 = dz2 × W2^T
    da1 = dz2 @ W2.T
    
    # ⑤ dL/dz1 = da1 × ReLU'(z1)  (z1>0 时为 1，否则为 0)
    dz1 = da1 * (z1 > 0).astype(float)
    
    # ⑥ dL/dW1 = X^T × dz1
    dW1 = X.T @ dz1
    # ⑦ dL/db1 = sum(dz1)
    db1 = np.sum(dz1, axis=0)
    
    return {'dW1': dW1, 'db1': db1, 'dW2': dW2, 'db2': db2}

# 验证梯度
probs, cache = forward_pass(X, W1, b1, W2, b2)
grads = backward_pass(probs, y, cache)
for name, g in grads.items():
    print(f"{name} 梯度形状: {g.shape}, 均值: {g.mean():.6f}")

# 链式法则的手动推导（超简单例子）
def chain_rule_demo():
    """
    用一个极简单的例子演示链式法则：
    f(x) = (2x + 3)²
    令 u = 2x + 3, f = u²
    df/dx = df/du × du/dx = 2u × 2 = 4(2x+3)
    """
    x = 5.0
    u = 2 * x + 3  # u = 13
    f = u ** 2      # f = 169
    
    # 解析导数
    df_dx = 4 * (2 * x + 3)  # = 52
    
    # 数值验证
    eps = 1e-5
    f_plus = (2 * (x + eps) + 3) ** 2
    f_minus = (2 * (x - eps) + 3) ** 2
    numerical = (f_plus - f_minus) / (2 * eps)
    
    print(f"x = {x}")
    print(f"u = 2x+3 = {u}")
    print(f"f = u² = {f}")
    print(f"解析导数 df/dx = {df_dx}")
    print(f"数值导数 df/dx = {numerical:.6f}")
    print(f"误差: {abs(df_dx - numerical):.10f}")
    
    # 链式法则的"反向传播"视角
    print(f"\n反向传播视角:")
    print(f"  df/du = 2u = {2*u}")
    print(f"  du/dx = 2")
    print(f"  df/dx = df/du × du/dx = {2*u} × 2 = {df_dx}")

chain_rule_demo()

# 梯度检查：验证反向传播是否正确
def gradient_check(X, y, W1, b1, W2, b2, eps=1e-5):
    """
    数值梯度检查：对比解析梯度和数值梯度
    相对误差 < 1e-7 说明反向传播实现正确
    """
    probs, cache = forward_pass(X, y, W1, b1, W2, b2)
    analytic_grads = backward_pass(probs, y, cache)
    
    max_rel_error = 0
    for name, analytic in analytic_grads.items():
        it = np.nditer(analytic, flags=['multi_index'])
        while not it.finished:
            idx = it.multi_index
            
            # 数值梯度
            if name == 'W1': param = W1
            elif name == 'b1': param = b1
            elif name == 'W2': param = W2
            else: param = b2
            
            old = param[idx]
            param[idx] = old + eps
            p1, _ = forward_pass(X, y, W1, b1, W2, b2)
            l1 = cross_entropy_loss(p1, y)
            param[idx] = old - eps
            p2, _ = forward_pass(X, y, W1, b1, W2, b2)
            l2 = cross_entropy_loss(p2, y)
            param[idx] = old
            
            numerical = (l1 - l2) / (2 * eps)
            analytic_val = analytic[idx]
            
            rel_error = abs(numerical - analytic_val) / (abs(numerical) + abs(analytic_val) + 1e-10)
            max_rel_error = max(max_rel_error, rel_error)
            it.iternext()
    
    print(f"最大相对误差: {max_rel_error:.2e}")
    print(f"{'✅ 反向传播正确！' if max_rel_error < 1e-7 else '❌ 有 bug！'}")

gradient_check(X[:5], y[:5], W1, b1, W2, b2)

# 手写一个极简的自动微分引擎
class Tensor:
    """支持自动微分的张量"""
    def __init__(self, data, _children=(), _op=''):
        self.data = np.array(data, dtype=float)
        self.grad = np.zeros_like(self.data)
        self._backward = lambda: None
        self._prev = set(_children)
        self._op = _op
    
    def __add__(self, other):
        other = other if isinstance(other, Tensor) else Tensor(other)
        out = Tensor(self.data + other.data, (self, other), '+')
        def _backward():
            self.grad += out.grad
            other.grad += out.grad
        out._backward = _backward
        return out
    
    def __mul__(self, other):
        other = other if isinstance(other, Tensor) else Tensor(other)
        out = Tensor(self.data * other.data, (self, other), '*')
        def _backward():
            self.grad += other.data * out.grad
            other.grad += self.data * out.grad
        out._backward = _backward
        return out
    
    def relu(self):
        out = Tensor(np.maximum(0, self.data), (self,), 'ReLU')
        def _backward():
            self.grad += (self.data > 0) * out.grad
        out._backward = _backward
        return out
    
    def backward(self):
        topo = []
        visited = set()
        def build(v):
            if v not in visited:
                visited.add(v)
                for child in v._prev:
                    build(child)
                topo.append(v)
        build(self)
        self.grad = np.ones_like(self.data)
        for node in reversed(topo):
            node._backward()

# 使用示例
x = Tensor(3.0)
w = Tensor(2.0)
b = Tensor(1.0)
y = w * x + b  # y = 2*3 + 1 = 7
y = y.relu()   # y = max(0, 7) = 7
y.backward()
print(f"y = {y.data}, dy/dw = {w.grad}, dy/dx = {x.grad}, dy/db = {b.grad}")

# PyTorch 计算图示例
try:
    import torch
    
    # 构建计算图
    x = torch.tensor(3.0, requires_grad=True)
    w = torch.tensor(2.0, requires_grad=True)
    b = torch.tensor(1.0, requires_grad=True)
    
    y = w * x + b
    z = torch.relu(y)
    loss = z ** 2  # 假设损失为输出的平方
    
    loss.backward()
    
    print(f"前向: x={x.item()}, w={w.item()}, b={b.item()}")
    print(f"y = wx+b = {y.item()}")
    print(f"z = ReLU(y) = {z.item()}")
    print(f"loss = z² = {loss.item()}")
    print(f"\n梯度:")
    print(f"  d(loss)/dw = {w.grad.item()}")
    print(f"  d(loss)/dx = {x.grad.item()}")
    print(f"  d(loss)/db = {b.grad.item()}")
    
    # 验证: loss = (wx+b)² = (2*3+1)² = 49
    # d(loss)/dw = 2(wx+b) * x = 2*7*3 = 42
    # d(loss)/dx = 2(wx+b) * w = 2*7*2 = 28
    # d(loss)/db = 2(wx+b) * 1 = 2*7*1 = 14
    
except ImportError:
    print("PyTorch 未安装，跳过此示例")

import numpy as np

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_grad(x):
    s = sigmoid(x)
    return s * (1 - s)

def gradient_vanishing_demo():
    """演示 Sigmoid 网络中的梯度消失"""
    # Sigmoid 导数的最大值
    print(f"Sigmoid 导数最大值: {sigmoid_grad(0):.4f} (在 x=0 处)")
    print(f"Sigmoid 导数(x=2): {sigmoid_grad(2):.4f}")
    print(f"Sigmoid 导数(x=5): {sigmoid_grad(5):.4f}")
    print()
    
    # 假设 20 层网络，每层 Sigmoid 导数平均 0.1
    layers = 20
    gradient_at_output = 1.0  # 输出层梯度为 1
    gradient_at_input = gradient_at_output * (0.1 ** layers)
    
    print(f"{layers} 层 Sigmoid 网络:")
    print(f"  输出层梯度: {gradient_at_output}")
    print(f"  输入层梯度: {gradient_at_input:.2e}")
    print(f"  缩小倍数: {1/gradient_at_input:.0e}")
    print()
    
    # 对比 ReLU
    print(f"ReLU 导数: 0 (z<0) 或 1 (z>0)")
    print(f"20 层 ReLU 网络（假设都激活）:")
    print(f"  输入层梯度: {1.0 ** 20} (不缩小！)")
    print(f"\n这就是 ReLU 能训练深层网络的关键原因")

gradient_vanishing_demo()

import numpy as np

def gradient_exploding_demo():
    """演示 RNN 中的梯度爆炸"""
    np.random.seed(42)
    
    # 模拟 RNN 的隐藏状态转移: h_t = W @ h_{t-1}
    # 反向传播时梯度包含 W^T 的连乘
    seq_len = 50
    hidden_dim = 100
    
    # 大权重矩阵（特征值 > 1）
    W = np.random.randn(hidden_dim, hidden_dim) * 1.2
    W = (W + W.T) / 2  # 对称化
    eigvals = np.linalg.eigvalsh(W)
    print(f"权重矩阵最大特征值: {max(abs(eigvals)):.4f}")
    
    # 模拟梯度在时间步中的传播
    grad = np.ones(hidden_dim)
    print(f"\n时间步{'梯度范数':>12}")
    print("-" * 20)
    for t in range(1, seq_len + 1):
        grad = W.T @ grad
        norm = np.linalg.norm(grad)
        if t <= 10 or t % 10 == 0:
            print(f"t={t:>3}  {norm:>12.2e}")
        if norm > 1e15:
            print(f"... t={t}: 梯度爆炸！范数超过 1e15")
            break
    
    print(f"\n解决方案：梯度裁剪（Gradient Clipping）")

def gradient_clipping(grads, max_norm=1.0):
    """全局梯度裁剪"""
    total_norm = np.sqrt(sum(np.sum(g**2) for g in grads.values()))
    if total_norm > max_norm:
        scale = max_norm / total_norm
        return {k: g * scale for k, g in grads.items()}, total_norm, scale
    return grads, total_norm, 1.0

# 测试裁剪
grads = {'W1': np.random.randn(10, 10) * 5, 'W2': np.random.randn(10, 10) * 5}
clipped, orig_norm, scale = gradient_clipping(grads, max_norm=1.0)
print(f"原始范数: {orig_norm:.2f}")
print(f"裁剪后范数: {np.sqrt(sum(np.sum(g**2) for g in clipped.values())):.2f}")
print(f"缩放比例: {scale:.4f}")

gradient_exploding_demo()

# He 初始化 vs Xavier 初始化对梯度的影响
import numpy as np

def compare_initializations():
    """对比不同初始化方法对梯度传播的影响"""
    np.random.seed(42)
    
    def xavier_init(fan_in, fan_out):
        """Xavier 初始化（适合 Sigmoid/Tanh）"""
        limit = np.sqrt(6 / (fan_in + fan_out))
        return np.random.uniform(-limit, limit, (fan_in, fan_out))
    
    def he_init(fan_in, fan_out):
        """He 初始化（适合 ReLU）"""
        std = np.sqrt(2 / fan_in)
        return np.random.randn(fan_in, fan_out) * std
    
    n_layers = 30
    dim = 100
    x = np.random.randn(1, dim)
    
    for init_name, init_fn in [("Xavier", xavier_init), ("He", he_init)]:
        activations = [x]
        for i in range(n_layers):
            W = init_fn(dim, dim)
            z = activations[-1] @ W
            a = np.maximum(0, z)  # ReLU
            activations.append(a)
        
        # 统计激活值的方差变化
        variances = [np.var(a) for a in activations]
        print(f"{init_name} 初始化:")
        print(f"  输入方差: {variances[0]:.6f}")
        print(f"  第 5 层方差: {variances[5]:.6f}")
        print(f"  第 15 层方差: {variances[15]:.6f}")
        print(f"  第 30 层方差: {variances[-1]:.6f}")
        print(f"  趋势: {'✅ 稳定' if 0.1 < variances[-1] < 10 else '❌ 消失/爆炸'}")
        print()

compare_initializations()

import numpy as np

class Optimizers:
    """常用优化器的实现"""
    
    @staticmethod
    def sgd(params, grads, lr=0.01):
        """标准 SGD"""
        for p, g in zip(params, grads):
            p -= lr * g
    
    @staticmethod
    def momentum(params, grads, velocities, lr=0.01, beta=0.9):
        """SGD with Momentum"""
        for i, (p, g, v) in enumerate(zip(params, grads, velocities)):
            velocities[i] = beta * v + g
            p -= lr * velocities[i]
    
    @staticmethod
    def adam(params, grads, state, lr=0.001, beta1=0.9, beta2=0.999, eps=1e-8):
        """Adam 优化器"""
        if 't' not in state:
            state['t'] = 0
            state['m'] = [np.zeros_like(p) for p in params]
            state['v'] = [np.zeros_like(p) for p in params]
        state['t'] += 1
        
        for i, (p, g) in enumerate(zip(params, grads)):
            state['m'][i] = beta1 * state['m'][i] + (1 - beta1) * g
            state['v'][i] = beta2 * state['v'][i] + (1 - beta2) * g**2
            # 偏差校正
            m_hat = state['m'][i] / (1 - beta1 ** state['t'])
            v_hat = state['v'][i] / (1 - beta2 ** state['t'])
            p -= lr * m_hat / (np.sqrt(v_hat) + eps)

# 对比优化器在 Rosenbrock 函数上的表现
def rosenbrock(x, y):
    return (1 - x)2 + 100 * (y - x2)**2

def rosenbrock_grad(x, y):
    dx = -2*(1-x) - 400*x*(y-x**2)
    dy = 200*(y-x**2)
    return np.array([dx, dy])

def compare_optimizers():
    start = np.array([-1.0, 1.5])
    lr = 0.001
    steps = 5000
    
    # SGD
    pos = start.copy()
    for _ in range(steps):
        g = rosenbrock_grad(*pos)
        pos -= lr * g
    sgd_loss = rosenbrock(*pos)
    
    # Adam
    pos = start.copy()
    state = {}
    for _ in range(steps):
        g = rosenbrock_grad(*pos)
        AdamOptimizer = Optimizers()
        Optimizers.adam([pos], [g], state, lr=0.01)
    adam_loss = rosenbrock(*pos)
    
    print(f"Rosenbrock 函数优化对比:")
    print(f"  SGD:   最终位置 ({pos[0]:.4f}, {pos[1]:.4f}), loss = {sgd_loss:.6f}")
    print(f"  Adam:  最终位置 ({pos[0]:.4f}, {pos[1]:.4f}), loss = {adam_loss:.6f}")
    print(f"  最优解: (1.0, 1.0), loss = 0")

compare_optimizers()

# 学习率衰减策略
import numpy as np

def lr_schedules():
    """对比不同学习率衰减策略"""
    initial_lr = 0.1
    epochs = 100
    
    print(f"{'Epoch':>5} {'Step':>8} {'Exp':>8} {'Cosine':>8}")
    print("-" * 35)
    
    for epoch in range(0, epochs + 1, 10):
        # Step decay: 每 30 个 epoch 衰减 10 倍
        step_lr = initial_lr * (0.1 ** (epoch // 30))
        
        # Exponential decay
        exp_lr = initial_lr * (0.95 ** epoch)
        
        # Cosine annealing
        cosine_lr = initial_lr * 0.5 * (1 + np.cos(np.pi * epoch / epochs))
        
        print(f"{epoch:>5} {step_lr:>8.4f} {exp_lr:>8.4f} {cosine_lr:>8.4f}")
    
    print(f"\n学习率策略选择:")
    print(f"  Step Decay: 简单有效，CV 任务常用")
    print(f"  Exponential: 平滑衰减，适合稳定训练")
    print(f"  Cosine: 目前最流行，Transformer 训练首选")
    print(f"  Warmup: 训练初期线性增长到峰值，再衰减")

lr_schedules()

import torch
import torch.nn as nn
import torch.optim as optim

# 定义一个简单的两层神经网络
class SimpleNet(nn.Module):
    def __init__(self, input_dim=4, hidden_dim=8, num_classes=3):
        super().__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.relu = nn.ReLU()
        self.fc2 = nn.Linear(hidden_dim, num_classes)
    
    def forward(self, x):
        x = self.fc1(x)
        x = self.relu(x)
        x = self.fc2(x)
        return x

# 创建模型、损失函数、优化器
model = SimpleNet()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.01)

# 模拟训练
torch.manual_seed(42)
X = torch.randn(32, 4)
y = torch.randint(0, 3, (32,))

print("训练前参数示例:")
print(f"  fc1.weight 均值: {model.fc1.weight.data.mean():.6f}")

for epoch in range(100):
    optimizer.zero_grad()       # 清除旧梯度
    output = model(X)           # 前向传播
    loss = criterion(output, y) # 计算损失
    loss.backward()             # 反向传播
    optimizer.step()            # 更新参数
    
    if (epoch + 1) % 20 == 0:
        pred = output.argmax(dim=1)
        acc = (pred == y).float().mean().item()
        print(f"Epoch {epoch+1:>3}: loss={loss.item():.4f}, acc={acc:.2%}")

print(f"\n训练后参数示例:")
print(f"  fc1.weight 均值: {model.fc1.weight.data.mean():.6f}")
print(f"  fc1.weight 梯度均值: {model.fc1.weight.grad.mean():.6f}")

# 理解 autograd 的计算图和梯度累积
import torch

def autograd_internals():
    """深入理解 PyTorch autograd 的机制"""
    
    # 1. requires_grad 的作用
    a = torch.tensor(2.0, requires_grad=True)
    b = torch.tensor(3.0)  # 不需要梯度
    c = a * b + 1  # c 需要梯度（因为 a 需要）
    c.backward()
    print(f"1. a.grad = {a.grad.item()}  (d/da (2a*3+1) = 3)")
    
    # 2. 梯度累积（不清零会叠加）
    x = torch.tensor(1.0, requires_grad=True)
    y = x ** 2
    y.backward()
    print(f"2. 第一次 backward: x.grad = {x.grad.item()}")
    y = x ** 2
    y.backward()
    print(f"   第二次 backward: x.grad = {x.grad.item()} (累积了!)")
    
    # 3. 无梯度上下文（推理时节省内存）
    model = nn.Linear(10, 5)
    with torch.no_grad():
        out = model(torch.randn(1, 10))
    print(f"3. no_grad 模式下输出.requires_grad = {out.requires_grad}")
    
    # 4. 查看计算图
    x = torch.tensor(2.0, requires_grad=True)
    y = torch.tensor(3.0, requires_grad=True)
    z = x * y + x ** 2
    print(f"4. z 的创建函数: {z.grad_fn}")
    print(f"   z 的输入: {z.grad_fn.next_functions}")
    
    # 5. detach: 从计算图中分离
    w = z.detach()
    print(f"5. z.detach().requires_grad = {w.requires_grad}")

autograd_internals()

# 梯度累积：用有限显存训练大 batch
import torch
import torch.nn as nn

def gradient_accumulation_demo():
    """
    当显存不够放一个大 batch 时，可以用梯度累积模拟
    将 batch=32 分成 4 个 sub-batch=8，累积 4 次梯度后更新一次
    """
    torch.manual_seed(42)
    
    model = nn.Linear(10, 2)
    criterion = nn.MSELoss()
    optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
    
    X = torch.randn(32, 10)
    y = torch.randn(32, 2)
    
    # 方法 1: 直接大 batch
    optimizer.zero_grad()
    output1 = model(X)
    loss1 = criterion(output1, y)
    loss1.backward()
    w1_grad = model.weight.grad.clone()
    
    # 方法 2: 梯度累积（4 个 sub-batch）
    model.zero_grad()
    accum_steps = 4
    sub_batch = 32 // accum_steps
    for i in range(accum_steps):
        X_sub = X[i*sub_batch:(i+1)*sub_batch]
        y_sub = y[i*sub_batch:(i+1)*sub_batch]
        output = model(X_sub)
        loss = criterion(output, y_sub) / accum_steps  # 重要：除以累积步数
        loss.backward()
    
    w2_grad = model.weight.grad.clone()
    
    print("梯度累积验证:")
    print(f"  大 batch 梯度均值: {w1_grad.mean():.6f}")
    print(f"  累积梯度均值: {w2_grad.mean():.6f}")
    print(f"  差异: {abs(w1_grad.mean() - w2_grad.mean()):.8f}")
    print(f"  {'✅ 梯度累积正确' if abs(w1_grad.mean() - w2_grad.mean()) < 1e-6 else '❌ 有差异'}")

gradient_accumulation_demo()

import torch
import torch.nn as nn

def debug_training():
    """系统化的训练调试工具"""
    torch.manual_seed(42)
    
    model = nn.Sequential(
        nn.Linear(10, 20),
        nn.ReLU(),
        nn.Linear(20, 5)
    )
    criterion = nn.MSELoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
    
    # 调试 1: 小数据过拟合测试
    X_small = torch.randn(5, 10)
    y_small = torch.randn(5, 5)
    
    print("=== 调试 1: 小数据过拟合测试 ===")
    for epoch in range(200):
        optimizer.zero_grad()
        output = model(X_small)
        loss = criterion(output, y_small)
        loss.backward()
        optimizer.step()
        if (epoch + 1) % 50 == 0:
            print(f"  Epoch {epoch+1}: loss = {loss.item():.6f}")
    
    if loss.item() < 0.01:
        print("  ✅ 能过拟合小数据，代码框架正确")
    else:
        print("  ❌ 无法过拟合小数据，检查代码 bug")
    
    # 调试 2: 梯度监控
    print("\n=== 调试 2: 梯度监控 ===")
    X = torch.randn(32, 10)
    y = torch.randn(32, 5)
    optimizer.zero_grad()
    output = model(X)
    loss = criterion(output, y)
    loss.backward()
    
    for name, param in model.named_parameters():
        if param.grad is not None:
            g = param.grad
            print(f"  {name:>20}: mean={g.mean():.6f}, "
                  f"std={g.std():.6f}, max={g.abs().max():.6f}")
        else:
            print(f"  {name:>20}: ⚠️ 梯度为 None (requires_grad={param.requires_grad})")

debug_training()

# 常见反向传播 bug 排查清单
import torch
import torch.nn as nn

def common_backward_bugs():
    """演示和排查常见的反向传播 bug"""
    
    print("常见反向传播 Bug 清单:")
    print()
    
    # Bug 1: 忘记 requires_grad
    x = torch.tensor([1.0, 2.0])  # 默认 requires_grad=False
    w = torch.tensor([0.5, 0.5], requires_grad=True)
    y = (x * w).sum()
    y.backward()
    print(f"Bug 1: x.grad = {x.grad} (应为 None, 因为 x 没有 requires_grad)")
    print(f"       w.grad = {w.grad} ✅")
    print()
    
    # Bug 2: 在训练循环中忘记 zero_grad
    model = nn.Linear(2, 1)
    opt = torch.optim.SGD(model.parameters(), lr=0.01)
    x = torch.randn(1, 2)
    y = torch.randn(1, 1)
    
    opt.zero_grad()
    out1 = model(x)
    l1 = ((out1 - y) ** 2).mean()
    l1.backward()
    grad1 = model.weight.grad.clone()
    
    # 不 zero_grad 直接再来一次
    out2 = model(x)
    l2 = ((out2 - y) ** 2).mean()
    l2.backward()
    grad2 = model.weight.grad.clone()
    
    print(f"Bug 2: 第一次梯度 = {grad1[0,0]:.6f}")
    print(f"       累积后梯度 = {grad2[0,0]:.6f} (翻倍了!)")
    print(f"       修复: 每次 backward 前调用 optimizer.zero_grad()")
    print()
    
    # Bug 3: in-place 操作破坏计算图
    print("Bug 3: In-place 操作")
    x = torch.tensor([1.0], requires_grad=True)
    y = x * 2
    try:
        y.add_(1.0)  # in-place 操作
        z = y.sum()
        z.backward()
    except RuntimeError as e:
        print(f"  ❌ {str(e)[:60]}...")
        print(f"  修复: 使用 y = y + 1.0 代替 y.add_(1.0)")

common_backward_bugs()