神经网络基础：前向传播、反向传播与激活函数

import numpy as np

# ===== 前向传播的 NumPy 实现 =====

def relu(z):
    """ReLU 激活函数"""
    return np.maximum(0, z)

def softmax(z):
    """Softmax 激活函数（数值稳定版本）"""
    exp_z = np.exp(z - np.max(z, axis=1, keepdims=True))
    return exp_z / np.sum(exp_z, axis=1, keepdims=True)

class SimpleMLP:
    """两层全连接神经网络"""
    def __init__(self, input_size, hidden_size, output_size):
        # Xavier 初始化
        self.W1 = np.random.randn(input_size, hidden_size) * np.sqrt(2.0 / input_size)
        self.b1 = np.zeros((1, hidden_size))
        self.W2 = np.random.randn(hidden_size, output_size) * np.sqrt(2.0 / hidden_size)
        self.b2 = np.zeros((1, output_size))
    
    def forward(self, X):
        """前向传播"""
        self.X = X
        self.z1 = X @ self.W1 + self.b1
        self.a1 = relu(self.z1)
        self.z2 = self.a1 @ self.W2 + self.b2
        self.a2 = softmax(self.z2)
        return self.a2
    
    def predict(self, X):
        """预测类别"""
        probs = self.forward(X)
        return np.argmax(probs, axis=1)

# 测试
np.random.seed(42)
X = np.random.randn(32, 4)  # batch_size=32, input_dim=4
net = SimpleMLP(input_size=4, hidden_size=16, output_size=3)
output = net.forward(X)
print(f"输出形状: {output.shape}")
print(f"前 5 个样本的预测类别: {net.predict(X[:5])}")
print(f"样本 0 各类别概率: {output[0]} (sum={output[0].sum():.4f})")

import numpy as np
import matplotlib.pyplot as plt

# ===== 激活函数可视化 =====
x = np.linspace(-5, 5, 500)

def sigmoid(x): return 1 / (1 + np.exp(-x))
def tanh(x): return np.tanh(x)
def relu(x): return np.maximum(0, x)
def leaky_relu(x, alpha=0.01): return np.where(x > 0, x, alpha * x)
def gelu(x): return 0.5 * x * (1 + np.tanh(np.sqrt(2/np.pi) * (x + 0.044715 * x**3)))
def silu(x): return x * sigmoid(x)

functions = {
    'Sigmoid': (sigmoid, '#e74c3c'),
    'Tanh': (tanh, '#3498db'),
    'ReLU': (relu, '#2ecc71'),
    'Leaky ReLU': (leaky_relu, '#9b59b6'),
    'GELU': (gelu, '#f39c12'),
    'SiLU': (silu, '#1abc9c'),
}

fig, axes = plt.subplots(2, 3, figsize=(15, 10))
for ax, (name, (func, color)) in zip(axes.ravel(), functions.items()):
    y = func(x)
    ax.plot(x, y, color=color, linewidth=2, label=name)
    ax.plot(x, np.gradient(y, x), '--', alpha=0.5, label='导数')
    ax.axhline(0, color='gray', linewidth=0.5)
    ax.axvline(0, color='gray', linewidth=0.5)
    ax.set_title(name, fontsize=12, fontweight='bold')
    ax.legend(fontsize=9)
    ax.grid(True, alpha=0.3)

plt.tight_layout()
plt.show()

import numpy as np
from sklearn.datasets import load_digits
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import warnings
warnings.filterwarnings('ignore')

# ===== 1. 加载并准备数据 =====
digits = load_digits()
X = digits.data / 16.0  # 归一化到 [0, 1]
y = digits.target  # 0-9 共 10 类

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# ===== 2. 神经网络实现 =====
class MLP:
    def __init__(self, layer_sizes, learning_rate=0.01, reg_lambda=0.001):
        self.lr = learning_rate
        self.reg_lambda = reg_lambda
        # 参数初始化 (He initialization for ReLU)
        self.weights = []
        self.biases = []
        for i in range(len(layer_sizes) - 1):
            w = np.random.randn(layer_sizes[i], layer_sizes[i+1]) * np.sqrt(2.0 / layer_sizes[i])
            b = np.zeros((1, layer_sizes[i+1]))
            self.weights.append(w)
            self.biases.append(b)
    
    def _relu(self, z):
        return np.maximum(0, z)
    
    def _relu_derivative(self, z):
        return (z > 0).astype(float)
    
    def _softmax(self, z):
        exp_z = np.exp(z - np.max(z, axis=1, keepdims=True))
        return exp_z / np.sum(exp_z, axis=1, keepdims=True)
    
    def _one_hot(self, y, num_classes):
        n = len(y)
        oh = np.zeros((n, num_classes))
        oh[np.arange(n), y] = 1
        return oh
    
    def forward(self, X):
        self.activations = [X]
        self.z_values = []
        a = X
        for i in range(len(self.weights) - 1):
            z = a @ self.weights[i] + self.biases[i]
            self.z_values.append(z)
            a = self._relu(z)
            self.activations.append(a)
        # 输出层 (Softmax)
        z = a @ self.weights[-1] + self.biases[-1]
        self.z_values.append(z)
        a = self._softmax(z)
        self.activations.append(a)
        return a
    
    def backward(self, y_onehot):
        m = y_onehot.shape[0]
        # 输出层梯度
        delta = (self.activations[-1] - y_onehot) / m
        
        d_weights = []
        d_biases = []
        for i in range(len(self.weights) - 1, 0, -1):
            dW = self.activations[i].T @ delta + self.reg_lambda * self.weights[i]
            db = np.sum(delta, axis=0, keepdims=True)
            d_weights.insert(0, dW)
            d_biases.insert(0, db)
            delta = (delta @ self.weights[i].T) * self._relu_derivative(self.z_values[i-1])
        # 第一层
        dW = self.activations[0].T @ delta + self.reg_lambda * self.weights[0]
        db = np.sum(delta, axis=0, keepdims=True)
        d_weights.insert(0, dW)
        d_biases.insert(0, db)
        return d_weights, d_biases
    
    def update(self, d_weights, d_biases):
        for i in range(len(self.weights)):
            self.weights[i] -= self.lr * d_weights[i]
            self.biases[i] -= self.lr * d_biases[i]
    
    def train(self, X, y, epochs=500, batch_size=64, verbose=True):
        num_classes = len(np.unique(y))
        y_onehot = self._one_hot(y, num_classes)
        n = X.shape[0]
        
        for epoch in range(epochs):
            # Mini-batch
            indices = np.random.permutation(n)
            X_shuffled = X[indices]
            y_shuffled = y_onehot[indices]
            
            for start in range(0, n, batch_size):
                X_batch = X_shuffled[start:start+batch_size]
                y_batch = y_shuffled[start:start+batch_size]
                
                self.forward(X_batch)
                dW, db = self.backward(y_batch)
                self.update(dW, db)
            
            if verbose and (epoch + 1) % 100 == 0:
                output = self.forward(X)
                loss = -np.mean(np.sum(y_onehot * np.log(output + 1e-8), axis=1))
                acc = np.mean(np.argmax(output, axis=1) == y)
                print(f"Epoch {epoch+1:4d} | Loss: {loss:.4f} | Train Acc: {acc:.4f}")
    
    def predict(self, X):
        output = self.forward(X)
        return np.argmax(output, axis=1)

# ===== 3. 训练与评估 =====
np.random.seed(42)
model = MLP(layer_sizes=[64, 128, 64, 10], learning_rate=0.01, reg_lambda=0.001)
model.train(X_train, y_train, epochs=500, batch_size=64, verbose=True)

# 测试集评估
y_pred = model.predict(X_test)
test_acc = np.mean(y_pred == y_test)
print(f"\n测试集准确率: {test_acc:.4f}")

# 混淆矩阵
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test, y_pred)
print("\n混淆矩阵 (行=真实, 列=预测):")
print(cm)