信息论基础

import numpy as np

def self_information(prob: float, base: float = 2) -> float:
    """计算单个事件的自信息"""
    if prob <= 0 or prob > 1:
        raise ValueError("Probability must be in (0, 1]")
    return -np.log(prob) / np.log(base)

print(f"公平硬币正面: {self_information(0.5):.3f} bits")
print(f"骰子出现3点: {self_information(1/6):.3f} bits")

# 独立事件信息量的可加性验证
def joint_information(p1: float, p2: float, base: float = 2) -> tuple:
    i1 = self_information(p1, base)
    i2 = self_information(p2, base)
    i_joint = self_information(p1 * p2, base)
    return i1, i2, i_joint, i1 + i2

i1, i2, i_joint, i_sum = joint_information(0.5, 0.25)
print(f"I(x)+I(y)={i_sum:.3f}, I(x,y)={i_joint:.3f}")
assert abs(i_sum - i_joint) < 1e-10  # 可加性成立

def entropy(probs: list[float], base: float = 2) -> float:
    """计算离散分布的信息熵"""
    probs = np.array(probs)
    assert np.isclose(probs.sum(), 1.0), "Probabilities must sum to 1"
    probs = probs[probs > 0]  # 过滤零概率
    return -np.sum(probs * np.log(probs) / np.log(base))

# 伯努利分布的熵
print(f"p=0.5: H={entropy([0.5, 0.5]):.3f} bits")
print(f"p=0.9: H={entropy([0.9, 0.1]):.3f} bits")
print(f"p=1.0: H={entropy([1.0, 0.0]):.3f} bits")

# 验证: 均匀分布的熵最大
def max_entropy_check(n_categories: int) -> float:
    uniform = [1.0 / n_categories] * n_categories
    skewed = [0.9] + [0.1 / (n_categories - 1)] * (n_categories - 1)
    return entropy(uniform), entropy(skewed)

for n in [2, 4, 10, 100]:
    h_uni, h_ske = max_entropy_check(n)
    print(f"n={n}: 均匀={h_uni:.3f}, 偏斜={h_ske:.3f}, 理论=log2({n})={np.log2(n):.3f}")

def cross_entropy(p_true: list[float], q_pred: list[float]) -> float:
    """计算两个分布之间的交叉熵"""
    p = np.array(p_true)
    q = np.array(q_pred)
    q = np.clip(q, 1e-15, 1.0)  # 数值稳定
    return -np.sum(p * np.log(q))

# 真实分布 vs 预测分布
p_true = [0.7, 0.2, 0.1]
q_good = [0.65, 0.25, 0.10]  # 接近真实
q_bad  = [0.2, 0.3, 0.5]     # 偏离真实
print(f"H(P,Q_good)={cross_entropy(p_true, q_good):.4f}")
print(f"H(P,Q_bad)={cross_entropy(p_true, q_bad):.4f}")

# 二分类交叉熵损失
def binary_cross_entropy(y_true: float, y_pred: float) -> float:
    eps = 1e-15
    y_pred = np.clip(y_pred, eps, 1 - eps)
    return -(y_true * np.log(y_pred) + (1 - y_true) * np.log(1 - y_pred))

# 不同预测质量的损失
for pred in [0.5, 0.7, 0.9, 0.99]:
    loss = binary_cross_entropy(1.0, pred)
    print(f"y=1, p={pred:.2f}, BCE={loss:.4f}")

def kl_divergence(p: list[float], q: list[float]) -> float:
    """计算 KL 散度 D_KL(P || Q)"""
    p = np.array(p)
    q = np.clip(np.array(q), 1e-15, 1.0)
    return np.sum(p * np.log(p / q))

p = [0.5, 0.3, 0.2]
q1 = [0.5, 0.3, 0.2]  # 相同
q2 = [0.3, 0.4, 0.3]  # 不同
print(f"D_KL(P||P) = {kl_divergence(p, p):.6f}")
print(f"D_KL(P||Q) = {kl_divergence(p, q2):.6f}")
print(f"D_KL(Q||P) = {kl_divergence(q2, p):.6f}")

# JS 散度——对称化的分布差异度量
def js_divergence(p: list[float], q: list[float]) -> float:
    p, q = np.array(p), np.array(q)
    m = 0.5 * (p + q)
    return 0.5 * kl_divergence(p, m) + 0.5 * kl_divergence(q, m)

# 比较 KL 散度的非对称性与 JS 散度的对称性
print(f"KL(P||Q)={kl_divergence(p,q2):.4f}, KL(Q||P)={kl_divergence(q2,p):.4f}")
print(f"JS(P||Q)={js_divergence(p,q2):.4f} (对称)")

def mutual_information(joint: np.ndarray) -> float:
    """从联合分布计算互信息"""
    joint = joint / joint.sum()  # 归一化
    p_x = joint.sum(axis=1)
    p_y = joint.sum(axis=0)
    outer = np.outer(p_x, p_y)
    mi = 0.0
    for i in range(joint.shape[0]):
        for j in range(joint.shape[1]):
            if joint[i,j] > 0 and outer[i,j] > 0:
                mi += joint[i,j] * np.log2(joint[i,j] / outer[i,j])
    return mi

joint = np.array([[0.4, 0.1], [0.1, 0.4]])
print(f"I(X;Y) = {mutual_information(joint):.4f} bits")

# 互信息与熵的关系验证
def conditional_entropy(joint: np.ndarray, axis: int = 1) -> float:
    joint = joint / joint.sum()
    marginal = joint.sum(axis=axis, keepdims=True)
    cond = np.where(joint > 0, joint * np.log2(joint / marginal), 0)
    return -cond.sum()

joint = np.array([[0.4, 0.1], [0.1, 0.4]])
h_x = entropy(joint.sum(axis=1))
h_xy = entropy(joint.flatten())
h_y_given_x = conditional_entropy(joint)
print(f"H(X)={h_x:.3f}, H(X,Y)={h_xy:.3f}")
print(f"I(X;Y)=H(X)-H(X|Y)={h_x - h_y_given_x:.3f}")

def softmax_cross_entropy(logits: np.ndarray, labels: np.ndarray) -> tuple:
    """Softmax + 交叉熵损失，返回损失和梯度"""
    exp_logits = np.exp(logits - np.max(logits))  # 数值稳定
    probs = exp_logits / exp_logits.sum()
    loss = -np.sum(labels * np.log(probs + 1e-15))
    grad = probs - labels  # 简洁的梯度形式
    return loss, grad

logits = np.array([2.0, 1.0, 0.1])
labels = np.array([1.0, 0.0, 0.0])
loss, grad = softmax_cross_entropy(logits, labels)
print(f"Loss: {loss:.4f}")
print(f"Gradient: {grad}")

# 标签平滑实现
def label_smoothing(labels: np.ndarray, alpha: float = 0.1) -> np.ndarray:
    """将独热编码标签平滑化"""
    k = len(labels)
    return (1 - alpha) * labels + alpha / k

original = np.array([1.0, 0.0, 0.0])
smoothed = label_smoothing(original, alpha=0.1)
print(f"原始: {original}")
print(f"平滑: {smoothed}")
print(f"平滑后交叉熵: {cross_entropy(smoothed, [0.8, 0.1, 0.1]):.4f}")

from collections import Counter

def calc_entropy(labels: list) -> float:
    """计算数据集的经验熵"""
    total = len(labels)
    counts = Counter(labels)
    probs = [c / total for c in counts.values()]
    return entropy(probs)

def information_gain(labels: list, feature: list) -> float:
    """计算特征的信息增益"""
    h_parent = calc_entropy(labels)
    total = len(labels)
    feature_vals = set(feature)
    h_children = 0.0
    for val in feature_vals:
        subset = [labels[i] for i in range(total) if feature[i] == val]
        h_children += (len(subset) / total) * calc_entropy(subset)
    return h_parent - h_children

# 示例数据集
labels =   ["Y","Y","Y","N","N","Y","Y","N","N","N"]
feature1 = ["A","A","A","A","B","B","B","B","B","B"]
print(f"IG(feature1) = {information_gain(labels, feature1):.4f}")

# 信息增益 vs 信息增益比
def gain_ratio(labels: list, feature: list) -> float:
    ig = information_gain(labels, feature)
    split_info = calc_entropy(feature)  # 特征本身的熵
    return ig / split_info if split_info > 0 else 0.0

# 比较多值特征 vs 少值特征
high_card = [f"x{i}" for i in range(10)]  # 10个唯一值
binary_feat = ["A","A","A","A","A","B","B","B","B","B"]

print(f"高基数: IG={information_gain(labels, high_card):.4f}, GR={gain_ratio(labels, high_card):.4f}")
print(f"二值: IG={information_gain(labels, binary_feat):.4f}, GR={gain_ratio(labels, binary_feat):.4f}")