决策树与随机森林实战

import numpy as np

def entropy(labels):
    """计算熵: H(S) = -Σ p_i log_2(p_i)"""
    _, counts = np.unique(labels, return_counts=True)
    probs = counts / len(labels)
    return -np.sum(probs * np.log2(probs + 1e-10))

def gini_impurity(labels):
    """计算基尼系数: G(S) = 1 - Σ p_i²"""
    _, counts = np.unique(labels, return_counts=True)
    probs = counts / len(labels)
    return 1 - np.sum(probs ** 2)

# 示例：二分类问题
labels_pure = np.array([1, 1, 1, 1, 1])
labels_mixed = np.array([1, 1, 0, 0, 0])
labels_balanced = np.array([1, 1, 1, 0, 0, 0])

print("=== 纯度对比 ===")
print(f"完全纯净  - Entropy: {entropy(labels_pure):.4f}, Gini: {gini_impurity(labels_pure):.4f}")
print(f"3:2 混合  - Entropy: {entropy(labels_mixed):.4f}, Gini: {gini_impurity(labels_mixed):.4f}")
print(f"3:3 均衡  - Entropy: {entropy(labels_balanced):.4f}, Gini: {gini_impurity(labels_balanced):.4f}")

class DecisionNode:
    """决策树节点"""
    def __init__(self, feature=None, threshold=None,
                 left=None, right=None, value=None):
        self.feature = feature
        self.threshold = threshold
        self.left = left
        self.right = right
        self.value = value  # 叶子节点的预测值

class DecisionTreeClassifier:
    """简化版 CART 分类树"""

    def __init__(self, max_depth=5, min_samples_split=2):
        self.max_depth = max_depth
        self.min_samples_split = min_samples_split
        self.root = None

    def _gini(self, y):
        if len(y) == 0:
            return 0
        probs = np.bincount(y) / len(y)
        return 1 - np.sum(probs ** 2)

    def _best_split(self, X, y):
        best_gain = -1
        best_feature = None
        best_threshold = None
        n_samples, n_features = X.shape
        parent_gini = self._gini(y)

        for feature in range(n_features):
            thresholds = np.unique(X[:, feature])
            for threshold in thresholds:
                left_mask = X[:, feature] <= threshold
                right_mask = ~left_mask
                if left_mask.sum() == 0 or right_mask.sum() == 0:
                    continue
                n_left = left_mask.sum()
                n_right = right_mask.sum()
                weighted_gini = (n_left * self._gini(y[left_mask]) +
                                 n_right * self._gini(y[right_mask])) / n_samples
                gain = parent_gini - weighted_gini
                if gain > best_gain:
                    best_gain = gain
                    best_feature = feature
                    best_threshold = threshold

        return best_feature, best_threshold, best_gain

    def _build_tree(self, X, y, depth):
        if (depth >= self.max_depth or
            len(y) < self.min_samples_split or
            self._gini(y) == 0):
            return DecisionNode(value=np.bincount(y).argmax())

        feature, threshold, gain = self._best_split(X, y)
        if feature is None or gain <= 0:
            return DecisionNode(value=np.bincount(y).argmax())

        left_mask = X[:, feature] <= threshold
        left = self._build_tree(X[left_mask], y[left_mask], depth + 1)
        right = self._build_tree(X[~left_mask], y[~left_mask], depth + 1)
        return DecisionNode(feature, threshold, left, right)

    def fit(self, X, y):
        self.root = self._build_tree(X, y, depth=0)
        return self

    def _predict_one(self, x, node):
        if node.value is not None:
            return node.value
        if x[node.feature] <= node.threshold:
            return self._predict_one(x, node.left)
        return self._predict_one(x, node.right)

    def predict(self, X):
        return np.array([self._predict_one(x, self.root) for x in X])

from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split

X, y = make_classification(n_samples=1000, n_features=20,
                           n_informative=10, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

rf = RandomForestClassifier(
    n_estimators=100,        # 树的数量
    max_depth=10,            # 每棵树的最大深度
    min_samples_split=5,     # 最小分裂样本数
    min_samples_leaf=2,      # 叶子最小样本数
    max_features="sqrt",     # 每步考虑的特征数
    random_state=42,
    n_jobs=-1                # 并行训练
)
rf.fit(X_train, y_train)

print(f"训练集准确率: {rf.score(X_train, y_train):.4f}")
print(f"测试集准确率: {rf.score(X_test, y_test):.4f}")

# 特征重要性
importances = rf.feature_importances_
top_features = np.argsort(importances)[::-1][:5]
print("Top 5 重要特征:")
for i, feat_idx in enumerate(top_features):
    print(f"  特征 {feat_idx}: {importances[feat_idx]:.4f}")

from sklearn.ensemble import RandomForestClassifier
from sklearn.inspection import permutation_importance
from sklearn.datasets import make_classification

X, y = make_classification(n_samples=1000, n_features=20,
                           n_informative=10, n_redundant=5,
                           random_state=42)

rf = RandomForestClassifier(n_estimators=100, random_state=42)
rf.fit(X, y)

# 方法一：基于不纯度的特征重要性
impurity_imp = rf.feature_importances_
print("=== 不纯度重要性 (Top 5) ===")
for i in np.argsort(impurity_imp)[::-1][:5]:
    print(f"  特征 {i:2d}: {impurity_imp[i]:.4f}")

# 方法二：排列重要性（更可靠）
result = permutation_importance(rf, X, y, n_repeats=10,
                                random_state=42, n_jobs=-1)
perm_imp = result.importances_mean
print("\n=== 排列重要性 (Top 5) ===")
for i in np.argsort(perm_imp)[::-1][:5]:
    print(f"  特征 {i:2d}: {perm_imp[i]:.4f} "
          f"(±{result.importances_std[i]:.4f})")