随机森林：Bagging 与特征重要性

import numpy as np
from sklearn.datasets import make_classification
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

X, y = make_classification(
    n_samples=1000, n_features=20, n_informative=10, random_state=42
)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# 单棵决策树
single_tree = DecisionTreeClassifier(random_state=42)
single_tree.fit(X_train, y_train)
print(f"单棵决策树准确率: {accuracy_score(y_test, single_tree.predict(X_test)):.4f}")

# 手动实现随机森林（简化版）
class SimpleRandomForest:
    def __init__(self, n_trees=100, max_depth=10, max_features='sqrt', random_state=None):
        self.n_trees = n_trees
        self.max_depth = max_depth
        self.max_features = max_features
        self.trees = []
        self.rng = np.random.RandomState(random_state)

    def fit(self, X, y):
        n_samples = X.shape[0]
        for _ in range(self.n_trees):
            # Bagging: 有放回采样
            indices = self.rng.choice(n_samples, n_samples, replace=True)
            X_boot, y_boot = X[indices], y[indices]

            tree = DecisionTreeClassifier(
                max_depth=self.max_depth,
                max_features=self.max_features,
                random_state=self.rng.randint(0, 10000)
            )
            tree.fit(X_boot, y_boot)
            self.trees.append(tree)

    def predict(self, X):
        predictions = np.array([tree.predict(X) for tree in self.trees])
        # 投票：选择得票最多的类别
        return np.array([
            np.bincount(predictions[:, i]).argmax()
            for i in range(X.shape[0])
        ])

my_rf = SimpleRandomForest(n_trees=100, max_depth=10, random_state=42)
my_rf.fit(X_train, y_train)
print(f"随机森林准确率: {accuracy_score(y_test, my_rf.predict(X_test)):.4f}")

import numpy as np
import matplotlib.pyplot as plt

# 演示 Bagging 降低方差的原理
np.random.seed(42)
n_datasets = 50
n_samples = 100

# 生成带噪声的数据
true_func = lambda x: np.sin(2 * x)
X = np.random.uniform(-3, 3, (n_datasets, n_samples))
y = true_func(X) + np.random.normal(0, 0.5, (n_datasets, n_samples))

# 用不同深度的决策树拟合
from sklearn.tree import DecisionTreeRegressor

x_test = np.linspace(-3, 3, 200).reshape(-1, 1)
single_preds = []
bagged_preds = []

for i in range(n_datasets):
    tree = DecisionTreeRegressor(max_depth=5, random_state=i)
    tree.fit(X[i].reshape(-1, 1), y[i])
    single_preds.append(tree.predict(x_test))

single_preds = np.array(single_preds)
bagged_preds = single_preds.mean(axis=0)

plt.figure(figsize=(12, 5))
plt.subplot(1, 2, 1)
for i in range(10):
    plt.plot(x_test, single_preds[i], alpha=0.3, color='blue')
plt.plot(x_test, true_func(x_test), 'r-', linewidth=2, label='真实函数')
plt.title('单棵决策树（高方差）')

plt.subplot(1, 2, 2)
plt.plot(x_test, bagged_preds, 'g-', linewidth=3, label='Bagging 平均')
plt.plot(x_test, true_func(x_test), 'r--', linewidth=2, label='真实函数')
plt.title('Bagging 平均（低方差）')
plt.tight_layout()
plt.show()

# 探索 max_features 对随机森林性能的影响
import numpy as np
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score
from sklearn.datasets import make_classification

X, y = make_classification(
    n_samples=2000, n_features=50, n_informative=15,
    n_redundant=10, random_state=42
)

max_features_options = ['sqrt', 'log2', 0.3, 0.5, 0.8, 1.0]

results = []
for mf in max_features_options:
    rf = RandomForestClassifier(
        n_estimators=100, max_features=mf, random_state=42, n_jobs=-1
    )
    scores = cross_val_score(rf, X, y, cv=5, scoring='accuracy')
    results.append({
        'max_features': mf,
        'mean': scores.mean(),
        'std': scores.std()
    })

print(f"{'max_features':>12} | {'准确率':>8} | {'标准差':>8}")
print("-" * 35)
for r in results:
    mf_str = str(r['max_features'])
    print(f"{mf_str:>12} | {r['mean']:>8.4f} | {r['std']:>8.4f}")

from sklearn.ensemble import RandomForestClassifier
from sklearn.inspection import permutation_importance
from sklearn.datasets import make_classification
import matplotlib.pyplot as plt
import numpy as np

# 生成数据（10 个信息特征 + 10 个噪声特征）
X, y = make_classification(
    n_samples=1000, n_features=20, n_informative=10,
    n_redundant=0, n_repeated=0, random_state=42
)

rf = RandomForestClassifier(n_estimators=100, random_state=42)
rf.fit(X, y)

# 方法 1: Gini 重要性
gini_importances = rf.feature_importances_

# 方法 2: 排列重要性
perm_importance = permutation_importance(
    rf, X, y, n_repeats=10, random_state=42, n_jobs=-1
)

# 可视化对比
fig, axes = plt.subplots(1, 2, figsize=(14, 6))

indices = np.argsort(gini_importances)[-15:]
axes[0].barh(range(len(indices)), gini_importances[indices])
axes[0].set_yticks(range(len(indices)))
axes[0].set_yticklabels([f'Feature {i}' for i in indices])
axes[0].set_title('Gini 重要性')

perm_means = perm_importance.importances_mean
perm_indices = np.argsort(perm_means)[-15:]
axes[1].barh(range(len(perm_indices)), perm_means[perm_indices])
axes[1].set_yticks(range(len(perm_indices)))
axes[1].set_yticklabels([f'Feature {i}' for i in perm_indices])
axes[1].set_title('排列重要性')
plt.tight_layout()
plt.show()

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score
import numpy as np

# 使用 OOB 评估
rf_oob = RandomForestClassifier(
    n_estimators=100, oob_score=True, random_state=42
)
rf_oob.fit(X_train, y_train)

print(f"OOB 得分: {rf_oob.oob_score_:.4f}")
print(f"测试集准确率: {rf_oob.score(X_test, y_test):.4f}")

# 用树的数量做实验，观察 OOB 得分的收敛性
n_trees_range = range(10, 500, 10)
oob_scores = []

for n in n_trees_range:
    rf = RandomForestClassifier(n_estimators=n, oob_score=True, random_state=42)
    rf.fit(X_train, y_train)
    oob_scores.append(rf.oob_score_)

import matplotlib.pyplot as plt
plt.figure(figsize=(10, 5))
plt.plot(list(n_trees_range), oob_scores, 'b-', linewidth=2)
plt.axhline(y=rf_oob.oob_score_, color='r', linestyle='--', label=f'最终 OOB: {rf_oob.oob_score_:.4f}')
plt.xlabel('树的数量')
plt.ylabel('OOB 得分')
plt.title('OOB 得分随树数量的收敛')
plt.legend()
plt.grid(True, alpha=0.3)
plt.show()

from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
import time

X, y = make_classification(
    n_samples=5000, n_features=30, n_informative=15, random_state=42
)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

classifiers = {
    'Random Forest': RandomForestClassifier(n_estimators=100, random_state=42),
    'Gradient Boosting': GradientBoostingClassifier(
        n_estimators=100, max_depth=5, learning_rate=0.1, random_state=42
    ),
}

for name, clf in classifiers.items():
    start = time.time()
    clf.fit(X_train, y_train)
    train_time = time.time() - start

    start = time.time()
    acc = clf.score(X_test, y_test)
    pred_time = time.time() - start

    print(f"{name:25s} | 准确率: {acc:.4f} | 训练: {train_time:.3f}s | 预测: {pred_time:.3f}s")

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import classification_report

# 完整的随机森林建模流程
# 步骤 1: 用随机森林做特征选择
rf = RandomForestClassifier(n_estimators=200, random_state=42, n_jobs=-1)
rf.fit(X_train, y_train)

# 选择重要性在前 60% 的特征
threshold = np.percentile(rf.feature_importances_, 40)
important_features = np.where(rf.feature_importances_ >= threshold)[0]
print(f"选择了 {len(important_features)}/{X_train.shape[1]} 个特征")

# 步骤 2: 用精选特征重新训练
X_train_sel = X_train[:, important_features]
X_test_sel = X_test[:, important_features]

# 步骤 3: 超参数调优
param_grid = {
    'n_estimators': [100, 200, 300],
    'max_depth': [None, 10, 20, 30],
    'min_samples_split': [2, 5, 10],
    'min_samples_leaf': [1, 2, 4],
}

rf_grid = RandomForestClassifier(random_state=42, n_jobs=-1)
grid = GridSearchCV(rf_grid, param_grid, cv=3, scoring='accuracy', n_jobs=-1)
grid.fit(X_train_sel, y_train)

print(f"最佳参数: {grid.best_params_}")
print(f"交叉验证准确率: {grid.best_score_:.4f}")
print(f"测试集准确率: {grid.score(X_test_sel, y_test):.4f}")

# 详细报告
y_pred = grid.predict(X_test_sel)
print(classification_report(y_test, y_pred))