XGBoost 原理与调参指南

import xgboost as xgb
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

# 加载数据
data = load_breast_cancer()
X_train, X_test, y_train, y_test = train_test_split(
    data.data, data.target, test_size=0.2, random_state=42
)

# 最简配置即可使用
model = xgb.XGBClassifier(
    n_estimators=100,
    learning_rate=0.1,
    max_depth=3,
    random_state=42
)
model.fit(X_train, y_train)

preds = model.predict(X_test)
print(f"Accuracy: {accuracy_score(y_test, preds):.4f}")

import numpy as np

def xgb_optimal_leaf_weights(g, h, lambd=1.0):
    """
    计算 XGBoost 最优叶子权重: w* = -G / (H + lambda)
    
    g: 一阶梯度数组 (G_j = sumg_i)
    h: 二阶梯度数组 (H_j = sumh_i)
    lambd: L2 正则化系数
    """
    G = np.sum(g)
    H = np.sum(h)
    w_star = -G / (H + lambd)
    
    # 最优目标函数值: -1/2 * G^2/(H+lambda)
    obj_star = -0.5 * G**2 / (H + lambd)
    
    return w_star, obj_star

# 示例：二元分类，使用对数损失
# 假设某个叶子节点包含 5 个样本
y_true = np.array([1, 1, 0, 1, 0])
y_pred_prev = np.array([0.3, 0.6, 0.4, 0.7, 0.2])  # 上一轮预测

# 对数损失的梯度和 Hessian
g = y_pred_prev - y_true          # 一阶导数
h = y_pred_prev * (1 - y_pred_prev)  # 二阶导数

w, obj = xgb_optimal_leaf_weights(g, h, lambd=1.0)
print(f"最优叶子权重 w* = {w:.4f}")
print(f"该节点的最优目标值 Obj* = {obj:.4f}")

import numpy as np

def calculate_split_gain(G_left, H_left, G_right, H_right, G_total, H_total, lambd=1.0, gamma=0.0):
    """
    计算分裂增益:
    Gain = 1/2 * [G_L^2/(H_L+lambda) + G_R^2/(H_R+lambda) - (G_L+G_R)^2/(H_L+H_R+lambda)] - gamma
    """
    gain_left = G_left**2 / (H_left + lambd)
    gain_right = G_right**2 / (H_right + lambd)
    gain_parent = (G_left + G_right)**2 / (H_left + H_right + lambd)
    return 0.5 * (gain_left + gain_right - gain_parent) - gamma

# 模拟精确贪心算法：扫描所有候选分裂点
def find_best_split_exact(X, g, h, feature_idx, lambd=1.0, gamma=0.0):
    """精确贪心算法：尝试特征的所有唯一值"""
    sorted_indices = np.argsort(X[:, feature_idx])
    g_sorted = g[sorted_indices]
    h_sorted = h[sorted_indices]
    
    G_total = np.sum(g)
    H_total = np.sum(h)
    
    best_gain = -np.inf
    best_threshold = None
    
    G_left, H_left = 0.0, 0.0
    unique_vals = np.unique(X[:, feature_idx])
    
    for val in unique_vals[:-1]:  # 最后一个值不能作为分裂点
        mask = X[:, feature_idx] == val
        G_left += np.sum(g_sorted[:np.sum(X[:, feature_idx] <= val)])
        H_left += np.sum(h_sorted[:np.sum(X[:, feature_idx] <= val)])
        
        G_right = G_total - G_left
        H_right = H_total - H_left
        
        gain = calculate_split_gain(G_left, H_left, G_right, H_right,
                                     G_total, H_total, lambd, gamma)
        if gain > best_gain:
            best_gain = gain
            best_threshold = val
    
    return best_threshold, best_gain

print("精确贪心算法会扫描所有唯一值作为候选分裂点")
print("当数据量很大时，改用近似算法（分位数草图）")

import xgboost as xgb
import numpy as np
from sklearn.model_selection import cross_val_score

# 对比不同正则化强度的效果
params_grid = {
    "无正则化": {"gamma": 0, "reg_alpha": 0, "reg_lambda": 1, "max_depth": 6},
    "轻度正则化": {"gamma": 0.1, "reg_alpha": 0.01, "reg_lambda": 1, "max_depth": 5},
    "中度正则化": {"gamma": 0.5, "reg_alpha": 0.1, "reg_lambda": 5, "max_depth": 4},
    "强度正则化": {"gamma": 1.0, "reg_alpha": 1.0, "reg_lambda": 10, "max_depth": 3},
}

for name, params in params_grid.items():
    model = xgb.XGBClassifier(
        n_estimators=200,
        learning_rate=0.05,
        **params,
        random_state=42,
        use_label_encoder=False,
        eval_metric='logloss'
    )
    scores = cross_val_score(model, X_train, y_train, cv=5, scoring='accuracy')
    print(f"{name}: {scores.mean():.4f} (+/- {scores.std():.4f})")

import xgboost as xgb
from sklearn.model_selection import GridSearchCV, train_test_split

# 第一阶段：确定树结构参数
param_grid_stage1 = {
    'max_depth': [3, 5, 7],
    'min_child_weight': [1, 3, 5],
    'subsample': [0.7, 0.8, 1.0],
    'colsample_bytree': [0.7, 0.8, 1.0],
}

model = xgb.XGBClassifier(
    n_estimators=200,
    learning_rate=0.1,
    gamma=0,
    reg_alpha=0,
    reg_lambda=1,
    random_state=42,
    use_label_encoder=False,
    eval_metric='logloss'
)

grid = GridSearchCV(
    model, param_grid_stage1,
    cv=3, scoring='accuracy',
    n_jobs=-1, verbose=1
)
grid.fit(X_train, y_train)
print(f"最佳参数: {grid.best_params_}")
print(f"最佳分数: {grid.best_score_:.4f}")

# 第二阶段：精细调优（学习率 + 树数量）
best_model = xgb.XGBClassifier(
    **grid.best_params_,
    learning_rate=0.01,
    n_estimators=1000,
    random_state=42,
    use_label_encoder=False,
    eval_metric='logloss'
)

# 使用早停防止过拟合
best_model.fit(
    X_train, y_train,
    eval_set=[(X_test, y_test)],
    early_stopping_rounds=50,
    verbose=False
)
print(f"最优迭代轮数: {best_model.best_iteration}")
print(f"测试集准确率: {best_model.score(X_test, y_test):.4f}")

# 三个框架的 API 对比

# XGBoost
import xgboost as xgb
xgb_model = xgb.XGBClassifier(
    n_estimators=100,
    learning_rate=0.1,
    max_depth=5,
    random_state=42
)

# LightGBM
import lightgbm as lgb
lgb_model = lgb.LGBMClassifier(
    n_estimators=100,
    learning_rate=0.1,
    max_depth=5,
    random_state=42
)

# CatBoost（注意类别特征处理）
from catboost import CatBoostClassifier
cat_model = CatBoostClassifier(
    iterations=100,
    learning_rate=0.1,
    depth=5,
    cat_features=['category_col1', 'category_col2'],  # 原生支持！
    random_state=42,
    verbose=False
)

import numpy as np
import pandas as pd
import xgboost as xgb
from sklearn.datasets import fetch_california_housing
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import mean_squared_error, r2_score
import matplotlib.pyplot as plt

# ========== 1. 数据加载与探索 ==========
california = fetch_california_housing()
X = pd.DataFrame(california.data, columns=california.feature_names)
y = california.target

print(f"数据集: {X.shape[0]} 样本, {X.shape[1]} 特征")
print(f"目标范围: {y.min():.2f} - {y.max():.2f}")
print(f"缺失值:\n{X.isnull().sum()}")

# 特征工程：创建交互特征
X['Rooms_per_House'] = X['AveRooms'] / X['AveOccup']
X['Bedrooms_ratio'] = X['AveBedrms'] / X['AveRooms']
X['Population_density'] = X['Population'] / X['HouseAge'].clip(lower=1)

# ========== 2. 数据划分 ==========
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# ========== 3. 模型训练（带早停） ==========
model = xgb.XGBRegressor(
    n_estimators=1000,
    learning_rate=0.05,
    max_depth=5,
    min_child_weight=3,
    subsample=0.8,
    colsample_bytree=0.8,
    gamma=0.1,
    reg_alpha=0.1,
    reg_lambda=5,
    random_state=42,
    tree_method='hist',
    early_stopping_rounds=50,
)

model.fit(
    X_train, y_train,
    eval_set=[(X_test, y_test)],
    verbose=False
)

# ========== 4. 模型评估 ==========
y_pred = model.predict(X_test)
print(f"RMSE: {np.sqrt(mean_squared_error(y_test, y_pred)):.4f}")
print(f"R^2: {r2_score(y_test, y_pred):.4f}")
print(f"最优迭代轮数: {model.best_iteration}")

# 交叉验证
cv_scores = cross_val_score(
    xgb.XGBRegressor(
        n_estimators=model.best_iteration,
        learning_rate=0.05,
        max_depth=5,
        random_state=42
    ),
    X_train, y_train, cv=5,
    scoring='neg_root_mean_squared_error'
)
print(f"CV RMSE: {-cv_scores.mean():.4f} (+/- {cv_scores.std():.4f})")

# ========== 5. 特征重要性 ==========
importance = model.feature_importances_
feat_imp = pd.Series(importance, index=X.columns)
feat_imp = feat_imp.sort_values(ascending=False)
print("\n前5重要特征:")
print(feat_imp.head())

# ========== 6. 模型保存与加载 ==========
model.save_model('xgb_california_model.json')

loaded_model = xgb.XGBRegressor()
loaded_model.load_model('xgb_california_model.json')
print("\n模型已保存并可重新加载！")