【多层堆叠集成模型（Stacking Ensemble）详解】

from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
import numpy as np
import optuna
import lightgbm as lgb
import xgboost as xgb
from catboost import CatBoostRegressor
from sklearn.linear_model import LinearRegression
from sklearn.base import clone

class StackingEnsemble:
    def __init__(self, base_models, meta_models, n_layers=2, n_splits=5):
        """
        多层堆叠集成模型

        参数：
        base_models: 基模型列表（第一层）
        meta_models: 元模型列表（后续各层）
        n_layers: 堆叠层数
        n_splits: K折交叉验证数
        """
        self.base_models = base_models
        self.meta_models = meta_models
        self.n_layers = n_layers
        self.n_splits = n_splits
        self.layers = []
        
    def _generate_oof(self, X, y, models, test_data=None, layer_idx=0):
        """生成堆叠特征"""
        kf = KFold(n_splits=self.n_splits, shuffle=True, random_state=42)
        oof_train = np.zeros((X.shape[0], len(models)))  # 训练集的OOF预测
        oof_test = np.zeros((test_data.shape[0], len(models))) if test_data is not None else None
        
        # 对每个模型进行交叉验证训练
        for model_idx, model in enumerate(models):
            model_oof_train = np.zeros(X.shape[0])  # 每个模型对训练集的预测
            model_oof_test = np.zeros(test_data.shape[0]) if test_data is not None else None
            
            for fold_idx, (train_idx, val_idx) in enumerate(kf.split(X)):
                X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]
                X_val = X.iloc[val_idx]
                
                # 克隆模型以避免参数污染
                cloned_model = clone_model(model, layer_idx, fold_idx)
                cloned_model.fit(X_train, y_train)
                
                # 在验证集上预测，并更新训练集的OOF值
                model_oof_train[val_idx] = cloned_model.predict(X_val)
                
                # 如果有测试集，进行预测并平均
                if test_data is not None:
                    model_oof_test += cloned_model.predict(test_data)
            
            # 平均测试集预测
            if test_data is not None:
                model_oof_test /= self.n_splits
                
            oof_train[:, model_idx] = model_oof_train
            if test_data is not None:
                oof_test[:, model_idx] = model_oof_test
        
        return oof_train, oof_test
    
    def fit_predict(self, X_train, y_train, X_test):
        """多层堆叠处理"""
        current_X_train = X_train.copy()  # 当前层的训练集
        current_X_test = X_test.copy()    # 当前层的测试集
        
        # 逐层训练
        for layer in range(self.n_layers):
            print(f"Training Layer {layer+1}/{self.n_layers}")
            
            # 第一层使用基模型
            if layer == 0:
                models = self.base_models
            else:
                models = self.meta_models
                
            # 生成堆叠特征
            oof_train, oof_test = self._generate_oof(
                current_X_train, y_train, 
                models, 
                test_data=current_X_test,
                layer_idx=layer
            )
            
            # 保存当前层结果
            self.layers.append({
                'train': oof_train,
                'test': oof_test
            })
            
            # 更新下一层输入
            current_X_train = np.hstack([current_X_train, oof_train])
            current_X_test = np.hstack([current_X_test, oof_test])
            
        return current_X_train, current_X_test

def clone_model(model, layer_idx, fold_idx):
    """克隆模型并设置随机种子"""
    params = model.get_params()  # 获取模型的参数
    if 'random_state' in params:
        params['random_state'] = 42 + layer_idx * 100 + fold_idx
    return model.__class__(**params)

if __name__ == "__main__":
    # 初始化基模型（使用优化后的参数）
    lgb_model = lgb.LGBMRegressor(**best_lgb)
    xgb_model = xgb.XGBRegressor(**best_xgb)
    cat_model = CatBoostRegressor(**best_cat)
    
    # 定义多层堆叠结构
    ensemble = StackingEnsemble(
        base_models=[lgb_model, xgb_model, cat_model],  # 第一层模型
        meta_models=[             # 后续层模型（可不同）
            lgb.LGBMRegressor(),
            xgb.XGBRegressor(),
            CatBoostRegressor(verbose=False),
            LinearRegression()    # 添加线性模型作为元模型
        ],
        n_layers=3,               # 总层数（包含基模型层）
        n_splits=5                # K折交叉验证数
    )
    
    # 使用堆叠集成进行训练和预测
    X_train_stack, X_test_stack = ensemble.fit_predict(X_train, y_train, X_test)