由于需要 stacking 中每个基模型都需要对数据集进行划分后进行交叉训练，如果为每个模型都写这部分的代码会显得非常冗余，因此这里提供一种简便实现 stacking 的思路。
具体做法就是先实现一个父类，父类中实现了交叉训练的方法，因为这个方法对所有模型都是一致的，然后声明两个方法：train 和 predict，由于采用的基模型不同，这两个方法的具体实现也不同，因此需要在子类中实现。下面以 python 为例进行讲解
import numpy as np
from sklearn.model_selection import KFold
class BasicModel(object):
"""Parent class of basic models"""
def train(self, x_train, y_train, x_val, y_val):
"""return a trained model and eval metric of validation data"""
pass
def predict(self, model, x_test):
"""return the predicted result of test data"""
pass
def get_oof(self, x_train, y_train, x_test, n_folds = 5):
"""K-fold stacking"""
num_train, num_test = x_train.shape[0], x_test.shape[0]
oof_train = np.zeros((num_train,))
oof_test = np.zeros((num_test,))
oof_test_all_fold = np.zeros((num_test, n_folds))
aucs = []
KF = KFold(n_splits = n_folds, random_state=2017)
for i, (train_index, val_index) in enumerate(KF.split(x_train)):
print('{0} fold, train {1}, val {2}'.format(i,
len(train_index),
len(val_index)))
x_tra, y_tra = x_train[train_index], y_train[train_index]
x_val, y_val = x_train[val_index], y_train[val_index]
model, auc = self.train(x_tra, y_tra, x_val, y_val)
aucs.append(auc)
oof_train[val_index] = self.predict(model, x_val)
oof_test_all_fold[:, i] = self.predict(model, x_test)
oof_test = np.mean(oof_test_all_fold, axis=1)
print('all aucs {0}, average {1}'.format(aucs, np.mean(aucs)))
return oof_train, oof_test
上面最重要的就是进行 K-fold 训练的 get_oof 方法，该方法最终返回训练集和测试集在基模型上的预测结果，也就是两个一维向量，长度分别是训练集和测试集的样本数。
下面以两个基模型为例进行 stacking，分别是 xgboost 和 lightgbm，这两个模型都只需要实现 BasicModel 中的 train 和 predict 方法
第一个基模型
import xgboost as xgb
class XGBClassifier(BasicModel):
def __init__(self):
"""set parameters"""
self.num_rounds=1000
self.early_stopping_rounds = 15
self.params = {
'objective': 'binary:logistic',
'eta': 0.1,
'max_depth': 8,
'eval_metric': 'auc',
'seed': 0,
'silent' : 0
}
def train(self, x_train, y_train, x_val, y_val):
print('train with xgb model')
xgbtrain = xgb.DMatrix(x_train, y_train)
xgbval = xgb.DMatrix(x_val, y_val)
watchlist = [(xgbtrain,'train'), (xgbval, 'val')]
model = xgb.train(self.params,
xgbtrain,
self.num_rounds)
watchlist,
early_stopping_rounds = self.early_stopping_rounds)
return model, float(model.eval(xgbval).split()[1].split(':')[1])
def predict(self, model, x_test):
print('test with xgb model')
xgbtest = xgb.DMatrix(x_test)
return model.predict(xgbtest)
第二个基模型
import lightgbm as lgb
class LGBClassifier(BasicModel):
def __init__(self):
self.num_boost_round = 2000
self.early_stopping_rounds = 15
self.params = {
'task': 'train',
'boosting_type': 'dart',
'objective': 'binary',
'metric': {'auc', 'binary_logloss'},
'num_leaves': 80,
'learning_rate': 0.05,
# 'scale_pos_weight': 1.5,
'feature_fraction': 0.5,
'bagging_fraction': 1,
'bagging_freq': 5,
'max_bin': 300,
'is_unbalance': True,
'lambda_l2': 5.0,
'verbose' : -1
}
def train(self, x_train, y_train, x_val, y_val):
print('train with lgb model')
lgbtrain = lgb.Dataset(x_train, y_train)
lgbval = lgb.Dataset(x_val, y_val)
model = lgb.train(self.params,
lgbtrain,
valid_sets = lgbval,
verbose_eval = self.num_boost_round,
num_boost_round = self.num_boost_round)
early_stopping_rounds = self.early_stopping_rounds)
return model, model.best_score['valid_0']['auc']
def predict(self, model, x_test):
print('test with lgb model')
return model.predict(x_test, num_iteration=model.best_iteration)
下一个步骤就是将这两个基模型的输出作为第二层模型的输入，这里选用的第二层模型是 LogisticsRegression，
首先需要将各个基模型的输出 reshape 和 concatenate 成合适的大小
lgb_classifier = LGBClassifier() lgb_oof_train, lgb_oof_test = lgb_classifier.get_oof(x_train, y_train, x_test) xgb_classifier = XGBClassifier() xgb_oof_train, xgb_oof_test = xgb_classifier.get_oof(x_train, y_train, x_test) input_train = [xgb_oof_train, lgb_oof_train] input_test = [xgb_oof_test, lgb_oof_test] stacked_train = np.concatenate([f.reshape(-1, 1) for f in input_train], axis=1) stacked_test = np.concatenate([f.reshape(-1, 1) for f in input_test], axis=1)
然后用第二层模型进行训练和预测
from sklearn.linear_model import LinearRegression final_model = LinearRegression() final_model.fit(stacked_train, y_train) test_prediction = final_model.predict(stacked_test)
上述实现的完整代码见下面的链接
https://github.com/WuLC/MachineLearningAlgorithm/blob/master/python/Stacking.py
如有错漏，欢迎交流指正
