From 0f5f9c03850073ce756f01cd67b0b86aa0934ac7 Mon Sep 17 00:00:00 2001 From: Skipper Seabold Date: Wed, 20 May 2015 14:17:03 -0500 Subject: [PATCH 1/8] ENH: Allow early stopping in sklearn API. --- wrapper/xgboost.py | 118 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 112 insertions(+), 6 deletions(-) diff --git a/wrapper/xgboost.py b/wrapper/xgboost.py index 96f6c2573..35c24a1f2 100644 --- a/wrapper/xgboost.py +++ b/wrapper/xgboost.py @@ -772,7 +772,6 @@ def train(params, dtrain, num_boost_round=10, evals=(), obj=None, feval=None, ------- booster : a trained booster model """ - evals = list(evals) bst = Booster(params, [dtrain] + [d[0] for d in evals]) @@ -1074,6 +1073,8 @@ class XGBModel(XGBModelBase): params = super(XGBModel, self).get_params(deep=deep) if params['missing'] is np.nan: params['missing'] = None # sklearn doesn't handle nan. see #4725 + if not params.get('eval_metric', True): + del params['eval_metric'] # don't give as None param to Booster return params def get_xgb_params(self): @@ -1086,10 +1087,62 @@ class XGBModel(XGBModelBase): xgb_params.pop('nthread', None) return xgb_params - def fit(self, data, y): + def fit(self, X, y, eval_set=None, eval_metric=None, + early_stopping_rounds=None, feval=None): # pylint: disable=missing-docstring,invalid-name - train_dmatrix = DMatrix(data, label=y, missing=self.missing) - self._Booster = train(self.get_xgb_params(), train_dmatrix, self.n_estimators) + """ + Fit the gradient boosting model + + Parameters + ---------- + X : array_like + Feature matrix + y : array_like + Labels + eval_set : list, optional + A list of (X, y) tuple pairs to use as a validation set for + early-stopping + eval_metric : str, optional + Built-in evaluation metric to use. + early_stopping_rounds : int + Activates early stopping. Validation error needs to decrease at + least every round(s) to continue training. + Requires at least one item in evals. If there's more than one, + will use the last. Returns the model from the last iteration + (not the best one). If early stopping occurs, the model will + have two additional fields: bst.best_score and bst.best_iteration. + feval : function, optional + Custom evaluation metric to use. The call signature is + feval(y_predicted, y_true) where y_true will be a DMatrix object + such that you may need to call the get_label method. This objective + if always assumed to be minimized, so use -feval when appropriate. + """ + trainDmatrix = DMatrix(X, label=y, missing=self.missing) + + eval_results = {} + if eval_set is not None: + evals = list(DMatrix(x[0], label=x[1]) for x in eval_set) + evals = list(zip(evals, + ["validation_{}" for i in range(len(evals))])) + else: + evals = () + + params = self.get_xgb_params() + + if eval_metric is not None: + params.update({'eval_metric': eval_metric}) + + self._Booster = train(params, trainDmatrix, + self.n_estimators, evals=evals, + early_stopping_rounds=early_stopping_rounds, + evals_result=eval_results, feval=None) + if eval_results: + eval_results = {k: np.array(v, dtype=float) + for k, v in eval_results.items()} + eval_results = {k: np.array(v) for k, v in eval_results.items()} + self.eval_results_ = eval_results + self.best_score_ = self._Booster.best_score + self.best_iteration_ = self._Booster.best_iteration return self def predict(self, data): @@ -1117,8 +1170,39 @@ class XGBClassifier(XGBModel, XGBClassifierBase): colsample_bytree, base_score, seed, missing) - def fit(self, X, y, sample_weight=None): + def fit(self, X, y, sample_weight=None, eval_set=None, eval_metric=None, + early_stopping_rounds=None, feval=None): # pylint: disable = attribute-defined-outside-init,arguments-differ + """ + Fit gradient boosting classifier + + Parameters + ---------- + X : array_like + Feature matrix + y : array_like + Labels + sample_weight : array_like + Weight for each instance + eval_set : list, optional + A list of (X, y) pairs to use as a validation set for + early-stopping + eval_metric : str + Built-in evaluation metric to use. + early_stopping_rounds : int, optional + Activates early stopping. Validation error needs to decrease at + least every round(s) to continue training. + Requires at least one item in evals. If there's more than one, + will use the last. Returns the model from the last iteration + (not the best one). If early stopping occurs, the model will + have two additional fields: bst.best_score and bst.best_iteration. + feval : function, optional + Custom evaluation metric to use. The call signature is + feval(y_predicted, y_true) where y_true will be a DMatrix object + such that you may need to call the get_label method. This objective + if always assumed to be minimized, so use -feval when appropriate. + """ + eval_results = {} self.classes_ = list(np.unique(y)) self.n_classes_ = len(self.classes_) if self.n_classes_ > 2: @@ -1129,6 +1213,18 @@ class XGBClassifier(XGBModel, XGBClassifierBase): else: xgb_options = self.get_xgb_params() + if eval_metric is not None: + xgb_options.update({"eval_metric": eval_metric}) + + if eval_set is not None: + # TODO: use sample_weight if given? + evals = list(DMatrix(x[0], label=x[1]) for x in eval_set) + nevals = len(evals) + eval_names = ["validation_{}".format(i) for i in range(nevals)] + evals = list(zip(evals, eval_names)) + else: + evals = () + self._le = LabelEncoder().fit(y) training_labels = self._le.transform(y) @@ -1139,7 +1235,17 @@ class XGBClassifier(XGBModel, XGBClassifierBase): train_dmatrix = DMatrix(X, label=training_labels, missing=self.missing) - self._Booster = train(xgb_options, train_dmatrix, self.n_estimators) + self._Booster = train(xgb_options, train_dmatrix, self.n_estimators, + evals=evals, + early_stopping_rounds=early_stopping_rounds, + evals_result=eval_results, feval=feval) + + if eval_results: + eval_results = {k: np.array(v, dtype=float) + for k, v in eval_results.items()} + self.eval_results_ = eval_results + self.best_score_ = self._Booster.best_score + self.best_iteration_ = self._Booster.best_iteration return self From 3952b525b82d2d2a2019429e3a97fe0f1f331f0c Mon Sep 17 00:00:00 2001 From: Skipper Seabold Date: Wed, 20 May 2015 14:17:30 -0500 Subject: [PATCH 2/8] ENH: Allow possibly negative evaluation metrics. --- wrapper/xgboost.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wrapper/xgboost.py b/wrapper/xgboost.py index 35c24a1f2..bc52da633 100644 --- a/wrapper/xgboost.py +++ b/wrapper/xgboost.py @@ -795,7 +795,7 @@ def train(params, dtrain, num_boost_round=10, evals=(), obj=None, feval=None, sys.stderr.write(msg + '\n') if evals_result is not None: - res = re.findall(":([0-9.]+).", msg) + res = re.findall(":-?([0-9.]+).", msg) for key, val in zip(evals_name, res): evals_result[key].append(val) return bst @@ -842,7 +842,7 @@ def train(params, dtrain, num_boost_round=10, evals=(), obj=None, feval=None, sys.stderr.write(msg + '\n') if evals_result is not None: - res = re.findall(":([0-9.]+).", msg) + res = re.findall(":-([0-9.]+).", msg) for key, val in zip(evals_name, res): evals_result[key].append(val) From cf89ae64e2c198c9e5acde076e8be40b1aab2e92 Mon Sep 17 00:00:00 2001 From: Skipper Seabold Date: Wed, 20 May 2015 14:27:22 -0500 Subject: [PATCH 3/8] ENH: Allow for silent evaluation --- wrapper/xgboost.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/wrapper/xgboost.py b/wrapper/xgboost.py index bc52da633..a4acd5a7f 100644 --- a/wrapper/xgboost.py +++ b/wrapper/xgboost.py @@ -738,7 +738,7 @@ class Booster(object): def train(params, dtrain, num_boost_round=10, evals=(), obj=None, feval=None, - early_stopping_rounds=None, evals_result=None): + early_stopping_rounds=None, evals_result=None, verbose_eval=True): # pylint: disable=too-many-statements,too-many-branches, attribute-defined-outside-init """Train a booster with given parameters. @@ -793,7 +793,8 @@ def train(params, dtrain, num_boost_round=10, evals=(), obj=None, feval=None, else: msg = bst_eval_set.decode() - sys.stderr.write(msg + '\n') + if verbose_eval: + sys.stderr.write(msg + '\n') if evals_result is not None: res = re.findall(":-?([0-9.]+).", msg) for key, val in zip(evals_name, res): @@ -839,7 +840,8 @@ def train(params, dtrain, num_boost_round=10, evals=(), obj=None, feval=None, else: msg = bst_eval_set.decode() - sys.stderr.write(msg + '\n') + if verbose_eval: + sys.stderr.write(msg + '\n') if evals_result is not None: res = re.findall(":-([0-9.]+).", msg) @@ -1088,7 +1090,7 @@ class XGBModel(XGBModelBase): return xgb_params def fit(self, X, y, eval_set=None, eval_metric=None, - early_stopping_rounds=None, feval=None): + early_stopping_rounds=None, feval=None, verbose=True): # pylint: disable=missing-docstring,invalid-name """ Fit the gradient boosting model @@ -1116,6 +1118,9 @@ class XGBModel(XGBModelBase): feval(y_predicted, y_true) where y_true will be a DMatrix object such that you may need to call the get_label method. This objective if always assumed to be minimized, so use -feval when appropriate. + verbose : bool + If `verbose` and an evaluation set is used, writes the evaluation + metric measured on the validation set to stderr. """ trainDmatrix = DMatrix(X, label=y, missing=self.missing) @@ -1135,7 +1140,8 @@ class XGBModel(XGBModelBase): self._Booster = train(params, trainDmatrix, self.n_estimators, evals=evals, early_stopping_rounds=early_stopping_rounds, - evals_result=eval_results, feval=None) + evals_result=eval_results, feval=None, + verbose_eval=verbose) if eval_results: eval_results = {k: np.array(v, dtype=float) for k, v in eval_results.items()} @@ -1171,7 +1177,7 @@ class XGBClassifier(XGBModel, XGBClassifierBase): base_score, seed, missing) def fit(self, X, y, sample_weight=None, eval_set=None, eval_metric=None, - early_stopping_rounds=None, feval=None): + early_stopping_rounds=None, feval=None, versbose=True): # pylint: disable = attribute-defined-outside-init,arguments-differ """ Fit gradient boosting classifier @@ -1201,6 +1207,9 @@ class XGBClassifier(XGBModel, XGBClassifierBase): feval(y_predicted, y_true) where y_true will be a DMatrix object such that you may need to call the get_label method. This objective if always assumed to be minimized, so use -feval when appropriate. + verbose : bool + If `verbose` and an evaluation set is used, writes the evaluation + metric measured on the validation set to stderr. """ eval_results = {} self.classes_ = list(np.unique(y)) @@ -1238,7 +1247,8 @@ class XGBClassifier(XGBModel, XGBClassifierBase): self._Booster = train(xgb_options, train_dmatrix, self.n_estimators, evals=evals, early_stopping_rounds=early_stopping_rounds, - evals_result=eval_results, feval=feval) + evals_result=eval_results, feval=feval, + verbose_eval=verbose) if eval_results: eval_results = {k: np.array(v, dtype=float) From 46e9520a28b4aca9281c938a919620a8754cb4d9 Mon Sep 17 00:00:00 2001 From: Skipper Seabold Date: Wed, 20 May 2015 14:38:45 -0500 Subject: [PATCH 4/8] DOC: Document verbose_eval --- wrapper/xgboost.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wrapper/xgboost.py b/wrapper/xgboost.py index a4acd5a7f..a4ad84bf5 100644 --- a/wrapper/xgboost.py +++ b/wrapper/xgboost.py @@ -767,6 +767,9 @@ def train(params, dtrain, num_boost_round=10, evals=(), obj=None, feval=None, bst.best_score and bst.best_iteration. evals_result: dict This dictionary stores the evaluation results of all the items in watchlist + verbose_eval : bool + If `verbose_eval` then the evaluation metric on the validation set, if + given, is printed at each boosting stage. Returns ------- From 113285e1dc3fdc0c709e72a2cb985b3025360897 Mon Sep 17 00:00:00 2001 From: Skipper Seabold Date: Wed, 20 May 2015 14:39:48 -0500 Subject: [PATCH 5/8] DOC: Point to parameter.md for eval_metric --- wrapper/xgboost.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wrapper/xgboost.py b/wrapper/xgboost.py index a4ad84bf5..adb21a00b 100644 --- a/wrapper/xgboost.py +++ b/wrapper/xgboost.py @@ -1108,7 +1108,7 @@ class XGBModel(XGBModelBase): A list of (X, y) tuple pairs to use as a validation set for early-stopping eval_metric : str, optional - Built-in evaluation metric to use. + Built-in evaluation metric to use. See doc/parameter.md. early_stopping_rounds : int Activates early stopping. Validation error needs to decrease at least every round(s) to continue training. @@ -1197,7 +1197,7 @@ class XGBClassifier(XGBModel, XGBClassifierBase): A list of (X, y) pairs to use as a validation set for early-stopping eval_metric : str - Built-in evaluation metric to use. + Built-in evaluation metric to use. See doc/parameter.md. early_stopping_rounds : int, optional Activates early stopping. Validation error needs to decrease at least every round(s) to continue training. From b0f7ddaa2ee3411b33f95b42be46a3325b0ac23b Mon Sep 17 00:00:00 2001 From: Skipper Seabold Date: Tue, 30 Jun 2015 11:42:14 -0500 Subject: [PATCH 6/8] REF: Combine eval_metric and feval to one parameter --- wrapper/xgboost.py | 48 ++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/wrapper/xgboost.py b/wrapper/xgboost.py index adb21a00b..95e0bf6ff 100644 --- a/wrapper/xgboost.py +++ b/wrapper/xgboost.py @@ -1093,7 +1093,7 @@ class XGBModel(XGBModelBase): return xgb_params def fit(self, X, y, eval_set=None, eval_metric=None, - early_stopping_rounds=None, feval=None, verbose=True): + early_stopping_rounds=None, verbose=True): # pylint: disable=missing-docstring,invalid-name """ Fit the gradient boosting model @@ -1107,8 +1107,14 @@ class XGBModel(XGBModelBase): eval_set : list, optional A list of (X, y) tuple pairs to use as a validation set for early-stopping - eval_metric : str, optional - Built-in evaluation metric to use. See doc/parameter.md. + eval_metric : str, callable, optional + If a str, should be a built-in evaluation metric to use. See + doc/parameter.md. If callable, a custom evaluation metric. The call + signature is func(y_predicted, y_true) where y_true will be a + DMatrix object such that you may need to call the get_label + method. It must return a str, value pair where the str is a name + for the evaluation and value is the value of the evaluation + function. This objective is always minimized. early_stopping_rounds : int Activates early stopping. Validation error needs to decrease at least every round(s) to continue training. @@ -1116,11 +1122,6 @@ class XGBModel(XGBModelBase): will use the last. Returns the model from the last iteration (not the best one). If early stopping occurs, the model will have two additional fields: bst.best_score and bst.best_iteration. - feval : function, optional - Custom evaluation metric to use. The call signature is - feval(y_predicted, y_true) where y_true will be a DMatrix object - such that you may need to call the get_label method. This objective - if always assumed to be minimized, so use -feval when appropriate. verbose : bool If `verbose` and an evaluation set is used, writes the evaluation metric measured on the validation set to stderr. @@ -1137,13 +1138,17 @@ class XGBModel(XGBModelBase): params = self.get_xgb_params() + feval = eval_metric if callable(eval_metric) else None if eval_metric is not None: - params.update({'eval_metric': eval_metric}) + if callable(eval_metric): + eval_metric = None + else: + params.update({'eval_metric': eval_metric}) self._Booster = train(params, trainDmatrix, self.n_estimators, evals=evals, early_stopping_rounds=early_stopping_rounds, - evals_result=eval_results, feval=None, + evals_result=eval_results, feval=feval, verbose_eval=verbose) if eval_results: eval_results = {k: np.array(v, dtype=float) @@ -1180,7 +1185,7 @@ class XGBClassifier(XGBModel, XGBClassifierBase): base_score, seed, missing) def fit(self, X, y, sample_weight=None, eval_set=None, eval_metric=None, - early_stopping_rounds=None, feval=None, versbose=True): + early_stopping_rounds=None, verbose=True): # pylint: disable = attribute-defined-outside-init,arguments-differ """ Fit gradient boosting classifier @@ -1196,8 +1201,14 @@ class XGBClassifier(XGBModel, XGBClassifierBase): eval_set : list, optional A list of (X, y) pairs to use as a validation set for early-stopping - eval_metric : str - Built-in evaluation metric to use. See doc/parameter.md. + eval_metric : str, callable, optional + If a str, should be a built-in evaluation metric to use. See + doc/parameter.md. If callable, a custom evaluation metric. The call + signature is func(y_predicted, y_true) where y_true will be a + DMatrix object such that you may need to call the get_label + method. It must return a str, value pair where the str is a name + for the evaluation and value is the value of the evaluation + function. This objective is always minimized. early_stopping_rounds : int, optional Activates early stopping. Validation error needs to decrease at least every round(s) to continue training. @@ -1205,11 +1216,6 @@ class XGBClassifier(XGBModel, XGBClassifierBase): will use the last. Returns the model from the last iteration (not the best one). If early stopping occurs, the model will have two additional fields: bst.best_score and bst.best_iteration. - feval : function, optional - Custom evaluation metric to use. The call signature is - feval(y_predicted, y_true) where y_true will be a DMatrix object - such that you may need to call the get_label method. This objective - if always assumed to be minimized, so use -feval when appropriate. verbose : bool If `verbose` and an evaluation set is used, writes the evaluation metric measured on the validation set to stderr. @@ -1225,8 +1231,12 @@ class XGBClassifier(XGBModel, XGBClassifierBase): else: xgb_options = self.get_xgb_params() + feval = eval_metric if callable(eval_metric) else None if eval_metric is not None: - xgb_options.update({"eval_metric": eval_metric}) + if callable(eval_metric): + eval_metric = None + else: + xgb_options.update({"eval_metric": eval_metric}) if eval_set is not None: # TODO: use sample_weight if given? From 4a37b852a03b1320d1c41f948a4ec212a981ad1d Mon Sep 17 00:00:00 2001 From: Skipper Seabold Date: Tue, 30 Jun 2015 11:42:28 -0500 Subject: [PATCH 7/8] DOC: Add early stopping example --- demo/guide-python/sklearn_examples.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/demo/guide-python/sklearn_examples.py b/demo/guide-python/sklearn_examples.py index ce8c8d01e..56fed1dd2 100755 --- a/demo/guide-python/sklearn_examples.py +++ b/demo/guide-python/sklearn_examples.py @@ -8,7 +8,7 @@ import pickle import xgboost as xgb import numpy as np -from sklearn.cross_validation import KFold +from sklearn.cross_validation import KFold, train_test_split from sklearn.metrics import confusion_matrix, mean_squared_error from sklearn.grid_search import GridSearchCV from sklearn.datasets import load_iris, load_digits, load_boston @@ -65,3 +65,23 @@ print("Pickling sklearn API models") pickle.dump(clf, open("best_boston.pkl", "wb")) clf2 = pickle.load(open("best_boston.pkl", "rb")) print(np.allclose(clf.predict(X), clf2.predict(X))) + +# Early-stopping + +X = digits['data'] +y = digits['target'] +X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) +clf = xgb.XGBClassifier() +clf.fit(X_train, y_train, early_stopping_rounds=10, eval_metric="auc", + eval_set=[(X_test, y_test)]) + +# Custom evaluation function +from sklearn.metrics import log_loss + + +def log_loss_eval(y_pred, y_true): + return "log-loss", log_loss(y_true.get_label(), y_pred) + + +clf.fit(X_train, y_train, early_stopping_rounds=10, eval_metric=log_loss_eval, + eval_set=[(X_test, y_test)]) From b76db01c6605a19e852172e3f08d9a4613bf6361 Mon Sep 17 00:00:00 2001 From: Skipper Seabold Date: Wed, 8 Jul 2015 14:29:52 -0500 Subject: [PATCH 8/8] STY: Fix lint errors --- wrapper/xgboost.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/wrapper/xgboost.py b/wrapper/xgboost.py index 95e0bf6ff..27041376b 100644 --- a/wrapper/xgboost.py +++ b/wrapper/xgboost.py @@ -6,7 +6,7 @@ Version: 0.40 Authors: Tianqi Chen, Bing Xu Early stopping by Zygmunt Zając """ -# pylint: disable=too-many-arguments, too-many-locals, too-many-lines, invalid-name +# pylint: disable=too-many-arguments, too-many-locals, too-many-lines, invalid-name, fixme from __future__ import absolute_import import os @@ -784,7 +784,7 @@ def train(params, dtrain, num_boost_round=10, evals=(), obj=None, feval=None, else: evals_name = [d[1] for d in evals] evals_result.clear() - evals_result.update({key:[] for key in evals_name}) + evals_result.update({key: [] for key in evals_name}) if not early_stopping_rounds: for i in range(num_boost_round): @@ -1094,7 +1094,7 @@ class XGBModel(XGBModelBase): def fit(self, X, y, eval_set=None, eval_metric=None, early_stopping_rounds=None, verbose=True): - # pylint: disable=missing-docstring,invalid-name + # pylint: disable=missing-docstring,invalid-name,attribute-defined-outside-init """ Fit the gradient boosting model @@ -1131,8 +1131,8 @@ class XGBModel(XGBModelBase): eval_results = {} if eval_set is not None: evals = list(DMatrix(x[0], label=x[1]) for x in eval_set) - evals = list(zip(evals, - ["validation_{}" for i in range(len(evals))])) + evals = list(zip(evals, ["validation_{}".format(i) for i in + range(len(evals))])) else: evals = ()