Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move skl eval_metric and early_stopping rounds to model params. (#6751)

Browse Source 
A new parameter `custom_metric` is added to `train` and `cv` to distinguish the behaviour from the old `feval`.  And `feval` is deprecated.  The new `custom_metric` receives transformed prediction when the built-in objective is used.  This enables XGBoost to use cost functions from other libraries like scikit-learn directly without going through the definition of the link function.

`eval_metric` and `early_stopping_rounds` in sklearn interface are moved from `fit` to `__init__` and is now saved as part of the scikit-learn model.  The old ones in `fit` function are now deprecated. The new `eval_metric` in `__init__` has the same new behaviour as `custom_metric`.

Added more detailed documents for the behaviour of custom objective and metric.
This commit is contained in:
Jiaming Yuan
2021-10-28 17:20:20 +08:00
committed by GitHub
parent 6b074add66
commit 45aef75cca
13 changed files with 685 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
tests/python/test_callback.py
View File

@@ -173,10 +173,11 @@ class TestCallbacks:
def test_early_stopping_skl(self):
from sklearn.datasets import load_breast_cancer
X, y = load_breast_cancer(return_X_y=True)
cls = xgb.XGBClassifier()
early_stopping_rounds = 5
cls.fit(X, y, eval_set=[(X, y)],
early_stopping_rounds=early_stopping_rounds, eval_metric='error')
cls = xgb.XGBClassifier(
early_stopping_rounds=early_stopping_rounds, eval_metric='error'
)
cls.fit(X, y, eval_set=[(X, y)])
booster = cls.get_booster()
dump = booster.get_dump(dump_format='json')
assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
@@ -184,12 +185,10 @@ class TestCallbacks:
def test_early_stopping_custom_eval_skl(self):
from sklearn.datasets import load_breast_cancer
X, y = load_breast_cancer(return_X_y=True)
cls = xgb.XGBClassifier()
cls = xgb.XGBClassifier(eval_metric=tm.eval_error_metric_skl)
early_stopping_rounds = 5
early_stop = xgb.callback.EarlyStopping(rounds=early_stopping_rounds)
cls.fit(X, y, eval_set=[(X, y)],
eval_metric=tm.eval_error_metric,
callbacks=[early_stop])
cls.fit(X, y, eval_set=[(X, y)], callbacks=[early_stop])
booster = cls.get_booster()
dump = booster.get_dump(dump_format='json')
assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
@@ -198,41 +197,40 @@ class TestCallbacks:
from sklearn.datasets import load_breast_cancer
X, y = load_breast_cancer(return_X_y=True)
n_estimators = 100
cls = xgb.XGBClassifier(n_estimators=n_estimators)
cls = xgb.XGBClassifier(
n_estimators=n_estimators, eval_metric=tm.eval_error_metric_skl
)
early_stopping_rounds = 5
early_stop = xgb.callback.EarlyStopping(rounds=early_stopping_rounds,
save_best=True)
cls.fit(X, y, eval_set=[(X, y)],
eval_metric=tm.eval_error_metric, callbacks=[early_stop])
cls.fit(X, y, eval_set=[(X, y)], callbacks=[early_stop])
booster = cls.get_booster()
dump = booster.get_dump(dump_format='json')
assert len(dump) == booster.best_iteration + 1
early_stop = xgb.callback.EarlyStopping(rounds=early_stopping_rounds,
save_best=True)
cls = xgb.XGBClassifier(booster='gblinear', n_estimators=10)
cls = xgb.XGBClassifier(
booster='gblinear', n_estimators=10, eval_metric=tm.eval_error_metric_skl
)
with pytest.raises(ValueError):
cls.fit(X, y, eval_set=[(X, y)], eval_metric=tm.eval_error_metric,
callbacks=[early_stop])
cls.fit(X, y, eval_set=[(X, y)], callbacks=[early_stop])
# No error
early_stop = xgb.callback.EarlyStopping(rounds=early_stopping_rounds,
save_best=False)
xgb.XGBClassifier(booster='gblinear', n_estimators=10).fit(
X, y, eval_set=[(X, y)],
eval_metric=tm.eval_error_metric,
callbacks=[early_stop])
xgb.XGBClassifier(
booster='gblinear', n_estimators=10, eval_metric=tm.eval_error_metric_skl
).fit(X, y, eval_set=[(X, y)], callbacks=[early_stop])
def test_early_stopping_continuation(self):
from sklearn.datasets import load_breast_cancer
X, y = load_breast_cancer(return_X_y=True)
cls = xgb.XGBClassifier()
cls = xgb.XGBClassifier(eval_metric=tm.eval_error_metric_skl)
early_stopping_rounds = 5
early_stop = xgb.callback.EarlyStopping(rounds=early_stopping_rounds,
save_best=True)
cls.fit(X, y, eval_set=[(X, y)],
eval_metric=tm.eval_error_metric,
callbacks=[early_stop])
cls.fit(X, y, eval_set=[(X, y)], callbacks=[early_stop])
booster = cls.get_booster()
assert booster.num_boosted_rounds() == booster.best_iteration + 1
@@ -243,8 +241,8 @@ class TestCallbacks:
cls.load_model(path)
assert cls._Booster is not None
early_stopping_rounds = 3
cls.fit(X, y, eval_set=[(X, y)], eval_metric=tm.eval_error_metric,
early_stopping_rounds=early_stopping_rounds)
cls.set_params(eval_metric=tm.eval_error_metric_skl)
cls.fit(X, y, eval_set=[(X, y)], early_stopping_rounds=early_stopping_rounds)
booster = cls.get_booster()
assert booster.num_boosted_rounds() == \
booster.best_iteration + early_stopping_rounds + 1

1
tests/python/test_early_stopping.py
View File

@@ -7,7 +7,6 @@ rng = np.random.RandomState(1994)
class TestEarlyStopping:
@pytest.mark.skipif(**tm.no_sklearn())
def test_early_stopping_nonparallel(self):
from sklearn.datasets import load_digits

13
tests/python/test_with_dask.py
View File

@@ -1663,11 +1663,16 @@ class TestDaskCallbacks:
valid_X, valid_y = load_breast_cancer(return_X_y=True)
valid_X, valid_y = da.from_array(valid_X), da.from_array(valid_y)
cls = xgb.dask.DaskXGBClassifier(objective='binary:logistic', tree_method='hist',
n_estimators=1000)
cls = xgb.dask.DaskXGBClassifier(
objective='binary:logistic',
tree_method='hist',
n_estimators=1000,
eval_metric=tm.eval_error_metric_skl
)
cls.client = client
cls.fit(X, y, early_stopping_rounds=early_stopping_rounds,
eval_set=[(valid_X, valid_y)], eval_metric=tm.eval_error_metric)
cls.fit(
X, y, early_stopping_rounds=early_stopping_rounds, eval_set=[(valid_X, valid_y)]
)
booster = cls.get_booster()
dump = booster.get_dump(dump_format='json')
assert len(dump) - booster.best_iteration == early_stopping_rounds + 1

73
tests/python/test_with_sklearn.py
View File

@@ -1271,3 +1271,76 @@ def test_prediction_config():
reg.set_params(booster="gblinear")
assert reg._can_use_inplace_predict() is False
def test_evaluation_metric():
from sklearn.datasets import load_diabetes, load_digits
from sklearn.metrics import mean_absolute_error
X, y = load_diabetes(return_X_y=True)
n_estimators = 16
with tm.captured_output() as (out, err):
reg = xgb.XGBRegressor(
tree_method="hist",
eval_metric=mean_absolute_error,
n_estimators=n_estimators,
)
reg.fit(X, y, eval_set=[(X, y)])
lines = out.getvalue().strip().split('\n')
assert len(lines) == n_estimators
for line in lines:
assert line.find("mean_absolute_error") != -1
def metric(predt: np.ndarray, Xy: xgb.DMatrix):
y = Xy.get_label()
return "m", np.abs(predt - y).sum()
with pytest.warns(UserWarning):
reg = xgb.XGBRegressor(
tree_method="hist",
n_estimators=1,
)
reg.fit(X, y, eval_set=[(X, y)], eval_metric=metric)
def merror(y_true: np.ndarray, predt: np.ndarray):
n_samples = y_true.shape[0]
assert n_samples == predt.size
errors = np.zeros(y_true.shape[0])
errors[y != predt] = 1.0
return np.sum(errors) / n_samples
X, y = load_digits(n_class=10, return_X_y=True)
clf = xgb.XGBClassifier(
use_label_encoder=False,
tree_method="hist",
eval_metric=merror,
n_estimators=16,
objective="multi:softmax"
)
clf.fit(X, y, eval_set=[(X, y)])
custom = clf.evals_result()
clf = xgb.XGBClassifier(
use_label_encoder=False,
tree_method="hist",
eval_metric="merror",
n_estimators=16,
objective="multi:softmax"
)
clf.fit(X, y, eval_set=[(X, y)])
internal = clf.evals_result()
np.testing.assert_allclose(
custom["validation_0"]["merror"], internal["validation_0"]["merror"]
)
clf = xgb.XGBRFClassifier(
use_label_encoder=False,
tree_method="hist", n_estimators=16,
objective=tm.softprob_obj(10),
eval_metric=merror,
)
with pytest.raises(AssertionError):
# shape check inside the `merror` function
clf.fit(X, y, eval_set=[(X, y)])

11
tests/python/testing.py
View File

@@ -338,6 +338,7 @@ def non_increasing(L, tolerance=1e-4):
def eval_error_metric(predt, dtrain: xgb.DMatrix):
"""Evaluation metric for xgb.train"""
label = dtrain.get_label()
r = np.zeros(predt.shape)
gt = predt > 0.5
@@ -349,6 +350,16 @@ def eval_error_metric(predt, dtrain: xgb.DMatrix):
return 'CustomErr', np.sum(r)
def eval_error_metric_skl(y_true: np.ndarray, y_score: np.ndarray) -> float:
"""Evaluation metric that looks like metrics provided by sklearn."""
r = np.zeros(y_score.shape)
gt = y_score > 0.5
r[gt] = 1 - y_true[gt]
le = y_score <= 0.5
r[le] = y_true[le]
return np.sum(r)
def softmax(x):
e = np.exp(x)
return e / np.sum(e)