Rework Python callback functions. (#6199)

* Define a new callback interface for Python.
* Deprecate the old callbacks.
* Enable early stopping on dask.

tests/python/test_basic_models.py

@@ -8,7 +8,7 @@ import pytest
 import locale
 import tempfile
 
-dpath = 'demo/data/'
+dpath = os.path.join(tm.PROJECT_ROOT, 'demo/data/')
 dtrain = xgb.DMatrix(dpath + 'agaricus.txt.train')
 dtest = xgb.DMatrix(dpath + 'agaricus.txt.test')
 
@@ -110,84 +110,6 @@ class TestModels(unittest.TestCase):
             for jj in range(ii + 1, len(preds_list)):
                 assert np.sum(np.abs(preds_list[ii] - preds_list[jj])) > 0
 
-    def run_eta_decay(self, tree_method):
-        watchlist = [(dtest, 'eval'), (dtrain, 'train')]
-        num_round = 4
-
-        # learning_rates as a list
-        # init eta with 0 to check whether learning_rates work
-        param = {'max_depth': 2, 'eta': 0, 'verbosity': 0,
-                 'objective': 'binary:logistic', 'eval_metric': 'error',
-                 'tree_method': tree_method}
-        evals_result = {}
-        bst = xgb.train(param, dtrain, num_round, watchlist,
-                        callbacks=[xgb.callback.reset_learning_rate([
-                            0.8, 0.7, 0.6, 0.5
-                        ])],
-                        evals_result=evals_result)
-        eval_errors_0 = list(map(float, evals_result['eval']['error']))
-        assert isinstance(bst, xgb.core.Booster)
-        # validation error should decrease, if eta > 0
-        assert eval_errors_0[0] > eval_errors_0[-1]
-
-        # init learning_rate with 0 to check whether learning_rates work
-        param = {'max_depth': 2, 'learning_rate': 0, 'verbosity': 0,
-                 'objective': 'binary:logistic', 'eval_metric': 'error',
-                 'tree_method': tree_method}
-        evals_result = {}
-        bst = xgb.train(param, dtrain, num_round, watchlist,
-                        callbacks=[xgb.callback.reset_learning_rate(
-                            [0.8, 0.7, 0.6, 0.5])],
-                        evals_result=evals_result)
-        eval_errors_1 = list(map(float, evals_result['eval']['error']))
-        assert isinstance(bst, xgb.core.Booster)
-        # validation error should decrease, if learning_rate > 0
-        assert eval_errors_1[0] > eval_errors_1[-1]
-
-        # check if learning_rates override default value of eta/learning_rate
-        param = {
-            'max_depth': 2, 'verbosity': 0, 'objective': 'binary:logistic',
-            'eval_metric': 'error', 'tree_method': tree_method
-        }
-        evals_result = {}
-        bst = xgb.train(param, dtrain, num_round, watchlist,
-                        callbacks=[xgb.callback.reset_learning_rate(
-                            [0, 0, 0, 0]
-                        )],
-                        evals_result=evals_result)
-        eval_errors_2 = list(map(float, evals_result['eval']['error']))
-        assert isinstance(bst, xgb.core.Booster)
-        # validation error should not decrease, if eta/learning_rate = 0
-        assert eval_errors_2[0] == eval_errors_2[-1]
-
-        # learning_rates as a customized decay function
-        def eta_decay(ithround, num_boost_round):
-            return num_boost_round / (ithround + 1)
-
-        evals_result = {}
-        bst = xgb.train(param, dtrain, num_round, watchlist,
-                        callbacks=[
-                            xgb.callback.reset_learning_rate(eta_decay)
-                        ],
-                        evals_result=evals_result)
-        eval_errors_3 = list(map(float, evals_result['eval']['error']))
-
-        assert isinstance(bst, xgb.core.Booster)
-
-        assert eval_errors_3[0] == eval_errors_2[0]
-
-        for i in range(1, len(eval_errors_0)):
-            assert eval_errors_3[i] != eval_errors_2[i]
-
-    def test_eta_decay_hist(self):
-        self.run_eta_decay('hist')
-
-    def test_eta_decay_approx(self):
-        self.run_eta_decay('approx')
-
-    def test_eta_decay_exact(self):
-        self.run_eta_decay('exact')
-
     def test_boost_from_prediction(self):
         # Re-construct dtrain here to avoid modification
         margined = xgb.DMatrix(dpath + 'agaricus.txt.train')

tests/python/test_callback.py

@@ -0,0 +1,234 @@
+import xgboost as xgb
+import unittest
+import pytest
+import os
+import testing as tm
+import tempfile
+
+# We use the dataset for tests.
+pytestmark = pytest.mark.skipif(**tm.no_sklearn())
+
+
+class TestCallbacks(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        from sklearn.datasets import load_breast_cancer
+        X, y = load_breast_cancer(return_X_y=True)
+        cls.X = X
+        cls.y = y
+
+        split = int(X.shape[0]*0.8)
+        cls.X_train = X[: split, ...]
+        cls.y_train = y[: split, ...]
+        cls.X_valid = X[split:, ...]
+        cls.y_valid = y[split:, ...]
+
+    def test_evaluation_monitor(self):
+        D_train = xgb.DMatrix(self.X_train, self.y_train)
+        D_valid = xgb.DMatrix(self.X_valid, self.y_valid)
+        evals_result = {}
+        rounds = 10
+        xgb.train({'objective': 'binary:logistic',
+                   'eval_metric': 'error'}, D_train,
+                  evals=[(D_train, 'Train'), (D_valid, 'Valid')],
+                  num_boost_round=rounds,
+                  evals_result=evals_result,
+                  verbose_eval=True)
+        print('evals_result:', evals_result)
+        assert len(evals_result['Train']['error']) == rounds
+        assert len(evals_result['Valid']['error']) == rounds
+
+    def test_early_stopping(self):
+        D_train = xgb.DMatrix(self.X_train, self.y_train)
+        D_valid = xgb.DMatrix(self.X_valid, self.y_valid)
+        evals_result = {}
+        rounds = 30
+        early_stopping_rounds = 5
+        booster = xgb.train({'objective': 'binary:logistic',
+                             'eval_metric': 'error'}, D_train,
+                            evals=[(D_train, 'Train'), (D_valid, 'Valid')],
+                            num_boost_round=rounds,
+                            evals_result=evals_result,
+                            verbose_eval=True,
+                            early_stopping_rounds=early_stopping_rounds)
+        dump = booster.get_dump(dump_format='json')
+        assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
+
+    def test_early_stopping_custom_eval(self):
+        D_train = xgb.DMatrix(self.X_train, self.y_train)
+        D_valid = xgb.DMatrix(self.X_valid, self.y_valid)
+        early_stopping_rounds = 5
+        booster = xgb.train({'objective': 'binary:logistic',
+                             'eval_metric': 'error',
+                             'tree_method': 'hist'}, D_train,
+                            evals=[(D_train, 'Train'), (D_valid, 'Valid')],
+                            feval=tm.eval_error_metric,
+                            num_boost_round=1000,
+                            early_stopping_rounds=early_stopping_rounds,
+                            verbose_eval=False)
+        dump = booster.get_dump(dump_format='json')
+        assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
+
+    def test_early_stopping_customize(self):
+        D_train = xgb.DMatrix(self.X_train, self.y_train)
+        D_valid = xgb.DMatrix(self.X_valid, self.y_valid)
+        early_stopping_rounds = 5
+        early_stop = xgb.callback.EarlyStopping(rounds=early_stopping_rounds,
+                                                metric_name='CustomErr',
+                                                data_name='Train')
+        # Specify which dataset and which metric should be used for early stopping.
+        booster = xgb.train(
+            {'objective': 'binary:logistic',
+             'eval_metric': ['error', 'rmse'],
+             'tree_method': 'hist'}, D_train,
+            evals=[(D_train, 'Train'), (D_valid, 'Valid')],
+            feval=tm.eval_error_metric,
+            num_boost_round=1000,
+            callbacks=[early_stop],
+            verbose_eval=False)
+        dump = booster.get_dump(dump_format='json')
+        assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
+        assert len(early_stop.stopping_history['Train']['CustomErr']) == len(dump)
+
+    def test_early_stopping_skl(self):
+        from sklearn.datasets import load_breast_cancer
+        X, y = load_breast_cancer(return_X_y=True)
+        cls = xgb.XGBClassifier()
+        early_stopping_rounds = 5
+        cls.fit(X, y, eval_set=[(X, y)],
+                early_stopping_rounds=early_stopping_rounds, eval_metric='error')
+        booster = cls.get_booster()
+        dump = booster.get_dump(dump_format='json')
+        assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
+
+    def test_early_stopping_custom_eval_skl(self):
+        from sklearn.datasets import load_breast_cancer
+        X, y = load_breast_cancer(return_X_y=True)
+        cls = xgb.XGBClassifier()
+        early_stopping_rounds = 5
+        cls.fit(X, y, eval_set=[(X, y)],
+                early_stopping_rounds=early_stopping_rounds,
+                eval_metric=tm.eval_error_metric)
+        booster = cls.get_booster()
+        dump = booster.get_dump(dump_format='json')
+        assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
+
+    def run_eta_decay(self, tree_method, deprecated_callback):
+        if deprecated_callback:
+            scheduler = xgb.callback.reset_learning_rate
+        else:
+            scheduler = xgb.callback.LearningRateScheduler
+
+        dpath = os.path.join(tm.PROJECT_ROOT, 'demo/data/')
+        dtrain = xgb.DMatrix(dpath + 'agaricus.txt.train')
+        dtest = xgb.DMatrix(dpath + 'agaricus.txt.test')
+        watchlist = [(dtest, 'eval'), (dtrain, 'train')]
+        num_round = 4
+
+        # learning_rates as a list
+        # init eta with 0 to check whether learning_rates work
+        param = {'max_depth': 2, 'eta': 0, 'verbosity': 0,
+                 'objective': 'binary:logistic', 'eval_metric': 'error',
+                 'tree_method': tree_method}
+        evals_result = {}
+        bst = xgb.train(param, dtrain, num_round, watchlist,
+                        callbacks=[scheduler([
+                            0.8, 0.7, 0.6, 0.5
+                        ])],
+                        evals_result=evals_result)
+        eval_errors_0 = list(map(float, evals_result['eval']['error']))
+        assert isinstance(bst, xgb.core.Booster)
+        # validation error should decrease, if eta > 0
+        assert eval_errors_0[0] > eval_errors_0[-1]
+
+        # init learning_rate with 0 to check whether learning_rates work
+        param = {'max_depth': 2, 'learning_rate': 0, 'verbosity': 0,
+                 'objective': 'binary:logistic', 'eval_metric': 'error',
+                 'tree_method': tree_method}
+        evals_result = {}
+        bst = xgb.train(param, dtrain, num_round, watchlist,
+                        callbacks=[scheduler(
+                            [0.8, 0.7, 0.6, 0.5])],
+                        evals_result=evals_result)
+        eval_errors_1 = list(map(float, evals_result['eval']['error']))
+        assert isinstance(bst, xgb.core.Booster)
+        # validation error should decrease, if learning_rate > 0
+        assert eval_errors_1[0] > eval_errors_1[-1]
+
+        # check if learning_rates override default value of eta/learning_rate
+        param = {
+            'max_depth': 2, 'verbosity': 0, 'objective': 'binary:logistic',
+            'eval_metric': 'error', 'tree_method': tree_method
+        }
+        evals_result = {}
+        bst = xgb.train(param, dtrain, num_round, watchlist,
+                        callbacks=[scheduler(
+                            [0, 0, 0, 0]
+                        )],
+                        evals_result=evals_result)
+        eval_errors_2 = list(map(float, evals_result['eval']['error']))
+        assert isinstance(bst, xgb.core.Booster)
+        # validation error should not decrease, if eta/learning_rate = 0
+        assert eval_errors_2[0] == eval_errors_2[-1]
+
+        # learning_rates as a customized decay function
+        def eta_decay(ithround, num_boost_round=num_round):
+            return num_boost_round / (ithround + 1)
+
+        evals_result = {}
+        bst = xgb.train(param, dtrain, num_round, watchlist,
+                        callbacks=[
+                            scheduler(eta_decay)
+                        ],
+                        evals_result=evals_result)
+        eval_errors_3 = list(map(float, evals_result['eval']['error']))
+
+        assert isinstance(bst, xgb.core.Booster)
+
+        assert eval_errors_3[0] == eval_errors_2[0]
+
+        for i in range(1, len(eval_errors_0)):
+            assert eval_errors_3[i] != eval_errors_2[i]
+
+    def test_eta_decay_hist(self):
+        with pytest.deprecated_call():
+            self.run_eta_decay('hist', True)
+        self.run_eta_decay('hist', False)
+
+    def test_eta_decay_approx(self):
+        with pytest.deprecated_call():
+            self.run_eta_decay('approx', True)
+        self.run_eta_decay('approx', False)
+
+    def test_eta_decay_exact(self):
+        with pytest.deprecated_call():
+            self.run_eta_decay('exact', True)
+        self.run_eta_decay('exact', False)
+
+    def test_check_point(self):
+        from sklearn.datasets import load_breast_cancer
+        X, y = load_breast_cancer(return_X_y=True)
+        m = xgb.DMatrix(X, y)
+        with tempfile.TemporaryDirectory() as tmpdir:
+            check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
+                                                          iterations=1,
+                                                          name='model')
+            xgb.train({'objective': 'binary:logistic'}, m,
+                      num_boost_round=10,
+                      verbose_eval=False,
+                      callbacks=[check_point])
+            for i in range(1, 10):
+                assert os.path.exists(
+                    os.path.join(tmpdir, 'model_' + str(i) + '.json'))
+
+            check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
+                                                          iterations=1,
+                                                          as_pickle=True,
+                                                          name='model')
+            xgb.train({'objective': 'binary:logistic'}, m,
+                      num_boost_round=10,
+                      verbose_eval=False,
+                      callbacks=[check_point])
+            for i in range(1, 10):
+                assert os.path.exists(
+                    os.path.join(tmpdir, 'model_' + str(i) + '.pkl'))

tests/python/test_demos.py

@@ -119,6 +119,12 @@ def test_aft_demo():
     os.remove('aft_model.json')
 
 
+def test_callbacks_demo():
+    script = os.path.join(PYTHON_DEMO_DIR, 'callbacks.py')
+    cmd = ['python', script, '--plot=0']
+    subprocess.check_call(cmd)
+
+
 # gpu_acceleration is not tested due to covertype dataset is being too huge.
 # gamma regression is not tested as it requires running a R script first.
 # aft viz is not tested due to ploting is not controled

tests/python/test_with_dask.py

@@ -328,7 +328,7 @@ def test_sklearn_grid_search():
             reg.client = client
             model = GridSearchCV(reg, {'max_depth': [2, 4],
                                        'n_estimators': [5, 10]},
-                                 cv=2, verbose=1, iid=True)
+                                 cv=2, verbose=1)
             model.fit(X, y)
             # Expect unique results for each parameter value This confirms
             # sklearn is able to successfully update the parameter
@@ -705,3 +705,42 @@ class TestWithDask:
     @pytest.mark.gtest
     def test_quantile_same_on_all_workers(self):
         self.run_quantile('SameOnAllWorkers')
+
+
+class TestDaskCallbacks:
+    @pytest.mark.skipif(**tm.no_sklearn())
+    def test_early_stopping(self, client):
+        from sklearn.datasets import load_breast_cancer
+        X, y = load_breast_cancer(return_X_y=True)
+        X, y = da.from_array(X), da.from_array(y)
+        m = xgb.dask.DaskDMatrix(client, X, y)
+        early_stopping_rounds = 5
+        booster = xgb.dask.train(client, {'objective': 'binary:logistic',
+                                          'eval_metric': 'error',
+                                          'tree_method': 'hist'}, m,
+                                 evals=[(m, 'Train')],
+                                 num_boost_round=1000,
+                                 early_stopping_rounds=early_stopping_rounds)['booster']
+        assert hasattr(booster, 'best_score')
+        assert booster.best_iteration == 10
+        dump = booster.get_dump(dump_format='json')
+        assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
+
+    @pytest.mark.skipif(**tm.no_sklearn())
+    def test_early_stopping_custom_eval(self, client):
+        from sklearn.datasets import load_breast_cancer
+        X, y = load_breast_cancer(return_X_y=True)
+        X, y = da.from_array(X), da.from_array(y)
+        m = xgb.dask.DaskDMatrix(client, X, y)
+        early_stopping_rounds = 5
+        booster = xgb.dask.train(
+            client, {'objective': 'binary:logistic',
+                     'eval_metric': 'error',
+                     'tree_method': 'hist'}, m,
+            evals=[(m, 'Train')],
+            feval=tm.eval_error_metric,
+            num_boost_round=1000,
+            early_stopping_rounds=early_stopping_rounds)['booster']
+        assert hasattr(booster, 'best_score')
+        dump = booster.get_dump(dump_format='json')
+        assert len(dump) - booster.best_iteration == early_stopping_rounds + 1

tests/python/testing.py

@@ -240,6 +240,16 @@ def non_increasing(L, tolerance=1e-4):
     return all((y - x) < tolerance for x, y in zip(L, L[1:]))
 
 
+def eval_error_metric(predt, dtrain: xgb.DMatrix):
+    label = dtrain.get_label()
+    r = np.zeros(predt.shape)
+    gt = predt > 0.5
+    r[gt] = 1 - label[gt]
+    le = predt <= 0.5
+    r[le] = label[le]
+    return 'CustomErr', np.sum(r)
+
+
 CURDIR = os.path.normpath(os.path.abspath(os.path.dirname(__file__)))
 PROJECT_ROOT = os.path.normpath(
     os.path.join(CURDIR, os.path.pardir, os.path.pardir))