Add tolerance to early stopping. (#6942)

python-package/xgboost/callback.py

@@ -487,25 +487,44 @@ class EarlyStopping(TrainingCallback):
         Whether to maximize evaluation metric.  None means auto (discouraged).
     save_best
         Whether training should return the best model or the last model.
+    abs_tol
+        Absolute tolerance for early stopping condition.
+
+        .. versionadded:: 1.5.0
+
+        .. code-block:: python
+
+            clf = xgboost.XGBClassifier(tree_method="gpu_hist")
+            es = xgboost.callback.EarlyStopping(
+                rounds=2,
+                abs_tol=1e-3,
+                save_best=True,
+                maximize=False,
+                data_name="validation_0",
+                metric_name="mlogloss",
+            )
+
+            X, y = load_digits(return_X_y=True)
+            clf.fit(X, y, eval_set=[(X, y)], callbacks=[es])
     """
     def __init__(self,
                  rounds: int,
                  metric_name: Optional[str] = None,
                  data_name: Optional[str] = None,
                  maximize: Optional[bool] = None,
-                 save_best: Optional[bool] = False) -> None:
+                 save_best: Optional[bool] = False,
+                 abs_tol: float = 0) -> None:
         self.data = data_name
         self.metric_name = metric_name
         self.rounds = rounds
         self.save_best = save_best
         self.maximize = maximize
         self.stopping_history: CallbackContainer.EvalsLog = {}
+        self._tol = abs_tol
+        if self._tol < 0:
+            raise ValueError("tolerance must be greater or equal to 0.")
 
-        if self.maximize is not None:
-            if self.maximize:
-                self.improve_op = lambda x, y: x > y
-            else:
-                self.improve_op = lambda x, y: x < y
+        self.improve_op = None
 
         self.current_rounds: int = 0
         self.best_scores: dict = {}
@@ -517,18 +536,33 @@ class EarlyStopping(TrainingCallback):
         return model
 
     def _update_rounds(self, score, name, metric, model, epoch) -> bool:
-        # Just to be compatibility with old behavior before 1.3.  We should let
-        # user to decide.
+        def get_s(x):
+            """get score if it's cross validation history."""
+            return x[0] if isinstance(x, tuple) else x
+
+        def maximize(new, best):
+            return numpy.greater(get_s(new) + self._tol, get_s(best))
+
+        def minimize(new, best):
+            return numpy.greater(get_s(best) + self._tol, get_s(new))
+
         if self.maximize is None:
+            # Just to be compatibility with old behavior before 1.3.  We should let
+            # user to decide.
             maximize_metrics = ('auc', 'aucpr', 'map', 'ndcg', 'auc@',
                                 'aucpr@', 'map@', 'ndcg@')
             if any(metric.startswith(x) for x in maximize_metrics):
-                self.improve_op = lambda x, y: x > y
                 self.maximize = True
             else:
-                self.improve_op = lambda x, y: x < y
                 self.maximize = False
 
+        if self.maximize:
+            self.improve_op = maximize
+        else:
+            self.improve_op = minimize
+
+        assert self.improve_op
+
         if not self.stopping_history:  # First round
             self.current_rounds = 0
             self.stopping_history[name] = {}