Support custom metric in sklearn ranker. (#8786)

tests/python/test_with_sklearn.py

@@ -154,6 +154,32 @@ def test_ranking():
     np.testing.assert_almost_equal(pred, pred_orig)
 
 
+def test_ranking_metric() -> None:
+    from sklearn.metrics import roc_auc_score
+
+    X, y, qid, w = tm.make_ltr(512, 4, 3, 2)
+    # use auc for test as ndcg_score in sklearn works only on label gain instead of exp
+    # gain.
+    # note that the auc in sklearn is different from the one in XGBoost. The one in
+    # sklearn compares the number of mis-classified docs, while the one in xgboost
+    # compares the number of mis-classified pairs.
+    ltr = xgb.XGBRanker(
+        eval_metric=roc_auc_score, n_estimators=10, tree_method="hist", max_depth=2
+    )
+    ltr.fit(
+        X,
+        y,
+        qid=qid,
+        sample_weight=w,
+        eval_set=[(X, y)],
+        eval_qid=[qid],
+        sample_weight_eval_set=[w],
+        verbose=True,
+    )
+    results = ltr.evals_result()
+    assert results["validation_0"]["roc_auc_score"][-1] > 0.6
+
+
 def test_stacking_regression():
     from sklearn.datasets import load_diabetes
     from sklearn.ensemble import RandomForestRegressor, StackingRegressor
@@ -1426,10 +1452,10 @@ def test_weighted_evaluation_metric():
     X_train, X_test = X[:1600], X[1600:]
     y_train, y_test = y[:1600], y[1600:]
     weights_eval_set = np.random.choice([1, 2], len(X_test))
-    
+
     np.random.seed(0)
     weights_train = np.random.choice([1, 2], len(X_train))
-    
+
     clf = xgb.XGBClassifier(
         tree_method="hist",
         eval_metric=log_loss,