Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Re-implement ROC-AUC. (#6747)

Browse Source 
* Re-implement ROC-AUC.

* Binary
* MultiClass
* LTR
* Add documents.

This PR resolves a few issues:
  - Define a value when the dataset is invalid, which can happen if there's an
  empty dataset, or when the dataset contains only positive or negative values.
  - Define ROC-AUC for multi-class classification.
  - Define weighted average value for distributed setting.
  - A correct implementation for learning to rank task.  Previous
  implementation is just binary classification with averaging across groups,
  which doesn't measure ordered learning to rank.
This commit is contained in:
Jiaming Yuan
2021-03-20 16:52:40 +08:00
committed by GitHub
parent 4ee8340e79
commit bcc0277338
27 changed files with 1622 additions and 461 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
tests/cpp/metric/test_rank_metric.cc
View File

@@ -24,49 +24,6 @@ TEST(Metric, AMS) {
}
#endif
TEST(Metric, DeclareUnifiedTest(AUC)) {
auto tparam = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric * metric = xgboost::Metric::Create("auc", &tparam);
ASSERT_STREQ(metric->Name(), "auc");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}),
0.5f, 0.001f);
EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {}));
EXPECT_ANY_THROW(GetMetricEval(metric, {0, 0}, {0, 0}));
EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {1, 1}));
// AUC with instance weights
EXPECT_NEAR(GetMetricEval(metric,
{0.9f, 0.1f, 0.4f, 0.3f},
{0, 0, 1, 1},
{1.0f, 3.0f, 2.0f, 4.0f}),
0.75f, 0.001f);
// AUC for a ranking task without weights
EXPECT_NEAR(GetMetricEval(metric,
{0.9f, 0.1f, 0.4f, 0.3f, 0.7f},
{0, 1, 0, 1, 1},
{},
{0, 2, 5}),
0.25f, 0.001f);
// AUC for a ranking task with weights/group
EXPECT_NEAR(GetMetricEval(metric,
{0.9f, 0.1f, 0.4f, 0.3f, 0.7f},
{1, 0, 1, 0, 0},
{1, 2},
{0, 2, 5}),
0.75f, 0.001f);
// AUC metric for grouped datasets - exception scenarios
EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1, 2}, {0, 0, 0}, {}, {0, 2, 3}));
EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1, 2}, {1, 1, 1}, {}, {0, 2, 3}));
delete metric;
}
TEST(Metric, DeclareUnifiedTest(AUCPR)) {
auto tparam = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric *metric = xgboost::Metric::Create("aucpr", &tparam);