Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Re-implement ROC-AUC. (#6747)

Browse Source 
* Re-implement ROC-AUC.

* Binary
* MultiClass
* LTR
* Add documents.

This PR resolves a few issues:
  - Define a value when the dataset is invalid, which can happen if there's an
  empty dataset, or when the dataset contains only positive or negative values.
  - Define ROC-AUC for multi-class classification.
  - Define weighted average value for distributed setting.
  - A correct implementation for learning to rank task.  Previous
  implementation is just binary classification with averaging across groups,
  which doesn't measure ordered learning to rank.
This commit is contained in:
Jiaming Yuan
2021-03-20 16:52:40 +08:00
committed by GitHub
parent 4ee8340e79
commit bcc0277338
27 changed files with 1622 additions and 461 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
tests/cpp/common/test_common.cc
View File

@@ -1,11 +1,12 @@
#include <gtest/gtest.h>
#include <xgboost/span.h>
#include "../../../src/common/common.h"
namespace xgboost {
namespace common {
TEST(ArgSort, Basic) {
std::vector<float> inputs {3.0, 2.0, 1.0};
auto ret = ArgSort<bst_feature_t>(inputs);
auto ret = ArgSort<bst_feature_t>(Span<float>{inputs});
std::vector<bst_feature_t> sol{2, 1, 0};
ASSERT_EQ(ret, sol);
}

66
tests/cpp/common/test_ranking_utils.cu Normal file
View File

@@ -0,0 +1,66 @@
#include <gtest/gtest.h>
#include "../../../src/common/ranking_utils.cuh"
#include "../../../src/common/device_helpers.cuh"
namespace xgboost {
namespace common {
TEST(SegmentedTrapezoidThreads, Basic) {
size_t constexpr kElements = 24, kGroups = 3;
dh::device_vector<size_t> offset_ptr(kGroups + 1, 0);
offset_ptr[0] = 0;
offset_ptr[1] = 8;
offset_ptr[2] = 16;
offset_ptr[kGroups] = kElements;
size_t h = 1;
dh::device_vector<size_t> thread_ptr(kGroups + 1, 0);
size_t total = SegmentedTrapezoidThreads(dh::ToSpan(offset_ptr), dh::ToSpan(thread_ptr), h);
ASSERT_EQ(total, kElements - kGroups);
h = 2;
SegmentedTrapezoidThreads(dh::ToSpan(offset_ptr), dh::ToSpan(thread_ptr), h);
std::vector<size_t> h_thread_ptr(thread_ptr.size());
thrust::copy(thread_ptr.cbegin(), thread_ptr.cend(), h_thread_ptr.begin());
for (size_t i = 1; i < h_thread_ptr.size(); ++i) {
ASSERT_EQ(h_thread_ptr[i] - h_thread_ptr[i - 1], 13);
}
h = 7;
SegmentedTrapezoidThreads(dh::ToSpan(offset_ptr), dh::ToSpan(thread_ptr), h);
thrust::copy(thread_ptr.cbegin(), thread_ptr.cend(), h_thread_ptr.begin());
for (size_t i = 1; i < h_thread_ptr.size(); ++i) {
ASSERT_EQ(h_thread_ptr[i] - h_thread_ptr[i - 1], 28);
}
}
TEST(SegmentedTrapezoidThreads, Unravel) {
size_t i = 0, j = 0;
size_t constexpr kN = 8;
UnravelTrapeziodIdx(6, kN, &i, &j);
ASSERT_EQ(i, 0);
ASSERT_EQ(j, 7);
UnravelTrapeziodIdx(12, kN, &i, &j);
ASSERT_EQ(i, 1);
ASSERT_EQ(j, 7);
UnravelTrapeziodIdx(15, kN, &i, &j);
ASSERT_EQ(i, 2);
ASSERT_EQ(j, 5);
UnravelTrapeziodIdx(21, kN, &i, &j);
ASSERT_EQ(i, 3);
ASSERT_EQ(j, 7);
UnravelTrapeziodIdx(25, kN, &i, &j);
ASSERT_EQ(i, 5);
ASSERT_EQ(j, 6);
UnravelTrapeziodIdx(27, kN, &i, &j);
ASSERT_EQ(i, 6);
ASSERT_EQ(j, 7);
}
} // namespace common
} // namespace xgboost

133
tests/cpp/metric/test_auc.cc Normal file
View File

@@ -0,0 +1,133 @@
#include <xgboost/metric.h>
#include "../helpers.h"
namespace xgboost {
namespace metric {
TEST(Metric, DeclareUnifiedTest(BinaryAUC)) {
auto tparam = xgboost::CreateEmptyGenericParam(GPUIDX);
std::unique_ptr<Metric> uni_ptr {Metric::Create("auc", &tparam)};
Metric * metric = uni_ptr.get();
ASSERT_STREQ(metric->Name(), "auc");
// Binary
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1.0f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {1, 0}), 0.0f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {0, 0}, {0, 1}), 0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {1, 1}, {0, 1}), 0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {0, 0}, {1, 0}), 0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {1, 1}, {1, 0}), 0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {1, 0, 0}, {0, 0, 1}), 0.25f, 1e-10);
// Invalid dataset
MetaInfo info;
info.labels_ = {0, 0};
float auc = metric->Eval({1, 1}, info, false);
ASSERT_TRUE(std::isnan(auc));
info.labels_ = HostDeviceVector<float>{};
auc = metric->Eval(HostDeviceVector<float>{}, info, false);
ASSERT_TRUE(std::isnan(auc));
EXPECT_NEAR(GetMetricEval(metric, {0, 1, 0, 1}, {0, 1, 0, 1}), 1.0f, 1e-10);
// AUC with instance weights
EXPECT_NEAR(GetMetricEval(metric,
{0.9f, 0.1f, 0.4f, 0.3f},
{0, 0, 1, 1},
{1.0f, 3.0f, 2.0f, 4.0f}),
0.75f, 0.001f);
// regression test case
ASSERT_NEAR(GetMetricEval(
metric,
{0.79523796, 0.5201713, 0.79523796, 0.24273258, 0.53452194,
0.53452194, 0.24273258, 0.5201713, 0.79523796, 0.53452194,
0.24273258, 0.53452194, 0.79523796, 0.5201713, 0.24273258,
0.5201713, 0.5201713, 0.53452194, 0.5201713, 0.53452194},
{0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0}),
0.5, 1e-10);
}
TEST(Metric, DeclareUnifiedTest(MultiAUC)) {
auto tparam = CreateEmptyGenericParam(GPUIDX);
std::unique_ptr<Metric> uni_ptr{
Metric::Create("auc", &tparam)};
auto metric = uni_ptr.get();
// MultiClass
// 3x3
EXPECT_NEAR(GetMetricEval(metric,
{
1.0f, 0.0f, 0.0f, // p_0
0.0f, 1.0f, 0.0f, // p_1
0.0f, 0.0f, 1.0f // p_2
},
{0, 1, 2}),
1.0f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{
1.0f, 0.0f, 0.0f, // p_0
0.0f, 1.0f, 0.0f, // p_1
0.0f, 0.0f, 1.0f // p_2
},
{2, 1, 0}),
0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{
1.0f, 0.0f, 0.0f, // p_0
0.0f, 1.0f, 0.0f, // p_1
0.0f, 0.0f, 1.0f // p_2
},
{2, 0, 1}),
0.25f, 1e-10);
// invalid dataset
float auc = GetMetricEval(metric,
{
1.0f, 0.0f, 0.0f, // p_0
0.0f, 1.0f, 0.0f, // p_1
0.0f, 0.0f, 1.0f // p_2
},
{0, 1, 1}); // no class 2.
EXPECT_TRUE(std::isnan(auc)) << auc;
}
TEST(Metric, DeclareUnifiedTest(RankingAUC)) {
auto tparam = CreateEmptyGenericParam(GPUIDX);
std::unique_ptr<Metric> metric{Metric::Create("auc", &tparam)};
// single group
EXPECT_NEAR(GetMetricEval(metric.get(), {0.7f, 0.2f, 0.3f, 0.6f},
{1.0f, 0.8f, 0.4f, 0.2f}, /*weights=*/{},
{0, 4}),
0.5f, 1e-10);
// multi group
EXPECT_NEAR(GetMetricEval(metric.get(), {0, 1, 2, 0, 1, 2},
{0, 1, 2, 0, 1, 2}, /*weights=*/{}, {0, 3, 6}),
1.0f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric.get(), {0, 1, 2, 0, 1, 2},
{0, 1, 2, 0, 1, 2}, /*weights=*/{1.0f, 2.0f},
{0, 3, 6}),
1.0f, 1e-10);
// AUC metric for grouped datasets - exception scenarios
ASSERT_TRUE(std::isnan(
GetMetricEval(metric.get(), {0, 1, 2}, {0, 0, 0}, {}, {0, 2, 3})));
// regression case
HostDeviceVector<float> predt{0.33935383, 0.5149714, 0.32138085, 1.4547751,
1.2010975, 0.42651367, 0.23104341, 0.83610827,
0.8494239, 0.07136688, 0.5623144, 0.8086237,
1.5066161, -4.094787, 0.76887935, -2.4082742};
std::vector<bst_group_t> groups{0, 7, 16};
std::vector<float> labels{1., 0., 0., 1., 2., 1., 0., 0.,
0., 0., 0., 0., 1., 0., 1., 0.};
EXPECT_NEAR(GetMetricEval(metric.get(), std::move(predt), labels,
/*weights=*/{}, groups),
0.769841f, 1e-6);
}
} // namespace metric
} // namespace xgboost

5
tests/cpp/metric/test_auc.cu Normal file
View File

@@ -0,0 +1,5 @@
/*!
* Copyright 2021 XGBoost contributors
*/
// Dummy file to keep the CUDA conditional compile trick.
#include "test_auc.cc"

43
tests/cpp/metric/test_rank_metric.cc
View File

@@ -24,49 +24,6 @@ TEST(Metric, AMS) {
}
#endif
TEST(Metric, DeclareUnifiedTest(AUC)) {
auto tparam = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric * metric = xgboost::Metric::Create("auc", &tparam);
ASSERT_STREQ(metric->Name(), "auc");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}),
0.5f, 0.001f);
EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {}));
EXPECT_ANY_THROW(GetMetricEval(metric, {0, 0}, {0, 0}));
EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {1, 1}));
// AUC with instance weights
EXPECT_NEAR(GetMetricEval(metric,
{0.9f, 0.1f, 0.4f, 0.3f},
{0, 0, 1, 1},
{1.0f, 3.0f, 2.0f, 4.0f}),
0.75f, 0.001f);
// AUC for a ranking task without weights
EXPECT_NEAR(GetMetricEval(metric,
{0.9f, 0.1f, 0.4f, 0.3f, 0.7f},
{0, 1, 0, 1, 1},
{},
{0, 2, 5}),
0.25f, 0.001f);
// AUC for a ranking task with weights/group
EXPECT_NEAR(GetMetricEval(metric,
{0.9f, 0.1f, 0.4f, 0.3f, 0.7f},
{1, 0, 1, 0, 0},
{1, 2},
{0, 2, 5}),
0.75f, 0.001f);
// AUC metric for grouped datasets - exception scenarios
EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1, 2}, {0, 0, 0}, {}, {0, 2, 3}));
EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1, 2}, {1, 1, 1}, {}, {0, 2, 3}));
delete metric;
}
TEST(Metric, DeclareUnifiedTest(AUCPR)) {
auto tparam = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric *metric = xgboost::Metric::Create("aucpr", &tparam);