Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make sure metrics work with column-wise distributed training (#9020)

Browse Source
This commit is contained in:
Rong Ou 2023-04-17 12:48:23 -07:00 committed by GitHub
parent 191d0aa5cf
commit ba9d24ff7b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 1183 additions and 811 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/metric/auc.cc
View File

@ -116,8 +116,10 @@ double MultiClassOVR(Context const *ctx, common::Span<float const> predts, MetaI
// we have 2 averages going in here, first is among workers, second is among
// classes. allreduce sums up fp/tp auc for each class.
if (info.IsRowSplit()) {
collective::Allreduce<collective::Operation::kSum>(results.Values().data(),
results.Values().size());
}
double auc_sum{0};
double tp_sum{0};
for (size_t c = 0; c < n_classes; ++c) {
@ -290,7 +292,9 @@ class EvalAUC : public MetricNoCache {
}
std::array<double, 2> results{auc, static_cast<double>(valid_groups)};
if (info.IsRowSplit()) {
collective::Allreduce<collective::Operation::kSum>(results.data(), results.size());
}
auc = results[0];
valid_groups = static_cast<uint32_t>(results[1]);
@ -319,7 +323,9 @@ class EvalAUC : public MetricNoCache {
}
double local_area = fp * tp;
std::array<double, 2> result{auc, local_area};
if (info.IsRowSplit()) {
collective::Allreduce<collective::Operation::kSum>(result.data(), result.size());
}
std::tie(auc, local_area) = common::UnpackArr(std::move(result));
if (local_area <= 0) {
// the dataset across all workers have only positive or negative sample

8
src/metric/elementwise_metric.cu
View File

@ -198,7 +198,7 @@ class PseudoErrorLoss : public MetricNoCache {
return std::make_tuple(v, wt);
});
double dat[2]{result.Residue(), result.Weights()};
if (collective::IsDistributed()) {
if (info.IsRowSplit()) {
collective::Allreduce<collective::Operation::kSum>(dat, 2);
}
return EvalRowMAPE::GetFinal(dat[0], dat[1]);
@ -367,7 +367,9 @@ struct EvalEWiseBase : public MetricNoCache {
});
double dat[2]{result.Residue(), result.Weights()};
if (info.IsRowSplit()) {
collective::Allreduce<collective::Operation::kSum>(dat, 2);
}
return Policy::GetFinal(dat[0], dat[1]);
}
@ -439,7 +441,9 @@ class QuantileError : public MetricNoCache {
if (info.num_row_ == 0) {
// empty DMatrix on distributed env
double dat[2]{0.0, 0.0};
if (info.IsRowSplit()) {
collective::Allreduce<collective::Operation::kSum>(dat, 2);
}
CHECK_GT(dat[1], 0);
return dat[0] / dat[1];
}
@ -477,7 +481,9 @@ class QuantileError : public MetricNoCache {
return std::make_tuple(l, w);
});
double dat[2]{result.Residue(), result.Weights()};
if (info.IsRowSplit()) {
collective::Allreduce<collective::Operation::kSum>(dat, 2);
}
CHECK_GT(dat[1], 0);
return dat[0] / dat[1];
}

2
src/metric/multiclass_metric.cu
View File

@ -181,7 +181,9 @@ struct EvalMClassBase : public MetricNoCache {
dat[0] = result.Residue();
dat[1] = result.Weights();
}
if (info.IsRowSplit()) {
collective::Allreduce<collective::Operation::kSum>(dat, 2);
}
return Derived::GetFinal(dat[0], dat[1]);
}
/*!

14
src/metric/rank_metric.cc
View File

@ -244,7 +244,7 @@ struct EvalRank : public MetricNoCache, public EvalRankConfig {
exc.Rethrow();
}
if (collective::IsDistributed()) {
if (collective::IsDistributed() && info.IsRowSplit()) {
double dat[2]{sum_metric, static_cast<double>(ngroups)};
// approximately estimate the metric using mean
collective::Allreduce<collective::Operation::kSum>(dat, 2);
@ -401,9 +401,11 @@ class EvalRankWithCache : public Metric {
};
namespace {
double Finalize(double score, double sw) {
double Finalize(MetaInfo const& info, double score, double sw) {
std::array<double, 2> dat{score, sw};
if (info.IsRowSplit()) {
collective::Allreduce<collective::Operation::kSum>(dat.data(), dat.size());
}
if (sw > 0.0) {
score = score / sw;
}
@ -430,7 +432,7 @@ class EvalNDCG : public EvalRankWithCache<ltr::NDCGCache> {
std::shared_ptr<ltr::NDCGCache> p_cache) override {
if (ctx_->IsCUDA()) {
auto ndcg = cuda_impl::NDCGScore(ctx_, info, preds, minus_, p_cache);
return Finalize(ndcg.Residue(), ndcg.Weights());
return Finalize(info, ndcg.Residue(), ndcg.Weights());
}
// group local ndcg
@ -476,7 +478,7 @@ class EvalNDCG : public EvalRankWithCache<ltr::NDCGCache> {
sum_w = std::accumulate(weights.weights.cbegin(), weights.weights.cend(), 0.0);
}
auto ndcg = std::accumulate(linalg::cbegin(ndcg_gloc), linalg::cend(ndcg_gloc), 0.0);
return Finalize(ndcg, sum_w);
return Finalize(info, ndcg, sum_w);
}
};
@ -489,7 +491,7 @@ class EvalMAPScore : public EvalRankWithCache<ltr::MAPCache> {
std::shared_ptr<ltr::MAPCache> p_cache) override {
if (ctx_->IsCUDA()) {
auto map = cuda_impl::MAPScore(ctx_, info, predt, minus_, p_cache);
return Finalize(map.Residue(), map.Weights());
return Finalize(info, map.Residue(), map.Weights());
}
auto gptr = p_cache->DataGroupPtr(ctx_);
@ -532,7 +534,7 @@ class EvalMAPScore : public EvalRankWithCache<ltr::MAPCache> {
sw += weight[i];
}
auto sum = std::accumulate(map_gloc.cbegin(), map_gloc.cend(), 0.0);
return Finalize(sum, sw);
return Finalize(info, sum, sw);
}
};

2
src/metric/survival_metric.cu
View File

@ -212,7 +212,9 @@ struct EvalEWiseSurvivalBase : public MetricNoCache {
info.labels_upper_bound_, preds);
double dat[2]{result.Residue(), result.Weights()};
if (info.IsRowSplit()) {
collective::Allreduce<collective::Operation::kSum>(dat, 2);
}
return Policy::GetFinal(dat[0], dat[1]);
}

9
tests/cpp/helpers.cc
View File

@ -167,18 +167,20 @@ xgboost::bst_float GetMetricEval(xgboost::Metric* metric,
xgboost::HostDeviceVector<xgboost::bst_float> const& preds,
std::vector<xgboost::bst_float> labels,
std::vector<xgboost::bst_float> weights,
std::vector<xgboost::bst_uint> groups) {
std::vector<xgboost::bst_uint> groups,
xgboost::DataSplitMode data_split_mode) {
return GetMultiMetricEval(
metric, preds,
xgboost::linalg::Tensor<float, 2>{labels.begin(), labels.end(), {labels.size()}, -1}, weights,
groups);
groups, data_split_mode);
}
double GetMultiMetricEval(xgboost::Metric* metric,
xgboost::HostDeviceVector<xgboost::bst_float> const& preds,
xgboost::linalg::Tensor<float, 2> const& labels,
std::vector<xgboost::bst_float> weights,
std::vector<xgboost::bst_uint> groups) {
std::vector<xgboost::bst_uint> groups,
xgboost::DataSplitMode data_split_mode) {
std::shared_ptr<xgboost::DMatrix> p_fmat{xgboost::RandomDataGenerator{0, 0, 0}.GenerateDMatrix()};
auto& info = p_fmat->Info();
info.num_row_ = labels.Shape(0);
@ -186,6 +188,7 @@ double GetMultiMetricEval(xgboost::Metric* metric,
info.labels.Data()->Copy(*labels.Data());
info.weights_.HostVector() = weights;
info.group_ptr_ = groups;
info.data_split_mode = data_split_mode;
return metric->Evaluate(preds, p_fmat);
}

31
tests/cpp/helpers.h
View File

@ -39,6 +39,18 @@
#define GPUIDX -1
#endif
#if defined(__CUDACC__)
#define DeclareUnifiedDistributedTest(name) MGPU ## name
#else
#define DeclareUnifiedDistributedTest(name) name
#endif
#if defined(__CUDACC__)
#define WORLD_SIZE_FOR_TEST (xgboost::common::AllVisibleGPUs())
#else
#define WORLD_SIZE_FOR_TEST (3)
#endif
namespace xgboost {
class ObjFunction;
class Metric;
@ -92,13 +104,15 @@ xgboost::bst_float GetMetricEval(
xgboost::HostDeviceVector<xgboost::bst_float> const& preds,
std::vector<xgboost::bst_float> labels,
std::vector<xgboost::bst_float> weights = std::vector<xgboost::bst_float>(),
std::vector<xgboost::bst_uint> groups = std::vector<xgboost::bst_uint>());
std::vector<xgboost::bst_uint> groups = std::vector<xgboost::bst_uint>(),
xgboost::DataSplitMode data_split_Mode = xgboost::DataSplitMode::kRow);
double GetMultiMetricEval(xgboost::Metric* metric,
xgboost::HostDeviceVector<xgboost::bst_float> const& preds,
xgboost::linalg::Tensor<float, 2> const& labels,
std::vector<xgboost::bst_float> weights = {},
std::vector<xgboost::bst_uint> groups = {});
std::vector<xgboost::bst_uint> groups = {},
xgboost::DataSplitMode data_split_Mode = xgboost::DataSplitMode::kRow);
namespace xgboost {
@ -496,4 +510,17 @@ void RunWithInMemoryCommunicator(int32_t world_size, Function&& function, Args&&
thread.join();
}
}
class DeclareUnifiedDistributedTest(MetricTest) : public ::testing::Test {
protected:
int world_size_;
void SetUp() override {
world_size_ = WORLD_SIZE_FOR_TEST;
if (world_size_ <= 1) {
GTEST_SKIP() << "Skipping MGPU test with # GPUs = " << world_size_;
}
}
};
} // namespace xgboost

271
tests/cpp/metric/test_auc.cc
View File

@ -1,261 +1,68 @@
#include "test_auc.h"
#include <xgboost/metric.h>
#include "../helpers.h"
namespace xgboost {
namespace metric {
TEST(Metric, DeclareUnifiedTest(BinaryAUC)) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
std::unique_ptr<Metric> uni_ptr {Metric::Create("auc", &ctx)};
Metric * metric = uni_ptr.get();
ASSERT_STREQ(metric->Name(), "auc");
TEST(Metric, DeclareUnifiedTest(BinaryAUC)) { VerifyBinaryAUC(); }
// Binary
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1.0f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {1, 0}), 0.0f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {0, 0}, {0, 1}), 0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {1, 1}, {0, 1}), 0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {0, 0}, {1, 0}), 0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {1, 1}, {1, 0}), 0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {1, 0, 0}, {0, 0, 1}), 0.25f, 1e-10);
TEST(Metric, DeclareUnifiedTest(MultiClassAUC)) { VerifyMultiClassAUC(); }
// Invalid dataset
auto p_fmat = EmptyDMatrix();
MetaInfo& info = p_fmat->Info();
info.labels = linalg::Tensor<float, 2>{{0.0f, 0.0f}, {2}, -1};
float auc = metric->Evaluate({1, 1}, p_fmat);
ASSERT_TRUE(std::isnan(auc));
*info.labels.Data() = HostDeviceVector<float>{};
auc = metric->Evaluate(HostDeviceVector<float>{}, p_fmat);
ASSERT_TRUE(std::isnan(auc));
TEST(Metric, DeclareUnifiedTest(RankingAUC)) { VerifyRankingAUC(); }
EXPECT_NEAR(GetMetricEval(metric, {0, 1, 0, 1}, {0, 1, 0, 1}), 1.0f, 1e-10);
TEST(Metric, DeclareUnifiedTest(PRAUC)) { VerifyPRAUC(); }
// AUC with instance weights
EXPECT_NEAR(GetMetricEval(metric,
{0.9f, 0.1f, 0.4f, 0.3f},
{0, 0, 1, 1},
{1.0f, 3.0f, 2.0f, 4.0f}),
0.75f, 0.001f);
TEST(Metric, DeclareUnifiedTest(MultiClassPRAUC)) { VerifyMultiClassPRAUC(); }
// regression test case
ASSERT_NEAR(GetMetricEval(
metric,
{0.79523796, 0.5201713, 0.79523796, 0.24273258, 0.53452194,
0.53452194, 0.24273258, 0.5201713, 0.79523796, 0.53452194,
0.24273258, 0.53452194, 0.79523796, 0.5201713, 0.24273258,
0.5201713, 0.5201713, 0.53452194, 0.5201713, 0.53452194},
{0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0}),
0.5, 1e-10);
TEST(Metric, DeclareUnifiedTest(RankingPRAUC)) { VerifyRankingPRAUC(); }
TEST_F(DeclareUnifiedDistributedTest(MetricTest), BinaryAUCRowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyBinaryAUC, DataSplitMode::kRow);
}
TEST(Metric, DeclareUnifiedTest(MultiClassAUC)) {
auto ctx = CreateEmptyGenericParam(GPUIDX);
std::unique_ptr<Metric> uni_ptr{
Metric::Create("auc", &ctx)};
auto metric = uni_ptr.get();
// MultiClass
// 3x3
EXPECT_NEAR(GetMetricEval(metric,
{
1.0f, 0.0f, 0.0f, // p_0
0.0f, 1.0f, 0.0f, // p_1
0.0f, 0.0f, 1.0f // p_2
},
{0, 1, 2}),
1.0f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{
1.0f, 0.0f, 0.0f, // p_0
0.0f, 1.0f, 0.0f, // p_1
0.0f, 0.0f, 1.0f // p_2
},
{0, 1, 2},
{1.0f, 1.0f, 1.0f}),
1.0f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{
1.0f, 0.0f, 0.0f, // p_0
0.0f, 1.0f, 0.0f, // p_1
0.0f, 0.0f, 1.0f // p_2
},
{2, 1, 0}),
0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{
1.0f, 0.0f, 0.0f, // p_0
0.0f, 1.0f, 0.0f, // p_1
0.0f, 0.0f, 1.0f // p_2
},
{2, 0, 1}),
0.25f, 1e-10);
// invalid dataset
float auc = GetMetricEval(metric,
{
1.0f, 0.0f, 0.0f, // p_0
0.0f, 1.0f, 0.0f, // p_1
0.0f, 0.0f, 1.0f // p_2
},
{0, 1, 1}); // no class 2.
EXPECT_TRUE(std::isnan(auc)) << auc;
HostDeviceVector<float> predts{
0.0f, 1.0f, 0.0f,
1.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f,
0.0f, 0.0f, 1.0f,
};
std::vector<float> labels {1.0f, 0.0f, 2.0f, 1.0f};
auc = GetMetricEval(metric, predts, labels, {1.0f, 2.0f, 3.0f, 4.0f});
ASSERT_GT(auc, 0.714);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), BinaryAUCColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyBinaryAUC, DataSplitMode::kCol);
}
TEST(Metric, DeclareUnifiedTest(RankingAUC)) {
auto ctx = CreateEmptyGenericParam(GPUIDX);
std::unique_ptr<Metric> metric{Metric::Create("auc", &ctx)};
// single group
EXPECT_NEAR(GetMetricEval(metric.get(), {0.7f, 0.2f, 0.3f, 0.6f},
{1.0f, 0.8f, 0.4f, 0.2f}, /*weights=*/{},
{0, 4}),
0.5f, 1e-10);
// multi group
EXPECT_NEAR(GetMetricEval(metric.get(), {0, 1, 2, 0, 1, 2},
{0, 1, 2, 0, 1, 2}, /*weights=*/{}, {0, 3, 6}),
1.0f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric.get(), {0, 1, 2, 0, 1, 2},
{0, 1, 2, 0, 1, 2}, /*weights=*/{1.0f, 2.0f},
{0, 3, 6}),
1.0f, 1e-10);
// AUC metric for grouped datasets - exception scenarios
ASSERT_TRUE(std::isnan(
GetMetricEval(metric.get(), {0, 1, 2}, {0, 0, 0}, {}, {0, 2, 3})));
// regression case
HostDeviceVector<float> predt{0.33935383, 0.5149714, 0.32138085, 1.4547751,
1.2010975, 0.42651367, 0.23104341, 0.83610827,
0.8494239, 0.07136688, 0.5623144, 0.8086237,
1.5066161, -4.094787, 0.76887935, -2.4082742};
std::vector<bst_group_t> groups{0, 7, 16};
std::vector<float> labels{1., 0., 0., 1., 2., 1., 0., 0.,
0., 0., 0., 0., 1., 0., 1., 0.};
EXPECT_NEAR(GetMetricEval(metric.get(), std::move(predt), labels,
/*weights=*/{}, groups),
0.769841f, 1e-6);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiClassAUCRowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyMultiClassAUC, DataSplitMode::kRow);
}
TEST(Metric, DeclareUnifiedTest(PRAUC)) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric *metric = xgboost::Metric::Create("aucpr", &ctx);
ASSERT_STREQ(metric->Name(), "aucpr");
EXPECT_NEAR(GetMetricEval(metric, {0, 0, 1, 1}, {0, 0, 1, 1}), 1, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {0.1f, 0.9f, 0.1f, 0.9f}, {0, 0, 1, 1}),
0.5f, 0.001f);
EXPECT_NEAR(GetMetricEval(
metric,
{0.4f, 0.2f, 0.9f, 0.1f, 0.2f, 0.4f, 0.1f, 0.1f, 0.2f, 0.1f},
{0, 0, 0, 0, 0, 1, 0, 0, 1, 1}),
0.2908445f, 0.001f);
EXPECT_NEAR(GetMetricEval(
metric, {0.87f, 0.31f, 0.40f, 0.42f, 0.25f, 0.66f, 0.95f,
0.09f, 0.10f, 0.97f, 0.76f, 0.69f, 0.15f, 0.20f,
0.30f, 0.14f, 0.07f, 0.58f, 0.61f, 0.08f},
{0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1}),
0.2769199f, 0.001f);
auto auc = GetMetricEval(metric, {0, 1}, {});
ASSERT_TRUE(std::isnan(auc));
// AUCPR with instance weights
EXPECT_NEAR(GetMetricEval(metric,
{0.29f, 0.52f, 0.11f, 0.21f, 0.219f, 0.93f, 0.493f,
0.17f, 0.47f, 0.13f, 0.43f, 0.59f, 0.87f, 0.007f},
{0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0},
{1, 2, 7, 4, 5, 2.2f, 3.2f, 5, 6, 1, 2, 1.1f, 3.2f,
4.5f}), // weights
0.694435f, 0.001f);
// Both groups contain only pos or neg samples.
auc = GetMetricEval(metric,
{0, 0.1f, 0.3f, 0.5f, 0.7f},
{1, 1, 0, 0, 0},
{},
{0, 2, 5});
ASSERT_TRUE(std::isnan(auc));
delete metric;
TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiClassAUCColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyMultiClassAUC, DataSplitMode::kCol);
}
TEST(Metric, DeclareUnifiedTest(MultiClassPRAUC)) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
std::unique_ptr<Metric> metric{Metric::Create("aucpr", &ctx)};
float auc = 0;
std::vector<float> labels {1.0f, 0.0f, 2.0f};
HostDeviceVector<float> predts{
0.0f, 1.0f, 0.0f,
1.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f,
};
auc = GetMetricEval(metric.get(), predts, labels, {});
EXPECT_EQ(auc, 1.0f);
auc = GetMetricEval(metric.get(), predts, labels, {1.0f, 1.0f, 1.0f});
EXPECT_EQ(auc, 1.0f);
predts.HostVector() = {
0.0f, 1.0f, 0.0f,
1.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f,
0.0f, 0.0f, 1.0f,
};
labels = {1.0f, 0.0f, 2.0f, 1.0f};
auc = GetMetricEval(metric.get(), predts, labels, {1.0f, 2.0f, 3.0f, 4.0f});
ASSERT_GT(auc, 0.699);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), RankingAUCRowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyRankingAUC, DataSplitMode::kRow);
}
TEST(Metric, DeclareUnifiedTest(RankingPRAUC)) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), RankingAUCColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyRankingAUC, DataSplitMode::kCol);
}
std::unique_ptr<Metric> metric{Metric::Create("aucpr", &ctx)};
TEST_F(DeclareUnifiedDistributedTest(MetricTest), PRAUCRowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyPRAUC, DataSplitMode::kRow);
}
std::vector<float> labels {1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f};
std::vector<uint32_t> groups {0, 2, 6};
TEST_F(DeclareUnifiedDistributedTest(MetricTest), PRAUCColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyPRAUC, DataSplitMode::kCol);
}
float auc = 0;
auc = GetMetricEval(metric.get(), {1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f}, labels, {}, groups);
EXPECT_EQ(auc, 1.0f);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiClassPRAUCRowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyMultiClassPRAUC, DataSplitMode::kRow);
}
auc = GetMetricEval(metric.get(), {1.0f, 0.5f, 0.8f, 0.3f, 0.2f, 1.0f}, labels, {}, groups);
EXPECT_EQ(auc, 1.0f);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiClassPRAUCColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyMultiClassPRAUC, DataSplitMode::kCol);
}
auc = GetMetricEval(metric.get(), {1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f},
{1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f}, {}, groups);
ASSERT_TRUE(std::isnan(auc));
TEST_F(DeclareUnifiedDistributedTest(MetricTest), RankingPRAUCRowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyRankingPRAUC, DataSplitMode::kRow);
}
// Incorrect label
ASSERT_THROW(GetMetricEval(metric.get(), {1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f},
{1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 3.0f}, {}, groups),
dmlc::Error);
// AUCPR with groups and no weights
EXPECT_NEAR(GetMetricEval(
metric.get(), {0.87f, 0.31f, 0.40f, 0.42f, 0.25f, 0.66f, 0.95f,
0.09f, 0.10f, 0.97f, 0.76f, 0.69f, 0.15f, 0.20f,
0.30f, 0.14f, 0.07f, 0.58f, 0.61f, 0.08f},
{0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1},
{}, // weights
{0, 2, 5, 9, 14, 20}), // group info
0.556021f, 0.001f);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), RankingPRAUCColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyRankingPRAUC, DataSplitMode::kCol);
}
} // namespace metric
} // namespace xgboost

249
tests/cpp/metric/test_auc.h Normal file
View File

@ -0,0 +1,249 @@
/*!
* Copyright (c) 2023 by XGBoost Contributors
*/
#pragma once
#include <xgboost/metric.h>
#include "../helpers.h"
namespace xgboost {
namespace metric {
inline void VerifyBinaryAUC(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
std::unique_ptr<Metric> uni_ptr{Metric::Create("auc", &ctx)};
Metric* metric = uni_ptr.get();
ASSERT_STREQ(metric->Name(), "auc");
// Binary
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 1.0f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {1, 0}, {}, {}, data_split_mode), 0.0f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {0, 0}, {0, 1}, {}, {}, data_split_mode), 0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {1, 1}, {0, 1}, {}, {}, data_split_mode), 0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {0, 0}, {1, 0}, {}, {}, data_split_mode), 0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {1, 1}, {1, 0}, {}, {}, data_split_mode), 0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {1, 0, 0}, {0, 0, 1}, {}, {}, data_split_mode), 0.25f, 1e-10);
// Invalid dataset
auto p_fmat = EmptyDMatrix();
MetaInfo& info = p_fmat->Info();
info.labels = linalg::Tensor<float, 2>{{0.0f, 0.0f}, {2}, -1};
float auc = metric->Evaluate({1, 1}, p_fmat);
ASSERT_TRUE(std::isnan(auc));
*info.labels.Data() = HostDeviceVector<float>{};
auc = metric->Evaluate(HostDeviceVector<float>{}, p_fmat);
ASSERT_TRUE(std::isnan(auc));
EXPECT_NEAR(GetMetricEval(metric, {0, 1, 0, 1}, {0, 1, 0, 1}, {}, {}, data_split_mode), 1.0f,
1e-10);
// AUC with instance weights
EXPECT_NEAR(GetMetricEval(metric, {0.9f, 0.1f, 0.4f, 0.3f}, {0, 0, 1, 1},
{1.0f, 3.0f, 2.0f, 4.0f}, {}, data_split_mode),
0.75f, 0.001f);
// regression test case
ASSERT_NEAR(GetMetricEval(metric, {0.79523796, 0.5201713, 0.79523796, 0.24273258, 0.53452194,
0.53452194, 0.24273258, 0.5201713, 0.79523796, 0.53452194,
0.24273258, 0.53452194, 0.79523796, 0.5201713, 0.24273258,
0.5201713, 0.5201713, 0.53452194, 0.5201713, 0.53452194},
{0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0}, {}, {},
data_split_mode),
0.5, 1e-10);
}
inline void VerifyMultiClassAUC(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = CreateEmptyGenericParam(GPUIDX);
std::unique_ptr<Metric> uni_ptr{Metric::Create("auc", &ctx)};
auto metric = uni_ptr.get();
// MultiClass
// 3x3
EXPECT_NEAR(GetMetricEval(metric,
{
1.0f, 0.0f, 0.0f, // p_0
0.0f, 1.0f, 0.0f, // p_1
0.0f, 0.0f, 1.0f // p_2
},
{0, 1, 2}, {}, {}, data_split_mode),
1.0f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{
1.0f, 0.0f, 0.0f, // p_0
0.0f, 1.0f, 0.0f, // p_1
0.0f, 0.0f, 1.0f // p_2
},
{0, 1, 2}, {1.0f, 1.0f, 1.0f}, {}, data_split_mode),
1.0f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{
1.0f, 0.0f, 0.0f, // p_0
0.0f, 1.0f, 0.0f, // p_1
0.0f, 0.0f, 1.0f // p_2
},
{2, 1, 0}, {}, {}, data_split_mode),
0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{
1.0f, 0.0f, 0.0f, // p_0
0.0f, 1.0f, 0.0f, // p_1
0.0f, 0.0f, 1.0f // p_2
},
{2, 0, 1}, {}, {}, data_split_mode),
0.25f, 1e-10);
// invalid dataset
float auc = GetMetricEval(metric,
{
1.0f, 0.0f, 0.0f, // p_0
0.0f, 1.0f, 0.0f, // p_1
0.0f, 0.0f, 1.0f // p_2
},
{0, 1, 1}, {}, {}, data_split_mode); // no class 2.
EXPECT_TRUE(std::isnan(auc)) << auc;
HostDeviceVector<float> predts{
0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f,
};
std::vector<float> labels{1.0f, 0.0f, 2.0f, 1.0f};
auc = GetMetricEval(metric, predts, labels, {1.0f, 2.0f, 3.0f, 4.0f}, {}, data_split_mode);
ASSERT_GT(auc, 0.714);
}
inline void VerifyRankingAUC(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = CreateEmptyGenericParam(GPUIDX);
std::unique_ptr<Metric> metric{Metric::Create("auc", &ctx)};
// single group
EXPECT_NEAR(GetMetricEval(metric.get(), {0.7f, 0.2f, 0.3f, 0.6f}, {1.0f, 0.8f, 0.4f, 0.2f},
/*weights=*/{}, {0, 4}, data_split_mode),
0.5f, 1e-10);
// multi group
EXPECT_NEAR(GetMetricEval(metric.get(), {0, 1, 2, 0, 1, 2}, {0, 1, 2, 0, 1, 2}, /*weights=*/{},
{0, 3, 6}, data_split_mode),
1.0f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric.get(), {0, 1, 2, 0, 1, 2}, {0, 1, 2, 0, 1, 2},
/*weights=*/{1.0f, 2.0f}, {0, 3, 6}, data_split_mode),
1.0f, 1e-10);
// AUC metric for grouped datasets - exception scenarios
ASSERT_TRUE(std::isnan(
GetMetricEval(metric.get(), {0, 1, 2}, {0, 0, 0}, {}, {0, 2, 3}, data_split_mode)));
// regression case
HostDeviceVector<float> predt{
0.33935383, 0.5149714, 0.32138085, 1.4547751, 1.2010975, 0.42651367, 0.23104341, 0.83610827,
0.8494239, 0.07136688, 0.5623144, 0.8086237, 1.5066161, -4.094787, 0.76887935, -2.4082742};
std::vector<bst_group_t> groups{0, 7, 16};
std::vector<float> labels{1., 0., 0., 1., 2., 1., 0., 0., 0., 0., 0., 0., 1., 0., 1., 0.};
EXPECT_NEAR(GetMetricEval(metric.get(), std::move(predt), labels,
/*weights=*/{}, groups, data_split_mode),
0.769841f, 1e-6);
}
inline void VerifyPRAUC(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric* metric = xgboost::Metric::Create("aucpr", &ctx);
ASSERT_STREQ(metric->Name(), "aucpr");
EXPECT_NEAR(GetMetricEval(metric, {0, 0, 1, 1}, {0, 0, 1, 1}, {}, {}, data_split_mode), 1, 1e-10);
EXPECT_NEAR(
GetMetricEval(metric, {0.1f, 0.9f, 0.1f, 0.9f}, {0, 0, 1, 1}, {}, {}, data_split_mode), 0.5f,
0.001f);
EXPECT_NEAR(GetMetricEval(metric, {0.4f, 0.2f, 0.9f, 0.1f, 0.2f, 0.4f, 0.1f, 0.1f, 0.2f, 0.1f},
{0, 0, 0, 0, 0, 1, 0, 0, 1, 1}, {}, {}, data_split_mode),
0.2908445f, 0.001f);
EXPECT_NEAR(
GetMetricEval(metric, {0.87f, 0.31f, 0.40f, 0.42f, 0.25f, 0.66f, 0.95f, 0.09f, 0.10f, 0.97f,
0.76f, 0.69f, 0.15f, 0.20f, 0.30f, 0.14f, 0.07f, 0.58f, 0.61f, 0.08f},
{0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1}, {}, {},
data_split_mode),
0.2769199f, 0.001f);
auto auc = GetMetricEval(metric, {0, 1}, {}, {}, {}, data_split_mode);
ASSERT_TRUE(std::isnan(auc));
// AUCPR with instance weights
EXPECT_NEAR(GetMetricEval(metric,
{0.29f, 0.52f, 0.11f, 0.21f, 0.219f, 0.93f, 0.493f, 0.17f, 0.47f, 0.13f,
0.43f, 0.59f, 0.87f, 0.007f},
{0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0},
{1, 2, 7, 4, 5, 2.2f, 3.2f, 5, 6, 1, 2, 1.1f, 3.2f, 4.5f}, // weights
{}, data_split_mode),
0.694435f, 0.001f);
// Both groups contain only pos or neg samples.
auc = GetMetricEval(metric, {0, 0.1f, 0.3f, 0.5f, 0.7f}, {1, 1, 0, 0, 0}, {}, {0, 2, 5},
data_split_mode);
ASSERT_TRUE(std::isnan(auc));
delete metric;
}
inline void VerifyMultiClassPRAUC(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
std::unique_ptr<Metric> metric{Metric::Create("aucpr", &ctx)};
float auc = 0;
std::vector<float> labels{1.0f, 0.0f, 2.0f};
HostDeviceVector<float> predts{
0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f,
};
auc = GetMetricEval(metric.get(), predts, labels, {}, {}, data_split_mode);
EXPECT_EQ(auc, 1.0f);
auc = GetMetricEval(metric.get(), predts, labels, {1.0f, 1.0f, 1.0f}, {}, data_split_mode);
EXPECT_EQ(auc, 1.0f);
predts.HostVector() = {
0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f,
};
labels = {1.0f, 0.0f, 2.0f, 1.0f};
auc = GetMetricEval(metric.get(), predts, labels, {1.0f, 2.0f, 3.0f, 4.0f}, {}, data_split_mode);
ASSERT_GT(auc, 0.699);
}
inline void VerifyRankingPRAUC(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
std::unique_ptr<Metric> metric{Metric::Create("aucpr", &ctx)};
std::vector<float> labels{1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f};
std::vector<uint32_t> groups{0, 2, 6};
float auc = 0;
auc = GetMetricEval(metric.get(), {1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f}, labels, {}, groups,
data_split_mode);
EXPECT_EQ(auc, 1.0f);
auc = GetMetricEval(metric.get(), {1.0f, 0.5f, 0.8f, 0.3f, 0.2f, 1.0f}, labels, {}, groups,
data_split_mode);
EXPECT_EQ(auc, 1.0f);
auc = GetMetricEval(metric.get(), {1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f},
{1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f}, {}, groups, data_split_mode);
ASSERT_TRUE(std::isnan(auc));
// Incorrect label
ASSERT_THROW(GetMetricEval(metric.get(), {1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f},
{1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 3.0f}, {}, groups, data_split_mode),
dmlc::Error);
// AUCPR with groups and no weights
EXPECT_NEAR(
GetMetricEval(metric.get(),
{0.87f, 0.31f, 0.40f, 0.42f, 0.25f, 0.66f, 0.95f, 0.09f, 0.10f, 0.97f,
0.76f, 0.69f, 0.15f, 0.20f, 0.30f, 0.14f, 0.07f, 0.58f, 0.61f, 0.08f},
{0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1}, {}, // weights
{0, 2, 5, 9, 14, 20}, // group info
data_split_mode),
0.556021f, 0.001f);
}
} // namespace metric
} // namespace xgboost

391
tests/cpp/metric/test_elementwise_metric.cc
View File

@ -1,347 +1,108 @@
/**
* Copyright 2018-2023 by XGBoost contributors
*/
#include <xgboost/json.h>
#include <xgboost/metric.h>
#include <map>
#include <memory>
#include "../../../src/common/linalg_op.h"
#include "../helpers.h"
namespace xgboost {
namespace {
inline void CheckDeterministicMetricElementWise(StringView name, int32_t device) {
auto ctx = CreateEmptyGenericParam(device);
std::unique_ptr<Metric> metric{Metric::Create(name.c_str(), &ctx)};
HostDeviceVector<float> predts;
size_t n_samples = 2048;
auto p_fmat = EmptyDMatrix();
MetaInfo& info = p_fmat->Info();
info.labels.Reshape(n_samples, 1);
info.num_row_ = n_samples;
auto &h_labels = info.labels.Data()->HostVector();
auto &h_predts = predts.HostVector();
SimpleLCG lcg;
SimpleRealUniformDistribution<float> dist{0.0f, 1.0f};
h_labels.resize(n_samples);
h_predts.resize(n_samples);
for (size_t i = 0; i < n_samples; ++i) {
h_predts[i] = dist(&lcg);
h_labels[i] = dist(&lcg);
}
auto result = metric->Evaluate(predts, p_fmat);
for (size_t i = 0; i < 8; ++i) {
ASSERT_EQ(metric->Evaluate(predts, p_fmat), result);
}
}
} // anonymous namespace
} // namespace xgboost
#include "test_elementwise_metric.h"
namespace xgboost {
namespace metric {
TEST(Metric, DeclareUnifiedTest(RMSE)) { VerifyRMSE(); }
TEST(Metric, DeclareUnifiedTest(RMSE)) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric * metric = xgboost::Metric::Create("rmse", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "rmse");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}),
0.6403f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ -1, 1, 9, -9}),
2.8284f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ 1, 2, 9, 8}),
0.6708f, 0.001f);
delete metric;
TEST(Metric, DeclareUnifiedTest(RMSLE)) { VerifyRMSLE(); }
xgboost::CheckDeterministicMetricElementWise(xgboost::StringView{"rmse"}, GPUIDX);
TEST(Metric, DeclareUnifiedTest(MAE)) { VerifyMAE(); }
TEST(Metric, DeclareUnifiedTest(MAPE)) { VerifyMAPE(); }
TEST(Metric, DeclareUnifiedTest(MPHE)) { VerifyMPHE(); }
TEST(Metric, DeclareUnifiedTest(LogLoss)) { VerifyLogLoss(); }
TEST(Metric, DeclareUnifiedTest(Error)) { VerifyError(); }
TEST(Metric, DeclareUnifiedTest(PoissonNegLogLik)) { VerifyPoissonNegLogLik(); }
TEST(Metric, DeclareUnifiedTest(MultiRMSE)) { VerifyMultiRMSE(); }
TEST(Metric, DeclareUnifiedTest(Quantile)) { VerifyQuantile(); }
TEST_F(DeclareUnifiedDistributedTest(MetricTest), RMSERowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyRMSE, DataSplitMode::kRow);
}
TEST(Metric, DeclareUnifiedTest(RMSLE)) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric * metric = xgboost::Metric::Create("rmsle", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "rmsle");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.2f, 0.4f, 0.8f, 1.6f},
{1.0f, 1.0f, 1.0f, 1.0f, 1.0f}),
0.4063f, 1e-4);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.2f, 0.4f, 0.8f, 1.6f},
{1.0f, 1.0f, 1.0f, 1.0f, 1.0f},
{ 0, -1, 1, -9, 9}),
0.6212f, 1e-4);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.2f, 0.4f, 0.8f, 1.6f},
{1.0f, 1.0f, 1.0f, 1.0f, 1.0f},
{ 0, 1, 2, 9, 8}),
0.2415f, 1e-4);
delete metric;
xgboost::CheckDeterministicMetricElementWise(xgboost::StringView{"rmsle"}, GPUIDX);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), RMSEColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyRMSE, DataSplitMode::kCol);
}
TEST(Metric, DeclareUnifiedTest(MAE)) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric * metric = xgboost::Metric::Create("mae", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "mae");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}),
0.5f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ -1, 1, 9, -9}),
8.0f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ 1, 2, 9, 8}),
0.54f, 0.001f);
delete metric;
xgboost::CheckDeterministicMetricElementWise(xgboost::StringView{"mae"}, GPUIDX);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), RMSLERowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyRMSLE, DataSplitMode::kRow);
}
TEST(Metric, DeclareUnifiedTest(MAPE)) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric * metric = xgboost::Metric::Create("mape", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "mape");
EXPECT_NEAR(GetMetricEval(metric, {150, 300}, {100, 200}), 0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{50, 400, 500, 4000},
{100, 200, 500, 1000}),
1.125f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{50, 400, 500, 4000},
{100, 200, 500, 1000},
{ -1, 1, 9, -9}),
-26.5f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{50, 400, 500, 4000},
{100, 200, 500, 1000},
{ 1, 2, 9, 8}),
1.3250f, 0.001f);
delete metric;
xgboost::CheckDeterministicMetricElementWise(xgboost::StringView{"mape"}, GPUIDX);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), RMSLEColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyRMSLE, DataSplitMode::kCol);
}
TEST(Metric, DeclareUnifiedTest(MPHE)) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
std::unique_ptr<xgboost::Metric> metric{xgboost::Metric::Create("mphe", &ctx)};
metric->Configure({});
ASSERT_STREQ(metric->Name(), "mphe");
EXPECT_NEAR(GetMetricEval(metric.get(), {0, 1}, {0, 1}), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric.get(),
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}),
0.1751f, 1e-4);
EXPECT_NEAR(GetMetricEval(metric.get(),
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ -1, 1, 9, -9}),
3.4037f, 1e-4);
EXPECT_NEAR(GetMetricEval(metric.get(),
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ 1, 2, 9, 8}),
0.1922f, 1e-4);
xgboost::CheckDeterministicMetricElementWise(xgboost::StringView{"mphe"}, GPUIDX);
metric->Configure({{"huber_slope", "0.1"}});
EXPECT_NEAR(GetMetricEval(metric.get(),
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ 1, 2, 9, 8}),
0.0461686f, 1e-4);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), MAERowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyMAE, DataSplitMode::kRow);
}
TEST(Metric, DeclareUnifiedTest(LogLoss)) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric * metric = xgboost::Metric::Create("logloss", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "logloss");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.5f, 1e-17f, 1.0f+1e-17f, 0.9f},
{ 0, 0, 1, 1}),
0.1996f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}),
1.2039f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ -1, 1, 9, -9}),
21.9722f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ 1, 2, 9, 8}),
1.3138f, 0.001f);
delete metric;
xgboost::CheckDeterministicMetricElementWise(xgboost::StringView{"logloss"}, GPUIDX);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), MAEColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyMAE, DataSplitMode::kCol);
}
TEST(Metric, DeclareUnifiedTest(Error)) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric * metric = xgboost::Metric::Create("error", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "error");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}),
0.5f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ -1, 1, 9, -9}),
10.0f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ 1, 2, 9, 8}),
0.55f, 0.001f);
EXPECT_ANY_THROW(xgboost::Metric::Create("error@abc", &ctx));
delete metric;
metric = xgboost::Metric::Create("error@0.5f", &ctx);
metric->Configure({});
EXPECT_STREQ(metric->Name(), "error");
delete metric;
metric = xgboost::Metric::Create("error@0.1", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "error@0.1");
EXPECT_STREQ(metric->Name(), "error@0.1");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{-0.1f, -0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}),
0.25f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{-0.1f, -0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ -1, 1, 9, -9}),
9.0f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{-0.1f, -0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ 1, 2, 9, 8}),
0.45f, 0.001f);
delete metric;
xgboost::CheckDeterministicMetricElementWise(xgboost::StringView{"error@0.5"}, GPUIDX);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), MAPERowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyMAPE, DataSplitMode::kRow);
}
TEST(Metric, DeclareUnifiedTest(PoissionNegLogLik)) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric * metric = xgboost::Metric::Create("poisson-nloglik", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "poisson-nloglik");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.5f, 1e-17f, 1.0f+1e-17f, 0.9f},
{ 0, 0, 1, 1}),
0.6263f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}),
1.1019f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ -1, 1, 9, -9}),
13.3750f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ 1, 2, 9, 8}),
1.5783f, 0.001f);
delete metric;
xgboost::CheckDeterministicMetricElementWise(xgboost::StringView{"poisson-nloglik"}, GPUIDX);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), MAPEColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyMAPE, DataSplitMode::kCol);
}
TEST(Metric, DeclareUnifiedTest(MultiRMSE)) {
size_t n_samples = 32, n_targets = 8;
linalg::Tensor<float, 2> y{{n_samples, n_targets}, GPUIDX};
auto &h_y = y.Data()->HostVector();
std::iota(h_y.begin(), h_y.end(), 0);
HostDeviceVector<float> predt(n_samples * n_targets, 0);
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
std::unique_ptr<Metric> metric{Metric::Create("rmse", &ctx)};
metric->Configure({});
auto loss = GetMultiMetricEval(metric.get(), predt, y);
std::vector<float> weights(n_samples, 1);
auto loss_w = GetMultiMetricEval(metric.get(), predt, y, weights);
std::transform(h_y.cbegin(), h_y.cend(), h_y.begin(), [](auto &v) { return v * v; });
auto ret = std::sqrt(std::accumulate(h_y.cbegin(), h_y.cend(), 1.0, std::plus<>{}) / h_y.size());
ASSERT_FLOAT_EQ(ret, loss);
ASSERT_FLOAT_EQ(ret, loss_w);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), MPHERowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyMPHE, DataSplitMode::kRow);
}
TEST(Metric, DeclareUnifiedTest(Quantile)) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
std::unique_ptr<Metric> metric{Metric::Create("quantile", &ctx)};
TEST_F(DeclareUnifiedDistributedTest(MetricTest), MPHEColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyMPHE, DataSplitMode::kCol);
}
HostDeviceVector<float> predts{0.1f, 0.9f, 0.1f, 0.9f};
std::vector<float> labels{0.5f, 0.5f, 0.9f, 0.1f};
std::vector<float> weights{0.2f, 0.4f,0.6f, 0.8f};
TEST_F(DeclareUnifiedDistributedTest(MetricTest), LogLossRowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyLogLoss, DataSplitMode::kRow);
}
metric->Configure(Args{{"quantile_alpha", "[0.0]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights), 0.400f, 0.001f);
metric->Configure(Args{{"quantile_alpha", "[0.2]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights), 0.376f, 0.001f);
metric->Configure(Args{{"quantile_alpha", "[0.4]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights), 0.352f, 0.001f);
metric->Configure(Args{{"quantile_alpha", "[0.8]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights), 0.304f, 0.001f);
metric->Configure(Args{{"quantile_alpha", "[1.0]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights), 0.28f, 0.001f);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), LogLossColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyLogLoss, DataSplitMode::kCol);
}
metric->Configure(Args{{"quantile_alpha", "[0.0]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels), 0.3f, 0.001f);
metric->Configure(Args{{"quantile_alpha", "[0.2]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels), 0.3f, 0.001f);
metric->Configure(Args{{"quantile_alpha", "[0.4]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels), 0.3f, 0.001f);
metric->Configure(Args{{"quantile_alpha", "[0.8]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels), 0.3f, 0.001f);
metric->Configure(Args{{"quantile_alpha", "[1.0]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels), 0.3f, 0.001f);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), ErrorRowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyError, DataSplitMode::kRow);
}
TEST_F(DeclareUnifiedDistributedTest(MetricTest), ErrorColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyError, DataSplitMode::kCol);
}
TEST_F(DeclareUnifiedDistributedTest(MetricTest), PoissonNegLogLikRowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyPoissonNegLogLik, DataSplitMode::kRow);
}
TEST_F(DeclareUnifiedDistributedTest(MetricTest), PoissonNegLogLikColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyPoissonNegLogLik, DataSplitMode::kCol);
}
TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiRMSERowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyMultiRMSE, DataSplitMode::kRow);
}
TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiRMSEColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyMultiRMSE, DataSplitMode::kCol);
}
TEST_F(DeclareUnifiedDistributedTest(MetricTest), QuantileRowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyQuantile, DataSplitMode::kRow);
}
TEST_F(DeclareUnifiedDistributedTest(MetricTest), QuantileColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyQuantile, DataSplitMode::kCol);
}
} // namespace metric
} // namespace xgboost

385
tests/cpp/metric/test_elementwise_metric.h Normal file
View File

@ -0,0 +1,385 @@
/**
* Copyright 2018-2023 by XGBoost contributors
*/
#pragma once
#include <xgboost/json.h>
#include <xgboost/metric.h>
#include <map>
#include <memory>
#include "../../../src/common/linalg_op.h"
#include "../helpers.h"
namespace xgboost {
namespace metric {
inline void CheckDeterministicMetricElementWise(StringView name, int32_t device) {
auto ctx = CreateEmptyGenericParam(device);
std::unique_ptr<Metric> metric{Metric::Create(name.c_str(), &ctx)};
HostDeviceVector<float> predts;
size_t n_samples = 2048;
auto p_fmat = EmptyDMatrix();
MetaInfo& info = p_fmat->Info();
info.labels.Reshape(n_samples, 1);
info.num_row_ = n_samples;
auto &h_labels = info.labels.Data()->HostVector();
auto &h_predts = predts.HostVector();
SimpleLCG lcg;
SimpleRealUniformDistribution<float> dist{0.0f, 1.0f};
h_labels.resize(n_samples);
h_predts.resize(n_samples);
for (size_t i = 0; i < n_samples; ++i) {
h_predts[i] = dist(&lcg);
h_labels[i] = dist(&lcg);
}
auto result = metric->Evaluate(predts, p_fmat);
for (size_t i = 0; i < 8; ++i) {
ASSERT_EQ(metric->Evaluate(predts, p_fmat), result);
}
}
inline void VerifyRMSE(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric * metric = xgboost::Metric::Create("rmse", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "rmse");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}, {}, {}, data_split_mode),
0.6403f, 0.001f);
auto expected = 2.8284f;
if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) {
expected = sqrt(8.0f * collective::GetWorldSize());
}
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ -1, 1, 9, -9}, {}, data_split_mode),
expected, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ 1, 2, 9, 8}, {}, data_split_mode),
0.6708f, 0.001f);
delete metric;
CheckDeterministicMetricElementWise(StringView{"rmse"}, GPUIDX);
}
inline void VerifyRMSLE(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric * metric = xgboost::Metric::Create("rmsle", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "rmsle");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.2f, 0.4f, 0.8f, 1.6f},
{1.0f, 1.0f, 1.0f, 1.0f, 1.0f}, {}, {}, data_split_mode),
0.4063f, 1e-4);
auto expected = 0.6212f;
if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) {
expected = sqrt(0.3859f * collective::GetWorldSize());
}
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.2f, 0.4f, 0.8f, 1.6f},
{1.0f, 1.0f, 1.0f, 1.0f, 1.0f},
{ 0, -1, 1, -9, 9}, {}, data_split_mode),
expected, 1e-4);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.2f, 0.4f, 0.8f, 1.6f},
{1.0f, 1.0f, 1.0f, 1.0f, 1.0f},
{ 0, 1, 2, 9, 8}, {}, data_split_mode),
0.2415f, 1e-4);
delete metric;
CheckDeterministicMetricElementWise(StringView{"rmsle"}, GPUIDX);
}
inline void VerifyMAE(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric * metric = xgboost::Metric::Create("mae", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "mae");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}, {}, {}, data_split_mode),
0.5f, 0.001f);
auto expected = 8.0f;
if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) {
expected *= collective::GetWorldSize();
}
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ -1, 1, 9, -9}, {}, data_split_mode),
expected, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ 1, 2, 9, 8}, {}, data_split_mode),
0.54f, 0.001f);
delete metric;
CheckDeterministicMetricElementWise(StringView{"mae"}, GPUIDX);
}
inline void VerifyMAPE(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric * metric = xgboost::Metric::Create("mape", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "mape");
EXPECT_NEAR(GetMetricEval(metric, {150, 300}, {100, 200}, {}, {}, data_split_mode), 0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{50, 400, 500, 4000},
{100, 200, 500, 1000}, {}, {}, data_split_mode),
1.125f, 0.001f);
auto expected = -26.5f;
if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) {
expected *= collective::GetWorldSize();
}
EXPECT_NEAR(GetMetricEval(metric,
{50, 400, 500, 4000},
{100, 200, 500, 1000},
{ -1, 1, 9, -9}, {}, data_split_mode),
expected, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{50, 400, 500, 4000},
{100, 200, 500, 1000},
{ 1, 2, 9, 8}, {}, data_split_mode),
1.3250f, 0.001f);
delete metric;
CheckDeterministicMetricElementWise(StringView{"mape"}, GPUIDX);
}
inline void VerifyMPHE(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
std::unique_ptr<xgboost::Metric> metric{xgboost::Metric::Create("mphe", &ctx)};
metric->Configure({});
ASSERT_STREQ(metric->Name(), "mphe");
EXPECT_NEAR(GetMetricEval(metric.get(), {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric.get(),
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}, {}, {}, data_split_mode),
0.1751f, 1e-4);
auto expected = 3.40375f;
if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) {
expected *= collective::GetWorldSize();
}
EXPECT_NEAR(GetMetricEval(metric.get(),
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ -1, 1, 9, -9}, {}, data_split_mode),
expected, 1e-4);
EXPECT_NEAR(GetMetricEval(metric.get(),
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ 1, 2, 9, 8}, {}, data_split_mode),
0.1922f, 1e-4);
CheckDeterministicMetricElementWise(StringView{"mphe"}, GPUIDX);
metric->Configure({{"huber_slope", "0.1"}});
EXPECT_NEAR(GetMetricEval(metric.get(),
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ 1, 2, 9, 8}, {}, data_split_mode),
0.0461686f, 1e-4);
}
inline void VerifyLogLoss(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric * metric = xgboost::Metric::Create("logloss", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "logloss");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.5f, 1e-17f, 1.0f+1e-17f, 0.9f},
{ 0, 0, 1, 1}, {}, {}, data_split_mode),
0.1996f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}, {}, {}, data_split_mode),
1.2039f, 0.001f);
auto expected = 21.9722f;
if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) {
expected *= collective::GetWorldSize();
}
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ -1, 1, 9, -9}, {}, data_split_mode),
expected, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ 1, 2, 9, 8}, {}, data_split_mode),
1.3138f, 0.001f);
delete metric;
CheckDeterministicMetricElementWise(StringView{"logloss"}, GPUIDX);
}
inline void VerifyError(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric * metric = xgboost::Metric::Create("error", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "error");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}, {}, {}, data_split_mode),
0.5f, 0.001f);
auto expected = 10.0f;
if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) {
expected *= collective::GetWorldSize();
}
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ -1, 1, 9, -9}, {}, data_split_mode),
expected, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ 1, 2, 9, 8}, {}, data_split_mode),
0.55f, 0.001f);
EXPECT_ANY_THROW(xgboost::Metric::Create("error@abc", &ctx));
delete metric;
metric = xgboost::Metric::Create("error@0.5f", &ctx);
metric->Configure({});
EXPECT_STREQ(metric->Name(), "error");
delete metric;
metric = xgboost::Metric::Create("error@0.1", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "error@0.1");
EXPECT_STREQ(metric->Name(), "error@0.1");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{-0.1f, -0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}, {}, {}, data_split_mode),
0.25f, 0.001f);
expected = 9.0f;
if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) {
expected *= collective::GetWorldSize();
}
EXPECT_NEAR(GetMetricEval(metric,
{-0.1f, -0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ -1, 1, 9, -9}, {}, data_split_mode),
expected, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{-0.1f, -0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ 1, 2, 9, 8}, {}, data_split_mode),
0.45f, 0.001f);
delete metric;
CheckDeterministicMetricElementWise(StringView{"error@0.5"}, GPUIDX);
}
inline void VerifyPoissonNegLogLik(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric * metric = xgboost::Metric::Create("poisson-nloglik", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "poisson-nloglik");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0.5f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.5f, 1e-17f, 1.0f+1e-17f, 0.9f},
{ 0, 0, 1, 1}, {}, {}, data_split_mode),
0.6263f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}, {}, {}, data_split_mode),
1.1019f, 0.001f);
auto expected = 13.3750f;
if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) {
expected *= collective::GetWorldSize();
}
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ -1, 1, 9, -9}, {}, data_split_mode),
expected, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1},
{ 1, 2, 9, 8}, {}, data_split_mode),
1.5783f, 0.001f);
delete metric;
CheckDeterministicMetricElementWise(StringView{"poisson-nloglik"}, GPUIDX);
}
inline void VerifyMultiRMSE(DataSplitMode data_split_mode = DataSplitMode::kRow) {
size_t n_samples = 32, n_targets = 8;
linalg::Tensor<float, 2> y{{n_samples, n_targets}, GPUIDX};
auto &h_y = y.Data()->HostVector();
std::iota(h_y.begin(), h_y.end(), 0);
HostDeviceVector<float> predt(n_samples * n_targets, 0);
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
std::unique_ptr<Metric> metric{Metric::Create("rmse", &ctx)};
metric->Configure({});
auto loss = GetMultiMetricEval(metric.get(), predt, y, {}, {}, data_split_mode);
std::vector<float> weights(n_samples, 1);
auto loss_w = GetMultiMetricEval(metric.get(), predt, y, weights, {}, data_split_mode);
std::transform(h_y.cbegin(), h_y.cend(), h_y.begin(), [](auto &v) { return v * v; });
auto ret = std::sqrt(std::accumulate(h_y.cbegin(), h_y.cend(), 1.0, std::plus<>{}) / h_y.size());
ASSERT_FLOAT_EQ(ret, loss);
ASSERT_FLOAT_EQ(ret, loss_w);
}
inline void VerifyQuantile(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
std::unique_ptr<Metric> metric{Metric::Create("quantile", &ctx)};
HostDeviceVector<float> predts{0.1f, 0.9f, 0.1f, 0.9f};
std::vector<float> labels{0.5f, 0.5f, 0.9f, 0.1f};
std::vector<float> weights{0.2f, 0.4f, 0.6f, 0.8f};
metric->Configure(Args{{"quantile_alpha", "[0.0]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights, {}, data_split_mode), 0.400f,
0.001f);
metric->Configure(Args{{"quantile_alpha", "[0.2]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights, {}, data_split_mode), 0.376f,
0.001f);
metric->Configure(Args{{"quantile_alpha", "[0.4]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights, {}, data_split_mode), 0.352f,
0.001f);
metric->Configure(Args{{"quantile_alpha", "[0.8]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights, {}, data_split_mode), 0.304f,
0.001f);
metric->Configure(Args{{"quantile_alpha", "[1.0]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights, {}, data_split_mode), 0.28f,
0.001f);
metric->Configure(Args{{"quantile_alpha", "[0.0]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, {}, {}, data_split_mode), 0.3f, 0.001f);
metric->Configure(Args{{"quantile_alpha", "[0.2]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, {}, {}, data_split_mode), 0.3f, 0.001f);
metric->Configure(Args{{"quantile_alpha", "[0.4]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, {}, {}, data_split_mode), 0.3f, 0.001f);
metric->Configure(Args{{"quantile_alpha", "[0.8]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, {}, {}, data_split_mode), 0.3f, 0.001f);
metric->Configure(Args{{"quantile_alpha", "[1.0]"}});
EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, {}, {}, data_split_mode), 0.3f, 0.001f);
}
} // namespace metric
} // namespace xgboost

98
tests/cpp/metric/test_multiclass_metric.cc
View File

@ -1,87 +1,29 @@
// Copyright by Contributors
#include <xgboost/metric.h>
#include "test_multiclass_metric.h"
#include <string>
#include "../helpers.h"
namespace xgboost {
inline void CheckDeterministicMetricMultiClass(StringView name, int32_t device) {
auto ctx = CreateEmptyGenericParam(device);
std::unique_ptr<Metric> metric{Metric::Create(name.c_str(), &ctx)};
namespace metric {
HostDeviceVector<float> predts;
auto p_fmat = EmptyDMatrix();
MetaInfo& info = p_fmat->Info();
auto &h_predts = predts.HostVector();
TEST(Metric, DeclareUnifiedTest(MultiClassError)) { VerifyMultiClassError(); }
SimpleLCG lcg;
TEST(Metric, DeclareUnifiedTest(MultiClassLogLoss)) { VerifyMultiClassLogLoss(); }
size_t n_samples = 2048, n_classes = 4;
info.labels.Reshape(n_samples);
auto &h_labels = info.labels.Data()->HostVector();
h_predts.resize(n_samples * n_classes);
{
SimpleRealUniformDistribution<float> dist{0.0f, static_cast<float>(n_classes)};
for (size_t i = 0; i < n_samples; ++i) {
h_labels[i] = dist(&lcg);
}
}
{
SimpleRealUniformDistribution<float> dist{0.0f, 1.0f};
for (size_t i = 0; i < n_samples * n_classes; ++i) {
h_predts[i] = dist(&lcg);
}
}
auto result = metric->Evaluate(predts, p_fmat);
for (size_t i = 0; i < 8; ++i) {
ASSERT_EQ(metric->Evaluate(predts, p_fmat), result);
}
TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiClassErrorRowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyMultiClassError, DataSplitMode::kRow);
}
TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiClassErrorColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyMultiClassError, DataSplitMode::kCol);
}
TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiClassLogLossRowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyMultiClassLogLoss, DataSplitMode::kRow);
}
TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiClassLogLossColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyMultiClassLogLoss, DataSplitMode::kCol);
}
} // namespace metric
} // namespace xgboost
inline void TestMultiClassError(int device) {
auto ctx = xgboost::CreateEmptyGenericParam(device);
ctx.gpu_id = device;
xgboost::Metric * metric = xgboost::Metric::Create("merror", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "merror");
EXPECT_ANY_THROW(GetMetricEval(metric, {0}, {0, 0}));
EXPECT_NEAR(GetMetricEval(
metric, {1, 0, 0, 0, 1, 0, 0, 0, 1}, {0, 1, 2}), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f},
{0, 1, 2}),
0.666f, 0.001f);
delete metric;
}
TEST(Metric, DeclareUnifiedTest(MultiClassError)) {
TestMultiClassError(GPUIDX);
xgboost::CheckDeterministicMetricMultiClass(xgboost::StringView{"merror"}, GPUIDX);
}
inline void TestMultiClassLogLoss(int device) {
auto ctx = xgboost::CreateEmptyGenericParam(device);
ctx.gpu_id = device;
xgboost::Metric * metric = xgboost::Metric::Create("mlogloss", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "mlogloss");
EXPECT_ANY_THROW(GetMetricEval(metric, {0}, {0, 0}));
EXPECT_NEAR(GetMetricEval(
metric, {1, 0, 0, 0, 1, 0, 0, 0, 1}, {0, 1, 2}), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f},
{0, 1, 2}),
2.302f, 0.001f);
delete metric;
}
TEST(Metric, DeclareUnifiedTest(MultiClassLogLoss)) {
TestMultiClassLogLoss(GPUIDX);
xgboost::CheckDeterministicMetricMultiClass(xgboost::StringView{"mlogloss"}, GPUIDX);
}

91
tests/cpp/metric/test_multiclass_metric.h Normal file
View File

@ -0,0 +1,91 @@
// Copyright by Contributors
#include <xgboost/metric.h>
#include <string>
#include "../helpers.h"
namespace xgboost {
namespace metric {
inline void CheckDeterministicMetricMultiClass(StringView name, int32_t device) {
auto ctx = CreateEmptyGenericParam(device);
std::unique_ptr<Metric> metric{Metric::Create(name.c_str(), &ctx)};
HostDeviceVector<float> predts;
auto p_fmat = EmptyDMatrix();
MetaInfo& info = p_fmat->Info();
auto &h_predts = predts.HostVector();
SimpleLCG lcg;
size_t n_samples = 2048, n_classes = 4;
info.labels.Reshape(n_samples);
auto &h_labels = info.labels.Data()->HostVector();
h_predts.resize(n_samples * n_classes);
{
SimpleRealUniformDistribution<float> dist{0.0f, static_cast<float>(n_classes)};
for (size_t i = 0; i < n_samples; ++i) {
h_labels[i] = dist(&lcg);
}
}
{
SimpleRealUniformDistribution<float> dist{0.0f, 1.0f};
for (size_t i = 0; i < n_samples * n_classes; ++i) {
h_predts[i] = dist(&lcg);
}
}
auto result = metric->Evaluate(predts, p_fmat);
for (size_t i = 0; i < 8; ++i) {
ASSERT_EQ(metric->Evaluate(predts, p_fmat), result);
}
}
inline void TestMultiClassError(int device, DataSplitMode data_split_mode) {
auto ctx = xgboost::CreateEmptyGenericParam(device);
ctx.gpu_id = device;
xgboost::Metric * metric = xgboost::Metric::Create("merror", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "merror");
EXPECT_ANY_THROW(GetMetricEval(metric, {0}, {0, 0}, {}, {}, data_split_mode));
EXPECT_NEAR(GetMetricEval(
metric, {1, 0, 0, 0, 1, 0, 0, 0, 1}, {0, 1, 2}, {}, {}, data_split_mode), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f},
{0, 1, 2}, {}, {}, data_split_mode),
0.666f, 0.001f);
delete metric;
}
inline void VerifyMultiClassError(DataSplitMode data_split_mode = DataSplitMode::kRow) {
TestMultiClassError(GPUIDX, data_split_mode);
CheckDeterministicMetricMultiClass(StringView{"merror"}, GPUIDX);
}
inline void TestMultiClassLogLoss(int device, DataSplitMode data_split_mode) {
auto ctx = xgboost::CreateEmptyGenericParam(device);
ctx.gpu_id = device;
xgboost::Metric * metric = xgboost::Metric::Create("mlogloss", &ctx);
metric->Configure({});
ASSERT_STREQ(metric->Name(), "mlogloss");
EXPECT_ANY_THROW(GetMetricEval(metric, {0}, {0, 0}, {}, {}, data_split_mode));
EXPECT_NEAR(GetMetricEval(
metric, {1, 0, 0, 0, 1, 0, 0, 0, 1}, {0, 1, 2}, {}, {}, data_split_mode), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f},
{0, 1, 2}, {}, {}, data_split_mode),
2.302f, 0.001f);
delete metric;
}
inline void VerifyMultiClassLogLoss(DataSplitMode data_split_mode = DataSplitMode::kRow) {
TestMultiClassLogLoss(GPUIDX, data_split_mode);
CheckDeterministicMetricMultiClass(StringView{"mlogloss"}, GPUIDX);
}
} // namespace metric
} // namespace xgboost

190
tests/cpp/metric/test_rank_metric.cc
View File

@ -11,16 +11,20 @@
#include <memory> // for unique_ptr
#include <vector> // for vector
#include "test_rank_metric.h"
#include "../helpers.h" // for GetMetricEval, CreateEmptyGe...
#include "xgboost/base.h" // for bst_float, kRtEps
#include "xgboost/host_device_vector.h" // for HostDeviceVector
#include "xgboost/json.h" // for Json, String, Object
namespace xgboost {
namespace metric {
#if !defined(__CUDACC__)
TEST(Metric, AMS) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
EXPECT_ANY_THROW(xgboost::Metric::Create("ams", &ctx));
xgboost::Metric* metric = xgboost::Metric::Create("ams@0.5f", &ctx);
auto ctx = CreateEmptyGenericParam(GPUIDX);
EXPECT_ANY_THROW(Metric::Create("ams", &ctx));
Metric* metric = Metric::Create("ams@0.5f", &ctx);
ASSERT_STREQ(metric->Name(), "ams@0.5");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0.311f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
@ -29,7 +33,7 @@ TEST(Metric, AMS) {
0.29710f, 0.001f);
delete metric;
metric = xgboost::Metric::Create("ams@0", &ctx);
metric = Metric::Create("ams@0", &ctx);
ASSERT_STREQ(metric->Name(), "ams@0");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0.311f, 0.001f);
@ -37,172 +41,44 @@ TEST(Metric, AMS) {
}
#endif
TEST(Metric, DeclareUnifiedTest(Precision)) {
// When the limit for precision is not given, it takes the limit at
// std::numeric_limits<unsigned>::max(); hence all values are very small
// NOTE(AbdealiJK): Maybe this should be fixed to be num_row by default.
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric * metric = xgboost::Metric::Create("pre", &ctx);
ASSERT_STREQ(metric->Name(), "pre");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-7);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}),
0, 1e-7);
TEST(Metric, DeclareUnifiedTest(Precision)) { VerifyPrecision(); }
delete metric;
metric = xgboost::Metric::Create("pre@2", &ctx);
ASSERT_STREQ(metric->Name(), "pre@2");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0.5f, 1e-7);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}),
0.5f, 0.001f);
TEST(Metric, DeclareUnifiedTest(NDCG)) { VerifyNDCG(); }
EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {}));
TEST(Metric, DeclareUnifiedTest(MAP)) { VerifyMAP(); }
delete metric;
TEST(Metric, DeclareUnifiedTest(NDCGExpGain)) { VerifyNDCGExpGain(); }
TEST_F(DeclareUnifiedDistributedTest(MetricTest), PrecisionRowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyPrecision, DataSplitMode::kRow);
}
namespace xgboost {
namespace metric {
TEST(Metric, DeclareUnifiedTest(NDCG)) {
auto ctx = CreateEmptyGenericParam(GPUIDX);
Metric * metric = xgboost::Metric::Create("ndcg", &ctx);
ASSERT_STREQ(metric->Name(), "ndcg");
EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {}));
ASSERT_NEAR(GetMetricEval(metric,
xgboost::HostDeviceVector<xgboost::bst_float>{},
{}), 1, 1e-10);
ASSERT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}),
0.6509f, 0.001f);
delete metric;
metric = xgboost::Metric::Create("ndcg@2", &ctx);
ASSERT_STREQ(metric->Name(), "ndcg@2");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}),
0.3868f, 0.001f);
delete metric;
metric = xgboost::Metric::Create("ndcg@-", &ctx);
ASSERT_STREQ(metric->Name(), "ndcg-");
EXPECT_NEAR(GetMetricEval(metric,
xgboost::HostDeviceVector<xgboost::bst_float>{},
{}), 0, 1e-10);
ASSERT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1.f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}),
0.6509f, 0.001f);
delete metric;
metric = xgboost::Metric::Create("ndcg-", &ctx);
ASSERT_STREQ(metric->Name(), "ndcg-");
EXPECT_NEAR(GetMetricEval(metric,
xgboost::HostDeviceVector<xgboost::bst_float>{},
{}), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1.f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}),
0.6509f, 0.001f);
delete metric;
metric = xgboost::Metric::Create("ndcg@2-", &ctx);
ASSERT_STREQ(metric->Name(), "ndcg@2-");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1.f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}),
1.f - 0.3868f, 1.f - 0.001f);
delete metric;
TEST_F(DeclareUnifiedDistributedTest(MetricTest), PrecisionColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyPrecision, DataSplitMode::kCol);
}
TEST(Metric, DeclareUnifiedTest(MAP)) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
Metric * metric = xgboost::Metric::Create("map", &ctx);
ASSERT_STREQ(metric->Name(), "map");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, kRtEps);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}),
0.5f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
xgboost::HostDeviceVector<xgboost::bst_float>{},
std::vector<xgboost::bst_float>{}), 1, 1e-10);
// Rank metric with group info
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.2f, 0.8f, 0.4f, 1.7f},
{1, 1, 1, 0, 1, 0}, // Labels
{}, // Weights
{0, 2, 5, 6}), // Group info
0.8611f, 0.001f);
delete metric;
metric = xgboost::Metric::Create("map@-", &ctx);
ASSERT_STREQ(metric->Name(), "map-");
EXPECT_NEAR(GetMetricEval(metric,
xgboost::HostDeviceVector<xgboost::bst_float>{},
{}), 0, 1e-10);
delete metric;
metric = xgboost::Metric::Create("map-", &ctx);
ASSERT_STREQ(metric->Name(), "map-");
EXPECT_NEAR(GetMetricEval(metric,
xgboost::HostDeviceVector<xgboost::bst_float>{},
{}), 0, 1e-10);
delete metric;
metric = xgboost::Metric::Create("map@2", &ctx);
ASSERT_STREQ(metric->Name(), "map@2");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}),
0.25f, 0.001f);
delete metric;
TEST_F(DeclareUnifiedDistributedTest(MetricTest), NDCGRowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyNDCG, DataSplitMode::kRow);
}
TEST(Metric, DeclareUnifiedTest(NDCGExpGain)) {
Context ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), NDCGColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyNDCG, DataSplitMode::kCol);
}
auto p_fmat = xgboost::RandomDataGenerator{0, 0, 0}.GenerateDMatrix();
MetaInfo& info = p_fmat->Info();
info.labels = linalg::Matrix<float>{{10.0f, 0.0f, 0.0f, 1.0f, 5.0f}, {5}, ctx.gpu_id};
info.num_row_ = info.labels.Shape(0);
info.group_ptr_.resize(2);
info.group_ptr_[0] = 0;
info.group_ptr_[1] = info.num_row_;
HostDeviceVector<float> predt{{0.1f, 0.2f, 0.3f, 4.0f, 70.0f}};
TEST_F(DeclareUnifiedDistributedTest(MetricTest), MAPRowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyMAP, DataSplitMode::kRow);
}
std::unique_ptr<Metric> metric{Metric::Create("ndcg", &ctx)};
Json config{Object{}};
config["name"] = String{"ndcg"};
config["lambdarank_param"] = Object{};
config["lambdarank_param"]["ndcg_exp_gain"] = String{"true"};
config["lambdarank_param"]["lambdarank_num_pair_per_sample"] = String{"32"};
metric->LoadConfig(config);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), MAPColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyMAP, DataSplitMode::kCol);
}
auto ndcg = metric->Evaluate(predt, p_fmat);
ASSERT_NEAR(ndcg, 0.409738f, kRtEps);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), NDCGExpGainRowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyNDCGExpGain, DataSplitMode::kRow);
}
config["lambdarank_param"]["ndcg_exp_gain"] = String{"false"};
metric->LoadConfig(config);
ndcg = metric->Evaluate(predt, p_fmat);
ASSERT_NEAR(ndcg, 0.695694f, kRtEps);
predt.HostVector() = info.labels.Data()->HostVector();
ndcg = metric->Evaluate(predt, p_fmat);
ASSERT_NEAR(ndcg, 1.0, kRtEps);
TEST_F(DeclareUnifiedDistributedTest(MetricTest), NDCGExpGainColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyNDCGExpGain, DataSplitMode::kCol);
}
} // namespace metric
} // namespace xgboost

191
tests/cpp/metric/test_rank_metric.h Normal file
View File

@ -0,0 +1,191 @@
/**
* Copyright 2016-2023 by XGBoost Contributors
*/
#pragma once
#include <gtest/gtest.h> // for Test, EXPECT_NEAR, ASSERT_STREQ
#include <xgboost/context.h> // for Context
#include <xgboost/data.h> // for MetaInfo, DMatrix
#include <xgboost/linalg.h> // for Matrix
#include <xgboost/metric.h> // for Metric
#include <algorithm> // for max
#include <memory> // for unique_ptr
#include <vector> // for vector
#include "../helpers.h" // for GetMetricEval, CreateEmptyGe...
#include "xgboost/base.h" // for bst_float, kRtEps
#include "xgboost/host_device_vector.h" // for HostDeviceVector
#include "xgboost/json.h" // for Json, String, Object
namespace xgboost {
namespace metric {
inline void VerifyPrecision(DataSplitMode data_split_mode = DataSplitMode::kRow) {
// When the limit for precision is not given, it takes the limit at
// std::numeric_limits<unsigned>::max(); hence all values are very small
// NOTE(AbdealiJK): Maybe this should be fixed to be num_row by default.
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
xgboost::Metric * metric = xgboost::Metric::Create("pre", &ctx);
ASSERT_STREQ(metric->Name(), "pre");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-7);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}, {}, {}, data_split_mode),
0, 1e-7);
delete metric;
metric = xgboost::Metric::Create("pre@2", &ctx);
ASSERT_STREQ(metric->Name(), "pre@2");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0.5f, 1e-7);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}, {}, {}, data_split_mode),
0.5f, 0.001f);
EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {}, {}, {}, data_split_mode));
delete metric;
}
inline void VerifyNDCG(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = CreateEmptyGenericParam(GPUIDX);
Metric * metric = xgboost::Metric::Create("ndcg", &ctx);
ASSERT_STREQ(metric->Name(), "ndcg");
EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {}, {}, {}, data_split_mode));
ASSERT_NEAR(GetMetricEval(metric,
xgboost::HostDeviceVector<xgboost::bst_float>{},
{}, {}, {}, data_split_mode), 1, 1e-10);
ASSERT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 1, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}, {}, {}, data_split_mode),
0.6509f, 0.001f);
delete metric;
metric = xgboost::Metric::Create("ndcg@2", &ctx);
ASSERT_STREQ(metric->Name(), "ndcg@2");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 1, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}, {}, {}, data_split_mode),
0.3868f, 0.001f);
delete metric;
metric = xgboost::Metric::Create("ndcg@-", &ctx);
ASSERT_STREQ(metric->Name(), "ndcg-");
EXPECT_NEAR(GetMetricEval(metric,
xgboost::HostDeviceVector<xgboost::bst_float>{},
{}, {}, {}, data_split_mode), 0, 1e-10);
ASSERT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 1.f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}, {}, {}, data_split_mode),
0.6509f, 0.001f);
delete metric;
metric = xgboost::Metric::Create("ndcg-", &ctx);
ASSERT_STREQ(metric->Name(), "ndcg-");
EXPECT_NEAR(GetMetricEval(metric,
xgboost::HostDeviceVector<xgboost::bst_float>{},
{}, {}, {}, data_split_mode), 0, 1e-10);
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 1.f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}, {}, {}, data_split_mode),
0.6509f, 0.001f);
delete metric;
metric = xgboost::Metric::Create("ndcg@2-", &ctx);
ASSERT_STREQ(metric->Name(), "ndcg@2-");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 1.f, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}, {}, {}, data_split_mode),
1.f - 0.3868f, 1.f - 0.001f);
delete metric;
}
inline void VerifyMAP(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
Metric * metric = xgboost::Metric::Create("map", &ctx);
ASSERT_STREQ(metric->Name(), "map");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 1, kRtEps);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}, {}, {}, data_split_mode),
0.5f, 0.001f);
EXPECT_NEAR(GetMetricEval(metric,
xgboost::HostDeviceVector<xgboost::bst_float>{},
std::vector<xgboost::bst_float>{}, {}, {}, data_split_mode), 1, 1e-10);
// Rank metric with group info
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.2f, 0.8f, 0.4f, 1.7f},
{1, 1, 1, 0, 1, 0}, // Labels
{}, // Weights
{0, 2, 5, 6}, // Group info
data_split_mode),
0.8611f, 0.001f);
delete metric;
metric = xgboost::Metric::Create("map@-", &ctx);
ASSERT_STREQ(metric->Name(), "map-");
EXPECT_NEAR(GetMetricEval(metric,
xgboost::HostDeviceVector<xgboost::bst_float>{},
{}, {}, {}, data_split_mode), 0, 1e-10);
delete metric;
metric = xgboost::Metric::Create("map-", &ctx);
ASSERT_STREQ(metric->Name(), "map-");
EXPECT_NEAR(GetMetricEval(metric,
xgboost::HostDeviceVector<xgboost::bst_float>{},
{}, {}, {}, data_split_mode), 0, 1e-10);
delete metric;
metric = xgboost::Metric::Create("map@2", &ctx);
ASSERT_STREQ(metric->Name(), "map@2");
EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 1, 1e-10);
EXPECT_NEAR(GetMetricEval(metric,
{0.1f, 0.9f, 0.1f, 0.9f},
{ 0, 0, 1, 1}, {}, {}, data_split_mode),
0.25f, 0.001f);
delete metric;
}
inline void VerifyNDCGExpGain(DataSplitMode data_split_mode = DataSplitMode::kRow) {
Context ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
auto p_fmat = xgboost::RandomDataGenerator{0, 0, 0}.GenerateDMatrix();
MetaInfo& info = p_fmat->Info();
info.labels = linalg::Matrix<float>{{10.0f, 0.0f, 0.0f, 1.0f, 5.0f}, {5}, ctx.gpu_id};
info.num_row_ = info.labels.Shape(0);
info.group_ptr_.resize(2);
info.group_ptr_[0] = 0;
info.group_ptr_[1] = info.num_row_;
info.data_split_mode = data_split_mode;
HostDeviceVector<float> predt{{0.1f, 0.2f, 0.3f, 4.0f, 70.0f}};
std::unique_ptr<Metric> metric{Metric::Create("ndcg", &ctx)};
Json config{Object{}};
config["name"] = String{"ndcg"};
config["lambdarank_param"] = Object{};
config["lambdarank_param"]["ndcg_exp_gain"] = String{"true"};
config["lambdarank_param"]["lambdarank_num_pair_per_sample"] = String{"32"};
metric->LoadConfig(config);
auto ndcg = metric->Evaluate(predt, p_fmat);
ASSERT_NEAR(ndcg, 0.409738f, kRtEps);
config["lambdarank_param"]["ndcg_exp_gain"] = String{"false"};
metric->LoadConfig(config);
ndcg = metric->Evaluate(predt, p_fmat);
ASSERT_NEAR(ndcg, 0.695694f, kRtEps);
predt.HostVector() = info.labels.Data()->HostVector();
ndcg = metric->Evaluate(predt, p_fmat);
ASSERT_NEAR(ndcg, 1.0, kRtEps);
}
} // namespace metric
} // namespace xgboost

28
tests/cpp/metric/test_survival_metric.cu
View File

@ -46,9 +46,8 @@ inline void CheckDeterministicMetricElementWise(StringView name, int32_t device)
ASSERT_EQ(metric->Evaluate(predts, p_fmat), result);
}
}
} // anonymous namespace
TEST(Metric, DeclareUnifiedTest(AFTNegLogLik)) {
void VerifyAFTNegLogLik(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
/**
@ -63,6 +62,7 @@ TEST(Metric, DeclareUnifiedTest(AFTNegLogLik)) {
info.labels_upper_bound_.HostVector()
= { 100.0f, 20.0f, std::numeric_limits<bst_float>::infinity(), 200.0f };
info.weights_.HostVector() = std::vector<bst_float>();
info.data_split_mode = data_split_mode;
HostDeviceVector<bst_float> preds(4, std::log(64));
struct TestCase {
@ -78,7 +78,7 @@ TEST(Metric, DeclareUnifiedTest(AFTNegLogLik)) {
}
}
TEST(Metric, DeclareUnifiedTest(IntervalRegressionAccuracy)) {
void VerifyIntervalRegressionAccuracy(DataSplitMode data_split_mode = DataSplitMode::kRow) {
auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
auto p_fmat = EmptyDMatrix();
@ -87,6 +87,7 @@ TEST(Metric, DeclareUnifiedTest(IntervalRegressionAccuracy)) {
info.labels_lower_bound_.HostVector() = { 20.0f, 0.0f, 60.0f, 16.0f };
info.labels_upper_bound_.HostVector() = { 80.0f, 20.0f, 80.0f, 200.0f };
info.weights_.HostVector() = std::vector<bst_float>();
info.data_split_mode = data_split_mode;
HostDeviceVector<bst_float> preds(4, std::log(60.0f));
std::unique_ptr<Metric> metric(Metric::Create("interval-regression-accuracy", &ctx));
@ -102,6 +103,27 @@ TEST(Metric, DeclareUnifiedTest(IntervalRegressionAccuracy)) {
CheckDeterministicMetricElementWise(StringView{"interval-regression-accuracy"}, GPUIDX);
}
} // anonymous namespace
TEST(Metric, DeclareUnifiedTest(AFTNegLogLik)) { VerifyAFTNegLogLik(); }
TEST_F(DeclareUnifiedDistributedTest(MetricTest), AFTNegLogLikRowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyAFTNegLogLik, DataSplitMode::kRow);
}
TEST_F(DeclareUnifiedDistributedTest(MetricTest), AFTNegLogLikColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyAFTNegLogLik, DataSplitMode::kCol);
}
TEST(Metric, DeclareUnifiedTest(IntervalRegressionAccuracy)) { VerifyIntervalRegressionAccuracy(); }
TEST_F(DeclareUnifiedDistributedTest(MetricTest), IntervalRegressionAccuracyRowSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyIntervalRegressionAccuracy, DataSplitMode::kRow);
}
TEST_F(DeclareUnifiedDistributedTest(MetricTest), IntervalRegressionAccuracyColumnSplit) {
RunWithInMemoryCommunicator(world_size_, &VerifyIntervalRegressionAccuracy, DataSplitMode::kCol);
}
// Test configuration of AFT metric
TEST(AFTNegLogLikMetric, DeclareUnifiedTest(Configuration)) {