diff --git a/src/metric/auc.cc b/src/metric/auc.cc index a926c2c5b..2d4becfa8 100644 --- a/src/metric/auc.cc +++ b/src/metric/auc.cc @@ -116,8 +116,10 @@ double MultiClassOVR(Context const *ctx, common::Span predts, MetaI // we have 2 averages going in here, first is among workers, second is among // classes. allreduce sums up fp/tp auc for each class. - collective::Allreduce(results.Values().data(), - results.Values().size()); + if (info.IsRowSplit()) { + collective::Allreduce(results.Values().data(), + results.Values().size()); + } double auc_sum{0}; double tp_sum{0}; for (size_t c = 0; c < n_classes; ++c) { @@ -290,7 +292,9 @@ class EvalAUC : public MetricNoCache { } std::array results{auc, static_cast(valid_groups)}; - collective::Allreduce(results.data(), results.size()); + if (info.IsRowSplit()) { + collective::Allreduce(results.data(), results.size()); + } auc = results[0]; valid_groups = static_cast(results[1]); @@ -319,7 +323,9 @@ class EvalAUC : public MetricNoCache { } double local_area = fp * tp; std::array result{auc, local_area}; - collective::Allreduce(result.data(), result.size()); + if (info.IsRowSplit()) { + collective::Allreduce(result.data(), result.size()); + } std::tie(auc, local_area) = common::UnpackArr(std::move(result)); if (local_area <= 0) { // the dataset across all workers have only positive or negative sample diff --git a/src/metric/elementwise_metric.cu b/src/metric/elementwise_metric.cu index 9006bdfca..b1c764047 100644 --- a/src/metric/elementwise_metric.cu +++ b/src/metric/elementwise_metric.cu @@ -198,7 +198,7 @@ class PseudoErrorLoss : public MetricNoCache { return std::make_tuple(v, wt); }); double dat[2]{result.Residue(), result.Weights()}; - if (collective::IsDistributed()) { + if (info.IsRowSplit()) { collective::Allreduce(dat, 2); } return EvalRowMAPE::GetFinal(dat[0], dat[1]); @@ -367,7 +367,9 @@ struct EvalEWiseBase : public MetricNoCache { }); double dat[2]{result.Residue(), result.Weights()}; - collective::Allreduce(dat, 2); + if (info.IsRowSplit()) { + collective::Allreduce(dat, 2); + } return Policy::GetFinal(dat[0], dat[1]); } @@ -439,7 +441,9 @@ class QuantileError : public MetricNoCache { if (info.num_row_ == 0) { // empty DMatrix on distributed env double dat[2]{0.0, 0.0}; - collective::Allreduce(dat, 2); + if (info.IsRowSplit()) { + collective::Allreduce(dat, 2); + } CHECK_GT(dat[1], 0); return dat[0] / dat[1]; } @@ -477,7 +481,9 @@ class QuantileError : public MetricNoCache { return std::make_tuple(l, w); }); double dat[2]{result.Residue(), result.Weights()}; - collective::Allreduce(dat, 2); + if (info.IsRowSplit()) { + collective::Allreduce(dat, 2); + } CHECK_GT(dat[1], 0); return dat[0] / dat[1]; } diff --git a/src/metric/multiclass_metric.cu b/src/metric/multiclass_metric.cu index aed6e7f4b..a1d19dbc8 100644 --- a/src/metric/multiclass_metric.cu +++ b/src/metric/multiclass_metric.cu @@ -181,7 +181,9 @@ struct EvalMClassBase : public MetricNoCache { dat[0] = result.Residue(); dat[1] = result.Weights(); } - collective::Allreduce(dat, 2); + if (info.IsRowSplit()) { + collective::Allreduce(dat, 2); + } return Derived::GetFinal(dat[0], dat[1]); } /*! diff --git a/src/metric/rank_metric.cc b/src/metric/rank_metric.cc index a84d0edb1..62efd0876 100644 --- a/src/metric/rank_metric.cc +++ b/src/metric/rank_metric.cc @@ -244,7 +244,7 @@ struct EvalRank : public MetricNoCache, public EvalRankConfig { exc.Rethrow(); } - if (collective::IsDistributed()) { + if (collective::IsDistributed() && info.IsRowSplit()) { double dat[2]{sum_metric, static_cast(ngroups)}; // approximately estimate the metric using mean collective::Allreduce(dat, 2); @@ -401,9 +401,11 @@ class EvalRankWithCache : public Metric { }; namespace { -double Finalize(double score, double sw) { +double Finalize(MetaInfo const& info, double score, double sw) { std::array dat{score, sw}; - collective::Allreduce(dat.data(), dat.size()); + if (info.IsRowSplit()) { + collective::Allreduce(dat.data(), dat.size()); + } if (sw > 0.0) { score = score / sw; } @@ -430,7 +432,7 @@ class EvalNDCG : public EvalRankWithCache { std::shared_ptr p_cache) override { if (ctx_->IsCUDA()) { auto ndcg = cuda_impl::NDCGScore(ctx_, info, preds, minus_, p_cache); - return Finalize(ndcg.Residue(), ndcg.Weights()); + return Finalize(info, ndcg.Residue(), ndcg.Weights()); } // group local ndcg @@ -476,7 +478,7 @@ class EvalNDCG : public EvalRankWithCache { sum_w = std::accumulate(weights.weights.cbegin(), weights.weights.cend(), 0.0); } auto ndcg = std::accumulate(linalg::cbegin(ndcg_gloc), linalg::cend(ndcg_gloc), 0.0); - return Finalize(ndcg, sum_w); + return Finalize(info, ndcg, sum_w); } }; @@ -489,7 +491,7 @@ class EvalMAPScore : public EvalRankWithCache { std::shared_ptr p_cache) override { if (ctx_->IsCUDA()) { auto map = cuda_impl::MAPScore(ctx_, info, predt, minus_, p_cache); - return Finalize(map.Residue(), map.Weights()); + return Finalize(info, map.Residue(), map.Weights()); } auto gptr = p_cache->DataGroupPtr(ctx_); @@ -532,7 +534,7 @@ class EvalMAPScore : public EvalRankWithCache { sw += weight[i]; } auto sum = std::accumulate(map_gloc.cbegin(), map_gloc.cend(), 0.0); - return Finalize(sum, sw); + return Finalize(info, sum, sw); } }; diff --git a/src/metric/survival_metric.cu b/src/metric/survival_metric.cu index 8205f07a1..9b1773dc5 100644 --- a/src/metric/survival_metric.cu +++ b/src/metric/survival_metric.cu @@ -212,7 +212,9 @@ struct EvalEWiseSurvivalBase : public MetricNoCache { info.labels_upper_bound_, preds); double dat[2]{result.Residue(), result.Weights()}; - collective::Allreduce(dat, 2); + if (info.IsRowSplit()) { + collective::Allreduce(dat, 2); + } return Policy::GetFinal(dat[0], dat[1]); } diff --git a/tests/cpp/helpers.cc b/tests/cpp/helpers.cc index 0c0c9fc9f..a8b974f03 100644 --- a/tests/cpp/helpers.cc +++ b/tests/cpp/helpers.cc @@ -167,18 +167,20 @@ xgboost::bst_float GetMetricEval(xgboost::Metric* metric, xgboost::HostDeviceVector const& preds, std::vector labels, std::vector weights, - std::vector groups) { + std::vector groups, + xgboost::DataSplitMode data_split_mode) { return GetMultiMetricEval( metric, preds, xgboost::linalg::Tensor{labels.begin(), labels.end(), {labels.size()}, -1}, weights, - groups); + groups, data_split_mode); } double GetMultiMetricEval(xgboost::Metric* metric, xgboost::HostDeviceVector const& preds, xgboost::linalg::Tensor const& labels, std::vector weights, - std::vector groups) { + std::vector groups, + xgboost::DataSplitMode data_split_mode) { std::shared_ptr p_fmat{xgboost::RandomDataGenerator{0, 0, 0}.GenerateDMatrix()}; auto& info = p_fmat->Info(); info.num_row_ = labels.Shape(0); @@ -186,6 +188,7 @@ double GetMultiMetricEval(xgboost::Metric* metric, info.labels.Data()->Copy(*labels.Data()); info.weights_.HostVector() = weights; info.group_ptr_ = groups; + info.data_split_mode = data_split_mode; return metric->Evaluate(preds, p_fmat); } diff --git a/tests/cpp/helpers.h b/tests/cpp/helpers.h index 025feae3e..5e65a1636 100644 --- a/tests/cpp/helpers.h +++ b/tests/cpp/helpers.h @@ -39,6 +39,18 @@ #define GPUIDX -1 #endif +#if defined(__CUDACC__) +#define DeclareUnifiedDistributedTest(name) MGPU ## name +#else +#define DeclareUnifiedDistributedTest(name) name +#endif + +#if defined(__CUDACC__) +#define WORLD_SIZE_FOR_TEST (xgboost::common::AllVisibleGPUs()) +#else +#define WORLD_SIZE_FOR_TEST (3) +#endif + namespace xgboost { class ObjFunction; class Metric; @@ -92,13 +104,15 @@ xgboost::bst_float GetMetricEval( xgboost::HostDeviceVector const& preds, std::vector labels, std::vector weights = std::vector(), - std::vector groups = std::vector()); + std::vector groups = std::vector(), + xgboost::DataSplitMode data_split_Mode = xgboost::DataSplitMode::kRow); double GetMultiMetricEval(xgboost::Metric* metric, xgboost::HostDeviceVector const& preds, xgboost::linalg::Tensor const& labels, std::vector weights = {}, - std::vector groups = {}); + std::vector groups = {}, + xgboost::DataSplitMode data_split_Mode = xgboost::DataSplitMode::kRow); namespace xgboost { @@ -496,4 +510,17 @@ void RunWithInMemoryCommunicator(int32_t world_size, Function&& function, Args&& thread.join(); } } + +class DeclareUnifiedDistributedTest(MetricTest) : public ::testing::Test { + protected: + int world_size_; + + void SetUp() override { + world_size_ = WORLD_SIZE_FOR_TEST; + if (world_size_ <= 1) { + GTEST_SKIP() << "Skipping MGPU test with # GPUs = " << world_size_; + } + } +}; + } // namespace xgboost diff --git a/tests/cpp/metric/test_auc.cc b/tests/cpp/metric/test_auc.cc index 2a6738899..de42bba53 100644 --- a/tests/cpp/metric/test_auc.cc +++ b/tests/cpp/metric/test_auc.cc @@ -1,261 +1,68 @@ +#include "test_auc.h" + #include -#include "../helpers.h" namespace xgboost { namespace metric { -TEST(Metric, DeclareUnifiedTest(BinaryAUC)) { - auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); - std::unique_ptr uni_ptr {Metric::Create("auc", &ctx)}; - Metric * metric = uni_ptr.get(); - ASSERT_STREQ(metric->Name(), "auc"); +TEST(Metric, DeclareUnifiedTest(BinaryAUC)) { VerifyBinaryAUC(); } - // Binary - EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1.0f, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {1, 0}), 0.0f, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, {0, 0}, {0, 1}), 0.5f, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, {1, 1}, {0, 1}), 0.5f, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, {0, 0}, {1, 0}), 0.5f, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, {1, 1}, {1, 0}), 0.5f, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, {1, 0, 0}, {0, 0, 1}), 0.25f, 1e-10); +TEST(Metric, DeclareUnifiedTest(MultiClassAUC)) { VerifyMultiClassAUC(); } - // Invalid dataset - auto p_fmat = EmptyDMatrix(); - MetaInfo& info = p_fmat->Info(); - info.labels = linalg::Tensor{{0.0f, 0.0f}, {2}, -1}; - float auc = metric->Evaluate({1, 1}, p_fmat); - ASSERT_TRUE(std::isnan(auc)); - *info.labels.Data() = HostDeviceVector{}; - auc = metric->Evaluate(HostDeviceVector{}, p_fmat); - ASSERT_TRUE(std::isnan(auc)); +TEST(Metric, DeclareUnifiedTest(RankingAUC)) { VerifyRankingAUC(); } - EXPECT_NEAR(GetMetricEval(metric, {0, 1, 0, 1}, {0, 1, 0, 1}), 1.0f, 1e-10); +TEST(Metric, DeclareUnifiedTest(PRAUC)) { VerifyPRAUC(); } - // AUC with instance weights - EXPECT_NEAR(GetMetricEval(metric, - {0.9f, 0.1f, 0.4f, 0.3f}, - {0, 0, 1, 1}, - {1.0f, 3.0f, 2.0f, 4.0f}), - 0.75f, 0.001f); +TEST(Metric, DeclareUnifiedTest(MultiClassPRAUC)) { VerifyMultiClassPRAUC(); } - // regression test case - ASSERT_NEAR(GetMetricEval( - metric, - {0.79523796, 0.5201713, 0.79523796, 0.24273258, 0.53452194, - 0.53452194, 0.24273258, 0.5201713, 0.79523796, 0.53452194, - 0.24273258, 0.53452194, 0.79523796, 0.5201713, 0.24273258, - 0.5201713, 0.5201713, 0.53452194, 0.5201713, 0.53452194}, - {0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0}), - 0.5, 1e-10); +TEST(Metric, DeclareUnifiedTest(RankingPRAUC)) { VerifyRankingPRAUC(); } + +TEST_F(DeclareUnifiedDistributedTest(MetricTest), BinaryAUCRowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyBinaryAUC, DataSplitMode::kRow); } -TEST(Metric, DeclareUnifiedTest(MultiClassAUC)) { - auto ctx = CreateEmptyGenericParam(GPUIDX); - std::unique_ptr uni_ptr{ - Metric::Create("auc", &ctx)}; - auto metric = uni_ptr.get(); - - // MultiClass - // 3x3 - EXPECT_NEAR(GetMetricEval(metric, - { - 1.0f, 0.0f, 0.0f, // p_0 - 0.0f, 1.0f, 0.0f, // p_1 - 0.0f, 0.0f, 1.0f // p_2 - }, - {0, 1, 2}), - 1.0f, 1e-10); - - EXPECT_NEAR(GetMetricEval(metric, - { - 1.0f, 0.0f, 0.0f, // p_0 - 0.0f, 1.0f, 0.0f, // p_1 - 0.0f, 0.0f, 1.0f // p_2 - }, - {0, 1, 2}, - {1.0f, 1.0f, 1.0f}), - 1.0f, 1e-10); - - EXPECT_NEAR(GetMetricEval(metric, - { - 1.0f, 0.0f, 0.0f, // p_0 - 0.0f, 1.0f, 0.0f, // p_1 - 0.0f, 0.0f, 1.0f // p_2 - }, - {2, 1, 0}), - 0.5f, 1e-10); - - EXPECT_NEAR(GetMetricEval(metric, - { - 1.0f, 0.0f, 0.0f, // p_0 - 0.0f, 1.0f, 0.0f, // p_1 - 0.0f, 0.0f, 1.0f // p_2 - }, - {2, 0, 1}), - 0.25f, 1e-10); - - // invalid dataset - float auc = GetMetricEval(metric, - { - 1.0f, 0.0f, 0.0f, // p_0 - 0.0f, 1.0f, 0.0f, // p_1 - 0.0f, 0.0f, 1.0f // p_2 - }, - {0, 1, 1}); // no class 2. - EXPECT_TRUE(std::isnan(auc)) << auc; - - HostDeviceVector predts{ - 0.0f, 1.0f, 0.0f, - 1.0f, 0.0f, 0.0f, - 0.0f, 0.0f, 1.0f, - 0.0f, 0.0f, 1.0f, - }; - std::vector labels {1.0f, 0.0f, 2.0f, 1.0f}; - auc = GetMetricEval(metric, predts, labels, {1.0f, 2.0f, 3.0f, 4.0f}); - ASSERT_GT(auc, 0.714); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), BinaryAUCColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyBinaryAUC, DataSplitMode::kCol); } -TEST(Metric, DeclareUnifiedTest(RankingAUC)) { - auto ctx = CreateEmptyGenericParam(GPUIDX); - std::unique_ptr metric{Metric::Create("auc", &ctx)}; - - // single group - EXPECT_NEAR(GetMetricEval(metric.get(), {0.7f, 0.2f, 0.3f, 0.6f}, - {1.0f, 0.8f, 0.4f, 0.2f}, /*weights=*/{}, - {0, 4}), - 0.5f, 1e-10); - - // multi group - EXPECT_NEAR(GetMetricEval(metric.get(), {0, 1, 2, 0, 1, 2}, - {0, 1, 2, 0, 1, 2}, /*weights=*/{}, {0, 3, 6}), - 1.0f, 1e-10); - - EXPECT_NEAR(GetMetricEval(metric.get(), {0, 1, 2, 0, 1, 2}, - {0, 1, 2, 0, 1, 2}, /*weights=*/{1.0f, 2.0f}, - {0, 3, 6}), - 1.0f, 1e-10); - - // AUC metric for grouped datasets - exception scenarios - ASSERT_TRUE(std::isnan( - GetMetricEval(metric.get(), {0, 1, 2}, {0, 0, 0}, {}, {0, 2, 3}))); - - // regression case - HostDeviceVector predt{0.33935383, 0.5149714, 0.32138085, 1.4547751, - 1.2010975, 0.42651367, 0.23104341, 0.83610827, - 0.8494239, 0.07136688, 0.5623144, 0.8086237, - 1.5066161, -4.094787, 0.76887935, -2.4082742}; - std::vector groups{0, 7, 16}; - std::vector labels{1., 0., 0., 1., 2., 1., 0., 0., - 0., 0., 0., 0., 1., 0., 1., 0.}; - - EXPECT_NEAR(GetMetricEval(metric.get(), std::move(predt), labels, - /*weights=*/{}, groups), - 0.769841f, 1e-6); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiClassAUCRowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyMultiClassAUC, DataSplitMode::kRow); } -TEST(Metric, DeclareUnifiedTest(PRAUC)) { - auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); - - xgboost::Metric *metric = xgboost::Metric::Create("aucpr", &ctx); - ASSERT_STREQ(metric->Name(), "aucpr"); - EXPECT_NEAR(GetMetricEval(metric, {0, 0, 1, 1}, {0, 0, 1, 1}), 1, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, {0.1f, 0.9f, 0.1f, 0.9f}, {0, 0, 1, 1}), - 0.5f, 0.001f); - EXPECT_NEAR(GetMetricEval( - metric, - {0.4f, 0.2f, 0.9f, 0.1f, 0.2f, 0.4f, 0.1f, 0.1f, 0.2f, 0.1f}, - {0, 0, 0, 0, 0, 1, 0, 0, 1, 1}), - 0.2908445f, 0.001f); - EXPECT_NEAR(GetMetricEval( - metric, {0.87f, 0.31f, 0.40f, 0.42f, 0.25f, 0.66f, 0.95f, - 0.09f, 0.10f, 0.97f, 0.76f, 0.69f, 0.15f, 0.20f, - 0.30f, 0.14f, 0.07f, 0.58f, 0.61f, 0.08f}, - {0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1}), - 0.2769199f, 0.001f); - auto auc = GetMetricEval(metric, {0, 1}, {}); - ASSERT_TRUE(std::isnan(auc)); - - // AUCPR with instance weights - EXPECT_NEAR(GetMetricEval(metric, - {0.29f, 0.52f, 0.11f, 0.21f, 0.219f, 0.93f, 0.493f, - 0.17f, 0.47f, 0.13f, 0.43f, 0.59f, 0.87f, 0.007f}, - {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0}, - {1, 2, 7, 4, 5, 2.2f, 3.2f, 5, 6, 1, 2, 1.1f, 3.2f, - 4.5f}), // weights - 0.694435f, 0.001f); - - // Both groups contain only pos or neg samples. - auc = GetMetricEval(metric, - {0, 0.1f, 0.3f, 0.5f, 0.7f}, - {1, 1, 0, 0, 0}, - {}, - {0, 2, 5}); - ASSERT_TRUE(std::isnan(auc)); - delete metric; +TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiClassAUCColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyMultiClassAUC, DataSplitMode::kCol); } -TEST(Metric, DeclareUnifiedTest(MultiClassPRAUC)) { - auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); - - std::unique_ptr metric{Metric::Create("aucpr", &ctx)}; - - float auc = 0; - std::vector labels {1.0f, 0.0f, 2.0f}; - HostDeviceVector predts{ - 0.0f, 1.0f, 0.0f, - 1.0f, 0.0f, 0.0f, - 0.0f, 0.0f, 1.0f, - }; - auc = GetMetricEval(metric.get(), predts, labels, {}); - EXPECT_EQ(auc, 1.0f); - - auc = GetMetricEval(metric.get(), predts, labels, {1.0f, 1.0f, 1.0f}); - EXPECT_EQ(auc, 1.0f); - - predts.HostVector() = { - 0.0f, 1.0f, 0.0f, - 1.0f, 0.0f, 0.0f, - 0.0f, 0.0f, 1.0f, - 0.0f, 0.0f, 1.0f, - }; - labels = {1.0f, 0.0f, 2.0f, 1.0f}; - auc = GetMetricEval(metric.get(), predts, labels, {1.0f, 2.0f, 3.0f, 4.0f}); - ASSERT_GT(auc, 0.699); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), RankingAUCRowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyRankingAUC, DataSplitMode::kRow); } -TEST(Metric, DeclareUnifiedTest(RankingPRAUC)) { - auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), RankingAUCColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyRankingAUC, DataSplitMode::kCol); +} - std::unique_ptr metric{Metric::Create("aucpr", &ctx)}; +TEST_F(DeclareUnifiedDistributedTest(MetricTest), PRAUCRowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyPRAUC, DataSplitMode::kRow); +} - std::vector labels {1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f}; - std::vector groups {0, 2, 6}; +TEST_F(DeclareUnifiedDistributedTest(MetricTest), PRAUCColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyPRAUC, DataSplitMode::kCol); +} - float auc = 0; - auc = GetMetricEval(metric.get(), {1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f}, labels, {}, groups); - EXPECT_EQ(auc, 1.0f); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiClassPRAUCRowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyMultiClassPRAUC, DataSplitMode::kRow); +} - auc = GetMetricEval(metric.get(), {1.0f, 0.5f, 0.8f, 0.3f, 0.2f, 1.0f}, labels, {}, groups); - EXPECT_EQ(auc, 1.0f); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiClassPRAUCColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyMultiClassPRAUC, DataSplitMode::kCol); +} - auc = GetMetricEval(metric.get(), {1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f}, - {1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f}, {}, groups); - ASSERT_TRUE(std::isnan(auc)); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), RankingPRAUCRowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyRankingPRAUC, DataSplitMode::kRow); +} - // Incorrect label - ASSERT_THROW(GetMetricEval(metric.get(), {1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f}, - {1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 3.0f}, {}, groups), - dmlc::Error); - - // AUCPR with groups and no weights - EXPECT_NEAR(GetMetricEval( - metric.get(), {0.87f, 0.31f, 0.40f, 0.42f, 0.25f, 0.66f, 0.95f, - 0.09f, 0.10f, 0.97f, 0.76f, 0.69f, 0.15f, 0.20f, - 0.30f, 0.14f, 0.07f, 0.58f, 0.61f, 0.08f}, - {0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1}, - {}, // weights - {0, 2, 5, 9, 14, 20}), // group info - 0.556021f, 0.001f); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), RankingPRAUCColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyRankingPRAUC, DataSplitMode::kCol); } } // namespace metric } // namespace xgboost diff --git a/tests/cpp/metric/test_auc.h b/tests/cpp/metric/test_auc.h new file mode 100644 index 000000000..3baa53290 --- /dev/null +++ b/tests/cpp/metric/test_auc.h @@ -0,0 +1,249 @@ +/*! + * Copyright (c) 2023 by XGBoost Contributors + */ +#pragma once + +#include + +#include "../helpers.h" + +namespace xgboost { +namespace metric { + +inline void VerifyBinaryAUC(DataSplitMode data_split_mode = DataSplitMode::kRow) { + auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); + std::unique_ptr uni_ptr{Metric::Create("auc", &ctx)}; + Metric* metric = uni_ptr.get(); + ASSERT_STREQ(metric->Name(), "auc"); + + // Binary + EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 1.0f, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {1, 0}, {}, {}, data_split_mode), 0.0f, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, {0, 0}, {0, 1}, {}, {}, data_split_mode), 0.5f, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, {1, 1}, {0, 1}, {}, {}, data_split_mode), 0.5f, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, {0, 0}, {1, 0}, {}, {}, data_split_mode), 0.5f, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, {1, 1}, {1, 0}, {}, {}, data_split_mode), 0.5f, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, {1, 0, 0}, {0, 0, 1}, {}, {}, data_split_mode), 0.25f, 1e-10); + + // Invalid dataset + auto p_fmat = EmptyDMatrix(); + MetaInfo& info = p_fmat->Info(); + info.labels = linalg::Tensor{{0.0f, 0.0f}, {2}, -1}; + float auc = metric->Evaluate({1, 1}, p_fmat); + ASSERT_TRUE(std::isnan(auc)); + *info.labels.Data() = HostDeviceVector{}; + auc = metric->Evaluate(HostDeviceVector{}, p_fmat); + ASSERT_TRUE(std::isnan(auc)); + + EXPECT_NEAR(GetMetricEval(metric, {0, 1, 0, 1}, {0, 1, 0, 1}, {}, {}, data_split_mode), 1.0f, + 1e-10); + + // AUC with instance weights + EXPECT_NEAR(GetMetricEval(metric, {0.9f, 0.1f, 0.4f, 0.3f}, {0, 0, 1, 1}, + {1.0f, 3.0f, 2.0f, 4.0f}, {}, data_split_mode), + 0.75f, 0.001f); + + // regression test case + ASSERT_NEAR(GetMetricEval(metric, {0.79523796, 0.5201713, 0.79523796, 0.24273258, 0.53452194, + 0.53452194, 0.24273258, 0.5201713, 0.79523796, 0.53452194, + 0.24273258, 0.53452194, 0.79523796, 0.5201713, 0.24273258, + 0.5201713, 0.5201713, 0.53452194, 0.5201713, 0.53452194}, + {0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0}, {}, {}, + data_split_mode), + 0.5, 1e-10); +} + +inline void VerifyMultiClassAUC(DataSplitMode data_split_mode = DataSplitMode::kRow) { + auto ctx = CreateEmptyGenericParam(GPUIDX); + std::unique_ptr uni_ptr{Metric::Create("auc", &ctx)}; + auto metric = uni_ptr.get(); + + // MultiClass + // 3x3 + EXPECT_NEAR(GetMetricEval(metric, + { + 1.0f, 0.0f, 0.0f, // p_0 + 0.0f, 1.0f, 0.0f, // p_1 + 0.0f, 0.0f, 1.0f // p_2 + }, + {0, 1, 2}, {}, {}, data_split_mode), + 1.0f, 1e-10); + + EXPECT_NEAR(GetMetricEval(metric, + { + 1.0f, 0.0f, 0.0f, // p_0 + 0.0f, 1.0f, 0.0f, // p_1 + 0.0f, 0.0f, 1.0f // p_2 + }, + {0, 1, 2}, {1.0f, 1.0f, 1.0f}, {}, data_split_mode), + 1.0f, 1e-10); + + EXPECT_NEAR(GetMetricEval(metric, + { + 1.0f, 0.0f, 0.0f, // p_0 + 0.0f, 1.0f, 0.0f, // p_1 + 0.0f, 0.0f, 1.0f // p_2 + }, + {2, 1, 0}, {}, {}, data_split_mode), + 0.5f, 1e-10); + + EXPECT_NEAR(GetMetricEval(metric, + { + 1.0f, 0.0f, 0.0f, // p_0 + 0.0f, 1.0f, 0.0f, // p_1 + 0.0f, 0.0f, 1.0f // p_2 + }, + {2, 0, 1}, {}, {}, data_split_mode), + 0.25f, 1e-10); + + // invalid dataset + float auc = GetMetricEval(metric, + { + 1.0f, 0.0f, 0.0f, // p_0 + 0.0f, 1.0f, 0.0f, // p_1 + 0.0f, 0.0f, 1.0f // p_2 + }, + {0, 1, 1}, {}, {}, data_split_mode); // no class 2. + EXPECT_TRUE(std::isnan(auc)) << auc; + + HostDeviceVector predts{ + 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f, + }; + std::vector labels{1.0f, 0.0f, 2.0f, 1.0f}; + auc = GetMetricEval(metric, predts, labels, {1.0f, 2.0f, 3.0f, 4.0f}, {}, data_split_mode); + ASSERT_GT(auc, 0.714); +} + +inline void VerifyRankingAUC(DataSplitMode data_split_mode = DataSplitMode::kRow) { + auto ctx = CreateEmptyGenericParam(GPUIDX); + std::unique_ptr metric{Metric::Create("auc", &ctx)}; + + // single group + EXPECT_NEAR(GetMetricEval(metric.get(), {0.7f, 0.2f, 0.3f, 0.6f}, {1.0f, 0.8f, 0.4f, 0.2f}, + /*weights=*/{}, {0, 4}, data_split_mode), + 0.5f, 1e-10); + + // multi group + EXPECT_NEAR(GetMetricEval(metric.get(), {0, 1, 2, 0, 1, 2}, {0, 1, 2, 0, 1, 2}, /*weights=*/{}, + {0, 3, 6}, data_split_mode), + 1.0f, 1e-10); + + EXPECT_NEAR(GetMetricEval(metric.get(), {0, 1, 2, 0, 1, 2}, {0, 1, 2, 0, 1, 2}, + /*weights=*/{1.0f, 2.0f}, {0, 3, 6}, data_split_mode), + 1.0f, 1e-10); + + // AUC metric for grouped datasets - exception scenarios + ASSERT_TRUE(std::isnan( + GetMetricEval(metric.get(), {0, 1, 2}, {0, 0, 0}, {}, {0, 2, 3}, data_split_mode))); + + // regression case + HostDeviceVector predt{ + 0.33935383, 0.5149714, 0.32138085, 1.4547751, 1.2010975, 0.42651367, 0.23104341, 0.83610827, + 0.8494239, 0.07136688, 0.5623144, 0.8086237, 1.5066161, -4.094787, 0.76887935, -2.4082742}; + std::vector groups{0, 7, 16}; + std::vector labels{1., 0., 0., 1., 2., 1., 0., 0., 0., 0., 0., 0., 1., 0., 1., 0.}; + + EXPECT_NEAR(GetMetricEval(metric.get(), std::move(predt), labels, + /*weights=*/{}, groups, data_split_mode), + 0.769841f, 1e-6); +} + +inline void VerifyPRAUC(DataSplitMode data_split_mode = DataSplitMode::kRow) { + auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); + + xgboost::Metric* metric = xgboost::Metric::Create("aucpr", &ctx); + ASSERT_STREQ(metric->Name(), "aucpr"); + EXPECT_NEAR(GetMetricEval(metric, {0, 0, 1, 1}, {0, 0, 1, 1}, {}, {}, data_split_mode), 1, 1e-10); + EXPECT_NEAR( + GetMetricEval(metric, {0.1f, 0.9f, 0.1f, 0.9f}, {0, 0, 1, 1}, {}, {}, data_split_mode), 0.5f, + 0.001f); + EXPECT_NEAR(GetMetricEval(metric, {0.4f, 0.2f, 0.9f, 0.1f, 0.2f, 0.4f, 0.1f, 0.1f, 0.2f, 0.1f}, + {0, 0, 0, 0, 0, 1, 0, 0, 1, 1}, {}, {}, data_split_mode), + 0.2908445f, 0.001f); + EXPECT_NEAR( + GetMetricEval(metric, {0.87f, 0.31f, 0.40f, 0.42f, 0.25f, 0.66f, 0.95f, 0.09f, 0.10f, 0.97f, + 0.76f, 0.69f, 0.15f, 0.20f, 0.30f, 0.14f, 0.07f, 0.58f, 0.61f, 0.08f}, + {0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1}, {}, {}, + data_split_mode), + 0.2769199f, 0.001f); + auto auc = GetMetricEval(metric, {0, 1}, {}, {}, {}, data_split_mode); + ASSERT_TRUE(std::isnan(auc)); + + // AUCPR with instance weights + EXPECT_NEAR(GetMetricEval(metric, + {0.29f, 0.52f, 0.11f, 0.21f, 0.219f, 0.93f, 0.493f, 0.17f, 0.47f, 0.13f, + 0.43f, 0.59f, 0.87f, 0.007f}, + {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0}, + {1, 2, 7, 4, 5, 2.2f, 3.2f, 5, 6, 1, 2, 1.1f, 3.2f, 4.5f}, // weights + {}, data_split_mode), + 0.694435f, 0.001f); + + // Both groups contain only pos or neg samples. + auc = GetMetricEval(metric, {0, 0.1f, 0.3f, 0.5f, 0.7f}, {1, 1, 0, 0, 0}, {}, {0, 2, 5}, + data_split_mode); + ASSERT_TRUE(std::isnan(auc)); + delete metric; +} + +inline void VerifyMultiClassPRAUC(DataSplitMode data_split_mode = DataSplitMode::kRow) { + auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); + + std::unique_ptr metric{Metric::Create("aucpr", &ctx)}; + + float auc = 0; + std::vector labels{1.0f, 0.0f, 2.0f}; + HostDeviceVector predts{ + 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, + }; + auc = GetMetricEval(metric.get(), predts, labels, {}, {}, data_split_mode); + EXPECT_EQ(auc, 1.0f); + + auc = GetMetricEval(metric.get(), predts, labels, {1.0f, 1.0f, 1.0f}, {}, data_split_mode); + EXPECT_EQ(auc, 1.0f); + + predts.HostVector() = { + 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f, + }; + labels = {1.0f, 0.0f, 2.0f, 1.0f}; + auc = GetMetricEval(metric.get(), predts, labels, {1.0f, 2.0f, 3.0f, 4.0f}, {}, data_split_mode); + ASSERT_GT(auc, 0.699); +} + +inline void VerifyRankingPRAUC(DataSplitMode data_split_mode = DataSplitMode::kRow) { + auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); + + std::unique_ptr metric{Metric::Create("aucpr", &ctx)}; + + std::vector labels{1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f}; + std::vector groups{0, 2, 6}; + + float auc = 0; + auc = GetMetricEval(metric.get(), {1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f}, labels, {}, groups, + data_split_mode); + EXPECT_EQ(auc, 1.0f); + + auc = GetMetricEval(metric.get(), {1.0f, 0.5f, 0.8f, 0.3f, 0.2f, 1.0f}, labels, {}, groups, + data_split_mode); + EXPECT_EQ(auc, 1.0f); + + auc = GetMetricEval(metric.get(), {1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f}, + {1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f}, {}, groups, data_split_mode); + ASSERT_TRUE(std::isnan(auc)); + + // Incorrect label + ASSERT_THROW(GetMetricEval(metric.get(), {1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f}, + {1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 3.0f}, {}, groups, data_split_mode), + dmlc::Error); + + // AUCPR with groups and no weights + EXPECT_NEAR( + GetMetricEval(metric.get(), + {0.87f, 0.31f, 0.40f, 0.42f, 0.25f, 0.66f, 0.95f, 0.09f, 0.10f, 0.97f, + 0.76f, 0.69f, 0.15f, 0.20f, 0.30f, 0.14f, 0.07f, 0.58f, 0.61f, 0.08f}, + {0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1}, {}, // weights + {0, 2, 5, 9, 14, 20}, // group info + data_split_mode), + 0.556021f, 0.001f); +} +} // namespace metric +} // namespace xgboost diff --git a/tests/cpp/metric/test_elementwise_metric.cc b/tests/cpp/metric/test_elementwise_metric.cc index 9000cfc09..2407dde39 100644 --- a/tests/cpp/metric/test_elementwise_metric.cc +++ b/tests/cpp/metric/test_elementwise_metric.cc @@ -1,347 +1,108 @@ /** * Copyright 2018-2023 by XGBoost contributors */ -#include -#include - -#include -#include - -#include "../../../src/common/linalg_op.h" -#include "../helpers.h" - -namespace xgboost { -namespace { -inline void CheckDeterministicMetricElementWise(StringView name, int32_t device) { - auto ctx = CreateEmptyGenericParam(device); - std::unique_ptr metric{Metric::Create(name.c_str(), &ctx)}; - - HostDeviceVector predts; - size_t n_samples = 2048; - - auto p_fmat = EmptyDMatrix(); - MetaInfo& info = p_fmat->Info(); - info.labels.Reshape(n_samples, 1); - info.num_row_ = n_samples; - auto &h_labels = info.labels.Data()->HostVector(); - auto &h_predts = predts.HostVector(); - - SimpleLCG lcg; - SimpleRealUniformDistribution dist{0.0f, 1.0f}; - - h_labels.resize(n_samples); - h_predts.resize(n_samples); - - for (size_t i = 0; i < n_samples; ++i) { - h_predts[i] = dist(&lcg); - h_labels[i] = dist(&lcg); - } - - auto result = metric->Evaluate(predts, p_fmat); - for (size_t i = 0; i < 8; ++i) { - ASSERT_EQ(metric->Evaluate(predts, p_fmat), result); - } -} -} // anonymous namespace -} // namespace xgboost +#include "test_elementwise_metric.h" namespace xgboost { namespace metric { +TEST(Metric, DeclareUnifiedTest(RMSE)) { VerifyRMSE(); } -TEST(Metric, DeclareUnifiedTest(RMSE)) { - auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); - xgboost::Metric * metric = xgboost::Metric::Create("rmse", &ctx); - metric->Configure({}); - ASSERT_STREQ(metric->Name(), "rmse"); - EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}), - 0.6403f, 0.001f); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}, - { -1, 1, 9, -9}), - 2.8284f, 0.001f); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}, - { 1, 2, 9, 8}), - 0.6708f, 0.001f); - delete metric; +TEST(Metric, DeclareUnifiedTest(RMSLE)) { VerifyRMSLE(); } - xgboost::CheckDeterministicMetricElementWise(xgboost::StringView{"rmse"}, GPUIDX); +TEST(Metric, DeclareUnifiedTest(MAE)) { VerifyMAE(); } + +TEST(Metric, DeclareUnifiedTest(MAPE)) { VerifyMAPE(); } + +TEST(Metric, DeclareUnifiedTest(MPHE)) { VerifyMPHE(); } + +TEST(Metric, DeclareUnifiedTest(LogLoss)) { VerifyLogLoss(); } + +TEST(Metric, DeclareUnifiedTest(Error)) { VerifyError(); } + +TEST(Metric, DeclareUnifiedTest(PoissonNegLogLik)) { VerifyPoissonNegLogLik(); } + +TEST(Metric, DeclareUnifiedTest(MultiRMSE)) { VerifyMultiRMSE(); } + +TEST(Metric, DeclareUnifiedTest(Quantile)) { VerifyQuantile(); } + +TEST_F(DeclareUnifiedDistributedTest(MetricTest), RMSERowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyRMSE, DataSplitMode::kRow); } -TEST(Metric, DeclareUnifiedTest(RMSLE)) { - auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); - xgboost::Metric * metric = xgboost::Metric::Create("rmsle", &ctx); - metric->Configure({}); - ASSERT_STREQ(metric->Name(), "rmsle"); - EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.2f, 0.4f, 0.8f, 1.6f}, - {1.0f, 1.0f, 1.0f, 1.0f, 1.0f}), - 0.4063f, 1e-4); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.2f, 0.4f, 0.8f, 1.6f}, - {1.0f, 1.0f, 1.0f, 1.0f, 1.0f}, - { 0, -1, 1, -9, 9}), - 0.6212f, 1e-4); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.2f, 0.4f, 0.8f, 1.6f}, - {1.0f, 1.0f, 1.0f, 1.0f, 1.0f}, - { 0, 1, 2, 9, 8}), - 0.2415f, 1e-4); - delete metric; - - xgboost::CheckDeterministicMetricElementWise(xgboost::StringView{"rmsle"}, GPUIDX); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), RMSEColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyRMSE, DataSplitMode::kCol); } -TEST(Metric, DeclareUnifiedTest(MAE)) { - auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); - xgboost::Metric * metric = xgboost::Metric::Create("mae", &ctx); - metric->Configure({}); - ASSERT_STREQ(metric->Name(), "mae"); - EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}), - 0.5f, 0.001f); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}, - { -1, 1, 9, -9}), - 8.0f, 0.001f); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}, - { 1, 2, 9, 8}), - 0.54f, 0.001f); - delete metric; - - xgboost::CheckDeterministicMetricElementWise(xgboost::StringView{"mae"}, GPUIDX); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), RMSLERowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyRMSLE, DataSplitMode::kRow); } -TEST(Metric, DeclareUnifiedTest(MAPE)) { - auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); - xgboost::Metric * metric = xgboost::Metric::Create("mape", &ctx); - metric->Configure({}); - ASSERT_STREQ(metric->Name(), "mape"); - EXPECT_NEAR(GetMetricEval(metric, {150, 300}, {100, 200}), 0.5f, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, - {50, 400, 500, 4000}, - {100, 200, 500, 1000}), - 1.125f, 0.001f); - EXPECT_NEAR(GetMetricEval(metric, - {50, 400, 500, 4000}, - {100, 200, 500, 1000}, - { -1, 1, 9, -9}), - -26.5f, 0.001f); - EXPECT_NEAR(GetMetricEval(metric, - {50, 400, 500, 4000}, - {100, 200, 500, 1000}, - { 1, 2, 9, 8}), - 1.3250f, 0.001f); - delete metric; - - xgboost::CheckDeterministicMetricElementWise(xgboost::StringView{"mape"}, GPUIDX); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), RMSLEColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyRMSLE, DataSplitMode::kCol); } -TEST(Metric, DeclareUnifiedTest(MPHE)) { - auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); - std::unique_ptr metric{xgboost::Metric::Create("mphe", &ctx)}; - metric->Configure({}); - ASSERT_STREQ(metric->Name(), "mphe"); - EXPECT_NEAR(GetMetricEval(metric.get(), {0, 1}, {0, 1}), 0, 1e-10); - EXPECT_NEAR(GetMetricEval(metric.get(), - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}), - 0.1751f, 1e-4); - EXPECT_NEAR(GetMetricEval(metric.get(), - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}, - { -1, 1, 9, -9}), - 3.4037f, 1e-4); - EXPECT_NEAR(GetMetricEval(metric.get(), - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}, - { 1, 2, 9, 8}), - 0.1922f, 1e-4); - - xgboost::CheckDeterministicMetricElementWise(xgboost::StringView{"mphe"}, GPUIDX); - - metric->Configure({{"huber_slope", "0.1"}}); - EXPECT_NEAR(GetMetricEval(metric.get(), - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}, - { 1, 2, 9, 8}), - 0.0461686f, 1e-4); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), MAERowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyMAE, DataSplitMode::kRow); } -TEST(Metric, DeclareUnifiedTest(LogLoss)) { - auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); - xgboost::Metric * metric = xgboost::Metric::Create("logloss", &ctx); - metric->Configure({}); - ASSERT_STREQ(metric->Name(), "logloss"); - EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, - {0.5f, 1e-17f, 1.0f+1e-17f, 0.9f}, - { 0, 0, 1, 1}), - 0.1996f, 0.001f); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}), - 1.2039f, 0.001f); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}, - { -1, 1, 9, -9}), - 21.9722f, 0.001f); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}, - { 1, 2, 9, 8}), - 1.3138f, 0.001f); - delete metric; - - xgboost::CheckDeterministicMetricElementWise(xgboost::StringView{"logloss"}, GPUIDX); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), MAEColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyMAE, DataSplitMode::kCol); } -TEST(Metric, DeclareUnifiedTest(Error)) { - auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); - xgboost::Metric * metric = xgboost::Metric::Create("error", &ctx); - metric->Configure({}); - ASSERT_STREQ(metric->Name(), "error"); - EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}), - 0.5f, 0.001f); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}, - { -1, 1, 9, -9}), - 10.0f, 0.001f); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}, - { 1, 2, 9, 8}), - 0.55f, 0.001f); - - EXPECT_ANY_THROW(xgboost::Metric::Create("error@abc", &ctx)); - delete metric; - - metric = xgboost::Metric::Create("error@0.5f", &ctx); - metric->Configure({}); - EXPECT_STREQ(metric->Name(), "error"); - - delete metric; - - metric = xgboost::Metric::Create("error@0.1", &ctx); - metric->Configure({}); - ASSERT_STREQ(metric->Name(), "error@0.1"); - EXPECT_STREQ(metric->Name(), "error@0.1"); - EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, - {-0.1f, -0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}), - 0.25f, 0.001f); - EXPECT_NEAR(GetMetricEval(metric, - {-0.1f, -0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}, - { -1, 1, 9, -9}), - 9.0f, 0.001f); - EXPECT_NEAR(GetMetricEval(metric, - {-0.1f, -0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}, - { 1, 2, 9, 8}), - 0.45f, 0.001f); - delete metric; - - xgboost::CheckDeterministicMetricElementWise(xgboost::StringView{"error@0.5"}, GPUIDX); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), MAPERowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyMAPE, DataSplitMode::kRow); } -TEST(Metric, DeclareUnifiedTest(PoissionNegLogLik)) { - auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); - xgboost::Metric * metric = xgboost::Metric::Create("poisson-nloglik", &ctx); - metric->Configure({}); - ASSERT_STREQ(metric->Name(), "poisson-nloglik"); - EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0.5f, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, - {0.5f, 1e-17f, 1.0f+1e-17f, 0.9f}, - { 0, 0, 1, 1}), - 0.6263f, 0.001f); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}), - 1.1019f, 0.001f); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}, - { -1, 1, 9, -9}), - 13.3750f, 0.001f); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}, - { 1, 2, 9, 8}), - 1.5783f, 0.001f); - delete metric; - - xgboost::CheckDeterministicMetricElementWise(xgboost::StringView{"poisson-nloglik"}, GPUIDX); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), MAPEColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyMAPE, DataSplitMode::kCol); } -TEST(Metric, DeclareUnifiedTest(MultiRMSE)) { - size_t n_samples = 32, n_targets = 8; - linalg::Tensor y{{n_samples, n_targets}, GPUIDX}; - auto &h_y = y.Data()->HostVector(); - std::iota(h_y.begin(), h_y.end(), 0); - - HostDeviceVector predt(n_samples * n_targets, 0); - - auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); - std::unique_ptr metric{Metric::Create("rmse", &ctx)}; - metric->Configure({}); - - auto loss = GetMultiMetricEval(metric.get(), predt, y); - std::vector weights(n_samples, 1); - auto loss_w = GetMultiMetricEval(metric.get(), predt, y, weights); - - std::transform(h_y.cbegin(), h_y.cend(), h_y.begin(), [](auto &v) { return v * v; }); - auto ret = std::sqrt(std::accumulate(h_y.cbegin(), h_y.cend(), 1.0, std::plus<>{}) / h_y.size()); - ASSERT_FLOAT_EQ(ret, loss); - ASSERT_FLOAT_EQ(ret, loss_w); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), MPHERowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyMPHE, DataSplitMode::kRow); } -TEST(Metric, DeclareUnifiedTest(Quantile)) { - auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); - std::unique_ptr metric{Metric::Create("quantile", &ctx)}; +TEST_F(DeclareUnifiedDistributedTest(MetricTest), MPHEColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyMPHE, DataSplitMode::kCol); +} - HostDeviceVector predts{0.1f, 0.9f, 0.1f, 0.9f}; - std::vector labels{0.5f, 0.5f, 0.9f, 0.1f}; - std::vector weights{0.2f, 0.4f,0.6f, 0.8f}; +TEST_F(DeclareUnifiedDistributedTest(MetricTest), LogLossRowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyLogLoss, DataSplitMode::kRow); +} - metric->Configure(Args{{"quantile_alpha", "[0.0]"}}); - EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights), 0.400f, 0.001f); - metric->Configure(Args{{"quantile_alpha", "[0.2]"}}); - EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights), 0.376f, 0.001f); - metric->Configure(Args{{"quantile_alpha", "[0.4]"}}); - EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights), 0.352f, 0.001f); - metric->Configure(Args{{"quantile_alpha", "[0.8]"}}); - EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights), 0.304f, 0.001f); - metric->Configure(Args{{"quantile_alpha", "[1.0]"}}); - EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights), 0.28f, 0.001f); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), LogLossColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyLogLoss, DataSplitMode::kCol); +} - metric->Configure(Args{{"quantile_alpha", "[0.0]"}}); - EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels), 0.3f, 0.001f); - metric->Configure(Args{{"quantile_alpha", "[0.2]"}}); - EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels), 0.3f, 0.001f); - metric->Configure(Args{{"quantile_alpha", "[0.4]"}}); - EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels), 0.3f, 0.001f); - metric->Configure(Args{{"quantile_alpha", "[0.8]"}}); - EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels), 0.3f, 0.001f); - metric->Configure(Args{{"quantile_alpha", "[1.0]"}}); - EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels), 0.3f, 0.001f); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), ErrorRowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyError, DataSplitMode::kRow); +} + +TEST_F(DeclareUnifiedDistributedTest(MetricTest), ErrorColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyError, DataSplitMode::kCol); +} + +TEST_F(DeclareUnifiedDistributedTest(MetricTest), PoissonNegLogLikRowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyPoissonNegLogLik, DataSplitMode::kRow); +} + +TEST_F(DeclareUnifiedDistributedTest(MetricTest), PoissonNegLogLikColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyPoissonNegLogLik, DataSplitMode::kCol); +} + +TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiRMSERowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyMultiRMSE, DataSplitMode::kRow); +} + +TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiRMSEColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyMultiRMSE, DataSplitMode::kCol); +} + +TEST_F(DeclareUnifiedDistributedTest(MetricTest), QuantileRowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyQuantile, DataSplitMode::kRow); +} + +TEST_F(DeclareUnifiedDistributedTest(MetricTest), QuantileColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyQuantile, DataSplitMode::kCol); } } // namespace metric } // namespace xgboost diff --git a/tests/cpp/metric/test_elementwise_metric.h b/tests/cpp/metric/test_elementwise_metric.h new file mode 100644 index 000000000..1b06194fe --- /dev/null +++ b/tests/cpp/metric/test_elementwise_metric.h @@ -0,0 +1,385 @@ +/** + * Copyright 2018-2023 by XGBoost contributors + */ +#pragma once +#include +#include + +#include +#include + +#include "../../../src/common/linalg_op.h" +#include "../helpers.h" + +namespace xgboost { +namespace metric { + +inline void CheckDeterministicMetricElementWise(StringView name, int32_t device) { + auto ctx = CreateEmptyGenericParam(device); + std::unique_ptr metric{Metric::Create(name.c_str(), &ctx)}; + + HostDeviceVector predts; + size_t n_samples = 2048; + + auto p_fmat = EmptyDMatrix(); + MetaInfo& info = p_fmat->Info(); + info.labels.Reshape(n_samples, 1); + info.num_row_ = n_samples; + auto &h_labels = info.labels.Data()->HostVector(); + auto &h_predts = predts.HostVector(); + + SimpleLCG lcg; + SimpleRealUniformDistribution dist{0.0f, 1.0f}; + + h_labels.resize(n_samples); + h_predts.resize(n_samples); + + for (size_t i = 0; i < n_samples; ++i) { + h_predts[i] = dist(&lcg); + h_labels[i] = dist(&lcg); + } + + auto result = metric->Evaluate(predts, p_fmat); + for (size_t i = 0; i < 8; ++i) { + ASSERT_EQ(metric->Evaluate(predts, p_fmat), result); + } +} + +inline void VerifyRMSE(DataSplitMode data_split_mode = DataSplitMode::kRow) { + auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); + xgboost::Metric * metric = xgboost::Metric::Create("rmse", &ctx); + metric->Configure({}); + ASSERT_STREQ(metric->Name(), "rmse"); + EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, {}, {}, data_split_mode), + 0.6403f, 0.001f); + auto expected = 2.8284f; + if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) { + expected = sqrt(8.0f * collective::GetWorldSize()); + } + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, + { -1, 1, 9, -9}, {}, data_split_mode), + expected, 0.001f); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, + { 1, 2, 9, 8}, {}, data_split_mode), + 0.6708f, 0.001f); + delete metric; + + CheckDeterministicMetricElementWise(StringView{"rmse"}, GPUIDX); +} + +inline void VerifyRMSLE(DataSplitMode data_split_mode = DataSplitMode::kRow) { + auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); + xgboost::Metric * metric = xgboost::Metric::Create("rmsle", &ctx); + metric->Configure({}); + ASSERT_STREQ(metric->Name(), "rmsle"); + EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.2f, 0.4f, 0.8f, 1.6f}, + {1.0f, 1.0f, 1.0f, 1.0f, 1.0f}, {}, {}, data_split_mode), + 0.4063f, 1e-4); + auto expected = 0.6212f; + if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) { + expected = sqrt(0.3859f * collective::GetWorldSize()); + } + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.2f, 0.4f, 0.8f, 1.6f}, + {1.0f, 1.0f, 1.0f, 1.0f, 1.0f}, + { 0, -1, 1, -9, 9}, {}, data_split_mode), + expected, 1e-4); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.2f, 0.4f, 0.8f, 1.6f}, + {1.0f, 1.0f, 1.0f, 1.0f, 1.0f}, + { 0, 1, 2, 9, 8}, {}, data_split_mode), + 0.2415f, 1e-4); + delete metric; + + CheckDeterministicMetricElementWise(StringView{"rmsle"}, GPUIDX); +} + +inline void VerifyMAE(DataSplitMode data_split_mode = DataSplitMode::kRow) { + auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); + xgboost::Metric * metric = xgboost::Metric::Create("mae", &ctx); + metric->Configure({}); + ASSERT_STREQ(metric->Name(), "mae"); + EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, {}, {}, data_split_mode), + 0.5f, 0.001f); + auto expected = 8.0f; + if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) { + expected *= collective::GetWorldSize(); + } + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, + { -1, 1, 9, -9}, {}, data_split_mode), + expected, 0.001f); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, + { 1, 2, 9, 8}, {}, data_split_mode), + 0.54f, 0.001f); + delete metric; + + CheckDeterministicMetricElementWise(StringView{"mae"}, GPUIDX); +} + +inline void VerifyMAPE(DataSplitMode data_split_mode = DataSplitMode::kRow) { + auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); + xgboost::Metric * metric = xgboost::Metric::Create("mape", &ctx); + metric->Configure({}); + ASSERT_STREQ(metric->Name(), "mape"); + EXPECT_NEAR(GetMetricEval(metric, {150, 300}, {100, 200}, {}, {}, data_split_mode), 0.5f, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, + {50, 400, 500, 4000}, + {100, 200, 500, 1000}, {}, {}, data_split_mode), + 1.125f, 0.001f); + auto expected = -26.5f; + if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) { + expected *= collective::GetWorldSize(); + } + EXPECT_NEAR(GetMetricEval(metric, + {50, 400, 500, 4000}, + {100, 200, 500, 1000}, + { -1, 1, 9, -9}, {}, data_split_mode), + expected, 0.001f); + EXPECT_NEAR(GetMetricEval(metric, + {50, 400, 500, 4000}, + {100, 200, 500, 1000}, + { 1, 2, 9, 8}, {}, data_split_mode), + 1.3250f, 0.001f); + delete metric; + + CheckDeterministicMetricElementWise(StringView{"mape"}, GPUIDX); +} + +inline void VerifyMPHE(DataSplitMode data_split_mode = DataSplitMode::kRow) { + auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); + std::unique_ptr metric{xgboost::Metric::Create("mphe", &ctx)}; + metric->Configure({}); + ASSERT_STREQ(metric->Name(), "mphe"); + EXPECT_NEAR(GetMetricEval(metric.get(), {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10); + EXPECT_NEAR(GetMetricEval(metric.get(), + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, {}, {}, data_split_mode), + 0.1751f, 1e-4); + auto expected = 3.40375f; + if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) { + expected *= collective::GetWorldSize(); + } + EXPECT_NEAR(GetMetricEval(metric.get(), + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, + { -1, 1, 9, -9}, {}, data_split_mode), + expected, 1e-4); + EXPECT_NEAR(GetMetricEval(metric.get(), + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, + { 1, 2, 9, 8}, {}, data_split_mode), + 0.1922f, 1e-4); + + CheckDeterministicMetricElementWise(StringView{"mphe"}, GPUIDX); + + metric->Configure({{"huber_slope", "0.1"}}); + EXPECT_NEAR(GetMetricEval(metric.get(), + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, + { 1, 2, 9, 8}, {}, data_split_mode), + 0.0461686f, 1e-4); +} + +inline void VerifyLogLoss(DataSplitMode data_split_mode = DataSplitMode::kRow) { + auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); + xgboost::Metric * metric = xgboost::Metric::Create("logloss", &ctx); + metric->Configure({}); + ASSERT_STREQ(metric->Name(), "logloss"); + EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, + {0.5f, 1e-17f, 1.0f+1e-17f, 0.9f}, + { 0, 0, 1, 1}, {}, {}, data_split_mode), + 0.1996f, 0.001f); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, {}, {}, data_split_mode), + 1.2039f, 0.001f); + auto expected = 21.9722f; + if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) { + expected *= collective::GetWorldSize(); + } + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, + { -1, 1, 9, -9}, {}, data_split_mode), + expected, 0.001f); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, + { 1, 2, 9, 8}, {}, data_split_mode), + 1.3138f, 0.001f); + delete metric; + + CheckDeterministicMetricElementWise(StringView{"logloss"}, GPUIDX); +} + +inline void VerifyError(DataSplitMode data_split_mode = DataSplitMode::kRow) { + auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); + xgboost::Metric * metric = xgboost::Metric::Create("error", &ctx); + metric->Configure({}); + ASSERT_STREQ(metric->Name(), "error"); + EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, {}, {}, data_split_mode), + 0.5f, 0.001f); + auto expected = 10.0f; + if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) { + expected *= collective::GetWorldSize(); + } + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, + { -1, 1, 9, -9}, {}, data_split_mode), + expected, 0.001f); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, + { 1, 2, 9, 8}, {}, data_split_mode), + 0.55f, 0.001f); + + EXPECT_ANY_THROW(xgboost::Metric::Create("error@abc", &ctx)); + delete metric; + + metric = xgboost::Metric::Create("error@0.5f", &ctx); + metric->Configure({}); + EXPECT_STREQ(metric->Name(), "error"); + + delete metric; + + metric = xgboost::Metric::Create("error@0.1", &ctx); + metric->Configure({}); + ASSERT_STREQ(metric->Name(), "error@0.1"); + EXPECT_STREQ(metric->Name(), "error@0.1"); + EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, + {-0.1f, -0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, {}, {}, data_split_mode), + 0.25f, 0.001f); + expected = 9.0f; + if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) { + expected *= collective::GetWorldSize(); + } + EXPECT_NEAR(GetMetricEval(metric, + {-0.1f, -0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, + { -1, 1, 9, -9}, {}, data_split_mode), + expected, 0.001f); + EXPECT_NEAR(GetMetricEval(metric, + {-0.1f, -0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, + { 1, 2, 9, 8}, {}, data_split_mode), + 0.45f, 0.001f); + delete metric; + + CheckDeterministicMetricElementWise(StringView{"error@0.5"}, GPUIDX); +} + +inline void VerifyPoissonNegLogLik(DataSplitMode data_split_mode = DataSplitMode::kRow) { + auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); + xgboost::Metric * metric = xgboost::Metric::Create("poisson-nloglik", &ctx); + metric->Configure({}); + ASSERT_STREQ(metric->Name(), "poisson-nloglik"); + EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0.5f, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, + {0.5f, 1e-17f, 1.0f+1e-17f, 0.9f}, + { 0, 0, 1, 1}, {}, {}, data_split_mode), + 0.6263f, 0.001f); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, {}, {}, data_split_mode), + 1.1019f, 0.001f); + auto expected = 13.3750f; + if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) { + expected *= collective::GetWorldSize(); + } + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, + { -1, 1, 9, -9}, {}, data_split_mode), + expected, 0.001f); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, + { 1, 2, 9, 8}, {}, data_split_mode), + 1.5783f, 0.001f); + delete metric; + + CheckDeterministicMetricElementWise(StringView{"poisson-nloglik"}, GPUIDX); +} + +inline void VerifyMultiRMSE(DataSplitMode data_split_mode = DataSplitMode::kRow) { + size_t n_samples = 32, n_targets = 8; + linalg::Tensor y{{n_samples, n_targets}, GPUIDX}; + auto &h_y = y.Data()->HostVector(); + std::iota(h_y.begin(), h_y.end(), 0); + + HostDeviceVector predt(n_samples * n_targets, 0); + + auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); + std::unique_ptr metric{Metric::Create("rmse", &ctx)}; + metric->Configure({}); + + auto loss = GetMultiMetricEval(metric.get(), predt, y, {}, {}, data_split_mode); + std::vector weights(n_samples, 1); + auto loss_w = GetMultiMetricEval(metric.get(), predt, y, weights, {}, data_split_mode); + + std::transform(h_y.cbegin(), h_y.cend(), h_y.begin(), [](auto &v) { return v * v; }); + auto ret = std::sqrt(std::accumulate(h_y.cbegin(), h_y.cend(), 1.0, std::plus<>{}) / h_y.size()); + ASSERT_FLOAT_EQ(ret, loss); + ASSERT_FLOAT_EQ(ret, loss_w); +} + +inline void VerifyQuantile(DataSplitMode data_split_mode = DataSplitMode::kRow) { + auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); + std::unique_ptr metric{Metric::Create("quantile", &ctx)}; + + HostDeviceVector predts{0.1f, 0.9f, 0.1f, 0.9f}; + std::vector labels{0.5f, 0.5f, 0.9f, 0.1f}; + std::vector weights{0.2f, 0.4f, 0.6f, 0.8f}; + + metric->Configure(Args{{"quantile_alpha", "[0.0]"}}); + EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights, {}, data_split_mode), 0.400f, + 0.001f); + metric->Configure(Args{{"quantile_alpha", "[0.2]"}}); + EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights, {}, data_split_mode), 0.376f, + 0.001f); + metric->Configure(Args{{"quantile_alpha", "[0.4]"}}); + EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights, {}, data_split_mode), 0.352f, + 0.001f); + metric->Configure(Args{{"quantile_alpha", "[0.8]"}}); + EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights, {}, data_split_mode), 0.304f, + 0.001f); + metric->Configure(Args{{"quantile_alpha", "[1.0]"}}); + EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights, {}, data_split_mode), 0.28f, + 0.001f); + + metric->Configure(Args{{"quantile_alpha", "[0.0]"}}); + EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, {}, {}, data_split_mode), 0.3f, 0.001f); + metric->Configure(Args{{"quantile_alpha", "[0.2]"}}); + EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, {}, {}, data_split_mode), 0.3f, 0.001f); + metric->Configure(Args{{"quantile_alpha", "[0.4]"}}); + EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, {}, {}, data_split_mode), 0.3f, 0.001f); + metric->Configure(Args{{"quantile_alpha", "[0.8]"}}); + EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, {}, {}, data_split_mode), 0.3f, 0.001f); + metric->Configure(Args{{"quantile_alpha", "[1.0]"}}); + EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, {}, {}, data_split_mode), 0.3f, 0.001f); +} +} // namespace metric +} // namespace xgboost diff --git a/tests/cpp/metric/test_multiclass_metric.cc b/tests/cpp/metric/test_multiclass_metric.cc index 2465b11c8..bfb638924 100644 --- a/tests/cpp/metric/test_multiclass_metric.cc +++ b/tests/cpp/metric/test_multiclass_metric.cc @@ -1,87 +1,29 @@ // Copyright by Contributors -#include +#include "test_multiclass_metric.h" + #include -#include "../helpers.h" - namespace xgboost { -inline void CheckDeterministicMetricMultiClass(StringView name, int32_t device) { - auto ctx = CreateEmptyGenericParam(device); - std::unique_ptr metric{Metric::Create(name.c_str(), &ctx)}; +namespace metric { - HostDeviceVector predts; - auto p_fmat = EmptyDMatrix(); - MetaInfo& info = p_fmat->Info(); - auto &h_predts = predts.HostVector(); +TEST(Metric, DeclareUnifiedTest(MultiClassError)) { VerifyMultiClassError(); } - SimpleLCG lcg; +TEST(Metric, DeclareUnifiedTest(MultiClassLogLoss)) { VerifyMultiClassLogLoss(); } - size_t n_samples = 2048, n_classes = 4; - - info.labels.Reshape(n_samples); - auto &h_labels = info.labels.Data()->HostVector(); - h_predts.resize(n_samples * n_classes); - - { - SimpleRealUniformDistribution dist{0.0f, static_cast(n_classes)}; - for (size_t i = 0; i < n_samples; ++i) { - h_labels[i] = dist(&lcg); - } - } - - { - SimpleRealUniformDistribution dist{0.0f, 1.0f}; - for (size_t i = 0; i < n_samples * n_classes; ++i) { - h_predts[i] = dist(&lcg); - } - } - - auto result = metric->Evaluate(predts, p_fmat); - for (size_t i = 0; i < 8; ++i) { - ASSERT_EQ(metric->Evaluate(predts, p_fmat), result); - } +TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiClassErrorRowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyMultiClassError, DataSplitMode::kRow); } + +TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiClassErrorColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyMultiClassError, DataSplitMode::kCol); +} + +TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiClassLogLossRowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyMultiClassLogLoss, DataSplitMode::kRow); +} + +TEST_F(DeclareUnifiedDistributedTest(MetricTest), MultiClassLogLossColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyMultiClassLogLoss, DataSplitMode::kCol); +} +} // namespace metric } // namespace xgboost - -inline void TestMultiClassError(int device) { - auto ctx = xgboost::CreateEmptyGenericParam(device); - ctx.gpu_id = device; - xgboost::Metric * metric = xgboost::Metric::Create("merror", &ctx); - metric->Configure({}); - ASSERT_STREQ(metric->Name(), "merror"); - EXPECT_ANY_THROW(GetMetricEval(metric, {0}, {0, 0})); - EXPECT_NEAR(GetMetricEval( - metric, {1, 0, 0, 0, 1, 0, 0, 0, 1}, {0, 1, 2}), 0, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f}, - {0, 1, 2}), - 0.666f, 0.001f); - delete metric; -} - -TEST(Metric, DeclareUnifiedTest(MultiClassError)) { - TestMultiClassError(GPUIDX); - xgboost::CheckDeterministicMetricMultiClass(xgboost::StringView{"merror"}, GPUIDX); -} - -inline void TestMultiClassLogLoss(int device) { - auto ctx = xgboost::CreateEmptyGenericParam(device); - ctx.gpu_id = device; - xgboost::Metric * metric = xgboost::Metric::Create("mlogloss", &ctx); - metric->Configure({}); - ASSERT_STREQ(metric->Name(), "mlogloss"); - EXPECT_ANY_THROW(GetMetricEval(metric, {0}, {0, 0})); - EXPECT_NEAR(GetMetricEval( - metric, {1, 0, 0, 0, 1, 0, 0, 0, 1}, {0, 1, 2}), 0, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f}, - {0, 1, 2}), - 2.302f, 0.001f); - - delete metric; -} - -TEST(Metric, DeclareUnifiedTest(MultiClassLogLoss)) { - TestMultiClassLogLoss(GPUIDX); - xgboost::CheckDeterministicMetricMultiClass(xgboost::StringView{"mlogloss"}, GPUIDX); -} diff --git a/tests/cpp/metric/test_multiclass_metric.h b/tests/cpp/metric/test_multiclass_metric.h new file mode 100644 index 000000000..cd2b142fc --- /dev/null +++ b/tests/cpp/metric/test_multiclass_metric.h @@ -0,0 +1,91 @@ +// Copyright by Contributors +#include +#include + +#include "../helpers.h" + +namespace xgboost { +namespace metric { + +inline void CheckDeterministicMetricMultiClass(StringView name, int32_t device) { + auto ctx = CreateEmptyGenericParam(device); + std::unique_ptr metric{Metric::Create(name.c_str(), &ctx)}; + + HostDeviceVector predts; + auto p_fmat = EmptyDMatrix(); + MetaInfo& info = p_fmat->Info(); + auto &h_predts = predts.HostVector(); + + SimpleLCG lcg; + + size_t n_samples = 2048, n_classes = 4; + + info.labels.Reshape(n_samples); + auto &h_labels = info.labels.Data()->HostVector(); + h_predts.resize(n_samples * n_classes); + + { + SimpleRealUniformDistribution dist{0.0f, static_cast(n_classes)}; + for (size_t i = 0; i < n_samples; ++i) { + h_labels[i] = dist(&lcg); + } + } + + { + SimpleRealUniformDistribution dist{0.0f, 1.0f}; + for (size_t i = 0; i < n_samples * n_classes; ++i) { + h_predts[i] = dist(&lcg); + } + } + + auto result = metric->Evaluate(predts, p_fmat); + for (size_t i = 0; i < 8; ++i) { + ASSERT_EQ(metric->Evaluate(predts, p_fmat), result); + } +} + +inline void TestMultiClassError(int device, DataSplitMode data_split_mode) { + auto ctx = xgboost::CreateEmptyGenericParam(device); + ctx.gpu_id = device; + xgboost::Metric * metric = xgboost::Metric::Create("merror", &ctx); + metric->Configure({}); + ASSERT_STREQ(metric->Name(), "merror"); + EXPECT_ANY_THROW(GetMetricEval(metric, {0}, {0, 0}, {}, {}, data_split_mode)); + EXPECT_NEAR(GetMetricEval( + metric, {1, 0, 0, 0, 1, 0, 0, 0, 1}, {0, 1, 2}, {}, {}, data_split_mode), 0, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f}, + {0, 1, 2}, {}, {}, data_split_mode), + 0.666f, 0.001f); + delete metric; +} + +inline void VerifyMultiClassError(DataSplitMode data_split_mode = DataSplitMode::kRow) { + TestMultiClassError(GPUIDX, data_split_mode); + CheckDeterministicMetricMultiClass(StringView{"merror"}, GPUIDX); +} + +inline void TestMultiClassLogLoss(int device, DataSplitMode data_split_mode) { + auto ctx = xgboost::CreateEmptyGenericParam(device); + ctx.gpu_id = device; + xgboost::Metric * metric = xgboost::Metric::Create("mlogloss", &ctx); + metric->Configure({}); + ASSERT_STREQ(metric->Name(), "mlogloss"); + EXPECT_ANY_THROW(GetMetricEval(metric, {0}, {0, 0}, {}, {}, data_split_mode)); + EXPECT_NEAR(GetMetricEval( + metric, {1, 0, 0, 0, 1, 0, 0, 0, 1}, {0, 1, 2}, {}, {}, data_split_mode), 0, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f}, + {0, 1, 2}, {}, {}, data_split_mode), + 2.302f, 0.001f); + + delete metric; +} + +inline void VerifyMultiClassLogLoss(DataSplitMode data_split_mode = DataSplitMode::kRow) { + TestMultiClassLogLoss(GPUIDX, data_split_mode); + CheckDeterministicMetricMultiClass(StringView{"mlogloss"}, GPUIDX); +} + +} // namespace metric +} // namespace xgboost diff --git a/tests/cpp/metric/test_rank_metric.cc b/tests/cpp/metric/test_rank_metric.cc index 3e1028c48..430671305 100644 --- a/tests/cpp/metric/test_rank_metric.cc +++ b/tests/cpp/metric/test_rank_metric.cc @@ -11,16 +11,20 @@ #include // for unique_ptr #include // for vector +#include "test_rank_metric.h" #include "../helpers.h" // for GetMetricEval, CreateEmptyGe... #include "xgboost/base.h" // for bst_float, kRtEps #include "xgboost/host_device_vector.h" // for HostDeviceVector #include "xgboost/json.h" // for Json, String, Object +namespace xgboost { +namespace metric { + #if !defined(__CUDACC__) TEST(Metric, AMS) { - auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); - EXPECT_ANY_THROW(xgboost::Metric::Create("ams", &ctx)); - xgboost::Metric* metric = xgboost::Metric::Create("ams@0.5f", &ctx); + auto ctx = CreateEmptyGenericParam(GPUIDX); + EXPECT_ANY_THROW(Metric::Create("ams", &ctx)); + Metric* metric = Metric::Create("ams@0.5f", &ctx); ASSERT_STREQ(metric->Name(), "ams@0.5"); EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0.311f, 0.001f); EXPECT_NEAR(GetMetricEval(metric, @@ -29,7 +33,7 @@ TEST(Metric, AMS) { 0.29710f, 0.001f); delete metric; - metric = xgboost::Metric::Create("ams@0", &ctx); + metric = Metric::Create("ams@0", &ctx); ASSERT_STREQ(metric->Name(), "ams@0"); EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0.311f, 0.001f); @@ -37,172 +41,44 @@ TEST(Metric, AMS) { } #endif -TEST(Metric, DeclareUnifiedTest(Precision)) { - // When the limit for precision is not given, it takes the limit at - // std::numeric_limits::max(); hence all values are very small - // NOTE(AbdealiJK): Maybe this should be fixed to be num_row by default. - auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); - xgboost::Metric * metric = xgboost::Metric::Create("pre", &ctx); - ASSERT_STREQ(metric->Name(), "pre"); - EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-7); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}), - 0, 1e-7); +TEST(Metric, DeclareUnifiedTest(Precision)) { VerifyPrecision(); } - delete metric; - metric = xgboost::Metric::Create("pre@2", &ctx); - ASSERT_STREQ(metric->Name(), "pre@2"); - EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0.5f, 1e-7); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}), - 0.5f, 0.001f); +TEST(Metric, DeclareUnifiedTest(NDCG)) { VerifyNDCG(); } - EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {})); +TEST(Metric, DeclareUnifiedTest(MAP)) { VerifyMAP(); } - delete metric; +TEST(Metric, DeclareUnifiedTest(NDCGExpGain)) { VerifyNDCGExpGain(); } + +TEST_F(DeclareUnifiedDistributedTest(MetricTest), PrecisionRowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyPrecision, DataSplitMode::kRow); } -namespace xgboost { -namespace metric { -TEST(Metric, DeclareUnifiedTest(NDCG)) { - auto ctx = CreateEmptyGenericParam(GPUIDX); - Metric * metric = xgboost::Metric::Create("ndcg", &ctx); - ASSERT_STREQ(metric->Name(), "ndcg"); - EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {})); - ASSERT_NEAR(GetMetricEval(metric, - xgboost::HostDeviceVector{}, - {}), 1, 1e-10); - ASSERT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}), - 0.6509f, 0.001f); - - delete metric; - metric = xgboost::Metric::Create("ndcg@2", &ctx); - ASSERT_STREQ(metric->Name(), "ndcg@2"); - EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}), - 0.3868f, 0.001f); - - delete metric; - metric = xgboost::Metric::Create("ndcg@-", &ctx); - ASSERT_STREQ(metric->Name(), "ndcg-"); - EXPECT_NEAR(GetMetricEval(metric, - xgboost::HostDeviceVector{}, - {}), 0, 1e-10); - ASSERT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1.f, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}), - 0.6509f, 0.001f); - delete metric; - metric = xgboost::Metric::Create("ndcg-", &ctx); - ASSERT_STREQ(metric->Name(), "ndcg-"); - EXPECT_NEAR(GetMetricEval(metric, - xgboost::HostDeviceVector{}, - {}), 0, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1.f, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}), - 0.6509f, 0.001f); - - delete metric; - metric = xgboost::Metric::Create("ndcg@2-", &ctx); - ASSERT_STREQ(metric->Name(), "ndcg@2-"); - EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1.f, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}), - 1.f - 0.3868f, 1.f - 0.001f); - - delete metric; +TEST_F(DeclareUnifiedDistributedTest(MetricTest), PrecisionColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyPrecision, DataSplitMode::kCol); } -TEST(Metric, DeclareUnifiedTest(MAP)) { - auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); - Metric * metric = xgboost::Metric::Create("map", &ctx); - ASSERT_STREQ(metric->Name(), "map"); - EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, kRtEps); - - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}), - 0.5f, 0.001f); - EXPECT_NEAR(GetMetricEval(metric, - xgboost::HostDeviceVector{}, - std::vector{}), 1, 1e-10); - - // Rank metric with group info - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.2f, 0.8f, 0.4f, 1.7f}, - {1, 1, 1, 0, 1, 0}, // Labels - {}, // Weights - {0, 2, 5, 6}), // Group info - 0.8611f, 0.001f); - - delete metric; - metric = xgboost::Metric::Create("map@-", &ctx); - ASSERT_STREQ(metric->Name(), "map-"); - EXPECT_NEAR(GetMetricEval(metric, - xgboost::HostDeviceVector{}, - {}), 0, 1e-10); - - delete metric; - metric = xgboost::Metric::Create("map-", &ctx); - ASSERT_STREQ(metric->Name(), "map-"); - EXPECT_NEAR(GetMetricEval(metric, - xgboost::HostDeviceVector{}, - {}), 0, 1e-10); - - delete metric; - metric = xgboost::Metric::Create("map@2", &ctx); - ASSERT_STREQ(metric->Name(), "map@2"); - EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10); - EXPECT_NEAR(GetMetricEval(metric, - {0.1f, 0.9f, 0.1f, 0.9f}, - { 0, 0, 1, 1}), - 0.25f, 0.001f); - delete metric; +TEST_F(DeclareUnifiedDistributedTest(MetricTest), NDCGRowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyNDCG, DataSplitMode::kRow); } -TEST(Metric, DeclareUnifiedTest(NDCGExpGain)) { - Context ctx = xgboost::CreateEmptyGenericParam(GPUIDX); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), NDCGColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyNDCG, DataSplitMode::kCol); +} - auto p_fmat = xgboost::RandomDataGenerator{0, 0, 0}.GenerateDMatrix(); - MetaInfo& info = p_fmat->Info(); - info.labels = linalg::Matrix{{10.0f, 0.0f, 0.0f, 1.0f, 5.0f}, {5}, ctx.gpu_id}; - info.num_row_ = info.labels.Shape(0); - info.group_ptr_.resize(2); - info.group_ptr_[0] = 0; - info.group_ptr_[1] = info.num_row_; - HostDeviceVector predt{{0.1f, 0.2f, 0.3f, 4.0f, 70.0f}}; +TEST_F(DeclareUnifiedDistributedTest(MetricTest), MAPRowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyMAP, DataSplitMode::kRow); +} - std::unique_ptr metric{Metric::Create("ndcg", &ctx)}; - Json config{Object{}}; - config["name"] = String{"ndcg"}; - config["lambdarank_param"] = Object{}; - config["lambdarank_param"]["ndcg_exp_gain"] = String{"true"}; - config["lambdarank_param"]["lambdarank_num_pair_per_sample"] = String{"32"}; - metric->LoadConfig(config); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), MAPColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyMAP, DataSplitMode::kCol); +} - auto ndcg = metric->Evaluate(predt, p_fmat); - ASSERT_NEAR(ndcg, 0.409738f, kRtEps); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), NDCGExpGainRowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyNDCGExpGain, DataSplitMode::kRow); +} - config["lambdarank_param"]["ndcg_exp_gain"] = String{"false"}; - metric->LoadConfig(config); - - ndcg = metric->Evaluate(predt, p_fmat); - ASSERT_NEAR(ndcg, 0.695694f, kRtEps); - - predt.HostVector() = info.labels.Data()->HostVector(); - ndcg = metric->Evaluate(predt, p_fmat); - ASSERT_NEAR(ndcg, 1.0, kRtEps); +TEST_F(DeclareUnifiedDistributedTest(MetricTest), NDCGExpGainColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyNDCGExpGain, DataSplitMode::kCol); } } // namespace metric } // namespace xgboost diff --git a/tests/cpp/metric/test_rank_metric.h b/tests/cpp/metric/test_rank_metric.h new file mode 100644 index 000000000..318de961b --- /dev/null +++ b/tests/cpp/metric/test_rank_metric.h @@ -0,0 +1,191 @@ +/** + * Copyright 2016-2023 by XGBoost Contributors + */ +#pragma once +#include // for Test, EXPECT_NEAR, ASSERT_STREQ +#include // for Context +#include // for MetaInfo, DMatrix +#include // for Matrix +#include // for Metric + +#include // for max +#include // for unique_ptr +#include // for vector + +#include "../helpers.h" // for GetMetricEval, CreateEmptyGe... +#include "xgboost/base.h" // for bst_float, kRtEps +#include "xgboost/host_device_vector.h" // for HostDeviceVector +#include "xgboost/json.h" // for Json, String, Object + +namespace xgboost { +namespace metric { + +inline void VerifyPrecision(DataSplitMode data_split_mode = DataSplitMode::kRow) { + // When the limit for precision is not given, it takes the limit at + // std::numeric_limits::max(); hence all values are very small + // NOTE(AbdealiJK): Maybe this should be fixed to be num_row by default. + auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); + xgboost::Metric * metric = xgboost::Metric::Create("pre", &ctx); + ASSERT_STREQ(metric->Name(), "pre"); + EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-7); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, {}, {}, data_split_mode), + 0, 1e-7); + + delete metric; + metric = xgboost::Metric::Create("pre@2", &ctx); + ASSERT_STREQ(metric->Name(), "pre@2"); + EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0.5f, 1e-7); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, {}, {}, data_split_mode), + 0.5f, 0.001f); + + EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {}, {}, {}, data_split_mode)); + + delete metric; +} + +inline void VerifyNDCG(DataSplitMode data_split_mode = DataSplitMode::kRow) { + auto ctx = CreateEmptyGenericParam(GPUIDX); + Metric * metric = xgboost::Metric::Create("ndcg", &ctx); + ASSERT_STREQ(metric->Name(), "ndcg"); + EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {}, {}, {}, data_split_mode)); + ASSERT_NEAR(GetMetricEval(metric, + xgboost::HostDeviceVector{}, + {}, {}, {}, data_split_mode), 1, 1e-10); + ASSERT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 1, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, {}, {}, data_split_mode), + 0.6509f, 0.001f); + + delete metric; + metric = xgboost::Metric::Create("ndcg@2", &ctx); + ASSERT_STREQ(metric->Name(), "ndcg@2"); + EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 1, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, {}, {}, data_split_mode), + 0.3868f, 0.001f); + + delete metric; + metric = xgboost::Metric::Create("ndcg@-", &ctx); + ASSERT_STREQ(metric->Name(), "ndcg-"); + EXPECT_NEAR(GetMetricEval(metric, + xgboost::HostDeviceVector{}, + {}, {}, {}, data_split_mode), 0, 1e-10); + ASSERT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 1.f, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, {}, {}, data_split_mode), + 0.6509f, 0.001f); + delete metric; + metric = xgboost::Metric::Create("ndcg-", &ctx); + ASSERT_STREQ(metric->Name(), "ndcg-"); + EXPECT_NEAR(GetMetricEval(metric, + xgboost::HostDeviceVector{}, + {}, {}, {}, data_split_mode), 0, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 1.f, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, {}, {}, data_split_mode), + 0.6509f, 0.001f); + + delete metric; + metric = xgboost::Metric::Create("ndcg@2-", &ctx); + ASSERT_STREQ(metric->Name(), "ndcg@2-"); + EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 1.f, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, {}, {}, data_split_mode), + 1.f - 0.3868f, 1.f - 0.001f); + + delete metric; +} + +inline void VerifyMAP(DataSplitMode data_split_mode = DataSplitMode::kRow) { + auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); + Metric * metric = xgboost::Metric::Create("map", &ctx); + ASSERT_STREQ(metric->Name(), "map"); + EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 1, kRtEps); + + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, {}, {}, data_split_mode), + 0.5f, 0.001f); + EXPECT_NEAR(GetMetricEval(metric, + xgboost::HostDeviceVector{}, + std::vector{}, {}, {}, data_split_mode), 1, 1e-10); + + // Rank metric with group info + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.2f, 0.8f, 0.4f, 1.7f}, + {1, 1, 1, 0, 1, 0}, // Labels + {}, // Weights + {0, 2, 5, 6}, // Group info + data_split_mode), + 0.8611f, 0.001f); + + delete metric; + metric = xgboost::Metric::Create("map@-", &ctx); + ASSERT_STREQ(metric->Name(), "map-"); + EXPECT_NEAR(GetMetricEval(metric, + xgboost::HostDeviceVector{}, + {}, {}, {}, data_split_mode), 0, 1e-10); + + delete metric; + metric = xgboost::Metric::Create("map-", &ctx); + ASSERT_STREQ(metric->Name(), "map-"); + EXPECT_NEAR(GetMetricEval(metric, + xgboost::HostDeviceVector{}, + {}, {}, {}, data_split_mode), 0, 1e-10); + + delete metric; + metric = xgboost::Metric::Create("map@2", &ctx); + ASSERT_STREQ(metric->Name(), "map@2"); + EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 1, 1e-10); + EXPECT_NEAR(GetMetricEval(metric, + {0.1f, 0.9f, 0.1f, 0.9f}, + { 0, 0, 1, 1}, {}, {}, data_split_mode), + 0.25f, 0.001f); + delete metric; +} + +inline void VerifyNDCGExpGain(DataSplitMode data_split_mode = DataSplitMode::kRow) { + Context ctx = xgboost::CreateEmptyGenericParam(GPUIDX); + + auto p_fmat = xgboost::RandomDataGenerator{0, 0, 0}.GenerateDMatrix(); + MetaInfo& info = p_fmat->Info(); + info.labels = linalg::Matrix{{10.0f, 0.0f, 0.0f, 1.0f, 5.0f}, {5}, ctx.gpu_id}; + info.num_row_ = info.labels.Shape(0); + info.group_ptr_.resize(2); + info.group_ptr_[0] = 0; + info.group_ptr_[1] = info.num_row_; + info.data_split_mode = data_split_mode; + HostDeviceVector predt{{0.1f, 0.2f, 0.3f, 4.0f, 70.0f}}; + + std::unique_ptr metric{Metric::Create("ndcg", &ctx)}; + Json config{Object{}}; + config["name"] = String{"ndcg"}; + config["lambdarank_param"] = Object{}; + config["lambdarank_param"]["ndcg_exp_gain"] = String{"true"}; + config["lambdarank_param"]["lambdarank_num_pair_per_sample"] = String{"32"}; + metric->LoadConfig(config); + + auto ndcg = metric->Evaluate(predt, p_fmat); + ASSERT_NEAR(ndcg, 0.409738f, kRtEps); + + config["lambdarank_param"]["ndcg_exp_gain"] = String{"false"}; + metric->LoadConfig(config); + + ndcg = metric->Evaluate(predt, p_fmat); + ASSERT_NEAR(ndcg, 0.695694f, kRtEps); + + predt.HostVector() = info.labels.Data()->HostVector(); + ndcg = metric->Evaluate(predt, p_fmat); + ASSERT_NEAR(ndcg, 1.0, kRtEps); +} +} // namespace metric +} // namespace xgboost diff --git a/tests/cpp/metric/test_survival_metric.cu b/tests/cpp/metric/test_survival_metric.cu index 80d6b72e6..d7ac54860 100644 --- a/tests/cpp/metric/test_survival_metric.cu +++ b/tests/cpp/metric/test_survival_metric.cu @@ -46,9 +46,8 @@ inline void CheckDeterministicMetricElementWise(StringView name, int32_t device) ASSERT_EQ(metric->Evaluate(predts, p_fmat), result); } } -} // anonymous namespace -TEST(Metric, DeclareUnifiedTest(AFTNegLogLik)) { +void VerifyAFTNegLogLik(DataSplitMode data_split_mode = DataSplitMode::kRow) { auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); /** @@ -59,10 +58,11 @@ TEST(Metric, DeclareUnifiedTest(AFTNegLogLik)) { MetaInfo& info = p_fmat->Info(); info.num_row_ = 4; info.labels_lower_bound_.HostVector() - = { 100.0f, 0.0f, 60.0f, 16.0f }; + = { 100.0f, 0.0f, 60.0f, 16.0f }; info.labels_upper_bound_.HostVector() - = { 100.0f, 20.0f, std::numeric_limits::infinity(), 200.0f }; + = { 100.0f, 20.0f, std::numeric_limits::infinity(), 200.0f }; info.weights_.HostVector() = std::vector(); + info.data_split_mode = data_split_mode; HostDeviceVector preds(4, std::log(64)); struct TestCase { @@ -70,15 +70,15 @@ TEST(Metric, DeclareUnifiedTest(AFTNegLogLik)) { bst_float reference_value; }; for (const auto& test_case : std::vector{ {"normal", 2.1508f}, {"logistic", 2.1804f}, - {"extreme", 2.0706f} }) { + {"extreme", 2.0706f} }) { std::unique_ptr metric(Metric::Create("aft-nloglik", &ctx)); metric->Configure({ {"aft_loss_distribution", test_case.dist_type}, - {"aft_loss_distribution_scale", "1.0"} }); + {"aft_loss_distribution_scale", "1.0"} }); EXPECT_NEAR(metric->Evaluate(preds, p_fmat), test_case.reference_value, 1e-4); } } -TEST(Metric, DeclareUnifiedTest(IntervalRegressionAccuracy)) { +void VerifyIntervalRegressionAccuracy(DataSplitMode data_split_mode = DataSplitMode::kRow) { auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); auto p_fmat = EmptyDMatrix(); @@ -87,6 +87,7 @@ TEST(Metric, DeclareUnifiedTest(IntervalRegressionAccuracy)) { info.labels_lower_bound_.HostVector() = { 20.0f, 0.0f, 60.0f, 16.0f }; info.labels_upper_bound_.HostVector() = { 80.0f, 20.0f, 80.0f, 200.0f }; info.weights_.HostVector() = std::vector(); + info.data_split_mode = data_split_mode; HostDeviceVector preds(4, std::log(60.0f)); std::unique_ptr metric(Metric::Create("interval-regression-accuracy", &ctx)); @@ -102,6 +103,27 @@ TEST(Metric, DeclareUnifiedTest(IntervalRegressionAccuracy)) { CheckDeterministicMetricElementWise(StringView{"interval-regression-accuracy"}, GPUIDX); } +} // anonymous namespace + +TEST(Metric, DeclareUnifiedTest(AFTNegLogLik)) { VerifyAFTNegLogLik(); } + +TEST_F(DeclareUnifiedDistributedTest(MetricTest), AFTNegLogLikRowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyAFTNegLogLik, DataSplitMode::kRow); +} + +TEST_F(DeclareUnifiedDistributedTest(MetricTest), AFTNegLogLikColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyAFTNegLogLik, DataSplitMode::kCol); +} + +TEST(Metric, DeclareUnifiedTest(IntervalRegressionAccuracy)) { VerifyIntervalRegressionAccuracy(); } + +TEST_F(DeclareUnifiedDistributedTest(MetricTest), IntervalRegressionAccuracyRowSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyIntervalRegressionAccuracy, DataSplitMode::kRow); +} + +TEST_F(DeclareUnifiedDistributedTest(MetricTest), IntervalRegressionAccuracyColumnSplit) { + RunWithInMemoryCommunicator(world_size_, &VerifyIntervalRegressionAccuracy, DataSplitMode::kCol); +} // Test configuration of AFT metric TEST(AFTNegLogLikMetric, DeclareUnifiedTest(Configuration)) {