From 81b2ee1153c6673a5851cc6c07facd0d18500ec1 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Mon, 13 Feb 2023 22:15:05 +0800 Subject: [PATCH] Pass DMatrix into metric for caching. (#8790) --- include/xgboost/cache.h | 2 ++ include/xgboost/metric.h | 13 ++++++----- src/learner.cc | 2 +- src/metric/auc.cc | 3 ++- src/metric/elementwise_metric.cu | 8 +++---- src/metric/metric.cc | 11 ++++----- src/metric/metric_common.h | 13 +++++++++-- src/metric/multiclass_metric.cu | 6 ++--- src/metric/rank_metric.cc | 8 +++---- src/metric/rank_metric.cu | 19 ++++++++-------- src/metric/survival_metric.cu | 22 +++++++++--------- tests/cpp/helpers.cc | 7 +++--- tests/cpp/helpers.h | 5 +++++ tests/cpp/metric/test_auc.cc | 7 +++--- tests/cpp/metric/test_elementwise_metric.cc | 7 +++--- tests/cpp/metric/test_multiclass_metric.cc | 7 +++--- tests/cpp/metric/test_survival_metric.cu | 25 ++++++++++++--------- 17 files changed, 95 insertions(+), 70 deletions(-) diff --git a/include/xgboost/cache.h b/include/xgboost/cache.h index 246a92648..142c33a57 100644 --- a/include/xgboost/cache.h +++ b/include/xgboost/cache.h @@ -32,6 +32,8 @@ class DMatrixCache { CacheT& Value() { return *value; } }; + static constexpr std::size_t DefaultSize() { return 32; } + protected: std::unordered_map container_; std::queue queue_; diff --git a/include/xgboost/metric.h b/include/xgboost/metric.h index 2045e3355..2be6d5591 100644 --- a/include/xgboost/metric.h +++ b/include/xgboost/metric.h @@ -54,12 +54,15 @@ class Metric : public Configurable { out["name"] = String(this->Name()); } - /*! - * \brief evaluate a specific metric - * \param preds prediction - * \param info information, including label etc. + /** + * \brief Evaluate a metric with DMatrix as input. + * + * \param preds Prediction + * \param p_fmat DMatrix that contains related information like labels. */ - virtual double Eval(const HostDeviceVector& preds, const MetaInfo& info) = 0; + virtual double Evaluate(HostDeviceVector const& preds, + std::shared_ptr p_fmat) = 0; + /*! \return name of metric */ virtual const char* Name() const = 0; /*! \brief virtual destructor */ diff --git a/src/learner.cc b/src/learner.cc index ee2b1528b..390889e9c 100644 --- a/src/learner.cc +++ b/src/learner.cc @@ -1339,7 +1339,7 @@ class LearnerImpl : public LearnerIO { obj_->EvalTransform(&out); for (auto& ev : metrics_) { - os << '\t' << data_names[i] << '-' << ev->Name() << ':' << ev->Eval(out, m->Info()); + os << '\t' << data_names[i] << '-' << ev->Name() << ':' << ev->Evaluate(out, m); } } diff --git a/src/metric/auc.cc b/src/metric/auc.cc index 8248d24ba..9bedd95ee 100644 --- a/src/metric/auc.cc +++ b/src/metric/auc.cc @@ -16,6 +16,7 @@ #include "../common/math.h" #include "../common/optional_weight.h" // OptionalWeights +#include "metric_common.h" // MetricNoCache #include "xgboost/host_device_vector.h" #include "xgboost/linalg.h" #include "xgboost/metric.h" @@ -253,7 +254,7 @@ std::pair RankingAUC(std::vector const &predts, } template -class EvalAUC : public Metric { +class EvalAUC : public MetricNoCache { double Eval(const HostDeviceVector &preds, const MetaInfo &info) override { double auc {0}; if (ctx_->gpu_id != Context::kCpuId) { diff --git a/src/metric/elementwise_metric.cu b/src/metric/elementwise_metric.cu index 3df3bb65d..804bd300f 100644 --- a/src/metric/elementwise_metric.cu +++ b/src/metric/elementwise_metric.cu @@ -11,7 +11,7 @@ #include #include "../collective/communicator-inl.h" -#include "../common/common.h" +#include "../common/common.h" // MetricNoCache #include "../common/math.h" #include "../common/optional_weight.h" // OptionalWeights #include "../common/pseudo_huber.h" @@ -23,8 +23,8 @@ #if defined(XGBOOST_USE_CUDA) #include // thrust::cuda::par #include // thrust::plus<> -#include #include +#include #include "../common/device_helpers.cuh" #endif // XGBOOST_USE_CUDA @@ -167,7 +167,7 @@ struct EvalRowLogLoss { } }; -class PseudoErrorLoss : public Metric { +class PseudoErrorLoss : public MetricNoCache { PesudoHuberParam param_; public: @@ -339,7 +339,7 @@ struct EvalTweedieNLogLik { * \tparam Derived the name of subclass */ template -struct EvalEWiseBase : public Metric { +struct EvalEWiseBase : public MetricNoCache { EvalEWiseBase() = default; explicit EvalEWiseBase(char const* policy_param) : policy_{policy_param} {} diff --git a/src/metric/metric.cc b/src/metric/metric.cc index 2638c56ed..ebb579827 100644 --- a/src/metric/metric.cc +++ b/src/metric/metric.cc @@ -53,20 +53,21 @@ Metric::Create(const std::string& name, Context const* ctx) { return metric; } -Metric * -GPUMetric::CreateGPUMetric(const std::string& name, Context const* ctx) { +GPUMetric* GPUMetric::CreateGPUMetric(const std::string& name, Context const* ctx) { auto metric = CreateMetricImpl(name); if (metric == nullptr) { LOG(WARNING) << "Cannot find a GPU metric builder for metric " << name << ". Resorting to the CPU builder"; - return metric; + return nullptr; } // Narrowing reference only for the compiler to allow assignment to a base class member. // As such, using this narrowed reference to refer to derived members will be an illegal op. // This is moot, as this type is stateless. - static_cast(metric)->ctx_ = ctx; - return metric; + auto casted = static_cast(metric); + CHECK(casted); + casted->ctx_ = ctx; + return casted; } } // namespace xgboost diff --git a/src/metric/metric_common.h b/src/metric/metric_common.h index 747b70a63..064608ebf 100644 --- a/src/metric/metric_common.h +++ b/src/metric/metric_common.h @@ -13,12 +13,21 @@ namespace xgboost { struct Context; +// Metric that doesn't need to cache anything based on input data. +class MetricNoCache : public Metric { + public: + virtual double Eval(HostDeviceVector const &predts, MetaInfo const &info) = 0; + + double Evaluate(HostDeviceVector const &predts, std::shared_ptr p_fmat) final { + return this->Eval(predts, p_fmat->Info()); + } +}; // This creates a GPU metric instance dynamically and adds it to the GPU metric registry, if not // present already. This is created when there is a device ordinal present and if xgboost // is compiled with CUDA support -struct GPUMetric : Metric { - static Metric *CreateGPUMetric(const std::string &name, Context const *tparam); +struct GPUMetric : public MetricNoCache { + static GPUMetric *CreateGPUMetric(const std::string &name, Context const *tparam); }; /*! diff --git a/src/metric/multiclass_metric.cu b/src/metric/multiclass_metric.cu index 95d571e8a..aed6e7f4b 100644 --- a/src/metric/multiclass_metric.cu +++ b/src/metric/multiclass_metric.cu @@ -9,16 +9,16 @@ #include #include -#include "metric_common.h" #include "../collective/communicator-inl.h" #include "../common/math.h" #include "../common/threading_utils.h" +#include "metric_common.h" // MetricNoCache #if defined(XGBOOST_USE_CUDA) #include // thrust::cuda::par #include // thrust::plus<> -#include #include +#include #include "../common/device_helpers.cuh" #endif // XGBOOST_USE_CUDA @@ -162,7 +162,7 @@ class MultiClassMetricsReduction { * \tparam Derived the name of subclass */ template -struct EvalMClassBase : public Metric { +struct EvalMClassBase : public MetricNoCache { double Eval(const HostDeviceVector &preds, const MetaInfo &info) override { if (info.labels.Size() == 0) { CHECK_EQ(preds.Size(), 0); diff --git a/src/metric/rank_metric.cc b/src/metric/rank_metric.cc index 683a0f0ed..7ca0243f2 100644 --- a/src/metric/rank_metric.cc +++ b/src/metric/rank_metric.cc @@ -92,7 +92,7 @@ namespace metric { DMLC_REGISTRY_FILE_TAG(rank_metric); /*! \brief AMS: also records best threshold */ -struct EvalAMS : public Metric { +struct EvalAMS : public MetricNoCache { public: explicit EvalAMS(const char* param) { CHECK(param != nullptr) // NOLINT @@ -155,10 +155,10 @@ struct EvalAMS : public Metric { }; /*! \brief Evaluate rank list */ -struct EvalRank : public Metric, public EvalRankConfig { +struct EvalRank : public MetricNoCache, public EvalRankConfig { private: // This is used to compute the ranking metrics on the GPU - for training jobs that run on the GPU. - std::unique_ptr rank_gpu_; + std::unique_ptr rank_gpu_; public: double Eval(const HostDeviceVector& preds, const MetaInfo& info) override { @@ -322,7 +322,7 @@ struct EvalMAP : public EvalRank { }; /*! \brief Cox: Partial likelihood of the Cox proportional hazards model */ -struct EvalCox : public Metric { +struct EvalCox : public MetricNoCache { public: EvalCox() = default; double Eval(const HostDeviceVector& preds, const MetaInfo& info) override { diff --git a/src/metric/rank_metric.cu b/src/metric/rank_metric.cu index 30836067d..5f98db7a9 100644 --- a/src/metric/rank_metric.cu +++ b/src/metric/rank_metric.cu @@ -1,21 +1,20 @@ /** * Copyright 2020-2023 by XGBoost Contributors - * \file rank_metric.cu - * \brief prediction rank based metrics. - * \author Kailong Chen, Tianqi Chen */ #include - +#include // make_counting_iterator +#include // reduce #include -#include -#include -#include +#include // std::size_t +#include // std::shared_ptr +#include "../common/cuda_context.cuh" // CUDAContext #include "metric_common.h" - -#include "../common/math.h" -#include "../common/device_helpers.cuh" +#include "xgboost/base.h" // XGBOOST_DEVICE +#include "xgboost/context.h" // Context +#include "xgboost/data.h" // MetaInfo +#include "xgboost/host_device_vector.h" // HostDeviceVector namespace xgboost { namespace metric { diff --git a/src/metric/survival_metric.cu b/src/metric/survival_metric.cu index 46b92b292..8205f07a1 100644 --- a/src/metric/survival_metric.cu +++ b/src/metric/survival_metric.cu @@ -10,15 +10,14 @@ #include #include -#include "xgboost/json.h" -#include "xgboost/metric.h" -#include "xgboost/host_device_vector.h" - -#include "metric_common.h" #include "../collective/communicator-inl.h" #include "../common/math.h" #include "../common/survival_util.h" -#include "../common/threading_utils.h" +#include "../common/threading_utils.h" +#include "metric_common.h" // MetricNoCache +#include "xgboost/host_device_vector.h" +#include "xgboost/json.h" +#include "xgboost/metric.h" #if defined(XGBOOST_USE_CUDA) #include // thrust::cuda::par @@ -194,10 +193,9 @@ struct EvalAFTNLogLik { AFTParam param_; }; -template struct EvalEWiseSurvivalBase : public Metric { - explicit EvalEWiseSurvivalBase(Context const *ctx) { - ctx_ = ctx; - } +template +struct EvalEWiseSurvivalBase : public MetricNoCache { + explicit EvalEWiseSurvivalBase(Context const* ctx) { ctx_ = ctx; } EvalEWiseSurvivalBase() = default; void Configure(const Args& args) override { @@ -230,7 +228,7 @@ template struct EvalEWiseSurvivalBase : public Metric { // This class exists because we want to perform dispatch according to the distribution type at // configuration time, not at prediction time. -struct AFTNLogLikDispatcher : public Metric { +struct AFTNLogLikDispatcher : public MetricNoCache { const char* Name() const override { return "aft-nloglik"; } @@ -270,7 +268,7 @@ struct AFTNLogLikDispatcher : public Metric { private: AFTParam param_; - std::unique_ptr metric_; + std::unique_ptr metric_; }; XGBOOST_REGISTER_METRIC(AFTNLogLik, "aft-nloglik") diff --git a/tests/cpp/helpers.cc b/tests/cpp/helpers.cc index 2c3cb5094..fcaffa5c6 100644 --- a/tests/cpp/helpers.cc +++ b/tests/cpp/helpers.cc @@ -156,14 +156,15 @@ double GetMultiMetricEval(xgboost::Metric* metric, xgboost::linalg::Tensor const& labels, std::vector weights, std::vector groups) { - xgboost::MetaInfo info; + std::shared_ptr p_fmat{xgboost::RandomDataGenerator{0, 0, 0}.GenerateDMatrix()}; + auto& info = p_fmat->Info(); info.num_row_ = labels.Shape(0); info.labels.Reshape(labels.Shape()[0], labels.Shape()[1]); info.labels.Data()->Copy(*labels.Data()); info.weights_.HostVector() = weights; info.group_ptr_ = groups; - return metric->Eval(preds, info); + return metric->Evaluate(preds, p_fmat); } namespace xgboost { @@ -661,4 +662,4 @@ void DeleteRMMResource(RMMAllocator*) {} RMMAllocatorPtr SetUpRMMResourceForCppTests(int, char**) { return {nullptr, DeleteRMMResource}; } #endif // !defined(XGBOOST_USE_RMM) || XGBOOST_USE_RMM != 1 -} // namespace xgboost +} // namespace xgboost diff --git a/tests/cpp/helpers.h b/tests/cpp/helpers.h index 71424af18..63ef6ac50 100644 --- a/tests/cpp/helpers.h +++ b/tests/cpp/helpers.h @@ -301,6 +301,11 @@ class RandomDataGenerator { std::shared_ptr GenerateQuantileDMatrix(); }; +// Generate an empty DMatrix, mostly for its meta info. +inline std::shared_ptr EmptyDMatrix() { + return RandomDataGenerator{0, 0, 0.0}.GenerateDMatrix(); +} + inline std::vector GenerateRandomCategoricalSingleColumn(int n, size_t num_categories) { std::vector x(n); diff --git a/tests/cpp/metric/test_auc.cc b/tests/cpp/metric/test_auc.cc index 321f46cdc..2a6738899 100644 --- a/tests/cpp/metric/test_auc.cc +++ b/tests/cpp/metric/test_auc.cc @@ -20,12 +20,13 @@ TEST(Metric, DeclareUnifiedTest(BinaryAUC)) { EXPECT_NEAR(GetMetricEval(metric, {1, 0, 0}, {0, 0, 1}), 0.25f, 1e-10); // Invalid dataset - MetaInfo info; + auto p_fmat = EmptyDMatrix(); + MetaInfo& info = p_fmat->Info(); info.labels = linalg::Tensor{{0.0f, 0.0f}, {2}, -1}; - float auc = metric->Eval({1, 1}, info); + float auc = metric->Evaluate({1, 1}, p_fmat); ASSERT_TRUE(std::isnan(auc)); *info.labels.Data() = HostDeviceVector{}; - auc = metric->Eval(HostDeviceVector{}, info); + auc = metric->Evaluate(HostDeviceVector{}, p_fmat); ASSERT_TRUE(std::isnan(auc)); EXPECT_NEAR(GetMetricEval(metric, {0, 1, 0, 1}, {0, 1, 0, 1}), 1.0f, 1e-10); diff --git a/tests/cpp/metric/test_elementwise_metric.cc b/tests/cpp/metric/test_elementwise_metric.cc index 5ac50be2a..9000cfc09 100644 --- a/tests/cpp/metric/test_elementwise_metric.cc +++ b/tests/cpp/metric/test_elementwise_metric.cc @@ -19,7 +19,8 @@ inline void CheckDeterministicMetricElementWise(StringView name, int32_t device) HostDeviceVector predts; size_t n_samples = 2048; - MetaInfo info; + auto p_fmat = EmptyDMatrix(); + MetaInfo& info = p_fmat->Info(); info.labels.Reshape(n_samples, 1); info.num_row_ = n_samples; auto &h_labels = info.labels.Data()->HostVector(); @@ -36,9 +37,9 @@ inline void CheckDeterministicMetricElementWise(StringView name, int32_t device) h_labels[i] = dist(&lcg); } - auto result = metric->Eval(predts, info); + auto result = metric->Evaluate(predts, p_fmat); for (size_t i = 0; i < 8; ++i) { - ASSERT_EQ(metric->Eval(predts, info), result); + ASSERT_EQ(metric->Evaluate(predts, p_fmat), result); } } } // anonymous namespace diff --git a/tests/cpp/metric/test_multiclass_metric.cc b/tests/cpp/metric/test_multiclass_metric.cc index 7aa8f8e8e..2465b11c8 100644 --- a/tests/cpp/metric/test_multiclass_metric.cc +++ b/tests/cpp/metric/test_multiclass_metric.cc @@ -10,7 +10,8 @@ inline void CheckDeterministicMetricMultiClass(StringView name, int32_t device) std::unique_ptr metric{Metric::Create(name.c_str(), &ctx)}; HostDeviceVector predts; - MetaInfo info; + auto p_fmat = EmptyDMatrix(); + MetaInfo& info = p_fmat->Info(); auto &h_predts = predts.HostVector(); SimpleLCG lcg; @@ -35,9 +36,9 @@ inline void CheckDeterministicMetricMultiClass(StringView name, int32_t device) } } - auto result = metric->Eval(predts, info); + auto result = metric->Evaluate(predts, p_fmat); for (size_t i = 0; i < 8; ++i) { - ASSERT_EQ(metric->Eval(predts, info), result); + ASSERT_EQ(metric->Evaluate(predts, p_fmat), result); } } } // namespace xgboost diff --git a/tests/cpp/metric/test_survival_metric.cu b/tests/cpp/metric/test_survival_metric.cu index 0e472008b..80d6b72e6 100644 --- a/tests/cpp/metric/test_survival_metric.cu +++ b/tests/cpp/metric/test_survival_metric.cu @@ -18,7 +18,8 @@ inline void CheckDeterministicMetricElementWise(StringView name, int32_t device) metric->Configure(Args{}); HostDeviceVector predts; - MetaInfo info; + auto p_fmat = EmptyDMatrix(); + MetaInfo& info = p_fmat->Info(); auto &h_predts = predts.HostVector(); SimpleLCG lcg; @@ -40,9 +41,9 @@ inline void CheckDeterministicMetricElementWise(StringView name, int32_t device) h_upper[i] = 10; } - auto result = metric->Eval(predts, info); + auto result = metric->Evaluate(predts, p_fmat); for (size_t i = 0; i < 8; ++i) { - ASSERT_EQ(metric->Eval(predts, info), result); + ASSERT_EQ(metric->Evaluate(predts, p_fmat), result); } } } // anonymous namespace @@ -54,7 +55,8 @@ TEST(Metric, DeclareUnifiedTest(AFTNegLogLik)) { * Test aggregate output from the AFT metric over a small test data set. * This is unlike AFTLoss.* tests, which verify metric values over individual data points. **/ - MetaInfo info; + auto p_fmat = EmptyDMatrix(); + MetaInfo& info = p_fmat->Info(); info.num_row_ = 4; info.labels_lower_bound_.HostVector() = { 100.0f, 0.0f, 60.0f, 16.0f }; @@ -72,14 +74,15 @@ TEST(Metric, DeclareUnifiedTest(AFTNegLogLik)) { std::unique_ptr metric(Metric::Create("aft-nloglik", &ctx)); metric->Configure({ {"aft_loss_distribution", test_case.dist_type}, {"aft_loss_distribution_scale", "1.0"} }); - EXPECT_NEAR(metric->Eval(preds, info), test_case.reference_value, 1e-4); + EXPECT_NEAR(metric->Evaluate(preds, p_fmat), test_case.reference_value, 1e-4); } } TEST(Metric, DeclareUnifiedTest(IntervalRegressionAccuracy)) { auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX); - MetaInfo info; + auto p_fmat = EmptyDMatrix(); + MetaInfo& info = p_fmat->Info(); info.num_row_ = 4; info.labels_lower_bound_.HostVector() = { 20.0f, 0.0f, 60.0f, 16.0f }; info.labels_upper_bound_.HostVector() = { 80.0f, 20.0f, 80.0f, 200.0f }; @@ -87,15 +90,15 @@ TEST(Metric, DeclareUnifiedTest(IntervalRegressionAccuracy)) { HostDeviceVector preds(4, std::log(60.0f)); std::unique_ptr metric(Metric::Create("interval-regression-accuracy", &ctx)); - EXPECT_FLOAT_EQ(metric->Eval(preds, info), 0.75f); + EXPECT_FLOAT_EQ(metric->Evaluate(preds, p_fmat), 0.75f); info.labels_lower_bound_.HostVector()[2] = 70.0f; - EXPECT_FLOAT_EQ(metric->Eval(preds, info), 0.50f); + EXPECT_FLOAT_EQ(metric->Evaluate(preds, p_fmat), 0.50f); info.labels_upper_bound_.HostVector()[2] = std::numeric_limits::infinity(); - EXPECT_FLOAT_EQ(metric->Eval(preds, info), 0.50f); + EXPECT_FLOAT_EQ(metric->Evaluate(preds, p_fmat), 0.50f); info.labels_upper_bound_.HostVector()[3] = std::numeric_limits::infinity(); - EXPECT_FLOAT_EQ(metric->Eval(preds, info), 0.50f); + EXPECT_FLOAT_EQ(metric->Evaluate(preds, p_fmat), 0.50f); info.labels_lower_bound_.HostVector()[0] = 70.0f; - EXPECT_FLOAT_EQ(metric->Eval(preds, info), 0.25f); + EXPECT_FLOAT_EQ(metric->Evaluate(preds, p_fmat), 0.25f); CheckDeterministicMetricElementWise(StringView{"interval-regression-accuracy"}, GPUIDX); }