diff --git a/demo/guide-python/quantile_regression.py b/demo/guide-python/quantile_regression.py index e8c5486c8..d92115bf0 100644 --- a/demo/guide-python/quantile_regression.py +++ b/demo/guide-python/quantile_regression.py @@ -53,9 +53,8 @@ def quantile_loss(args: argparse.Namespace) -> None: "tree_method": "hist", "quantile_alpha": alpha, # Let's try not to overfit. - "learning_rate": 0.01, - "max_depth": 3, - "min_child_weight": 16.0, + "learning_rate": 0.04, + "max_depth": 5, }, Xy, num_boost_round=32, @@ -80,9 +79,8 @@ def quantile_loss(args: argparse.Namespace) -> None: "objective": "reg:squarederror", "tree_method": "hist", # Let's try not to overfit. - "learning_rate": 0.01, - "max_depth": 3, - "min_child_weight": 16.0, + "learning_rate": 0.04, + "max_depth": 5, }, Xy, num_boost_round=32, diff --git a/include/xgboost/objective.h b/include/xgboost/objective.h index 0341a27a1..a04d2e453 100644 --- a/include/xgboost/objective.h +++ b/include/xgboost/objective.h @@ -116,12 +116,13 @@ class ObjFunction : public Configurable { * * \param position The leaf index for each rows. * \param info MetaInfo providing labels and weights. + * \param learning_rate The learning rate for current iteration. * \param prediction Model prediction after transformation. * \param group_idx The group index for this tree, 0 when it's not multi-target or multi-class. * \param p_tree Tree that needs to be updated. */ virtual void UpdateTreeLeaf(HostDeviceVector const& /*position*/, - MetaInfo const& /*info*/, + MetaInfo const& /*info*/, float /*learning_rate*/, HostDeviceVector const& /*prediction*/, std::int32_t /*group_idx*/, RegTree* /*p_tree*/) const {} diff --git a/include/xgboost/tree_updater.h b/include/xgboost/tree_updater.h index 5cf8fb05c..59f4c2cf8 100644 --- a/include/xgboost/tree_updater.h +++ b/include/xgboost/tree_updater.h @@ -24,6 +24,9 @@ #include namespace xgboost { +namespace tree { +struct TrainParam; +} class Json; struct Context; @@ -56,8 +59,10 @@ class TreeUpdater : public Configurable { * tree can be used. */ virtual bool HasNodePosition() const { return false; } - /*! + /** * \brief perform update to the tree models + * + * \param param Hyper-parameter for constructing trees. * \param gpair the gradient pair statistics of the data * \param data The data matrix passed to the updater. * \param out_position The leaf index for each row. The index is negated if that row is @@ -67,8 +72,8 @@ class TreeUpdater : public Configurable { * but maybe different random seeds, usually one tree is passed in at a time, * there can be multiple trees when we train random forest style model */ - virtual void Update(HostDeviceVector* gpair, DMatrix* data, - common::Span> out_position, + virtual void Update(tree::TrainParam const* param, HostDeviceVector* gpair, + DMatrix* data, common::Span> out_position, const std::vector& out_trees) = 0; /*! diff --git a/src/gbm/gbtree.cc b/src/gbm/gbtree.cc index dc280217e..39f38c289 100644 --- a/src/gbm/gbtree.cc +++ b/src/gbm/gbtree.cc @@ -32,15 +32,14 @@ #include "xgboost/string_view.h" #include "xgboost/tree_updater.h" -namespace xgboost { -namespace gbm { - +namespace xgboost::gbm { DMLC_REGISTRY_FILE_TAG(gbtree); -void GBTree::Configure(const Args& cfg) { +void GBTree::Configure(Args const& cfg) { this->cfg_ = cfg; std::string updater_seq = tparam_.updater_seq; tparam_.UpdateAllowUnknown(cfg); + tree_param_.UpdateAllowUnknown(cfg); model_.Configure(cfg); @@ -235,9 +234,11 @@ void GBTree::UpdateTreeLeaf(DMatrix const* p_fmat, HostDeviceVector const CHECK_EQ(model_.param.num_parallel_tree, trees.size()); CHECK_EQ(model_.param.num_parallel_tree, 1) << "Boosting random forest is not supported for current objective."; + CHECK_EQ(trees.size(), model_.param.num_parallel_tree); for (std::size_t tree_idx = 0; tree_idx < trees.size(); ++tree_idx) { auto const& position = node_position.at(tree_idx); - obj->UpdateTreeLeaf(position, p_fmat->Info(), predictions, group_idx, trees[tree_idx].get()); + obj->UpdateTreeLeaf(position, p_fmat->Info(), tree_param_.learning_rate / trees.size(), + predictions, group_idx, trees[tree_idx].get()); } } @@ -388,9 +389,15 @@ void GBTree::BoostNewTrees(HostDeviceVector* gpair, DMatrix* p_fma CHECK(out_position); out_position->resize(new_trees.size()); + + // Rescale learning rate according to the size of trees + auto lr = tree_param_.learning_rate; + tree_param_.learning_rate /= static_cast(new_trees.size()); for (auto& up : updaters_) { - up->Update(gpair, p_fmat, common::Span>{*out_position}, new_trees); + up->Update(&tree_param_, gpair, p_fmat, + common::Span>{*out_position}, new_trees); } + tree_param_.learning_rate = lr; } void GBTree::CommitModel(std::vector>>&& new_trees) { @@ -404,6 +411,8 @@ void GBTree::CommitModel(std::vector>>&& ne void GBTree::LoadConfig(Json const& in) { CHECK_EQ(get(in["name"]), "gbtree"); FromJson(in["gbtree_train_param"], &tparam_); + FromJson(in["tree_train_param"], &tree_param_); + // Process type cannot be kUpdate from loaded model // This would cause all trees to be pushed to trees_to_update // e.g. updating a model, then saving and loading it would result in an empty model @@ -451,6 +460,7 @@ void GBTree::SaveConfig(Json* p_out) const { auto& out = *p_out; out["name"] = String("gbtree"); out["gbtree_train_param"] = ToJson(tparam_); + out["tree_train_param"] = ToJson(tree_param_); // Process type cannot be kUpdate from loaded model // This would cause all trees to be pushed to trees_to_update @@ -1058,5 +1068,4 @@ XGBOOST_REGISTER_GBM(Dart, "dart") GBTree* p = new Dart(booster_config, ctx); return p; }); -} // namespace gbm -} // namespace xgboost +} // namespace xgboost::gbm diff --git a/src/gbm/gbtree.h b/src/gbm/gbtree.h index 6bf98916f..10e6c415f 100644 --- a/src/gbm/gbtree.h +++ b/src/gbm/gbtree.h @@ -20,6 +20,7 @@ #include "../common/common.h" #include "../common/timer.h" +#include "../tree/param.h" // TrainParam #include "gbtree_model.h" #include "xgboost/base.h" #include "xgboost/data.h" @@ -405,8 +406,8 @@ class GBTree : public GradientBooster { p_fmat, out_contribs, model_, tree_end, nullptr, approximate); } - std::vector DumpModel(const FeatureMap& fmap, bool with_stats, - std::string format) const override { + [[nodiscard]] std::vector DumpModel(const FeatureMap& fmap, bool with_stats, + std::string format) const override { return model_.DumpModel(fmap, with_stats, this->ctx_->Threads(), format); } @@ -428,6 +429,8 @@ class GBTree : public GradientBooster { GBTreeModel model_; // training parameter GBTreeTrainParam tparam_; + // Tree training parameter + tree::TrainParam tree_param_; // ----training fields---- bool showed_updater_warning_ {false}; bool specified_updater_ {false}; diff --git a/src/objective/adaptive.cc b/src/objective/adaptive.cc index fb22f0049..f5f35c846 100644 --- a/src/objective/adaptive.cc +++ b/src/objective/adaptive.cc @@ -76,7 +76,7 @@ void EncodeTreeLeafHost(Context const* ctx, RegTree const& tree, } void UpdateTreeLeafHost(Context const* ctx, std::vector const& position, - std::int32_t group_idx, MetaInfo const& info, + std::int32_t group_idx, MetaInfo const& info, float learning_rate, HostDeviceVector const& predt, float alpha, RegTree* p_tree) { auto& tree = *p_tree; @@ -87,7 +87,7 @@ void UpdateTreeLeafHost(Context const* ctx, std::vector const& posit size_t n_leaf = nidx.size(); if (nptr.empty()) { std::vector quantiles; - UpdateLeafValues(&quantiles, nidx, p_tree); + UpdateLeafValues(&quantiles, nidx, learning_rate, p_tree); return; } @@ -133,12 +133,13 @@ void UpdateTreeLeafHost(Context const* ctx, std::vector const& posit quantiles.at(k) = q; }); - UpdateLeafValues(&quantiles, nidx, p_tree); + UpdateLeafValues(&quantiles, nidx, learning_rate, p_tree); } #if !defined(XGBOOST_USE_CUDA) void UpdateTreeLeafDevice(Context const*, common::Span, std::int32_t, - MetaInfo const&, HostDeviceVector const&, float, RegTree*) { + MetaInfo const&, float learning_rate, HostDeviceVector const&, + float, RegTree*) { common::AssertGPUSupport(); } #endif // !defined(XGBOOST_USE_CUDA) diff --git a/src/objective/adaptive.cu b/src/objective/adaptive.cu index 71731e9c4..76627cf6d 100644 --- a/src/objective/adaptive.cu +++ b/src/objective/adaptive.cu @@ -140,7 +140,7 @@ void EncodeTreeLeafDevice(Context const* ctx, common::Span pos } void UpdateTreeLeafDevice(Context const* ctx, common::Span position, - std::int32_t group_idx, MetaInfo const& info, + std::int32_t group_idx, MetaInfo const& info, float learning_rate, HostDeviceVector const& predt, float alpha, RegTree* p_tree) { dh::safe_cuda(cudaSetDevice(ctx->gpu_id)); dh::device_vector ridx; @@ -151,7 +151,7 @@ void UpdateTreeLeafDevice(Context const* ctx, common::Span pos if (nptr.Empty()) { std::vector quantiles; - UpdateLeafValues(&quantiles, nidx.ConstHostVector(), p_tree); + UpdateLeafValues(&quantiles, nidx.ConstHostVector(), learning_rate, p_tree); } HostDeviceVector quantiles; @@ -186,7 +186,7 @@ void UpdateTreeLeafDevice(Context const* ctx, common::Span pos w_it + d_weights.size(), &quantiles); } - UpdateLeafValues(&quantiles.HostVector(), nidx.ConstHostVector(), p_tree); + UpdateLeafValues(&quantiles.HostVector(), nidx.ConstHostVector(), learning_rate, p_tree); } } // namespace detail } // namespace obj diff --git a/src/objective/adaptive.h b/src/objective/adaptive.h index ca81cac2a..fef920ec9 100644 --- a/src/objective/adaptive.h +++ b/src/objective/adaptive.h @@ -36,7 +36,7 @@ inline void FillMissingLeaf(std::vector const& maybe_missing, } inline void UpdateLeafValues(std::vector* p_quantiles, std::vector const& nidx, - RegTree* p_tree) { + float learning_rate, RegTree* p_tree) { auto& tree = *p_tree; auto& quantiles = *p_quantiles; auto const& h_node_idx = nidx; @@ -71,7 +71,7 @@ inline void UpdateLeafValues(std::vector* p_quantiles, std::vector position, - std::int32_t group_idx, MetaInfo const& info, + std::int32_t group_idx, MetaInfo const& info, float learning_rate, HostDeviceVector const& predt, float alpha, RegTree* p_tree); void UpdateTreeLeafHost(Context const* ctx, std::vector const& position, - std::int32_t group_idx, MetaInfo const& info, + std::int32_t group_idx, MetaInfo const& info, float learning_rate, HostDeviceVector const& predt, float alpha, RegTree* p_tree); } // namespace detail inline void UpdateTreeLeaf(Context const* ctx, HostDeviceVector const& position, - std::int32_t group_idx, MetaInfo const& info, + std::int32_t group_idx, MetaInfo const& info, float learning_rate, HostDeviceVector const& predt, float alpha, RegTree* p_tree) { if (ctx->IsCPU()) { - detail::UpdateTreeLeafHost(ctx, position.ConstHostVector(), group_idx, info, predt, alpha, - p_tree); + detail::UpdateTreeLeafHost(ctx, position.ConstHostVector(), group_idx, info, learning_rate, + predt, alpha, p_tree); } else { position.SetDevice(ctx->gpu_id); - detail::UpdateTreeLeafDevice(ctx, position.ConstDeviceSpan(), group_idx, info, predt, alpha, - p_tree); + detail::UpdateTreeLeafDevice(ctx, position.ConstDeviceSpan(), group_idx, info, learning_rate, + predt, alpha, p_tree); } } } // namespace obj diff --git a/src/objective/quantile_obj.cu b/src/objective/quantile_obj.cu index f699338e1..3b9204251 100644 --- a/src/objective/quantile_obj.cu +++ b/src/objective/quantile_obj.cu @@ -183,10 +183,11 @@ class QuantileRegression : public ObjFunction { } void UpdateTreeLeaf(HostDeviceVector const& position, MetaInfo const& info, - HostDeviceVector const& prediction, std::int32_t group_idx, - RegTree* p_tree) const override { + float learning_rate, HostDeviceVector const& prediction, + std::int32_t group_idx, RegTree* p_tree) const override { auto alpha = param_.quantile_alpha[group_idx]; - ::xgboost::obj::UpdateTreeLeaf(ctx_, position, group_idx, info, prediction, alpha, p_tree); + ::xgboost::obj::UpdateTreeLeaf(ctx_, position, group_idx, info, learning_rate, prediction, + alpha, p_tree); } void Configure(Args const& args) override { diff --git a/src/objective/regression_obj.cu b/src/objective/regression_obj.cu index 2edaff0b0..d7999f8c1 100644 --- a/src/objective/regression_obj.cu +++ b/src/objective/regression_obj.cu @@ -742,9 +742,10 @@ class MeanAbsoluteError : public ObjFunction { } void UpdateTreeLeaf(HostDeviceVector const& position, MetaInfo const& info, - HostDeviceVector const& prediction, std::int32_t group_idx, - RegTree* p_tree) const override { - ::xgboost::obj::UpdateTreeLeaf(ctx_, position, group_idx, info, prediction, 0.5, p_tree); + float learning_rate, HostDeviceVector const& prediction, + std::int32_t group_idx, RegTree* p_tree) const override { + ::xgboost::obj::UpdateTreeLeaf(ctx_, position, group_idx, info, learning_rate, prediction, 0.5, + p_tree); } const char* DefaultEvalMetric() const override { return "mae"; } diff --git a/src/tree/hist/evaluate_splits.h b/src/tree/hist/evaluate_splits.h index d90eb93f4..31a61fb9d 100644 --- a/src/tree/hist/evaluate_splits.h +++ b/src/tree/hist/evaluate_splits.h @@ -17,13 +17,11 @@ #include "../../common/random.h" #include "../../data/gradient_index.h" #include "../constraints.h" -#include "../param.h" +#include "../param.h" // for TrainParam #include "../split_evaluator.h" #include "xgboost/context.h" -namespace xgboost { -namespace tree { - +namespace xgboost::tree { template class HistEvaluator { private: @@ -36,7 +34,7 @@ class HistEvaluator { private: Context const* ctx_; - TrainParam param_; + TrainParam const* param_; std::shared_ptr column_sampler_; TreeEvaluator tree_evaluator_; bool is_col_split_{false}; @@ -55,8 +53,9 @@ class HistEvaluator { } } - bool IsValid(GradStats const &left, GradStats const &right) const { - return left.GetHess() >= param_.min_child_weight && right.GetHess() >= param_.min_child_weight; + [[nodiscard]] bool IsValid(GradStats const &left, GradStats const &right) const { + return left.GetHess() >= param_->min_child_weight && + right.GetHess() >= param_->min_child_weight; } /** @@ -95,9 +94,10 @@ class HistEvaluator { right_sum = GradStats{hist[i]}; left_sum.SetSubstract(parent.stats, right_sum); if (IsValid(left_sum, right_sum)) { - auto missing_left_chg = static_cast( - evaluator.CalcSplitGain(param_, nidx, fidx, GradStats{left_sum}, GradStats{right_sum}) - - parent.root_gain); + auto missing_left_chg = + static_cast(evaluator.CalcSplitGain(*param_, nidx, fidx, GradStats{left_sum}, + GradStats{right_sum}) - + parent.root_gain); best.Update(missing_left_chg, fidx, split_pt, true, true, left_sum, right_sum); } @@ -105,9 +105,10 @@ class HistEvaluator { right_sum.Add(missing); left_sum.SetSubstract(parent.stats, right_sum); if (IsValid(left_sum, right_sum)) { - auto missing_right_chg = static_cast( - evaluator.CalcSplitGain(param_, nidx, fidx, GradStats{left_sum}, GradStats{right_sum}) - - parent.root_gain); + auto missing_right_chg = + static_cast(evaluator.CalcSplitGain(*param_, nidx, fidx, GradStats{left_sum}, + GradStats{right_sum}) - + parent.root_gain); best.Update(missing_right_chg, fidx, split_pt, false, true, left_sum, right_sum); } } @@ -152,7 +153,7 @@ class HistEvaluator { bst_bin_t f_begin = cut_ptr[fidx]; bst_bin_t f_end = cut_ptr[fidx + 1]; bst_bin_t n_bins_feature{f_end - f_begin}; - auto n_bins = std::min(param_.max_cat_threshold, n_bins_feature); + auto n_bins = std::min(param_->max_cat_threshold, n_bins_feature); // statistics on both sides of split GradStats left_sum; @@ -181,9 +182,9 @@ class HistEvaluator { right_sum.SetSubstract(parent.stats, left_sum); // missing on right } if (IsValid(left_sum, right_sum)) { - auto loss_chg = - evaluator.CalcSplitGain(param_, nidx, fidx, GradStats{left_sum}, GradStats{right_sum}) - - parent.root_gain; + auto loss_chg = evaluator.CalcSplitGain(*param_, nidx, fidx, GradStats{left_sum}, + GradStats{right_sum}) - + parent.root_gain; // We don't have a numeric split point, nan here is a dummy split. if (best.Update(loss_chg, fidx, std::numeric_limits::quiet_NaN(), d_step == 1, true, left_sum, right_sum)) { @@ -256,7 +257,7 @@ class HistEvaluator { if (d_step > 0) { // forward enumeration: split at right bound of each bin loss_chg = - static_cast(evaluator.CalcSplitGain(param_, nidx, fidx, GradStats{left_sum}, + static_cast(evaluator.CalcSplitGain(*param_, nidx, fidx, GradStats{left_sum}, GradStats{right_sum}) - parent.root_gain); split_pt = cut_val[i]; // not used for partition based @@ -264,7 +265,7 @@ class HistEvaluator { } else { // backward enumeration: split at left bound of each bin loss_chg = - static_cast(evaluator.CalcSplitGain(param_, nidx, fidx, GradStats{right_sum}, + static_cast(evaluator.CalcSplitGain(*param_, nidx, fidx, GradStats{right_sum}, GradStats{left_sum}) - parent.root_gain); if (i == imin) { @@ -326,7 +327,7 @@ class HistEvaluator { } if (is_cat) { auto n_bins = cut_ptrs.at(fidx + 1) - cut_ptrs[fidx]; - if (common::UseOneHot(n_bins, param_.max_cat_to_onehot)) { + if (common::UseOneHot(n_bins, param_->max_cat_to_onehot)) { EnumerateOneHot(cut, histogram, fidx, nidx, evaluator, best); } else { std::vector sorted_idx(n_bins); @@ -334,8 +335,8 @@ class HistEvaluator { auto feat_hist = histogram.subspan(cut_ptrs[fidx], n_bins); // Sort the histogram to get contiguous partitions. std::stable_sort(sorted_idx.begin(), sorted_idx.end(), [&](size_t l, size_t r) { - auto ret = evaluator.CalcWeightCat(param_, feat_hist[l]) < - evaluator.CalcWeightCat(param_, feat_hist[r]); + auto ret = evaluator.CalcWeightCat(*param_, feat_hist[l]) < + evaluator.CalcWeightCat(*param_, feat_hist[r]); return ret; }); EnumeratePart<+1>(cut, sorted_idx, histogram, fidx, nidx, evaluator, best); @@ -382,24 +383,22 @@ class HistEvaluator { GradStats parent_sum = candidate.split.left_sum; parent_sum.Add(candidate.split.right_sum); - auto base_weight = - evaluator.CalcWeight(candidate.nid, param_, GradStats{parent_sum}); - + auto base_weight = evaluator.CalcWeight(candidate.nid, *param_, GradStats{parent_sum}); auto left_weight = - evaluator.CalcWeight(candidate.nid, param_, GradStats{candidate.split.left_sum}); + evaluator.CalcWeight(candidate.nid, *param_, GradStats{candidate.split.left_sum}); auto right_weight = - evaluator.CalcWeight(candidate.nid, param_, GradStats{candidate.split.right_sum}); + evaluator.CalcWeight(candidate.nid, *param_, GradStats{candidate.split.right_sum}); if (candidate.split.is_cat) { tree.ExpandCategorical( candidate.nid, candidate.split.SplitIndex(), candidate.split.cat_bits, - candidate.split.DefaultLeft(), base_weight, left_weight * param_.learning_rate, - right_weight * param_.learning_rate, candidate.split.loss_chg, parent_sum.GetHess(), + candidate.split.DefaultLeft(), base_weight, left_weight * param_->learning_rate, + right_weight * param_->learning_rate, candidate.split.loss_chg, parent_sum.GetHess(), candidate.split.left_sum.GetHess(), candidate.split.right_sum.GetHess()); } else { tree.ExpandNode(candidate.nid, candidate.split.SplitIndex(), candidate.split.split_value, candidate.split.DefaultLeft(), base_weight, - left_weight * param_.learning_rate, right_weight * param_.learning_rate, + left_weight * param_->learning_rate, right_weight * param_->learning_rate, candidate.split.loss_chg, parent_sum.GetHess(), candidate.split.left_sum.GetHess(), candidate.split.right_sum.GetHess()); } @@ -415,11 +414,11 @@ class HistEvaluator { max_node = std::max(candidate.nid, max_node); snode_.resize(tree.GetNodes().size()); snode_.at(left_child).stats = candidate.split.left_sum; - snode_.at(left_child).root_gain = evaluator.CalcGain( - candidate.nid, param_, GradStats{candidate.split.left_sum}); + snode_.at(left_child).root_gain = + evaluator.CalcGain(candidate.nid, *param_, GradStats{candidate.split.left_sum}); snode_.at(right_child).stats = candidate.split.right_sum; - snode_.at(right_child).root_gain = evaluator.CalcGain( - candidate.nid, param_, GradStats{candidate.split.right_sum}); + snode_.at(right_child).root_gain = + evaluator.CalcGain(candidate.nid, *param_, GradStats{candidate.split.right_sum}); interaction_constraints_.Split(candidate.nid, tree[candidate.nid].SplitIndex(), left_child, @@ -429,31 +428,31 @@ class HistEvaluator { auto Evaluator() const { return tree_evaluator_.GetEvaluator(); } auto const& Stats() const { return snode_; } - float InitRoot(GradStats const& root_sum) { + float InitRoot(GradStats const &root_sum) { snode_.resize(1); auto root_evaluator = tree_evaluator_.GetEvaluator(); snode_[0].stats = GradStats{root_sum.GetGrad(), root_sum.GetHess()}; - snode_[0].root_gain = root_evaluator.CalcGain(RegTree::kRoot, param_, - GradStats{snode_[0].stats}); - auto weight = root_evaluator.CalcWeight(RegTree::kRoot, param_, - GradStats{snode_[0].stats}); + snode_[0].root_gain = + root_evaluator.CalcGain(RegTree::kRoot, *param_, GradStats{snode_[0].stats}); + auto weight = root_evaluator.CalcWeight(RegTree::kRoot, *param_, GradStats{snode_[0].stats}); return weight; } public: // The column sampler must be constructed by caller since we need to preserve the rng // for the entire training session. - explicit HistEvaluator(Context const* ctx, TrainParam const ¶m, MetaInfo const &info, + explicit HistEvaluator(Context const *ctx, TrainParam const *param, MetaInfo const &info, std::shared_ptr sampler) - : ctx_{ctx}, param_{param}, + : ctx_{ctx}, + param_{param}, column_sampler_{std::move(sampler)}, - tree_evaluator_{param, static_cast(info.num_col_), Context::kCpuId}, + tree_evaluator_{*param, static_cast(info.num_col_), Context::kCpuId}, is_col_split_{info.data_split_mode == DataSplitMode::kCol} { - interaction_constraints_.Configure(param, info.num_col_); + interaction_constraints_.Configure(*param, info.num_col_); column_sampler_->Init(ctx, info.num_col_, info.feature_weights.HostVector(), - param_.colsample_bynode, param_.colsample_bylevel, - param_.colsample_bytree); + param_->colsample_bynode, param_->colsample_bylevel, + param_->colsample_bytree); } }; @@ -488,6 +487,5 @@ void UpdatePredictionCacheImpl(Context const *ctx, RegTree const *p_last_tree, }); } } -} // namespace tree -} // namespace xgboost +} // namespace xgboost::tree #endif // XGBOOST_TREE_HIST_EVALUATE_SPLITS_H_ diff --git a/src/tree/updater_approx.cc b/src/tree/updater_approx.cc index 4a939ff02..2bc3ff543 100644 --- a/src/tree/updater_approx.cc +++ b/src/tree/updater_approx.cc @@ -23,8 +23,7 @@ #include "xgboost/tree_model.h" #include "xgboost/tree_updater.h" -namespace xgboost { -namespace tree { +namespace xgboost::tree { DMLC_REGISTRY_FILE_TAG(updater_approx); @@ -41,7 +40,7 @@ auto BatchSpec(TrainParam const &p, common::Span hess) { class GloablApproxBuilder { protected: - TrainParam param_; + TrainParam const* param_; std::shared_ptr col_sampler_; HistEvaluator evaluator_; HistogramBuilder histogram_builder_; @@ -64,7 +63,7 @@ class GloablApproxBuilder { bst_bin_t n_total_bins = 0; partitioner_.clear(); // Generating the GHistIndexMatrix is quite slow, is there a way to speed it up? - for (auto const &page : p_fmat->GetBatches(BatchSpec(param_, hess, task_))) { + for (auto const &page : p_fmat->GetBatches(BatchSpec(*param_, hess, task_))) { if (n_total_bins == 0) { n_total_bins = page.cut.TotalBins(); feature_values_ = page.cut; @@ -75,7 +74,7 @@ class GloablApproxBuilder { n_batches_++; } - histogram_builder_.Reset(n_total_bins, BatchSpec(param_, hess), ctx_->Threads(), n_batches_, + histogram_builder_.Reset(n_total_bins, BatchSpec(*param_, hess), ctx_->Threads(), n_batches_, collective::IsDistributed(), p_fmat->IsColumnSplit()); monitor_->Stop(__func__); } @@ -96,7 +95,7 @@ class GloablApproxBuilder { std::vector nodes{best}; size_t i = 0; auto space = ConstructHistSpace(partitioner_, nodes); - for (auto const &page : p_fmat->GetBatches(BatchSpec(param_, hess))) { + for (auto const &page : p_fmat->GetBatches(BatchSpec(*param_, hess))) { histogram_builder_.BuildHist(i, space, page, p_tree, partitioner_.at(i).Partitions(), nodes, {}, gpair); i++; @@ -105,7 +104,7 @@ class GloablApproxBuilder { auto weight = evaluator_.InitRoot(root_sum); p_tree->Stat(RegTree::kRoot).sum_hess = root_sum.GetHess(); p_tree->Stat(RegTree::kRoot).base_weight = weight; - (*p_tree)[RegTree::kRoot].SetLeaf(param_.learning_rate * weight); + (*p_tree)[RegTree::kRoot].SetLeaf(param_->learning_rate * weight); auto const &histograms = histogram_builder_.Histogram(); auto ft = p_fmat->Info().feature_types.ConstHostSpan(); @@ -147,7 +146,7 @@ class GloablApproxBuilder { size_t i = 0; auto space = ConstructHistSpace(partitioner_, nodes_to_build); - for (auto const &page : p_fmat->GetBatches(BatchSpec(param_, hess))) { + for (auto const &page : p_fmat->GetBatches(BatchSpec(*param_, hess))) { histogram_builder_.BuildHist(i, space, page, p_tree, partitioner_.at(i).Partitions(), nodes_to_build, nodes_to_sub, gpair); i++; @@ -168,10 +167,10 @@ class GloablApproxBuilder { } public: - explicit GloablApproxBuilder(TrainParam param, MetaInfo const &info, Context const *ctx, + explicit GloablApproxBuilder(TrainParam const *param, MetaInfo const &info, Context const *ctx, std::shared_ptr column_sampler, ObjInfo task, common::Monitor *monitor) - : param_{std::move(param)}, + : param_{param}, col_sampler_{std::move(column_sampler)}, evaluator_{ctx, param_, info, col_sampler_}, ctx_{ctx}, @@ -183,7 +182,7 @@ class GloablApproxBuilder { p_last_tree_ = p_tree; this->InitData(p_fmat, hess); - Driver driver(param_); + Driver driver(*param_); auto &tree = *p_tree; driver.Push({this->InitRoot(p_fmat, gpair, hess, p_tree)}); auto expand_set = driver.Pop(); @@ -213,7 +212,7 @@ class GloablApproxBuilder { monitor_->Start("UpdatePosition"); size_t page_id = 0; - for (auto const &page : p_fmat->GetBatches(BatchSpec(param_, hess))) { + for (auto const &page : p_fmat->GetBatches(BatchSpec(*param_, hess))) { partitioner_.at(page_id).UpdatePosition(ctx_, page, applied, p_tree); page_id++; } @@ -250,7 +249,6 @@ class GloablApproxBuilder { * iteration. */ class GlobalApproxUpdater : public TreeUpdater { - TrainParam param_; common::Monitor monitor_; // specializations for different histogram precision. std::unique_ptr pimpl_; @@ -265,15 +263,9 @@ class GlobalApproxUpdater : public TreeUpdater { monitor_.Init(__func__); } - void Configure(const Args &args) override { param_.UpdateAllowUnknown(args); } - void LoadConfig(Json const &in) override { - auto const &config = get(in); - FromJson(config.at("train_param"), &this->param_); - } - void SaveConfig(Json *p_out) const override { - auto &out = *p_out; - out["train_param"] = ToJson(param_); - } + void Configure(Args const &) override {} + void LoadConfig(Json const &) override {} + void SaveConfig(Json *) const override {} void InitData(TrainParam const ¶m, HostDeviceVector const *gpair, linalg::Matrix *sampled) { @@ -283,20 +275,17 @@ class GlobalApproxUpdater : public TreeUpdater { SampleGradient(ctx_, param, sampled->HostView()); } - char const *Name() const override { return "grow_histmaker"; } + [[nodiscard]] char const *Name() const override { return "grow_histmaker"; } - void Update(HostDeviceVector *gpair, DMatrix *m, + void Update(TrainParam const *param, HostDeviceVector *gpair, DMatrix *m, common::Span> out_position, const std::vector &trees) override { - float lr = param_.learning_rate; - param_.learning_rate = lr / trees.size(); - - pimpl_ = std::make_unique(param_, m->Info(), ctx_, column_sampler_, task_, + pimpl_ = std::make_unique(param, m->Info(), ctx_, column_sampler_, task_, &monitor_); linalg::Matrix h_gpair; // Obtain the hessian values for weighted sketching - InitData(param_, gpair, &h_gpair); + InitData(*param, gpair, &h_gpair); std::vector hess(h_gpair.Size()); auto const &s_gpair = h_gpair.Data()->ConstHostVector(); std::transform(s_gpair.begin(), s_gpair.end(), hess.begin(), @@ -304,12 +293,11 @@ class GlobalApproxUpdater : public TreeUpdater { cached_ = m; - size_t t_idx = 0; + std::size_t t_idx = 0; for (auto p_tree : trees) { this->pimpl_->UpdateTree(m, s_gpair, hess, p_tree, &out_position[t_idx]); ++t_idx; } - param_.learning_rate = lr; } bool UpdatePredictionCache(const DMatrix *data, linalg::VectorView out_preds) override { @@ -320,7 +308,7 @@ class GlobalApproxUpdater : public TreeUpdater { return true; } - bool HasNodePosition() const override { return true; } + [[nodiscard]] bool HasNodePosition() const override { return true; } }; DMLC_REGISTRY_FILE_TAG(grow_histmaker); @@ -330,5 +318,4 @@ XGBOOST_REGISTER_TREE_UPDATER(GlobalHistMaker, "grow_histmaker") "Tree constructor that uses approximate histogram construction " "for each node.") .set_body([](Context const *ctx, ObjInfo task) { return new GlobalApproxUpdater(ctx, task); }); -} // namespace tree -} // namespace xgboost +} // namespace xgboost::tree diff --git a/src/tree/updater_colmaker.cc b/src/tree/updater_colmaker.cc index 197c8fe5c..070bfe578 100644 --- a/src/tree/updater_colmaker.cc +++ b/src/tree/updater_colmaker.cc @@ -1,5 +1,5 @@ -/*! - * Copyright 2014-2022 by XGBoost Contributors +/** + * Copyright 2014-2023 by XGBoost Contributors * \file updater_colmaker.cc * \brief use columnwise update to construct a tree * \author Tianqi Chen @@ -17,8 +17,7 @@ #include "../common/random.h" #include "split_evaluator.h" -namespace xgboost { -namespace tree { +namespace xgboost::tree { DMLC_REGISTRY_FILE_TAG(updater_colmaker); @@ -57,18 +56,15 @@ class ColMaker: public TreeUpdater { public: explicit ColMaker(Context const *ctx) : TreeUpdater(ctx) {} void Configure(const Args &args) override { - param_.UpdateAllowUnknown(args); colmaker_param_.UpdateAllowUnknown(args); } void LoadConfig(Json const& in) override { auto const& config = get(in); - FromJson(config.at("train_param"), &this->param_); FromJson(config.at("colmaker_train_param"), &this->colmaker_param_); } - void SaveConfig(Json* p_out) const override { - auto& out = *p_out; - out["train_param"] = ToJson(param_); + void SaveConfig(Json *p_out) const override { + auto &out = *p_out; out["colmaker_train_param"] = ToJson(colmaker_param_); } @@ -95,7 +91,7 @@ class ColMaker: public TreeUpdater { } } - void Update(HostDeviceVector *gpair, DMatrix *dmat, + void Update(TrainParam const *param, HostDeviceVector *gpair, DMatrix *dmat, common::Span> /*out_position*/, const std::vector &trees) override { if (collective::IsDistributed()) { @@ -108,22 +104,16 @@ class ColMaker: public TreeUpdater { } this->LazyGetColumnDensity(dmat); // rescale learning rate according to size of trees - float lr = param_.learning_rate; - param_.learning_rate = lr / trees.size(); - interaction_constraints_.Configure(param_, dmat->Info().num_row_); + interaction_constraints_.Configure(*param, dmat->Info().num_row_); // build tree for (auto tree : trees) { CHECK(ctx_); - Builder builder(param_, colmaker_param_, interaction_constraints_, ctx_, - column_densities_); + Builder builder(*param, colmaker_param_, interaction_constraints_, ctx_, column_densities_); builder.Update(gpair->ConstHostVector(), dmat, tree); } - param_.learning_rate = lr; } protected: - // training parameter - TrainParam param_; ColMakerTrainParam colmaker_param_; // SplitEvaluator that will be cloned for each Builder std::vector column_densities_; @@ -614,5 +604,4 @@ class ColMaker: public TreeUpdater { XGBOOST_REGISTER_TREE_UPDATER(ColMaker, "grow_colmaker") .describe("Grow tree with parallelization over columns.") .set_body([](Context const *ctx, ObjInfo) { return new ColMaker(ctx); }); -} // namespace tree -} // namespace xgboost +} // namespace xgboost::tree diff --git a/src/tree/updater_gpu_hist.cu b/src/tree/updater_gpu_hist.cu index 6345c44d5..32b3f4a03 100644 --- a/src/tree/updater_gpu_hist.cu +++ b/src/tree/updater_gpu_hist.cu @@ -1,5 +1,5 @@ -/*! - * Copyright 2017-2022 XGBoost contributors +/** + * Copyright 2017-2023 by XGBoost contributors */ #include #include @@ -756,7 +756,6 @@ class GPUHistMaker : public TreeUpdater { void Configure(const Args& args) override { // Used in test to count how many configurations are performed LOG(DEBUG) << "[GPU Hist]: Configure"; - param_.UpdateAllowUnknown(args); hist_maker_param_.UpdateAllowUnknown(args); dh::CheckComputeCapability(); initialised_ = false; @@ -768,32 +767,26 @@ class GPUHistMaker : public TreeUpdater { auto const& config = get(in); FromJson(config.at("gpu_hist_train_param"), &this->hist_maker_param_); initialised_ = false; - FromJson(config.at("train_param"), ¶m_); } void SaveConfig(Json* p_out) const override { auto& out = *p_out; out["gpu_hist_train_param"] = ToJson(hist_maker_param_); - out["train_param"] = ToJson(param_); } ~GPUHistMaker() { // NOLINT dh::GlobalMemoryLogger().Log(); } - void Update(HostDeviceVector* gpair, DMatrix* dmat, + void Update(TrainParam const* param, HostDeviceVector* gpair, DMatrix* dmat, common::Span> out_position, const std::vector& trees) override { monitor_.Start("Update"); - // rescale learning rate according to size of trees - float lr = param_.learning_rate; - param_.learning_rate = lr / trees.size(); - // build tree try { size_t t_idx{0}; for (xgboost::RegTree* tree : trees) { - this->UpdateTree(gpair, dmat, tree, &out_position[t_idx]); + this->UpdateTree(param, gpair, dmat, tree, &out_position[t_idx]); if (hist_maker_param_.debug_synchronize) { this->CheckTreesSynchronized(tree); @@ -804,12 +797,10 @@ class GPUHistMaker : public TreeUpdater { } catch (const std::exception& e) { LOG(FATAL) << "Exception in gpu_hist: " << e.what() << std::endl; } - - param_.learning_rate = lr; monitor_.Stop("Update"); } - void InitDataOnce(DMatrix* dmat) { + void InitDataOnce(TrainParam const* param, DMatrix* dmat) { CHECK_GE(ctx_->gpu_id, 0) << "Must have at least one device"; info_ = &dmat->Info(); @@ -818,24 +809,24 @@ class GPUHistMaker : public TreeUpdater { collective::Broadcast(&column_sampling_seed, sizeof(column_sampling_seed), 0); BatchParam batch_param{ - ctx_->gpu_id, - param_.max_bin, + ctx_->gpu_id, + param->max_bin, }; auto page = (*dmat->GetBatches(batch_param).begin()).Impl(); dh::safe_cuda(cudaSetDevice(ctx_->gpu_id)); info_->feature_types.SetDevice(ctx_->gpu_id); maker.reset(new GPUHistMakerDevice( - ctx_, page, info_->feature_types.ConstDeviceSpan(), info_->num_row_, param_, + ctx_, page, info_->feature_types.ConstDeviceSpan(), info_->num_row_, *param, column_sampling_seed, info_->num_col_, batch_param)); p_last_fmat_ = dmat; initialised_ = true; } - void InitData(DMatrix* dmat, RegTree const* p_tree) { + void InitData(TrainParam const* param, DMatrix* dmat, RegTree const* p_tree) { if (!initialised_) { monitor_.Start("InitDataOnce"); - this->InitDataOnce(dmat); + this->InitDataOnce(param, dmat); monitor_.Stop("InitDataOnce"); } p_last_tree_ = p_tree; @@ -856,10 +847,10 @@ class GPUHistMaker : public TreeUpdater { CHECK(*local_tree == reference_tree); } - void UpdateTree(HostDeviceVector* gpair, DMatrix* p_fmat, RegTree* p_tree, - HostDeviceVector* p_out_position) { + void UpdateTree(TrainParam const* param, HostDeviceVector* gpair, DMatrix* p_fmat, + RegTree* p_tree, HostDeviceVector* p_out_position) { monitor_.Start("InitData"); - this->InitData(p_fmat, p_tree); + this->InitData(param, p_fmat, p_tree); monitor_.Stop("InitData"); gpair->SetDevice(ctx_->gpu_id); @@ -878,7 +869,6 @@ class GPUHistMaker : public TreeUpdater { return result; } - TrainParam param_; // NOLINT MetaInfo* info_{}; // NOLINT std::unique_ptr> maker; // NOLINT diff --git a/src/tree/updater_prune.cc b/src/tree/updater_prune.cc index bec49bf47..c591ce454 100644 --- a/src/tree/updater_prune.cc +++ b/src/tree/updater_prune.cc @@ -1,5 +1,5 @@ -/*! - * Copyright 2014-2022 by XGBoost Contributors +/** + * Copyright 2014-2023 by XGBoost Contributors * \file updater_prune.cc * \brief prune a tree given the statistics * \author Tianqi Chen @@ -8,13 +8,11 @@ #include +#include "../common/timer.h" +#include "./param.h" #include "xgboost/base.h" #include "xgboost/json.h" -#include "./param.h" -#include "../common/timer.h" -namespace xgboost { -namespace tree { - +namespace xgboost::tree { DMLC_REGISTRY_FILE_TAG(updater_prune); /*! \brief pruner that prunes a tree after growing finishes */ @@ -24,47 +22,31 @@ class TreePruner : public TreeUpdater { syncher_.reset(TreeUpdater::Create("sync", ctx_, task)); pruner_monitor_.Init("TreePruner"); } - char const* Name() const override { - return "prune"; - } - + [[nodiscard]] char const* Name() const override { return "prune"; } // set training parameter - void Configure(const Args& args) override { - param_.UpdateAllowUnknown(args); - syncher_->Configure(args); - } + void Configure(const Args& args) override { syncher_->Configure(args); } - void LoadConfig(Json const& in) override { - auto const& config = get(in); - FromJson(config.at("train_param"), &this->param_); - } - void SaveConfig(Json* p_out) const override { - auto& out = *p_out; - out["train_param"] = ToJson(param_); - } - bool CanModifyTree() const override { - return true; - } + void LoadConfig(Json const&) override {} + void SaveConfig(Json*) const override {} + [[nodiscard]] bool CanModifyTree() const override { return true; } // update the tree, do pruning - void Update(HostDeviceVector* gpair, DMatrix* p_fmat, + void Update(TrainParam const* param, HostDeviceVector* gpair, DMatrix* p_fmat, common::Span> out_position, const std::vector& trees) override { pruner_monitor_.Start("PrunerUpdate"); - // rescale learning rate according to size of trees - float lr = param_.learning_rate; - param_.learning_rate = lr / trees.size(); for (auto tree : trees) { - this->DoPrune(tree); + this->DoPrune(param, tree); } - param_.learning_rate = lr; - syncher_->Update(gpair, p_fmat, out_position, trees); + syncher_->Update(param, gpair, p_fmat, out_position, trees); pruner_monitor_.Stop("PrunerUpdate"); } private: // try to prune off current leaf - bst_node_t TryPruneLeaf(RegTree &tree, int nid, int depth, int npruned) { // NOLINT(*) + bst_node_t TryPruneLeaf(TrainParam const* param, RegTree* p_tree, int nid, int depth, + int npruned) { + auto& tree = *p_tree; CHECK(tree[nid].IsLeaf()); if (tree[nid].IsRoot()) { return npruned; @@ -77,22 +59,22 @@ class TreePruner : public TreeUpdater { auto right = tree[pid].RightChild(); bool balanced = tree[left].IsLeaf() && right != RegTree::kInvalidNodeId && tree[right].IsLeaf(); - if (balanced && param_.NeedPrune(s.loss_chg, depth)) { + if (balanced && param->NeedPrune(s.loss_chg, depth)) { // need to be pruned - tree.ChangeToLeaf(pid, param_.learning_rate * s.base_weight); + tree.ChangeToLeaf(pid, param->learning_rate * s.base_weight); // tail recursion - return this->TryPruneLeaf(tree, pid, depth - 1, npruned + 2); + return this->TryPruneLeaf(param, p_tree, pid, depth - 1, npruned + 2); } else { return npruned; } } /*! \brief do pruning of a tree */ - void DoPrune(RegTree* p_tree) { + void DoPrune(TrainParam const* param, RegTree* p_tree) { auto& tree = *p_tree; bst_node_t npruned = 0; for (int nid = 0; nid < tree.param.num_nodes; ++nid) { if (tree[nid].IsLeaf() && !tree[nid].IsDeleted()) { - npruned = this->TryPruneLeaf(tree, nid, tree.GetDepth(nid), npruned); + npruned = this->TryPruneLeaf(param, p_tree, nid, tree.GetDepth(nid), npruned); } } LOG(INFO) << "tree pruning end, " @@ -103,13 +85,10 @@ class TreePruner : public TreeUpdater { private: // synchronizer std::unique_ptr syncher_; - // training parameter - TrainParam param_; common::Monitor pruner_monitor_; }; XGBOOST_REGISTER_TREE_UPDATER(TreePruner, "prune") .describe("Pruner that prune the tree according to statistics.") .set_body([](Context const* ctx, ObjInfo task) { return new TreePruner(ctx, task); }); -} // namespace tree -} // namespace xgboost +} // namespace xgboost::tree diff --git a/src/tree/updater_quantile_hist.cc b/src/tree/updater_quantile_hist.cc index e2cb820c2..ad30442f6 100644 --- a/src/tree/updater_quantile_hist.cc +++ b/src/tree/updater_quantile_hist.cc @@ -28,21 +28,14 @@ namespace tree { DMLC_REGISTRY_FILE_TAG(updater_quantile_hist); -void QuantileHistMaker::Configure(const Args &args) { - param_.UpdateAllowUnknown(args); -} - -void QuantileHistMaker::Update(HostDeviceVector *gpair, DMatrix *dmat, +void QuantileHistMaker::Update(TrainParam const *param, HostDeviceVector *gpair, + DMatrix *dmat, common::Span> out_position, const std::vector &trees) { - // rescale learning rate according to size of trees - float lr = param_.learning_rate; - param_.learning_rate = lr / trees.size(); - // build tree const size_t n_trees = trees.size(); if (!pimpl_) { - pimpl_.reset(new Builder(n_trees, param_, dmat, task_, ctx_)); + pimpl_.reset(new Builder(n_trees, param, dmat, task_, ctx_)); } size_t t_idx{0}; @@ -51,8 +44,6 @@ void QuantileHistMaker::Update(HostDeviceVector *gpair, DMatrix *d this->pimpl_->UpdateTree(gpair, dmat, p_tree, &t_row_position); ++t_idx; } - - param_.learning_rate = lr; } bool QuantileHistMaker::UpdatePredictionCache(const DMatrix *data, @@ -107,7 +98,7 @@ CPUExpandEntry QuantileHistMaker::Builder::InitRoot( auto weight = evaluator_->InitRoot(GradStats{grad_stat}); p_tree->Stat(RegTree::kRoot).sum_hess = grad_stat.GetHess(); p_tree->Stat(RegTree::kRoot).base_weight = weight; - (*p_tree)[RegTree::kRoot].SetLeaf(param_.learning_rate * weight); + (*p_tree)[RegTree::kRoot].SetLeaf(param_->learning_rate * weight); std::vector entries{node}; monitor_->Start("EvaluateSplits"); @@ -173,7 +164,7 @@ void QuantileHistMaker::Builder::ExpandTree(DMatrix *p_fmat, RegTree *p_tree, HostDeviceVector *p_out_position) { monitor_->Start(__func__); - Driver driver(param_); + Driver driver(*param_); driver.Push(this->InitRoot(p_fmat, p_tree, gpair_h)); auto const &tree = *p_tree; auto expand_set = driver.Pop(); @@ -285,7 +276,7 @@ void QuantileHistMaker::Builder::InitData(DMatrix *fmat, const RegTree &tree, auto m_gpair = linalg::MakeTensorView(*gpair, {gpair->size(), static_cast(1)}, ctx_->gpu_id); - SampleGradient(ctx_, param_, m_gpair); + SampleGradient(ctx_, *param_, m_gpair); } // store a pointer to the tree diff --git a/src/tree/updater_quantile_hist.h b/src/tree/updater_quantile_hist.h index ea7000651..f2e562691 100644 --- a/src/tree/updater_quantile_hist.h +++ b/src/tree/updater_quantile_hist.h @@ -35,49 +35,36 @@ #include "../common/partition_builder.h" #include "../common/column_matrix.h" -namespace xgboost { -namespace tree { -inline BatchParam HistBatch(TrainParam const& param) { - return {param.max_bin, param.sparse_threshold}; +namespace xgboost::tree { +inline BatchParam HistBatch(TrainParam const* param) { + return {param->max_bin, param->sparse_threshold}; } /*! \brief construct a tree using quantized feature values */ class QuantileHistMaker: public TreeUpdater { public: explicit QuantileHistMaker(Context const* ctx, ObjInfo task) : TreeUpdater(ctx), task_{task} {} - void Configure(const Args& args) override; + void Configure(const Args&) override {} - void Update(HostDeviceVector* gpair, DMatrix* dmat, + void Update(TrainParam const* param, HostDeviceVector* gpair, DMatrix* dmat, common::Span> out_position, const std::vector& trees) override; bool UpdatePredictionCache(const DMatrix *data, linalg::VectorView out_preds) override; - void LoadConfig(Json const& in) override { - auto const& config = get(in); - FromJson(config.at("train_param"), &this->param_); - } - void SaveConfig(Json* p_out) const override { - auto& out = *p_out; - out["train_param"] = ToJson(param_); - } + void LoadConfig(Json const&) override {} + void SaveConfig(Json*) const override {} - char const* Name() const override { - return "grow_quantile_histmaker"; - } - - bool HasNodePosition() const override { return true; } + [[nodiscard]] char const* Name() const override { return "grow_quantile_histmaker"; } + [[nodiscard]] bool HasNodePosition() const override { return true; } protected: - // training parameter - TrainParam param_; - // actual builder that runs the algorithm struct Builder { public: // constructor - explicit Builder(const size_t n_trees, const TrainParam& param, DMatrix const* fmat, + explicit Builder(const size_t n_trees, TrainParam const* param, DMatrix const* fmat, ObjInfo task, Context const* ctx) : n_trees_(n_trees), param_(param), @@ -115,7 +102,7 @@ class QuantileHistMaker: public TreeUpdater { private: const size_t n_trees_; - const TrainParam& param_; + TrainParam const* param_; std::shared_ptr column_sampler_{ std::make_shared()}; @@ -140,7 +127,6 @@ class QuantileHistMaker: public TreeUpdater { std::unique_ptr pimpl_; ObjInfo task_; }; -} // namespace tree -} // namespace xgboost +} // namespace xgboost::tree #endif // XGBOOST_TREE_UPDATER_QUANTILE_HIST_H_ diff --git a/src/tree/updater_refresh.cc b/src/tree/updater_refresh.cc index 864c704fa..ebda2a999 100644 --- a/src/tree/updater_refresh.cc +++ b/src/tree/updater_refresh.cc @@ -1,5 +1,5 @@ -/*! - * Copyright 2014-2022 by XGBoost Contributors +/** + * Copyright 2014-2023 by XGBoost Contributors * \file updater_refresh.cc * \brief refresh the statistics and leaf value on the tree on the dataset * \author Tianqi Chen @@ -16,8 +16,7 @@ #include "./param.h" #include "xgboost/json.h" -namespace xgboost { -namespace tree { +namespace xgboost::tree { DMLC_REGISTRY_FILE_TAG(updater_refresh); @@ -25,23 +24,14 @@ DMLC_REGISTRY_FILE_TAG(updater_refresh); class TreeRefresher : public TreeUpdater { public: explicit TreeRefresher(Context const *ctx) : TreeUpdater(ctx) {} - void Configure(const Args &args) override { param_.UpdateAllowUnknown(args); } - void LoadConfig(Json const& in) override { - auto const& config = get(in); - FromJson(config.at("train_param"), &this->param_); - } - void SaveConfig(Json* p_out) const override { - auto& out = *p_out; - out["train_param"] = ToJson(param_); - } - char const* Name() const override { - return "refresh"; - } - bool CanModifyTree() const override { - return true; - } + void Configure(const Args &) override {} + void LoadConfig(Json const &) override {} + void SaveConfig(Json *) const override {} + + [[nodiscard]] char const *Name() const override { return "refresh"; } + [[nodiscard]] bool CanModifyTree() const override { return true; } // update the tree, do pruning - void Update(HostDeviceVector *gpair, DMatrix *p_fmat, + void Update(TrainParam const *param, HostDeviceVector *gpair, DMatrix *p_fmat, common::Span> /*out_position*/, const std::vector &trees) override { if (trees.size() == 0) return; @@ -103,16 +93,11 @@ class TreeRefresher : public TreeUpdater { lazy_get_stats(); collective::Allreduce(&dmlc::BeginPtr(stemp[0])->sum_grad, stemp[0].size() * 2); - // rescale learning rate according to size of trees - float lr = param_.learning_rate; - param_.learning_rate = lr / trees.size(); int offset = 0; for (auto tree : trees) { - this->Refresh(dmlc::BeginPtr(stemp[0]) + offset, 0, tree); + this->Refresh(param, dmlc::BeginPtr(stemp[0]) + offset, 0, tree); offset += tree->param.num_nodes; } - // set learning rate back - param_.learning_rate = lr; } private: @@ -135,31 +120,27 @@ class TreeRefresher : public TreeUpdater { gstats[pid].Add(gpair[ridx]); } } - inline void Refresh(const GradStats *gstats, - int nid, RegTree *p_tree) { + inline void Refresh(TrainParam const *param, const GradStats *gstats, int nid, RegTree *p_tree) { RegTree &tree = *p_tree; tree.Stat(nid).base_weight = - static_cast(CalcWeight(param_, gstats[nid])); + static_cast(CalcWeight(*param, gstats[nid])); tree.Stat(nid).sum_hess = static_cast(gstats[nid].sum_hess); if (tree[nid].IsLeaf()) { - if (param_.refresh_leaf) { - tree[nid].SetLeaf(tree.Stat(nid).base_weight * param_.learning_rate); + if (param->refresh_leaf) { + tree[nid].SetLeaf(tree.Stat(nid).base_weight * param->learning_rate); } } else { - tree.Stat(nid).loss_chg = static_cast( - xgboost::tree::CalcGain(param_, gstats[tree[nid].LeftChild()]) + - xgboost::tree::CalcGain(param_, gstats[tree[nid].RightChild()]) - - xgboost::tree::CalcGain(param_, gstats[nid])); - this->Refresh(gstats, tree[nid].LeftChild(), p_tree); - this->Refresh(gstats, tree[nid].RightChild(), p_tree); + tree.Stat(nid).loss_chg = + static_cast(xgboost::tree::CalcGain(*param, gstats[tree[nid].LeftChild()]) + + xgboost::tree::CalcGain(*param, gstats[tree[nid].RightChild()]) - + xgboost::tree::CalcGain(*param, gstats[nid])); + this->Refresh(param, gstats, tree[nid].LeftChild(), p_tree); + this->Refresh(param, gstats, tree[nid].RightChild(), p_tree); } } - // training parameter - TrainParam param_; }; XGBOOST_REGISTER_TREE_UPDATER(TreeRefresher, "refresh") .describe("Refresher that refreshes the weight and statistics according to data.") .set_body([](Context const *ctx, ObjInfo) { return new TreeRefresher(ctx); }); -} // namespace tree -} // namespace xgboost +} // namespace xgboost::tree diff --git a/src/tree/updater_sync.cc b/src/tree/updater_sync.cc index a3f99362e..bb28bc4e6 100644 --- a/src/tree/updater_sync.cc +++ b/src/tree/updater_sync.cc @@ -1,5 +1,5 @@ -/*! - * Copyright 2014-2019 by Contributors +/** + * Copyright 2014-2013 by XBGoost Contributors * \file updater_sync.cc * \brief synchronize the tree in all distributed nodes */ @@ -13,8 +13,7 @@ #include "../common/io.h" #include "xgboost/json.h" -namespace xgboost { -namespace tree { +namespace xgboost::tree { DMLC_REGISTRY_FILE_TAG(updater_sync); @@ -30,11 +29,9 @@ class TreeSyncher : public TreeUpdater { void LoadConfig(Json const&) override {} void SaveConfig(Json*) const override {} - char const* Name() const override { - return "prune"; - } + [[nodiscard]] char const* Name() const override { return "prune"; } - void Update(HostDeviceVector*, DMatrix*, + void Update(TrainParam const*, HostDeviceVector*, DMatrix*, common::Span> /*out_position*/, const std::vector& trees) override { if (collective::GetWorldSize() == 1) return; @@ -57,5 +54,4 @@ class TreeSyncher : public TreeUpdater { XGBOOST_REGISTER_TREE_UPDATER(TreeSyncher, "sync") .describe("Syncher that synchronize the tree in all distributed nodes.") .set_body([](Context const* ctx, ObjInfo) { return new TreeSyncher(ctx); }); -} // namespace tree -} // namespace xgboost +} // namespace xgboost::tree diff --git a/tests/cpp/objective/test_regression_obj.cc b/tests/cpp/objective/test_regression_obj.cc index bfb292dc3..424f66aaf 100644 --- a/tests/cpp/objective/test_regression_obj.cc +++ b/tests/cpp/objective/test_regression_obj.cc @@ -6,8 +6,9 @@ #include #include -#include "../../../src/common/linalg_op.h" // begin,end +#include "../../../src/common/linalg_op.h" // for begin, end #include "../../../src/objective/adaptive.h" +#include "../../../src/tree/param.h" // for TrainParam #include "../helpers.h" #include "xgboost/base.h" #include "xgboost/data.h" @@ -408,9 +409,13 @@ TEST(Objective, DeclareUnifiedTest(AbsoluteError)) { h_predt[i] = labels[i] + i; } - obj->UpdateTreeLeaf(position, info, predt, 0, &tree); - ASSERT_EQ(tree[1].LeafValue(), -1); - ASSERT_EQ(tree[2].LeafValue(), -4); + tree::TrainParam param; + param.Init(Args{}); + auto lr = param.learning_rate; + + obj->UpdateTreeLeaf(position, info, param.learning_rate, predt, 0, &tree); + ASSERT_EQ(tree[1].LeafValue(), -1.0f * lr); + ASSERT_EQ(tree[2].LeafValue(), -4.0f * lr); } TEST(Objective, DeclareUnifiedTest(AbsoluteErrorLeaf)) { @@ -457,11 +462,16 @@ TEST(Objective, DeclareUnifiedTest(AbsoluteErrorLeaf)) { ASSERT_EQ(tree.GetNumLeaves(), 4); auto empty_leaf = tree[4].LeafValue(); - obj->UpdateTreeLeaf(position, info, predt, t, &tree); - ASSERT_EQ(tree[3].LeafValue(), -5); - ASSERT_EQ(tree[4].LeafValue(), empty_leaf); - ASSERT_EQ(tree[5].LeafValue(), -10); - ASSERT_EQ(tree[6].LeafValue(), -14); + + tree::TrainParam param; + param.Init(Args{}); + auto lr = param.learning_rate; + + obj->UpdateTreeLeaf(position, info, lr, predt, t, &tree); + ASSERT_EQ(tree[3].LeafValue(), -5.0f * lr); + ASSERT_EQ(tree[4].LeafValue(), empty_leaf * lr); + ASSERT_EQ(tree[5].LeafValue(), -10.0f * lr); + ASSERT_EQ(tree[6].LeafValue(), -14.0f * lr); } } diff --git a/tests/cpp/tree/hist/test_evaluate_splits.cc b/tests/cpp/tree/hist/test_evaluate_splits.cc index 984f46881..fc94f3130 100644 --- a/tests/cpp/tree/hist/test_evaluate_splits.cc +++ b/tests/cpp/tree/hist/test_evaluate_splits.cc @@ -24,7 +24,7 @@ void TestEvaluateSplits(bool force_read_by_column) { auto dmat = RandomDataGenerator(kRows, kCols, 0).Seed(3).GenerateDMatrix(); - auto evaluator = HistEvaluator{&ctx, param, dmat->Info(), sampler}; + auto evaluator = HistEvaluator{&ctx, ¶m, dmat->Info(), sampler}; common::HistCollection hist; std::vector row_gpairs = { {1.23f, 0.24f}, {0.24f, 0.25f}, {0.26f, 0.27f}, {2.27f, 0.28f}, @@ -96,7 +96,7 @@ TEST(HistEvaluator, Apply) { param.UpdateAllowUnknown(Args{{"min_child_weight", "0"}, {"reg_lambda", "0.0"}}); auto dmat = RandomDataGenerator(kNRows, kNCols, 0).Seed(3).GenerateDMatrix(); auto sampler = std::make_shared(); - auto evaluator_ = HistEvaluator{&ctx, param, dmat->Info(), sampler}; + auto evaluator_ = HistEvaluator{&ctx, ¶m, dmat->Info(), sampler}; CPUExpandEntry entry{0, 0, 10.0f}; entry.split.left_sum = GradStats{0.4, 0.6f}; @@ -123,7 +123,7 @@ TEST_F(TestPartitionBasedSplit, CPUHist) { // check the evaluator is returning the optimal split std::vector ft{FeatureType::kCategorical}; auto sampler = std::make_shared(); - HistEvaluator evaluator{&ctx, param_, info_, sampler}; + HistEvaluator evaluator{&ctx, ¶m_, info_, sampler}; evaluator.InitRoot(GradStats{total_gpair_}); RegTree tree; std::vector entries(1); @@ -153,7 +153,7 @@ auto CompareOneHotAndPartition(bool onehot) { RandomDataGenerator(kRows, kCols, 0).Seed(3).Type(ft).MaxCategory(n_cats).GenerateDMatrix(); auto sampler = std::make_shared(); - auto evaluator = HistEvaluator{&ctx, param, dmat->Info(), sampler}; + auto evaluator = HistEvaluator{&ctx, ¶m, dmat->Info(), sampler}; std::vector entries(1); for (auto const &gmat : dmat->GetBatches({32, param.sparse_threshold})) { @@ -204,7 +204,7 @@ TEST_F(TestCategoricalSplitWithMissing, HistEvaluator) { info.num_col_ = 1; info.feature_types = {FeatureType::kCategorical}; Context ctx; - auto evaluator = HistEvaluator{&ctx, param_, info, sampler}; + auto evaluator = HistEvaluator{&ctx, ¶m_, info, sampler}; evaluator.InitRoot(GradStats{parent_sum_}); std::vector entries(1); diff --git a/tests/cpp/tree/test_evaluate_splits.h b/tests/cpp/tree/test_evaluate_splits.h index 2421b8ba0..a74739faa 100644 --- a/tests/cpp/tree/test_evaluate_splits.h +++ b/tests/cpp/tree/test_evaluate_splits.h @@ -1,5 +1,5 @@ -/*! - * Copyright 2022 by XGBoost Contributors +/** + * Copyright 2022-2023 by XGBoost Contributors */ #include #include @@ -12,8 +12,7 @@ #include "../../../src/tree/split_evaluator.h" #include "../helpers.h" -namespace xgboost { -namespace tree { +namespace xgboost::tree { /** * \brief Enumerate all possible partitions for categorical split. */ @@ -151,5 +150,4 @@ class TestCategoricalSplitWithMissing : public testing::Test { ASSERT_EQ(right_sum.GetHess(), parent_sum_.GetHess() - left_sum.GetHess()); } }; -} // namespace tree -} // namespace xgboost +} // namespace xgboost::tree diff --git a/tests/cpp/tree/test_gpu_hist.cu b/tests/cpp/tree/test_gpu_hist.cu index 100a4c393..e828d1379 100644 --- a/tests/cpp/tree/test_gpu_hist.cu +++ b/tests/cpp/tree/test_gpu_hist.cu @@ -1,5 +1,5 @@ -/*! - * Copyright 2017-2022 XGBoost contributors +/** + * Copyright 2017-2023 by XGBoost contributors */ #include #include @@ -13,6 +13,7 @@ #include "../../../src/common/common.h" #include "../../../src/data/sparse_page_source.h" #include "../../../src/tree/constraints.cuh" +#include "../../../src/tree/param.h" // for TrainParam #include "../../../src/tree/updater_gpu_common.cuh" #include "../../../src/tree/updater_gpu_hist.cu" #include "../filesystem.h" // dmlc::TemporaryDirectory @@ -21,8 +22,7 @@ #include "xgboost/context.h" #include "xgboost/json.h" -namespace xgboost { -namespace tree { +namespace xgboost::tree { TEST(GpuHist, DeviceHistogram) { // Ensures that node allocates correctly after reaching `kStopGrowingSize`. dh::safe_cuda(cudaSetDevice(0)); @@ -83,11 +83,12 @@ void TestBuildHist(bool use_shared_memory_histograms) { int const kNRows = 16, kNCols = 8; TrainParam param; - std::vector> args { - {"max_depth", "6"}, - {"max_leaves", "0"}, + Args args{ + {"max_depth", "6"}, + {"max_leaves", "0"}, }; param.Init(args); + auto page = BuildEllpackPage(kNRows, kNCols); BatchParam batch_param{}; Context ctx{CreateEmptyGenericParam(0)}; @@ -168,7 +169,6 @@ void TestHistogramIndexImpl() { int constexpr kNRows = 1000, kNCols = 10; // Build 2 matrices and build a histogram maker with that - Context ctx(CreateEmptyGenericParam(0)); tree::GPUHistMaker hist_maker{&ctx, ObjInfo{ObjInfo::kRegression}}, hist_maker_ext{&ctx, ObjInfo{ObjInfo::kRegression}}; @@ -179,15 +179,14 @@ void TestHistogramIndexImpl() { std::unique_ptr hist_maker_ext_dmat( CreateSparsePageDMatrixWithRC(kNRows, kNCols, 128UL, true, tempdir)); - std::vector> training_params = { - {"max_depth", "10"}, - {"max_leaves", "0"} - }; + Args training_params = {{"max_depth", "10"}, {"max_leaves", "0"}}; + TrainParam param; + param.UpdateAllowUnknown(training_params); hist_maker.Configure(training_params); - hist_maker.InitDataOnce(hist_maker_dmat.get()); + hist_maker.InitDataOnce(¶m, hist_maker_dmat.get()); hist_maker_ext.Configure(training_params); - hist_maker_ext.InitDataOnce(hist_maker_ext_dmat.get()); + hist_maker_ext.InitDataOnce(¶m, hist_maker_ext_dmat.get()); // Extract the device maker from the histogram makers and from that its compressed // histogram index @@ -237,13 +236,15 @@ void UpdateTree(HostDeviceVector* gpair, DMatrix* dmat, {"subsample", std::to_string(subsample)}, {"sampling_method", sampling_method}, }; + TrainParam param; + param.UpdateAllowUnknown(args); Context ctx(CreateEmptyGenericParam(0)); tree::GPUHistMaker hist_maker{&ctx,ObjInfo{ObjInfo::kRegression}}; - hist_maker.Configure(args); std::vector> position(1); - hist_maker.Update(gpair, dmat, common::Span>{position}, {tree}); + hist_maker.Update(¶m, gpair, dmat, common::Span>{position}, + {tree}); auto cache = linalg::VectorView{preds->DeviceSpan(), {preds->Size()}, 0}; hist_maker.UpdatePredictionCache(dmat, cache); } @@ -391,13 +392,11 @@ TEST(GpuHist, ConfigIO) { Json j_updater { Object() }; updater->SaveConfig(&j_updater); ASSERT_TRUE(IsA(j_updater["gpu_hist_train_param"])); - ASSERT_TRUE(IsA(j_updater["train_param"])); updater->LoadConfig(j_updater); Json j_updater_roundtrip { Object() }; updater->SaveConfig(&j_updater_roundtrip); ASSERT_TRUE(IsA(j_updater_roundtrip["gpu_hist_train_param"])); - ASSERT_TRUE(IsA(j_updater_roundtrip["train_param"])); ASSERT_EQ(j_updater, j_updater_roundtrip); } @@ -414,5 +413,4 @@ TEST(GpuHist, MaxDepth) { ASSERT_THROW({learner->UpdateOneIter(0, p_mat);}, dmlc::Error); } -} // namespace tree -} // namespace xgboost +} // namespace xgboost::tree diff --git a/tests/cpp/tree/test_histmaker.cc b/tests/cpp/tree/test_histmaker.cc index 809f66c22..20340f539 100644 --- a/tests/cpp/tree/test_histmaker.cc +++ b/tests/cpp/tree/test_histmaker.cc @@ -5,11 +5,10 @@ #include #include +#include "../../../src/tree/param.h" // for TrainParam #include "../helpers.h" -namespace xgboost { -namespace tree { - +namespace xgboost::tree { std::shared_ptr GenerateDMatrix(std::size_t rows, std::size_t cols){ return RandomDataGenerator{rows, cols, 0.6f}.Seed(3).GenerateDMatrix(); } @@ -45,11 +44,11 @@ TEST(GrowHistMaker, InteractionConstraint) std::unique_ptr updater{ TreeUpdater::Create("grow_histmaker", &ctx, ObjInfo{ObjInfo::kRegression})}; - updater->Configure(Args{ - {"interaction_constraints", "[[0, 1]]"}, - {"num_feature", std::to_string(kCols)}}); + TrainParam param; + param.UpdateAllowUnknown( + Args{{"interaction_constraints", "[[0, 1]]"}, {"num_feature", std::to_string(kCols)}}); std::vector> position(1); - updater->Update(p_gradients.get(), p_dmat.get(), position, {&tree}); + updater->Update(¶m, p_gradients.get(), p_dmat.get(), position, {&tree}); ASSERT_EQ(tree.NumExtraNodes(), 4); ASSERT_EQ(tree[0].SplitIndex(), 1); @@ -64,9 +63,10 @@ TEST(GrowHistMaker, InteractionConstraint) std::unique_ptr updater{ TreeUpdater::Create("grow_histmaker", &ctx, ObjInfo{ObjInfo::kRegression})}; - updater->Configure(Args{{"num_feature", std::to_string(kCols)}}); std::vector> position(1); - updater->Update(p_gradients.get(), p_dmat.get(), position, {&tree}); + TrainParam param; + param.Init(Args{}); + updater->Update(¶m, p_gradients.get(), p_dmat.get(), position, {&tree}); ASSERT_EQ(tree.NumExtraNodes(), 10); ASSERT_EQ(tree[0].SplitIndex(), 1); @@ -83,7 +83,6 @@ void TestColumnSplit(int32_t rows, int32_t cols, RegTree const& expected_tree) { Context ctx; std::unique_ptr updater{ TreeUpdater::Create("grow_histmaker", &ctx, ObjInfo{ObjInfo::kRegression})}; - updater->Configure(Args{{"num_feature", std::to_string(cols)}}); std::vector> position(1); std::unique_ptr sliced{ @@ -91,7 +90,9 @@ void TestColumnSplit(int32_t rows, int32_t cols, RegTree const& expected_tree) { RegTree tree; tree.param.num_feature = cols; - updater->Update(p_gradients.get(), sliced.get(), position, {&tree}); + TrainParam param; + param.Init(Args{}); + updater->Update(¶m, p_gradients.get(), sliced.get(), position, {&tree}); EXPECT_EQ(tree.NumExtraNodes(), 10); EXPECT_EQ(tree[0].SplitIndex(), 1); @@ -115,14 +116,13 @@ TEST(GrowHistMaker, ColumnSplit) { Context ctx; std::unique_ptr updater{ TreeUpdater::Create("grow_histmaker", &ctx, ObjInfo{ObjInfo::kRegression})}; - updater->Configure(Args{{"num_feature", std::to_string(kCols)}}); std::vector> position(1); - updater->Update(p_gradients.get(), p_dmat.get(), position, {&expected_tree}); + TrainParam param; + param.Init(Args{}); + updater->Update(¶m, p_gradients.get(), p_dmat.get(), position, {&expected_tree}); } auto constexpr kWorldSize = 2; RunWithInMemoryCommunicator(kWorldSize, TestColumnSplit, kRows, kCols, std::cref(expected_tree)); } - -} // namespace tree -} // namespace xgboost +} // namespace xgboost::tree diff --git a/tests/cpp/tree/test_prediction_cache.cc b/tests/cpp/tree/test_prediction_cache.cc index dc41b3edd..f4e67d836 100644 --- a/tests/cpp/tree/test_prediction_cache.cc +++ b/tests/cpp/tree/test_prediction_cache.cc @@ -7,6 +7,7 @@ #include +#include "../../../src/tree/param.h" // for TrainParam #include "../helpers.h" namespace xgboost { @@ -75,9 +76,11 @@ class TestPredictionCache : public ::testing::Test { RegTree tree; std::vector trees{&tree}; auto gpair = GenerateRandomGradients(n_samples_); - updater->Configure(Args{{"max_bin", "64"}}); + tree::TrainParam param; + param.UpdateAllowUnknown(Args{{"max_bin", "64"}}); + std::vector> position(1); - updater->Update(&gpair, Xy_.get(), position, trees); + updater->Update(¶m, &gpair, Xy_.get(), position, trees); HostDeviceVector out_prediction_cached; out_prediction_cached.SetDevice(ctx.gpu_id); out_prediction_cached.Resize(n_samples_); diff --git a/tests/cpp/tree/test_prune.cc b/tests/cpp/tree/test_prune.cc index 9dd3ec30a..258396976 100644 --- a/tests/cpp/tree/test_prune.cc +++ b/tests/cpp/tree/test_prune.cc @@ -1,20 +1,20 @@ -/*! - * Copyright 2018-2019 by Contributors +/** + * Copyright 2018-2023 by XGBoost Contributors */ +#include #include #include -#include #include -#include -#include -#include -#include +#include +#include +#include +#include + +#include "../../../src/tree/param.h" // for TrainParam #include "../helpers.h" -namespace xgboost { -namespace tree { - +namespace xgboost::tree { TEST(Updater, Prune) { int constexpr kCols = 16; @@ -36,28 +36,30 @@ TEST(Updater, Prune) { tree.param.UpdateAllowUnknown(cfg); std::vector trees {&tree}; // prepare pruner + TrainParam param; + param.UpdateAllowUnknown(cfg); + std::unique_ptr pruner( TreeUpdater::Create("prune", &ctx, ObjInfo{ObjInfo::kRegression})); - pruner->Configure(cfg); // loss_chg < min_split_loss; std::vector> position(trees.size()); tree.ExpandNode(0, 0, 0, true, 0.0f, 0.3f, 0.4f, 0.0f, 0.0f, /*left_sum=*/0.0f, /*right_sum=*/0.0f); - pruner->Update(&gpair, p_dmat.get(), position, trees); + pruner->Update(¶m, &gpair, p_dmat.get(), position, trees); ASSERT_EQ(tree.NumExtraNodes(), 0); // loss_chg > min_split_loss; tree.ExpandNode(0, 0, 0, true, 0.0f, 0.3f, 0.4f, 11.0f, 0.0f, /*left_sum=*/0.0f, /*right_sum=*/0.0f); - pruner->Update(&gpair, p_dmat.get(), position, trees); + pruner->Update(¶m, &gpair, p_dmat.get(), position, trees); ASSERT_EQ(tree.NumExtraNodes(), 2); // loss_chg == min_split_loss; tree.Stat(0).loss_chg = 10; - pruner->Update(&gpair, p_dmat.get(), position, trees); + pruner->Update(¶m, &gpair, p_dmat.get(), position, trees); ASSERT_EQ(tree.NumExtraNodes(), 2); @@ -71,10 +73,10 @@ TEST(Updater, Prune) { 0, 0.5f, true, 0.3, 0.4, 0.5, /*loss_chg=*/19.0f, 0.0f, /*left_sum=*/0.0f, /*right_sum=*/0.0f); - cfg.emplace_back("max_depth", "1"); - pruner->Configure(cfg); - pruner->Update(&gpair, p_dmat.get(), position, trees); + cfg.emplace_back("max_depth", "1"); + param.UpdateAllowUnknown(cfg); + pruner->Update(¶m, &gpair, p_dmat.get(), position, trees); ASSERT_EQ(tree.NumExtraNodes(), 2); tree.ExpandNode(tree[0].LeftChild(), @@ -82,9 +84,9 @@ TEST(Updater, Prune) { /*loss_chg=*/18.0f, 0.0f, /*left_sum=*/0.0f, /*right_sum=*/0.0f); cfg.emplace_back("min_split_loss", "0"); - pruner->Configure(cfg); - pruner->Update(&gpair, p_dmat.get(), position, trees); + param.UpdateAllowUnknown(cfg); + + pruner->Update(¶m, &gpair, p_dmat.get(), position, trees); ASSERT_EQ(tree.NumExtraNodes(), 2); } -} // namespace tree -} // namespace xgboost +} // namespace xgboost::tree diff --git a/tests/cpp/tree/test_refresh.cc b/tests/cpp/tree/test_refresh.cc index 953d2eea4..870022724 100644 --- a/tests/cpp/tree/test_refresh.cc +++ b/tests/cpp/tree/test_refresh.cc @@ -1,14 +1,15 @@ -/*! - * Copyright 2018-2019 by Contributors +/** + * Copyright 2018-2013 by XGBoost Contributors */ +#include #include #include -#include -#include -#include #include +#include +#include +#include "../../../src/tree/param.h" // for TrainParam #include "../helpers.h" namespace xgboost { @@ -43,9 +44,11 @@ TEST(Updater, Refresh) { tree.Stat(cleft).base_weight = 1.2; tree.Stat(cright).base_weight = 1.3; - refresher->Configure(cfg); std::vector> position; - refresher->Update(&gpair, p_dmat.get(), position, trees); + tree::TrainParam param; + param.UpdateAllowUnknown(cfg); + + refresher->Update(¶m, &gpair, p_dmat.get(), position, trees); bst_float constexpr kEps = 1e-6; ASSERT_NEAR(-0.183392, tree[cright].LeafValue(), kEps); diff --git a/tests/cpp/tree/test_tree_stat.cc b/tests/cpp/tree/test_tree_stat.cc index 5b52534c1..4757bb3c1 100644 --- a/tests/cpp/tree/test_tree_stat.cc +++ b/tests/cpp/tree/test_tree_stat.cc @@ -1,7 +1,11 @@ +/** + * Copyright 2020-2023 by XGBoost Contributors + */ #include #include #include +#include "../../../src/tree/param.h" // for TrainParam #include "../helpers.h" namespace xgboost { @@ -21,6 +25,9 @@ class UpdaterTreeStatTest : public ::testing::Test { } void RunTest(std::string updater) { + tree::TrainParam param; + param.Init(Args{}); + Context ctx(updater == "grow_gpu_hist" ? CreateEmptyGenericParam(0) : CreateEmptyGenericParam(Context::kCpuId)); auto up = std::unique_ptr{ @@ -29,7 +36,7 @@ class UpdaterTreeStatTest : public ::testing::Test { RegTree tree; tree.param.num_feature = kCols; std::vector> position(1); - up->Update(&gpairs_, p_dmat_.get(), position, {&tree}); + up->Update(¶m, &gpairs_, p_dmat_.get(), position, {&tree}); tree.WalkTree([&tree](bst_node_t nidx) { if (tree[nidx].IsLeaf()) { @@ -69,28 +76,33 @@ class UpdaterEtaTest : public ::testing::Test { void RunTest(std::string updater) { Context ctx(updater == "grow_gpu_hist" ? CreateEmptyGenericParam(0) : CreateEmptyGenericParam(Context::kCpuId)); + float eta = 0.4; auto up_0 = std::unique_ptr{ TreeUpdater::Create(updater, &ctx, ObjInfo{ObjInfo::kClassification})}; - up_0->Configure(Args{{"eta", std::to_string(eta)}}); + up_0->Configure(Args{}); + tree::TrainParam param0; + param0.Init(Args{{"eta", std::to_string(eta)}}); auto up_1 = std::unique_ptr{ TreeUpdater::Create(updater, &ctx, ObjInfo{ObjInfo::kClassification})}; up_1->Configure(Args{{"eta", "1.0"}}); + tree::TrainParam param1; + param1.Init(Args{{"eta", "1.0"}}); for (size_t iter = 0; iter < 4; ++iter) { RegTree tree_0; { tree_0.param.num_feature = kCols; std::vector> position(1); - up_0->Update(&gpairs_, p_dmat_.get(), position, {&tree_0}); + up_0->Update(¶m0, &gpairs_, p_dmat_.get(), position, {&tree_0}); } RegTree tree_1; { tree_1.param.num_feature = kCols; std::vector> position(1); - up_1->Update(&gpairs_, p_dmat_.get(), position, {&tree_1}); + up_1->Update(¶m1, &gpairs_, p_dmat_.get(), position, {&tree_1}); } tree_0.WalkTree([&](bst_node_t nidx) { if (tree_0[nidx].IsLeaf()) { @@ -139,17 +151,18 @@ class TestMinSplitLoss : public ::testing::Test { // test gamma {"gamma", std::to_string(gamma)}}; + tree::TrainParam param; + param.UpdateAllowUnknown(args); Context ctx(updater == "grow_gpu_hist" ? CreateEmptyGenericParam(0) : CreateEmptyGenericParam(Context::kCpuId)); - std::cout << ctx.gpu_id << std::endl; auto up = std::unique_ptr{ TreeUpdater::Create(updater, &ctx, ObjInfo{ObjInfo::kRegression})}; - up->Configure(args); + up->Configure({}); RegTree tree; std::vector> position(1); - up->Update(&gpair_, dmat_.get(), position, {&tree}); + up->Update(¶m, &gpair_, dmat_.get(), position, {&tree}); auto n_nodes = tree.NumExtraNodes(); return n_nodes; diff --git a/tests/python-gpu/test_gpu_basic_models.py b/tests/python-gpu/test_gpu_basic_models.py index 83d1a2557..a6f50c224 100644 --- a/tests/python-gpu/test_gpu_basic_models.py +++ b/tests/python-gpu/test_gpu_basic_models.py @@ -42,9 +42,15 @@ class TestGPUBasicModels: def test_custom_objective(self): self.cpu_test_bm.run_custom_objective("gpu_hist") - def test_eta_decay_gpu_hist(self): + def test_eta_decay(self): self.cpu_test_cb.run_eta_decay('gpu_hist') + @pytest.mark.parametrize( + "objective", ["binary:logistic", "reg:absoluteerror", "reg:quantileerror"] + ) + def test_eta_decay_leaf_output(self, objective) -> None: + self.cpu_test_cb.run_eta_decay_leaf_output("gpu_hist", objective) + def test_deterministic_gpu_hist(self): kRows = 1000 kCols = 64 diff --git a/tests/python/test_callback.py b/tests/python/test_callback.py index 3e972345b..fabf8672e 100644 --- a/tests/python/test_callback.py +++ b/tests/python/test_callback.py @@ -1,3 +1,4 @@ +import json import os import tempfile from contextlib import nullcontext @@ -355,47 +356,125 @@ class TestCallbacks: with warning_check: xgb.cv(param, dtrain, num_round, callbacks=[scheduler(eta_decay)]) - @pytest.mark.parametrize("tree_method", ["hist", "approx", "exact"]) + def run_eta_decay_leaf_output(self, tree_method: str, objective: str) -> None: + # check decay has effect on leaf output. + num_round = 4 + scheduler = xgb.callback.LearningRateScheduler + + dpath = tm.data_dir(__file__) + dtrain = xgb.DMatrix(os.path.join(dpath, "agaricus.txt.train")) + dtest = xgb.DMatrix(os.path.join(dpath, "agaricus.txt.test")) + watchlist = [(dtest, 'eval'), (dtrain, 'train')] + + param = { + "max_depth": 2, + "objective": objective, + "eval_metric": "error", + "tree_method": tree_method, + } + if objective == "reg:quantileerror": + param["quantile_alpha"] = 0.3 + + def eta_decay_0(i): + return num_round / (i + 1) + + bst0 = xgb.train( + param, + dtrain, + num_round, + watchlist, + callbacks=[scheduler(eta_decay_0)], + ) + + def eta_decay_1(i: int) -> float: + if i > 1: + return 5.0 + return num_round / (i + 1) + + bst1 = xgb.train( + param, + dtrain, + num_round, + watchlist, + callbacks=[scheduler(eta_decay_1)], + ) + bst_json0 = bst0.save_raw(raw_format="json") + bst_json1 = bst1.save_raw(raw_format="json") + + j0 = json.loads(bst_json0) + j1 = json.loads(bst_json1) + + tree_2th_0 = j0["learner"]["gradient_booster"]["model"]["trees"][2] + tree_2th_1 = j1["learner"]["gradient_booster"]["model"]["trees"][2] + assert tree_2th_0["base_weights"] == tree_2th_1["base_weights"] + assert tree_2th_0["split_conditions"] == tree_2th_1["split_conditions"] + + tree_3th_0 = j0["learner"]["gradient_booster"]["model"]["trees"][3] + tree_3th_1 = j1["learner"]["gradient_booster"]["model"]["trees"][3] + assert tree_3th_0["base_weights"] != tree_3th_1["base_weights"] + assert tree_3th_0["split_conditions"] != tree_3th_1["split_conditions"] + + @pytest.mark.parametrize("tree_method", ["hist", "approx", "approx"]) def test_eta_decay(self, tree_method): self.run_eta_decay(tree_method) + @pytest.mark.parametrize( + "tree_method,objective", + [ + ("hist", "binary:logistic"), + ("hist", "reg:absoluteerror"), + ("hist", "reg:quantileerror"), + ("approx", "binary:logistic"), + ("approx", "reg:absoluteerror"), + ("approx", "reg:quantileerror"), + ], + ) + def test_eta_decay_leaf_output(self, tree_method: str, objective: str) -> None: + self.run_eta_decay_leaf_output(tree_method, objective) + def test_check_point(self): from sklearn.datasets import load_breast_cancer + X, y = load_breast_cancer(return_X_y=True) m = xgb.DMatrix(X, y) with tempfile.TemporaryDirectory() as tmpdir: - check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir, - iterations=1, - name='model') - xgb.train({'objective': 'binary:logistic'}, m, - num_boost_round=10, - verbose_eval=False, - callbacks=[check_point]) + check_point = xgb.callback.TrainingCheckPoint( + directory=tmpdir, iterations=1, name="model" + ) + xgb.train( + {"objective": "binary:logistic"}, + m, + num_boost_round=10, + verbose_eval=False, + callbacks=[check_point], + ) for i in range(1, 10): - assert os.path.exists( - os.path.join(tmpdir, 'model_' + str(i) + '.json')) + assert os.path.exists(os.path.join(tmpdir, "model_" + str(i) + ".json")) - check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir, - iterations=1, - as_pickle=True, - name='model') - xgb.train({'objective': 'binary:logistic'}, m, - num_boost_round=10, - verbose_eval=False, - callbacks=[check_point]) + check_point = xgb.callback.TrainingCheckPoint( + directory=tmpdir, iterations=1, as_pickle=True, name="model" + ) + xgb.train( + {"objective": "binary:logistic"}, + m, + num_boost_round=10, + verbose_eval=False, + callbacks=[check_point], + ) for i in range(1, 10): - assert os.path.exists( - os.path.join(tmpdir, 'model_' + str(i) + '.pkl')) + assert os.path.exists(os.path.join(tmpdir, "model_" + str(i) + ".pkl")) def test_callback_list(self): X, y = tm.get_california_housing() m = xgb.DMatrix(X, y) callbacks = [xgb.callback.EarlyStopping(rounds=10)] for i in range(4): - xgb.train({'objective': 'reg:squarederror', - 'eval_metric': 'rmse'}, m, - evals=[(m, 'Train')], - num_boost_round=1, - verbose_eval=True, - callbacks=callbacks) + xgb.train( + {"objective": "reg:squarederror", "eval_metric": "rmse"}, + m, + evals=[(m, "Train")], + num_boost_round=1, + verbose_eval=True, + callbacks=callbacks, + ) assert len(callbacks) == 1 diff --git a/tests/python/test_pickling.py b/tests/python/test_pickling.py index 161a5fd4e..2f4d77bf0 100644 --- a/tests/python/test_pickling.py +++ b/tests/python/test_pickling.py @@ -51,11 +51,8 @@ class TestPickling: def test_model_pickling_json(self): def check(config): - updater = config["learner"]["gradient_booster"]["updater"] - if params["tree_method"] == "exact": - subsample = updater["grow_colmaker"]["train_param"]["subsample"] - else: - subsample = updater["grow_quantile_histmaker"]["train_param"]["subsample"] + tree_param = config["learner"]["gradient_booster"]["tree_train_param"] + subsample = tree_param["subsample"] assert float(subsample) == 0.5 params = {"nthread": 8, "tree_method": "hist", "subsample": 0.5} diff --git a/tests/python/test_updaters.py b/tests/python/test_updaters.py index c54fb9267..be72793e7 100644 --- a/tests/python/test_updaters.py +++ b/tests/python/test_updaters.py @@ -447,7 +447,8 @@ class TestTreeMethod: { "tree_method": tree_method, "objective": "reg:absoluteerror", - "subsample": 0.8 + "subsample": 0.8, + "eta": 1.0, }, Xy, num_boost_round=10, diff --git a/tests/python/test_with_sklearn.py b/tests/python/test_with_sklearn.py index 2f08f1951..bc7a3e94e 100644 --- a/tests/python/test_with_sklearn.py +++ b/tests/python/test_with_sklearn.py @@ -1018,14 +1018,18 @@ def test_XGBClassifier_resume(): def test_constraint_parameters(): - reg = xgb.XGBRegressor(interaction_constraints='[[0, 1], [2, 3, 4]]') + reg = xgb.XGBRegressor(interaction_constraints="[[0, 1], [2, 3, 4]]") X = np.random.randn(10, 10) y = np.random.randn(10) reg.fit(X, y) config = json.loads(reg.get_booster().save_config()) - assert config['learner']['gradient_booster']['updater']['grow_colmaker'][ - 'train_param']['interaction_constraints'] == '[[0, 1], [2, 3, 4]]' + assert ( + config["learner"]["gradient_booster"]["tree_train_param"][ + "interaction_constraints" + ] + == "[[0, 1], [2, 3, 4]]" + ) def test_parameter_validation(): diff --git a/tests/test_distributed/test_with_spark/test_spark_local_cluster.py b/tests/test_distributed/test_with_spark/test_spark_local_cluster.py index 56003082f..528b770ff 100644 --- a/tests/test_distributed/test_with_spark/test_spark_local_cluster.py +++ b/tests/test_distributed/test_with_spark/test_spark_local_cluster.py @@ -422,10 +422,10 @@ class XgboostLocalClusterTestCase(SparkLocalClusterTestCase): self.assertTrue(hasattr(classifier, "max_depth")) self.assertEqual(classifier.getOrDefault(classifier.max_depth), 7) booster_config = json.loads(model.get_booster().save_config()) - max_depth = booster_config["learner"]["gradient_booster"]["updater"][ - "grow_histmaker" - ]["train_param"]["max_depth"] - self.assertEqual(int(max_depth), 7) + max_depth = booster_config["learner"]["gradient_booster"]["tree_train_param"][ + "max_depth" + ] + assert int(max_depth) == 7 def test_repartition(self): # The following test case has a few partitioned datasets that are either