Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support learning rate for zero-hessian objectives. (#8866)

Browse Source
This commit is contained in:
Jiaming Yuan 2023-03-06 20:33:28 +08:00 committed by GitHub
parent 173096a6a7
commit 228a46e8ad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 464 additions and 434 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
demo/guide-python/quantile_regression.py
View File

@ -53,9 +53,8 @@ def quantile_loss(args: argparse.Namespace) -> None:
"tree_method": "hist",
"quantile_alpha": alpha,
# Let's try not to overfit.
"learning_rate": 0.01,
"max_depth": 3,
"min_child_weight": 16.0,
"learning_rate": 0.04,
"max_depth": 5,
},
Xy,
num_boost_round=32,
@ -80,9 +79,8 @@ def quantile_loss(args: argparse.Namespace) -> None:
"objective": "reg:squarederror",
"tree_method": "hist",
# Let's try not to overfit.
"learning_rate": 0.01,
"max_depth": 3,
"min_child_weight": 16.0,
"learning_rate": 0.04,
"max_depth": 5,
},
Xy,
num_boost_round=32,

3
include/xgboost/objective.h
View File

@ -116,12 +116,13 @@ class ObjFunction : public Configurable {
*
* \param position The leaf index for each rows.
* \param info MetaInfo providing labels and weights.
* \param learning_rate The learning rate for current iteration.
* \param prediction Model prediction after transformation.
* \param group_idx The group index for this tree, 0 when it's not multi-target or multi-class.
* \param p_tree Tree that needs to be updated.
*/
virtual void UpdateTreeLeaf(HostDeviceVector<bst_node_t> const& /*position*/,
MetaInfo const& /*info*/,
MetaInfo const& /*info*/, float /*learning_rate*/,
HostDeviceVector<float> const& /*prediction*/,
std::int32_t /*group_idx*/, RegTree* /*p_tree*/) const {}

11
include/xgboost/tree_updater.h
View File

@ -24,6 +24,9 @@
#include <vector>
namespace xgboost {
namespace tree {
struct TrainParam;
}
class Json;
struct Context;
@ -56,8 +59,10 @@ class TreeUpdater : public Configurable {
* tree can be used.
*/
virtual bool HasNodePosition() const { return false; }
/*!
/**
* \brief perform update to the tree models
*
* \param param Hyper-parameter for constructing trees.
* \param gpair the gradient pair statistics of the data
* \param data The data matrix passed to the updater.
* \param out_position The leaf index for each row. The index is negated if that row is
@ -67,8 +72,8 @@ class TreeUpdater : public Configurable {
* but maybe different random seeds, usually one tree is passed in at a time,
* there can be multiple trees when we train random forest style model
*/
virtual void Update(HostDeviceVector<GradientPair>* gpair, DMatrix* data,
common::Span<HostDeviceVector<bst_node_t>> out_position,
virtual void Update(tree::TrainParam const* param, HostDeviceVector<GradientPair>* gpair,
DMatrix* data, common::Span<HostDeviceVector<bst_node_t>> out_position,
const std::vector<RegTree*>& out_trees) = 0;
/*!

25
src/gbm/gbtree.cc
View File

@ -32,15 +32,14 @@
#include "xgboost/string_view.h"
#include "xgboost/tree_updater.h"
namespace xgboost {
namespace gbm {
namespace xgboost::gbm {
DMLC_REGISTRY_FILE_TAG(gbtree);
void GBTree::Configure(const Args& cfg) {
void GBTree::Configure(Args const& cfg) {
this->cfg_ = cfg;
std::string updater_seq = tparam_.updater_seq;
tparam_.UpdateAllowUnknown(cfg);
tree_param_.UpdateAllowUnknown(cfg);
model_.Configure(cfg);
@ -235,9 +234,11 @@ void GBTree::UpdateTreeLeaf(DMatrix const* p_fmat, HostDeviceVector<float> const
CHECK_EQ(model_.param.num_parallel_tree, trees.size());
CHECK_EQ(model_.param.num_parallel_tree, 1)
<< "Boosting random forest is not supported for current objective.";
CHECK_EQ(trees.size(), model_.param.num_parallel_tree);
for (std::size_t tree_idx = 0; tree_idx < trees.size(); ++tree_idx) {
auto const& position = node_position.at(tree_idx);
obj->UpdateTreeLeaf(position, p_fmat->Info(), predictions, group_idx, trees[tree_idx].get());
obj->UpdateTreeLeaf(position, p_fmat->Info(), tree_param_.learning_rate / trees.size(),
predictions, group_idx, trees[tree_idx].get());
}
}
@ -388,9 +389,15 @@ void GBTree::BoostNewTrees(HostDeviceVector<GradientPair>* gpair, DMatrix* p_fma
CHECK(out_position);
out_position->resize(new_trees.size());
// Rescale learning rate according to the size of trees
auto lr = tree_param_.learning_rate;
tree_param_.learning_rate /= static_cast<float>(new_trees.size());
for (auto& up : updaters_) {
up->Update(gpair, p_fmat, common::Span<HostDeviceVector<bst_node_t>>{*out_position}, new_trees);
up->Update(&tree_param_, gpair, p_fmat,
common::Span<HostDeviceVector<bst_node_t>>{*out_position}, new_trees);
}
tree_param_.learning_rate = lr;
}
void GBTree::CommitModel(std::vector<std::vector<std::unique_ptr<RegTree>>>&& new_trees) {
@ -404,6 +411,8 @@ void GBTree::CommitModel(std::vector<std::vector<std::unique_ptr<RegTree>>>&& ne
void GBTree::LoadConfig(Json const& in) {
CHECK_EQ(get<String>(in["name"]), "gbtree");
FromJson(in["gbtree_train_param"], &tparam_);
FromJson(in["tree_train_param"], &tree_param_);
// Process type cannot be kUpdate from loaded model
// This would cause all trees to be pushed to trees_to_update
// e.g. updating a model, then saving and loading it would result in an empty model
@ -451,6 +460,7 @@ void GBTree::SaveConfig(Json* p_out) const {
auto& out = *p_out;
out["name"] = String("gbtree");
out["gbtree_train_param"] = ToJson(tparam_);
out["tree_train_param"] = ToJson(tree_param_);
// Process type cannot be kUpdate from loaded model
// This would cause all trees to be pushed to trees_to_update
@ -1058,5 +1068,4 @@ XGBOOST_REGISTER_GBM(Dart, "dart")
GBTree* p = new Dart(booster_config, ctx);
return p;
});
} // namespace gbm
} // namespace xgboost
} // namespace xgboost::gbm

7
src/gbm/gbtree.h
View File

@ -20,6 +20,7 @@
#include "../common/common.h"
#include "../common/timer.h"
#include "../tree/param.h" // TrainParam
#include "gbtree_model.h"
#include "xgboost/base.h"
#include "xgboost/data.h"
@ -405,8 +406,8 @@ class GBTree : public GradientBooster {
p_fmat, out_contribs, model_, tree_end, nullptr, approximate);
}
std::vector<std::string> DumpModel(const FeatureMap& fmap, bool with_stats,
std::string format) const override {
[[nodiscard]] std::vector<std::string> DumpModel(const FeatureMap& fmap, bool with_stats,
std::string format) const override {
return model_.DumpModel(fmap, with_stats, this->ctx_->Threads(), format);
}
@ -428,6 +429,8 @@ class GBTree : public GradientBooster {
GBTreeModel model_;
// training parameter
GBTreeTrainParam tparam_;
// Tree training parameter
tree::TrainParam tree_param_;
// ----training fields----
bool showed_updater_warning_ {false};
bool specified_updater_ {false};

9
src/objective/adaptive.cc
View File

@ -76,7 +76,7 @@ void EncodeTreeLeafHost(Context const* ctx, RegTree const& tree,
}
void UpdateTreeLeafHost(Context const* ctx, std::vector<bst_node_t> const& position,
std::int32_t group_idx, MetaInfo const& info,
std::int32_t group_idx, MetaInfo const& info, float learning_rate,
HostDeviceVector<float> const& predt, float alpha, RegTree* p_tree) {
auto& tree = *p_tree;
@ -87,7 +87,7 @@ void UpdateTreeLeafHost(Context const* ctx, std::vector<bst_node_t> const& posit
size_t n_leaf = nidx.size();
if (nptr.empty()) {
std::vector<float> quantiles;
UpdateLeafValues(&quantiles, nidx, p_tree);
UpdateLeafValues(&quantiles, nidx, learning_rate, p_tree);
return;
}
@ -133,12 +133,13 @@ void UpdateTreeLeafHost(Context const* ctx, std::vector<bst_node_t> const& posit
quantiles.at(k) = q;
});
UpdateLeafValues(&quantiles, nidx, p_tree);
UpdateLeafValues(&quantiles, nidx, learning_rate, p_tree);
}
#if !defined(XGBOOST_USE_CUDA)
void UpdateTreeLeafDevice(Context const*, common::Span<bst_node_t const>, std::int32_t,
MetaInfo const&, HostDeviceVector<float> const&, float, RegTree*) {
MetaInfo const&, float learning_rate, HostDeviceVector<float> const&,
float, RegTree*) {
common::AssertGPUSupport();
}
#endif // !defined(XGBOOST_USE_CUDA)

6
src/objective/adaptive.cu
View File

@ -140,7 +140,7 @@ void EncodeTreeLeafDevice(Context const* ctx, common::Span<bst_node_t const> pos
}
void UpdateTreeLeafDevice(Context const* ctx, common::Span<bst_node_t const> position,
std::int32_t group_idx, MetaInfo const& info,
std::int32_t group_idx, MetaInfo const& info, float learning_rate,
HostDeviceVector<float> const& predt, float alpha, RegTree* p_tree) {
dh::safe_cuda(cudaSetDevice(ctx->gpu_id));
dh::device_vector<size_t> ridx;
@ -151,7 +151,7 @@ void UpdateTreeLeafDevice(Context const* ctx, common::Span<bst_node_t const> pos
if (nptr.Empty()) {
std::vector<float> quantiles;
UpdateLeafValues(&quantiles, nidx.ConstHostVector(), p_tree);
UpdateLeafValues(&quantiles, nidx.ConstHostVector(), learning_rate, p_tree);
}
HostDeviceVector<float> quantiles;
@ -186,7 +186,7 @@ void UpdateTreeLeafDevice(Context const* ctx, common::Span<bst_node_t const> pos
w_it + d_weights.size(), &quantiles);
}
UpdateLeafValues(&quantiles.HostVector(), nidx.ConstHostVector(), p_tree);
UpdateLeafValues(&quantiles.HostVector(), nidx.ConstHostVector(), learning_rate, p_tree);
}
} // namespace detail
} // namespace obj

18
src/objective/adaptive.h
View File

@ -36,7 +36,7 @@ inline void FillMissingLeaf(std::vector<bst_node_t> const& maybe_missing,
}
inline void UpdateLeafValues(std::vector<float>* p_quantiles, std::vector<bst_node_t> const& nidx,
RegTree* p_tree) {
float learning_rate, RegTree* p_tree) {
auto& tree = *p_tree;
auto& quantiles = *p_quantiles;
auto const& h_node_idx = nidx;
@ -71,7 +71,7 @@ inline void UpdateLeafValues(std::vector<float>* p_quantiles, std::vector<bst_no
auto nidx = h_node_idx[i];
auto q = quantiles[i];
CHECK(tree[nidx].IsLeaf());
tree[nidx].SetLeaf(q);
tree[nidx].SetLeaf(q * learning_rate);
}
}
@ -85,24 +85,24 @@ inline std::size_t IdxY(MetaInfo const& info, bst_group_t group_idx) {
}
void UpdateTreeLeafDevice(Context const* ctx, common::Span<bst_node_t const> position,
std::int32_t group_idx, MetaInfo const& info,
std::int32_t group_idx, MetaInfo const& info, float learning_rate,
HostDeviceVector<float> const& predt, float alpha, RegTree* p_tree);
void UpdateTreeLeafHost(Context const* ctx, std::vector<bst_node_t> const& position,
std::int32_t group_idx, MetaInfo const& info,
std::int32_t group_idx, MetaInfo const& info, float learning_rate,
HostDeviceVector<float> const& predt, float alpha, RegTree* p_tree);
} // namespace detail
inline void UpdateTreeLeaf(Context const* ctx, HostDeviceVector<bst_node_t> const& position,
std::int32_t group_idx, MetaInfo const& info,
std::int32_t group_idx, MetaInfo const& info, float learning_rate,
HostDeviceVector<float> const& predt, float alpha, RegTree* p_tree) {
if (ctx->IsCPU()) {
detail::UpdateTreeLeafHost(ctx, position.ConstHostVector(), group_idx, info, predt, alpha,
p_tree);
detail::UpdateTreeLeafHost(ctx, position.ConstHostVector(), group_idx, info, learning_rate,
predt, alpha, p_tree);
} else {
position.SetDevice(ctx->gpu_id);
detail::UpdateTreeLeafDevice(ctx, position.ConstDeviceSpan(), group_idx, info, predt, alpha,
p_tree);
detail::UpdateTreeLeafDevice(ctx, position.ConstDeviceSpan(), group_idx, info, learning_rate,
predt, alpha, p_tree);
}
}
} // namespace obj

7
src/objective/quantile_obj.cu
View File

@ -183,10 +183,11 @@ class QuantileRegression : public ObjFunction {
}
void UpdateTreeLeaf(HostDeviceVector<bst_node_t> const& position, MetaInfo const& info,
HostDeviceVector<float> const& prediction, std::int32_t group_idx,
RegTree* p_tree) const override {
float learning_rate, HostDeviceVector<float> const& prediction,
std::int32_t group_idx, RegTree* p_tree) const override {
auto alpha = param_.quantile_alpha[group_idx];
::xgboost::obj::UpdateTreeLeaf(ctx_, position, group_idx, info, prediction, alpha, p_tree);
::xgboost::obj::UpdateTreeLeaf(ctx_, position, group_idx, info, learning_rate, prediction,
alpha, p_tree);
}
void Configure(Args const& args) override {

7
src/objective/regression_obj.cu
View File

@ -742,9 +742,10 @@ class MeanAbsoluteError : public ObjFunction {
}
void UpdateTreeLeaf(HostDeviceVector<bst_node_t> const& position, MetaInfo const& info,
HostDeviceVector<float> const& prediction, std::int32_t group_idx,
RegTree* p_tree) const override {
::xgboost::obj::UpdateTreeLeaf(ctx_, position, group_idx, info, prediction, 0.5, p_tree);
float learning_rate, HostDeviceVector<float> const& prediction,
std::int32_t group_idx, RegTree* p_tree) const override {
::xgboost::obj::UpdateTreeLeaf(ctx_, position, group_idx, info, learning_rate, prediction, 0.5,
p_tree);
}
const char* DefaultEvalMetric() const override { return "mae"; }

92
src/tree/hist/evaluate_splits.h
View File

@ -17,13 +17,11 @@
#include "../../common/random.h"
#include "../../data/gradient_index.h"
#include "../constraints.h"
#include "../param.h"
#include "../param.h" // for TrainParam
#include "../split_evaluator.h"
#include "xgboost/context.h"
namespace xgboost {
namespace tree {
namespace xgboost::tree {
template <typename ExpandEntry>
class HistEvaluator {
private:
@ -36,7 +34,7 @@ class HistEvaluator {
private:
Context const* ctx_;
TrainParam param_;
TrainParam const* param_;
std::shared_ptr<common::ColumnSampler> column_sampler_;
TreeEvaluator tree_evaluator_;
bool is_col_split_{false};
@ -55,8 +53,9 @@ class HistEvaluator {
}
}
bool IsValid(GradStats const &left, GradStats const &right) const {
return left.GetHess() >= param_.min_child_weight && right.GetHess() >= param_.min_child_weight;
[[nodiscard]] bool IsValid(GradStats const &left, GradStats const &right) const {
return left.GetHess() >= param_->min_child_weight &&
right.GetHess() >= param_->min_child_weight;
}
/**
@ -95,9 +94,10 @@ class HistEvaluator {
right_sum = GradStats{hist[i]};
left_sum.SetSubstract(parent.stats, right_sum);
if (IsValid(left_sum, right_sum)) {
auto missing_left_chg = static_cast<float>(
evaluator.CalcSplitGain(param_, nidx, fidx, GradStats{left_sum}, GradStats{right_sum}) -
parent.root_gain);
auto missing_left_chg =
static_cast<float>(evaluator.CalcSplitGain(*param_, nidx, fidx, GradStats{left_sum},
GradStats{right_sum}) -
parent.root_gain);
best.Update(missing_left_chg, fidx, split_pt, true, true, left_sum, right_sum);
}
@ -105,9 +105,10 @@ class HistEvaluator {
right_sum.Add(missing);
left_sum.SetSubstract(parent.stats, right_sum);
if (IsValid(left_sum, right_sum)) {
auto missing_right_chg = static_cast<float>(
evaluator.CalcSplitGain(param_, nidx, fidx, GradStats{left_sum}, GradStats{right_sum}) -
parent.root_gain);
auto missing_right_chg =
static_cast<float>(evaluator.CalcSplitGain(*param_, nidx, fidx, GradStats{left_sum},
GradStats{right_sum}) -
parent.root_gain);
best.Update(missing_right_chg, fidx, split_pt, false, true, left_sum, right_sum);
}
}
@ -152,7 +153,7 @@ class HistEvaluator {
bst_bin_t f_begin = cut_ptr[fidx];
bst_bin_t f_end = cut_ptr[fidx + 1];
bst_bin_t n_bins_feature{f_end - f_begin};
auto n_bins = std::min(param_.max_cat_threshold, n_bins_feature);
auto n_bins = std::min(param_->max_cat_threshold, n_bins_feature);
// statistics on both sides of split
GradStats left_sum;
@ -181,9 +182,9 @@ class HistEvaluator {
right_sum.SetSubstract(parent.stats, left_sum); // missing on right
}
if (IsValid(left_sum, right_sum)) {
auto loss_chg =
evaluator.CalcSplitGain(param_, nidx, fidx, GradStats{left_sum}, GradStats{right_sum}) -
parent.root_gain;
auto loss_chg = evaluator.CalcSplitGain(*param_, nidx, fidx, GradStats{left_sum},
GradStats{right_sum}) -
parent.root_gain;
// We don't have a numeric split point, nan here is a dummy split.
if (best.Update(loss_chg, fidx, std::numeric_limits<float>::quiet_NaN(), d_step == 1, true,
left_sum, right_sum)) {
@ -256,7 +257,7 @@ class HistEvaluator {
if (d_step > 0) {
// forward enumeration: split at right bound of each bin
loss_chg =
static_cast<float>(evaluator.CalcSplitGain(param_, nidx, fidx, GradStats{left_sum},
static_cast<float>(evaluator.CalcSplitGain(*param_, nidx, fidx, GradStats{left_sum},
GradStats{right_sum}) -
parent.root_gain);
split_pt = cut_val[i]; // not used for partition based
@ -264,7 +265,7 @@ class HistEvaluator {
} else {
// backward enumeration: split at left bound of each bin
loss_chg =
static_cast<float>(evaluator.CalcSplitGain(param_, nidx, fidx, GradStats{right_sum},
static_cast<float>(evaluator.CalcSplitGain(*param_, nidx, fidx, GradStats{right_sum},
GradStats{left_sum}) -
parent.root_gain);
if (i == imin) {
@ -326,7 +327,7 @@ class HistEvaluator {
}
if (is_cat) {
auto n_bins = cut_ptrs.at(fidx + 1) - cut_ptrs[fidx];
if (common::UseOneHot(n_bins, param_.max_cat_to_onehot)) {
if (common::UseOneHot(n_bins, param_->max_cat_to_onehot)) {
EnumerateOneHot(cut, histogram, fidx, nidx, evaluator, best);
} else {
std::vector<size_t> sorted_idx(n_bins);
@ -334,8 +335,8 @@ class HistEvaluator {
auto feat_hist = histogram.subspan(cut_ptrs[fidx], n_bins);
// Sort the histogram to get contiguous partitions.
std::stable_sort(sorted_idx.begin(), sorted_idx.end(), [&](size_t l, size_t r) {
auto ret = evaluator.CalcWeightCat(param_, feat_hist[l]) <
evaluator.CalcWeightCat(param_, feat_hist[r]);
auto ret = evaluator.CalcWeightCat(*param_, feat_hist[l]) <
evaluator.CalcWeightCat(*param_, feat_hist[r]);
return ret;
});
EnumeratePart<+1>(cut, sorted_idx, histogram, fidx, nidx, evaluator, best);
@ -382,24 +383,22 @@ class HistEvaluator {
GradStats parent_sum = candidate.split.left_sum;
parent_sum.Add(candidate.split.right_sum);
auto base_weight =
evaluator.CalcWeight(candidate.nid, param_, GradStats{parent_sum});
auto base_weight = evaluator.CalcWeight(candidate.nid, *param_, GradStats{parent_sum});
auto left_weight =
evaluator.CalcWeight(candidate.nid, param_, GradStats{candidate.split.left_sum});
evaluator.CalcWeight(candidate.nid, *param_, GradStats{candidate.split.left_sum});
auto right_weight =
evaluator.CalcWeight(candidate.nid, param_, GradStats{candidate.split.right_sum});
evaluator.CalcWeight(candidate.nid, *param_, GradStats{candidate.split.right_sum});
if (candidate.split.is_cat) {
tree.ExpandCategorical(
candidate.nid, candidate.split.SplitIndex(), candidate.split.cat_bits,
candidate.split.DefaultLeft(), base_weight, left_weight * param_.learning_rate,
right_weight * param_.learning_rate, candidate.split.loss_chg, parent_sum.GetHess(),
candidate.split.DefaultLeft(), base_weight, left_weight * param_->learning_rate,
right_weight * param_->learning_rate, candidate.split.loss_chg, parent_sum.GetHess(),
candidate.split.left_sum.GetHess(), candidate.split.right_sum.GetHess());
} else {
tree.ExpandNode(candidate.nid, candidate.split.SplitIndex(), candidate.split.split_value,
candidate.split.DefaultLeft(), base_weight,
left_weight * param_.learning_rate, right_weight * param_.learning_rate,
left_weight * param_->learning_rate, right_weight * param_->learning_rate,
candidate.split.loss_chg, parent_sum.GetHess(),
candidate.split.left_sum.GetHess(), candidate.split.right_sum.GetHess());
}
@ -415,11 +414,11 @@ class HistEvaluator {
max_node = std::max(candidate.nid, max_node);
snode_.resize(tree.GetNodes().size());
snode_.at(left_child).stats = candidate.split.left_sum;
snode_.at(left_child).root_gain = evaluator.CalcGain(
candidate.nid, param_, GradStats{candidate.split.left_sum});
snode_.at(left_child).root_gain =
evaluator.CalcGain(candidate.nid, *param_, GradStats{candidate.split.left_sum});
snode_.at(right_child).stats = candidate.split.right_sum;
snode_.at(right_child).root_gain = evaluator.CalcGain(
candidate.nid, param_, GradStats{candidate.split.right_sum});
snode_.at(right_child).root_gain =
evaluator.CalcGain(candidate.nid, *param_, GradStats{candidate.split.right_sum});
interaction_constraints_.Split(candidate.nid,
tree[candidate.nid].SplitIndex(), left_child,
@ -429,31 +428,31 @@ class HistEvaluator {
auto Evaluator() const { return tree_evaluator_.GetEvaluator(); }
auto const& Stats() const { return snode_; }
float InitRoot(GradStats const& root_sum) {
float InitRoot(GradStats const &root_sum) {
snode_.resize(1);
auto root_evaluator = tree_evaluator_.GetEvaluator();
snode_[0].stats = GradStats{root_sum.GetGrad(), root_sum.GetHess()};
snode_[0].root_gain = root_evaluator.CalcGain(RegTree::kRoot, param_,
GradStats{snode_[0].stats});
auto weight = root_evaluator.CalcWeight(RegTree::kRoot, param_,
GradStats{snode_[0].stats});
snode_[0].root_gain =
root_evaluator.CalcGain(RegTree::kRoot, *param_, GradStats{snode_[0].stats});
auto weight = root_evaluator.CalcWeight(RegTree::kRoot, *param_, GradStats{snode_[0].stats});
return weight;
}
public:
// The column sampler must be constructed by caller since we need to preserve the rng
// for the entire training session.
explicit HistEvaluator(Context const* ctx, TrainParam const &param, MetaInfo const &info,
explicit HistEvaluator(Context const *ctx, TrainParam const *param, MetaInfo const &info,
std::shared_ptr<common::ColumnSampler> sampler)
: ctx_{ctx}, param_{param},
: ctx_{ctx},
param_{param},
column_sampler_{std::move(sampler)},
tree_evaluator_{param, static_cast<bst_feature_t>(info.num_col_), Context::kCpuId},
tree_evaluator_{*param, static_cast<bst_feature_t>(info.num_col_), Context::kCpuId},
is_col_split_{info.data_split_mode == DataSplitMode::kCol} {
interaction_constraints_.Configure(param, info.num_col_);
interaction_constraints_.Configure(*param, info.num_col_);
column_sampler_->Init(ctx, info.num_col_, info.feature_weights.HostVector(),
param_.colsample_bynode, param_.colsample_bylevel,
param_.colsample_bytree);
param_->colsample_bynode, param_->colsample_bylevel,
param_->colsample_bytree);
}
};
@ -488,6 +487,5 @@ void UpdatePredictionCacheImpl(Context const *ctx, RegTree const *p_last_tree,
});
}
}
} // namespace tree
} // namespace xgboost
} // namespace xgboost::tree
#endif // XGBOOST_TREE_HIST_EVALUATE_SPLITS_H_

55
src/tree/updater_approx.cc
View File

@ -23,8 +23,7 @@
#include "xgboost/tree_model.h"
#include "xgboost/tree_updater.h"
namespace xgboost {
namespace tree {
namespace xgboost::tree {
DMLC_REGISTRY_FILE_TAG(updater_approx);
@ -41,7 +40,7 @@ auto BatchSpec(TrainParam const &p, common::Span<float> hess) {
class GloablApproxBuilder {
protected:
TrainParam param_;
TrainParam const* param_;
std::shared_ptr<common::ColumnSampler> col_sampler_;
HistEvaluator<CPUExpandEntry> evaluator_;
HistogramBuilder<CPUExpandEntry> histogram_builder_;
@ -64,7 +63,7 @@ class GloablApproxBuilder {
bst_bin_t n_total_bins = 0;
partitioner_.clear();
// Generating the GHistIndexMatrix is quite slow, is there a way to speed it up?
for (auto const &page : p_fmat->GetBatches<GHistIndexMatrix>(BatchSpec(param_, hess, task_))) {
for (auto const &page : p_fmat->GetBatches<GHistIndexMatrix>(BatchSpec(*param_, hess, task_))) {
if (n_total_bins == 0) {
n_total_bins = page.cut.TotalBins();
feature_values_ = page.cut;
@ -75,7 +74,7 @@ class GloablApproxBuilder {
n_batches_++;
}
histogram_builder_.Reset(n_total_bins, BatchSpec(param_, hess), ctx_->Threads(), n_batches_,
histogram_builder_.Reset(n_total_bins, BatchSpec(*param_, hess), ctx_->Threads(), n_batches_,
collective::IsDistributed(), p_fmat->IsColumnSplit());
monitor_->Stop(__func__);
}
@ -96,7 +95,7 @@ class GloablApproxBuilder {
std::vector<CPUExpandEntry> nodes{best};
size_t i = 0;
auto space = ConstructHistSpace(partitioner_, nodes);
for (auto const &page : p_fmat->GetBatches<GHistIndexMatrix>(BatchSpec(param_, hess))) {
for (auto const &page : p_fmat->GetBatches<GHistIndexMatrix>(BatchSpec(*param_, hess))) {
histogram_builder_.BuildHist(i, space, page, p_tree, partitioner_.at(i).Partitions(), nodes,
{}, gpair);
i++;
@ -105,7 +104,7 @@ class GloablApproxBuilder {
auto weight = evaluator_.InitRoot(root_sum);
p_tree->Stat(RegTree::kRoot).sum_hess = root_sum.GetHess();
p_tree->Stat(RegTree::kRoot).base_weight = weight;
(*p_tree)[RegTree::kRoot].SetLeaf(param_.learning_rate * weight);
(*p_tree)[RegTree::kRoot].SetLeaf(param_->learning_rate * weight);
auto const &histograms = histogram_builder_.Histogram();
auto ft = p_fmat->Info().feature_types.ConstHostSpan();
@ -147,7 +146,7 @@ class GloablApproxBuilder {
size_t i = 0;
auto space = ConstructHistSpace(partitioner_, nodes_to_build);
for (auto const &page : p_fmat->GetBatches<GHistIndexMatrix>(BatchSpec(param_, hess))) {
for (auto const &page : p_fmat->GetBatches<GHistIndexMatrix>(BatchSpec(*param_, hess))) {
histogram_builder_.BuildHist(i, space, page, p_tree, partitioner_.at(i).Partitions(),
nodes_to_build, nodes_to_sub, gpair);
i++;
@ -168,10 +167,10 @@ class GloablApproxBuilder {
}
public:
explicit GloablApproxBuilder(TrainParam param, MetaInfo const &info, Context const *ctx,
explicit GloablApproxBuilder(TrainParam const *param, MetaInfo const &info, Context const *ctx,
std::shared_ptr<common::ColumnSampler> column_sampler, ObjInfo task,
common::Monitor *monitor)
: param_{std::move(param)},
: param_{param},
col_sampler_{std::move(column_sampler)},
evaluator_{ctx, param_, info, col_sampler_},
ctx_{ctx},
@ -183,7 +182,7 @@ class GloablApproxBuilder {
p_last_tree_ = p_tree;
this->InitData(p_fmat, hess);
Driver<CPUExpandEntry> driver(param_);
Driver<CPUExpandEntry> driver(*param_);
auto &tree = *p_tree;
driver.Push({this->InitRoot(p_fmat, gpair, hess, p_tree)});
auto expand_set = driver.Pop();
@ -213,7 +212,7 @@ class GloablApproxBuilder {
monitor_->Start("UpdatePosition");
size_t page_id = 0;
for (auto const &page : p_fmat->GetBatches<GHistIndexMatrix>(BatchSpec(param_, hess))) {
for (auto const &page : p_fmat->GetBatches<GHistIndexMatrix>(BatchSpec(*param_, hess))) {
partitioner_.at(page_id).UpdatePosition(ctx_, page, applied, p_tree);
page_id++;
}
@ -250,7 +249,6 @@ class GloablApproxBuilder {
* iteration.
*/
class GlobalApproxUpdater : public TreeUpdater {
TrainParam param_;
common::Monitor monitor_;
// specializations for different histogram precision.
std::unique_ptr<GloablApproxBuilder> pimpl_;
@ -265,15 +263,9 @@ class GlobalApproxUpdater : public TreeUpdater {
monitor_.Init(__func__);
}
void Configure(const Args &args) override { param_.UpdateAllowUnknown(args); }
void LoadConfig(Json const &in) override {
auto const &config = get<Object const>(in);
FromJson(config.at("train_param"), &this->param_);
}
void SaveConfig(Json *p_out) const override {
auto &out = *p_out;
out["train_param"] = ToJson(param_);
}
void Configure(Args const &) override {}
void LoadConfig(Json const &) override {}
void SaveConfig(Json *) const override {}
void InitData(TrainParam const &param, HostDeviceVector<GradientPair> const *gpair,
linalg::Matrix<GradientPair> *sampled) {
@ -283,20 +275,17 @@ class GlobalApproxUpdater : public TreeUpdater {
SampleGradient(ctx_, param, sampled->HostView());
}
char const *Name() const override { return "grow_histmaker"; }
[[nodiscard]] char const *Name() const override { return "grow_histmaker"; }
void Update(HostDeviceVector<GradientPair> *gpair, DMatrix *m,
void Update(TrainParam const *param, HostDeviceVector<GradientPair> *gpair, DMatrix *m,
common::Span<HostDeviceVector<bst_node_t>> out_position,
const std::vector<RegTree *> &trees) override {
float lr = param_.learning_rate;
param_.learning_rate = lr / trees.size();
pimpl_ = std::make_unique<GloablApproxBuilder>(param_, m->Info(), ctx_, column_sampler_, task_,
pimpl_ = std::make_unique<GloablApproxBuilder>(param, m->Info(), ctx_, column_sampler_, task_,
&monitor_);
linalg::Matrix<GradientPair> h_gpair;
// Obtain the hessian values for weighted sketching
InitData(param_, gpair, &h_gpair);
InitData(*param, gpair, &h_gpair);
std::vector<float> hess(h_gpair.Size());
auto const &s_gpair = h_gpair.Data()->ConstHostVector();
std::transform(s_gpair.begin(), s_gpair.end(), hess.begin(),
@ -304,12 +293,11 @@ class GlobalApproxUpdater : public TreeUpdater {
cached_ = m;
size_t t_idx = 0;
std::size_t t_idx = 0;
for (auto p_tree : trees) {
this->pimpl_->UpdateTree(m, s_gpair, hess, p_tree, &out_position[t_idx]);
++t_idx;
}
param_.learning_rate = lr;
}
bool UpdatePredictionCache(const DMatrix *data, linalg::VectorView<float> out_preds) override {
@ -320,7 +308,7 @@ class GlobalApproxUpdater : public TreeUpdater {
return true;
}
bool HasNodePosition() const override { return true; }
[[nodiscard]] bool HasNodePosition() const override { return true; }
};
DMLC_REGISTRY_FILE_TAG(grow_histmaker);
@ -330,5 +318,4 @@ XGBOOST_REGISTER_TREE_UPDATER(GlobalHistMaker, "grow_histmaker")
"Tree constructor that uses approximate histogram construction "
"for each node.")
.set_body([](Context const *ctx, ObjInfo task) { return new GlobalApproxUpdater(ctx, task); });
} // namespace tree
} // namespace xgboost
} // namespace xgboost::tree

29
src/tree/updater_colmaker.cc
View File

@ -1,5 +1,5 @@
/*!
* Copyright 2014-2022 by XGBoost Contributors
/**
* Copyright 2014-2023 by XGBoost Contributors
* \file updater_colmaker.cc
* \brief use columnwise update to construct a tree
* \author Tianqi Chen
@ -17,8 +17,7 @@
#include "../common/random.h"
#include "split_evaluator.h"
namespace xgboost {
namespace tree {
namespace xgboost::tree {
DMLC_REGISTRY_FILE_TAG(updater_colmaker);
@ -57,18 +56,15 @@ class ColMaker: public TreeUpdater {
public:
explicit ColMaker(Context const *ctx) : TreeUpdater(ctx) {}
void Configure(const Args &args) override {
param_.UpdateAllowUnknown(args);
colmaker_param_.UpdateAllowUnknown(args);
}
void LoadConfig(Json const& in) override {
auto const& config = get<Object const>(in);
FromJson(config.at("train_param"), &this->param_);
FromJson(config.at("colmaker_train_param"), &this->colmaker_param_);
}
void SaveConfig(Json* p_out) const override {
auto& out = *p_out;
out["train_param"] = ToJson(param_);
void SaveConfig(Json *p_out) const override {
auto &out = *p_out;
out["colmaker_train_param"] = ToJson(colmaker_param_);
}
@ -95,7 +91,7 @@ class ColMaker: public TreeUpdater {
}
}
void Update(HostDeviceVector<GradientPair> *gpair, DMatrix *dmat,
void Update(TrainParam const *param, HostDeviceVector<GradientPair> *gpair, DMatrix *dmat,
common::Span<HostDeviceVector<bst_node_t>> /*out_position*/,
const std::vector<RegTree *> &trees) override {
if (collective::IsDistributed()) {
@ -108,22 +104,16 @@ class ColMaker: public TreeUpdater {
}
this->LazyGetColumnDensity(dmat);
// rescale learning rate according to size of trees
float lr = param_.learning_rate;
param_.learning_rate = lr / trees.size();
interaction_constraints_.Configure(param_, dmat->Info().num_row_);
interaction_constraints_.Configure(*param, dmat->Info().num_row_);
// build tree
for (auto tree : trees) {
CHECK(ctx_);
Builder builder(param_, colmaker_param_, interaction_constraints_, ctx_,
column_densities_);
Builder builder(*param, colmaker_param_, interaction_constraints_, ctx_, column_densities_);
builder.Update(gpair->ConstHostVector(), dmat, tree);
}
param_.learning_rate = lr;
}
protected:
// training parameter
TrainParam param_;
ColMakerTrainParam colmaker_param_;
// SplitEvaluator that will be cloned for each Builder
std::vector<float> column_densities_;
@ -614,5 +604,4 @@ class ColMaker: public TreeUpdater {
XGBOOST_REGISTER_TREE_UPDATER(ColMaker, "grow_colmaker")
.describe("Grow tree with parallelization over columns.")
.set_body([](Context const *ctx, ObjInfo) { return new ColMaker(ctx); });
} // namespace tree
} // namespace xgboost
} // namespace xgboost::tree

36
src/tree/updater_gpu_hist.cu
View File

@ -1,5 +1,5 @@
/*!
* Copyright 2017-2022 XGBoost contributors
/**
* Copyright 2017-2023 by XGBoost contributors
*/
#include <thrust/copy.h>
#include <thrust/reduce.h>
@ -756,7 +756,6 @@ class GPUHistMaker : public TreeUpdater {
void Configure(const Args& args) override {
// Used in test to count how many configurations are performed
LOG(DEBUG) << "[GPU Hist]: Configure";
param_.UpdateAllowUnknown(args);
hist_maker_param_.UpdateAllowUnknown(args);
dh::CheckComputeCapability();
initialised_ = false;
@ -768,32 +767,26 @@ class GPUHistMaker : public TreeUpdater {
auto const& config = get<Object const>(in);
FromJson(config.at("gpu_hist_train_param"), &this->hist_maker_param_);
initialised_ = false;
FromJson(config.at("train_param"), &param_);
}
void SaveConfig(Json* p_out) const override {
auto& out = *p_out;
out["gpu_hist_train_param"] = ToJson(hist_maker_param_);
out["train_param"] = ToJson(param_);
}
~GPUHistMaker() { // NOLINT
dh::GlobalMemoryLogger().Log();
}
void Update(HostDeviceVector<GradientPair>* gpair, DMatrix* dmat,
void Update(TrainParam const* param, HostDeviceVector<GradientPair>* gpair, DMatrix* dmat,
common::Span<HostDeviceVector<bst_node_t>> out_position,
const std::vector<RegTree*>& trees) override {
monitor_.Start("Update");
// rescale learning rate according to size of trees
float lr = param_.learning_rate;
param_.learning_rate = lr / trees.size();
// build tree
try {
size_t t_idx{0};
for (xgboost::RegTree* tree : trees) {
this->UpdateTree(gpair, dmat, tree, &out_position[t_idx]);
this->UpdateTree(param, gpair, dmat, tree, &out_position[t_idx]);
if (hist_maker_param_.debug_synchronize) {
this->CheckTreesSynchronized(tree);
@ -804,12 +797,10 @@ class GPUHistMaker : public TreeUpdater {
} catch (const std::exception& e) {
LOG(FATAL) << "Exception in gpu_hist: " << e.what() << std::endl;
}
param_.learning_rate = lr;
monitor_.Stop("Update");
}
void InitDataOnce(DMatrix* dmat) {
void InitDataOnce(TrainParam const* param, DMatrix* dmat) {
CHECK_GE(ctx_->gpu_id, 0) << "Must have at least one device";
info_ = &dmat->Info();
@ -818,24 +809,24 @@ class GPUHistMaker : public TreeUpdater {
collective::Broadcast(&column_sampling_seed, sizeof(column_sampling_seed), 0);
BatchParam batch_param{
ctx_->gpu_id,
param_.max_bin,
ctx_->gpu_id,
param->max_bin,
};
auto page = (*dmat->GetBatches<EllpackPage>(batch_param).begin()).Impl();
dh::safe_cuda(cudaSetDevice(ctx_->gpu_id));
info_->feature_types.SetDevice(ctx_->gpu_id);
maker.reset(new GPUHistMakerDevice<GradientSumT>(
ctx_, page, info_->feature_types.ConstDeviceSpan(), info_->num_row_, param_,
ctx_, page, info_->feature_types.ConstDeviceSpan(), info_->num_row_, *param,
column_sampling_seed, info_->num_col_, batch_param));
p_last_fmat_ = dmat;
initialised_ = true;
}
void InitData(DMatrix* dmat, RegTree const* p_tree) {
void InitData(TrainParam const* param, DMatrix* dmat, RegTree const* p_tree) {
if (!initialised_) {
monitor_.Start("InitDataOnce");
this->InitDataOnce(dmat);
this->InitDataOnce(param, dmat);
monitor_.Stop("InitDataOnce");
}
p_last_tree_ = p_tree;
@ -856,10 +847,10 @@ class GPUHistMaker : public TreeUpdater {
CHECK(*local_tree == reference_tree);
}
void UpdateTree(HostDeviceVector<GradientPair>* gpair, DMatrix* p_fmat, RegTree* p_tree,
HostDeviceVector<bst_node_t>* p_out_position) {
void UpdateTree(TrainParam const* param, HostDeviceVector<GradientPair>* gpair, DMatrix* p_fmat,
RegTree* p_tree, HostDeviceVector<bst_node_t>* p_out_position) {
monitor_.Start("InitData");
this->InitData(p_fmat, p_tree);
this->InitData(param, p_fmat, p_tree);
monitor_.Stop("InitData");
gpair->SetDevice(ctx_->gpu_id);
@ -878,7 +869,6 @@ class GPUHistMaker : public TreeUpdater {
return result;
}
TrainParam param_; // NOLINT
MetaInfo* info_{}; // NOLINT
std::unique_ptr<GPUHistMakerDevice<GradientSumT>> maker; // NOLINT

65
src/tree/updater_prune.cc
View File

@ -1,5 +1,5 @@
/*!
* Copyright 2014-2022 by XGBoost Contributors
/**
* Copyright 2014-2023 by XGBoost Contributors
* \file updater_prune.cc
* \brief prune a tree given the statistics
* \author Tianqi Chen
@ -8,13 +8,11 @@
#include <memory>
#include "../common/timer.h"
#include "./param.h"
#include "xgboost/base.h"
#include "xgboost/json.h"
#include "./param.h"
#include "../common/timer.h"
namespace xgboost {
namespace tree {
namespace xgboost::tree {
DMLC_REGISTRY_FILE_TAG(updater_prune);
/*! \brief pruner that prunes a tree after growing finishes */
@ -24,47 +22,31 @@ class TreePruner : public TreeUpdater {
syncher_.reset(TreeUpdater::Create("sync", ctx_, task));
pruner_monitor_.Init("TreePruner");
}
char const* Name() const override {
return "prune";
}
[[nodiscard]] char const* Name() const override { return "prune"; }
// set training parameter
void Configure(const Args& args) override {
param_.UpdateAllowUnknown(args);
syncher_->Configure(args);
}
void Configure(const Args& args) override { syncher_->Configure(args); }
void LoadConfig(Json const& in) override {
auto const& config = get<Object const>(in);
FromJson(config.at("train_param"), &this->param_);
}
void SaveConfig(Json* p_out) const override {
auto& out = *p_out;
out["train_param"] = ToJson(param_);
}
bool CanModifyTree() const override {
return true;
}
void LoadConfig(Json const&) override {}
void SaveConfig(Json*) const override {}
[[nodiscard]] bool CanModifyTree() const override { return true; }
// update the tree, do pruning
void Update(HostDeviceVector<GradientPair>* gpair, DMatrix* p_fmat,
void Update(TrainParam const* param, HostDeviceVector<GradientPair>* gpair, DMatrix* p_fmat,
common::Span<HostDeviceVector<bst_node_t>> out_position,
const std::vector<RegTree*>& trees) override {
pruner_monitor_.Start("PrunerUpdate");
// rescale learning rate according to size of trees
float lr = param_.learning_rate;
param_.learning_rate = lr / trees.size();
for (auto tree : trees) {
this->DoPrune(tree);
this->DoPrune(param, tree);
}
param_.learning_rate = lr;
syncher_->Update(gpair, p_fmat, out_position, trees);
syncher_->Update(param, gpair, p_fmat, out_position, trees);
pruner_monitor_.Stop("PrunerUpdate");
}
private:
// try to prune off current leaf
bst_node_t TryPruneLeaf(RegTree &tree, int nid, int depth, int npruned) { // NOLINT(*)
bst_node_t TryPruneLeaf(TrainParam const* param, RegTree* p_tree, int nid, int depth,
int npruned) {
auto& tree = *p_tree;
CHECK(tree[nid].IsLeaf());
if (tree[nid].IsRoot()) {
return npruned;
@ -77,22 +59,22 @@ class TreePruner : public TreeUpdater {
auto right = tree[pid].RightChild();
bool balanced = tree[left].IsLeaf() &&
right != RegTree::kInvalidNodeId && tree[right].IsLeaf();
if (balanced && param_.NeedPrune(s.loss_chg, depth)) {
if (balanced && param->NeedPrune(s.loss_chg, depth)) {
// need to be pruned
tree.ChangeToLeaf(pid, param_.learning_rate * s.base_weight);
tree.ChangeToLeaf(pid, param->learning_rate * s.base_weight);
// tail recursion
return this->TryPruneLeaf(tree, pid, depth - 1, npruned + 2);
return this->TryPruneLeaf(param, p_tree, pid, depth - 1, npruned + 2);
} else {
return npruned;
}
}
/*! \brief do pruning of a tree */
void DoPrune(RegTree* p_tree) {
void DoPrune(TrainParam const* param, RegTree* p_tree) {
auto& tree = *p_tree;
bst_node_t npruned = 0;
for (int nid = 0; nid < tree.param.num_nodes; ++nid) {
if (tree[nid].IsLeaf() && !tree[nid].IsDeleted()) {
npruned = this->TryPruneLeaf(tree, nid, tree.GetDepth(nid), npruned);
npruned = this->TryPruneLeaf(param, p_tree, nid, tree.GetDepth(nid), npruned);
}
}
LOG(INFO) << "tree pruning end, "
@ -103,13 +85,10 @@ class TreePruner : public TreeUpdater {
private:
// synchronizer
std::unique_ptr<TreeUpdater> syncher_;
// training parameter
TrainParam param_;
common::Monitor pruner_monitor_;
};
XGBOOST_REGISTER_TREE_UPDATER(TreePruner, "prune")
.describe("Pruner that prune the tree according to statistics.")
.set_body([](Context const* ctx, ObjInfo task) { return new TreePruner(ctx, task); });
} // namespace tree
} // namespace xgboost
} // namespace xgboost::tree

21
src/tree/updater_quantile_hist.cc
View File

@ -28,21 +28,14 @@ namespace tree {
DMLC_REGISTRY_FILE_TAG(updater_quantile_hist);
void QuantileHistMaker::Configure(const Args &args) {
param_.UpdateAllowUnknown(args);
}
void QuantileHistMaker::Update(HostDeviceVector<GradientPair> *gpair, DMatrix *dmat,
void QuantileHistMaker::Update(TrainParam const *param, HostDeviceVector<GradientPair> *gpair,
DMatrix *dmat,
common::Span<HostDeviceVector<bst_node_t>> out_position,
const std::vector<RegTree *> &trees) {
// rescale learning rate according to size of trees
float lr = param_.learning_rate;
param_.learning_rate = lr / trees.size();
// build tree
const size_t n_trees = trees.size();
if (!pimpl_) {
pimpl_.reset(new Builder(n_trees, param_, dmat, task_, ctx_));
pimpl_.reset(new Builder(n_trees, param, dmat, task_, ctx_));
}
size_t t_idx{0};
@ -51,8 +44,6 @@ void QuantileHistMaker::Update(HostDeviceVector<GradientPair> *gpair, DMatrix *d
this->pimpl_->UpdateTree(gpair, dmat, p_tree, &t_row_position);
++t_idx;
}
param_.learning_rate = lr;
}
bool QuantileHistMaker::UpdatePredictionCache(const DMatrix *data,
@ -107,7 +98,7 @@ CPUExpandEntry QuantileHistMaker::Builder::InitRoot(
auto weight = evaluator_->InitRoot(GradStats{grad_stat});
p_tree->Stat(RegTree::kRoot).sum_hess = grad_stat.GetHess();
p_tree->Stat(RegTree::kRoot).base_weight = weight;
(*p_tree)[RegTree::kRoot].SetLeaf(param_.learning_rate * weight);
(*p_tree)[RegTree::kRoot].SetLeaf(param_->learning_rate * weight);
std::vector<CPUExpandEntry> entries{node};
monitor_->Start("EvaluateSplits");
@ -173,7 +164,7 @@ void QuantileHistMaker::Builder::ExpandTree(DMatrix *p_fmat, RegTree *p_tree,
HostDeviceVector<bst_node_t> *p_out_position) {
monitor_->Start(__func__);
Driver<CPUExpandEntry> driver(param_);
Driver<CPUExpandEntry> driver(*param_);
driver.Push(this->InitRoot(p_fmat, p_tree, gpair_h));
auto const &tree = *p_tree;
auto expand_set = driver.Pop();
@ -285,7 +276,7 @@ void QuantileHistMaker::Builder::InitData(DMatrix *fmat, const RegTree &tree,
auto m_gpair =
linalg::MakeTensorView(*gpair, {gpair->size(), static_cast<std::size_t>(1)}, ctx_->gpu_id);
SampleGradient(ctx_, param_, m_gpair);
SampleGradient(ctx_, *param_, m_gpair);
}
// store a pointer to the tree

38
src/tree/updater_quantile_hist.h
View File

@ -35,49 +35,36 @@
#include "../common/partition_builder.h"
#include "../common/column_matrix.h"
namespace xgboost {
namespace tree {
inline BatchParam HistBatch(TrainParam const& param) {
return {param.max_bin, param.sparse_threshold};
namespace xgboost::tree {
inline BatchParam HistBatch(TrainParam const* param) {
return {param->max_bin, param->sparse_threshold};
}
/*! \brief construct a tree using quantized feature values */
class QuantileHistMaker: public TreeUpdater {
public:
explicit QuantileHistMaker(Context const* ctx, ObjInfo task) : TreeUpdater(ctx), task_{task} {}
void Configure(const Args& args) override;
void Configure(const Args&) override {}
void Update(HostDeviceVector<GradientPair>* gpair, DMatrix* dmat,
void Update(TrainParam const* param, HostDeviceVector<GradientPair>* gpair, DMatrix* dmat,
common::Span<HostDeviceVector<bst_node_t>> out_position,
const std::vector<RegTree*>& trees) override;
bool UpdatePredictionCache(const DMatrix *data,
linalg::VectorView<float> out_preds) override;
void LoadConfig(Json const& in) override {
auto const& config = get<Object const>(in);
FromJson(config.at("train_param"), &this->param_);
}
void SaveConfig(Json* p_out) const override {
auto& out = *p_out;
out["train_param"] = ToJson(param_);
}
void LoadConfig(Json const&) override {}
void SaveConfig(Json*) const override {}
char const* Name() const override {
return "grow_quantile_histmaker";
}
bool HasNodePosition() const override { return true; }
[[nodiscard]] char const* Name() const override { return "grow_quantile_histmaker"; }
[[nodiscard]] bool HasNodePosition() const override { return true; }
protected:
// training parameter
TrainParam param_;
// actual builder that runs the algorithm
struct Builder {
public:
// constructor
explicit Builder(const size_t n_trees, const TrainParam& param, DMatrix const* fmat,
explicit Builder(const size_t n_trees, TrainParam const* param, DMatrix const* fmat,
ObjInfo task, Context const* ctx)
: n_trees_(n_trees),
param_(param),
@ -115,7 +102,7 @@ class QuantileHistMaker: public TreeUpdater {
private:
const size_t n_trees_;
const TrainParam& param_;
TrainParam const* param_;
std::shared_ptr<common::ColumnSampler> column_sampler_{
std::make_shared<common::ColumnSampler>()};
@ -140,7 +127,6 @@ class QuantileHistMaker: public TreeUpdater {
std::unique_ptr<Builder> pimpl_;
ObjInfo task_;
};
} // namespace tree
} // namespace xgboost
} // namespace xgboost::tree
#endif // XGBOOST_TREE_UPDATER_QUANTILE_HIST_H_

63
src/tree/updater_refresh.cc
View File

@ -1,5 +1,5 @@
/*!
* Copyright 2014-2022 by XGBoost Contributors
/**
* Copyright 2014-2023 by XGBoost Contributors
* \file updater_refresh.cc
* \brief refresh the statistics and leaf value on the tree on the dataset
* \author Tianqi Chen
@ -16,8 +16,7 @@
#include "./param.h"
#include "xgboost/json.h"
namespace xgboost {
namespace tree {
namespace xgboost::tree {
DMLC_REGISTRY_FILE_TAG(updater_refresh);
@ -25,23 +24,14 @@ DMLC_REGISTRY_FILE_TAG(updater_refresh);
class TreeRefresher : public TreeUpdater {
public:
explicit TreeRefresher(Context const *ctx) : TreeUpdater(ctx) {}
void Configure(const Args &args) override { param_.UpdateAllowUnknown(args); }
void LoadConfig(Json const& in) override {
auto const& config = get<Object const>(in);
FromJson(config.at("train_param"), &this->param_);
}
void SaveConfig(Json* p_out) const override {
auto& out = *p_out;
out["train_param"] = ToJson(param_);
}
char const* Name() const override {
return "refresh";
}
bool CanModifyTree() const override {
return true;
}
void Configure(const Args &) override {}
void LoadConfig(Json const &) override {}
void SaveConfig(Json *) const override {}
[[nodiscard]] char const *Name() const override { return "refresh"; }
[[nodiscard]] bool CanModifyTree() const override { return true; }
// update the tree, do pruning
void Update(HostDeviceVector<GradientPair> *gpair, DMatrix *p_fmat,
void Update(TrainParam const *param, HostDeviceVector<GradientPair> *gpair, DMatrix *p_fmat,
common::Span<HostDeviceVector<bst_node_t>> /*out_position*/,
const std::vector<RegTree *> &trees) override {
if (trees.size() == 0) return;
@ -103,16 +93,11 @@ class TreeRefresher : public TreeUpdater {
lazy_get_stats();
collective::Allreduce<collective::Operation::kSum>(&dmlc::BeginPtr(stemp[0])->sum_grad,
stemp[0].size() * 2);
// rescale learning rate according to size of trees
float lr = param_.learning_rate;
param_.learning_rate = lr / trees.size();
int offset = 0;
for (auto tree : trees) {
this->Refresh(dmlc::BeginPtr(stemp[0]) + offset, 0, tree);
this->Refresh(param, dmlc::BeginPtr(stemp[0]) + offset, 0, tree);
offset += tree->param.num_nodes;
}
// set learning rate back
param_.learning_rate = lr;
}
private:
@ -135,31 +120,27 @@ class TreeRefresher : public TreeUpdater {
gstats[pid].Add(gpair[ridx]);
}
}
inline void Refresh(const GradStats *gstats,
int nid, RegTree *p_tree) {
inline void Refresh(TrainParam const *param, const GradStats *gstats, int nid, RegTree *p_tree) {
RegTree &tree = *p_tree;
tree.Stat(nid).base_weight =
static_cast<bst_float>(CalcWeight(param_, gstats[nid]));
static_cast<bst_float>(CalcWeight(*param, gstats[nid]));
tree.Stat(nid).sum_hess = static_cast<bst_float>(gstats[nid].sum_hess);
if (tree[nid].IsLeaf()) {
if (param_.refresh_leaf) {
tree[nid].SetLeaf(tree.Stat(nid).base_weight * param_.learning_rate);
if (param->refresh_leaf) {
tree[nid].SetLeaf(tree.Stat(nid).base_weight * param->learning_rate);
}
} else {
tree.Stat(nid).loss_chg = static_cast<bst_float>(
xgboost::tree::CalcGain(param_, gstats[tree[nid].LeftChild()]) +
xgboost::tree::CalcGain(param_, gstats[tree[nid].RightChild()]) -
xgboost::tree::CalcGain(param_, gstats[nid]));
this->Refresh(gstats, tree[nid].LeftChild(), p_tree);
this->Refresh(gstats, tree[nid].RightChild(), p_tree);
tree.Stat(nid).loss_chg =
static_cast<bst_float>(xgboost::tree::CalcGain(*param, gstats[tree[nid].LeftChild()]) +
xgboost::tree::CalcGain(*param, gstats[tree[nid].RightChild()]) -
xgboost::tree::CalcGain(*param, gstats[nid]));
this->Refresh(param, gstats, tree[nid].LeftChild(), p_tree);
this->Refresh(param, gstats, tree[nid].RightChild(), p_tree);
}
}
// training parameter
TrainParam param_;
};
XGBOOST_REGISTER_TREE_UPDATER(TreeRefresher, "refresh")
.describe("Refresher that refreshes the weight and statistics according to data.")
.set_body([](Context const *ctx, ObjInfo) { return new TreeRefresher(ctx); });
} // namespace tree
} // namespace xgboost
} // namespace xgboost::tree

16
src/tree/updater_sync.cc
View File

@ -1,5 +1,5 @@
/*!
* Copyright 2014-2019 by Contributors
/**
* Copyright 2014-2013 by XBGoost Contributors
* \file updater_sync.cc
* \brief synchronize the tree in all distributed nodes
*/
@ -13,8 +13,7 @@
#include "../common/io.h"
#include "xgboost/json.h"
namespace xgboost {
namespace tree {
namespace xgboost::tree {
DMLC_REGISTRY_FILE_TAG(updater_sync);
@ -30,11 +29,9 @@ class TreeSyncher : public TreeUpdater {
void LoadConfig(Json const&) override {}
void SaveConfig(Json*) const override {}
char const* Name() const override {
return "prune";
}
[[nodiscard]] char const* Name() const override { return "prune"; }
void Update(HostDeviceVector<GradientPair>*, DMatrix*,
void Update(TrainParam const*, HostDeviceVector<GradientPair>*, DMatrix*,
common::Span<HostDeviceVector<bst_node_t>> /*out_position*/,
const std::vector<RegTree*>& trees) override {
if (collective::GetWorldSize() == 1) return;
@ -57,5 +54,4 @@ class TreeSyncher : public TreeUpdater {
XGBOOST_REGISTER_TREE_UPDATER(TreeSyncher, "sync")
.describe("Syncher that synchronize the tree in all distributed nodes.")
.set_body([](Context const* ctx, ObjInfo) { return new TreeSyncher(ctx); });
} // namespace tree
} // namespace xgboost
} // namespace xgboost::tree

28
tests/cpp/objective/test_regression_obj.cc
View File

@ -6,8 +6,9 @@
#include <xgboost/json.h>
#include <xgboost/objective.h>
#include "../../../src/common/linalg_op.h" // begin,end
#include "../../../src/common/linalg_op.h" // for begin, end
#include "../../../src/objective/adaptive.h"
#include "../../../src/tree/param.h" // for TrainParam
#include "../helpers.h"
#include "xgboost/base.h"
#include "xgboost/data.h"
@ -408,9 +409,13 @@ TEST(Objective, DeclareUnifiedTest(AbsoluteError)) {
h_predt[i] = labels[i] + i;
}
obj->UpdateTreeLeaf(position, info, predt, 0, &tree);
ASSERT_EQ(tree[1].LeafValue(), -1);
ASSERT_EQ(tree[2].LeafValue(), -4);
tree::TrainParam param;
param.Init(Args{});
auto lr = param.learning_rate;
obj->UpdateTreeLeaf(position, info, param.learning_rate, predt, 0, &tree);
ASSERT_EQ(tree[1].LeafValue(), -1.0f * lr);
ASSERT_EQ(tree[2].LeafValue(), -4.0f * lr);
}
TEST(Objective, DeclareUnifiedTest(AbsoluteErrorLeaf)) {
@ -457,11 +462,16 @@ TEST(Objective, DeclareUnifiedTest(AbsoluteErrorLeaf)) {
ASSERT_EQ(tree.GetNumLeaves(), 4);
auto empty_leaf = tree[4].LeafValue();
obj->UpdateTreeLeaf(position, info, predt, t, &tree);
ASSERT_EQ(tree[3].LeafValue(), -5);
ASSERT_EQ(tree[4].LeafValue(), empty_leaf);
ASSERT_EQ(tree[5].LeafValue(), -10);
ASSERT_EQ(tree[6].LeafValue(), -14);
tree::TrainParam param;
param.Init(Args{});
auto lr = param.learning_rate;
obj->UpdateTreeLeaf(position, info, lr, predt, t, &tree);
ASSERT_EQ(tree[3].LeafValue(), -5.0f * lr);
ASSERT_EQ(tree[4].LeafValue(), empty_leaf * lr);
ASSERT_EQ(tree[5].LeafValue(), -10.0f * lr);
ASSERT_EQ(tree[6].LeafValue(), -14.0f * lr);
}
}

10
tests/cpp/tree/hist/test_evaluate_splits.cc
View File

@ -24,7 +24,7 @@ void TestEvaluateSplits(bool force_read_by_column) {
auto dmat = RandomDataGenerator(kRows, kCols, 0).Seed(3).GenerateDMatrix();
auto evaluator = HistEvaluator<CPUExpandEntry>{&ctx, param, dmat->Info(), sampler};
auto evaluator = HistEvaluator<CPUExpandEntry>{&ctx, &param, dmat->Info(), sampler};
common::HistCollection hist;
std::vector<GradientPair> row_gpairs = {
{1.23f, 0.24f}, {0.24f, 0.25f}, {0.26f, 0.27f}, {2.27f, 0.28f},
@ -96,7 +96,7 @@ TEST(HistEvaluator, Apply) {
param.UpdateAllowUnknown(Args{{"min_child_weight", "0"}, {"reg_lambda", "0.0"}});
auto dmat = RandomDataGenerator(kNRows, kNCols, 0).Seed(3).GenerateDMatrix();
auto sampler = std::make_shared<common::ColumnSampler>();
auto evaluator_ = HistEvaluator<CPUExpandEntry>{&ctx, param, dmat->Info(), sampler};
auto evaluator_ = HistEvaluator<CPUExpandEntry>{&ctx, &param, dmat->Info(), sampler};
CPUExpandEntry entry{0, 0, 10.0f};
entry.split.left_sum = GradStats{0.4, 0.6f};
@ -123,7 +123,7 @@ TEST_F(TestPartitionBasedSplit, CPUHist) {
// check the evaluator is returning the optimal split
std::vector<FeatureType> ft{FeatureType::kCategorical};
auto sampler = std::make_shared<common::ColumnSampler>();
HistEvaluator<CPUExpandEntry> evaluator{&ctx, param_, info_, sampler};
HistEvaluator<CPUExpandEntry> evaluator{&ctx, &param_, info_, sampler};
evaluator.InitRoot(GradStats{total_gpair_});
RegTree tree;
std::vector<CPUExpandEntry> entries(1);
@ -153,7 +153,7 @@ auto CompareOneHotAndPartition(bool onehot) {
RandomDataGenerator(kRows, kCols, 0).Seed(3).Type(ft).MaxCategory(n_cats).GenerateDMatrix();
auto sampler = std::make_shared<common::ColumnSampler>();
auto evaluator = HistEvaluator<CPUExpandEntry>{&ctx, param, dmat->Info(), sampler};
auto evaluator = HistEvaluator<CPUExpandEntry>{&ctx, &param, dmat->Info(), sampler};
std::vector<CPUExpandEntry> entries(1);
for (auto const &gmat : dmat->GetBatches<GHistIndexMatrix>({32, param.sparse_threshold})) {
@ -204,7 +204,7 @@ TEST_F(TestCategoricalSplitWithMissing, HistEvaluator) {
info.num_col_ = 1;
info.feature_types = {FeatureType::kCategorical};
Context ctx;
auto evaluator = HistEvaluator<CPUExpandEntry>{&ctx, param_, info, sampler};
auto evaluator = HistEvaluator<CPUExpandEntry>{&ctx, &param_, info, sampler};
evaluator.InitRoot(GradStats{parent_sum_});
std::vector<CPUExpandEntry> entries(1);

10
tests/cpp/tree/test_evaluate_splits.h
View File

@ -1,5 +1,5 @@
/*!
* Copyright 2022 by XGBoost Contributors
/**
* Copyright 2022-2023 by XGBoost Contributors
*/
#include <gtest/gtest.h>
#include <xgboost/data.h>
@ -12,8 +12,7 @@
#include "../../../src/tree/split_evaluator.h"
#include "../helpers.h"
namespace xgboost {
namespace tree {
namespace xgboost::tree {
/**
* \brief Enumerate all possible partitions for categorical split.
*/
@ -151,5 +150,4 @@ class TestCategoricalSplitWithMissing : public testing::Test {
ASSERT_EQ(right_sum.GetHess(), parent_sum_.GetHess() - left_sum.GetHess());
}
};
} // namespace tree
} // namespace xgboost
} // namespace xgboost::tree

38
tests/cpp/tree/test_gpu_hist.cu
View File

@ -1,5 +1,5 @@
/*!
* Copyright 2017-2022 XGBoost contributors
/**
* Copyright 2017-2023 by XGBoost contributors
*/
#include <gtest/gtest.h>
#include <thrust/device_vector.h>
@ -13,6 +13,7 @@
#include "../../../src/common/common.h"
#include "../../../src/data/sparse_page_source.h"
#include "../../../src/tree/constraints.cuh"
#include "../../../src/tree/param.h" // for TrainParam
#include "../../../src/tree/updater_gpu_common.cuh"
#include "../../../src/tree/updater_gpu_hist.cu"
#include "../filesystem.h" // dmlc::TemporaryDirectory
@ -21,8 +22,7 @@
#include "xgboost/context.h"
#include "xgboost/json.h"
namespace xgboost {
namespace tree {
namespace xgboost::tree {
TEST(GpuHist, DeviceHistogram) {
// Ensures that node allocates correctly after reaching `kStopGrowingSize`.
dh::safe_cuda(cudaSetDevice(0));
@ -83,11 +83,12 @@ void TestBuildHist(bool use_shared_memory_histograms) {
int const kNRows = 16, kNCols = 8;
TrainParam param;
std::vector<std::pair<std::string, std::string>> args {
{"max_depth", "6"},
{"max_leaves", "0"},
Args args{
{"max_depth", "6"},
{"max_leaves", "0"},
};
param.Init(args);
auto page = BuildEllpackPage(kNRows, kNCols);
BatchParam batch_param{};
Context ctx{CreateEmptyGenericParam(0)};
@ -168,7 +169,6 @@ void TestHistogramIndexImpl() {
int constexpr kNRows = 1000, kNCols = 10;
// Build 2 matrices and build a histogram maker with that
Context ctx(CreateEmptyGenericParam(0));
tree::GPUHistMaker hist_maker{&ctx, ObjInfo{ObjInfo::kRegression}},
hist_maker_ext{&ctx, ObjInfo{ObjInfo::kRegression}};
@ -179,15 +179,14 @@ void TestHistogramIndexImpl() {
std::unique_ptr<DMatrix> hist_maker_ext_dmat(
CreateSparsePageDMatrixWithRC(kNRows, kNCols, 128UL, true, tempdir));
std::vector<std::pair<std::string, std::string>> training_params = {
{"max_depth", "10"},
{"max_leaves", "0"}
};
Args training_params = {{"max_depth", "10"}, {"max_leaves", "0"}};
TrainParam param;
param.UpdateAllowUnknown(training_params);
hist_maker.Configure(training_params);
hist_maker.InitDataOnce(hist_maker_dmat.get());
hist_maker.InitDataOnce(&param, hist_maker_dmat.get());
hist_maker_ext.Configure(training_params);
hist_maker_ext.InitDataOnce(hist_maker_ext_dmat.get());
hist_maker_ext.InitDataOnce(&param, hist_maker_ext_dmat.get());
// Extract the device maker from the histogram makers and from that its compressed
// histogram index
@ -237,13 +236,15 @@ void UpdateTree(HostDeviceVector<GradientPair>* gpair, DMatrix* dmat,
{"subsample", std::to_string(subsample)},
{"sampling_method", sampling_method},
};
TrainParam param;
param.UpdateAllowUnknown(args);
Context ctx(CreateEmptyGenericParam(0));
tree::GPUHistMaker hist_maker{&ctx,ObjInfo{ObjInfo::kRegression}};
hist_maker.Configure(args);
std::vector<HostDeviceVector<bst_node_t>> position(1);
hist_maker.Update(gpair, dmat, common::Span<HostDeviceVector<bst_node_t>>{position}, {tree});
hist_maker.Update(&param, gpair, dmat, common::Span<HostDeviceVector<bst_node_t>>{position},
{tree});
auto cache = linalg::VectorView<float>{preds->DeviceSpan(), {preds->Size()}, 0};
hist_maker.UpdatePredictionCache(dmat, cache);
}
@ -391,13 +392,11 @@ TEST(GpuHist, ConfigIO) {
Json j_updater { Object() };
updater->SaveConfig(&j_updater);
ASSERT_TRUE(IsA<Object>(j_updater["gpu_hist_train_param"]));
ASSERT_TRUE(IsA<Object>(j_updater["train_param"]));
updater->LoadConfig(j_updater);
Json j_updater_roundtrip { Object() };
updater->SaveConfig(&j_updater_roundtrip);
ASSERT_TRUE(IsA<Object>(j_updater_roundtrip["gpu_hist_train_param"]));
ASSERT_TRUE(IsA<Object>(j_updater_roundtrip["train_param"]));
ASSERT_EQ(j_updater, j_updater_roundtrip);
}
@ -414,5 +413,4 @@ TEST(GpuHist, MaxDepth) {
ASSERT_THROW({learner->UpdateOneIter(0, p_mat);}, dmlc::Error);
}
} // namespace tree
} // namespace xgboost
} // namespace xgboost::tree

32
tests/cpp/tree/test_histmaker.cc
View File

@ -5,11 +5,10 @@
#include <xgboost/tree_model.h>
#include <xgboost/tree_updater.h>
#include "../../../src/tree/param.h" // for TrainParam
#include "../helpers.h"
namespace xgboost {
namespace tree {
namespace xgboost::tree {
std::shared_ptr<DMatrix> GenerateDMatrix(std::size_t rows, std::size_t cols){
return RandomDataGenerator{rows, cols, 0.6f}.Seed(3).GenerateDMatrix();
}
@ -45,11 +44,11 @@ TEST(GrowHistMaker, InteractionConstraint)
std::unique_ptr<TreeUpdater> updater{
TreeUpdater::Create("grow_histmaker", &ctx, ObjInfo{ObjInfo::kRegression})};
updater->Configure(Args{
{"interaction_constraints", "[[0, 1]]"},
{"num_feature", std::to_string(kCols)}});
TrainParam param;
param.UpdateAllowUnknown(
Args{{"interaction_constraints", "[[0, 1]]"}, {"num_feature", std::to_string(kCols)}});
std::vector<HostDeviceVector<bst_node_t>> position(1);
updater->Update(p_gradients.get(), p_dmat.get(), position, {&tree});
updater->Update(&param, p_gradients.get(), p_dmat.get(), position, {&tree});
ASSERT_EQ(tree.NumExtraNodes(), 4);
ASSERT_EQ(tree[0].SplitIndex(), 1);
@ -64,9 +63,10 @@ TEST(GrowHistMaker, InteractionConstraint)
std::unique_ptr<TreeUpdater> updater{
TreeUpdater::Create("grow_histmaker", &ctx, ObjInfo{ObjInfo::kRegression})};
updater->Configure(Args{{"num_feature", std::to_string(kCols)}});
std::vector<HostDeviceVector<bst_node_t>> position(1);
updater->Update(p_gradients.get(), p_dmat.get(), position, {&tree});
TrainParam param;
param.Init(Args{});
updater->Update(&param, p_gradients.get(), p_dmat.get(), position, {&tree});
ASSERT_EQ(tree.NumExtraNodes(), 10);
ASSERT_EQ(tree[0].SplitIndex(), 1);
@ -83,7 +83,6 @@ void TestColumnSplit(int32_t rows, int32_t cols, RegTree const& expected_tree) {
Context ctx;
std::unique_ptr<TreeUpdater> updater{
TreeUpdater::Create("grow_histmaker", &ctx, ObjInfo{ObjInfo::kRegression})};
updater->Configure(Args{{"num_feature", std::to_string(cols)}});
std::vector<HostDeviceVector<bst_node_t>> position(1);
std::unique_ptr<DMatrix> sliced{
@ -91,7 +90,9 @@ void TestColumnSplit(int32_t rows, int32_t cols, RegTree const& expected_tree) {
RegTree tree;
tree.param.num_feature = cols;
updater->Update(p_gradients.get(), sliced.get(), position, {&tree});
TrainParam param;
param.Init(Args{});
updater->Update(&param, p_gradients.get(), sliced.get(), position, {&tree});
EXPECT_EQ(tree.NumExtraNodes(), 10);
EXPECT_EQ(tree[0].SplitIndex(), 1);
@ -115,14 +116,13 @@ TEST(GrowHistMaker, ColumnSplit) {
Context ctx;
std::unique_ptr<TreeUpdater> updater{
TreeUpdater::Create("grow_histmaker", &ctx, ObjInfo{ObjInfo::kRegression})};
updater->Configure(Args{{"num_feature", std::to_string(kCols)}});
std::vector<HostDeviceVector<bst_node_t>> position(1);
updater->Update(p_gradients.get(), p_dmat.get(), position, {&expected_tree});
TrainParam param;
param.Init(Args{});
updater->Update(&param, p_gradients.get(), p_dmat.get(), position, {&expected_tree});
}
auto constexpr kWorldSize = 2;
RunWithInMemoryCommunicator(kWorldSize, TestColumnSplit, kRows, kCols, std::cref(expected_tree));
}
} // namespace tree
} // namespace xgboost
} // namespace xgboost::tree

7
tests/cpp/tree/test_prediction_cache.cc
View File

@ -7,6 +7,7 @@
#include <memory>
#include "../../../src/tree/param.h" // for TrainParam
#include "../helpers.h"
namespace xgboost {
@ -75,9 +76,11 @@ class TestPredictionCache : public ::testing::Test {
RegTree tree;
std::vector<RegTree *> trees{&tree};
auto gpair = GenerateRandomGradients(n_samples_);
updater->Configure(Args{{"max_bin", "64"}});
tree::TrainParam param;
param.UpdateAllowUnknown(Args{{"max_bin", "64"}});
std::vector<HostDeviceVector<bst_node_t>> position(1);
updater->Update(&gpair, Xy_.get(), position, trees);
updater->Update(&param, &gpair, Xy_.get(), position, trees);
HostDeviceVector<float> out_prediction_cached;
out_prediction_cached.SetDevice(ctx.gpu_id);
out_prediction_cached.Resize(n_samples_);

44
tests/cpp/tree/test_prune.cc
View File

@ -1,20 +1,20 @@
/*!
* Copyright 2018-2019 by Contributors
/**
* Copyright 2018-2023 by XGBoost Contributors
*/
#include <gtest/gtest.h>
#include <xgboost/data.h>
#include <xgboost/host_device_vector.h>
#include <xgboost/tree_updater.h>
#include <xgboost/learner.h>
#include <gtest/gtest.h>
#include <vector>
#include <string>
#include <memory>
#include <xgboost/tree_updater.h>
#include <memory>
#include <string>
#include <vector>
#include "../../../src/tree/param.h" // for TrainParam
#include "../helpers.h"
namespace xgboost {
namespace tree {
namespace xgboost::tree {
TEST(Updater, Prune) {
int constexpr kCols = 16;
@ -36,28 +36,30 @@ TEST(Updater, Prune) {
tree.param.UpdateAllowUnknown(cfg);
std::vector<RegTree*> trees {&tree};
// prepare pruner
TrainParam param;
param.UpdateAllowUnknown(cfg);
std::unique_ptr<TreeUpdater> pruner(
TreeUpdater::Create("prune", &ctx, ObjInfo{ObjInfo::kRegression}));
pruner->Configure(cfg);
// loss_chg < min_split_loss;
std::vector<HostDeviceVector<bst_node_t>> position(trees.size());
tree.ExpandNode(0, 0, 0, true, 0.0f, 0.3f, 0.4f, 0.0f, 0.0f,
/*left_sum=*/0.0f, /*right_sum=*/0.0f);
pruner->Update(&gpair, p_dmat.get(), position, trees);
pruner->Update(&param, &gpair, p_dmat.get(), position, trees);
ASSERT_EQ(tree.NumExtraNodes(), 0);
// loss_chg > min_split_loss;
tree.ExpandNode(0, 0, 0, true, 0.0f, 0.3f, 0.4f, 11.0f, 0.0f,
/*left_sum=*/0.0f, /*right_sum=*/0.0f);
pruner->Update(&gpair, p_dmat.get(), position, trees);
pruner->Update(&param, &gpair, p_dmat.get(), position, trees);
ASSERT_EQ(tree.NumExtraNodes(), 2);
// loss_chg == min_split_loss;
tree.Stat(0).loss_chg = 10;
pruner->Update(&gpair, p_dmat.get(), position, trees);
pruner->Update(&param, &gpair, p_dmat.get(), position, trees);
ASSERT_EQ(tree.NumExtraNodes(), 2);
@ -71,10 +73,10 @@ TEST(Updater, Prune) {
0, 0.5f, true, 0.3, 0.4, 0.5,
/*loss_chg=*/19.0f, 0.0f,
/*left_sum=*/0.0f, /*right_sum=*/0.0f);
cfg.emplace_back("max_depth", "1");
pruner->Configure(cfg);
pruner->Update(&gpair, p_dmat.get(), position, trees);
cfg.emplace_back("max_depth", "1");
param.UpdateAllowUnknown(cfg);
pruner->Update(&param, &gpair, p_dmat.get(), position, trees);
ASSERT_EQ(tree.NumExtraNodes(), 2);
tree.ExpandNode(tree[0].LeftChild(),
@ -82,9 +84,9 @@ TEST(Updater, Prune) {
/*loss_chg=*/18.0f, 0.0f,
/*left_sum=*/0.0f, /*right_sum=*/0.0f);
cfg.emplace_back("min_split_loss", "0");
pruner->Configure(cfg);
pruner->Update(&gpair, p_dmat.get(), position, trees);
param.UpdateAllowUnknown(cfg);
pruner->Update(&param, &gpair, p_dmat.get(), position, trees);
ASSERT_EQ(tree.NumExtraNodes(), 2);
}
} // namespace tree
} // namespace xgboost
} // namespace xgboost::tree

17
tests/cpp/tree/test_refresh.cc
View File

@ -1,14 +1,15 @@
/*!
* Copyright 2018-2019 by Contributors
/**
* Copyright 2018-2013 by XGBoost Contributors
*/
#include <gtest/gtest.h>
#include <xgboost/host_device_vector.h>
#include <xgboost/tree_updater.h>
#include <gtest/gtest.h>
#include <vector>
#include <string>
#include <memory>
#include <string>
#include <vector>
#include "../../../src/tree/param.h" // for TrainParam
#include "../helpers.h"
namespace xgboost {
@ -43,9 +44,11 @@ TEST(Updater, Refresh) {
tree.Stat(cleft).base_weight = 1.2;
tree.Stat(cright).base_weight = 1.3;
refresher->Configure(cfg);
std::vector<HostDeviceVector<bst_node_t>> position;
refresher->Update(&gpair, p_dmat.get(), position, trees);
tree::TrainParam param;
param.UpdateAllowUnknown(cfg);
refresher->Update(&param, &gpair, p_dmat.get(), position, trees);
bst_float constexpr kEps = 1e-6;
ASSERT_NEAR(-0.183392, tree[cright].LeafValue(), kEps);

27
tests/cpp/tree/test_tree_stat.cc
View File

@ -1,7 +1,11 @@
/**
* Copyright 2020-2023 by XGBoost Contributors
*/
#include <gtest/gtest.h>
#include <xgboost/tree_model.h>
#include <xgboost/tree_updater.h>
#include "../../../src/tree/param.h" // for TrainParam
#include "../helpers.h"
namespace xgboost {
@ -21,6 +25,9 @@ class UpdaterTreeStatTest : public ::testing::Test {
}
void RunTest(std::string updater) {
tree::TrainParam param;
param.Init(Args{});
Context ctx(updater == "grow_gpu_hist" ? CreateEmptyGenericParam(0)
: CreateEmptyGenericParam(Context::kCpuId));
auto up = std::unique_ptr<TreeUpdater>{
@ -29,7 +36,7 @@ class UpdaterTreeStatTest : public ::testing::Test {
RegTree tree;
tree.param.num_feature = kCols;
std::vector<HostDeviceVector<bst_node_t>> position(1);
up->Update(&gpairs_, p_dmat_.get(), position, {&tree});
up->Update(&param, &gpairs_, p_dmat_.get(), position, {&tree});
tree.WalkTree([&tree](bst_node_t nidx) {
if (tree[nidx].IsLeaf()) {
@ -69,28 +76,33 @@ class UpdaterEtaTest : public ::testing::Test {
void RunTest(std::string updater) {
Context ctx(updater == "grow_gpu_hist" ? CreateEmptyGenericParam(0)
: CreateEmptyGenericParam(Context::kCpuId));
float eta = 0.4;
auto up_0 = std::unique_ptr<TreeUpdater>{
TreeUpdater::Create(updater, &ctx, ObjInfo{ObjInfo::kClassification})};
up_0->Configure(Args{{"eta", std::to_string(eta)}});
up_0->Configure(Args{});
tree::TrainParam param0;
param0.Init(Args{{"eta", std::to_string(eta)}});
auto up_1 = std::unique_ptr<TreeUpdater>{
TreeUpdater::Create(updater, &ctx, ObjInfo{ObjInfo::kClassification})};
up_1->Configure(Args{{"eta", "1.0"}});
tree::TrainParam param1;
param1.Init(Args{{"eta", "1.0"}});
for (size_t iter = 0; iter < 4; ++iter) {
RegTree tree_0;
{
tree_0.param.num_feature = kCols;
std::vector<HostDeviceVector<bst_node_t>> position(1);
up_0->Update(&gpairs_, p_dmat_.get(), position, {&tree_0});
up_0->Update(&param0, &gpairs_, p_dmat_.get(), position, {&tree_0});
}
RegTree tree_1;
{
tree_1.param.num_feature = kCols;
std::vector<HostDeviceVector<bst_node_t>> position(1);
up_1->Update(&gpairs_, p_dmat_.get(), position, {&tree_1});
up_1->Update(&param1, &gpairs_, p_dmat_.get(), position, {&tree_1});
}
tree_0.WalkTree([&](bst_node_t nidx) {
if (tree_0[nidx].IsLeaf()) {
@ -139,17 +151,18 @@ class TestMinSplitLoss : public ::testing::Test {
// test gamma
{"gamma", std::to_string(gamma)}};
tree::TrainParam param;
param.UpdateAllowUnknown(args);
Context ctx(updater == "grow_gpu_hist" ? CreateEmptyGenericParam(0)
: CreateEmptyGenericParam(Context::kCpuId));
std::cout << ctx.gpu_id << std::endl;
auto up = std::unique_ptr<TreeUpdater>{
TreeUpdater::Create(updater, &ctx, ObjInfo{ObjInfo::kRegression})};
up->Configure(args);
up->Configure({});
RegTree tree;
std::vector<HostDeviceVector<bst_node_t>> position(1);
up->Update(&gpair_, dmat_.get(), position, {&tree});
up->Update(&param, &gpair_, dmat_.get(), position, {&tree});
auto n_nodes = tree.NumExtraNodes();
return n_nodes;

8
tests/python-gpu/test_gpu_basic_models.py
View File

@ -42,9 +42,15 @@ class TestGPUBasicModels:
def test_custom_objective(self):
self.cpu_test_bm.run_custom_objective("gpu_hist")
def test_eta_decay_gpu_hist(self):
def test_eta_decay(self):
self.cpu_test_cb.run_eta_decay('gpu_hist')
@pytest.mark.parametrize(
"objective", ["binary:logistic", "reg:absoluteerror", "reg:quantileerror"]
)
def test_eta_decay_leaf_output(self, objective) -> None:
self.cpu_test_cb.run_eta_decay_leaf_output("gpu_hist", objective)
def test_deterministic_gpu_hist(self):
kRows = 1000
kCols = 64

131
tests/python/test_callback.py
View File

@ -1,3 +1,4 @@
import json
import os
import tempfile
from contextlib import nullcontext
@ -355,47 +356,125 @@ class TestCallbacks:
with warning_check:
xgb.cv(param, dtrain, num_round, callbacks=[scheduler(eta_decay)])
@pytest.mark.parametrize("tree_method", ["hist", "approx", "exact"])
def run_eta_decay_leaf_output(self, tree_method: str, objective: str) -> None:
# check decay has effect on leaf output.
num_round = 4
scheduler = xgb.callback.LearningRateScheduler
dpath = tm.data_dir(__file__)
dtrain = xgb.DMatrix(os.path.join(dpath, "agaricus.txt.train"))
dtest = xgb.DMatrix(os.path.join(dpath, "agaricus.txt.test"))
watchlist = [(dtest, 'eval'), (dtrain, 'train')]
param = {
"max_depth": 2,
"objective": objective,
"eval_metric": "error",
"tree_method": tree_method,
}
if objective == "reg:quantileerror":
param["quantile_alpha"] = 0.3
def eta_decay_0(i):
return num_round / (i + 1)
bst0 = xgb.train(
param,
dtrain,
num_round,
watchlist,
callbacks=[scheduler(eta_decay_0)],
)
def eta_decay_1(i: int) -> float:
if i > 1:
return 5.0
return num_round / (i + 1)
bst1 = xgb.train(
param,
dtrain,
num_round,
watchlist,
callbacks=[scheduler(eta_decay_1)],
)
bst_json0 = bst0.save_raw(raw_format="json")
bst_json1 = bst1.save_raw(raw_format="json")
j0 = json.loads(bst_json0)
j1 = json.loads(bst_json1)
tree_2th_0 = j0["learner"]["gradient_booster"]["model"]["trees"][2]
tree_2th_1 = j1["learner"]["gradient_booster"]["model"]["trees"][2]
assert tree_2th_0["base_weights"] == tree_2th_1["base_weights"]
assert tree_2th_0["split_conditions"] == tree_2th_1["split_conditions"]
tree_3th_0 = j0["learner"]["gradient_booster"]["model"]["trees"][3]
tree_3th_1 = j1["learner"]["gradient_booster"]["model"]["trees"][3]
assert tree_3th_0["base_weights"] != tree_3th_1["base_weights"]
assert tree_3th_0["split_conditions"] != tree_3th_1["split_conditions"]
@pytest.mark.parametrize("tree_method", ["hist", "approx", "approx"])
def test_eta_decay(self, tree_method):
self.run_eta_decay(tree_method)
@pytest.mark.parametrize(
"tree_method,objective",
[
("hist", "binary:logistic"),
("hist", "reg:absoluteerror"),
("hist", "reg:quantileerror"),
("approx", "binary:logistic"),
("approx", "reg:absoluteerror"),
("approx", "reg:quantileerror"),
],
)
def test_eta_decay_leaf_output(self, tree_method: str, objective: str) -> None:
self.run_eta_decay_leaf_output(tree_method, objective)
def test_check_point(self):
from sklearn.datasets import load_breast_cancer
X, y = load_breast_cancer(return_X_y=True)
m = xgb.DMatrix(X, y)
with tempfile.TemporaryDirectory() as tmpdir:
check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
iterations=1,
name='model')
xgb.train({'objective': 'binary:logistic'}, m,
num_boost_round=10,
verbose_eval=False,
callbacks=[check_point])
check_point = xgb.callback.TrainingCheckPoint(
directory=tmpdir, iterations=1, name="model"
)
xgb.train(
{"objective": "binary:logistic"},
m,
num_boost_round=10,
verbose_eval=False,
callbacks=[check_point],
)
for i in range(1, 10):
assert os.path.exists(
os.path.join(tmpdir, 'model_' + str(i) + '.json'))
assert os.path.exists(os.path.join(tmpdir, "model_" + str(i) + ".json"))
check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
iterations=1,
as_pickle=True,
name='model')
xgb.train({'objective': 'binary:logistic'}, m,
num_boost_round=10,
verbose_eval=False,
callbacks=[check_point])
check_point = xgb.callback.TrainingCheckPoint(
directory=tmpdir, iterations=1, as_pickle=True, name="model"
)
xgb.train(
{"objective": "binary:logistic"},
m,
num_boost_round=10,
verbose_eval=False,
callbacks=[check_point],
)
for i in range(1, 10):
assert os.path.exists(
os.path.join(tmpdir, 'model_' + str(i) + '.pkl'))
assert os.path.exists(os.path.join(tmpdir, "model_" + str(i) + ".pkl"))
def test_callback_list(self):
X, y = tm.get_california_housing()
m = xgb.DMatrix(X, y)
callbacks = [xgb.callback.EarlyStopping(rounds=10)]
for i in range(4):
xgb.train({'objective': 'reg:squarederror',
'eval_metric': 'rmse'}, m,
evals=[(m, 'Train')],
num_boost_round=1,
verbose_eval=True,
callbacks=callbacks)
xgb.train(
{"objective": "reg:squarederror", "eval_metric": "rmse"},
m,
evals=[(m, "Train")],
num_boost_round=1,
verbose_eval=True,
callbacks=callbacks,
)
assert len(callbacks) == 1

7
tests/python/test_pickling.py
View File

@ -51,11 +51,8 @@ class TestPickling:
def test_model_pickling_json(self):
def check(config):
updater = config["learner"]["gradient_booster"]["updater"]
if params["tree_method"] == "exact":
subsample = updater["grow_colmaker"]["train_param"]["subsample"]
else:
subsample = updater["grow_quantile_histmaker"]["train_param"]["subsample"]
tree_param = config["learner"]["gradient_booster"]["tree_train_param"]
subsample = tree_param["subsample"]
assert float(subsample) == 0.5
params = {"nthread": 8, "tree_method": "hist", "subsample": 0.5}

3
tests/python/test_updaters.py
View File

@ -447,7 +447,8 @@ class TestTreeMethod:
{
"tree_method": tree_method,
"objective": "reg:absoluteerror",
"subsample": 0.8
"subsample": 0.8,
"eta": 1.0,
},
Xy,
num_boost_round=10,

10
tests/python/test_with_sklearn.py
View File

@ -1018,14 +1018,18 @@ def test_XGBClassifier_resume():
def test_constraint_parameters():
reg = xgb.XGBRegressor(interaction_constraints='[[0, 1], [2, 3, 4]]')
reg = xgb.XGBRegressor(interaction_constraints="[[0, 1], [2, 3, 4]]")
X = np.random.randn(10, 10)
y = np.random.randn(10)
reg.fit(X, y)
config = json.loads(reg.get_booster().save_config())
assert config['learner']['gradient_booster']['updater']['grow_colmaker'][
'train_param']['interaction_constraints'] == '[[0, 1], [2, 3, 4]]'
assert (
config["learner"]["gradient_booster"]["tree_train_param"][
"interaction_constraints"
]
== "[[0, 1], [2, 3, 4]]"
)
def test_parameter_validation():

8
tests/test_distributed/test_with_spark/test_spark_local_cluster.py
View File

@ -422,10 +422,10 @@ class XgboostLocalClusterTestCase(SparkLocalClusterTestCase):
self.assertTrue(hasattr(classifier, "max_depth"))
self.assertEqual(classifier.getOrDefault(classifier.max_depth), 7)
booster_config = json.loads(model.get_booster().save_config())
max_depth = booster_config["learner"]["gradient_booster"]["updater"][
"grow_histmaker"
]["train_param"]["max_depth"]
self.assertEqual(int(max_depth), 7)
max_depth = booster_config["learner"]["gradient_booster"]["tree_train_param"][
"max_depth"
]
assert int(max_depth) == 7
def test_repartition(self):
# The following test case has a few partitioned datasets that are either