Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[POC] Experimental support for l1 error. (#7812)

Browse Source 
Support adaptive tree, a feature supported by both sklearn and lightgbm.  The tree leaf is recomputed based on residue of labels and predictions after construction.

For l1 error, the optimal value is the median (50 percentile).

This is marked as experimental support for the following reasons:
- The value is not well defined for distributed training, where we might have empty leaves for local workers. Right now I just use the original leaf value for computing the average with other workers, which might cause significant errors.
- Some follow-ups are required, for exact, pruner, and optimization for quantile function. Also, we need to calculate the initial estimation.
This commit is contained in:
Jiaming Yuan
2022-04-26 21:41:55 +08:00
committed by GitHub
parent ad06172c6b
commit fdf533f2b9
64 changed files with 1727 additions and 336 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
tests/cpp/tree/gpu_hist/test_row_partitioner.cu
View File

@@ -1,7 +1,8 @@
/*!
* Copyright 2019-2021 by XGBoost Contributors
* Copyright 2019-2022 by XGBoost Contributors
*/
#include <gtest/gtest.h>
#include <algorithm>
#include <vector>
#include <thrust/device_vector.h>
@@ -10,6 +11,10 @@
#include "../../../../src/tree/gpu_hist/row_partitioner.cuh"
#include "../../helpers.h"
#include "xgboost/base.h"
#include "xgboost/generic_parameters.h"
#include "xgboost/task.h"
#include "xgboost/tree_model.h"
namespace xgboost {
namespace tree {
@@ -103,17 +108,58 @@ TEST(RowPartitioner, Basic) { TestUpdatePosition(); }
void TestFinalise() {
const int kNumRows = 10;
ObjInfo task{ObjInfo::kRegression, false, false};
HostDeviceVector<bst_node_t> position;
Context ctx;
ctx.gpu_id = 0;
{
RowPartitioner rp(0, kNumRows);
rp.FinalisePosition(
&ctx, task, &position,
[=] __device__(RowPartitioner::RowIndexT ridx, int position) { return 7; },
[] XGBOOST_DEVICE(size_t idx) { return false; });
auto position = rp.GetPositionHost();
for (auto p : position) {
EXPECT_EQ(p, 7);
}
}
/**
* Test for sampling.
*/
dh::device_vector<float> hess(kNumRows);
for (size_t i = 0; i < hess.size(); ++i) {
// removed rows, 0, 3, 6, 9
if (i % 3 == 0) {
hess[i] = 0;
} else {
hess[i] = i;
}
}
auto d_hess = dh::ToSpan(hess);
RowPartitioner rp(0, kNumRows);
rp.FinalisePosition([=]__device__(RowPartitioner::RowIndexT ridx, int position)
{
return 7;
});
auto position = rp.GetPositionHost();
for(auto p:position)
{
EXPECT_EQ(p, 7);
rp.FinalisePosition(
&ctx, task, &position,
[] __device__(RowPartitioner::RowIndexT ridx, bst_node_t position) {
return ridx % 2 == 0 ? 1 : 2;
},
[d_hess] __device__(size_t ridx) { return d_hess[ridx] - 0.f == 0.f; });
auto const& h_position = position.ConstHostVector();
for (size_t ridx = 0; ridx < h_position.size(); ++ridx) {
if (ridx % 3 == 0) {
ASSERT_LT(h_position[ridx], 0);
} else {
ASSERT_EQ(h_position[ridx], ridx % 2 == 0 ? 1 : 2);
}
}
}
TEST(RowPartitioner, Finalise) { TestFinalise(); }
void TestIncorrectRow() {

78
tests/cpp/tree/test_approx.cc
View File

@@ -26,7 +26,7 @@ TEST(Approx, Partitioner) {
std::transform(grad.HostVector().cbegin(), grad.HostVector().cend(), hess.begin(),
[](auto gpair) { return gpair.GetHess(); });
for (auto const &page : Xy->GetBatches<GHistIndexMatrix>({64, hess, true})) {
for (auto const& page : Xy->GetBatches<GHistIndexMatrix>({64, hess, true})) {
bst_feature_t const split_ind = 0;
{
auto min_value = page.cut.MinValues()[split_ind];
@@ -44,9 +44,9 @@ TEST(Approx, Partitioner) {
float split_value = page.cut.Values().at(ptr / 2);
RegTree tree;
GetSplit(&tree, split_value, &candidates);
auto left_nidx = tree[RegTree::kRoot].LeftChild();
partitioner.UpdatePosition(&ctx, page, candidates, &tree);
auto left_nidx = tree[RegTree::kRoot].LeftChild();
auto elem = partitioner[left_nidx];
ASSERT_LT(elem.Size(), n_samples);
ASSERT_GT(elem.Size(), 1);
@@ -54,6 +54,7 @@ TEST(Approx, Partitioner) {
auto value = page.cut.Values().at(page.index[*it]);
ASSERT_LE(value, split_value);
}
auto right_nidx = tree[RegTree::kRoot].RightChild();
elem = partitioner[right_nidx];
for (auto it = elem.begin; it != elem.end; ++it) {
@@ -63,5 +64,78 @@ TEST(Approx, Partitioner) {
}
}
}
namespace {
void TestLeafPartition(size_t n_samples) {
size_t const n_features = 2, base_rowid = 0;
common::RowSetCollection row_set;
ApproxRowPartitioner partitioner{n_samples, base_rowid};
auto Xy = RandomDataGenerator{n_samples, n_features, 0}.GenerateDMatrix(true);
GenericParameter ctx;
std::vector<CPUExpandEntry> candidates{{0, 0, 0.4}};
RegTree tree;
std::vector<float> hess(n_samples, 0);
// emulate sampling
auto not_sampled = [](size_t i) {
size_t const kSampleFactor{3};
return i % kSampleFactor != 0;
};
size_t n{0};
for (size_t i = 0; i < hess.size(); ++i) {
if (not_sampled(i)) {
hess[i] = 1.0f;
++n;
}
}
std::vector<size_t> h_nptr;
float split_value{0};
for (auto const& page : Xy->GetBatches<GHistIndexMatrix>({Context::kCpuId, 64})) {
bst_feature_t const split_ind = 0;
auto ptr = page.cut.Ptrs()[split_ind + 1];
split_value = page.cut.Values().at(ptr / 2);
GetSplit(&tree, split_value, &candidates);
partitioner.UpdatePosition(&ctx, page, candidates, &tree);
std::vector<bst_node_t> position;
partitioner.LeafPartition(&ctx, tree, hess, &position);
std::sort(position.begin(), position.end());
size_t beg = std::distance(
position.begin(),
std::find_if(position.begin(), position.end(), [&](bst_node_t nidx) { return nidx >= 0; }));
std::vector<size_t> nptr;
common::RunLengthEncode(position.cbegin() + beg, position.cend(), &nptr);
std::transform(nptr.begin(), nptr.end(), nptr.begin(), [&](size_t x) { return x + beg; });
auto n_uniques = std::unique(position.begin() + beg, position.end()) - (position.begin() + beg);
ASSERT_EQ(nptr.size(), n_uniques + 1);
ASSERT_EQ(nptr[0], beg);
ASSERT_EQ(nptr.back(), n_samples);
h_nptr = nptr;
}
if (h_nptr.front() == n_samples) {
return;
}
ASSERT_GE(h_nptr.size(), 2);
for (auto const& page : Xy->GetBatches<SparsePage>()) {
auto batch = page.GetView();
size_t left{0};
for (size_t i = 0; i < batch.Size(); ++i) {
if (not_sampled(i) && batch[i].front().fvalue < split_value) {
left++;
}
}
ASSERT_EQ(left, h_nptr[1] - h_nptr[0]); // equal to number of sampled assigned to left
}
}
} // anonymous namespace
TEST(Approx, LeafPartition) {
for (auto n_samples : {0ul, 1ul, 128ul, 256ul}) {
TestLeafPartition(n_samples);
}
}
} // namespace tree
} // namespace xgboost

23
tests/cpp/tree/test_gpu_hist.cu
View File

@@ -1,5 +1,5 @@
/*!
* Copyright 2017-2021 XGBoost contributors
* Copyright 2017-2022 XGBoost contributors
*/
#include <gtest/gtest.h>
#include <thrust/device_vector.h>
@@ -13,6 +13,7 @@
#include "../helpers.h"
#include "../histogram_helpers.h"
#include "xgboost/generic_parameters.h"
#include "xgboost/json.h"
#include "../../../src/data/sparse_page_source.h"
#include "../../../src/tree/updater_gpu_hist.cu"
@@ -22,7 +23,6 @@
namespace xgboost {
namespace tree {
TEST(GpuHist, DeviceHistogram) {
// Ensures that node allocates correctly after reaching `kStopGrowingSize`.
dh::safe_cuda(cudaSetDevice(0));
@@ -81,8 +81,9 @@ void TestBuildHist(bool use_shared_memory_histograms) {
param.Init(args);
auto page = BuildEllpackPage(kNRows, kNCols);
BatchParam batch_param{};
GPUHistMakerDevice<GradientSumT> maker(0, page.get(), {}, kNRows, param,
kNCols, kNCols, batch_param);
Context ctx{CreateEmptyGenericParam(0)};
GPUHistMakerDevice<GradientSumT> maker(&ctx, page.get(), {}, kNRows, param, kNCols, kNCols,
batch_param);
xgboost::SimpleLCG gen;
xgboost::SimpleRealUniformDistribution<bst_float> dist(0.0f, 1.0f);
HostDeviceVector<GradientPair> gpair(kNRows);
@@ -158,14 +159,14 @@ TEST(GpuHist, ApplySplit) {
BatchParam bparam;
bparam.gpu_id = 0;
bparam.max_bin = 3;
Context ctx{CreateEmptyGenericParam(0)};
for (auto& ellpack : m->GetBatches<EllpackPage>(bparam)){
auto impl = ellpack.Impl();
HostDeviceVector<FeatureType> feature_types(10, FeatureType::kCategorical);
feature_types.SetDevice(bparam.gpu_id);
tree::GPUHistMakerDevice<GradientPairPrecise> updater(
0, impl, feature_types.ConstDeviceSpan(), n_rows, tparam, 0, n_cols,
bparam);
&ctx, impl, feature_types.ConstDeviceSpan(), n_rows, tparam, 0, n_cols, bparam);
updater.ApplySplit(candidate, &tree);
ASSERT_EQ(tree.GetSplitTypes().size(), 3);
@@ -224,8 +225,9 @@ TEST(GpuHist, EvaluateRootSplit) {
// Initialize GPUHistMakerDevice
auto page = BuildEllpackPage(kNRows, kNCols);
BatchParam batch_param{};
GPUHistMakerDevice<GradientPairPrecise> maker(
0, page.get(), {}, kNRows, param, kNCols, kNCols, batch_param);
Context ctx{CreateEmptyGenericParam(0)};
GPUHistMakerDevice<GradientPairPrecise> maker(&ctx, page.get(), {}, kNRows, param, kNCols, kNCols,
batch_param);
// Initialize GPUHistMakerDevice::node_sum_gradients
maker.node_sum_gradients = {};
@@ -348,7 +350,8 @@ void UpdateTree(HostDeviceVector<GradientPair>* gpair, DMatrix* dmat,
GenericParameter generic_param(CreateEmptyGenericParam(0));
hist_maker.Configure(args, &generic_param);
hist_maker.Update(gpair, dmat, {tree});
std::vector<HostDeviceVector<bst_node_t>> position(1);
hist_maker.Update(gpair, dmat, common::Span<HostDeviceVector<bst_node_t>>{position}, {tree});
auto cache = linalg::VectorView<float>{preds->DeviceSpan(), {preds->Size()}, 0};
hist_maker.UpdatePredictionCache(dmat, cache);
}
@@ -483,7 +486,7 @@ TEST(GpuHist, ExternalMemoryWithSampling) {
auto preds_h = preds.ConstHostVector();
auto preds_ext_h = preds_ext.ConstHostVector();
for (int i = 0; i < kRows; i++) {
EXPECT_NEAR(preds_h[i], preds_ext_h[i], 1e-3);
ASSERT_NEAR(preds_h[i], preds_ext_h[i], 1e-3);
}
}

6
tests/cpp/tree/test_histmaker.cc
View File

@@ -39,7 +39,8 @@ TEST(GrowHistMaker, InteractionConstraint) {
updater->Configure(Args{
{"interaction_constraints", "[[0, 1]]"},
{"num_feature", std::to_string(kCols)}});
updater->Update(&gradients, p_dmat.get(), {&tree});
std::vector<HostDeviceVector<bst_node_t>> position(1);
updater->Update(&gradients, p_dmat.get(), position, {&tree});
ASSERT_EQ(tree.NumExtraNodes(), 4);
ASSERT_EQ(tree[0].SplitIndex(), 1);
@@ -55,7 +56,8 @@ TEST(GrowHistMaker, InteractionConstraint) {
std::unique_ptr<TreeUpdater> updater{
TreeUpdater::Create("grow_histmaker", &param, ObjInfo{ObjInfo::kRegression})};
updater->Configure(Args{{"num_feature", std::to_string(kCols)}});
updater->Update(&gradients, p_dmat.get(), {&tree});
std::vector<HostDeviceVector<bst_node_t>> position(1);
updater->Update(&gradients, p_dmat.get(), position, {&tree});
ASSERT_EQ(tree.NumExtraNodes(), 10);
ASSERT_EQ(tree[0].SplitIndex(), 1);

3
tests/cpp/tree/test_prediction_cache.cc
View File

@@ -77,7 +77,8 @@ class TestPredictionCache : public ::testing::Test {
std::vector<RegTree *> trees{&tree};
auto gpair = GenerateRandomGradients(n_samples_);
updater->Configure(Args{{"max_bin", "64"}});
updater->Update(&gpair, Xy_.get(), trees);
std::vector<HostDeviceVector<bst_node_t>> position(1);
updater->Update(&gpair, Xy_.get(), position, trees);
HostDeviceVector<float> out_prediction_cached;
out_prediction_cached.SetDevice(ctx.gpu_id);
out_prediction_cached.Resize(n_samples_);

11
tests/cpp/tree/test_prune.cc
View File

@@ -43,22 +43,23 @@ TEST(Updater, Prune) {
pruner->Configure(cfg);
// loss_chg < min_split_loss;
std::vector<HostDeviceVector<bst_node_t>> position(trees.size());
tree.ExpandNode(0, 0, 0, true, 0.0f, 0.3f, 0.4f, 0.0f, 0.0f,
/*left_sum=*/0.0f, /*right_sum=*/0.0f);
pruner->Update(&gpair, p_dmat.get(), trees);
pruner->Update(&gpair, p_dmat.get(), position, trees);
ASSERT_EQ(tree.NumExtraNodes(), 0);
// loss_chg > min_split_loss;
tree.ExpandNode(0, 0, 0, true, 0.0f, 0.3f, 0.4f, 11.0f, 0.0f,
/*left_sum=*/0.0f, /*right_sum=*/0.0f);
pruner->Update(&gpair, p_dmat.get(), trees);
pruner->Update(&gpair, p_dmat.get(), position, trees);
ASSERT_EQ(tree.NumExtraNodes(), 2);
// loss_chg == min_split_loss;
tree.Stat(0).loss_chg = 10;
pruner->Update(&gpair, p_dmat.get(), trees);
pruner->Update(&gpair, p_dmat.get(), position, trees);
ASSERT_EQ(tree.NumExtraNodes(), 2);
@@ -74,7 +75,7 @@ TEST(Updater, Prune) {
/*left_sum=*/0.0f, /*right_sum=*/0.0f);
cfg.emplace_back(std::make_pair("max_depth", "1"));
pruner->Configure(cfg);
pruner->Update(&gpair, p_dmat.get(), trees);
pruner->Update(&gpair, p_dmat.get(), position, trees);
ASSERT_EQ(tree.NumExtraNodes(), 2);
@@ -84,7 +85,7 @@ TEST(Updater, Prune) {
/*left_sum=*/0.0f, /*right_sum=*/0.0f);
cfg.emplace_back(std::make_pair("min_split_loss", "0"));
pruner->Configure(cfg);
pruner->Update(&gpair, p_dmat.get(), trees);
pruner->Update(&gpair, p_dmat.get(), position, trees);
ASSERT_EQ(tree.NumExtraNodes(), 2);
}
} // namespace tree

3
tests/cpp/tree/test_refresh.cc
View File

@@ -44,7 +44,8 @@ TEST(Updater, Refresh) {
tree.Stat(cright).base_weight = 1.3;
refresher->Configure(cfg);
refresher->Update(&gpair, p_dmat.get(), trees);
std::vector<HostDeviceVector<bst_node_t>> position;
refresher->Update(&gpair, p_dmat.get(), position, trees);
bst_float constexpr kEps = 1e-6;
ASSERT_NEAR(-0.183392, tree[cright].LeafValue(), kEps);

12
tests/cpp/tree/test_tree_stat.cc
View File

@@ -27,7 +27,8 @@ class UpdaterTreeStatTest : public ::testing::Test {
up->Configure(Args{});
RegTree tree;
tree.param.num_feature = kCols;
up->Update(&gpairs_, p_dmat_.get(), {&tree});
std::vector<HostDeviceVector<bst_node_t>> position(1);
up->Update(&gpairs_, p_dmat_.get(), position, {&tree});
tree.WalkTree([&tree](bst_node_t nidx) {
if (tree[nidx].IsLeaf()) {
@@ -87,13 +88,15 @@ class UpdaterEtaTest : public ::testing::Test {
RegTree tree_0;
{
tree_0.param.num_feature = kCols;
up_0->Update(&gpairs_, p_dmat_.get(), {&tree_0});
std::vector<HostDeviceVector<bst_node_t>> position(1);
up_0->Update(&gpairs_, p_dmat_.get(), position, {&tree_0});
}
RegTree tree_1;
{
tree_1.param.num_feature = kCols;
up_1->Update(&gpairs_, p_dmat_.get(), {&tree_1});
std::vector<HostDeviceVector<bst_node_t>> position(1);
up_1->Update(&gpairs_, p_dmat_.get(), position, {&tree_1});
}
tree_0.WalkTree([&](bst_node_t nidx) {
if (tree_0[nidx].IsLeaf()) {
@@ -149,7 +152,8 @@ class TestMinSplitLoss : public ::testing::Test {
up->Configure(args);
RegTree tree;
up->Update(&gpair_, dmat_.get(), {&tree});
std::vector<HostDeviceVector<bst_node_t>> position(1);
up->Update(&gpair_, dmat_.get(), position, {&tree});
auto n_nodes = tree.NumExtraNodes();
return n_nodes;