Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use integer gradients in gpu_hist split evaluation (#8274)

Browse Source
This commit is contained in:
Rory Mitchell
2022-10-11 12:16:27 +02:00
committed by GitHub
parent c68684ff4c
commit 210915c985
12 changed files with 224 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
tests/cpp/tree/gpu_hist/test_evaluate_splits.cu
View File

@@ -22,10 +22,10 @@ auto ZeroParam() {
} // anonymous namespace
inline GradientQuantizer DummyRoundingFactor() {
inline GradientQuantiser DummyRoundingFactor() {
thrust::device_vector<GradientPair> gpair(1);
gpair[0] = {1000.f, 1000.f}; // Tests should not exceed sum of 1000
return GradientQuantizer(dh::ToSpan(gpair));
return GradientQuantiser(dh::ToSpan(gpair));
}
thrust::device_vector<GradientPairInt64> ConvertToInteger(std::vector<GradientPairPrecise> x) {
@@ -48,16 +48,16 @@ TEST_F(TestCategoricalSplitWithMissing, GPUHistEvaluator) {
dh::device_vector<FeatureType> feature_types(feature_set.size(), FeatureType::kCategorical);
auto d_feature_types = dh::ToSpan(feature_types);
EvaluateSplitInputs input{1, 0, parent_sum_, dh::ToSpan(feature_set),
auto quantiser = DummyRoundingFactor();
EvaluateSplitInputs input{1, 0, quantiser.ToFixedPoint(parent_sum_), dh::ToSpan(feature_set),
dh::ToSpan(feature_histogram)};
EvaluateSplitSharedInputs shared_inputs{
param,
DummyRoundingFactor(),
quantiser,
d_feature_types,
cuts_.cut_ptrs_.ConstDeviceSpan(),
cuts_.cut_values_.ConstDeviceSpan(),
cuts_.min_vals_.ConstDeviceSpan(),
cuts_.min_vals_.ConstDeviceSpan(), false
};
GPUHistEvaluator evaluator{param_, static_cast<bst_feature_t>(feature_set.size()), 0};
@@ -67,7 +67,7 @@ TEST_F(TestCategoricalSplitWithMissing, GPUHistEvaluator) {
ASSERT_EQ(result.thresh, 1);
this->CheckResult(result.loss_chg, result.findex, result.fvalue, result.is_cat,
result.dir == kLeftDir, result.left_sum, result.right_sum);
result.dir == kLeftDir, quantiser.ToFloatingPoint(result.left_sum), quantiser.ToFloatingPoint(result.right_sum));
}
TEST(GpuHist, PartitionBasic) {
@@ -91,10 +91,10 @@ TEST(GpuHist, PartitionBasic) {
*std::max_element(cuts.cut_values_.HostVector().begin(), cuts.cut_values_.HostVector().end());
cuts.SetCategorical(true, max_cat);
d_feature_types = dh::ToSpan(feature_types);
auto quantiser = DummyRoundingFactor();
EvaluateSplitSharedInputs shared_inputs{
param,
DummyRoundingFactor(),
quantiser,
d_feature_types,
cuts.cut_ptrs_.ConstDeviceSpan(),
cuts.cut_values_.ConstDeviceSpan(),
@@ -107,7 +107,7 @@ TEST(GpuHist, PartitionBasic) {
{
// -1.0s go right
// -3.0s go left
GradientPairPrecise parent_sum(-5.0, 3.0);
auto parent_sum = quantiser.ToFixedPoint(GradientPairPrecise{-5.0, 3.0});
auto feature_histogram = ConvertToInteger({{-1.0, 1.0}, {-1.0, 1.0}, {-3.0, 1.0}});
EvaluateSplitInputs input{0, 0, parent_sum, dh::ToSpan(feature_set),
dh::ToSpan(feature_histogram)};
@@ -115,14 +115,13 @@ TEST(GpuHist, PartitionBasic) {
auto cats = std::bitset<32>(evaluator.GetHostNodeCats(input.nidx)[0]);
EXPECT_EQ(result.dir, kLeftDir);
EXPECT_EQ(cats, std::bitset<32>("11000000000000000000000000000000"));
EXPECT_FLOAT_EQ(result.left_sum.GetGrad() + result.right_sum.GetGrad(), parent_sum.GetGrad());
EXPECT_FLOAT_EQ(result.left_sum.GetHess() + result.right_sum.GetHess(), parent_sum.GetHess());
EXPECT_EQ(result.left_sum + result.right_sum, parent_sum);
}
{
// -1.0s go right
// -3.0s go left
GradientPairPrecise parent_sum(-7.0, 3.0);
auto parent_sum = quantiser.ToFixedPoint(GradientPairPrecise{-7.0, 3.0});
auto feature_histogram = ConvertToInteger({{-1.0, 1.0}, {-3.0, 1.0}, {-3.0, 1.0}});
EvaluateSplitInputs input{1, 0, parent_sum, dh::ToSpan(feature_set),
dh::ToSpan(feature_histogram)};
@@ -130,25 +129,23 @@ TEST(GpuHist, PartitionBasic) {
auto cats = std::bitset<32>(evaluator.GetHostNodeCats(input.nidx)[0]);
EXPECT_EQ(result.dir, kLeftDir);
EXPECT_EQ(cats, std::bitset<32>("10000000000000000000000000000000"));
EXPECT_FLOAT_EQ(result.left_sum.GetGrad() + result.right_sum.GetGrad(), parent_sum.GetGrad());
EXPECT_FLOAT_EQ(result.left_sum.GetHess() + result.right_sum.GetHess(), parent_sum.GetHess());
EXPECT_EQ(result.left_sum + result.right_sum, parent_sum);
}
{
// All -1.0, gain from splitting should be 0.0
GradientPairPrecise parent_sum(-3.0, 3.0);
auto parent_sum = quantiser.ToFixedPoint(GradientPairPrecise{-3.0, 3.0});
auto feature_histogram = ConvertToInteger({{-1.0, 1.0}, {-1.0, 1.0}, {-1.0, 1.0}});
EvaluateSplitInputs input{2, 0, parent_sum, dh::ToSpan(feature_set),
dh::ToSpan(feature_histogram)};
DeviceSplitCandidate result = evaluator.EvaluateSingleSplit(input, shared_inputs).split;
EXPECT_EQ(result.dir, kLeftDir);
EXPECT_FLOAT_EQ(result.loss_chg, 0.0f);
EXPECT_FLOAT_EQ(result.left_sum.GetGrad() + result.right_sum.GetGrad(), parent_sum.GetGrad());
EXPECT_FLOAT_EQ(result.left_sum.GetHess() + result.right_sum.GetHess(), parent_sum.GetHess());
EXPECT_EQ(result.left_sum + result.right_sum, parent_sum);
}
// With 3.0/3.0 missing values
// Forward, first 2 categories are selected, while the last one go to left along with missing value
{
GradientPairPrecise parent_sum(0.0, 6.0);
auto parent_sum = quantiser.ToFixedPoint(GradientPairPrecise{0.0, 6.0});
auto feature_histogram = ConvertToInteger({{-1.0, 1.0}, {-1.0, 1.0}, {-1.0, 1.0}});
EvaluateSplitInputs input{3, 0, parent_sum, dh::ToSpan(feature_set),
dh::ToSpan(feature_histogram)};
@@ -156,13 +153,12 @@ TEST(GpuHist, PartitionBasic) {
auto cats = std::bitset<32>(evaluator.GetHostNodeCats(input.nidx)[0]);
EXPECT_EQ(cats, std::bitset<32>("11000000000000000000000000000000"));
EXPECT_EQ(result.dir, kLeftDir);
EXPECT_FLOAT_EQ(result.left_sum.GetGrad() + result.right_sum.GetGrad(), parent_sum.GetGrad());
EXPECT_FLOAT_EQ(result.left_sum.GetHess() + result.right_sum.GetHess(), parent_sum.GetHess());
EXPECT_EQ(result.left_sum + result.right_sum, parent_sum);
}
{
// -1.0s go right
// -3.0s go left
GradientPairPrecise parent_sum(-5.0, 3.0);
auto parent_sum = quantiser.ToFixedPoint(GradientPairPrecise{-5.0, 3.0});
auto feature_histogram = ConvertToInteger({{-1.0, 1.0}, {-3.0, 1.0}, {-1.0, 1.0}});
EvaluateSplitInputs input{4, 0, parent_sum, dh::ToSpan(feature_set),
dh::ToSpan(feature_histogram)};
@@ -170,21 +166,19 @@ TEST(GpuHist, PartitionBasic) {
auto cats = std::bitset<32>(evaluator.GetHostNodeCats(input.nidx)[0]);
EXPECT_EQ(result.dir, kLeftDir);
EXPECT_EQ(cats, std::bitset<32>("10100000000000000000000000000000"));
EXPECT_FLOAT_EQ(result.left_sum.GetGrad() + result.right_sum.GetGrad(), parent_sum.GetGrad());
EXPECT_FLOAT_EQ(result.left_sum.GetHess() + result.right_sum.GetHess(), parent_sum.GetHess());
EXPECT_EQ(result.left_sum + result.right_sum, parent_sum);
}
{
// -1.0s go right
// -3.0s go left
GradientPairPrecise parent_sum(-5.0, 3.0);
auto parent_sum = quantiser.ToFixedPoint(GradientPairPrecise{-5.0, 3.0});
auto feature_histogram = ConvertToInteger({{-3.0, 1.0}, {-1.0, 1.0}, {-3.0, 1.0}});
EvaluateSplitInputs input{5, 0, parent_sum, dh::ToSpan(feature_set),
dh::ToSpan(feature_histogram)};
DeviceSplitCandidate result = evaluator.EvaluateSingleSplit(input, shared_inputs).split;
auto cats = std::bitset<32>(evaluator.GetHostNodeCats(input.nidx)[0]);
EXPECT_EQ(cats, std::bitset<32>("01000000000000000000000000000000"));
EXPECT_FLOAT_EQ(result.left_sum.GetGrad() + result.right_sum.GetGrad(), parent_sum.GetGrad());
EXPECT_FLOAT_EQ(result.left_sum.GetHess() + result.right_sum.GetHess(), parent_sum.GetHess());
EXPECT_EQ(result.left_sum + result.right_sum, parent_sum);
}
}
@@ -209,9 +203,10 @@ TEST(GpuHist, PartitionTwoFeatures) {
*std::max_element(cuts.cut_values_.HostVector().begin(), cuts.cut_values_.HostVector().end());
cuts.SetCategorical(true, max_cat);
auto quantiser = DummyRoundingFactor();
EvaluateSplitSharedInputs shared_inputs{
param,
DummyRoundingFactor(),
quantiser,
d_feature_types,
cuts.cut_ptrs_.ConstDeviceSpan(),
cuts.cut_values_.ConstDeviceSpan(),
@@ -222,7 +217,7 @@ TEST(GpuHist, PartitionTwoFeatures) {
evaluator.Reset(cuts, dh::ToSpan(feature_types), feature_set.size(), tparam, 0);
{
GradientPairPrecise parent_sum(-6.0, 3.0);
auto parent_sum = quantiser.ToFixedPoint(GradientPairPrecise{-6.0, 3.0});
auto feature_histogram = ConvertToInteger({ {-2.0, 1.0}, {-2.0, 1.0}, {-2.0, 1.0}, {-1.0, 1.0}, {-1.0, 1.0}, {-4.0, 1.0}});
EvaluateSplitInputs input{0, 0, parent_sum, dh::ToSpan(feature_set),
dh::ToSpan(feature_histogram)};
@@ -230,12 +225,11 @@ TEST(GpuHist, PartitionTwoFeatures) {
auto cats = std::bitset<32>(evaluator.GetHostNodeCats(input.nidx)[0]);
EXPECT_EQ(result.findex, 1);
EXPECT_EQ(cats, std::bitset<32>("11000000000000000000000000000000"));
EXPECT_FLOAT_EQ(result.left_sum.GetGrad() + result.right_sum.GetGrad(), parent_sum.GetGrad());
EXPECT_FLOAT_EQ(result.left_sum.GetHess() + result.right_sum.GetHess(), parent_sum.GetHess());
EXPECT_EQ(result.left_sum + result.right_sum, parent_sum);
}
{
GradientPairPrecise parent_sum(-6.0, 3.0);
auto parent_sum = quantiser.ToFixedPoint(GradientPairPrecise{-6.0, 3.0});
auto feature_histogram = ConvertToInteger({ {-2.0, 1.0}, {-2.0, 1.0}, {-2.0, 1.0}, {-1.0, 1.0}, {-2.5, 1.0}, {-2.5, 1.0}});
EvaluateSplitInputs input{1, 0, parent_sum, dh::ToSpan(feature_set),
dh::ToSpan(feature_histogram)};
@@ -243,8 +237,7 @@ TEST(GpuHist, PartitionTwoFeatures) {
auto cats = std::bitset<32>(evaluator.GetHostNodeCats(input.nidx)[0]);
EXPECT_EQ(result.findex, 1);
EXPECT_EQ(cats, std::bitset<32>("10000000000000000000000000000000"));
EXPECT_FLOAT_EQ(result.left_sum.GetGrad() + result.right_sum.GetGrad(), parent_sum.GetGrad());
EXPECT_FLOAT_EQ(result.left_sum.GetHess() + result.right_sum.GetHess(), parent_sum.GetHess());
EXPECT_EQ(result.left_sum + result.right_sum, parent_sum);
}
}
@@ -269,9 +262,10 @@ TEST(GpuHist, PartitionTwoNodes) {
*std::max_element(cuts.cut_values_.HostVector().begin(), cuts.cut_values_.HostVector().end());
cuts.SetCategorical(true, max_cat);
auto quantiser = DummyRoundingFactor();
EvaluateSplitSharedInputs shared_inputs{
param,
DummyRoundingFactor(),
quantiser,
d_feature_types,
cuts.cut_ptrs_.ConstDeviceSpan(),
cuts.cut_values_.ConstDeviceSpan(),
@@ -282,7 +276,7 @@ TEST(GpuHist, PartitionTwoNodes) {
evaluator.Reset(cuts, dh::ToSpan(feature_types), feature_set.size(), tparam, 0);
{
GradientPairPrecise parent_sum(-6.0, 3.0);
auto parent_sum = quantiser.ToFixedPoint(GradientPairPrecise{-6.0, 3.0});
auto feature_histogram_a = ConvertToInteger({{-1.0, 1.0}, {-2.5, 1.0}, {-2.5, 1.0},
{-1.0, 1.0}, {-1.0, 1.0}, {-4.0, 1.0}});
thrust::device_vector<EvaluateSplitInputs> inputs(2);
@@ -303,7 +297,8 @@ TEST(GpuHist, PartitionTwoNodes) {
}
void TestEvaluateSingleSplit(bool is_categorical) {
GradientPairPrecise parent_sum(0.0, 1.0);
auto quantiser = DummyRoundingFactor();
auto parent_sum = quantiser.ToFixedPoint(GradientPairPrecise{0.0, 1.0});
TrainParam tparam = ZeroParam();
GPUTrainingParam param{tparam};
@@ -327,7 +322,7 @@ void TestEvaluateSingleSplit(bool is_categorical) {
dh::ToSpan(feature_histogram)};
EvaluateSplitSharedInputs shared_inputs{
param,
DummyRoundingFactor(),
quantiser,
d_feature_types,
cuts.cut_ptrs_.ConstDeviceSpan(),
cuts.cut_values_.ConstDeviceSpan(),
@@ -345,10 +340,7 @@ void TestEvaluateSingleSplit(bool is_categorical) {
} else {
EXPECT_EQ(result.fvalue, 11.0);
}
EXPECT_FLOAT_EQ(result.left_sum.GetGrad() + result.right_sum.GetGrad(),
parent_sum.GetGrad());
EXPECT_FLOAT_EQ(result.left_sum.GetHess() + result.right_sum.GetHess(),
parent_sum.GetHess());
EXPECT_EQ(result.left_sum + result.right_sum, parent_sum);
}
TEST(GpuHist, EvaluateSingleSplit) {
@@ -360,7 +352,8 @@ TEST(GpuHist, EvaluateSingleCategoricalSplit) {
}
TEST(GpuHist, EvaluateSingleSplitMissing) {
GradientPairPrecise parent_sum(1.0, 1.5);
auto quantiser = DummyRoundingFactor();
auto parent_sum = quantiser.ToFixedPoint(GradientPairPrecise{1.0, 1.5});
TrainParam tparam = ZeroParam();
GPUTrainingParam param{tparam};
@@ -377,7 +370,7 @@ TEST(GpuHist, EvaluateSingleSplitMissing) {
dh::ToSpan(feature_histogram)};
EvaluateSplitSharedInputs shared_inputs{
param,
DummyRoundingFactor(),
quantiser,
{},
dh::ToSpan(feature_segments),
dh::ToSpan(feature_values),
@@ -390,8 +383,8 @@ TEST(GpuHist, EvaluateSingleSplitMissing) {
EXPECT_EQ(result.findex, 0);
EXPECT_EQ(result.fvalue, 1.0);
EXPECT_EQ(result.dir, kRightDir);
EXPECT_EQ(result.left_sum, GradientPairPrecise(-0.5, 0.5));
EXPECT_EQ(result.right_sum, GradientPairPrecise(1.5, 1.0));
EXPECT_EQ(result.left_sum,quantiser.ToFixedPoint(GradientPairPrecise(-0.5, 0.5)));
EXPECT_EQ(result.right_sum, quantiser.ToFixedPoint(GradientPairPrecise(1.5, 1.0)));
}
TEST(GpuHist, EvaluateSingleSplitEmpty) {
@@ -409,7 +402,8 @@ TEST(GpuHist, EvaluateSingleSplitEmpty) {
// Feature 0 has a better split, but the algorithm must select feature 1
TEST(GpuHist, EvaluateSingleSplitFeatureSampling) {
GradientPairPrecise parent_sum(0.0, 1.0);
auto quantiser = DummyRoundingFactor();
auto parent_sum = quantiser.ToFixedPoint(GradientPairPrecise{0.0, 1.0});
TrainParam tparam = ZeroParam();
tparam.UpdateAllowUnknown(Args{});
GPUTrainingParam param{tparam};
@@ -429,7 +423,7 @@ TEST(GpuHist, EvaluateSingleSplitFeatureSampling) {
dh::ToSpan(feature_histogram)};
EvaluateSplitSharedInputs shared_inputs{
param,
DummyRoundingFactor(),
quantiser,
{},
dh::ToSpan(feature_segments),
dh::ToSpan(feature_values),
@@ -441,13 +435,14 @@ TEST(GpuHist, EvaluateSingleSplitFeatureSampling) {
EXPECT_EQ(result.findex, 1);
EXPECT_EQ(result.fvalue, 11.0);
EXPECT_EQ(result.left_sum, GradientPairPrecise(-0.5, 0.5));
EXPECT_EQ(result.right_sum, GradientPairPrecise(0.5, 0.5));
EXPECT_EQ(result.left_sum,quantiser.ToFixedPoint(GradientPairPrecise(-0.5, 0.5)));
EXPECT_EQ(result.right_sum, quantiser.ToFixedPoint(GradientPairPrecise(0.5, 0.5)));
}
// Features 0 and 1 have identical gain, the algorithm must select 0
TEST(GpuHist, EvaluateSingleSplitBreakTies) {
GradientPairPrecise parent_sum(0.0, 1.0);
auto quantiser = DummyRoundingFactor();
auto parent_sum = quantiser.ToFixedPoint(GradientPairPrecise{0.0, 1.0});
TrainParam tparam = ZeroParam();
tparam.UpdateAllowUnknown(Args{});
GPUTrainingParam param{tparam};
@@ -467,7 +462,7 @@ TEST(GpuHist, EvaluateSingleSplitBreakTies) {
dh::ToSpan(feature_histogram)};
EvaluateSplitSharedInputs shared_inputs{
param,
DummyRoundingFactor(),
quantiser,
{},
dh::ToSpan(feature_segments),
dh::ToSpan(feature_values),
@@ -483,7 +478,8 @@ TEST(GpuHist, EvaluateSingleSplitBreakTies) {
TEST(GpuHist, EvaluateSplits) {
thrust::device_vector<DeviceSplitCandidate> out_splits(2);
GradientPairPrecise parent_sum(0.0, 1.0);
auto quantiser = DummyRoundingFactor();
auto parent_sum = quantiser.ToFixedPoint(GradientPairPrecise{0.0, 1.0});
TrainParam tparam = ZeroParam();
tparam.UpdateAllowUnknown(Args{});
GPUTrainingParam param{tparam};
@@ -510,7 +506,7 @@ TEST(GpuHist, EvaluateSplits) {
dh::ToSpan(feature_histogram_right)};
EvaluateSplitSharedInputs shared_inputs{
param,
DummyRoundingFactor(),
quantiser,
{},
dh::ToSpan(feature_segments),
dh::ToSpan(feature_values),
@@ -543,18 +539,18 @@ TEST_F(TestPartitionBasedSplit, GpuHist) {
evaluator.Reset(cuts_, dh::ToSpan(ft), info_.num_col_, param_, 0);
// Convert the sample histogram to fixed point
auto rounding = DummyRoundingFactor();
auto quantiser = DummyRoundingFactor();
thrust::host_vector<GradientPairInt64> h_hist;
for(auto e: hist_[0]){
h_hist.push_back(rounding.ToFixedPoint({float(e.GetGrad()),float(e.GetHess())}));
h_hist.push_back(quantiser.ToFixedPoint(e));
}
dh::device_vector<GradientPairInt64> d_hist = h_hist;
dh::device_vector<bst_feature_t> feature_set{std::vector<bst_feature_t>{0}};
EvaluateSplitInputs input{0, 0, total_gpair_, dh::ToSpan(feature_set), dh::ToSpan(d_hist)};
EvaluateSplitInputs input{0, 0, quantiser.ToFixedPoint(total_gpair_), dh::ToSpan(feature_set), dh::ToSpan(d_hist)};
EvaluateSplitSharedInputs shared_inputs{
GPUTrainingParam{param_},
rounding,
quantiser,
dh::ToSpan(ft),
cuts_.cut_ptrs_.ConstDeviceSpan(),
cuts_.cut_values_.ConstDeviceSpan(),