Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Feature interaction for GPU Hist. (#4534)

Browse Source 
* GPU hist Interaction Constraints.
* Duplicate related parameters.
* Add tests for CPU interaction constraint.
* Add better error reporting.
* Thorough tests.
This commit is contained in:
Jiaming Yuan
2019-06-19 18:11:02 +08:00
committed by GitHub
parent 570374effe
commit ae05948e32
14 changed files with 1201 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
tests/cpp/tree/test_bitfield.cu Normal file
View File

@@ -0,0 +1,60 @@
/*!
* Copyright 2019 XGBoost contributors
*/
#include <gtest/gtest.h>
#include <thrust/copy.h>
#include <thrust/device_vector.h>
#include <vector>
#include "../../../src/tree/constraints.cuh"
#include "../../../src/common/device_helpers.cuh"
namespace xgboost {
__global__ void TestSetKernel(BitField bits) {
auto tid = threadIdx.x + blockIdx.x * blockDim.x;
if (tid < bits.Size()) {
bits.Set(tid);
}
}
TEST(BitField, Set) {
dh::device_vector<BitField::value_type> storage;
uint32_t constexpr kBits = 128;
storage.resize(128);
auto bits = BitField(dh::ToSpan(storage));
TestSetKernel<<<1, kBits>>>(bits);
std::vector<BitField::value_type> h_storage(storage.size());
thrust::copy(storage.begin(), storage.end(), h_storage.begin());
BitField outputs {
common::Span<BitField::value_type>{h_storage.data(),
h_storage.data() + h_storage.size()}};
for (size_t i = 0; i < kBits; ++i) {
ASSERT_TRUE(outputs.Check(i));
}
}
__global__ void TestOrKernel(BitField lhs, BitField rhs) {
lhs |= rhs;
}
TEST(BitField, And) {
uint32_t constexpr kBits = 128;
dh::device_vector<BitField::value_type> lhs_storage(kBits);
dh::device_vector<BitField::value_type> rhs_storage(kBits);
auto lhs = BitField(dh::ToSpan(lhs_storage));
auto rhs = BitField(dh::ToSpan(rhs_storage));
thrust::fill(lhs_storage.begin(), lhs_storage.end(), 0UL);
thrust::fill(rhs_storage.begin(), rhs_storage.end(), ~static_cast<BitField::value_type>(0UL));
TestOrKernel<<<1, kBits>>>(lhs, rhs);
std::vector<BitField::value_type> h_storage(lhs_storage.size());
thrust::copy(lhs_storage.begin(), lhs_storage.end(), h_storage.begin());
BitField outputs {{h_storage.data(), h_storage.data() + h_storage.size()}};
for (size_t i = 0; i < kBits; ++i) {
ASSERT_TRUE(outputs.Check(i));
}
}
} // namespace xgboost

321
tests/cpp/tree/test_constraints.cu Normal file
View File

@@ -0,0 +1,321 @@
/*!
* Copyright 2019 XGBoost contributors
*/
#include <gtest/gtest.h>
#include <thrust/copy.h>
#include <thrust/device_vector.h>
#include <cinttypes>
#include <string>
#include <bitset>
#include <set>
#include "../../../src/tree/constraints.cuh"
#include "../../../src/tree/param.h"
#include "../../../src/common/device_helpers.cuh"
namespace xgboost {
namespace {
struct FConstraintWrapper : public FeatureInteractionConstraint {
common::Span<BitField> GetNodeConstraints() {
return FeatureInteractionConstraint::s_node_constraints_;
}
FConstraintWrapper(tree::TrainParam param, int32_t n_features) :
FeatureInteractionConstraint(param, n_features) {}
dh::device_vector<int32_t> const& GetDSets() const {
return d_sets_;
}
dh::device_vector<int32_t> const& GetDSetsPtr() const {
return d_sets_ptr_;
}
};
std::string GetConstraintsStr() {
std::string const constraints_str = R"constraint([[1, 2], [3, 4, 5]])constraint";
return constraints_str;
}
tree::TrainParam GetParameter() {
std::vector<std::pair<std::string, std::string>> args{
{"interaction_constraints", GetConstraintsStr()}
};
tree::TrainParam param;
param.Init(args);
return param;
}
void CompareBitField(BitField d_field, std::set<uint32_t> positions) {
std::vector<BitField::value_type> h_field_storage(d_field.bits_.size());
thrust::copy(thrust::device_ptr<BitField::value_type>(d_field.bits_.data()),
thrust::device_ptr<BitField::value_type>(
d_field.bits_.data() + d_field.bits_.size()),
h_field_storage.data());
BitField h_field;
h_field.bits_ = {h_field_storage.data(), h_field_storage.data() + h_field_storage.size()};
for (size_t i = 0; i < h_field.Size(); ++i) {
if (positions.find(i) != positions.cend()) {
ASSERT_TRUE(h_field.Check(i));
} else {
ASSERT_FALSE(h_field.Check(i));
}
}
}
} // anonymous namespace
TEST(FeatureInteractionConstraint, Init) {
{
int32_t constexpr kFeatures = 6;
tree::TrainParam param = GetParameter();
FConstraintWrapper constraints(param, kFeatures);
ASSERT_EQ(constraints.Features(), kFeatures);
common::Span<BitField> s_nodes_constraints = constraints.GetNodeConstraints();
for (BitField const& d_node : s_nodes_constraints) {
std::vector<BitField::value_type> h_node_storage(d_node.bits_.size());
thrust::copy(thrust::device_ptr<BitField::value_type>(d_node.bits_.data()),
thrust::device_ptr<BitField::value_type>(
d_node.bits_.data() + d_node.bits_.size()),
h_node_storage.data());
BitField h_node;
h_node.bits_ = {h_node_storage.data(), h_node_storage.data() + h_node_storage.size()};
// no feature is attached to node.
for (size_t i = 0; i < h_node.Size(); ++i) {
ASSERT_FALSE(h_node.Check(i));
}
}
}
{
// Test one feature in multiple sets
int32_t constexpr kFeatures = 7;
tree::TrainParam param = GetParameter();
param.interaction_constraints = R"([[0, 1, 3], [3, 5, 6]])";
FConstraintWrapper constraints(param, kFeatures);
std::vector<int32_t> h_sets {0, 0, 0, 1, 1, 1};
std::vector<int32_t> h_sets_ptr {0, 1, 2, 2, 4, 4, 5, 6};
auto d_sets = constraints.GetDSets();
ASSERT_EQ(h_sets.size(), d_sets.size());
auto d_sets_ptr = constraints.GetDSetsPtr();
ASSERT_EQ(h_sets_ptr, d_sets_ptr);
for (size_t i = 0; i < h_sets.size(); ++i) {
ASSERT_EQ(h_sets[i], d_sets[i]);
}
for (size_t i = 0; i < h_sets_ptr.size(); ++i) {
ASSERT_EQ(h_sets_ptr[i], d_sets_ptr[i]);
}
}
{
// Test having more than 1 BitField::value_type
int32_t constexpr kFeatures = 129;
tree::TrainParam param = GetParameter();
param.interaction_constraints = R"([[0, 1, 3], [3, 5, 128], [127, 128]])";
FConstraintWrapper constraints(param, kFeatures);
auto d_sets = constraints.GetDSets();
auto d_sets_ptr = constraints.GetDSetsPtr();
auto _128_beg = d_sets_ptr[128];
auto _128_end = d_sets_ptr[128 + 1];
ASSERT_EQ(_128_end - _128_beg, 2);
ASSERT_EQ(d_sets[_128_beg], 1);
ASSERT_EQ(d_sets[_128_end-1], 2);
}
}
TEST(FeatureInteractionConstraint, Split) {
tree::TrainParam param = GetParameter();
int32_t constexpr kFeatures = 6;
FConstraintWrapper constraints(param, kFeatures);
{
BitField d_node[3];
constraints.Split(0, /*feature_id=*/1, 1, 2);
for (size_t nid = 0; nid < 3; ++nid) {
d_node[nid] = constraints.GetNodeConstraints()[nid];
ASSERT_EQ(d_node[nid].bits_.size(), 1);
CompareBitField(d_node[nid], {1, 2});
}
}
{
BitField d_node[5];
constraints.Split(1, /*feature_id=*/0, /*left_id=*/3, /*right_id=*/4);
for (auto nid : {1, 3, 4}) {
d_node[nid] = constraints.GetNodeConstraints()[nid];
CompareBitField(d_node[nid], {0, 1, 2});
}
for (auto nid : {0, 2}) {
d_node[nid] = constraints.GetNodeConstraints()[nid];
CompareBitField(d_node[nid], {1, 2});
}
}
}
TEST(FeatureInteractionConstraint, QueryNode) {
tree::TrainParam param = GetParameter();
int32_t constexpr kFeatures = 6;
FConstraintWrapper constraints(param, kFeatures);
{
auto span = constraints.QueryNode(0);
ASSERT_EQ(span.size(), 0);
}
{
constraints.Split(/*node_id=*/ 0, /*feature_id=*/ 1, 1, 2);
auto span = constraints.QueryNode(0);
std::vector<int32_t> h_result (span.size());
thrust::copy(thrust::device_ptr<int32_t>(span.data()),
thrust::device_ptr<int32_t>(span.data() + span.size()),
h_result.begin());
ASSERT_EQ(h_result.size(), 2);
ASSERT_EQ(h_result[0], 1);
ASSERT_EQ(h_result[1], 2);
}
{
constraints.Split(1, /*feature_id=*/0, 3, 4);
auto span = constraints.QueryNode(1);
std::vector<int32_t> h_result (span.size());
thrust::copy(thrust::device_ptr<int32_t>(span.data()),
thrust::device_ptr<int32_t>(span.data() + span.size()),
h_result.begin());
ASSERT_EQ(h_result.size(), 3);
ASSERT_EQ(h_result[0], 0);
ASSERT_EQ(h_result[1], 1);
ASSERT_EQ(h_result[2], 2);
// same as parent
span = constraints.QueryNode(3);
h_result.resize(span.size());
thrust::copy(thrust::device_ptr<int32_t>(span.data()),
thrust::device_ptr<int32_t>(span.data() + span.size()),
h_result.begin());
ASSERT_EQ(h_result.size(), 3);
ASSERT_EQ(h_result[0], 0);
ASSERT_EQ(h_result[1], 1);
ASSERT_EQ(h_result[2], 2);
}
{
tree::TrainParam large_param = GetParameter();
large_param.interaction_constraints = R"([[1, 139], [244, 0], [139, 221]])";
FConstraintWrapper large_features(large_param, 256);
large_features.Split(0, 139, 1, 2);
auto span = large_features.QueryNode(0);
std::vector<int32_t> h_result (span.size());
thrust::copy(thrust::device_ptr<int32_t>(span.data()),
thrust::device_ptr<int32_t>(span.data() + span.size()),
h_result.begin());
ASSERT_EQ(h_result.size(), 3);
ASSERT_EQ(h_result[0], 1);
ASSERT_EQ(h_result[1], 139);
ASSERT_EQ(h_result[2], 221);
}
}
namespace {
void CompareFeatureList(common::Span<int32_t> s_output, std::vector<int32_t> solution) {
std::vector<int32_t> h_output(s_output.size());
thrust::copy(thrust::device_ptr<int32_t>(s_output.data()),
thrust::device_ptr<int32_t>(s_output.data() + s_output.size()),
h_output.begin());
ASSERT_EQ(h_output.size(), solution.size());
for (size_t i = 0; i < solution.size(); ++i) {
ASSERT_EQ(h_output[i], solution[i]);
}
}
} // anonymous namespace
TEST(FeatureInteractionConstraint, Query) {
{
tree::TrainParam param = GetParameter();
int32_t constexpr kFeatures = 6;
FConstraintWrapper constraints(param, kFeatures);
std::vector<int32_t> h_input_feature_list {0, 1, 2, 3, 4, 5};
dh::device_vector<int32_t> d_input_feature_list (h_input_feature_list);
common::Span<int32_t> s_input_feature_list = dh::ToSpan(d_input_feature_list);
auto s_output = constraints.Query(s_input_feature_list, 0);
CompareFeatureList(s_output, h_input_feature_list);
}
{
tree::TrainParam param = GetParameter();
int32_t constexpr kFeatures = 6;
FConstraintWrapper constraints(param, kFeatures);
constraints.Split(/*node_id=*/0, /*feature_id=*/1, /*left_id=*/1, /*right_id=*/2);
constraints.Split(/*node_id=*/1, /*feature_id=*/0, /*left_id=*/3, /*right_id=*/4);
constraints.Split(/*node_id=*/4, /*feature_id=*/3, /*left_id=*/5, /*right_id=*/6);
/*
* (node id) [allowed features]
*
* (0) [1, 2]
* / \
* {split at 0} \
* / \
* (1)[0, 1, 2] (2)[1, 2]
* / \
* / {split at 3}
* / \
* (3)[0, 1, 2] (4)[0, 1, 2, 3, 4, 5]
*
*/
std::vector<int32_t> h_input_feature_list {0, 1, 2, 3, 4, 5};
dh::device_vector<int32_t> d_input_feature_list (h_input_feature_list);
common::Span<int32_t> s_input_feature_list = dh::ToSpan(d_input_feature_list);
auto s_output = constraints.Query(s_input_feature_list, 1);
CompareFeatureList(s_output, {0, 1, 2});
s_output = constraints.Query(s_input_feature_list, 2);
CompareFeatureList(s_output, {1, 2});
s_output = constraints.Query(s_input_feature_list, 3);
CompareFeatureList(s_output, {0, 1, 2});
s_output = constraints.Query(s_input_feature_list, 4);
CompareFeatureList(s_output, {0, 1, 2, 3, 4, 5});
s_output = constraints.Query(s_input_feature_list, 5);
CompareFeatureList(s_output, {0, 1, 2, 3, 4, 5});
s_output = constraints.Query(s_input_feature_list, 6);
CompareFeatureList(s_output, {0, 1, 2, 3, 4, 5});
}
// Test shared feature
{
tree::TrainParam param = GetParameter();
int32_t constexpr kFeatures = 6;
std::string const constraints_str = R"constraint([[1, 2], [2, 3, 4]])constraint";
param.interaction_constraints = constraints_str;
FConstraintWrapper constraints(param, kFeatures);
constraints.Split(/*node_id=*/0, /*feature_id=*/2, /*left_id=*/1, /*right_id=*/2);
std::vector<int32_t> h_input_feature_list {0, 1, 2, 3, 4, 5};
dh::device_vector<int32_t> d_input_feature_list (h_input_feature_list);
common::Span<int32_t> s_input_feature_list = dh::ToSpan(d_input_feature_list);
auto s_output = constraints.Query(s_input_feature_list, 1);
CompareFeatureList(s_output, {1, 2, 3, 4});
}
// Test choosing free feature in root
{
tree::TrainParam param = GetParameter();
int32_t constexpr kFeatures = 6;
std::string const constraints_str = R"constraint([[0, 1]])constraint";
param.interaction_constraints = constraints_str;
FConstraintWrapper constraints(param, kFeatures);
std::vector<int32_t> h_input_feature_list {0, 1, 2, 3, 4, 5};
dh::device_vector<int32_t> d_input_feature_list (h_input_feature_list);
common::Span<int32_t> s_input_feature_list = dh::ToSpan(d_input_feature_list);
constraints.Split(/*node_id=*/0, /*feature_id=*/2, /*left_id=*/1, /*right_id=*/2);
auto s_output = constraints.Query(s_input_feature_list, 1);
CompareFeatureList(s_output, {2});
s_output = constraints.Query(s_input_feature_list, 2);
CompareFeatureList(s_output, {2});
}
}
} // namespace xgboost

67
tests/cpp/tree/test_gpu_hist.cu
View File

@@ -1,7 +1,6 @@
/*!
* Copyright 2017-2018 XGBoost contributors
* Copyright 2017-2019 XGBoost contributors
*/
#include <thrust/device_vector.h>
#include <xgboost/base.h>
#include <random>
@@ -16,6 +15,7 @@
#include "../../../src/tree/updater_gpu_hist.cu"
#include "../../../src/tree/updater_gpu_common.cuh"
#include "../../../src/common/common.h"
#include "../../../src/tree/constraints.cuh"
namespace xgboost {
namespace tree {
@@ -91,11 +91,13 @@ void BuildGidx(DeviceShard<GradientSumT>* shard, int n_rows, int n_cols,
TEST(GpuHist, BuildGidxDense) {
int constexpr kNRows = 16, kNCols = 8;
TrainParam param;
param.max_depth = 1;
param.max_leaves = 0;
DeviceShard<GradientPairPrecise> shard(0, 0, 0, kNRows, param, kNCols);
tree::TrainParam param;
std::vector<std::pair<std::string, std::string>> args {
{"max_depth", "1"},
{"max_leaves", "0"},
};
param.Init(args);
DeviceShard<GradientPairPrecise> shard(0, 0, 0, kNRows, param, kNCols, kNCols);
BuildGidx(&shard, kNRows, kNCols);
std::vector<common::CompressedByteT> h_gidx_buffer(shard.gidx_buffer.size());
@@ -130,10 +132,14 @@ TEST(GpuHist, BuildGidxDense) {
TEST(GpuHist, BuildGidxSparse) {
int constexpr kNRows = 16, kNCols = 8;
TrainParam param;
param.max_depth = 1;
param.max_leaves = 0;
std::vector<std::pair<std::string, std::string>> args {
{"max_depth", "1"},
{"max_leaves", "0"},
};
param.Init(args);
DeviceShard<GradientPairPrecise> shard(0, 0, 0, kNRows, param, kNCols);
DeviceShard<GradientPairPrecise> shard(0, 0, 0, kNRows, param, kNCols,
kNCols);
BuildGidx(&shard, kNRows, kNCols, 0.9f);
std::vector<common::CompressedByteT> h_gidx_buffer(shard.gidx_buffer.size());
@@ -173,10 +179,13 @@ void TestBuildHist(bool use_shared_memory_histograms) {
int const kNRows = 16, kNCols = 8;
TrainParam param;
param.max_depth = 6;
param.max_leaves = 0;
DeviceShard<GradientSumT> shard(0, 0, 0, kNRows, param, kNCols);
std::vector<std::pair<std::string, std::string>> args {
{"max_depth", "6"},
{"max_leaves", "0"},
};
param.Init(args);
DeviceShard<GradientSumT> shard(0, 0, 0, kNRows, param, kNCols,
kNCols);
BuildGidx(&shard, kNRows, kNCols);
xgboost::SimpleLCG gen;
@@ -263,17 +272,21 @@ TEST(GpuHist, EvaluateSplits) {
constexpr int kNCols = 8;
TrainParam param;
param.max_depth = 1;
param.colsample_bynode = 1;
param.colsample_bylevel = 1;
param.colsample_bytree = 1;
param.min_child_weight = 0.01;
// Disable all parameters.
param.reg_alpha = 0.0;
param.reg_lambda = 0;
param.max_delta_step = 0.0;
std::vector<std::pair<std::string, std::string>> args {
{"max_depth", "1"},
{"max_leaves", "0"},
// Disable all other parameters.
{"colsample_bynode", "1"},
{"colsample_bylevel", "1"},
{"colsample_bytree", "1"},
{"min_child_weight", "0.01"},
{"reg_alpha", "0"},
{"reg_lambda", "0"},
{"max_delta_step", "0"}
};
param.Init(args);
for (size_t i = 0; i < kNCols; ++i) {
param.monotone_constraints.emplace_back(0);
}
@@ -282,7 +295,8 @@ TEST(GpuHist, EvaluateSplits) {
// Initialize DeviceShard
std::unique_ptr<DeviceShard<GradientPairPrecise>> shard{
new DeviceShard<GradientPairPrecise>(0, 0, 0, kNRows, param, kNCols)};
new DeviceShard<GradientPairPrecise>(0, 0, 0, kNRows, param, kNCols,
kNCols)};
// Initialize DeviceShard::node_sum_gradients
shard->node_sum_gradients = {{6.4f, 12.8f}};
@@ -352,14 +366,13 @@ TEST(GpuHist, ApplySplit) {
TrainParam param;
std::vector<std::pair<std::string, std::string>> args = {};
param.InitAllowUnknown(args);
// Initialize shard
for (size_t i = 0; i < kNCols; ++i) {
param.monotone_constraints.emplace_back(0);
}
std::unique_ptr<DeviceShard<GradientPairPrecise>> shard{
new DeviceShard<GradientPairPrecise>(0, 0, 0, kNRows, param, kNCols)};
new DeviceShard<GradientPairPrecise>(0, 0, 0, kNRows, param, kNCols,
kNCols)};
shard->ridx_segments.resize(3); // 3 nodes.
shard->node_sum_gradients.resize(3);

57
tests/cpp/tree/test_split_evaluator.cc Normal file
View File

@@ -0,0 +1,57 @@
#include <gtest/gtest.h>
#include <xgboost/logging.h>
#include <memory>
#include "../../../src/tree/split_evaluator.h"
namespace xgboost {
namespace tree {
TEST(SplitEvaluator, Interaction) {
std::string constraints_str = R"interaction([[0, 1], [1, 2, 3]])interaction";
std::vector<std::pair<std::string, std::string>> args{
{"interaction_constraints", constraints_str},
{"num_feature", "8"}};
{
std::unique_ptr<SplitEvaluator> eval{
SplitEvaluator::Create("elastic_net,interaction")};
eval->Init(args);
eval->AddSplit(0, 1, 2, /*feature_id=*/4, 0, 0);
eval->AddSplit(2, 3, 4, /*feature_id=*/5, 0, 0);
ASSERT_FALSE(eval->CheckFeatureConstraint(2, /*feature_id=*/0));
ASSERT_FALSE(eval->CheckFeatureConstraint(2, /*feature_id=*/1));
ASSERT_TRUE(eval->CheckFeatureConstraint(2, /*feature_id=*/4));
ASSERT_FALSE(eval->CheckFeatureConstraint(2, /*feature_id=*/5));
std::vector<int32_t> accepted_features; // for node 3
for (int32_t f = 0; f < 8; ++f) {
if (eval->CheckFeatureConstraint(3, f)) {
accepted_features.emplace_back(f);
}
}
std::vector<int32_t> solutions{4, 5};
ASSERT_EQ(accepted_features.size(), solutions.size());
for (int32_t f = 0; f < accepted_features.size(); ++f) {
ASSERT_EQ(accepted_features[f], solutions[f]);
}
}
{
std::unique_ptr<SplitEvaluator> eval{
SplitEvaluator::Create("elastic_net,interaction")};
eval->Init(args);
eval->AddSplit(/*node_id=*/0, /*left_id=*/1, /*right_id=*/2, /*feature_id=*/4, 0, 0);
std::vector<int32_t> accepted_features; // for node 1
for (int32_t f = 0; f < 8; ++f) {
if (eval->CheckFeatureConstraint(1, f)) {
accepted_features.emplace_back(f);
}
}
ASSERT_EQ(accepted_features.size(), 1);
ASSERT_EQ(accepted_features[0], 4);
}
}
} // namespace tree
} // namespace xgboost