Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Batch UpdatePosition using cudaMemcpy (#7964)

Browse Source
This commit is contained in:
Rory Mitchell 2022-06-30 17:52:40 +02:00 committed by GitHub
parent 2407381c3d
commit bc4f802b17
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 441 additions and 516 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/common/device_helpers.cuh
View File

@ -1939,4 +1939,25 @@ class CUDAStream {
CUDAStreamView View() const { return CUDAStreamView{stream_}; } CUDAStreamView View() const { return CUDAStreamView{stream_}; }
void Sync() { this->View().Sync(); } void Sync() { this->View().Sync(); }
}; };
// Force nvcc to load data as constant
template <typename T>
class LDGIterator {
using DeviceWordT = typename cub::UnitWord<T>::DeviceWord;
static constexpr std::size_t kNumWords = sizeof(T) / sizeof(DeviceWordT);
const T *ptr_;
public:
explicit LDGIterator(const T *ptr) : ptr_(ptr) {}
__device__ T operator[](std::size_t idx) const {
DeviceWordT tmp[kNumWords];
static_assert(sizeof(tmp) == sizeof(T), "Expect sizes to be equal.");
#pragma unroll
for (int i = 0; i < kNumWords; i++) {
tmp[i] = __ldg(reinterpret_cast<const DeviceWordT *>(ptr_ + idx) + i);
}
return *reinterpret_cast<const T *>(tmp);
}
};
} // namespace dh } // namespace dh

156
src/tree/gpu_hist/row_partitioner.cu
View File

@ -1,174 +1,46 @@
/*! /*!
* Copyright 2017-2021 XGBoost contributors * Copyright 2017-2022 XGBoost contributors
*/ */
#include <thrust/iterator/discard_iterator.h> #include <thrust/iterator/discard_iterator.h>
#include <thrust/iterator/transform_output_iterator.h> #include <thrust/iterator/transform_output_iterator.h>
#include <thrust/sequence.h> #include <thrust/sequence.h>
#include <vector> #include <vector>
#include "../../common/device_helpers.cuh" #include "../../common/device_helpers.cuh"
#include "row_partitioner.cuh" #include "row_partitioner.cuh"
namespace xgboost { namespace xgboost {
namespace tree { namespace tree {
struct IndexFlagTuple {
size_t idx;
size_t flag;
};
struct IndexFlagOp {
__device__ IndexFlagTuple operator()(const IndexFlagTuple& a,
const IndexFlagTuple& b) const {
return {b.idx, a.flag + b.flag};
}
};
struct WriteResultsFunctor {
bst_node_t left_nidx;
common::Span<bst_node_t> position_in;
common::Span<bst_node_t> position_out;
common::Span<RowPartitioner::RowIndexT> ridx_in;
common::Span<RowPartitioner::RowIndexT> ridx_out;
int64_t* d_left_count;
__device__ IndexFlagTuple operator()(const IndexFlagTuple& x) {
// the ex_scan_result represents how many rows have been assigned to left
// node so far during scan.
int scatter_address;
if (position_in[x.idx] == left_nidx) {
scatter_address = x.flag - 1; // -1 because inclusive scan
} else {
// current number of rows belong to right node + total number of rows
// belong to left node
scatter_address = (x.idx - x.flag) + *d_left_count;
}
// copy the node id to output
position_out[scatter_address] = position_in[x.idx];
ridx_out[scatter_address] = ridx_in[x.idx];
// Discard
return {};
}
};
// Implement partitioning via single scan operation using transform output to
// write the result
void RowPartitioner::SortPosition(common::Span<bst_node_t> position,
common::Span<bst_node_t> position_out,
common::Span<RowIndexT> ridx,
common::Span<RowIndexT> ridx_out,
bst_node_t left_nidx, bst_node_t,
int64_t* d_left_count, cudaStream_t stream) {
WriteResultsFunctor write_results{left_nidx, position, position_out,
ridx, ridx_out, d_left_count};
auto discard_write_iterator =
thrust::make_transform_output_iterator(dh::TypedDiscard<IndexFlagTuple>(), write_results);
auto counting = thrust::make_counting_iterator(0llu);
auto input_iterator = dh::MakeTransformIterator<IndexFlagTuple>(
counting, [=] __device__(size_t idx) {
return IndexFlagTuple{idx, static_cast<size_t>(position[idx] == left_nidx)};
});
size_t temp_bytes = 0;
cub::DeviceScan::InclusiveScan(nullptr, temp_bytes, input_iterator,
discard_write_iterator, IndexFlagOp(),
position.size(), stream);
dh::TemporaryArray<int8_t> temp(temp_bytes);
cub::DeviceScan::InclusiveScan(temp.data().get(), temp_bytes, input_iterator,
discard_write_iterator, IndexFlagOp(),
position.size(), stream);
}
void Reset(int device_idx, common::Span<RowPartitioner::RowIndexT> ridx,
common::Span<bst_node_t> position) {
dh::safe_cuda(cudaSetDevice(device_idx));
CHECK_EQ(ridx.size(), position.size());
dh::LaunchN(ridx.size(), [=] __device__(size_t idx) {
ridx[idx] = idx;
position[idx] = 0;
});
}
RowPartitioner::RowPartitioner(int device_idx, size_t num_rows) RowPartitioner::RowPartitioner(int device_idx, size_t num_rows)
: device_idx_(device_idx), ridx_a_(num_rows), position_a_(num_rows), : device_idx_(device_idx), ridx_(num_rows), ridx_tmp_(num_rows) {
ridx_b_(num_rows), position_b_(num_rows) {
dh::safe_cuda(cudaSetDevice(device_idx_)); dh::safe_cuda(cudaSetDevice(device_idx_));
ridx_ = dh::DoubleBuffer<RowIndexT>{&ridx_a_, &ridx_b_}; ridx_segments_.emplace_back(NodePositionInfo{Segment(0, num_rows)});
position_ = dh::DoubleBuffer<bst_node_t>{&position_a_, &position_b_}; thrust::sequence(thrust::device, ridx_.data(), ridx_.data() + ridx_.size());
ridx_segments_.emplace_back(static_cast<size_t>(0), num_rows); dh::safe_cuda(cudaStreamCreate(&stream_));
Reset(device_idx, ridx_.CurrentSpan(), position_.CurrentSpan());
left_counts_.resize(256);
thrust::fill(left_counts_.begin(), left_counts_.end(), 0);
streams_.resize(2);
for (auto& stream : streams_) {
dh::safe_cuda(cudaStreamCreate(&stream));
}
} }
RowPartitioner::~RowPartitioner() { RowPartitioner::~RowPartitioner() {
dh::safe_cuda(cudaSetDevice(device_idx_)); dh::safe_cuda(cudaSetDevice(device_idx_));
for (auto& stream : streams_) { dh::safe_cuda(cudaStreamDestroy(stream_));
dh::safe_cuda(cudaStreamDestroy(stream));
}
} }
common::Span<const RowPartitioner::RowIndexT> RowPartitioner::GetRows( common::Span<const RowPartitioner::RowIndexT> RowPartitioner::GetRows(bst_node_t nidx) {
bst_node_t nidx) { auto segment = ridx_segments_.at(nidx).segment;
auto segment = ridx_segments_.at(nidx); return dh::ToSpan(ridx_).subspan(segment.begin, segment.Size());
// Return empty span here as a valid result
// Will error if we try to construct a span from a pointer with size 0
if (segment.Size() == 0) {
return {};
}
return ridx_.CurrentSpan().subspan(segment.begin, segment.Size());
} }
common::Span<const RowPartitioner::RowIndexT> RowPartitioner::GetRows() { common::Span<const RowPartitioner::RowIndexT> RowPartitioner::GetRows() {
return ridx_.CurrentSpan(); return dh::ToSpan(ridx_);
} }
common::Span<const bst_node_t> RowPartitioner::GetPosition() { std::vector<RowPartitioner::RowIndexT> RowPartitioner::GetRowsHost(bst_node_t nidx) {
return position_.CurrentSpan();
}
std::vector<RowPartitioner::RowIndexT> RowPartitioner::GetRowsHost(
bst_node_t nidx) {
auto span = GetRows(nidx); auto span = GetRows(nidx);
std::vector<RowIndexT> rows(span.size()); std::vector<RowIndexT> rows(span.size());
dh::CopyDeviceSpanToVector(&rows, span); dh::CopyDeviceSpanToVector(&rows, span);
return rows; return rows;
} }
std::vector<bst_node_t> RowPartitioner::GetPositionHost() {
auto span = GetPosition();
std::vector<bst_node_t> position(span.size());
dh::CopyDeviceSpanToVector(&position, span);
return position;
}
void RowPartitioner::SortPositionAndCopy(const Segment& segment,
bst_node_t left_nidx,
bst_node_t right_nidx,
int64_t* d_left_count,
cudaStream_t stream) {
SortPosition(
// position_in
common::Span<bst_node_t>(position_.Current() + segment.begin,
segment.Size()),
// position_out
common::Span<bst_node_t>(position_.Other() + segment.begin,
segment.Size()),
// row index in
common::Span<RowIndexT>(ridx_.Current() + segment.begin, segment.Size()),
// row index out
common::Span<RowIndexT>(ridx_.Other() + segment.begin, segment.Size()),
left_nidx, right_nidx, d_left_count, stream);
// Copy back key/value
const auto d_position_current = position_.Current() + segment.begin;
const auto d_position_other = position_.Other() + segment.begin;
const auto d_ridx_current = ridx_.Current() + segment.begin;
const auto d_ridx_other = ridx_.Other() + segment.begin;
dh::LaunchN(segment.Size(), stream, [=] __device__(size_t idx) {
d_position_current[idx] = d_position_other[idx];
d_ridx_current[idx] = d_ridx_other[idx];
});
}
}; // namespace tree }; // namespace tree
}; // namespace xgboost }; // namespace xgboost

385
src/tree/gpu_hist/row_partitioner.cuh
View File

@ -2,33 +2,193 @@
* Copyright 2017-2022 XGBoost contributors * Copyright 2017-2022 XGBoost contributors
*/ */
#pragma once #pragma once
#include <thrust/execution_policy.h>
#include <limits> #include <limits>
#include <vector> #include <vector>
#include "xgboost/base.h"
#include "../../common/device_helpers.cuh" #include "../../common/device_helpers.cuh"
#include "xgboost/base.h"
#include "xgboost/generic_parameters.h" #include "xgboost/generic_parameters.h"
#include "xgboost/task.h" #include "xgboost/task.h"
#include "xgboost/tree_model.h" #include "xgboost/tree_model.h"
namespace xgboost { namespace xgboost {
namespace tree { namespace tree {
/*! \brief Count how many rows are assigned to left node. */
__forceinline__ __device__ void AtomicIncrement(int64_t* d_count, bool increment) { /** \brief Used to demarcate a contiguous set of row indices associated with
#if __CUDACC_VER_MAJOR__ > 8 * some tree node. */
int mask = __activemask(); struct Segment {
unsigned ballot = __ballot_sync(mask, increment); bst_uint begin{0};
int leader = __ffs(mask) - 1; bst_uint end{0};
if (threadIdx.x % 32 == leader) {
atomicAdd(reinterpret_cast<unsigned long long*>(d_count), // NOLINT Segment() = default;
static_cast<unsigned long long>(__popc(ballot))); // NOLINT
Segment(bst_uint begin, bst_uint end) : begin(begin), end(end) { CHECK_GE(end, begin); }
__host__ __device__ size_t Size() const { return end - begin; }
};
// TODO(Rory): Can be larger. To be tuned alongside other batch operations.
static const int kMaxUpdatePositionBatchSize = 32;
template <typename OpDataT>
struct PerNodeData {
Segment segment;
OpDataT data;
};
template <typename BatchIterT>
__device__ __forceinline__ void AssignBatch(BatchIterT batch_info, std::size_t global_thread_idx,
int* batch_idx, std::size_t* item_idx) {
bst_uint sum = 0;
for (int i = 0; i < kMaxUpdatePositionBatchSize; i++) {
if (sum + batch_info[i].segment.Size() > global_thread_idx) {
*batch_idx = i;
*item_idx = (global_thread_idx - sum) + batch_info[i].segment.begin;
break;
} }
#else sum += batch_info[i].segment.Size();
unsigned ballot = __ballot(increment); }
if (threadIdx.x % 32 == 0) { }
atomicAdd(reinterpret_cast<unsigned long long*>(d_count), // NOLINT
static_cast<unsigned long long>(__popc(ballot))); // NOLINT template <int kBlockSize, typename RowIndexT, typename OpDataT>
__global__ __launch_bounds__(kBlockSize) void SortPositionCopyKernel(
dh::LDGIterator<PerNodeData<OpDataT>> batch_info, common::Span<RowIndexT> d_ridx,
const common::Span<const RowIndexT> ridx_tmp, std::size_t total_rows) {
for (auto idx : dh::GridStrideRange<std::size_t>(0, total_rows)) {
int batch_idx;
std::size_t item_idx;
AssignBatch(batch_info, idx, &batch_idx, &item_idx);
d_ridx[item_idx] = ridx_tmp[item_idx];
}
}
// We can scan over this tuple, where the scan gives us information on how to partition inputs
// according to the flag
struct IndexFlagTuple {
bst_uint idx; // The location of the item we are working on in ridx_
bst_uint flag_scan; // This gets populated after scanning
int batch_idx; // Which node in the batch does this item belong to
bool flag; // Result of op (is this item going left?)
};
struct IndexFlagOp {
__device__ IndexFlagTuple operator()(const IndexFlagTuple& a, const IndexFlagTuple& b) const {
// Segmented scan - resets if we cross batch boundaries
if (a.batch_idx == b.batch_idx) {
// Accumulate the flags, everything else stays the same
return {b.idx, a.flag_scan + b.flag_scan, b.batch_idx, b.flag};
} else {
return b;
}
}
};
template <typename OpDataT>
struct WriteResultsFunctor {
dh::LDGIterator<PerNodeData<OpDataT>> batch_info;
const bst_uint* ridx_in;
bst_uint* ridx_out;
bst_uint* counts;
__device__ IndexFlagTuple operator()(const IndexFlagTuple& x) {
std::size_t scatter_address;
const Segment& segment = batch_info[x.batch_idx].segment;
if (x.flag) {
bst_uint num_previous_flagged = x.flag_scan - 1; // -1 because inclusive scan
scatter_address = segment.begin + num_previous_flagged;
} else {
bst_uint num_previous_unflagged = (x.idx - segment.begin) - x.flag_scan;
scatter_address = segment.end - num_previous_unflagged - 1;
}
ridx_out[scatter_address] = ridx_in[x.idx];
if (x.idx == (segment.end - 1)) {
// Write out counts
counts[x.batch_idx] = x.flag_scan;
}
// Discard
return {};
}
};
template <typename RowIndexT, typename OpT, typename OpDataT>
void SortPositionBatch(common::Span<const PerNodeData<OpDataT>> d_batch_info,
common::Span<RowIndexT> ridx, common::Span<RowIndexT> ridx_tmp,
common::Span<bst_uint> d_counts, std::size_t total_rows, OpT op,
dh::device_vector<int8_t>* tmp, cudaStream_t stream) {
dh::LDGIterator<PerNodeData<OpDataT>> batch_info_itr(d_batch_info.data());
WriteResultsFunctor<OpDataT> write_results{batch_info_itr, ridx.data(), ridx_tmp.data(),
d_counts.data()};
auto discard_write_iterator =
thrust::make_transform_output_iterator(dh::TypedDiscard<IndexFlagTuple>(), write_results);
auto counting = thrust::make_counting_iterator(0llu);
auto input_iterator =
dh::MakeTransformIterator<IndexFlagTuple>(counting, [=] __device__(size_t idx) {
int batch_idx;
std::size_t item_idx;
AssignBatch(batch_info_itr, idx, &batch_idx, &item_idx);
auto op_res = op(ridx[item_idx], batch_info_itr[batch_idx].data);
return IndexFlagTuple{bst_uint(item_idx), op_res, batch_idx, op_res};
});
size_t temp_bytes = 0;
if (tmp->empty()) {
cub::DeviceScan::InclusiveScan(nullptr, temp_bytes, input_iterator, discard_write_iterator,
IndexFlagOp(), total_rows, stream);
tmp->resize(temp_bytes);
}
temp_bytes = tmp->size();
cub::DeviceScan::InclusiveScan(tmp->data().get(), temp_bytes, input_iterator,
discard_write_iterator, IndexFlagOp(), total_rows, stream);
constexpr int kBlockSize = 256;
// Value found by experimentation
const int kItemsThread = 12;
const int grid_size = xgboost::common::DivRoundUp(total_rows, kBlockSize * kItemsThread);
SortPositionCopyKernel<kBlockSize, RowIndexT, OpDataT>
<<<grid_size, kBlockSize, 0, stream>>>(batch_info_itr, ridx, ridx_tmp, total_rows);
}
struct NodePositionInfo {
Segment segment;
bst_node_t left_child = -1;
bst_node_t right_child = -1;
__device__ bool IsLeaf() { return left_child == -1; }
};
__device__ __forceinline__ int GetPositionFromSegments(std::size_t idx,
const NodePositionInfo* d_node_info) {
int position = 0;
NodePositionInfo node = d_node_info[position];
while (!node.IsLeaf()) {
NodePositionInfo left = d_node_info[node.left_child];
NodePositionInfo right = d_node_info[node.right_child];
if (idx >= left.segment.begin && idx < left.segment.end) {
position = node.left_child;
node = left;
} else if (idx >= right.segment.begin && idx < right.segment.end) {
position = node.right_child;
node = right;
} else {
KERNEL_CHECK(false);
}
}
return position;
}
template <int kBlockSize, typename RowIndexT, typename OpT>
__global__ __launch_bounds__(kBlockSize) void FinalisePositionKernel(
const common::Span<const NodePositionInfo> d_node_info,
const common::Span<const RowIndexT> d_ridx, common::Span<bst_node_t> d_out_position, OpT op) {
for (auto idx : dh::GridStrideRange<std::size_t>(0, d_ridx.size())) {
auto position = GetPositionFromSegments(idx, d_node_info.data());
RowIndexT ridx = d_ridx[idx];
bst_node_t new_position = op(ridx, position);
d_out_position[ridx] = new_position;
} }
#endif
} }
/** \brief Class responsible for tracking subsets of rows as we add splits and /** \brief Class responsible for tracking subsets of rows as we add splits and
@ -36,7 +196,6 @@ __forceinline__ __device__ void AtomicIncrement(int64_t* d_count, bool increment
class RowPartitioner { class RowPartitioner {
public: public:
using RowIndexT = bst_uint; using RowIndexT = bst_uint;
struct Segment;
static constexpr bst_node_t kIgnoredTreePosition = -1; static constexpr bst_node_t kIgnoredTreePosition = -1;
private: private:
@ -49,23 +208,20 @@ class RowPartitioner {
* node id -> segment -> indices of rows belonging to node * node id -> segment -> indices of rows belonging to node
*/ */
/*! \brief Range of row index for each node, pointers into ridx below. */ /*! \brief Range of row index for each node, pointers into ridx below. */
std::vector<Segment> ridx_segments_;
dh::TemporaryArray<RowIndexT> ridx_a_; std::vector<NodePositionInfo> ridx_segments_;
dh::TemporaryArray<RowIndexT> ridx_b_;
dh::TemporaryArray<bst_node_t> position_a_;
dh::TemporaryArray<bst_node_t> position_b_;
/*! \brief mapping for node id -> rows. /*! \brief mapping for node id -> rows.
* This looks like: * This looks like:
* node id | 1 | 2 | * node id | 1 | 2 |
* rows idx | 3, 5, 1 | 13, 31 | * rows idx | 3, 5, 1 | 13, 31 |
*/ */
dh::DoubleBuffer<RowIndexT> ridx_; dh::TemporaryArray<RowIndexT> ridx_;
/*! \brief mapping for row -> node id. */ // Staging area for sorting ridx
dh::DoubleBuffer<bst_node_t> position_; dh::TemporaryArray<RowIndexT> ridx_tmp_;
dh::caching_device_vector<int64_t> dh::device_vector<int8_t> tmp_;
left_counts_; // Useful to keep a bunch of zeroed memory for sort position
std::vector<cudaStream_t> streams_;
dh::PinnedMemory pinned_; dh::PinnedMemory pinned_;
dh::PinnedMemory pinned2_;
cudaStream_t stream_;
public: public:
RowPartitioner(int device_idx, size_t num_rows); RowPartitioner(int device_idx, size_t num_rows);
@ -83,73 +239,74 @@ class RowPartitioner {
*/ */
common::Span<const RowIndexT> GetRows(); common::Span<const RowIndexT> GetRows();
/**
* \brief Gets the tree position of all training instances.
*/
common::Span<const bst_node_t> GetPosition();
/** /**
* \brief Convenience method for testing * \brief Convenience method for testing
*/ */
std::vector<RowIndexT> GetRowsHost(bst_node_t nidx); std::vector<RowIndexT> GetRowsHost(bst_node_t nidx);
/**
* \brief Convenience method for testing
*/
std::vector<bst_node_t> GetPositionHost();
/** /**
* \brief Updates the tree position for set of training instances being split * \brief Updates the tree position for set of training instances being split
* into left and right child nodes. Accepts a user-defined lambda specifying * into left and right child nodes. Accepts a user-defined lambda specifying
* which branch each training instance should go down. * which branch each training instance should go down.
* *
* \tparam UpdatePositionOpT * \tparam UpdatePositionOpT
* \param nidx The index of the node being split. * \tparam OpDataT
* \param left_nidx The left child index. * \param nidx The index of the nodes being split.
* \param right_nidx The right child index. * \param left_nidx The left child indices.
* \param op Device lambda. Should provide the row index as an * \param right_nidx The right child indices.
* argument and return the new position for this training instance. * \param op_data User-defined data provided as the second argument to op
* \param op Device lambda with the row index as the first argument and op_data as the
* second. Returns true if this training instance goes on the left partition.
*/ */
template <typename UpdatePositionOpT> template <typename UpdatePositionOpT, typename OpDataT>
void UpdatePosition(bst_node_t nidx, bst_node_t left_nidx, void UpdatePositionBatch(const std::vector<bst_node_t>& nidx,
bst_node_t right_nidx, UpdatePositionOpT op) { const std::vector<bst_node_t>& left_nidx,
Segment segment = ridx_segments_.at(nidx); // rows belongs to node nidx const std::vector<bst_node_t>& right_nidx,
auto d_ridx = ridx_.CurrentSpan(); const std::vector<OpDataT>& op_data, UpdatePositionOpT op) {
auto d_position = position_.CurrentSpan(); if (nidx.empty()) return;
if (left_counts_.size() <= static_cast<size_t>(nidx)) { CHECK_EQ(nidx.size(), left_nidx.size());
left_counts_.resize((nidx * 2) + 1); CHECK_EQ(nidx.size(), right_nidx.size());
thrust::fill(left_counts_.begin(), left_counts_.end(), 0); CHECK_EQ(nidx.size(), op_data.size());
auto h_batch_info = pinned2_.GetSpan<PerNodeData<OpDataT>>(nidx.size());
dh::TemporaryArray<PerNodeData<OpDataT>> d_batch_info(nidx.size());
std::size_t total_rows = 0;
for (int i = 0; i < nidx.size(); i++) {
h_batch_info[i] = {ridx_segments_.at(nidx.at(i)).segment, op_data.at(i)};
total_rows += ridx_segments_.at(nidx.at(i)).segment.Size();
} }
// Now we divide the row segment into left and right node. dh::safe_cuda(cudaMemcpyAsync(d_batch_info.data().get(), h_batch_info.data(),
h_batch_info.size() * sizeof(PerNodeData<OpDataT>),
cudaMemcpyDefault, stream_));
int64_t* d_left_count = left_counts_.data().get() + nidx; // Temporary arrays
// Launch 1 thread for each row auto h_counts = pinned_.GetSpan<bst_uint>(nidx.size(), 0);
dh::LaunchN<1, 128>(segment.Size(), [segment, op, left_nidx, right_nidx, d_ridx, d_left_count, dh::TemporaryArray<bst_uint> d_counts(nidx.size(), 0);
d_position] __device__(size_t idx) {
// LaunchN starts from zero, so we restore the row index by adding segment.begin
idx += segment.begin;
RowIndexT ridx = d_ridx[idx];
bst_node_t new_position = op(ridx); // new node id
KERNEL_CHECK(new_position == left_nidx || new_position == right_nidx);
AtomicIncrement(d_left_count, new_position == left_nidx);
d_position[idx] = new_position;
});
// Overlap device to host memory copy (left_count) with sort
int64_t &left_count = pinned_.GetSpan<int64_t>(1)[0];
dh::safe_cuda(cudaMemcpyAsync(&left_count, d_left_count, sizeof(int64_t),
cudaMemcpyDeviceToHost, streams_[0]));
SortPositionAndCopy(segment, left_nidx, right_nidx, d_left_count, streams_[1]); // Partition the rows according to the operator
SortPositionBatch<RowIndexT, UpdatePositionOpT, OpDataT>(
dh::ToSpan(d_batch_info), dh::ToSpan(ridx_), dh::ToSpan(ridx_tmp_), dh::ToSpan(d_counts),
total_rows, op, &tmp_, stream_);
dh::safe_cuda(cudaMemcpyAsync(h_counts.data(), d_counts.data().get(), h_counts.size_bytes(),
cudaMemcpyDefault, stream_));
// TODO(Rory): this synchronisation hurts performance a lot
// Future optimisation should find a way to skip this
dh::safe_cuda(cudaStreamSynchronize(stream_));
dh::safe_cuda(cudaStreamSynchronize(streams_[0])); // Update segments
for (int i = 0; i < nidx.size(); i++) {
auto segment = ridx_segments_.at(nidx[i]).segment;
auto left_count = h_counts[i];
CHECK_LE(left_count, segment.Size()); CHECK_LE(left_count, segment.Size());
CHECK_GE(left_count, 0);
ridx_segments_.resize(std::max(static_cast<bst_node_t>(ridx_segments_.size()), ridx_segments_.resize(std::max(static_cast<bst_node_t>(ridx_segments_.size()),
std::max(left_nidx, right_nidx) + 1)); std::max(left_nidx[i], right_nidx[i]) + 1));
ridx_segments_[left_nidx] = ridx_segments_[nidx[i]] = NodePositionInfo{segment, left_nidx[i], right_nidx[i]};
Segment(segment.begin, segment.begin + left_count); ridx_segments_[left_nidx[i]] =
ridx_segments_[right_nidx] = NodePositionInfo{Segment(segment.begin, segment.begin + left_count)};
Segment(segment.begin + left_count, segment.end); ridx_segments_[right_nidx[i]] =
NodePositionInfo{Segment(segment.begin + left_count, segment.end)};
}
} }
/** /**
@ -165,69 +322,21 @@ class RowPartitioner {
* argument and return the new position for this training instance. * argument and return the new position for this training instance.
* \param sampled A device lambda to inform the partitioner whether a row is sampled. * \param sampled A device lambda to inform the partitioner whether a row is sampled.
*/ */
template <typename FinalisePositionOpT, typename Sampledp> template <typename FinalisePositionOpT>
void FinalisePosition(Context const* ctx, ObjInfo task, void FinalisePosition(common::Span<bst_node_t> d_out_position, FinalisePositionOpT op) {
HostDeviceVector<bst_node_t>* p_out_position, FinalisePositionOpT op, dh::TemporaryArray<NodePositionInfo> d_node_info_storage(ridx_segments_.size());
Sampledp sampledp) { dh::safe_cuda(cudaMemcpyAsync(d_node_info_storage.data().get(), ridx_segments_.data(),
auto d_position = position_.Current(); sizeof(NodePositionInfo) * ridx_segments_.size(),
const auto d_ridx = ridx_.Current(); cudaMemcpyDefault, stream_));
if (!task.UpdateTreeLeaf()) {
dh::LaunchN(position_.Size(), [=] __device__(size_t idx) {
auto position = d_position[idx];
RowIndexT ridx = d_ridx[idx];
bst_node_t new_position = op(ridx, position);
if (new_position == kIgnoredTreePosition) {
return;
}
d_position[idx] = new_position;
});
return;
}
p_out_position->SetDevice(ctx->gpu_id); constexpr int kBlockSize = 512;
p_out_position->Resize(position_.Size()); const int kItemsThread = 8;
auto sorted_position = p_out_position->DevicePointer(); const int grid_size = xgboost::common::DivRoundUp(ridx_.size(), kBlockSize * kItemsThread);
dh::LaunchN(position_.Size(), [=] __device__(size_t idx) { common::Span<const RowIndexT> d_ridx(ridx_.data().get(), ridx_.size());
auto position = d_position[idx]; FinalisePositionKernel<kBlockSize><<<grid_size, kBlockSize, 0, stream_>>>(
RowIndexT ridx = d_ridx[idx]; dh::ToSpan(d_node_info_storage), d_ridx, d_out_position, op);
bst_node_t new_position = op(ridx, position);
sorted_position[ridx] = sampledp(ridx) ? ~new_position : new_position;
if (new_position == kIgnoredTreePosition) {
return;
} }
d_position[idx] = new_position;
});
}
/**
* \brief Optimised routine for sorting key value pairs into left and right
* segments. Based on a single pass of exclusive scan, uses iterators to
* redirect inputs and outputs.
*/
void SortPosition(common::Span<bst_node_t> position,
common::Span<bst_node_t> position_out,
common::Span<RowIndexT> ridx,
common::Span<RowIndexT> ridx_out, bst_node_t left_nidx,
bst_node_t right_nidx, int64_t* d_left_count,
cudaStream_t stream = nullptr);
/*! \brief Sort row indices according to position. */
void SortPositionAndCopy(const Segment& segment, bst_node_t left_nidx,
bst_node_t right_nidx, int64_t* d_left_count,
cudaStream_t stream);
/** \brief Used to demarcate a contiguous set of row indices associated with
* some tree node. */
struct Segment {
size_t begin { 0 };
size_t end { 0 };
Segment() = default;
Segment(size_t begin, size_t end) : begin(begin), end(end) {
CHECK_GE(end, begin);
}
size_t Size() const { return end - begin; }
};
}; };
}; // namespace tree }; // namespace tree
}; // namespace xgboost }; // namespace xgboost

156
src/tree/updater_gpu_hist.cu
View File

@ -182,10 +182,11 @@ struct GPUHistMakerDevice {
std::unique_ptr<RowPartitioner> row_partitioner; std::unique_ptr<RowPartitioner> row_partitioner;
DeviceHistogramStorage<GradientSumT> hist{}; DeviceHistogramStorage<GradientSumT> hist{};
dh::caching_device_vector<GradientPair> d_gpair; // storage for gpair; dh::device_vector<GradientPair> d_gpair; // storage for gpair;
common::Span<GradientPair> gpair; common::Span<GradientPair> gpair;
dh::caching_device_vector<int> monotone_constraints; dh::device_vector<int> monotone_constraints;
dh::device_vector<float> update_predictions;
/*! \brief Sum gradient for each node. */ /*! \brief Sum gradient for each node. */
std::vector<GradientPairPrecise> node_sum_gradients; std::vector<GradientPairPrecise> node_sum_gradients;
@ -356,36 +357,49 @@ struct GPUHistMakerDevice {
return true; return true;
} }
void UpdatePosition(const GPUExpandEntry &e, RegTree* p_tree) { // Extra data for each node that is passed
// to the update position function
struct NodeSplitData {
RegTree::Node split_node;
FeatureType split_type;
common::CatBitField node_cats;
};
void UpdatePosition(const std::vector<GPUExpandEntry>& candidates, RegTree* p_tree) {
if (candidates.empty()) return;
std::vector<int> nidx(candidates.size());
std::vector<int> left_nidx(candidates.size());
std::vector<int> right_nidx(candidates.size());
std::vector<NodeSplitData> split_data(candidates.size());
for (int i = 0; i < candidates.size(); i++) {
auto& e = candidates[i];
RegTree::Node split_node = (*p_tree)[e.nid]; RegTree::Node split_node = (*p_tree)[e.nid];
auto split_type = p_tree->NodeSplitType(e.nid); auto split_type = p_tree->NodeSplitType(e.nid);
auto d_matrix = page->GetDeviceAccessor(ctx_->gpu_id); nidx.at(i) = e.nid;
auto node_cats = e.split.split_cats.Bits(); left_nidx.at(i) = split_node.LeftChild();
right_nidx.at(i) = split_node.RightChild();
split_data.at(i) = NodeSplitData{split_node, split_type, e.split.split_cats};
}
row_partitioner->UpdatePosition( auto d_matrix = page->GetDeviceAccessor(ctx_->gpu_id);
e.nid, split_node.LeftChild(), split_node.RightChild(), row_partitioner->UpdatePositionBatch(
[=] __device__(bst_uint ridx) { nidx, left_nidx, right_nidx, split_data,
[=] __device__(bst_uint ridx, const NodeSplitData& data) {
// given a row index, returns the node id it belongs to // given a row index, returns the node id it belongs to
bst_float cut_value = bst_float cut_value = d_matrix.GetFvalue(ridx, data.split_node.SplitIndex());
d_matrix.GetFvalue(ridx, split_node.SplitIndex());
// Missing value // Missing value
bst_node_t new_position = 0;
if (isnan(cut_value)) {
new_position = split_node.DefaultChild();
} else {
bool go_left = true; bool go_left = true;
if (split_type == FeatureType::kCategorical) { if (isnan(cut_value)) {
go_left = common::Decision<false>(node_cats, cut_value, split_node.DefaultLeft()); go_left = data.split_node.DefaultLeft();
} else { } else {
go_left = cut_value <= split_node.SplitCond(); if (data.split_type == FeatureType::kCategorical) {
} go_left = common::Decision<false>(data.node_cats.Bits(), cut_value,
if (go_left) { data.split_node.DefaultLeft());
new_position = split_node.LeftChild();
} else { } else {
new_position = split_node.RightChild(); go_left = cut_value <= data.split_node.SplitCond();
} }
} }
return new_position; return go_left;
}); });
} }
@ -394,6 +408,16 @@ struct GPUHistMakerDevice {
// prediction cache // prediction cache
void FinalisePosition(RegTree const* p_tree, DMatrix* p_fmat, ObjInfo task, void FinalisePosition(RegTree const* p_tree, DMatrix* p_fmat, ObjInfo task,
HostDeviceVector<bst_node_t>* p_out_position) { HostDeviceVector<bst_node_t>* p_out_position) {
// Prediction cache will not be used with external memory
if (!p_fmat->SingleColBlock()) {
if (task.UpdateTreeLeaf()) {
LOG(FATAL) << "Current objective function can not be used with external memory.";
}
p_out_position->Resize(0);
update_predictions.clear();
return;
}
dh::TemporaryArray<RegTree::Node> d_nodes(p_tree->GetNodes().size()); dh::TemporaryArray<RegTree::Node> d_nodes(p_tree->GetNodes().size());
dh::safe_cuda(cudaMemcpyAsync(d_nodes.data().get(), p_tree->GetNodes().data(), dh::safe_cuda(cudaMemcpyAsync(d_nodes.data().get(), p_tree->GetNodes().data(),
d_nodes.size() * sizeof(RegTree::Node), d_nodes.size() * sizeof(RegTree::Node),
@ -412,25 +436,9 @@ struct GPUHistMakerDevice {
dh::CopyToD(categories_segments, &d_categories_segments); dh::CopyToD(categories_segments, &d_categories_segments);
} }
if (row_partitioner->GetRows().size() != p_fmat->Info().num_row_) {
row_partitioner.reset(); // Release the device memory first before reallocating
row_partitioner.reset(new RowPartitioner(ctx_->gpu_id, p_fmat->Info().num_row_));
}
if (task.UpdateTreeLeaf() && !p_fmat->SingleColBlock() && param.subsample != 1.0) {
// see comment in the `FinalisePositionInPage`.
LOG(FATAL) << "Current objective function can not be used with subsampled external memory.";
}
if (page->n_rows == p_fmat->Info().num_row_) {
FinalisePositionInPage(page, dh::ToSpan(d_nodes), dh::ToSpan(d_split_types), FinalisePositionInPage(page, dh::ToSpan(d_nodes), dh::ToSpan(d_split_types),
dh::ToSpan(d_categories), dh::ToSpan(d_categories_segments), task, dh::ToSpan(d_categories), dh::ToSpan(d_categories_segments),
p_out_position); p_out_position);
} else {
for (auto const& batch : p_fmat->GetBatches<EllpackPage>(batch_param)) {
FinalisePositionInPage(batch.Impl(), dh::ToSpan(d_nodes), dh::ToSpan(d_split_types),
dh::ToSpan(d_categories), dh::ToSpan(d_categories_segments), task,
p_out_position);
}
}
} }
void FinalisePositionInPage(EllpackPageImpl const *page, void FinalisePositionInPage(EllpackPageImpl const *page,
@ -438,13 +446,15 @@ struct GPUHistMakerDevice {
common::Span<FeatureType const> d_feature_types, common::Span<FeatureType const> d_feature_types,
common::Span<uint32_t const> categories, common::Span<uint32_t const> categories,
common::Span<RegTree::Segment> categories_segments, common::Span<RegTree::Segment> categories_segments,
ObjInfo task,
HostDeviceVector<bst_node_t>* p_out_position) { HostDeviceVector<bst_node_t>* p_out_position) {
auto d_matrix = page->GetDeviceAccessor(ctx_->gpu_id); auto d_matrix = page->GetDeviceAccessor(ctx_->gpu_id);
auto d_gpair = this->gpair; auto d_gpair = this->gpair;
row_partitioner->FinalisePosition( update_predictions.resize(row_partitioner->GetRows().size());
ctx_, task, p_out_position, auto d_update_predictions = dh::ToSpan(update_predictions);
[=] __device__(size_t row_id, int position) { p_out_position->SetDevice(ctx_->gpu_id);
p_out_position->Resize(row_partitioner->GetRows().size());
auto new_position_op = [=] __device__(size_t row_id, int position) {
// What happens if user prune the tree? // What happens if user prune the tree?
if (!d_matrix.IsInRange(row_id)) { if (!d_matrix.IsInRange(row_id)) {
return RowPartitioner::kIgnoredTreePosition; return RowPartitioner::kIgnoredTreePosition;
@ -459,8 +469,7 @@ struct GPUHistMakerDevice {
} else { } else {
bool go_left = true; bool go_left = true;
if (common::IsCat(d_feature_types, position)) { if (common::IsCat(d_feature_types, position)) {
auto node_cats = auto node_cats = categories.subspan(categories_segments[position].beg,
categories.subspan(categories_segments[position].beg,
categories_segments[position].size); categories_segments[position].size);
go_left = common::Decision<false>(node_cats, element, node.DefaultLeft()); go_left = common::Decision<false>(node_cats, element, node.DefaultLeft());
} else { } else {
@ -472,45 +481,38 @@ struct GPUHistMakerDevice {
position = node.RightChild(); position = node.RightChild();
} }
} }
node = d_nodes[position]; node = d_nodes[position];
} }
d_update_predictions[row_id] = node.LeafValue();
return position; return position;
}, }; // NOLINT
[d_gpair] __device__(size_t ridx) {
// FIXME(jiamingy): Doesn't work when sampling is used with external memory as auto d_out_position = p_out_position->DeviceSpan();
// the sampler compacts the gradient vector. row_partitioner->FinalisePosition(d_out_position, new_position_op);
return d_gpair[ridx].GetHess() - .0f == 0.f;
dh::LaunchN(row_partitioner->GetRows().size(), [=] __device__(size_t idx) {
bst_node_t position = d_out_position[idx];
d_update_predictions[idx] = d_nodes[position].LeafValue();
bool is_row_sampled = d_gpair[idx].GetHess() - .0f == 0.f;
d_out_position[idx] = is_row_sampled ? ~position : position;
}); });
} }
void UpdatePredictionCache(linalg::VectorView<float> out_preds_d, RegTree const* p_tree) { bool UpdatePredictionCache(linalg::VectorView<float> out_preds_d, RegTree const* p_tree) {
if (update_predictions.empty()) {
return false;
}
CHECK(p_tree); CHECK(p_tree);
dh::safe_cuda(cudaSetDevice(ctx_->gpu_id)); dh::safe_cuda(cudaSetDevice(ctx_->gpu_id));
CHECK_EQ(out_preds_d.DeviceIdx(), ctx_->gpu_id); CHECK_EQ(out_preds_d.DeviceIdx(), ctx_->gpu_id);
auto d_ridx = row_partitioner->GetRows(); auto d_update_predictions = dh::ToSpan(update_predictions);
CHECK_EQ(out_preds_d.Size(), d_update_predictions.size());
GPUTrainingParam param_d(param); dh::LaunchN(out_preds_d.Size(), [=] XGBOOST_DEVICE(size_t idx) mutable {
dh::TemporaryArray<GradientPairPrecise> device_node_sum_gradients(node_sum_gradients.size()); out_preds_d(idx) += d_update_predictions[idx];
dh::safe_cuda(cudaMemcpyAsync(device_node_sum_gradients.data().get(), node_sum_gradients.data(),
sizeof(GradientPairPrecise) * node_sum_gradients.size(),
cudaMemcpyHostToDevice));
auto d_position = row_partitioner->GetPosition();
auto d_node_sum_gradients = device_node_sum_gradients.data().get();
auto tree_evaluator = evaluator_.GetEvaluator();
auto const& h_nodes = p_tree->GetNodes();
dh::caching_device_vector<RegTree::Node> nodes(h_nodes.size());
dh::safe_cuda(cudaMemcpyAsync(nodes.data().get(), h_nodes.data(),
h_nodes.size() * sizeof(RegTree::Node), cudaMemcpyHostToDevice));
auto d_nodes = dh::ToSpan(nodes);
dh::LaunchN(d_ridx.size(), [=] XGBOOST_DEVICE(size_t idx) mutable {
bst_node_t nidx = d_position[idx];
auto weight = d_nodes[nidx].LeafValue();
out_preds_d(d_ridx[idx]) += weight;
}); });
row_partitioner.reset(); return true;
} }
// num histograms is the number of contiguous histograms in memory to reduce over // num histograms is the number of contiguous histograms in memory to reduce over
@ -684,14 +686,12 @@ struct GPUHistMakerDevice {
auto new_candidates = auto new_candidates =
pinned.GetSpan<GPUExpandEntry>(filtered_expand_set.size() * 2, GPUExpandEntry()); pinned.GetSpan<GPUExpandEntry>(filtered_expand_set.size() * 2, GPUExpandEntry());
for (const auto& e : filtered_expand_set) {
monitor.Start("UpdatePosition"); monitor.Start("UpdatePosition");
// Update position is only run when child is valid, instead of right after apply // Update position is only run when child is valid, instead of right after apply
// split (as in approx tree method). Hense we have the finalise position call // split (as in approx tree method). Hense we have the finalise position call
// in GPU Hist. // in GPU Hist.
this->UpdatePosition(e, p_tree); this->UpdatePosition(filtered_expand_set, p_tree);
monitor.Stop("UpdatePosition"); monitor.Stop("UpdatePosition");
}
monitor.Start("BuildHist"); monitor.Start("BuildHist");
this->BuildHistLeftRight(filtered_expand_set, reducer, tree); this->BuildHistLeftRight(filtered_expand_set, reducer, tree);
@ -844,9 +844,9 @@ class GPUHistMaker : public TreeUpdater {
return false; return false;
} }
monitor_.Start("UpdatePredictionCache"); monitor_.Start("UpdatePredictionCache");
maker->UpdatePredictionCache(p_out_preds, p_last_tree_); bool result = maker->UpdatePredictionCache(p_out_preds, p_last_tree_);
monitor_.Stop("UpdatePredictionCache"); monitor_.Stop("UpdatePredictionCache");
return true; return result;
} }
TrainParam param_; // NOLINT TrainParam param_; // NOLINT

161
tests/cpp/tree/gpu_hist/test_row_partitioner.cu
View File

@ -19,49 +19,7 @@
namespace xgboost { namespace xgboost {
namespace tree { namespace tree {
void TestSortPosition(const std::vector<int>& position_in, int left_idx, void TestUpdatePositionBatch() {
int right_idx) {
dh::safe_cuda(cudaSetDevice(0));
std::vector<int64_t> left_count = {
std::count(position_in.begin(), position_in.end(), left_idx)};
dh::caching_device_vector<int64_t> d_left_count = left_count;
dh::caching_device_vector<int> position = position_in;
dh::caching_device_vector<int> position_out(position.size());
dh::caching_device_vector<RowPartitioner::RowIndexT> ridx(position.size());
thrust::sequence(ridx.begin(), ridx.end());
dh::caching_device_vector<RowPartitioner::RowIndexT> ridx_out(ridx.size());
RowPartitioner rp(0,10);
rp.SortPosition(
common::Span<int>(position.data().get(), position.size()),
common::Span<int>(position_out.data().get(), position_out.size()),
common::Span<RowPartitioner::RowIndexT>(ridx.data().get(), ridx.size()),
common::Span<RowPartitioner::RowIndexT>(ridx_out.data().get(), ridx_out.size()), left_idx,
right_idx, d_left_count.data().get(), nullptr);
thrust::host_vector<int> position_result = position_out;
thrust::host_vector<int> ridx_result = ridx_out;
// Check position is sorted
EXPECT_TRUE(std::is_sorted(position_result.begin(), position_result.end()));
// Check row indices are sorted inside left and right segment
EXPECT_TRUE(
std::is_sorted(ridx_result.begin(), ridx_result.begin() + left_count[0]));
EXPECT_TRUE(
std::is_sorted(ridx_result.begin() + left_count[0], ridx_result.end()));
// Check key value pairs are the same
for (auto i = 0ull; i < ridx_result.size(); i++) {
EXPECT_EQ(position_result[i], position_in[ridx_result[i]]);
}
}
TEST(GpuHist, SortPosition) {
TestSortPosition({1, 2, 1, 2, 1}, 1, 2);
TestSortPosition({1, 1, 1, 1}, 1, 2);
TestSortPosition({2, 2, 2, 2}, 1, 2);
TestSortPosition({1, 2, 1, 2, 3}, 1, 2);
}
void TestUpdatePosition() {
const int kNumRows = 10; const int kNumRows = 10;
RowPartitioner rp(0, kNumRows); RowPartitioner rp(0, kNumRows);
auto rows = rp.GetRowsHost(0); auto rows = rp.GetRowsHost(0);
@ -69,16 +27,11 @@ void TestUpdatePosition() {
for (auto i = 0ull; i < kNumRows; i++) { for (auto i = 0ull; i < kNumRows; i++) {
EXPECT_EQ(rows[i], i); EXPECT_EQ(rows[i], i);
} }
std::vector<int> extra_data = {0};
// Send the first five training instances to the right node // Send the first five training instances to the right node
// and the second 5 to the left node // and the second 5 to the left node
rp.UpdatePosition(0, 1, 2, rp.UpdatePositionBatch({0}, {1}, {2}, extra_data, [=] __device__(RowPartitioner::RowIndexT ridx, int) {
[=] __device__(RowPartitioner::RowIndexT ridx) { return ridx > 4;
if (ridx > 4) {
return 1;
}
else {
return 2;
}
}); });
rows = rp.GetRowsHost(1); rows = rp.GetRowsHost(1);
for (auto r : rows) { for (auto r : rows) {
@ -90,88 +43,58 @@ void TestUpdatePosition() {
} }
// Split the left node again // Split the left node again
rp.UpdatePosition(1, 3, 4, [=]__device__(RowPartitioner::RowIndexT ridx) rp.UpdatePositionBatch({1}, {3}, {4}, extra_data,[=] __device__(RowPartitioner::RowIndexT ridx, int) {
{ return ridx < 7;
if (ridx < 7) {
return 3
;
}
return 4;
}); });
EXPECT_EQ(rp.GetRows(3).size(), 2); EXPECT_EQ(rp.GetRows(3).size(), 2);
EXPECT_EQ(rp.GetRows(4).size(), 3); EXPECT_EQ(rp.GetRows(4).size(), 3);
// Check position is as expected
EXPECT_EQ(rp.GetPositionHost(), std::vector<bst_node_t>({3,3,4,4,4,2,2,2,2,2}));
} }
TEST(RowPartitioner, Basic) { TestUpdatePosition(); } TEST(RowPartitioner, Batch) { TestUpdatePositionBatch(); }
void TestFinalise() { void TestSortPositionBatch(const std::vector<int>& ridx_in, const std::vector<Segment>& segments) {
const int kNumRows = 10; thrust::device_vector<uint32_t> ridx = ridx_in;
thrust::device_vector<uint32_t> ridx_tmp(ridx_in.size());
thrust::device_vector<bst_uint> counts(segments.size());
ObjInfo task{ObjInfo::kRegression, false, false}; auto op = [=] __device__(auto ridx, int data) { return ridx % 2 == 0; };
HostDeviceVector<bst_node_t> position; std::vector<int> op_data(segments.size());
Context ctx; std::vector<PerNodeData<int>> h_batch_info(segments.size());
ctx.gpu_id = 0; dh::TemporaryArray<PerNodeData<int>> d_batch_info(segments.size());
{ std::size_t total_rows = 0;
RowPartitioner rp(0, kNumRows); for (int i = 0; i < segments.size(); i++) {
rp.FinalisePosition( h_batch_info[i] = {segments.at(i), 0};
&ctx, task, &position, total_rows += segments.at(i).Size();
[=] __device__(RowPartitioner::RowIndexT ridx, int position) { return 7; },
[] XGBOOST_DEVICE(size_t) { return false; });
auto position = rp.GetPositionHost();
for (auto p : position) {
EXPECT_EQ(p, 7);
}
} }
dh::safe_cuda(cudaMemcpyAsync(d_batch_info.data().get(), h_batch_info.data(),
h_batch_info.size() * sizeof(PerNodeData<int>), cudaMemcpyDefault,
nullptr));
dh::device_vector<int8_t> tmp;
SortPositionBatch<uint32_t, decltype(op), int>(dh::ToSpan(d_batch_info), dh::ToSpan(ridx),
dh::ToSpan(ridx_tmp), dh::ToSpan(counts),
total_rows, op, &tmp, nullptr);
/** auto op_without_data = [=] __device__(auto ridx) { return ridx % 2 == 0; };
* Test for sampling. for (int i = 0; i < segments.size(); i++) {
*/ auto begin = ridx.begin() + segments[i].begin;
dh::device_vector<float> hess(kNumRows); auto end = ridx.begin() + segments[i].end;
for (size_t i = 0; i < hess.size(); ++i) { bst_uint count = counts[i];
// removed rows, 0, 3, 6, 9 auto left_partition_count =
if (i % 3 == 0) { thrust::count_if(thrust::device, begin, begin + count, op_without_data);
hess[i] = 0; EXPECT_EQ(left_partition_count, count);
} else { auto right_partition_count =
hess[i] = i; thrust::count_if(thrust::device, begin + count, end, op_without_data);
} EXPECT_EQ(right_partition_count, 0);
}
auto d_hess = dh::ToSpan(hess);
RowPartitioner rp(0, kNumRows);
rp.FinalisePosition(
&ctx, task, &position,
[] __device__(RowPartitioner::RowIndexT ridx, bst_node_t position) {
return ridx % 2 == 0 ? 1 : 2;
},
[d_hess] __device__(size_t ridx) { return d_hess[ridx] - 0.f == 0.f; });
auto const& h_position = position.ConstHostVector();
for (size_t ridx = 0; ridx < h_position.size(); ++ridx) {
if (ridx % 3 == 0) {
ASSERT_LT(h_position[ridx], 0);
} else {
ASSERT_EQ(h_position[ridx], ridx % 2 == 0 ? 1 : 2);
}
} }
} }
TEST(RowPartitioner, Finalise) { TestFinalise(); } TEST(GpuHist, SortPositionBatch) {
TestSortPositionBatch({0, 1, 2, 3, 4, 5}, {{0, 3}, {3, 6}});
void TestIncorrectRow() { TestSortPositionBatch({0, 1, 2, 3, 4, 5}, {{0, 1}, {3, 6}});
RowPartitioner rp(0, 1); TestSortPositionBatch({0, 1, 2, 3, 4, 5}, {{0, 6}});
rp.UpdatePosition(0, 1, 2, [=]__device__ (RowPartitioner::RowIndexT ridx) TestSortPositionBatch({0, 1, 2, 3, 4, 5}, {{3, 6}, {0, 2}});
{
return 4; // This is not the left branch or the right branch
});
} }
TEST(RowPartitionerDeathTest, IncorrectRow) {
ASSERT_DEATH({ TestIncorrectRow(); },".*");
}
} // namespace tree } // namespace tree
} // namespace xgboost } // namespace xgboost