Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Copy data from Ellpack to GHist. (#8215)

Browse Source
This commit is contained in:
Jiaming Yuan
2022-09-06 23:05:49 +08:00
committed by GitHub
parent 7ee10e3dbd
commit 441ffc017a
16 changed files with 466 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/common/algorithm.cuh Normal file
View File

@@ -0,0 +1,27 @@
/*!
* Copyright 2022 by XGBoost Contributors
*/
#pragma once
#include <thrust/binary_search.h> // thrust::upper_bound
#include <thrust/execution_policy.h> // thrust::seq
#include "xgboost/base.h"
#include "xgboost/span.h"
namespace xgboost {
namespace common {
namespace cuda {
template <typename It>
size_t XGBOOST_DEVICE SegmentId(It first, It last, size_t idx) {
size_t segment_id = thrust::upper_bound(thrust::seq, first, last, idx) - 1 - first;
return segment_id;
}
template <typename T>
size_t XGBOOST_DEVICE SegmentId(Span<T> segments_ptr, size_t idx) {
return SegmentId(segments_ptr.cbegin(), segments_ptr.cend(), idx);
}
} // namespace cuda
} // namespace common
} // namespace xgboost

16
src/common/algorithm.h Normal file
View File

@@ -0,0 +1,16 @@
/*!
* Copyright 2022 by XGBoost Contributors
*/
#pragma once
#include <algorithm> // std::upper_bound
#include <cinttypes> // std::size_t
namespace xgboost {
namespace common {
template <typename It, typename Idx>
auto SegmentId(It first, It last, Idx idx) {
std::size_t segment_id = std::upper_bound(first, last, idx) - 1 - first;
return segment_id;
}
} // namespace common
} // namespace xgboost

153
src/common/column_matrix.h
View File

@@ -18,6 +18,7 @@
#include "../data/adapter.h"
#include "../data/gradient_index.h"
#include "algorithm.h"
#include "hist_util.h"
namespace xgboost {
@@ -135,6 +136,22 @@ class DenseColumnIter : public Column<BinIdxT> {
class ColumnMatrix {
void InitStorage(GHistIndexMatrix const& gmat, double sparse_threshold);
template <typename ColumnBinT, typename BinT, typename RIdx>
void SetBinSparse(BinT bin_id, RIdx rid, bst_feature_t fid, ColumnBinT* local_index) {
if (type_[fid] == kDenseColumn) {
ColumnBinT* begin = &local_index[feature_offsets_[fid]];
begin[rid] = bin_id - index_base_[fid];
// not thread-safe with bool vector. FIXME(jiamingy): We can directly assign
// kMissingId to the index to avoid missing flags.
missing_flags_[feature_offsets_[fid] + rid] = false;
} else {
ColumnBinT* begin = &local_index[feature_offsets_[fid]];
begin[num_nonzeros_[fid]] = bin_id - index_base_[fid];
row_ind_[feature_offsets_[fid] + num_nonzeros_[fid]] = rid;
++num_nonzeros_[fid];
}
}
public:
// get number of features
bst_feature_t GetNumFeature() const { return static_cast<bst_feature_t>(type_.size()); }
@@ -144,34 +161,66 @@ class ColumnMatrix {
this->InitStorage(gmat, sparse_threshold);
}
template <typename Batch>
void PushBatch(int32_t n_threads, Batch const& batch, float missing, GHistIndexMatrix const& gmat,
size_t base_rowid) {
// pre-fill index_ for dense columns
auto n_features = gmat.Features();
if (!any_missing_) {
missing_flags_.resize(feature_offsets_[n_features], false);
// row index is compressed, we need to dispatch it.
DispatchBinType(gmat.index.GetBinTypeSize(), [&, size = batch.Size(), n_features = n_features,
n_threads = n_threads](auto t) {
using RowBinIdxT = decltype(t);
SetIndexNoMissing(base_rowid, gmat.index.data<RowBinIdxT>(), size, n_features, n_threads);
});
} else {
missing_flags_.resize(feature_offsets_[n_features], true);
SetIndexMixedColumns(base_rowid, batch, gmat, n_features, missing);
}
}
// construct column matrix from GHistIndexMatrix
void Init(SparsePage const& page, const GHistIndexMatrix& gmat, double sparse_threshold,
int32_t n_threads) {
/**
* \brief Initialize ColumnMatrix from GHistIndexMatrix with reference to the original
* SparsePage.
*/
void InitFromSparse(SparsePage const& page, const GHistIndexMatrix& gmat, double sparse_threshold,
int32_t n_threads) {
auto batch = data::SparsePageAdapterBatch{page.GetView()};
this->InitStorage(gmat, sparse_threshold);
// ignore base row id here as we always has one column matrix for each sparse page.
this->PushBatch(n_threads, batch, std::numeric_limits<float>::quiet_NaN(), gmat, 0);
}
/**
* \brief Initialize ColumnMatrix from GHistIndexMatrix without reference to actual
* data.
*
* This function requires a binary search for each bin to get back the feature index
* for those bins.
*/
void InitFromGHist(Context const* ctx, GHistIndexMatrix const& gmat) {
auto n_threads = ctx->Threads();
if (!any_missing_) {
// row index is compressed, we need to dispatch it.
DispatchBinType(gmat.index.GetBinTypeSize(), [&, size = gmat.Size(), n_threads = n_threads,
n_features = gmat.Features()](auto t) {
using RowBinIdxT = decltype(t);
SetIndexNoMissing(gmat.base_rowid, gmat.index.data<RowBinIdxT>(), size, n_features,
n_threads);
});
} else {
SetIndexMixedColumns(gmat);
}
}
/**
* \brief Push batch of data for Quantile DMatrix support.
*
* \param batch Input data wrapped inside a adapter batch.
* \param gmat The row-major histogram index that contains index for ALL data.
* \param base_rowid The beginning row index for current batch.
*/
template <typename Batch>
void PushBatch(int32_t n_threads, Batch const& batch, float missing, GHistIndexMatrix const& gmat,
size_t base_rowid) {
// pre-fill index_ for dense columns
if (!any_missing_) {
// row index is compressed, we need to dispatch it.
// use base_rowid from input parameter as gmat is a single matrix that contains all
// the histogram index instead of being only a batch.
DispatchBinType(gmat.index.GetBinTypeSize(), [&, size = batch.Size(), n_threads = n_threads,
n_features = gmat.Features()](auto t) {
using RowBinIdxT = decltype(t);
SetIndexNoMissing(base_rowid, gmat.index.data<RowBinIdxT>(), size, n_features, n_threads);
});
} else {
SetIndexMixedColumns(base_rowid, batch, gmat, missing);
}
}
/* Set the number of bytes based on numeric limit of maximum number of bins provided by user */
void SetTypeSize(size_t max_bin_per_feat) {
if ((max_bin_per_feat - 1) <= static_cast<int>(std::numeric_limits<uint8_t>::max())) {
@@ -210,6 +259,7 @@ class ColumnMatrix {
template <typename RowBinIdxT>
void SetIndexNoMissing(bst_row_t base_rowid, RowBinIdxT const* row_index, const size_t n_samples,
const size_t n_features, int32_t n_threads) {
missing_flags_.resize(feature_offsets_[n_features], false);
DispatchBinType(bins_type_size_, [&](auto t) {
using ColumnBinT = decltype(t);
auto column_index = Span<ColumnBinT>{reinterpret_cast<ColumnBinT*>(index_.data()),
@@ -232,29 +282,16 @@ class ColumnMatrix {
*/
template <typename Batch>
void SetIndexMixedColumns(size_t base_rowid, Batch const& batch, const GHistIndexMatrix& gmat,
size_t n_features, float missing) {
float missing) {
auto n_features = gmat.Features();
missing_flags_.resize(feature_offsets_[n_features], true);
auto const* row_index = gmat.index.data<uint32_t>() + gmat.row_ptr[base_rowid];
auto is_valid = data::IsValidFunctor {missing};
num_nonzeros_.resize(n_features, 0);
auto is_valid = data::IsValidFunctor{missing};
DispatchBinType(bins_type_size_, [&](auto t) {
using ColumnBinT = decltype(t);
ColumnBinT* local_index = reinterpret_cast<ColumnBinT*>(index_.data());
num_nonzeros_.resize(n_features, 0);
auto get_bin_idx = [&](auto bin_id, auto rid, bst_feature_t fid) {
if (type_[fid] == kDenseColumn) {
ColumnBinT* begin = reinterpret_cast<ColumnBinT*>(&local_index[feature_offsets_[fid]]);
begin[rid] = bin_id - index_base_[fid];
// not thread-safe with bool vector. FIXME(jiamingy): We can directly assign
// kMissingId to the index to avoid missing flags.
missing_flags_[feature_offsets_[fid] + rid] = false;
} else {
ColumnBinT* begin = reinterpret_cast<ColumnBinT*>(&local_index[feature_offsets_[fid]]);
begin[num_nonzeros_[fid]] = bin_id - index_base_[fid];
row_ind_[feature_offsets_[fid] + num_nonzeros_[fid]] = rid;
++num_nonzeros_[fid];
}
};
size_t const batch_size = batch.Size();
size_t k{0};
for (size_t rid = 0; rid < batch_size; ++rid) {
@@ -264,7 +301,7 @@ class ColumnMatrix {
if (is_valid(coo)) {
auto fid = coo.column_idx;
const uint32_t bin_id = row_index[k];
get_bin_idx(bin_id, rid + base_rowid, fid);
SetBinSparse(bin_id, rid + base_rowid, fid, local_index);
++k;
}
}
@@ -272,6 +309,40 @@ class ColumnMatrix {
});
}
/**
* \brief Set column index for both dense and sparse columns, but with only GHistMatrix
* available and requires a search for each bin.
*/
void SetIndexMixedColumns(const GHistIndexMatrix& gmat) {
auto n_features = gmat.Features();
missing_flags_.resize(feature_offsets_[n_features], true);
auto const* row_index = gmat.index.data<uint32_t>() + gmat.row_ptr[gmat.base_rowid];
num_nonzeros_.resize(n_features, 0);
auto const& ptrs = gmat.cut.Ptrs();
DispatchBinType(bins_type_size_, [&](auto t) {
using ColumnBinT = decltype(t);
ColumnBinT* local_index = reinterpret_cast<ColumnBinT*>(index_.data());
auto const batch_size = gmat.Size();
size_t k{0};
for (size_t ridx = 0; ridx < batch_size; ++ridx) {
auto r_beg = gmat.row_ptr[ridx];
auto r_end = gmat.row_ptr[ridx + 1];
bst_feature_t fidx{0};
for (size_t j = r_beg; j < r_end; ++j) {
const uint32_t bin_idx = row_index[k];
// find the feature index for current bin.
while (bin_idx >= ptrs[fidx + 1]) {
fidx++;
}
SetBinSparse(bin_idx, ridx, fidx, local_index);
++k;
}
}
});
}
BinTypeSize GetTypeSize() const { return bins_type_size_; }
auto GetColumnType(bst_feature_t fidx) const { return type_[fidx]; }

13
src/common/device_helpers.cuh
View File

@@ -35,6 +35,7 @@
#include "xgboost/global_config.h"
#include "common.h"
#include "algorithm.cuh"
#ifdef XGBOOST_USE_NCCL
#include "nccl.h"
@@ -1556,17 +1557,7 @@ XGBOOST_DEVICE thrust::transform_iterator<FuncT, IterT, ReturnT> MakeTransformIt
return thrust::transform_iterator<FuncT, IterT, ReturnT>(iter, func);
}
template <typename It>
size_t XGBOOST_DEVICE SegmentId(It first, It last, size_t idx) {
size_t segment_id = thrust::upper_bound(thrust::seq, first, last, idx) -
1 - first;
return segment_id;
}
template <typename T>
size_t XGBOOST_DEVICE SegmentId(xgboost::common::Span<T> segments_ptr, size_t idx) {
return SegmentId(segments_ptr.cbegin(), segments_ptr.cend(), idx);
}
using xgboost::common::cuda::SegmentId; // import it for compatibility
namespace detail {
template <typename Key, typename KeyOutIt>

35
src/common/hist_util.h
View File

@@ -22,6 +22,7 @@
#include "row_set.h"
#include "threading_utils.h"
#include "timer.h"
#include "algorithm.h" // SegmentId
namespace xgboost {
class GHistIndexMatrix;
@@ -130,19 +131,23 @@ class HistogramCuts {
/**
* \brief Search the bin index for categorical feature.
*/
bst_bin_t SearchCatBin(float value, bst_feature_t fidx) const {
auto const &ptrs = this->Ptrs();
auto const &vals = this->Values();
bst_bin_t SearchCatBin(float value, bst_feature_t fidx, std::vector<uint32_t> const& ptrs,
std::vector<float> const& vals) const {
auto end = ptrs.at(fidx + 1) + vals.cbegin();
auto beg = ptrs[fidx] + vals.cbegin();
// Truncates the value in case it's not perfectly rounded.
auto v = static_cast<float>(common::AsCat(value));
auto v = static_cast<float>(common::AsCat(value));
auto bin_idx = std::lower_bound(beg, end, v) - vals.cbegin();
if (bin_idx == ptrs.at(fidx + 1)) {
bin_idx -= 1;
}
return bin_idx;
}
bst_bin_t SearchCatBin(float value, bst_feature_t fidx) const {
auto const& ptrs = this->Ptrs();
auto const& vals = this->Values();
return this->SearchCatBin(value, fidx, ptrs, vals);
}
bst_bin_t SearchCatBin(Entry const& e) const { return SearchCatBin(e.fvalue, e.index); }
};
@@ -189,6 +194,28 @@ auto DispatchBinType(BinTypeSize type, Fn&& fn) {
* storage class.
*/
struct Index {
// Inside the compressor, bin_idx is the index for cut value across all features. By
// subtracting it with starting pointer of each feature, we can reduce it to smaller
// value and store it with smaller types. Usable only with dense data.
//
// For sparse input we have to store an addition feature index (similar to sparse matrix
// formats like CSR) for each bin in index field to choose the right offset.
template <typename T>
struct CompressBin {
uint32_t const* offsets;
template <typename Bin, typename Feat>
auto operator()(Bin bin_idx, Feat fidx) const {
return static_cast<T>(bin_idx - offsets[fidx]);
}
};
template <typename T>
CompressBin<T> MakeCompressor() const {
uint32_t const* offsets = this->Offset();
return CompressBin<T>{offsets};
}
Index() { SetBinTypeSize(binTypeSize_); }
Index(const Index& i) = delete;
Index& operator=(Index i) = delete;

11
src/data/ellpack_page.cu
View File

@@ -547,4 +547,15 @@ EllpackDeviceAccessor EllpackPageImpl::GetDeviceAccessor(
NumSymbols()),
feature_types};
}
EllpackDeviceAccessor EllpackPageImpl::GetHostAccessor(
common::Span<FeatureType const> feature_types) const {
return {Context::kCpuId,
cuts_,
is_dense,
row_stride,
base_rowid,
n_rows,
common::CompressedIterator<uint32_t>(gidx_buffer.ConstHostPointer(), NumSymbols()),
feature_types};
}
} // namespace xgboost

19
src/data/ellpack_page.cuh
View File

@@ -43,12 +43,18 @@ struct EllpackDeviceAccessor {
base_rowid(base_rowid),
n_rows(n_rows) ,gidx_iter(gidx_iter),
feature_types{feature_types} {
cuts.cut_values_.SetDevice(device);
cuts.cut_ptrs_.SetDevice(device);
cuts.min_vals_.SetDevice(device);
gidx_fvalue_map = cuts.cut_values_.ConstDeviceSpan();
feature_segments = cuts.cut_ptrs_.ConstDeviceSpan();
min_fvalue = cuts.min_vals_.ConstDeviceSpan();
if (device == Context::kCpuId) {
gidx_fvalue_map = cuts.cut_values_.ConstHostSpan();
feature_segments = cuts.cut_ptrs_.ConstHostSpan();
min_fvalue = cuts.min_vals_.ConstHostSpan();
} else {
cuts.cut_values_.SetDevice(device);
cuts.cut_ptrs_.SetDevice(device);
cuts.min_vals_.SetDevice(device);
gidx_fvalue_map = cuts.cut_values_.ConstDeviceSpan();
feature_segments = cuts.cut_ptrs_.ConstDeviceSpan();
min_fvalue = cuts.min_vals_.ConstDeviceSpan();
}
}
// Get a matrix element, uses binary search for look up Return NaN if missing
// Given a row index and a feature index, returns the corresponding cut value
@@ -202,6 +208,7 @@ class EllpackPageImpl {
EllpackDeviceAccessor
GetDeviceAccessor(int device,
common::Span<FeatureType const> feature_types = {}) const;
EllpackDeviceAccessor GetHostAccessor(common::Span<FeatureType const> feature_types = {}) const;
private:
/*!

10
src/data/gradient_index.cc
View File

@@ -53,7 +53,7 @@ GHistIndexMatrix::GHistIndexMatrix(DMatrix *p_fmat, bst_bin_t max_bins_per_feat,
// hist
CHECK(!sorted_sketch);
for (auto const &page : p_fmat->GetBatches<SparsePage>()) {
this->columns_->Init(page, *this, sparse_thresh, n_threads);
this->columns_->InitFromSparse(page, *this, sparse_thresh, n_threads);
}
}
}
@@ -66,6 +66,12 @@ GHistIndexMatrix::GHistIndexMatrix(MetaInfo const &info, common::HistogramCuts &
max_num_bins(max_bin_per_feat),
isDense_{info.num_col_ * info.num_row_ == info.num_nonzero_} {}
#if !defined(XGBOOST_USE_CUDA)
GHistIndexMatrix::GHistIndexMatrix(Context const *, MetaInfo const &, EllpackPage const &,
BatchParam const &) {
common::AssertGPUSupport();
}
#endif // defined(XGBOOST_USE_CUDA)
GHistIndexMatrix::~GHistIndexMatrix() = default;
@@ -99,7 +105,7 @@ GHistIndexMatrix::GHistIndexMatrix(SparsePage const &batch, common::Span<Feature
this->PushBatch(batch, ft, n_threads);
this->columns_ = std::make_unique<common::ColumnMatrix>();
if (!std::isnan(sparse_thresh)) {
this->columns_->Init(batch, *this, sparse_thresh, n_threads);
this->columns_->InitFromSparse(batch, *this, sparse_thresh, n_threads);
}
}

111
src/data/gradient_index.cu Normal file
View File

@@ -0,0 +1,111 @@
/*!
* Copyright 2022 by XGBoost Contributors
*/
#include <memory> // std::unique_ptr
#include "../common/column_matrix.h"
#include "../common/hist_util.h" // Index
#include "ellpack_page.cuh"
#include "gradient_index.h"
#include "xgboost/data.h"
namespace xgboost {
// Similar to GHistIndexMatrix::SetIndexData, but without the need for adaptor or bin
// searching. Is there a way to unify the code?
template <typename BinT, typename CompressOffset>
void SetIndexData(Context const* ctx, EllpackPageImpl const* page,
std::vector<size_t>* p_hit_count_tloc, CompressOffset&& get_offset,
GHistIndexMatrix* out) {
auto accessor = page->GetHostAccessor();
auto const kNull = static_cast<bst_bin_t>(accessor.NullValue());
common::Span<BinT> index_data_span = {out->index.data<BinT>(), out->index.Size()};
auto n_bins_total = page->Cuts().TotalBins();
auto& hit_count_tloc = *p_hit_count_tloc;
hit_count_tloc.clear();
hit_count_tloc.resize(ctx->Threads() * n_bins_total, 0);
common::ParallelFor(page->Size(), ctx->Threads(), [&](auto i) {
auto tid = omp_get_thread_num();
size_t in_rbegin = page->row_stride * i;
size_t out_rbegin = out->row_ptr[i];
auto r_size = out->row_ptr[i + 1] - out->row_ptr[i];
for (size_t j = 0; j < r_size; ++j) {
auto bin_idx = accessor.gidx_iter[in_rbegin + j];
assert(bin_idx != kNull);
index_data_span[out_rbegin + j] = get_offset(bin_idx, j);
++hit_count_tloc[tid * n_bins_total + bin_idx];
}
});
}
void GetRowPtrFromEllpack(Context const* ctx, EllpackPageImpl const* page,
std::vector<size_t>* p_out) {
auto& row_ptr = *p_out;
row_ptr.resize(page->Size() + 1, 0);
if (page->is_dense) {
std::fill(row_ptr.begin() + 1, row_ptr.end(), page->row_stride);
} else {
auto accessor = page->GetHostAccessor();
auto const kNull = static_cast<bst_bin_t>(accessor.NullValue());
common::ParallelFor(page->Size(), ctx->Threads(), [&](auto i) {
size_t ibegin = page->row_stride * i;
for (size_t j = 0; j < page->row_stride; ++j) {
bst_bin_t bin_idx = accessor.gidx_iter[ibegin + j];
if (bin_idx != kNull) {
row_ptr[i + 1]++;
}
}
});
}
std::partial_sum(row_ptr.begin(), row_ptr.end(), row_ptr.begin());
}
GHistIndexMatrix::GHistIndexMatrix(Context const* ctx, MetaInfo const& info,
EllpackPage const& in_page, BatchParam const& p)
: max_num_bins{p.max_bin} {
auto page = in_page.Impl();
isDense_ = page->is_dense;
CHECK_EQ(info.num_row_, in_page.Size());
this->cut = page->Cuts();
// pull to host early, prevent race condition
this->cut.Ptrs();
this->cut.Values();
this->cut.MinValues();
this->ResizeIndex(info.num_nonzero_, page->is_dense);
if (page->is_dense) {
this->index.SetBinOffset(page->Cuts().Ptrs());
}
auto n_bins_total = page->Cuts().TotalBins();
GetRowPtrFromEllpack(ctx, page, &this->row_ptr);
if (page->is_dense) {
common::DispatchBinType(this->index.GetBinTypeSize(), [&](auto dtype) {
using T = decltype(dtype);
::xgboost::SetIndexData<T>(ctx, page, &hit_count_tloc_, index.MakeCompressor<T>(), this);
});
} else {
// no compression
::xgboost::SetIndexData<uint32_t>(
ctx, page, &hit_count_tloc_, [&](auto bin_idx, auto) { return bin_idx; }, this);
}
this->hit_count.resize(n_bins_total, 0);
this->GatherHitCount(ctx->Threads(), n_bins_total);
// sanity checks
CHECK_EQ(this->Features(), info.num_col_);
CHECK_EQ(this->Size(), info.num_row_);
CHECK(this->cut.cut_ptrs_.HostCanRead());
CHECK(this->cut.cut_values_.HostCanRead());
CHECK(this->cut.min_vals_.HostCanRead());
this->columns_ = std::make_unique<common::ColumnMatrix>(*this, p.sparse_thresh);
this->columns_->InitFromGHist(ctx, *this);
}
} // namespace xgboost

45
src/data/gradient_index.h
View File

@@ -69,7 +69,7 @@ class GHistIndexMatrix {
if (is_valid(elem)) {
bst_bin_t bin_idx{-1};
if (common::IsCat(ft, elem.column_idx)) {
bin_idx = cut.SearchCatBin(elem.value, elem.column_idx);
bin_idx = cut.SearchCatBin(elem.value, elem.column_idx, ptrs, values);
} else {
bin_idx = cut.SearchBin(elem.value, elem.column_idx, ptrs, values);
}
@@ -81,6 +81,17 @@ class GHistIndexMatrix {
});
}
// Gather hit_count from all threads
void GatherHitCount(int32_t n_threads, bst_bin_t n_bins_total) {
CHECK_EQ(hit_count.size(), n_bins_total);
common::ParallelFor(n_bins_total, n_threads, [&](bst_omp_uint idx) {
for (int32_t tid = 0; tid < n_threads; ++tid) {
hit_count[idx] += hit_count_tloc_[tid * n_bins_total + idx];
hit_count_tloc_[tid * n_bins_total + idx] = 0; // reset for next batch
}
});
}
template <typename Batch, typename IsValid>
void PushBatchImpl(int32_t n_threads, Batch const& batch, size_t rbegin, IsValid&& is_valid,
common::Span<FeatureType const> ft) {
@@ -95,33 +106,20 @@ class GHistIndexMatrix {
if (isDense_) {
index.SetBinOffset(cut.Ptrs());
}
uint32_t const* offsets = index.Offset();
if (isDense_) {
// Inside the lambda functions, bin_idx is the index for cut value across all
// features. By subtracting it with starting pointer of each feature, we can reduce
// it to smaller value and compress it to smaller types.
common::DispatchBinType(index.GetBinTypeSize(), [&](auto dtype) {
using T = decltype(dtype);
common::Span<T> index_data_span = {index.data<T>(), index.Size()};
SetIndexData(
index_data_span, rbegin, ft, batch_threads, batch, is_valid, n_bins_total,
[offsets](auto bin_idx, auto fidx) { return static_cast<T>(bin_idx - offsets[fidx]); });
SetIndexData(index_data_span, rbegin, ft, batch_threads, batch, is_valid, n_bins_total,
index.MakeCompressor<T>());
});
} else {
/* For sparse DMatrix we have to store index of feature for each bin
in index field to chose right offset. So offset is nullptr and index is
not reduced */
common::Span<uint32_t> index_data_span = {index.data<uint32_t>(), n_index};
// no compression
SetIndexData(index_data_span, rbegin, ft, batch_threads, batch, is_valid, n_bins_total,
[](auto idx, auto) { return idx; });
}
common::ParallelFor(n_bins_total, n_threads, [&](bst_omp_uint idx) {
for (int32_t tid = 0; tid < n_threads; ++tid) {
hit_count[idx] += hit_count_tloc_[tid * n_bins_total + idx];
hit_count_tloc_[tid * n_bins_total + idx] = 0; // reset for next batch
}
});
this->GatherHitCount(n_threads, n_bins_total);
}
public:
@@ -129,12 +127,12 @@ class GHistIndexMatrix {
std::vector<size_t> row_ptr;
/*! \brief The index data */
common::Index index;
/*! \brief hit count of each index */
/*! \brief hit count of each index, used for constructing the ColumnMatrix */
std::vector<size_t> hit_count;
/*! \brief The corresponding cuts */
common::HistogramCuts cut;
/*! \brief max_bin for each feature. */
size_t max_num_bins;
bst_bin_t max_num_bins;
/*! \brief base row index for current page (used by external memory) */
size_t base_rowid{0};
@@ -149,6 +147,13 @@ class GHistIndexMatrix {
* for push batch.
*/
GHistIndexMatrix(MetaInfo const& info, common::HistogramCuts&& cuts, bst_bin_t max_bin_per_feat);
/**
* \brief Constructor fro Iterative DMatrix where we might copy an existing ellpack page
* to host gradient index.
*/
GHistIndexMatrix(Context const* ctx, MetaInfo const& info, EllpackPage const& page,
BatchParam const& p);
/**
* \brief Constructor for external memory.
*/

11
src/data/iterative_dmatrix.cc
View File

@@ -205,12 +205,11 @@ void IterativeDMatrix::InitFromCPU(DataIterHandle iter_handle, float missing,
BatchSet<GHistIndexMatrix> IterativeDMatrix::GetGradientIndex(BatchParam const& param) {
CheckParam(param);
CHECK(ghist_) << R"(`QuantileDMatrix` is not initialized with CPU data but used for CPU training.
Possible solutions:
- Use `DMatrix` instead.
- Use CPU input for `QuantileDMatrix`.
- Run training on GPU.
)";
if (!ghist_) {
CHECK(ellpack_);
ghist_ = std::make_shared<GHistIndexMatrix>(&ctx_, Info(), *ellpack_, param);
}
auto begin_iter =
BatchIterator<GHistIndexMatrix>(new SimpleBatchIteratorImpl<GHistIndexMatrix>(ghist_));
return BatchSet<GHistIndexMatrix>(begin_iter);

13
src/data/iterative_dmatrix.h
View File

@@ -29,20 +29,17 @@ namespace data {
* `QuantileDMatrix` is an intermediate storage for quantilization results including
* quantile cuts and histogram index. Quantilization is designed to be performed on stream
* of data (or batches of it). As a result, the `QuantileDMatrix` is also designed to work
* with batches of data. During initializaion, it will walk through the data multiple
* times iteratively in order to perform quantilization. This design can help us reduce
* memory usage significantly by avoiding data concatenation along with removing the CSR
* matrix `SparsePage`. However, it has its limitation (can be fixed if needed):
* with batches of data. During initializaion, it walks through the data multiple times
* iteratively in order to perform quantilization. This design helps us reduce memory
* usage significantly by avoiding data concatenation along with removing the CSR matrix
* `SparsePage`. However, it has its limitation (can be fixed if needed):
*
* - It's only supported by hist tree method (both CPU and GPU) since approx requires a
* re-calculation of quantiles for each iteration. We can fix this by retaining a
* reference to the callback if there are feature requests.
*
* - The CPU format and the GPU format are different, the former uses a CSR + CSC for
* histogram index while the latter uses only Ellpack. This results into a design that
* we can obtain the GPU format from CPU but the other way around is not yet
* supported. We can search the bin value from ellpack to recover the feature index when
* we support copying data from GPU to CPU.
* histogram index while the latter uses only Ellpack.
*/
class IterativeDMatrix : public DMatrix {
MetaInfo info_;