Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reducing memory consumption for 'hist' method on CPU (#5334)

Browse Source
This commit is contained in:
ShvetsKS
2020-03-28 04:45:52 +03:00
committed by GitHub
parent 13b10a6370
commit 27a8e36fc3
7 changed files with 849 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
tests/cpp/common/test_column_matrix.cc
View File

@@ -9,28 +9,46 @@ namespace xgboost {
namespace common {
TEST(DenseColumn, Test) {
auto dmat = RandomDataGenerator(100, 10, 0.0).GenerateDMatix();
GHistIndexMatrix gmat;
gmat.Init(dmat.get(), 256);
ColumnMatrix column_matrix;
column_matrix.Init(gmat, 0.2);
uint64_t max_num_bins[] = {static_cast<uint64_t>(std::numeric_limits<uint8_t>::max()) + 1,
static_cast<uint64_t>(std::numeric_limits<uint16_t>::max()) + 1,
static_cast<uint64_t>(std::numeric_limits<uint16_t>::max()) + 2};
for (size_t max_num_bin : max_num_bins) {
auto dmat = RandomDataGenerator(100, 10, 0.0).GenerateDMatix();
GHistIndexMatrix gmat;
gmat.Init(dmat.get(), max_num_bin);
ColumnMatrix column_matrix;
column_matrix.Init(gmat, 0.2);
for (auto i = 0ull; i < dmat->Info().num_row_; i++) {
for (auto j = 0ull; j < dmat->Info().num_col_; j++) {
auto col = column_matrix.GetColumn(j);
ASSERT_EQ(gmat.index[i * dmat->Info().num_col_ + j],
col.GetGlobalBinIdx(i));
for (auto i = 0ull; i < dmat->Info().num_row_; i++) {
for (auto j = 0ull; j < dmat->Info().num_col_; j++) {
switch (column_matrix.GetTypeSize()) {
case UINT8_BINS_TYPE_SIZE: {
auto col = column_matrix.GetColumn<uint8_t>(j);
ASSERT_EQ(gmat.index[i * dmat->Info().num_col_ + j],
(*col.get()).GetGlobalBinIdx(i));
}
break;
case UINT16_BINS_TYPE_SIZE: {
auto col = column_matrix.GetColumn<uint16_t>(j);
ASSERT_EQ(gmat.index[i * dmat->Info().num_col_ + j],
(*col.get()).GetGlobalBinIdx(i));
}
break;
case UINT32_BINS_TYPE_SIZE: {
auto col = column_matrix.GetColumn<uint32_t>(j);
ASSERT_EQ(gmat.index[i * dmat->Info().num_col_ + j],
(*col.get()).GetGlobalBinIdx(i));
}
break;
}
}
}
}
}
TEST(SparseColumn, Test) {
auto dmat = RandomDataGenerator(100, 1, 0.85).GenerateDMatix();
GHistIndexMatrix gmat;
gmat.Init(dmat.get(), 256);
ColumnMatrix column_matrix;
column_matrix.Init(gmat, 0.5);
auto col = column_matrix.GetColumn(0);
template<typename BinIdxType>
inline void CheckSparseColumn(const Column<BinIdxType>& col_input, const GHistIndexMatrix& gmat) {
const SparseColumn<BinIdxType>& col = static_cast<const SparseColumn<BinIdxType>& >(col_input);
ASSERT_EQ(col.Size(), gmat.index.size());
for (auto i = 0ull; i < col.Size(); i++) {
ASSERT_EQ(gmat.index[gmat.row_ptr[col.GetRowIdx(i)]],
@@ -38,20 +56,77 @@ TEST(SparseColumn, Test) {
}
}
TEST(DenseColumnWithMissing, Test) {
auto dmat = RandomDataGenerator(100, 1, 0.5).GenerateDMatix();
GHistIndexMatrix gmat;
gmat.Init(dmat.get(), 256);
ColumnMatrix column_matrix;
column_matrix.Init(gmat, 0.2);
auto col = column_matrix.GetColumn(0);
TEST(SparseColumn, Test) {
uint64_t max_num_bins[] = {static_cast<uint64_t>(std::numeric_limits<uint8_t>::max()) + 1,
static_cast<uint64_t>(std::numeric_limits<uint16_t>::max()) + 1,
static_cast<uint64_t>(std::numeric_limits<uint16_t>::max()) + 2};
for (size_t max_num_bin : max_num_bins) {
auto dmat = RandomDataGenerator(100, 1, 0.85).GenerateDMatix();
GHistIndexMatrix gmat;
gmat.Init(dmat.get(), max_num_bin);
ColumnMatrix column_matrix;
column_matrix.Init(gmat, 0.5);
switch (column_matrix.GetTypeSize()) {
case UINT8_BINS_TYPE_SIZE: {
auto col = column_matrix.GetColumn<uint8_t>(0);
CheckSparseColumn(*col.get(), gmat);
}
break;
case UINT16_BINS_TYPE_SIZE: {
auto col = column_matrix.GetColumn<uint16_t>(0);
CheckSparseColumn(*col.get(), gmat);
}
break;
case UINT32_BINS_TYPE_SIZE: {
auto col = column_matrix.GetColumn<uint32_t>(0);
CheckSparseColumn(*col.get(), gmat);
}
break;
}
}
}
template<typename BinIdxType>
inline void CheckColumWithMissingValue(const Column<BinIdxType>& col_input,
const GHistIndexMatrix& gmat) {
const DenseColumn<BinIdxType>& col = static_cast<const DenseColumn<BinIdxType>& >(col_input);
for (auto i = 0ull; i < col.Size(); i++) {
if (col.IsMissing(i)) continue;
EXPECT_EQ(gmat.index[gmat.row_ptr[col.GetRowIdx(i)]],
EXPECT_EQ(gmat.index[gmat.row_ptr[i]],
col.GetGlobalBinIdx(i));
}
}
TEST(DenseColumnWithMissing, Test) {
uint64_t max_num_bins[] = { static_cast<uint64_t>(std::numeric_limits<uint8_t>::max()) + 1,
static_cast<uint64_t>(std::numeric_limits<uint16_t>::max()) + 1,
static_cast<uint64_t>(std::numeric_limits<uint16_t>::max()) + 2 };
for (size_t max_num_bin : max_num_bins) {
auto dmat = RandomDataGenerator(100, 1, 0.5).GenerateDMatix();
GHistIndexMatrix gmat;
gmat.Init(dmat.get(), max_num_bin);
ColumnMatrix column_matrix;
column_matrix.Init(gmat, 0.2);
switch (column_matrix.GetTypeSize()) {
case UINT8_BINS_TYPE_SIZE: {
auto col = column_matrix.GetColumn<uint8_t>(0);
CheckColumWithMissingValue(*col.get(), gmat);
}
break;
case UINT16_BINS_TYPE_SIZE: {
auto col = column_matrix.GetColumn<uint16_t>(0);
CheckColumWithMissingValue(*col.get(), gmat);
}
break;
case UINT32_BINS_TYPE_SIZE: {
auto col = column_matrix.GetColumn<uint32_t>(0);
CheckColumWithMissingValue(*col.get(), gmat);
}
break;
}
}
}
void TestGHistIndexMatrixCreation(size_t nthreads) {
dmlc::TemporaryDirectory tmpdir;
std::string filename = tmpdir.path + "/big.libsvm";