Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix external memory for get column batches. (#4622)

Browse Source 
* Fix external memory for get column batches.

This fixes two bugs:

* Use PushCSC for get column batches.
* Don't remove the created temporary directory before finishing test.

* Check all pages.
This commit is contained in:
Jiaming Yuan
2019-06-30 09:56:49 +08:00
committed by GitHub
parent a30176907f
commit 45876bf41b
14 changed files with 90 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests/cpp/predictor/test_cpu_predictor.cc
View File

@@ -56,7 +56,9 @@ TEST(cpu_predictor, Test) {
}
TEST(cpu_predictor, ExternalMemoryTest) {
std::unique_ptr<DMatrix> dmat = CreateSparsePageDMatrix(12, 64);
dmlc::TemporaryDirectory tmpdir;
std::string filename = tmpdir.path + "/big.libsvm";
std::unique_ptr<DMatrix> dmat = CreateSparsePageDMatrix(12, 64, filename);
auto lparam = CreateEmptyGenericParam(0, 0);
std::unique_ptr<Predictor> cpu_predictor =
std::unique_ptr<Predictor>(Predictor::Create("cpu_predictor", &lparam));

14
tests/cpp/predictor/test_gpu_predictor.cu
View File

@@ -97,7 +97,9 @@ TEST(gpu_predictor, ExternalMemoryTest) {
gbm::GBTreeModel model = CreateTestModel();
int n_col = 3;
model.param.num_feature = n_col;
std::unique_ptr<DMatrix> dmat = CreateSparsePageDMatrix(32, 64);
dmlc::TemporaryDirectory tmpdir;
std::string filename = tmpdir.path + "/big.libsvm";
std::unique_ptr<DMatrix> dmat = CreateSparsePageDMatrix(32, 64, filename);
// Test predict batch
HostDeviceVector<float> out_predictions;
@@ -268,9 +270,13 @@ TEST(gpu_predictor, MGPU_ExternalMemoryTest) {
const int n_classes = 3;
model.param.num_output_group = n_classes;
std::vector<std::unique_ptr<DMatrix>> dmats;
dmats.push_back(CreateSparsePageDMatrix(9, 64UL));
dmats.push_back(CreateSparsePageDMatrix(128, 128UL));
dmats.push_back(CreateSparsePageDMatrix(1024, 1024UL));
dmlc::TemporaryDirectory tmpdir;
std::string file0 = tmpdir.path + "/big_0.libsvm";
std::string file1 = tmpdir.path + "/big_1.libsvm";
std::string file2 = tmpdir.path + "/big_2.libsvm";
dmats.push_back(CreateSparsePageDMatrix(9, 64UL, file0));
dmats.push_back(CreateSparsePageDMatrix(128, 128UL, file1));
dmats.push_back(CreateSparsePageDMatrix(1024, 1024UL, file2));
for (const auto& dmat: dmats) {
// Test predict batch