Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[IO] Enable external memory

Browse Source
This commit is contained in:
tqchen
2016-01-10 12:13:39 -08:00
parent 5f28617d7d
commit ef1021e759
11 changed files with 654 additions and 626 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/data/data.cc
View File

@@ -8,6 +8,8 @@
#include "./sparse_batch_page.h"
#include "./simple_dmatrix.h"
#include "./simple_csr_source.h"
#include "./sparse_page_source.h"
#include "./sparse_page_dmatrix.h"
#include "../common/io.h"
namespace xgboost {
@@ -151,8 +153,11 @@ DMatrix* DMatrix::Create(dmlc::Parser<uint32_t>* parser,
source->CopyFrom(parser);
return DMatrix::Create(std::move(source), cache_prefix);
} else {
LOG(FATAL) << "external memory not yet implemented";
return nullptr;
if (!data::SparsePageSource::CacheExist(cache_prefix)) {
data::SparsePageSource::Create(parser, cache_prefix);
}
std::unique_ptr<data::SparsePageSource> source(new data::SparsePageSource(cache_prefix));
return DMatrix::Create(std::move(source), cache_prefix);
}
}
@@ -165,6 +170,10 @@ void DMatrix::SaveToLocalFile(const std::string& fname) {
DMatrix* DMatrix::Create(std::unique_ptr<DataSource>&& source,
const std::string& cache_prefix) {
return new data::SimpleDMatrix(std::move(source));
if (cache_prefix.length() == 0) {
return new data::SimpleDMatrix(std::move(source));
} else {
return new data::SparsePageDMatrix(std::move(source), cache_prefix);
}
}
} // namespace xgboost