Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add a test for cpu predictor using external memory (#4308)

Browse Source 
* add a test for cpu predictor using external memory

* allow different page size for testing
This commit is contained in:
Rong Ou
2019-04-09 18:25:10 -07:00
committed by Rory Mitchell
parent b72eab3e07
commit 81c1cd40ca
7 changed files with 95 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
include/xgboost/data.h
View File

@@ -433,12 +433,14 @@ class DMatrix {
* \param load_row_split Flag to read in part of rows, divided among the workers in distributed mode.
* \param file_format The format type of the file, used for dmlc::Parser::Create.
* By default "auto" will be able to load in both local binary file.
* \param page_size Page size for external memory.
* \return The created DMatrix.
*/
static DMatrix* Load(const std::string& uri,
bool silent,
bool load_row_split,
const std::string& file_format = "auto");
const std::string& file_format = "auto",
const size_t page_size = kPageSize);
/*!
* \brief create a new DMatrix, by wrapping a row_iterator, and meta info.
* \param source The source iterator of the data, the create function takes ownership of the source.
@@ -454,6 +456,7 @@ class DMatrix {
* \param parser The input data parser
* \param cache_prefix The path to prefix of temporary cache file of the DMatrix when used in external memory mode.
* This can be nullptr for common cases, and in-memory mode will be used.
* \param page_size Page size for external memory.
* \sa dmlc::Parser
* \note dmlc-core provides efficient distributed data parser for libsvm format.
* User can create and register customized parser to load their own format using DMLC_REGISTER_DATA_PARSER.
@@ -461,7 +464,11 @@ class DMatrix {
* \return A created DMatrix.
*/
static DMatrix* Create(dmlc::Parser<uint32_t>* parser,
const std::string& cache_prefix = "");
const std::string& cache_prefix = "",
const size_t page_size = kPageSize);
/*! \brief page size 32 MB */
static const size_t kPageSize = 32UL << 20UL;
};
// implementation of inline functions