External data adapters (#5044)

* Use external data adapters as lightweight intermediate layer between external data and DMatrix
This commit is contained in:
Rory Mitchell
2019-12-04 10:56:17 +13:00
committed by GitHub
parent f2277e7106
commit e3c34c79be
15 changed files with 1058 additions and 593 deletions

View File

@@ -465,6 +465,20 @@ class DMatrix {
*/
static DMatrix* Create(std::unique_ptr<DataSource<SparsePage>>&& source,
const std::string& cache_prefix = "");
/**
* \brief Creates a new DMatrix from an external data adapter.
*
* \tparam AdapterT Type of the adapter.
* \param adapter View onto an external data.
* \param missing Values to count as missing.
* \param nthread Number of threads for construction.
*
* \return a Created DMatrix.
*/
template <typename AdapterT>
static DMatrix* Create(AdapterT* adapter, float missing, int nthread);
/*!
* \brief Create a DMatrix by loading data from parser.
* Parser can later be deleted after the DMatrix i created.