Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[DISK] Add shard option to disk

Browse Source
This commit is contained in:
tqchen
2016-01-19 13:39:10 -08:00
parent 72961d914b
commit 2230f1273f
8 changed files with 356 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/data/sparse_page_source.h
View File

@@ -71,14 +71,14 @@ class SparsePageSource : public DataSource {
RowBatch batch_;
/*! \brief page currently on hold. */
SparsePage *page_;
/*! \brief The cache predix of the dataset. */
std::string cache_prefix_;
/*! \brief internal clock ptr */
size_t clock_ptr_;
/*! \brief file pointer to the row blob file. */
std::unique_ptr<dmlc::SeekStream> fi_;
std::vector<std::unique_ptr<dmlc::SeekStream> > files_;
/*! \brief Sparse page format file. */
std::unique_ptr<SparsePage::Format> format_;
std::vector<std::unique_ptr<SparsePage::Format> > formats_;
/*! \brief internal prefetcher. */
dmlc::ThreadedIter<SparsePage> prefetcher_;
std::vector<std::unique_ptr<dmlc::ThreadedIter<SparsePage> > > prefetchers_;
};
} // namespace data
} // namespace xgboost