Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

make rowbatch page flexible

Browse Source
This commit is contained in:
tqchen 2014-09-01 20:44:15 -07:00
parent 7d1e9f06d4
commit 9d3e09ff2a
1 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/io/page_dmatrix-inl.hpp
View File

@ -16,7 +16,7 @@ namespace io {
/*! \brief page structure that can be used to store a rowbatch */ /*! \brief page structure that can be used to store a rowbatch */
struct RowBatchPage { struct RowBatchPage {
public: public:
RowBatchPage(void) { RowBatchPage(size_t page_size) : kPageSize(page_size) {
data_ = new int[kPageSize]; data_ = new int[kPageSize];
utils::Assert(data_ != NULL, "fail to allocate row batch page"); utils::Assert(data_ != NULL, "fail to allocate row batch page");
this->Clear(); this->Clear();
@ -82,8 +82,6 @@ struct RowBatchPage {
inline int Size(void) const { inline int Size(void) const {
return data_[0]; return data_[0];
} }
/*! \brief page size 64 MB */
static const size_t kPageSize = 64 << 18;
private: private:
/*! \return number of elements */ /*! \return number of elements */
@ -98,6 +96,8 @@ struct RowBatchPage {
inline RowBatch::Entry* data_ptr(int i) { inline RowBatch::Entry* data_ptr(int i) {
return (RowBatch::Entry*)(&data_[1]) + i; return (RowBatch::Entry*)(&data_[1]) + i;
} }
// page size
const size_t kPageSize;
// content of data // content of data
int *data_; int *data_;
}; };
@ -137,7 +137,7 @@ class ThreadRowPageIterator: public utils::IIterator<RowBatch> {
*/ */
inline static void Save(utils::IIterator<RowBatch> *iter, inline static void Save(utils::IIterator<RowBatch> *iter,
utils::IStream &fo) { utils::IStream &fo) {
RowBatchPage page; RowBatchPage page(kPageSize);
iter->BeforeFirst(); iter->BeforeFirst();
while (iter->Next()) { while (iter->Next()) {
const RowBatch &batch = iter->Value(); const RowBatch &batch = iter->Value();
@ -151,6 +151,8 @@ class ThreadRowPageIterator: public utils::IIterator<RowBatch> {
} }
if (page.Size() != 0) page.Save(fo); if (page.Size() != 0) page.Save(fo);
} }
/*! \brief page size 64 MB */
static const size_t kPageSize = 64 << 18;
private: private:
// base row id // base row id
size_t base_rowid_; size_t base_rowid_;
@ -178,7 +180,7 @@ class ThreadRowPageIterator: public utils::IIterator<RowBatch> {
return val->Load(fi); return val->Load(fi);
} }
inline PagePtr Create(void) { inline PagePtr Create(void) {
PagePtr a = new RowBatchPage(); PagePtr a = new RowBatchPage(kPageSize);
return a; return a;
} }
inline void FreeSpace(PagePtr &a) { inline void FreeSpace(PagePtr &a) {