Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

various fix of page

Browse Source
This commit is contained in:
tqchen 2014-08-29 20:54:24 -07:00
parent ce772c2f3e
commit 7bc1c3ee79
4 changed files with 14 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/io/io.cpp
View File

@ -37,8 +37,7 @@ DataMatrix* LoadDataMatrix(const char *fname, bool silent, bool savebuffer) {
} }
void SaveDataMatrix(const DataMatrix &dmat, const char *fname, bool silent) { void SaveDataMatrix(const DataMatrix &dmat, const char *fname, bool silent) {
if (!strcmp(fname + strlen(fname) - 5, ".page")) { if (!strcmp(fname + strlen(fname) - 5, ".page")) {
DMatrixPage::Save(fname, dmat, silent); DMatrixPage::Save(fname, dmat, silent);
return; return;
} }
@ -46,7 +45,9 @@ void SaveDataMatrix(const DataMatrix &dmat, const char *fname, bool silent) {
const DMatrixSimple *p_dmat = static_cast<const DMatrixSimple*>(&dmat); const DMatrixSimple *p_dmat = static_cast<const DMatrixSimple*>(&dmat);
p_dmat->SaveBinary(fname, silent); p_dmat->SaveBinary(fname, silent);
} else { } else {
utils::Error("not implemented"); DMatrixSimple smat;
smat.CopyFrom(dmat);
smat.SaveBinary(fname, silent);
} }
} }

12
src/io/page_dmatrix-inl.hpp
View File

@ -32,7 +32,7 @@ struct RowBatchPage {
const size_t dsize = row.length * sizeof(RowBatch::Entry); const size_t dsize = row.length * sizeof(RowBatch::Entry);
if (FreeBytes() < dsize+ sizeof(int)) return false; if (FreeBytes() < dsize+ sizeof(int)) return false;
row_ptr(Size() + 1) = row_ptr(Size()) + row.length; row_ptr(Size() + 1) = row_ptr(Size()) + row.length;
memcpy(data_ptr(Size()) , row.data, dsize); memcpy(data_ptr(row_ptr(Size())) , row.data, dsize);
++ data_[0]; ++ data_[0];
return true; return true;
} }
@ -48,13 +48,18 @@ struct RowBatchPage {
batch.data_ptr = this->data_ptr(0); batch.data_ptr = this->data_ptr(0);
batch.size = static_cast<size_t>(this->Size()); batch.size = static_cast<size_t>(this->Size());
std::vector<size_t> &rptr = *p_rptr; std::vector<size_t> &rptr = *p_rptr;
rptr.resize(this->Size()+1); rptr.resize(this->Size() + 1);
for (size_t i = 0; i < rptr.size(); ++i) { for (size_t i = 0; i < rptr.size(); ++i) {
rptr[i] = static_cast<size_t>(this->row_ptr(i)); rptr[i] = static_cast<size_t>(this->row_ptr(i));
} }
batch.ind_ptr = &rptr[0]; batch.ind_ptr = &rptr[0];
return batch; return batch;
} }
/*! \brief get i-th row from the batch */
inline RowBatch::Inst operator[](size_t i) {
return RowBatch::Inst(data_ptr(0) + row_ptr(i),
static_cast<bst_uint>(row_ptr(i+1) - row_ptr(i)));
}
/*! /*!
* \brief clear the page, cleanup the content * \brief clear the page, cleanup the content
*/ */
@ -77,7 +82,7 @@ struct RowBatchPage {
return data_[0]; return data_[0];
} }
/*! \brief page size 64 MB */ /*! \brief page size 64 MB */
static const size_t kPageSize = 64 << 18; static const size_t kPageSize = 64 << 8;
private: private:
/*! \return number of elements */ /*! \return number of elements */
@ -112,7 +117,6 @@ class ThreadRowPageIterator: public utils::IIterator<RowBatch> {
itr.BeforeFirst(); itr.BeforeFirst();
isend_ = false; isend_ = false;
base_rowid_ = 0; base_rowid_ = 0;
utils::Assert(this->LoadNextPage(), "ThreadRowPageIterator");
} }
virtual bool Next(void) { virtual bool Next(void) {
if(!this->LoadNextPage()) return false; if(!this->LoadNextPage()) return false;

2
src/io/simple_dmatrix-inl.hpp
View File

@ -44,8 +44,8 @@ class DMatrixSimple : public DataMatrix {
} }
/*! \brief copy content data from source matrix */ /*! \brief copy content data from source matrix */
inline void CopyFrom(const DataMatrix &src) { inline void CopyFrom(const DataMatrix &src) {
this->info = src.info;
this->Clear(); this->Clear();
this->info = src.info;
// clone data content in thos matrix // clone data content in thos matrix
utils::IIterator<RowBatch> *iter = src.fmat()->RowIterator(); utils::IIterator<RowBatch> *iter = src.fmat()->RowIterator();
iter->BeforeFirst(); iter->BeforeFirst();

2
src/io/simple_fmatrix-inl.hpp
View File

@ -150,7 +150,7 @@ class FMatrixS : public IFMatrix{
iter_->BeforeFirst(); iter_->BeforeFirst();
while (iter_->Next()) { while (iter_->Next()) {
const RowBatch &batch = iter_->Value(); const RowBatch &batch = iter_->Value();
for (size_t i = 0; i < batch.size; ++i) { for (size_t i = 0; i < batch.size; ++i) {
if (pkeep == 1.0f || random::SampleBinary(pkeep)) { if (pkeep == 1.0f || random::SampleBinary(pkeep)) {
buffered_rowset_.push_back(static_cast<bst_uint>(batch.base_rowid+i)); buffered_rowset_.push_back(static_cast<bst_uint>(batch.base_rowid+i));
RowBatch::Inst inst = batch[i]; RowBatch::Inst inst = batch[i];