some lint
This commit is contained in:
parent
e6e467ad60
commit
401d648372
@ -5,6 +5,7 @@
|
|||||||
* row iterator based on sparse page
|
* row iterator based on sparse page
|
||||||
* \author Tianqi Chen
|
* \author Tianqi Chen
|
||||||
*/
|
*/
|
||||||
|
#include <vector>
|
||||||
#include "../data.h"
|
#include "../data.h"
|
||||||
#include "../utils/iterator.h"
|
#include "../utils/iterator.h"
|
||||||
#include "../utils/thread_buffer.h"
|
#include "../utils/thread_buffer.h"
|
||||||
@ -15,7 +16,7 @@ namespace io {
|
|||||||
/*! \brief page structure that can be used to store a rowbatch */
|
/*! \brief page structure that can be used to store a rowbatch */
|
||||||
struct RowBatchPage {
|
struct RowBatchPage {
|
||||||
public:
|
public:
|
||||||
RowBatchPage(size_t page_size) : kPageSize(page_size) {
|
explicit RowBatchPage(size_t page_size) : kPageSize(page_size) {
|
||||||
data_ = new int[kPageSize];
|
data_ = new int[kPageSize];
|
||||||
utils::Assert(data_ != NULL, "fail to allocate row batch page");
|
utils::Assert(data_ != NULL, "fail to allocate row batch page");
|
||||||
this->Clear();
|
this->Clear();
|
||||||
@ -33,7 +34,7 @@ struct RowBatchPage {
|
|||||||
if (FreeBytes() < dsize+ sizeof(int)) return false;
|
if (FreeBytes() < dsize+ sizeof(int)) return false;
|
||||||
row_ptr(Size() + 1) = row_ptr(Size()) + row.length;
|
row_ptr(Size() + 1) = row_ptr(Size()) + row.length;
|
||||||
memcpy(data_ptr(row_ptr(Size())) , row.data, dsize);
|
memcpy(data_ptr(row_ptr(Size())) , row.data, dsize);
|
||||||
++ data_[0];
|
++data_[0];
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
/*!
|
/*!
|
||||||
@ -85,8 +86,8 @@ struct RowBatchPage {
|
|||||||
private:
|
private:
|
||||||
/*! \return number of elements */
|
/*! \return number of elements */
|
||||||
inline size_t FreeBytes(void) {
|
inline size_t FreeBytes(void) {
|
||||||
return (kPageSize - (Size() + 2)) * sizeof(int)
|
return (kPageSize - (Size() + 2)) * sizeof(int) -
|
||||||
- row_ptr(Size()) * sizeof(RowBatch::Entry) ;
|
row_ptr(Size()) * sizeof(RowBatch::Entry);
|
||||||
}
|
}
|
||||||
/*! \brief equivalent row pointer at i */
|
/*! \brief equivalent row pointer at i */
|
||||||
inline int& row_ptr(int i) {
|
inline int& row_ptr(int i) {
|
||||||
@ -108,8 +109,7 @@ class ThreadRowPageIterator: public utils::IIterator<RowBatch> {
|
|||||||
page_ = NULL;
|
page_ = NULL;
|
||||||
base_rowid_ = 0;
|
base_rowid_ = 0;
|
||||||
}
|
}
|
||||||
virtual ~ThreadRowPageIterator(void) {
|
virtual ~ThreadRowPageIterator(void) {}
|
||||||
}
|
|
||||||
virtual void Init(void) {
|
virtual void Init(void) {
|
||||||
}
|
}
|
||||||
virtual void BeforeFirst(void) {
|
virtual void BeforeFirst(void) {
|
||||||
@ -117,12 +117,12 @@ class ThreadRowPageIterator: public utils::IIterator<RowBatch> {
|
|||||||
base_rowid_ = 0;
|
base_rowid_ = 0;
|
||||||
}
|
}
|
||||||
virtual bool Next(void) {
|
virtual bool Next(void) {
|
||||||
if(!itr.Next(page_)) return false;
|
if (!itr.Next(page_)) return false;
|
||||||
out_ = page_->GetRowBatch(&tmp_ptr_, base_rowid_);
|
out_ = page_->GetRowBatch(&tmp_ptr_, base_rowid_);
|
||||||
base_rowid_ += out_.size;
|
base_rowid_ += out_.size;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
virtual const RowBatch &Value(void) const{
|
virtual const RowBatch &Value(void) const {
|
||||||
return out_;
|
return out_;
|
||||||
}
|
}
|
||||||
/*! \brief load and initialize the iterator with fi */
|
/*! \brief load and initialize the iterator with fi */
|
||||||
@ -152,6 +152,7 @@ class ThreadRowPageIterator: public utils::IIterator<RowBatch> {
|
|||||||
}
|
}
|
||||||
/*! \brief page size 64 MB */
|
/*! \brief page size 64 MB */
|
||||||
static const size_t kPageSize = 64 << 18;
|
static const size_t kPageSize = 64 << 18;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
// base row id
|
// base row id
|
||||||
size_t base_rowid_;
|
size_t base_rowid_;
|
||||||
@ -195,7 +196,7 @@ class ThreadRowPageIterator: public utils::IIterator<RowBatch> {
|
|||||||
|
|
||||||
protected:
|
protected:
|
||||||
PagePtr page_;
|
PagePtr page_;
|
||||||
utils::ThreadBuffer<PagePtr,Factory> itr;
|
utils::ThreadBuffer<PagePtr, Factory> itr;
|
||||||
};
|
};
|
||||||
|
|
||||||
/*! \brief data matrix using page */
|
/*! \brief data matrix using page */
|
||||||
@ -213,7 +214,7 @@ class DMatrixPageBase : public DataMatrix {
|
|||||||
/*! \brief load and initialize the iterator with fi */
|
/*! \brief load and initialize the iterator with fi */
|
||||||
inline void Load(utils::FileStream &fi,
|
inline void Load(utils::FileStream &fi,
|
||||||
bool silent = false,
|
bool silent = false,
|
||||||
const char *fname = NULL){
|
const char *fname = NULL) {
|
||||||
int tmagic;
|
int tmagic;
|
||||||
utils::Check(fi.Read(&tmagic, sizeof(tmagic)) != 0, "invalid input file format");
|
utils::Check(fi.Read(&tmagic, sizeof(tmagic)) != 0, "invalid input file format");
|
||||||
utils::Check(tmagic == magic, "invalid format,magic number mismatch");
|
utils::Check(tmagic == magic, "invalid format,magic number mismatch");
|
||||||
@ -229,7 +230,7 @@ class DMatrixPageBase : public DataMatrix {
|
|||||||
utils::Printf("\n");
|
utils::Printf("\n");
|
||||||
}
|
}
|
||||||
if (info.group_ptr.size() != 0) {
|
if (info.group_ptr.size() != 0) {
|
||||||
utils::Printf("data contains %u groups\n", (unsigned)info.group_ptr.size()-1);
|
utils::Printf("data contains %u groups\n", (unsigned)info.group_ptr.size() - 1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -249,8 +250,8 @@ class DMatrixPageBase : public DataMatrix {
|
|||||||
}
|
}
|
||||||
/*! \brief magic number used to identify DMatrix */
|
/*! \brief magic number used to identify DMatrix */
|
||||||
static const int kMagic = TKMagic;
|
static const int kMagic = TKMagic;
|
||||||
protected:
|
|
||||||
|
|
||||||
|
protected:
|
||||||
/*! \brief row iterator */
|
/*! \brief row iterator */
|
||||||
ThreadRowPageIterator *iter_;
|
ThreadRowPageIterator *iter_;
|
||||||
};
|
};
|
||||||
|
|||||||
@ -5,6 +5,9 @@
|
|||||||
* sparse page manager for fmatrix
|
* sparse page manager for fmatrix
|
||||||
* \author Tianqi Chen
|
* \author Tianqi Chen
|
||||||
*/
|
*/
|
||||||
|
#include <vector>
|
||||||
|
#include <string>
|
||||||
|
#include <algorithm>
|
||||||
#include "../data.h"
|
#include "../data.h"
|
||||||
#include "../utils/iterator.h"
|
#include "../utils/iterator.h"
|
||||||
#include "../utils/io.h"
|
#include "../utils/io.h"
|
||||||
@ -52,6 +55,7 @@ class CSCMatrixManager {
|
|||||||
batch.col_data = BeginPtr(col_data);
|
batch.col_data = BeginPtr(col_data);
|
||||||
return batch;
|
return batch;
|
||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
/*! \brief buffer space, not to be changed since ready */
|
/*! \brief buffer space, not to be changed since ready */
|
||||||
std::vector<ColBatch::Entry> buffer;
|
std::vector<ColBatch::Entry> buffer;
|
||||||
@ -124,6 +128,7 @@ class CSCMatrixManager {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
/*! \brief fill a page with */
|
/*! \brief fill a page with */
|
||||||
inline bool TryFill(size_t cidx, Page *p_page) {
|
inline bool TryFill(size_t cidx, Page *p_page) {
|
||||||
@ -177,17 +182,18 @@ class ThreadColPageIterator : public utils::IIterator<ColBatch> {
|
|||||||
virtual bool Next(void) {
|
virtual bool Next(void) {
|
||||||
// page to be loaded
|
// page to be loaded
|
||||||
CSCMatrixManager::PagePtr page;
|
CSCMatrixManager::PagePtr page;
|
||||||
if(!itr_.Next(page)) return false;
|
if (!itr_.Next(page)) return false;
|
||||||
out_ = page->GetBatch();
|
out_ = page->GetBatch();
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
virtual const ColBatch &Value(void) const{
|
virtual const ColBatch &Value(void) const {
|
||||||
return out_;
|
return out_;
|
||||||
}
|
}
|
||||||
inline const std::vector<size_t> &col_ptr(void) const {
|
inline const std::vector<size_t> &col_ptr(void) const {
|
||||||
return itr_.get_factory().col_ptr();
|
return itr_.get_factory().col_ptr();
|
||||||
}
|
}
|
||||||
inline void SetColSet(const std::vector<bst_uint> &cset, bool setall = false) {
|
inline void SetColSet(const std::vector<bst_uint> &cset,
|
||||||
|
bool setall = false) {
|
||||||
itr_.get_factory().SetColSet(cset, setall);
|
itr_.get_factory().SetColSet(cset, setall);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -195,9 +201,8 @@ class ThreadColPageIterator : public utils::IIterator<ColBatch> {
|
|||||||
// output data
|
// output data
|
||||||
ColBatch out_;
|
ColBatch out_;
|
||||||
// internal iterator
|
// internal iterator
|
||||||
utils::ThreadBuffer<CSCMatrixManager::PagePtr,CSCMatrixManager> itr_;
|
utils::ThreadBuffer<CSCMatrixManager::PagePtr, CSCMatrixManager> itr_;
|
||||||
};
|
};
|
||||||
|
|
||||||
/*!
|
/*!
|
||||||
* \brief sparse matrix that support column access
|
* \brief sparse matrix that support column access
|
||||||
*/
|
*/
|
||||||
@ -295,7 +300,6 @@ class FMatrixPage : public IFMatrix {
|
|||||||
utils::FileStream fo(utils::FopenCheck(fname, "wb+"));
|
utils::FileStream fo(utils::FopenCheck(fname, "wb+"));
|
||||||
// use 64M buffer
|
// use 64M buffer
|
||||||
utils::SparseCSRFileBuilder<ColBatch::Entry> builder(&fo, buffer_size);
|
utils::SparseCSRFileBuilder<ColBatch::Entry> builder(&fo, buffer_size);
|
||||||
|
|
||||||
// start working
|
// start working
|
||||||
row_iter_->BeforeFirst();
|
row_iter_->BeforeFirst();
|
||||||
while (row_iter_->Next()) {
|
while (row_iter_->Next()) {
|
||||||
@ -350,7 +354,7 @@ class FMatrixPage : public IFMatrix {
|
|||||||
|
|
||||||
class DMatrixColPage : public DMatrixPageBase<0xffffab03> {
|
class DMatrixColPage : public DMatrixPageBase<0xffffab03> {
|
||||||
public:
|
public:
|
||||||
DMatrixColPage(const char *fname) {
|
explicit DMatrixColPage(const char *fname) {
|
||||||
std::string fext = fname;
|
std::string fext = fname;
|
||||||
fext += ".col";
|
fext += ".col";
|
||||||
fmat_ = new FMatrixPage(iter_, fext.c_str());
|
fmat_ = new FMatrixPage(iter_, fext.c_str());
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user