Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Squashed 'subtree/rabit/' changes from d1d2ab4..4fe8d1d

Browse Source 
4fe8d1d ok io
a5d77ca checkin new dmlc interface

git-subtree-dir: subtree/rabit
git-subtree-split: 4fe8d1d66bb66e4293bbff0ea3207020e38be6a7
This commit is contained in:
tqchen 2015-04-29 20:22:11 -07:00
parent afe0a552e0
commit 18164e677a
1 changed files with 22 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
include/dmlc/io.h
View File

@ -12,6 +12,13 @@
#include <ostream> #include <ostream>
#include <streambuf> #include <streambuf>
// include uint64_t only to make io standalone
#ifdef _MSC_VER
typedef unsigned __int64 uint64_t;
#else
#include <cinttypes>
#endif
/*! \brief namespace for dmlc */ /*! \brief namespace for dmlc */
namespace dmlc { namespace dmlc {
/*! /*!
@ -128,12 +135,25 @@ class InputSplit {
/*! \brief size of the memory region */ /*! \brief size of the memory region */
size_t size; size_t size;
}; };
/*!
* \brief hint the inputsplit how large the chunk size
* it should return when implementing NextChunk
* this is a hint so may not be enforced,
* but InputSplit will try adjust its internal buffer
* size to the hinted value
* \param chunk_size the chunk size
*/
virtual void HintChunkSize(size_t chunk_size) {}
/*! \brief reset the position of InputSplit to beginning */ /*! \brief reset the position of InputSplit to beginning */
virtual void BeforeFirst(void) = 0; virtual void BeforeFirst(void) = 0;
/*! /*!
* \brief get the next record, the returning value * \brief get the next record, the returning value
* is valid until next call to NextRecord or NextChunk * is valid until next call to NextRecord or NextChunk
* caller can modify the memory content of out_rec * caller can modify the memory content of out_rec
*
* For text, out_rec contains a single line
* For recordio, out_rec contains one record content(with header striped)
*
* \param out_rec used to store the result * \param out_rec used to store the result
* \return true if we can successfully get next record * \return true if we can successfully get next record
* false if we reached end of split * false if we reached end of split
@ -144,7 +164,7 @@ class InputSplit {
* \brief get a chunk of memory that can contain multiple records, * \brief get a chunk of memory that can contain multiple records,
* the caller needs to parse the content of the resulting chunk, * the caller needs to parse the content of the resulting chunk,
* for text file, out_chunk can contain data of multiple lines * for text file, out_chunk can contain data of multiple lines
* for recordio, out_chunk can contain data of multiple records * for recordio, out_chunk can contain multiple records(including headers)
* *
* This function ensures there won't be partial record in the chunk * This function ensures there won't be partial record in the chunk
* caller can modify the memory content of out_chunk, * caller can modify the memory content of out_chunk,
@ -157,6 +177,7 @@ class InputSplit {
* \return true if we can successfully get next record * \return true if we can successfully get next record
* false if we reached end of split * false if we reached end of split
* \sa InputSplit::Create for definition of record * \sa InputSplit::Create for definition of record
* \sa RecordIOChunkReader to parse recordio content from out_chunk
*/ */
virtual bool NextChunk(Blob *out_chunk) = 0; virtual bool NextChunk(Blob *out_chunk) = 0;
/*! \brief destructor*/ /*! \brief destructor*/