57b5d7873f
28ca7be add linear readme ca4b20f add linear readme 1133628 add linear readme 6a11676 update docs a607047 Update build.sh 2c1cfd8 complete yarn 4f28e32 change formater 2fbda81 fix stdin input 3258bcf checkin yarn master 67ebf81 allow setup from env variables 9b6bf57 fix hdfs 395d5c2 add make system 88ce767 refactor io, initial hdfs file access need test 19be870 chgs a1bd3c6 Merge branch 'master' of ssh://github.com/tqchen/rabit 1a573f9 introduce input split 29476f1 fix timer issue git-subtree-dir: subtree/rabit git-subtree-split: 28ca7becbdf6503e6b1398588a969efb164c9701
62 lines
1.7 KiB
C++
62 lines
1.7 KiB
C++
#ifndef RABIT_LEARN_IO_IO_H_
|
|
#define RABIT_LEARN_IO_IO_H_
|
|
/*!
|
|
* \file io.h
|
|
* \brief Input/Output utils that handles read/write
|
|
* of files in distrubuted enviroment
|
|
* \author Tianqi Chen
|
|
*/
|
|
#include "../../include/rabit_serializable.h"
|
|
|
|
/*! \brief whether compile with HDFS support */
|
|
#ifndef RABIT_USE_HDFS
|
|
#define RABIT_USE_HDFS 0
|
|
#endif
|
|
|
|
/*! \brief io interface */
|
|
namespace rabit {
|
|
/*!
|
|
* \brief namespace to handle input split and filesystem interfacing
|
|
*/
|
|
namespace io {
|
|
typedef utils::ISeekStream ISeekStream;
|
|
/*!
|
|
* \brief user facing input split helper,
|
|
* can be used to get the partition of data used by current node
|
|
*/
|
|
class InputSplit {
|
|
public:
|
|
/*!
|
|
* \brief get next line, store into out_data
|
|
* \param out_data the string that stores the line data,
|
|
* \n is not included
|
|
* \return true of next line was found, false if we read all the lines
|
|
*/
|
|
virtual bool NextLine(std::string *out_data) = 0;
|
|
/*! \brief destructor*/
|
|
virtual ~InputSplit(void) {}
|
|
};
|
|
/*!
|
|
* \brief create input split given a uri
|
|
* \param uri the uri of the input, can contain hdfs prefix
|
|
* \param part the part id of current input
|
|
* \param nsplit total number of splits
|
|
*/
|
|
inline InputSplit *CreateInputSplit(const char *uri,
|
|
unsigned part,
|
|
unsigned nsplit);
|
|
/*!
|
|
* \brief create an stream, the stream must be able to close
|
|
* the underlying resources(files) when deleted
|
|
*
|
|
* \param uri the uri of the input, can contain hdfs prefix
|
|
* \param mode can be 'w' or 'r' for read or write
|
|
*/
|
|
inline IStream *CreateStream(const char *uri, const char *mode);
|
|
} // namespace io
|
|
} // namespace rabit
|
|
|
|
#include "./io-inl.h"
|
|
#include "./base64-inl.h"
|
|
#endif // RABIT_LEARN_IO_IO_H_
|