75bf97b575
59e63bc minor 6233050 ok 14477f9 add namenode 75a6d34 add libhdfs opts e3c76bf minmum fix 8b3c435 chg 2035799 test code 7751b2b add debug 7690313 ok bd346b4 ok faba1dc add testload 6f7783e add testload e5f0340 ok 3ed9ec8 chg e552ac4 ask for more ram in am b2505e3 only stop nm when sucess bc696c9 add queue info f3e867e add option queue 5dc843c refactor fileio cd9c81b quick fix 1e23af2 add virtual destructor to iseekstream f165ffb fix hdfs 8cc6508 allow demo to pass in env fad4d69 ok 0fd6197 fix more 7423837 fix more d25de54 add temporal solution, run_yarn_prog.py e5a9e31 final attempt ed3bee8 add command back 0774000 add hdfs to resource 9b66e7e fix hadoop 6812f14 ok 08e1c16 change hadoop prefix back to hadoop home d6b6828 Update build.sh 146e069 bugfix: logical boundary for ring buffer 19cb685 ok 4cf3c13 Merge branch 'master' of ssh://github.com/tqchen/rabit 20daddb add tracker c57dad8 add ringbased passing and batch schedule 295d8a1 update 994cb02 add sge 014c866 OK git-subtree-dir: subtree/rabit git-subtree-split: 59e63bc1354c9ff516d72d9a6468f6c431627202
116 lines
3.4 KiB
C++
116 lines
3.4 KiB
C++
/*!
|
|
* Copyright (c) 2014 by Contributors
|
|
* \file io.h
|
|
* \brief utilities with different serializable implementations
|
|
* \author Tianqi Chen
|
|
*/
|
|
#ifndef RABIT_UTILS_IO_H_
|
|
#define RABIT_UTILS_IO_H_
|
|
#include <cstdio>
|
|
#include <vector>
|
|
#include <cstring>
|
|
#include <string>
|
|
#include <algorithm>
|
|
#include "./utils.h"
|
|
#include "../rabit_serializable.h"
|
|
|
|
namespace rabit {
|
|
namespace utils {
|
|
/*! \brief interface of i/o stream that support seek */
|
|
class ISeekStream: public IStream {
|
|
public:
|
|
// virtual destructor
|
|
virtual ~ISeekStream(void) {}
|
|
/*! \brief seek to certain position of the file */
|
|
virtual void Seek(size_t pos) = 0;
|
|
/*! \brief tell the position of the stream */
|
|
virtual size_t Tell(void) = 0;
|
|
/*! \return whether we are at end of file */
|
|
virtual bool AtEnd(void) const = 0;
|
|
};
|
|
|
|
/*! \brief fixed size memory buffer */
|
|
struct MemoryFixSizeBuffer : public ISeekStream {
|
|
public:
|
|
MemoryFixSizeBuffer(void *p_buffer, size_t buffer_size)
|
|
: p_buffer_(reinterpret_cast<char*>(p_buffer)),
|
|
buffer_size_(buffer_size) {
|
|
curr_ptr_ = 0;
|
|
}
|
|
virtual ~MemoryFixSizeBuffer(void) {}
|
|
virtual size_t Read(void *ptr, size_t size) {
|
|
utils::Assert(curr_ptr_ + size <= buffer_size_,
|
|
"read can not have position excceed buffer length");
|
|
size_t nread = std::min(buffer_size_ - curr_ptr_, size);
|
|
if (nread != 0) std::memcpy(ptr, p_buffer_ + curr_ptr_, nread);
|
|
curr_ptr_ += nread;
|
|
return nread;
|
|
}
|
|
virtual void Write(const void *ptr, size_t size) {
|
|
if (size == 0) return;
|
|
utils::Assert(curr_ptr_ + size <= buffer_size_,
|
|
"write position exceed fixed buffer size");
|
|
std::memcpy(p_buffer_ + curr_ptr_, ptr, size);
|
|
curr_ptr_ += size;
|
|
}
|
|
virtual void Seek(size_t pos) {
|
|
curr_ptr_ = static_cast<size_t>(pos);
|
|
}
|
|
virtual size_t Tell(void) {
|
|
return curr_ptr_;
|
|
}
|
|
virtual bool AtEnd(void) const {
|
|
return curr_ptr_ == buffer_size_;
|
|
}
|
|
private:
|
|
/*! \brief in memory buffer */
|
|
char *p_buffer_;
|
|
/*! \brief current pointer */
|
|
size_t buffer_size_;
|
|
/*! \brief current pointer */
|
|
size_t curr_ptr_;
|
|
}; // class MemoryFixSizeBuffer
|
|
|
|
/*! \brief a in memory buffer that can be read and write as stream interface */
|
|
struct MemoryBufferStream : public ISeekStream {
|
|
public:
|
|
explicit MemoryBufferStream(std::string *p_buffer)
|
|
: p_buffer_(p_buffer) {
|
|
curr_ptr_ = 0;
|
|
}
|
|
virtual ~MemoryBufferStream(void) {}
|
|
virtual size_t Read(void *ptr, size_t size) {
|
|
utils::Assert(curr_ptr_ <= p_buffer_->length(),
|
|
"read can not have position excceed buffer length");
|
|
size_t nread = std::min(p_buffer_->length() - curr_ptr_, size);
|
|
if (nread != 0) std::memcpy(ptr, &(*p_buffer_)[0] + curr_ptr_, nread);
|
|
curr_ptr_ += nread;
|
|
return nread;
|
|
}
|
|
virtual void Write(const void *ptr, size_t size) {
|
|
if (size == 0) return;
|
|
if (curr_ptr_ + size > p_buffer_->length()) {
|
|
p_buffer_->resize(curr_ptr_+size);
|
|
}
|
|
std::memcpy(&(*p_buffer_)[0] + curr_ptr_, ptr, size);
|
|
curr_ptr_ += size;
|
|
}
|
|
virtual void Seek(size_t pos) {
|
|
curr_ptr_ = static_cast<size_t>(pos);
|
|
}
|
|
virtual size_t Tell(void) {
|
|
return curr_ptr_;
|
|
}
|
|
virtual bool AtEnd(void) const {
|
|
return curr_ptr_ == p_buffer_->length();
|
|
}
|
|
private:
|
|
/*! \brief in memory buffer */
|
|
std::string *p_buffer_;
|
|
/*! \brief current pointer */
|
|
size_t curr_ptr_;
|
|
}; // class MemoryBufferStream
|
|
} // namespace utils
|
|
} // namespace rabit
|
|
#endif // RABIT_UTILS_IO_H_
|