diff --git a/.gitignore b/.gitignore index c780cc15f..94a8c1059 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,5 @@ _* # Jetbrain .idea cmake-build-debug/ + + diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..8e517101f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "dmlc-core"] + path = dmlc-core + url = https://github.com/dmlc/dmlc-core diff --git a/.travis.yml b/.travis.yml index 933513718..ba01f4236 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,8 @@ env: - TASK=doc - TASK=build - TASK=mpi-build - - TASK=test + - TASK=cmake-build + - TASK=test CXX=g++-4.8 # dependent apt packages dist: xenial @@ -32,7 +33,6 @@ addons: - libopenmpi-dev before_install: - - git clone https://github.com/dmlc/dmlc-core - export TRAVIS=dmlc-core/scripts/travis/ - source ${TRAVIS}/travis_setup_env.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 4a7c557a9..19882c326 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.0) -project(rabit VERSION 0.0.0) +project(rabit VERSION 0.2.0) option(RABIT_BUILD_TESTS "Build rabit tests" OFF) option(RABIT_BUILD_MPI "Build MPI" OFF) @@ -10,9 +10,12 @@ add_library(rabit_base src/allreduce_base.cc src/engine_base.cc src/c_api.cc) add_library(rabit_empty src/engine_empty.cc src/c_api.cc) set(rabit_libs rabit rabit_base rabit_empty) + +set_target_properties(rabit rabit_base rabit_empty PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON) + if(RABIT_BUILD_MPI) find_package(MPI REQUIRED) - add_library(rabit_mpi src/engine_mpi.cc) + add_library(rabit_mpi src/engine_mpi.cc ${MPI_INCLUDE_PATH}) target_link_libraries(rabit_mpi ${MPI_CXX_LIBRARIES}) list(APPEND rabit_libs rabit_mpi) endif() @@ -27,7 +30,7 @@ foreach(lib ${rabit_libs}) #include "./internal/utils.h" target_include_directories(${lib} PUBLIC "$" - "$" + "$" ) endforeach() diff --git a/Makefile b/Makefile index 816fdeb2e..2af5b9b26 100644 --- a/Makefile +++ b/Makefile @@ -95,10 +95,10 @@ lib/librabit_empty.a: $(BPATH)/engine_empty.o $(BPATH)/c_api.o lib/librabit_mpi.a lib/librabit_mpi.so: $(MPIOBJ) $(OBJ) : - $(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c %.cc, $^) ) -I include/ + $(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c %.cc, $^) ) -I include/ -I $(DMLC)/include $(MPIOBJ) : - $(MPICXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c %.cc, $^) ) + $(MPICXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c %.cc, $^) ) -I $(DMLC)/include $(ALIB): ar cr $@ $+ diff --git a/dmlc-core b/dmlc-core new file mode 160000 index 000000000..15362f8fc --- /dev/null +++ b/dmlc-core @@ -0,0 +1 @@ +Subproject commit 15362f8fcc7345d60de13a676a2cbd3ffdc3f064 diff --git a/include/dmlc/README.md b/include/dmlc/README.md deleted file mode 100644 index 846cec006..000000000 --- a/include/dmlc/README.md +++ /dev/null @@ -1,4 +0,0 @@ -This folder is part of dmlc-core library, this allows rabit to use unified stream interface with other dmlc projects. - -- Since it is only interface dependency DMLC core is not required to compile rabit -- To compile project that uses dmlc-core functions, link to libdmlc.a (provided by dmlc-core) will be required. diff --git a/include/dmlc/base.h b/include/dmlc/base.h deleted file mode 100644 index 11a084bdb..000000000 --- a/include/dmlc/base.h +++ /dev/null @@ -1,283 +0,0 @@ -/*! - * Copyright (c) 2015 by Contributors - * \file base.h - * \brief defines configuration macros - */ -#ifndef DMLC_BASE_H_ -#define DMLC_BASE_H_ - -/*! \brief whether use glog for logging */ -#ifndef DMLC_USE_GLOG -#define DMLC_USE_GLOG 0 -#endif // DMLC_USE_GLOG - -/*! - * \brief whether throw dmlc::Error instead of - * directly calling abort when FATAL error occured - * NOTE: this may still not be perfect. - * do not use FATAL and CHECK in destructors - */ -#ifndef DMLC_LOG_FATAL_THROW -#define DMLC_LOG_FATAL_THROW 1 -#endif // DMLC_LOG_FATAL_THROW - -/*! - * \brief whether always log a message before throw - * This can help identify the error that cannot be catched. - */ -#ifndef DMLC_LOG_BEFORE_THROW -#define DMLC_LOG_BEFORE_THROW 1 -#endif // DMLC_LOG_BEFORE_THROW - -/*! - * \brief Whether to use customized logger, - * whose output can be decided by other libraries. - */ -#ifndef DMLC_LOG_CUSTOMIZE -#define DMLC_LOG_CUSTOMIZE 0 -#endif // DMLC_LOG_CUSTOMIZE - -/*! - * \brief Wheter to print stack trace for fatal error, - * enabled on linux when using gcc. - */ -#if (!defined(DMLC_LOG_STACK_TRACE) \ - && defined(__GNUC__) && !defined(__MINGW32__) \ - && !defined(__sun) && !defined(__SVR4)) -#define DMLC_LOG_STACK_TRACE 1 -#endif // guards - -/*! \brief whether compile with hdfs support */ -#ifndef DMLC_USE_HDFS -#define DMLC_USE_HDFS 0 -#endif // DMLC_USE_HDFS - -/*! \brief whether compile with s3 support */ -#ifndef DMLC_USE_S3 -#define DMLC_USE_S3 0 -#endif // DMLC_USE_S3 - -/*! \brief whether or not use parameter server */ -#ifndef DMLC_USE_PS -#define DMLC_USE_PS 0 -#endif // DMLC_USE_PS - -/*! \brief whether or not use c++11 support */ -#ifndef DMLC_USE_CXX11 -#if defined(__GXX_EXPERIMENTAL_CXX0X__) || defined(_MSC_VER) -#define DMLC_USE_CXX11 1 -#else -#define DMLC_USE_CXX11 (__cplusplus >= 201103L) -#endif // defined(__GXX_EXPERIMENTAL_CXX0X__) || defined(_MSC_VER) -#endif // DMLC_USE_CXX11 - -/*! \brief strict CXX11 support */ -#ifndef DMLC_STRICT_CXX11 -#if defined(_MSC_VER) -#define DMLC_STRICT_CXX11 1 -#else -#define DMLC_STRICT_CXX11 (__cplusplus >= 201103L) -#endif // defined(_MSC_VER) -#endif // DMLC_STRICT_CXX11 - -/*! \brief Whether cxx11 thread local is supported */ -#ifndef DMLC_CXX11_THREAD_LOCAL -#if defined(_MSC_VER) -#if (_MSC_VER >= 1900) -#define DMLC_CXX11_THREAD_LOCAL 1 -#else -#define DMLC_CXX11_THREAD_LOCAL 0 -#endif // (_MSC_VER >= 1900) -#else -#define DMLC_CXX11_THREAD_LOCAL (__cplusplus >= 201103L) -#endif // defined(_MSC_VER) -#endif // DMLC_CXX11_THREAD_LOCAL - - -/*! \brief whether RTTI is enabled */ -#ifndef DMLC_ENABLE_RTTI -#define DMLC_ENABLE_RTTI 1 -#endif // DMLC_ENABLE_RTTI - -/// check if g++ is before 4.6 -#if DMLC_USE_CXX11 && defined(__GNUC__) && !defined(__clang_version__) -#if __GNUC__ == 4 && __GNUC_MINOR__ < 6 -#pragma message("Will need g++-4.6 or higher to compile all" \ - "the features in dmlc-core, " \ - "compile without c++0x, some features may be disabled") -#undef DMLC_USE_CXX11 -#define DMLC_USE_CXX11 0 -#endif // __GNUC__ == 4 && __GNUC_MINOR__ < 6 -#endif // DMLC_USE_CXX11 && defined(__GNUC__) && !defined(__clang_version__) - - -/*! - * \brief Enable std::thread related modules, - * Used to disable some module in mingw compile. - */ -#ifndef DMLC_ENABLE_STD_THREAD -#define DMLC_ENABLE_STD_THREAD DMLC_USE_CXX11 -#endif // DMLC_ENABLE_STD_THREAD - -/*! \brief whether enable regex support, actually need g++-4.9 or higher*/ -#ifndef DMLC_USE_REGEX -#define DMLC_USE_REGEX DMLC_STRICT_CXX11 -#endif // DMLC_USE_REGEX - -/*! \brief helper macro to supress unused warning */ -#if defined(__GNUC__) -#define DMLC_ATTRIBUTE_UNUSED __attribute__((unused)) -#else -#define DMLC_ATTRIBUTE_UNUSED -#endif // defined(__GNUC__) - -/*! \brief helper macro to generate string concat */ -#define DMLC_STR_CONCAT_(__x, __y) __x##__y -#define DMLC_STR_CONCAT(__x, __y) DMLC_STR_CONCAT_(__x, __y) - -/*! - * \brief Disable copy constructor and assignment operator. - * - * If C++11 is supported, both copy and move constructors and - * assignment operators are deleted explicitly. Otherwise, they are - * only declared but not implemented. Place this macro in private - * section if C++11 is not available. - */ -#ifndef DISALLOW_COPY_AND_ASSIGN -# if DMLC_USE_CXX11 -# define DISALLOW_COPY_AND_ASSIGN(T) \ - T(T const&) = delete; \ - T(T&&) = delete; \ - T& operator=(T const&) = delete; \ - T& operator=(T&&) = delete -# else -# define DISALLOW_COPY_AND_ASSIGN(T) \ - T(T const&); \ - T& operator=(T const&) -# endif // DMLC_USE_CXX11 -#endif // DISALLOW_COPY_AND_ASSIGN - -/// -/// code block to handle optionally loading -/// -#if !defined(__GNUC__) -#define fopen64 std::fopen -#endif // !defined(__GNUC__) - -#if (defined __MINGW32__) && !(defined __MINGW64__) -#define fopen64 std::fopen -#endif // (defined __MINGW32__) && !(defined __MINGW64__) - -#ifdef _MSC_VER - -# if _MSC_VER < 1900 -// NOTE: sprintf_s is not equivalent to snprintf, -// they are equivalent when success, which is sufficient for our case -# define snprintf sprintf_s -# define vsnprintf vsprintf_s -# endif // _MSC_VER < 1900 - -#else - -# ifdef _FILE_OFFSET_BITS -# if _FILE_OFFSET_BITS == 32 -# pragma message("Warning: FILE OFFSET BITS defined to be 32 bit") -# endif // _FILE_OFFSET_BITS == 32 -# endif // _FILE_OFFSET_BITS - -# ifdef __APPLE__ -# define off64_t off_t -# define fopen64 std::fopen -# endif // __APPLE__ - -extern "C" { -#include -} -#endif // _MSC_VER - -#ifdef _MSC_VER -//! \cond Doxygen_Suppress -typedef signed char int8_t; -typedef __int16 int16_t; -typedef __int32 int32_t; -typedef __int64 int64_t; -typedef unsigned char uint8_t; -typedef unsigned __int16 uint16_t; -typedef unsigned __int32 uint32_t; -typedef unsigned __int64 uint64_t; -//! \endcond -#else -#include -#endif // _MSC_VER - -#include -#include - -#if defined(_MSC_VER) && _MSC_VER < 1900 -#define noexcept_true throw () -#define noexcept_false -#define noexcept(a) noexcept_##a -#endif // defined(_MSC_VER) && _MSC_VER < 1900 - -#if DMLC_USE_CXX11 -#define DMLC_THROW_EXCEPTION noexcept(false) -#define DMLC_NO_EXCEPTION noexcept(true) -#else -#define DMLC_THROW_EXCEPTION -#define DMLC_NO_EXCEPTION -#endif // DMLC_USE_CXX11 - -/*! \brief namespace for dmlc */ -namespace dmlc { -/*! - * \brief safely get the beginning address of a vector - * \param vec input vector - * \return beginning address of a vector - */ -template -inline T *BeginPtr(std::vector &vec) { // NOLINT(*) - if (vec.size() == 0) { - return NULL; - } else { - return &vec[0]; - } -} -/*! - * \brief get the beginning address of a const vector - * \param vec input vector - * \return beginning address of a vector - */ -template -inline const T *BeginPtr(const std::vector &vec) { - if (vec.size() == 0) { - return NULL; - } else { - return &vec[0]; - } -} -/*! - * \brief get the beginning address of a string - * \param str input string - * \return beginning address of a string - */ -inline char* BeginPtr(std::string &str) { // NOLINT(*) - if (str.length() == 0) return NULL; - return &str[0]; -} -/*! - * \brief get the beginning address of a const string - * \param str input string - * \return beginning address of a string - */ -inline const char* BeginPtr(const std::string &str) { - if (str.length() == 0) return NULL; - return &str[0]; -} -} // namespace dmlc - -#if defined(_MSC_VER) && _MSC_VER < 1900 -#define constexpr const -#define alignof __alignof -#endif // defined(_MSC_VER) && _MSC_VER < 1900 - -#endif // DMLC_BASE_H_ diff --git a/include/dmlc/io.h b/include/dmlc/io.h deleted file mode 100644 index 1965a7ad2..000000000 --- a/include/dmlc/io.h +++ /dev/null @@ -1,424 +0,0 @@ -/*! - * Copyright (c) 2015 by Contributors - * \file io.h - * \brief defines serializable interface of dmlc - */ -#ifndef DMLC_IO_H_ -#define DMLC_IO_H_ -#include -#include -#include -#include -#include -#include - -#include "base.h" - -// include uint64_t only to make io standalone -#ifdef _MSC_VER -/*! \brief uint64 */ -typedef unsigned __int64 uint64_t; -#else -#include -#endif - -/*! \brief namespace for dmlc */ -namespace dmlc { -/*! - * \brief interface of stream I/O for serialization - */ -class Stream { // NOLINT(*) - public: - /*! - * \brief reads data from a stream - * \param ptr pointer to a memory buffer - * \param size block size - * \return the size of data read - */ - virtual size_t Read(void *ptr, size_t size) = 0; - /*! - * \brief writes data to a stream - * \param ptr pointer to a memory buffer - * \param size block size - */ - virtual void Write(const void *ptr, size_t size) = 0; - /*! \brief virtual destructor */ - virtual ~Stream(void) {} - /*! - * \brief generic factory function - * create an stream, the stream will close the underlying files upon deletion - * - * \param uri the uri of the input currently we support - * hdfs://, s3://, and file:// by default file:// will be used - * \param flag can be "w", "r", "a" - * \param allow_null whether NULL can be returned, or directly report error - * \return the created stream, can be NULL when allow_null == true and file do not exist - */ - static Stream *Create(const char *uri, - const char* const flag, - bool allow_null = false); - // helper functions to write/read different data structures - /*! - * \brief writes a data to stream - * - * dmlc::Stream support Write/Read of most STL - * composites and base types. - * If the data type is not supported, a compile time error will - * be issued. - * - * \param data data to be written - * \tparam T the data type to be written - */ - template - inline void Write(const T &data); - /*! - * \brief loads a data from stream. - * - * dmlc::Stream support Write/Read of most STL - * composites and base types. - * If the data type is not supported, a compile time error will - * be issued. - * - * \param out_data place holder of data to be deserialized - * \return whether the load was successful - */ - template - inline bool Read(T *out_data); -}; - -/*! \brief interface of i/o stream that support seek */ -class SeekStream: public Stream { - public: - // virtual destructor - virtual ~SeekStream(void) {} - /*! \brief seek to certain position of the file */ - virtual void Seek(size_t pos) = 0; - /*! \brief tell the position of the stream */ - virtual size_t Tell(void) = 0; - /*! - * \brief generic factory function - * create an SeekStream for read only, - * the stream will close the underlying files upon deletion - * error will be reported and the system will exit when create failed - * \param uri the uri of the input currently we support - * hdfs://, s3://, and file:// by default file:// will be used - * \param allow_null whether NULL can be returned, or directly report error - * \return the created stream, can be NULL when allow_null == true and file do not exist - */ - static SeekStream *CreateForRead(const char *uri, - bool allow_null = false); -}; - -/*! \brief interface for serializable objects */ -class Serializable { - public: - /*! \brief virtual destructor */ - virtual ~Serializable() {} - /*! - * \brief load the model from a stream - * \param fi stream where to load the model from - */ - virtual void Load(Stream *fi) = 0; - /*! - * \brief saves the model to a stream - * \param fo stream where to save the model to - */ - virtual void Save(Stream *fo) const = 0; -}; - -/*! - * \brief input split creates that allows reading - * of records from split of data, - * independent part that covers all the dataset - * - * see InputSplit::Create for definition of record - */ -class InputSplit { - public: - /*! \brief a blob of memory region */ - struct Blob { - /*! \brief points to start of the memory region */ - void *dptr; - /*! \brief size of the memory region */ - size_t size; - }; - /*! - * \brief hint the inputsplit how large the chunk size - * it should return when implementing NextChunk - * this is a hint so may not be enforced, - * but InputSplit will try adjust its internal buffer - * size to the hinted value - * \param chunk_size the chunk size - */ - virtual void HintChunkSize(size_t chunk_size) {} - /*! \brief get the total size of the InputSplit */ - virtual size_t GetTotalSize(void) = 0; - /*! \brief reset the position of InputSplit to beginning */ - virtual void BeforeFirst(void) = 0; - /*! - * \brief get the next record, the returning value - * is valid until next call to NextRecord or NextChunk - * caller can modify the memory content of out_rec - * - * For text, out_rec contains a single line - * For recordio, out_rec contains one record content(with header striped) - * - * \param out_rec used to store the result - * \return true if we can successfully get next record - * false if we reached end of split - * \sa InputSplit::Create for definition of record - */ - virtual bool NextRecord(Blob *out_rec) = 0; - /*! - * \brief get a chunk of memory that can contain multiple records, - * the caller needs to parse the content of the resulting chunk, - * for text file, out_chunk can contain data of multiple lines - * for recordio, out_chunk can contain multiple records(including headers) - * - * This function ensures there won't be partial record in the chunk - * caller can modify the memory content of out_chunk, - * the memory is valid until next call to NextRecord or NextChunk - * - * Usually NextRecord is sufficient, NextChunk can be used by some - * multi-threaded parsers to parse the input content - * - * \param out_chunk used to store the result - * \return true if we can successfully get next record - * false if we reached end of split - * \sa InputSplit::Create for definition of record - * \sa RecordIOChunkReader to parse recordio content from out_chunk - */ - virtual bool NextChunk(Blob *out_chunk) = 0; - /*! \brief destructor*/ - virtual ~InputSplit(void) {} - /*! - * \brief reset the Input split to a certain part id, - * The InputSplit will be pointed to the head of the new specified segment. - * This feature may not be supported by every implementation of InputSplit. - * \param part_index The part id of the new input. - * \param num_parts The total number of parts. - */ - virtual void ResetPartition(unsigned part_index, unsigned num_parts) = 0; - /*! - * \brief factory function: - * create input split given a uri - * \param uri the uri of the input, can contain hdfs prefix - * \param part_index the part id of current input - * \param num_parts total number of splits - * \param type type of record - * List of possible types: "text", "recordio" - * - "text": - * text file, each line is treated as a record - * input split will split on '\\n' or '\\r' - * - "recordio": - * binary recordio file, see recordio.h - * \return a new input split - * \sa InputSplit::Type - */ - static InputSplit* Create(const char *uri, - unsigned part_index, - unsigned num_parts, - const char *type); -}; - -/*! - * \brief a std::ostream class that can can wrap Stream objects, - * can use ostream with that output to underlying Stream - * - * Usage example: - * \code - * - * Stream *fs = Stream::Create("hdfs:///test.txt", "w"); - * dmlc::ostream os(fs); - * os << "hello world" << std::endl; - * delete fs; - * \endcode - */ -class ostream : public std::basic_ostream { - public: - /*! - * \brief construct std::ostream type - * \param stream the Stream output to be used - * \param buffer_size internal streambuf size - */ - explicit ostream(Stream *stream, - size_t buffer_size = (1 << 10)) - : std::basic_ostream(NULL), buf_(buffer_size) { - this->set_stream(stream); - } - // explictly synchronize the buffer - virtual ~ostream() DMLC_NO_EXCEPTION { - buf_.pubsync(); - } - /*! - * \brief set internal stream to be stream, reset states - * \param stream new stream as output - */ - inline void set_stream(Stream *stream) { - buf_.set_stream(stream); - this->rdbuf(&buf_); - } - - /*! \return how many bytes we written so far */ - inline size_t bytes_written(void) const { - return buf_.bytes_out(); - } - - private: - // internal streambuf - class OutBuf : public std::streambuf { - public: - explicit OutBuf(size_t buffer_size) - : stream_(NULL), buffer_(buffer_size), bytes_out_(0) { - if (buffer_size == 0) buffer_.resize(2); - } - // set stream to the buffer - inline void set_stream(Stream *stream); - - inline size_t bytes_out() const { return bytes_out_; } - private: - /*! \brief internal stream by StreamBuf */ - Stream *stream_; - /*! \brief internal buffer */ - std::vector buffer_; - /*! \brief number of bytes written so far */ - size_t bytes_out_; - // override sync - inline int_type sync(void); - // override overflow - inline int_type overflow(int c); - }; - /*! \brief buffer of the stream */ - OutBuf buf_; -}; - -/*! - * \brief a std::istream class that can can wrap Stream objects, - * can use istream with that output to underlying Stream - * - * Usage example: - * \code - * - * Stream *fs = Stream::Create("hdfs:///test.txt", "r"); - * dmlc::istream is(fs); - * is >> mydata; - * delete fs; - * \endcode - */ -class istream : public std::basic_istream { - public: - /*! - * \brief construct std::ostream type - * \param stream the Stream output to be used - * \param buffer_size internal buffer size - */ - explicit istream(Stream *stream, - size_t buffer_size = (1 << 10)) - : std::basic_istream(NULL), buf_(buffer_size) { - this->set_stream(stream); - } - virtual ~istream() DMLC_NO_EXCEPTION {} - /*! - * \brief set internal stream to be stream, reset states - * \param stream new stream as output - */ - inline void set_stream(Stream *stream) { - buf_.set_stream(stream); - this->rdbuf(&buf_); - } - /*! \return how many bytes we read so far */ - inline size_t bytes_read(void) const { - return buf_.bytes_read(); - } - - private: - // internal streambuf - class InBuf : public std::streambuf { - public: - explicit InBuf(size_t buffer_size) - : stream_(NULL), bytes_read_(0), - buffer_(buffer_size) { - if (buffer_size == 0) buffer_.resize(2); - } - // set stream to the buffer - inline void set_stream(Stream *stream); - // return how many bytes read so far - inline size_t bytes_read(void) const { - return bytes_read_; - } - private: - /*! \brief internal stream by StreamBuf */ - Stream *stream_; - /*! \brief how many bytes we read so far */ - size_t bytes_read_; - /*! \brief internal buffer */ - std::vector buffer_; - // override underflow - inline int_type underflow(); - }; - /*! \brief input buffer */ - InBuf buf_; -}; -} // namespace dmlc - -#include "./serializer.h" - -namespace dmlc { -// implementations of inline functions -template -inline void Stream::Write(const T &data) { - serializer::Handler::Write(this, data); -} -template -inline bool Stream::Read(T *out_data) { - return serializer::Handler::Read(this, out_data); -} - -// implementations for ostream -inline void ostream::OutBuf::set_stream(Stream *stream) { - if (stream_ != NULL) this->pubsync(); - this->stream_ = stream; - this->setp(&buffer_[0], &buffer_[0] + buffer_.size() - 1); -} -inline int ostream::OutBuf::sync(void) { - if (stream_ == NULL) return -1; - std::ptrdiff_t n = pptr() - pbase(); - stream_->Write(pbase(), n); - this->pbump(-static_cast(n)); - bytes_out_ += n; - return 0; -} -inline int ostream::OutBuf::overflow(int c) { - *(this->pptr()) = c; - std::ptrdiff_t n = pptr() - pbase(); - this->pbump(-static_cast(n)); - if (c == EOF) { - stream_->Write(pbase(), n); - bytes_out_ += n; - } else { - stream_->Write(pbase(), n + 1); - bytes_out_ += n + 1; - } - return c; -} - -// implementations for istream -inline void istream::InBuf::set_stream(Stream *stream) { - stream_ = stream; - this->setg(&buffer_[0], &buffer_[0], &buffer_[0]); -} -inline int istream::InBuf::underflow() { - char *bhead = &buffer_[0]; - if (this->gptr() == this->egptr()) { - size_t sz = stream_->Read(bhead, buffer_.size()); - this->setg(bhead, bhead, bhead + sz); - bytes_read_ += sz; - } - if (this->gptr() == this->egptr()) { - return traits_type::eof(); - } else { - return traits_type::to_int_type(*gptr()); - } -} -} // namespace dmlc -#endif // DMLC_IO_H_ diff --git a/include/dmlc/serializer.h b/include/dmlc/serializer.h deleted file mode 100644 index af50c248c..000000000 --- a/include/dmlc/serializer.h +++ /dev/null @@ -1,382 +0,0 @@ -/*! - * Copyright (c) 2015 by Contributors - * \file serializer.h - * \brief serializer template class that helps serialization. - * This file do not need to be directly used by most user. - */ -#ifndef DMLC_SERIALIZER_H_ -#define DMLC_SERIALIZER_H_ - -#include -#include -#include -#include -#include -#include -#include - -#include "./base.h" -#include "./io.h" -#include "./type_traits.h" - -#if DMLC_USE_CXX11 -#include -#include -#endif - -namespace dmlc { -/*! \brief internal namespace for serializers */ -namespace serializer { -/*! - * \brief generic serialization handler - * \tparam T the type to be serialized - */ -template -struct Handler; - -//! \cond Doxygen_Suppress -/*! - * \brief Serializer that redirect calls by condition - * \tparam cond the condition - * \tparam Then the serializer used for then condition - * \tparam Else the serializer used for else condition - * \tparam Return the type of data the serializer handles - */ -template -struct IfThenElse; - -template -struct IfThenElse { - inline static void Write(Stream *strm, const T &data) { - Then::Write(strm, data); - } - inline static bool Read(Stream *strm, T *data) { - return Then::Read(strm, data); - } -}; -template -struct IfThenElse { - inline static void Write(Stream *strm, const T &data) { - Else::Write(strm, data); - } - inline static bool Read(Stream *strm, T *data) { - return Else::Read(strm, data); - } -}; - -/*! \brief Serializer for POD(plain-old-data) data */ -template -struct PODHandler { - inline static void Write(Stream *strm, const T &data) { - strm->Write(&data, sizeof(T)); - } - inline static bool Read(Stream *strm, T *dptr) { - return strm->Read((void*)dptr, sizeof(T)) == sizeof(T); // NOLINT(*) - } -}; - -// serializer for class that have save/load function -template -struct SaveLoadClassHandler { - inline static void Write(Stream *strm, const T &data) { - data.Save(strm); - } - inline static bool Read(Stream *strm, T *data) { - return data->Load(strm); - } -}; - -/*! - * \brief dummy class for undefined serialization. - * This is used to generate error message when user tries to - * serialize something that is not supported. - * \tparam T the type to be serialized - */ -template -struct UndefinedSerializerFor { -}; - -/*! - * \brief Serializer handler for std::vector where T is POD type. - * \tparam T element type - */ -template -struct PODVectorHandler { - inline static void Write(Stream *strm, const std::vector &vec) { - uint64_t sz = static_cast(vec.size()); - strm->Write(&sz, sizeof(sz)); - if (sz != 0) { - strm->Write(&vec[0], sizeof(T) * vec.size()); - } - } - inline static bool Read(Stream *strm, std::vector *out_vec) { - uint64_t sz; - if (strm->Read(&sz, sizeof(sz)) != sizeof(sz)) return false; - size_t size = static_cast(sz); - out_vec->resize(size); - if (sz != 0) { - size_t nbytes = sizeof(T) * size; - return strm->Read(&(*out_vec)[0], nbytes) == nbytes; - } - return true; - } -}; - -/*! - * \brief Serializer handler for std::vector where T can be composed type - * \tparam T element type - */ -template -struct ComposeVectorHandler { - inline static void Write(Stream *strm, const std::vector &vec) { - uint64_t sz = static_cast(vec.size()); - strm->Write(&sz, sizeof(sz)); - for (size_t i = 0; i < vec.size(); ++i) { - Handler::Write(strm, vec[i]); - } - } - inline static bool Read(Stream *strm, std::vector *out_vec) { - uint64_t sz; - if (strm->Read(&sz, sizeof(sz)) != sizeof(sz)) return false; - size_t size = static_cast(sz); - out_vec->resize(size); - for (size_t i = 0; i < size; ++i) { - if (!Handler::Read(strm, &(*out_vec)[i])) return false; - } - return true; - } -}; - -/*! - * \brief Serializer handler for std::basic_string where T is POD type. - * \tparam T element type - */ -template -struct PODStringHandler { - inline static void Write(Stream *strm, const std::basic_string &vec) { - uint64_t sz = static_cast(vec.length()); - strm->Write(&sz, sizeof(sz)); - if (sz != 0) { - strm->Write(&vec[0], sizeof(T) * vec.length()); - } - } - inline static bool Read(Stream *strm, std::basic_string *out_vec) { - uint64_t sz; - if (strm->Read(&sz, sizeof(sz)) != sizeof(sz)) return false; - size_t size = static_cast(sz); - out_vec->resize(size); - if (sz != 0) { - size_t nbytes = sizeof(T) * size; - return strm->Read(&(*out_vec)[0], nbytes) == nbytes; - } - return true; - } -}; - -/*! \brief Serializer for std::pair */ -template -struct PairHandler { - inline static void Write(Stream *strm, const std::pair &data) { - Handler::Write(strm, data.first); - Handler::Write(strm, data.second); - } - inline static bool Read(Stream *strm, std::pair *data) { - return Handler::Read(strm, &(data->first)) && - Handler::Read(strm, &(data->second)); - } -}; - -// set type handler that can handle most collection type case -template -struct CollectionHandler { - inline static void Write(Stream *strm, const ContainerType &data) { - typedef typename ContainerType::value_type ElemType; - // dump data to vector - std::vector vdata(data.begin(), data.end()); - // serialize the vector - Handler >::Write(strm, vdata); - } - inline static bool Read(Stream *strm, ContainerType *data) { - typedef typename ContainerType::value_type ElemType; - std::vector vdata; - if (!Handler >::Read(strm, &vdata)) return false; - data->clear(); - data->insert(vdata.begin(), vdata.end()); - return true; - } -}; - - -// handler that can handle most list type case -// this type insert function takes additional iterator -template -struct ListHandler { - inline static void Write(Stream *strm, const ListType &data) { - typedef typename ListType::value_type ElemType; - // dump data to vector - std::vector vdata(data.begin(), data.end()); - // serialize the vector - Handler >::Write(strm, vdata); - } - inline static bool Read(Stream *strm, ListType *data) { - typedef typename ListType::value_type ElemType; - std::vector vdata; - if (!Handler >::Read(strm, &vdata)) return false; - data->clear(); - data->insert(data->begin(), vdata.begin(), vdata.end()); - return true; - } -}; - -//! \endcond - -/*! - * \brief generic serialization handler for type T - * - * User can define specialization of this class to support - * composite serialization of their own class. - * - * \tparam T the type to be serialized - */ -template -struct Handler { - /*! - * \brief write data to stream - * \param strm the stream we write the data. - * \param data the data obeject to be serialized - */ - inline static void Write(Stream *strm, const T &data) { - IfThenElse::value, - PODHandler, - IfThenElse::value, - SaveLoadClassHandler, - UndefinedSerializerFor, T>, - T> - ::Write(strm, data); - } - /*! - * \brief read data to stream - * \param strm the stream to read the data. - * \param data the pointer to the data obeject to read - * \return whether the read is successful - */ - inline static bool Read(Stream *strm, T *data) { - return IfThenElse::value, - PODHandler, - IfThenElse::value, - SaveLoadClassHandler, - UndefinedSerializerFor, T>, - T> - ::Read(strm, data); - } -}; - -//! \cond Doxygen_Suppress -template -struct Handler > { - inline static void Write(Stream *strm, const std::vector &data) { - IfThenElse::value, - PODVectorHandler, - ComposeVectorHandler, std::vector > - ::Write(strm, data); - } - inline static bool Read(Stream *strm, std::vector *data) { - return IfThenElse::value, - PODVectorHandler, - ComposeVectorHandler, - std::vector > - ::Read(strm, data); - } -}; - -template -struct Handler > { - inline static void Write(Stream *strm, const std::basic_string &data) { - IfThenElse::value, - PODStringHandler, - UndefinedSerializerFor, - std::basic_string > - ::Write(strm, data); - } - inline static bool Read(Stream *strm, std::basic_string *data) { - return IfThenElse::value, - PODStringHandler, - UndefinedSerializerFor, - std::basic_string > - ::Read(strm, data); - } -}; - -template -struct Handler > { - inline static void Write(Stream *strm, const std::pair &data) { - IfThenElse::value && dmlc::is_pod::value, - PODHandler >, - PairHandler, - std::pair > - ::Write(strm, data); - } - inline static bool Read(Stream *strm, std::pair *data) { - return IfThenElse::value && dmlc::is_pod::value, - PODHandler >, - PairHandler, - std::pair > - ::Read(strm, data); - } -}; - -template -struct Handler > - : public CollectionHandler > { -}; - -template -struct Handler > - : public CollectionHandler > { -}; - -template -struct Handler > - : public CollectionHandler > { -}; - -template -struct Handler > - : public CollectionHandler > { -}; - -template -struct Handler > - : public ListHandler > { -}; - -template -struct Handler > - : public ListHandler > { -}; - -#if DMLC_USE_CXX11 -template -struct Handler > - : public CollectionHandler > { -}; - -template -struct Handler > - : public CollectionHandler > { -}; - -template -struct Handler > - : public CollectionHandler > { -}; - -template -struct Handler > - : public CollectionHandler > { -}; -#endif -//! \endcond -} // namespace serializer -} // namespace dmlc -#endif // DMLC_SERIALIZER_H_ diff --git a/include/dmlc/type_traits.h b/include/dmlc/type_traits.h deleted file mode 100644 index ef0af06ab..000000000 --- a/include/dmlc/type_traits.h +++ /dev/null @@ -1,171 +0,0 @@ -/*! - * Copyright (c) 2015 by Contributors - * \file type_traits.h - * \brief type traits information header - */ -#ifndef DMLC_TYPE_TRAITS_H_ -#define DMLC_TYPE_TRAITS_H_ - -#include "./base.h" -#if DMLC_USE_CXX11 -#include -#endif // DMLC_USE_CXX11 -#include - -namespace dmlc { -/*! - * \brief whether a type is pod type - * \tparam T the type to query - */ -template -struct is_pod { -#if DMLC_USE_CXX11 - /*! \brief the value of the traits */ - static const bool value = std::is_pod::value; -#else - /*! \brief the value of the traits */ - static const bool value = false; -#endif // DMLC_USE_CXX11 -}; - - -/*! - * \brief whether a type is integer type - * \tparam T the type to query - */ -template -struct is_integral { -#if DMLC_USE_CXX11 - /*! \brief the value of the traits */ - static const bool value = std::is_integral::value; -#else - /*! \brief the value of the traits */ - static const bool value = false; -#endif // DMLC_USE_CXX11 -}; - -/*! - * \brief whether a type is floating point type - * \tparam T the type to query - */ -template -struct is_floating_point { -#if DMLC_USE_CXX11 - /*! \brief the value of the traits */ - static const bool value = std::is_floating_point::value; -#else - /*! \brief the value of the traits */ - static const bool value = false; -#endif // DMLC_USE_CXX11 -}; - -/*! - * \brief whether a type is arithemetic type - * \tparam T the type to query - */ -template -struct is_arithmetic { -#if DMLC_USE_CXX11 - /*! \brief the value of the traits */ - static const bool value = std::is_arithmetic::value; -#else - /*! \brief the value of the traits */ - static const bool value = (dmlc::is_integral::value || - dmlc::is_floating_point::value); -#endif // DMLC_USE_CXX11 -}; - -/*! - * \brief the string representation of type name - * \tparam T the type to query - * \return a const string of typename. - */ -template -inline const char* type_name() { - return ""; -} - -/*! - * \brief whether a type have save/load function - * \tparam T the type to query - */ -template -struct has_saveload { - /*! \brief the value of the traits */ - static const bool value = false; -}; - -/*! - * \brief template to select type based on condition - * For example, IfThenElseType::Type will give int - * \tparam cond the condition - * \tparam Then the typename to be returned if cond is true - * \tparam The typename to be returned if cond is false -*/ -template -struct IfThenElseType; - -/*! \brief macro to quickly declare traits information */ -#define DMLC_DECLARE_TRAITS(Trait, Type, Value) \ - template<> \ - struct Trait { \ - static const bool value = Value; \ - } - -/*! \brief macro to quickly declare traits information */ -#define DMLC_DECLARE_TYPE_NAME(Type, Name) \ - template<> \ - inline const char* type_name() { \ - return Name; \ - } - -//! \cond Doxygen_Suppress -// declare special traits when C++11 is not available -#if DMLC_USE_CXX11 == 0 -DMLC_DECLARE_TRAITS(is_pod, char, true); -DMLC_DECLARE_TRAITS(is_pod, int8_t, true); -DMLC_DECLARE_TRAITS(is_pod, int16_t, true); -DMLC_DECLARE_TRAITS(is_pod, int32_t, true); -DMLC_DECLARE_TRAITS(is_pod, int64_t, true); -DMLC_DECLARE_TRAITS(is_pod, uint8_t, true); -DMLC_DECLARE_TRAITS(is_pod, uint16_t, true); -DMLC_DECLARE_TRAITS(is_pod, uint32_t, true); -DMLC_DECLARE_TRAITS(is_pod, uint64_t, true); -DMLC_DECLARE_TRAITS(is_pod, float, true); -DMLC_DECLARE_TRAITS(is_pod, double, true); - -DMLC_DECLARE_TRAITS(is_integral, char, true); -DMLC_DECLARE_TRAITS(is_integral, int8_t, true); -DMLC_DECLARE_TRAITS(is_integral, int16_t, true); -DMLC_DECLARE_TRAITS(is_integral, int32_t, true); -DMLC_DECLARE_TRAITS(is_integral, int64_t, true); -DMLC_DECLARE_TRAITS(is_integral, uint8_t, true); -DMLC_DECLARE_TRAITS(is_integral, uint16_t, true); -DMLC_DECLARE_TRAITS(is_integral, uint32_t, true); -DMLC_DECLARE_TRAITS(is_integral, uint64_t, true); - -DMLC_DECLARE_TRAITS(is_floating_point, float, true); -DMLC_DECLARE_TRAITS(is_floating_point, double, true); - -#endif // DMLC_USE_CXX11 - -DMLC_DECLARE_TYPE_NAME(float, "float"); -DMLC_DECLARE_TYPE_NAME(double, "double"); -DMLC_DECLARE_TYPE_NAME(int, "int"); -DMLC_DECLARE_TYPE_NAME(uint32_t, "int (non-negative)"); -DMLC_DECLARE_TYPE_NAME(uint64_t, "long (non-negative)"); -DMLC_DECLARE_TYPE_NAME(std::string, "string"); -DMLC_DECLARE_TYPE_NAME(bool, "boolean"); - -template -struct IfThenElseType { - typedef Then Type; -}; - -template -struct IfThenElseType { - typedef Else Type; -}; -//! \endcond -} // namespace dmlc -#endif // DMLC_TYPE_TRAITS_H_ diff --git a/include/rabit/serializable.h b/include/rabit/serializable.h index 4a3c2a115..514b7a70f 100644 --- a/include/rabit/serializable.h +++ b/include/rabit/serializable.h @@ -9,7 +9,7 @@ #include #include #include "./internal/utils.h" -#include "../dmlc/io.h" +#include "../../dmlc-core/include/dmlc/io.h" namespace rabit { /*! diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh index 1321b19f1..12e78f65c 100755 --- a/scripts/travis_script.sh +++ b/scripts/travis_script.sh @@ -25,3 +25,9 @@ if [ ${TASK} == "test" ]; then ../scripts/travis_runtest.sh || exit -1 fi +if [ ${TASK} == "cmake-build" ]; then + mkdir build + cd build + cmake .. + make all || exit -1 +fi \ No newline at end of file diff --git a/src/allreduce_robust.cc b/src/allreduce_robust.cc index a9a89af07..43871e010 100644 --- a/src/allreduce_robust.cc +++ b/src/allreduce_robust.cc @@ -50,12 +50,13 @@ void AllreduceRobust::Shutdown(void) { // execute check ack step, load happens here utils::Assert(RecoverExec(NULL, 0, ActionSummary::kCheckAck, ActionSummary::kSpecialOp), "Shutdown: check ack must return true"); -#ifdef __APPLE__ - // In OSX, one worker shutdowns and closes sockets while rest still run kCheckAck - // This cause rest workers checkandrecover and hang inf, https://github.com/dmlc/xgboost/pull/3818 + + // one worker shutdowns and closes sockets while rest still run kCheckAck, + // seems has something to do with time-wait state in tcp connection, + // this cause rest workers checkandrecover and hang inf, + // https://github.com/dmlc/xgboost/pull/3818 // TODO(Chen Qin): a fundamental fix for this - sleep(2); -#endif // __APPLE__ + sleep(1); AllreduceBase::Shutdown(); } /*! diff --git a/test/Makefile b/test/Makefile index 28cf29bdf..5a6ba2aa8 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,6 +1,6 @@ export MPICXX = mpicxx export LDFLAGS= -L../lib -pthread -lm -export CFLAGS = -Wall -O3 -msse2 -Wno-unknown-pragmas -fPIC -I../include -std=c++11 +export CFLAGS = -Wall -O3 -msse2 -Wno-unknown-pragmas -fPIC -I../include -I ../dmlc-core/include -std=c++11 OS := $(shell uname)