[rabit harden] replace hardcopy dmlc-core headers with submodule links (#86)
* backport dmlc header changes to rabit * use gitmodule to reference latest dmlc header files * include ref to dmlc-core fix cmake * update cmake file, add cmake build traivs task * try force using g++-4.8 * per feedback, update cmake
This commit is contained in:
Chen Qin
Nan Zhu
parent
785d7e54d3
commit
ecd4bf7aae
2
.gitignore
vendored
2
.gitignore
vendored
@ -40,3 +40,5 @@ _*
|
||||
# Jetbrain
|
||||
.idea
|
||||
cmake-build-debug/
|
||||
|
||||
|
||||
|
||||
3
.gitmodules
vendored
Normal file
3
.gitmodules
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
[submodule "dmlc-core"]
|
||||
path = dmlc-core
|
||||
url = https://github.com/dmlc/dmlc-core
|
||||
@ -9,7 +9,8 @@ env:
|
||||
- TASK=doc
|
||||
- TASK=build
|
||||
- TASK=mpi-build
|
||||
- TASK=test
|
||||
- TASK=cmake-build
|
||||
- TASK=test CXX=g++-4.8
|
||||
|
||||
# dependent apt packages
|
||||
dist: xenial
|
||||
@ -32,7 +33,6 @@ addons:
|
||||
- libopenmpi-dev
|
||||
|
||||
before_install:
|
||||
- git clone https://github.com/dmlc/dmlc-core
|
||||
- export TRAVIS=dmlc-core/scripts/travis/
|
||||
- source ${TRAVIS}/travis_setup_env.sh
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
cmake_minimum_required(VERSION 3.0)
|
||||
|
||||
project(rabit VERSION 0.0.0)
|
||||
project(rabit VERSION 0.2.0)
|
||||
|
||||
option(RABIT_BUILD_TESTS "Build rabit tests" OFF)
|
||||
option(RABIT_BUILD_MPI "Build MPI" OFF)
|
||||
@ -10,9 +10,12 @@ add_library(rabit_base src/allreduce_base.cc src/engine_base.cc src/c_api.cc)
|
||||
add_library(rabit_empty src/engine_empty.cc src/c_api.cc)
|
||||
|
||||
set(rabit_libs rabit rabit_base rabit_empty)
|
||||
|
||||
set_target_properties(rabit rabit_base rabit_empty PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON)
|
||||
|
||||
if(RABIT_BUILD_MPI)
|
||||
find_package(MPI REQUIRED)
|
||||
add_library(rabit_mpi src/engine_mpi.cc)
|
||||
add_library(rabit_mpi src/engine_mpi.cc ${MPI_INCLUDE_PATH})
|
||||
target_link_libraries(rabit_mpi ${MPI_CXX_LIBRARIES})
|
||||
list(APPEND rabit_libs rabit_mpi)
|
||||
endif()
|
||||
@ -27,7 +30,7 @@ foreach(lib ${rabit_libs})
|
||||
#include "./internal/utils.h"
|
||||
target_include_directories(${lib} PUBLIC
|
||||
"$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include>"
|
||||
"$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include/rabit>"
|
||||
"$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/dmlc-core/include>"
|
||||
)
|
||||
endforeach()
|
||||
|
||||
|
||||
4
Makefile
4
Makefile
@ -95,10 +95,10 @@ lib/librabit_empty.a: $(BPATH)/engine_empty.o $(BPATH)/c_api.o
|
||||
lib/librabit_mpi.a lib/librabit_mpi.so: $(MPIOBJ)
|
||||
|
||||
$(OBJ) :
|
||||
$(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c %.cc, $^) ) -I include/
|
||||
$(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c %.cc, $^) ) -I include/ -I $(DMLC)/include
|
||||
|
||||
$(MPIOBJ) :
|
||||
$(MPICXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c %.cc, $^) )
|
||||
$(MPICXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c %.cc, $^) ) -I $(DMLC)/include
|
||||
|
||||
$(ALIB):
|
||||
ar cr $@ $+
|
||||
|
||||
1
dmlc-core
Submodule
1
dmlc-core
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit 15362f8fcc7345d60de13a676a2cbd3ffdc3f064
|
||||
@ -1,4 +0,0 @@
|
||||
This folder is part of dmlc-core library, this allows rabit to use unified stream interface with other dmlc projects.
|
||||
|
||||
- Since it is only interface dependency DMLC core is not required to compile rabit
|
||||
- To compile project that uses dmlc-core functions, link to libdmlc.a (provided by dmlc-core) will be required.
|
||||
@ -1,283 +0,0 @@
|
||||
/*!
|
||||
* Copyright (c) 2015 by Contributors
|
||||
* \file base.h
|
||||
* \brief defines configuration macros
|
||||
*/
|
||||
#ifndef DMLC_BASE_H_
|
||||
#define DMLC_BASE_H_
|
||||
|
||||
/*! \brief whether use glog for logging */
|
||||
#ifndef DMLC_USE_GLOG
|
||||
#define DMLC_USE_GLOG 0
|
||||
#endif // DMLC_USE_GLOG
|
||||
|
||||
/*!
|
||||
* \brief whether throw dmlc::Error instead of
|
||||
* directly calling abort when FATAL error occured
|
||||
* NOTE: this may still not be perfect.
|
||||
* do not use FATAL and CHECK in destructors
|
||||
*/
|
||||
#ifndef DMLC_LOG_FATAL_THROW
|
||||
#define DMLC_LOG_FATAL_THROW 1
|
||||
#endif // DMLC_LOG_FATAL_THROW
|
||||
|
||||
/*!
|
||||
* \brief whether always log a message before throw
|
||||
* This can help identify the error that cannot be catched.
|
||||
*/
|
||||
#ifndef DMLC_LOG_BEFORE_THROW
|
||||
#define DMLC_LOG_BEFORE_THROW 1
|
||||
#endif // DMLC_LOG_BEFORE_THROW
|
||||
|
||||
/*!
|
||||
* \brief Whether to use customized logger,
|
||||
* whose output can be decided by other libraries.
|
||||
*/
|
||||
#ifndef DMLC_LOG_CUSTOMIZE
|
||||
#define DMLC_LOG_CUSTOMIZE 0
|
||||
#endif // DMLC_LOG_CUSTOMIZE
|
||||
|
||||
/*!
|
||||
* \brief Wheter to print stack trace for fatal error,
|
||||
* enabled on linux when using gcc.
|
||||
*/
|
||||
#if (!defined(DMLC_LOG_STACK_TRACE) \
|
||||
&& defined(__GNUC__) && !defined(__MINGW32__) \
|
||||
&& !defined(__sun) && !defined(__SVR4))
|
||||
#define DMLC_LOG_STACK_TRACE 1
|
||||
#endif // guards
|
||||
|
||||
/*! \brief whether compile with hdfs support */
|
||||
#ifndef DMLC_USE_HDFS
|
||||
#define DMLC_USE_HDFS 0
|
||||
#endif // DMLC_USE_HDFS
|
||||
|
||||
/*! \brief whether compile with s3 support */
|
||||
#ifndef DMLC_USE_S3
|
||||
#define DMLC_USE_S3 0
|
||||
#endif // DMLC_USE_S3
|
||||
|
||||
/*! \brief whether or not use parameter server */
|
||||
#ifndef DMLC_USE_PS
|
||||
#define DMLC_USE_PS 0
|
||||
#endif // DMLC_USE_PS
|
||||
|
||||
/*! \brief whether or not use c++11 support */
|
||||
#ifndef DMLC_USE_CXX11
|
||||
#if defined(__GXX_EXPERIMENTAL_CXX0X__) || defined(_MSC_VER)
|
||||
#define DMLC_USE_CXX11 1
|
||||
#else
|
||||
#define DMLC_USE_CXX11 (__cplusplus >= 201103L)
|
||||
#endif // defined(__GXX_EXPERIMENTAL_CXX0X__) || defined(_MSC_VER)
|
||||
#endif // DMLC_USE_CXX11
|
||||
|
||||
/*! \brief strict CXX11 support */
|
||||
#ifndef DMLC_STRICT_CXX11
|
||||
#if defined(_MSC_VER)
|
||||
#define DMLC_STRICT_CXX11 1
|
||||
#else
|
||||
#define DMLC_STRICT_CXX11 (__cplusplus >= 201103L)
|
||||
#endif // defined(_MSC_VER)
|
||||
#endif // DMLC_STRICT_CXX11
|
||||
|
||||
/*! \brief Whether cxx11 thread local is supported */
|
||||
#ifndef DMLC_CXX11_THREAD_LOCAL
|
||||
#if defined(_MSC_VER)
|
||||
#if (_MSC_VER >= 1900)
|
||||
#define DMLC_CXX11_THREAD_LOCAL 1
|
||||
#else
|
||||
#define DMLC_CXX11_THREAD_LOCAL 0
|
||||
#endif // (_MSC_VER >= 1900)
|
||||
#else
|
||||
#define DMLC_CXX11_THREAD_LOCAL (__cplusplus >= 201103L)
|
||||
#endif // defined(_MSC_VER)
|
||||
#endif // DMLC_CXX11_THREAD_LOCAL
|
||||
|
||||
|
||||
/*! \brief whether RTTI is enabled */
|
||||
#ifndef DMLC_ENABLE_RTTI
|
||||
#define DMLC_ENABLE_RTTI 1
|
||||
#endif // DMLC_ENABLE_RTTI
|
||||
|
||||
/// check if g++ is before 4.6
|
||||
#if DMLC_USE_CXX11 && defined(__GNUC__) && !defined(__clang_version__)
|
||||
#if __GNUC__ == 4 && __GNUC_MINOR__ < 6
|
||||
#pragma message("Will need g++-4.6 or higher to compile all" \
|
||||
"the features in dmlc-core, " \
|
||||
"compile without c++0x, some features may be disabled")
|
||||
#undef DMLC_USE_CXX11
|
||||
#define DMLC_USE_CXX11 0
|
||||
#endif // __GNUC__ == 4 && __GNUC_MINOR__ < 6
|
||||
#endif // DMLC_USE_CXX11 && defined(__GNUC__) && !defined(__clang_version__)
|
||||
|
||||
|
||||
/*!
|
||||
* \brief Enable std::thread related modules,
|
||||
* Used to disable some module in mingw compile.
|
||||
*/
|
||||
#ifndef DMLC_ENABLE_STD_THREAD
|
||||
#define DMLC_ENABLE_STD_THREAD DMLC_USE_CXX11
|
||||
#endif // DMLC_ENABLE_STD_THREAD
|
||||
|
||||
/*! \brief whether enable regex support, actually need g++-4.9 or higher*/
|
||||
#ifndef DMLC_USE_REGEX
|
||||
#define DMLC_USE_REGEX DMLC_STRICT_CXX11
|
||||
#endif // DMLC_USE_REGEX
|
||||
|
||||
/*! \brief helper macro to supress unused warning */
|
||||
#if defined(__GNUC__)
|
||||
#define DMLC_ATTRIBUTE_UNUSED __attribute__((unused))
|
||||
#else
|
||||
#define DMLC_ATTRIBUTE_UNUSED
|
||||
#endif // defined(__GNUC__)
|
||||
|
||||
/*! \brief helper macro to generate string concat */
|
||||
#define DMLC_STR_CONCAT_(__x, __y) __x##__y
|
||||
#define DMLC_STR_CONCAT(__x, __y) DMLC_STR_CONCAT_(__x, __y)
|
||||
|
||||
/*!
|
||||
* \brief Disable copy constructor and assignment operator.
|
||||
*
|
||||
* If C++11 is supported, both copy and move constructors and
|
||||
* assignment operators are deleted explicitly. Otherwise, they are
|
||||
* only declared but not implemented. Place this macro in private
|
||||
* section if C++11 is not available.
|
||||
*/
|
||||
#ifndef DISALLOW_COPY_AND_ASSIGN
|
||||
# if DMLC_USE_CXX11
|
||||
# define DISALLOW_COPY_AND_ASSIGN(T) \
|
||||
T(T const&) = delete; \
|
||||
T(T&&) = delete; \
|
||||
T& operator=(T const&) = delete; \
|
||||
T& operator=(T&&) = delete
|
||||
# else
|
||||
# define DISALLOW_COPY_AND_ASSIGN(T) \
|
||||
T(T const&); \
|
||||
T& operator=(T const&)
|
||||
# endif // DMLC_USE_CXX11
|
||||
#endif // DISALLOW_COPY_AND_ASSIGN
|
||||
|
||||
///
|
||||
/// code block to handle optionally loading
|
||||
///
|
||||
#if !defined(__GNUC__)
|
||||
#define fopen64 std::fopen
|
||||
#endif // !defined(__GNUC__)
|
||||
|
||||
#if (defined __MINGW32__) && !(defined __MINGW64__)
|
||||
#define fopen64 std::fopen
|
||||
#endif // (defined __MINGW32__) && !(defined __MINGW64__)
|
||||
|
||||
#ifdef _MSC_VER
|
||||
|
||||
# if _MSC_VER < 1900
|
||||
// NOTE: sprintf_s is not equivalent to snprintf,
|
||||
// they are equivalent when success, which is sufficient for our case
|
||||
# define snprintf sprintf_s
|
||||
# define vsnprintf vsprintf_s
|
||||
# endif // _MSC_VER < 1900
|
||||
|
||||
#else
|
||||
|
||||
# ifdef _FILE_OFFSET_BITS
|
||||
# if _FILE_OFFSET_BITS == 32
|
||||
# pragma message("Warning: FILE OFFSET BITS defined to be 32 bit")
|
||||
# endif // _FILE_OFFSET_BITS == 32
|
||||
# endif // _FILE_OFFSET_BITS
|
||||
|
||||
# ifdef __APPLE__
|
||||
# define off64_t off_t
|
||||
# define fopen64 std::fopen
|
||||
# endif // __APPLE__
|
||||
|
||||
extern "C" {
|
||||
#include <sys/types.h>
|
||||
}
|
||||
#endif // _MSC_VER
|
||||
|
||||
#ifdef _MSC_VER
|
||||
//! \cond Doxygen_Suppress
|
||||
typedef signed char int8_t;
|
||||
typedef __int16 int16_t;
|
||||
typedef __int32 int32_t;
|
||||
typedef __int64 int64_t;
|
||||
typedef unsigned char uint8_t;
|
||||
typedef unsigned __int16 uint16_t;
|
||||
typedef unsigned __int32 uint32_t;
|
||||
typedef unsigned __int64 uint64_t;
|
||||
//! \endcond
|
||||
#else
|
||||
#include <inttypes.h>
|
||||
#endif // _MSC_VER
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#if defined(_MSC_VER) && _MSC_VER < 1900
|
||||
#define noexcept_true throw ()
|
||||
#define noexcept_false
|
||||
#define noexcept(a) noexcept_##a
|
||||
#endif // defined(_MSC_VER) && _MSC_VER < 1900
|
||||
|
||||
#if DMLC_USE_CXX11
|
||||
#define DMLC_THROW_EXCEPTION noexcept(false)
|
||||
#define DMLC_NO_EXCEPTION noexcept(true)
|
||||
#else
|
||||
#define DMLC_THROW_EXCEPTION
|
||||
#define DMLC_NO_EXCEPTION
|
||||
#endif // DMLC_USE_CXX11
|
||||
|
||||
/*! \brief namespace for dmlc */
|
||||
namespace dmlc {
|
||||
/*!
|
||||
* \brief safely get the beginning address of a vector
|
||||
* \param vec input vector
|
||||
* \return beginning address of a vector
|
||||
*/
|
||||
template<typename T>
|
||||
inline T *BeginPtr(std::vector<T> &vec) { // NOLINT(*)
|
||||
if (vec.size() == 0) {
|
||||
return NULL;
|
||||
} else {
|
||||
return &vec[0];
|
||||
}
|
||||
}
|
||||
/*!
|
||||
* \brief get the beginning address of a const vector
|
||||
* \param vec input vector
|
||||
* \return beginning address of a vector
|
||||
*/
|
||||
template<typename T>
|
||||
inline const T *BeginPtr(const std::vector<T> &vec) {
|
||||
if (vec.size() == 0) {
|
||||
return NULL;
|
||||
} else {
|
||||
return &vec[0];
|
||||
}
|
||||
}
|
||||
/*!
|
||||
* \brief get the beginning address of a string
|
||||
* \param str input string
|
||||
* \return beginning address of a string
|
||||
*/
|
||||
inline char* BeginPtr(std::string &str) { // NOLINT(*)
|
||||
if (str.length() == 0) return NULL;
|
||||
return &str[0];
|
||||
}
|
||||
/*!
|
||||
* \brief get the beginning address of a const string
|
||||
* \param str input string
|
||||
* \return beginning address of a string
|
||||
*/
|
||||
inline const char* BeginPtr(const std::string &str) {
|
||||
if (str.length() == 0) return NULL;
|
||||
return &str[0];
|
||||
}
|
||||
} // namespace dmlc
|
||||
|
||||
#if defined(_MSC_VER) && _MSC_VER < 1900
|
||||
#define constexpr const
|
||||
#define alignof __alignof
|
||||
#endif // defined(_MSC_VER) && _MSC_VER < 1900
|
||||
|
||||
#endif // DMLC_BASE_H_
|
||||
@ -1,424 +0,0 @@
|
||||
/*!
|
||||
* Copyright (c) 2015 by Contributors
|
||||
* \file io.h
|
||||
* \brief defines serializable interface of dmlc
|
||||
*/
|
||||
#ifndef DMLC_IO_H_
|
||||
#define DMLC_IO_H_
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <istream>
|
||||
#include <ostream>
|
||||
#include <streambuf>
|
||||
|
||||
#include "base.h"
|
||||
|
||||
// include uint64_t only to make io standalone
|
||||
#ifdef _MSC_VER
|
||||
/*! \brief uint64 */
|
||||
typedef unsigned __int64 uint64_t;
|
||||
#else
|
||||
#include <inttypes.h>
|
||||
#endif
|
||||
|
||||
/*! \brief namespace for dmlc */
|
||||
namespace dmlc {
|
||||
/*!
|
||||
* \brief interface of stream I/O for serialization
|
||||
*/
|
||||
class Stream { // NOLINT(*)
|
||||
public:
|
||||
/*!
|
||||
* \brief reads data from a stream
|
||||
* \param ptr pointer to a memory buffer
|
||||
* \param size block size
|
||||
* \return the size of data read
|
||||
*/
|
||||
virtual size_t Read(void *ptr, size_t size) = 0;
|
||||
/*!
|
||||
* \brief writes data to a stream
|
||||
* \param ptr pointer to a memory buffer
|
||||
* \param size block size
|
||||
*/
|
||||
virtual void Write(const void *ptr, size_t size) = 0;
|
||||
/*! \brief virtual destructor */
|
||||
virtual ~Stream(void) {}
|
||||
/*!
|
||||
* \brief generic factory function
|
||||
* create an stream, the stream will close the underlying files upon deletion
|
||||
*
|
||||
* \param uri the uri of the input currently we support
|
||||
* hdfs://, s3://, and file:// by default file:// will be used
|
||||
* \param flag can be "w", "r", "a"
|
||||
* \param allow_null whether NULL can be returned, or directly report error
|
||||
* \return the created stream, can be NULL when allow_null == true and file do not exist
|
||||
*/
|
||||
static Stream *Create(const char *uri,
|
||||
const char* const flag,
|
||||
bool allow_null = false);
|
||||
// helper functions to write/read different data structures
|
||||
/*!
|
||||
* \brief writes a data to stream
|
||||
*
|
||||
* dmlc::Stream support Write/Read of most STL
|
||||
* composites and base types.
|
||||
* If the data type is not supported, a compile time error will
|
||||
* be issued.
|
||||
*
|
||||
* \param data data to be written
|
||||
* \tparam T the data type to be written
|
||||
*/
|
||||
template<typename T>
|
||||
inline void Write(const T &data);
|
||||
/*!
|
||||
* \brief loads a data from stream.
|
||||
*
|
||||
* dmlc::Stream support Write/Read of most STL
|
||||
* composites and base types.
|
||||
* If the data type is not supported, a compile time error will
|
||||
* be issued.
|
||||
*
|
||||
* \param out_data place holder of data to be deserialized
|
||||
* \return whether the load was successful
|
||||
*/
|
||||
template<typename T>
|
||||
inline bool Read(T *out_data);
|
||||
};
|
||||
|
||||
/*! \brief interface of i/o stream that support seek */
|
||||
class SeekStream: public Stream {
|
||||
public:
|
||||
// virtual destructor
|
||||
virtual ~SeekStream(void) {}
|
||||
/*! \brief seek to certain position of the file */
|
||||
virtual void Seek(size_t pos) = 0;
|
||||
/*! \brief tell the position of the stream */
|
||||
virtual size_t Tell(void) = 0;
|
||||
/*!
|
||||
* \brief generic factory function
|
||||
* create an SeekStream for read only,
|
||||
* the stream will close the underlying files upon deletion
|
||||
* error will be reported and the system will exit when create failed
|
||||
* \param uri the uri of the input currently we support
|
||||
* hdfs://, s3://, and file:// by default file:// will be used
|
||||
* \param allow_null whether NULL can be returned, or directly report error
|
||||
* \return the created stream, can be NULL when allow_null == true and file do not exist
|
||||
*/
|
||||
static SeekStream *CreateForRead(const char *uri,
|
||||
bool allow_null = false);
|
||||
};
|
||||
|
||||
/*! \brief interface for serializable objects */
|
||||
class Serializable {
|
||||
public:
|
||||
/*! \brief virtual destructor */
|
||||
virtual ~Serializable() {}
|
||||
/*!
|
||||
* \brief load the model from a stream
|
||||
* \param fi stream where to load the model from
|
||||
*/
|
||||
virtual void Load(Stream *fi) = 0;
|
||||
/*!
|
||||
* \brief saves the model to a stream
|
||||
* \param fo stream where to save the model to
|
||||
*/
|
||||
virtual void Save(Stream *fo) const = 0;
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief input split creates that allows reading
|
||||
* of records from split of data,
|
||||
* independent part that covers all the dataset
|
||||
*
|
||||
* see InputSplit::Create for definition of record
|
||||
*/
|
||||
class InputSplit {
|
||||
public:
|
||||
/*! \brief a blob of memory region */
|
||||
struct Blob {
|
||||
/*! \brief points to start of the memory region */
|
||||
void *dptr;
|
||||
/*! \brief size of the memory region */
|
||||
size_t size;
|
||||
};
|
||||
/*!
|
||||
* \brief hint the inputsplit how large the chunk size
|
||||
* it should return when implementing NextChunk
|
||||
* this is a hint so may not be enforced,
|
||||
* but InputSplit will try adjust its internal buffer
|
||||
* size to the hinted value
|
||||
* \param chunk_size the chunk size
|
||||
*/
|
||||
virtual void HintChunkSize(size_t chunk_size) {}
|
||||
/*! \brief get the total size of the InputSplit */
|
||||
virtual size_t GetTotalSize(void) = 0;
|
||||
/*! \brief reset the position of InputSplit to beginning */
|
||||
virtual void BeforeFirst(void) = 0;
|
||||
/*!
|
||||
* \brief get the next record, the returning value
|
||||
* is valid until next call to NextRecord or NextChunk
|
||||
* caller can modify the memory content of out_rec
|
||||
*
|
||||
* For text, out_rec contains a single line
|
||||
* For recordio, out_rec contains one record content(with header striped)
|
||||
*
|
||||
* \param out_rec used to store the result
|
||||
* \return true if we can successfully get next record
|
||||
* false if we reached end of split
|
||||
* \sa InputSplit::Create for definition of record
|
||||
*/
|
||||
virtual bool NextRecord(Blob *out_rec) = 0;
|
||||
/*!
|
||||
* \brief get a chunk of memory that can contain multiple records,
|
||||
* the caller needs to parse the content of the resulting chunk,
|
||||
* for text file, out_chunk can contain data of multiple lines
|
||||
* for recordio, out_chunk can contain multiple records(including headers)
|
||||
*
|
||||
* This function ensures there won't be partial record in the chunk
|
||||
* caller can modify the memory content of out_chunk,
|
||||
* the memory is valid until next call to NextRecord or NextChunk
|
||||
*
|
||||
* Usually NextRecord is sufficient, NextChunk can be used by some
|
||||
* multi-threaded parsers to parse the input content
|
||||
*
|
||||
* \param out_chunk used to store the result
|
||||
* \return true if we can successfully get next record
|
||||
* false if we reached end of split
|
||||
* \sa InputSplit::Create for definition of record
|
||||
* \sa RecordIOChunkReader to parse recordio content from out_chunk
|
||||
*/
|
||||
virtual bool NextChunk(Blob *out_chunk) = 0;
|
||||
/*! \brief destructor*/
|
||||
virtual ~InputSplit(void) {}
|
||||
/*!
|
||||
* \brief reset the Input split to a certain part id,
|
||||
* The InputSplit will be pointed to the head of the new specified segment.
|
||||
* This feature may not be supported by every implementation of InputSplit.
|
||||
* \param part_index The part id of the new input.
|
||||
* \param num_parts The total number of parts.
|
||||
*/
|
||||
virtual void ResetPartition(unsigned part_index, unsigned num_parts) = 0;
|
||||
/*!
|
||||
* \brief factory function:
|
||||
* create input split given a uri
|
||||
* \param uri the uri of the input, can contain hdfs prefix
|
||||
* \param part_index the part id of current input
|
||||
* \param num_parts total number of splits
|
||||
* \param type type of record
|
||||
* List of possible types: "text", "recordio"
|
||||
* - "text":
|
||||
* text file, each line is treated as a record
|
||||
* input split will split on '\\n' or '\\r'
|
||||
* - "recordio":
|
||||
* binary recordio file, see recordio.h
|
||||
* \return a new input split
|
||||
* \sa InputSplit::Type
|
||||
*/
|
||||
static InputSplit* Create(const char *uri,
|
||||
unsigned part_index,
|
||||
unsigned num_parts,
|
||||
const char *type);
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief a std::ostream class that can can wrap Stream objects,
|
||||
* can use ostream with that output to underlying Stream
|
||||
*
|
||||
* Usage example:
|
||||
* \code
|
||||
*
|
||||
* Stream *fs = Stream::Create("hdfs:///test.txt", "w");
|
||||
* dmlc::ostream os(fs);
|
||||
* os << "hello world" << std::endl;
|
||||
* delete fs;
|
||||
* \endcode
|
||||
*/
|
||||
class ostream : public std::basic_ostream<char> {
|
||||
public:
|
||||
/*!
|
||||
* \brief construct std::ostream type
|
||||
* \param stream the Stream output to be used
|
||||
* \param buffer_size internal streambuf size
|
||||
*/
|
||||
explicit ostream(Stream *stream,
|
||||
size_t buffer_size = (1 << 10))
|
||||
: std::basic_ostream<char>(NULL), buf_(buffer_size) {
|
||||
this->set_stream(stream);
|
||||
}
|
||||
// explictly synchronize the buffer
|
||||
virtual ~ostream() DMLC_NO_EXCEPTION {
|
||||
buf_.pubsync();
|
||||
}
|
||||
/*!
|
||||
* \brief set internal stream to be stream, reset states
|
||||
* \param stream new stream as output
|
||||
*/
|
||||
inline void set_stream(Stream *stream) {
|
||||
buf_.set_stream(stream);
|
||||
this->rdbuf(&buf_);
|
||||
}
|
||||
|
||||
/*! \return how many bytes we written so far */
|
||||
inline size_t bytes_written(void) const {
|
||||
return buf_.bytes_out();
|
||||
}
|
||||
|
||||
private:
|
||||
// internal streambuf
|
||||
class OutBuf : public std::streambuf {
|
||||
public:
|
||||
explicit OutBuf(size_t buffer_size)
|
||||
: stream_(NULL), buffer_(buffer_size), bytes_out_(0) {
|
||||
if (buffer_size == 0) buffer_.resize(2);
|
||||
}
|
||||
// set stream to the buffer
|
||||
inline void set_stream(Stream *stream);
|
||||
|
||||
inline size_t bytes_out() const { return bytes_out_; }
|
||||
private:
|
||||
/*! \brief internal stream by StreamBuf */
|
||||
Stream *stream_;
|
||||
/*! \brief internal buffer */
|
||||
std::vector<char> buffer_;
|
||||
/*! \brief number of bytes written so far */
|
||||
size_t bytes_out_;
|
||||
// override sync
|
||||
inline int_type sync(void);
|
||||
// override overflow
|
||||
inline int_type overflow(int c);
|
||||
};
|
||||
/*! \brief buffer of the stream */
|
||||
OutBuf buf_;
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief a std::istream class that can can wrap Stream objects,
|
||||
* can use istream with that output to underlying Stream
|
||||
*
|
||||
* Usage example:
|
||||
* \code
|
||||
*
|
||||
* Stream *fs = Stream::Create("hdfs:///test.txt", "r");
|
||||
* dmlc::istream is(fs);
|
||||
* is >> mydata;
|
||||
* delete fs;
|
||||
* \endcode
|
||||
*/
|
||||
class istream : public std::basic_istream<char> {
|
||||
public:
|
||||
/*!
|
||||
* \brief construct std::ostream type
|
||||
* \param stream the Stream output to be used
|
||||
* \param buffer_size internal buffer size
|
||||
*/
|
||||
explicit istream(Stream *stream,
|
||||
size_t buffer_size = (1 << 10))
|
||||
: std::basic_istream<char>(NULL), buf_(buffer_size) {
|
||||
this->set_stream(stream);
|
||||
}
|
||||
virtual ~istream() DMLC_NO_EXCEPTION {}
|
||||
/*!
|
||||
* \brief set internal stream to be stream, reset states
|
||||
* \param stream new stream as output
|
||||
*/
|
||||
inline void set_stream(Stream *stream) {
|
||||
buf_.set_stream(stream);
|
||||
this->rdbuf(&buf_);
|
||||
}
|
||||
/*! \return how many bytes we read so far */
|
||||
inline size_t bytes_read(void) const {
|
||||
return buf_.bytes_read();
|
||||
}
|
||||
|
||||
private:
|
||||
// internal streambuf
|
||||
class InBuf : public std::streambuf {
|
||||
public:
|
||||
explicit InBuf(size_t buffer_size)
|
||||
: stream_(NULL), bytes_read_(0),
|
||||
buffer_(buffer_size) {
|
||||
if (buffer_size == 0) buffer_.resize(2);
|
||||
}
|
||||
// set stream to the buffer
|
||||
inline void set_stream(Stream *stream);
|
||||
// return how many bytes read so far
|
||||
inline size_t bytes_read(void) const {
|
||||
return bytes_read_;
|
||||
}
|
||||
private:
|
||||
/*! \brief internal stream by StreamBuf */
|
||||
Stream *stream_;
|
||||
/*! \brief how many bytes we read so far */
|
||||
size_t bytes_read_;
|
||||
/*! \brief internal buffer */
|
||||
std::vector<char> buffer_;
|
||||
// override underflow
|
||||
inline int_type underflow();
|
||||
};
|
||||
/*! \brief input buffer */
|
||||
InBuf buf_;
|
||||
};
|
||||
} // namespace dmlc
|
||||
|
||||
#include "./serializer.h"
|
||||
|
||||
namespace dmlc {
|
||||
// implementations of inline functions
|
||||
template<typename T>
|
||||
inline void Stream::Write(const T &data) {
|
||||
serializer::Handler<T>::Write(this, data);
|
||||
}
|
||||
template<typename T>
|
||||
inline bool Stream::Read(T *out_data) {
|
||||
return serializer::Handler<T>::Read(this, out_data);
|
||||
}
|
||||
|
||||
// implementations for ostream
|
||||
inline void ostream::OutBuf::set_stream(Stream *stream) {
|
||||
if (stream_ != NULL) this->pubsync();
|
||||
this->stream_ = stream;
|
||||
this->setp(&buffer_[0], &buffer_[0] + buffer_.size() - 1);
|
||||
}
|
||||
inline int ostream::OutBuf::sync(void) {
|
||||
if (stream_ == NULL) return -1;
|
||||
std::ptrdiff_t n = pptr() - pbase();
|
||||
stream_->Write(pbase(), n);
|
||||
this->pbump(-static_cast<int>(n));
|
||||
bytes_out_ += n;
|
||||
return 0;
|
||||
}
|
||||
inline int ostream::OutBuf::overflow(int c) {
|
||||
*(this->pptr()) = c;
|
||||
std::ptrdiff_t n = pptr() - pbase();
|
||||
this->pbump(-static_cast<int>(n));
|
||||
if (c == EOF) {
|
||||
stream_->Write(pbase(), n);
|
||||
bytes_out_ += n;
|
||||
} else {
|
||||
stream_->Write(pbase(), n + 1);
|
||||
bytes_out_ += n + 1;
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
// implementations for istream
|
||||
inline void istream::InBuf::set_stream(Stream *stream) {
|
||||
stream_ = stream;
|
||||
this->setg(&buffer_[0], &buffer_[0], &buffer_[0]);
|
||||
}
|
||||
inline int istream::InBuf::underflow() {
|
||||
char *bhead = &buffer_[0];
|
||||
if (this->gptr() == this->egptr()) {
|
||||
size_t sz = stream_->Read(bhead, buffer_.size());
|
||||
this->setg(bhead, bhead, bhead + sz);
|
||||
bytes_read_ += sz;
|
||||
}
|
||||
if (this->gptr() == this->egptr()) {
|
||||
return traits_type::eof();
|
||||
} else {
|
||||
return traits_type::to_int_type(*gptr());
|
||||
}
|
||||
}
|
||||
} // namespace dmlc
|
||||
#endif // DMLC_IO_H_
|
||||
@ -1,382 +0,0 @@
|
||||
/*!
|
||||
* Copyright (c) 2015 by Contributors
|
||||
* \file serializer.h
|
||||
* \brief serializer template class that helps serialization.
|
||||
* This file do not need to be directly used by most user.
|
||||
*/
|
||||
#ifndef DMLC_SERIALIZER_H_
|
||||
#define DMLC_SERIALIZER_H_
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <list>
|
||||
#include <deque>
|
||||
#include <utility>
|
||||
|
||||
#include "./base.h"
|
||||
#include "./io.h"
|
||||
#include "./type_traits.h"
|
||||
|
||||
#if DMLC_USE_CXX11
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#endif
|
||||
|
||||
namespace dmlc {
|
||||
/*! \brief internal namespace for serializers */
|
||||
namespace serializer {
|
||||
/*!
|
||||
* \brief generic serialization handler
|
||||
* \tparam T the type to be serialized
|
||||
*/
|
||||
template<typename T>
|
||||
struct Handler;
|
||||
|
||||
//! \cond Doxygen_Suppress
|
||||
/*!
|
||||
* \brief Serializer that redirect calls by condition
|
||||
* \tparam cond the condition
|
||||
* \tparam Then the serializer used for then condition
|
||||
* \tparam Else the serializer used for else condition
|
||||
* \tparam Return the type of data the serializer handles
|
||||
*/
|
||||
template<bool cond, typename Then, typename Else, typename Return>
|
||||
struct IfThenElse;
|
||||
|
||||
template<typename Then, typename Else, typename T>
|
||||
struct IfThenElse<true, Then, Else, T> {
|
||||
inline static void Write(Stream *strm, const T &data) {
|
||||
Then::Write(strm, data);
|
||||
}
|
||||
inline static bool Read(Stream *strm, T *data) {
|
||||
return Then::Read(strm, data);
|
||||
}
|
||||
};
|
||||
template<typename Then, typename Else, typename T>
|
||||
struct IfThenElse<false, Then, Else, T> {
|
||||
inline static void Write(Stream *strm, const T &data) {
|
||||
Else::Write(strm, data);
|
||||
}
|
||||
inline static bool Read(Stream *strm, T *data) {
|
||||
return Else::Read(strm, data);
|
||||
}
|
||||
};
|
||||
|
||||
/*! \brief Serializer for POD(plain-old-data) data */
|
||||
template<typename T>
|
||||
struct PODHandler {
|
||||
inline static void Write(Stream *strm, const T &data) {
|
||||
strm->Write(&data, sizeof(T));
|
||||
}
|
||||
inline static bool Read(Stream *strm, T *dptr) {
|
||||
return strm->Read((void*)dptr, sizeof(T)) == sizeof(T); // NOLINT(*)
|
||||
}
|
||||
};
|
||||
|
||||
// serializer for class that have save/load function
|
||||
template<typename T>
|
||||
struct SaveLoadClassHandler {
|
||||
inline static void Write(Stream *strm, const T &data) {
|
||||
data.Save(strm);
|
||||
}
|
||||
inline static bool Read(Stream *strm, T *data) {
|
||||
return data->Load(strm);
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief dummy class for undefined serialization.
|
||||
* This is used to generate error message when user tries to
|
||||
* serialize something that is not supported.
|
||||
* \tparam T the type to be serialized
|
||||
*/
|
||||
template<typename T>
|
||||
struct UndefinedSerializerFor {
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief Serializer handler for std::vector<T> where T is POD type.
|
||||
* \tparam T element type
|
||||
*/
|
||||
template<typename T>
|
||||
struct PODVectorHandler {
|
||||
inline static void Write(Stream *strm, const std::vector<T> &vec) {
|
||||
uint64_t sz = static_cast<uint64_t>(vec.size());
|
||||
strm->Write(&sz, sizeof(sz));
|
||||
if (sz != 0) {
|
||||
strm->Write(&vec[0], sizeof(T) * vec.size());
|
||||
}
|
||||
}
|
||||
inline static bool Read(Stream *strm, std::vector<T> *out_vec) {
|
||||
uint64_t sz;
|
||||
if (strm->Read(&sz, sizeof(sz)) != sizeof(sz)) return false;
|
||||
size_t size = static_cast<size_t>(sz);
|
||||
out_vec->resize(size);
|
||||
if (sz != 0) {
|
||||
size_t nbytes = sizeof(T) * size;
|
||||
return strm->Read(&(*out_vec)[0], nbytes) == nbytes;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief Serializer handler for std::vector<T> where T can be composed type
|
||||
* \tparam T element type
|
||||
*/
|
||||
template<typename T>
|
||||
struct ComposeVectorHandler {
|
||||
inline static void Write(Stream *strm, const std::vector<T> &vec) {
|
||||
uint64_t sz = static_cast<uint64_t>(vec.size());
|
||||
strm->Write(&sz, sizeof(sz));
|
||||
for (size_t i = 0; i < vec.size(); ++i) {
|
||||
Handler<T>::Write(strm, vec[i]);
|
||||
}
|
||||
}
|
||||
inline static bool Read(Stream *strm, std::vector<T> *out_vec) {
|
||||
uint64_t sz;
|
||||
if (strm->Read(&sz, sizeof(sz)) != sizeof(sz)) return false;
|
||||
size_t size = static_cast<size_t>(sz);
|
||||
out_vec->resize(size);
|
||||
for (size_t i = 0; i < size; ++i) {
|
||||
if (!Handler<T>::Read(strm, &(*out_vec)[i])) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief Serializer handler for std::basic_string<T> where T is POD type.
|
||||
* \tparam T element type
|
||||
*/
|
||||
template<typename T>
|
||||
struct PODStringHandler {
|
||||
inline static void Write(Stream *strm, const std::basic_string<T> &vec) {
|
||||
uint64_t sz = static_cast<uint64_t>(vec.length());
|
||||
strm->Write(&sz, sizeof(sz));
|
||||
if (sz != 0) {
|
||||
strm->Write(&vec[0], sizeof(T) * vec.length());
|
||||
}
|
||||
}
|
||||
inline static bool Read(Stream *strm, std::basic_string<T> *out_vec) {
|
||||
uint64_t sz;
|
||||
if (strm->Read(&sz, sizeof(sz)) != sizeof(sz)) return false;
|
||||
size_t size = static_cast<size_t>(sz);
|
||||
out_vec->resize(size);
|
||||
if (sz != 0) {
|
||||
size_t nbytes = sizeof(T) * size;
|
||||
return strm->Read(&(*out_vec)[0], nbytes) == nbytes;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
/*! \brief Serializer for std::pair */
|
||||
template<typename TA, typename TB>
|
||||
struct PairHandler {
|
||||
inline static void Write(Stream *strm, const std::pair<TA, TB> &data) {
|
||||
Handler<TA>::Write(strm, data.first);
|
||||
Handler<TB>::Write(strm, data.second);
|
||||
}
|
||||
inline static bool Read(Stream *strm, std::pair<TA, TB> *data) {
|
||||
return Handler<TA>::Read(strm, &(data->first)) &&
|
||||
Handler<TB>::Read(strm, &(data->second));
|
||||
}
|
||||
};
|
||||
|
||||
// set type handler that can handle most collection type case
|
||||
template<typename ContainerType>
|
||||
struct CollectionHandler {
|
||||
inline static void Write(Stream *strm, const ContainerType &data) {
|
||||
typedef typename ContainerType::value_type ElemType;
|
||||
// dump data to vector
|
||||
std::vector<ElemType> vdata(data.begin(), data.end());
|
||||
// serialize the vector
|
||||
Handler<std::vector<ElemType> >::Write(strm, vdata);
|
||||
}
|
||||
inline static bool Read(Stream *strm, ContainerType *data) {
|
||||
typedef typename ContainerType::value_type ElemType;
|
||||
std::vector<ElemType> vdata;
|
||||
if (!Handler<std::vector<ElemType> >::Read(strm, &vdata)) return false;
|
||||
data->clear();
|
||||
data->insert(vdata.begin(), vdata.end());
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// handler that can handle most list type case
|
||||
// this type insert function takes additional iterator
|
||||
template<typename ListType>
|
||||
struct ListHandler {
|
||||
inline static void Write(Stream *strm, const ListType &data) {
|
||||
typedef typename ListType::value_type ElemType;
|
||||
// dump data to vector
|
||||
std::vector<ElemType> vdata(data.begin(), data.end());
|
||||
// serialize the vector
|
||||
Handler<std::vector<ElemType> >::Write(strm, vdata);
|
||||
}
|
||||
inline static bool Read(Stream *strm, ListType *data) {
|
||||
typedef typename ListType::value_type ElemType;
|
||||
std::vector<ElemType> vdata;
|
||||
if (!Handler<std::vector<ElemType> >::Read(strm, &vdata)) return false;
|
||||
data->clear();
|
||||
data->insert(data->begin(), vdata.begin(), vdata.end());
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
//! \endcond
|
||||
|
||||
/*!
|
||||
* \brief generic serialization handler for type T
|
||||
*
|
||||
* User can define specialization of this class to support
|
||||
* composite serialization of their own class.
|
||||
*
|
||||
* \tparam T the type to be serialized
|
||||
*/
|
||||
template<typename T>
|
||||
struct Handler {
|
||||
/*!
|
||||
* \brief write data to stream
|
||||
* \param strm the stream we write the data.
|
||||
* \param data the data obeject to be serialized
|
||||
*/
|
||||
inline static void Write(Stream *strm, const T &data) {
|
||||
IfThenElse<dmlc::is_pod<T>::value,
|
||||
PODHandler<T>,
|
||||
IfThenElse<dmlc::has_saveload<T>::value,
|
||||
SaveLoadClassHandler<T>,
|
||||
UndefinedSerializerFor<T>, T>,
|
||||
T>
|
||||
::Write(strm, data);
|
||||
}
|
||||
/*!
|
||||
* \brief read data to stream
|
||||
* \param strm the stream to read the data.
|
||||
* \param data the pointer to the data obeject to read
|
||||
* \return whether the read is successful
|
||||
*/
|
||||
inline static bool Read(Stream *strm, T *data) {
|
||||
return IfThenElse<dmlc::is_pod<T>::value,
|
||||
PODHandler<T>,
|
||||
IfThenElse<dmlc::has_saveload<T>::value,
|
||||
SaveLoadClassHandler<T>,
|
||||
UndefinedSerializerFor<T>, T>,
|
||||
T>
|
||||
::Read(strm, data);
|
||||
}
|
||||
};
|
||||
|
||||
//! \cond Doxygen_Suppress
|
||||
template<typename T>
|
||||
struct Handler<std::vector<T> > {
|
||||
inline static void Write(Stream *strm, const std::vector<T> &data) {
|
||||
IfThenElse<dmlc::is_pod<T>::value,
|
||||
PODVectorHandler<T>,
|
||||
ComposeVectorHandler<T>, std::vector<T> >
|
||||
::Write(strm, data);
|
||||
}
|
||||
inline static bool Read(Stream *strm, std::vector<T> *data) {
|
||||
return IfThenElse<dmlc::is_pod<T>::value,
|
||||
PODVectorHandler<T>,
|
||||
ComposeVectorHandler<T>,
|
||||
std::vector<T> >
|
||||
::Read(strm, data);
|
||||
}
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
struct Handler<std::basic_string<T> > {
|
||||
inline static void Write(Stream *strm, const std::basic_string<T> &data) {
|
||||
IfThenElse<dmlc::is_pod<T>::value,
|
||||
PODStringHandler<T>,
|
||||
UndefinedSerializerFor<T>,
|
||||
std::basic_string<T> >
|
||||
::Write(strm, data);
|
||||
}
|
||||
inline static bool Read(Stream *strm, std::basic_string<T> *data) {
|
||||
return IfThenElse<dmlc::is_pod<T>::value,
|
||||
PODStringHandler<T>,
|
||||
UndefinedSerializerFor<T>,
|
||||
std::basic_string<T> >
|
||||
::Read(strm, data);
|
||||
}
|
||||
};
|
||||
|
||||
template<typename TA, typename TB>
|
||||
struct Handler<std::pair<TA, TB> > {
|
||||
inline static void Write(Stream *strm, const std::pair<TA, TB> &data) {
|
||||
IfThenElse<dmlc::is_pod<TA>::value && dmlc::is_pod<TB>::value,
|
||||
PODHandler<std::pair<TA, TB> >,
|
||||
PairHandler<TA, TB>,
|
||||
std::pair<TA, TB> >
|
||||
::Write(strm, data);
|
||||
}
|
||||
inline static bool Read(Stream *strm, std::pair<TA, TB> *data) {
|
||||
return IfThenElse<dmlc::is_pod<TA>::value && dmlc::is_pod<TB>::value,
|
||||
PODHandler<std::pair<TA, TB> >,
|
||||
PairHandler<TA, TB>,
|
||||
std::pair<TA, TB> >
|
||||
::Read(strm, data);
|
||||
}
|
||||
};
|
||||
|
||||
template<typename K, typename V>
|
||||
struct Handler<std::map<K, V> >
|
||||
: public CollectionHandler<std::map<K, V> > {
|
||||
};
|
||||
|
||||
template<typename K, typename V>
|
||||
struct Handler<std::multimap<K, V> >
|
||||
: public CollectionHandler<std::multimap<K, V> > {
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
struct Handler<std::set<T> >
|
||||
: public CollectionHandler<std::set<T> > {
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
struct Handler<std::multiset<T> >
|
||||
: public CollectionHandler<std::multiset<T> > {
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
struct Handler<std::list<T> >
|
||||
: public ListHandler<std::list<T> > {
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
struct Handler<std::deque<T> >
|
||||
: public ListHandler<std::deque<T> > {
|
||||
};
|
||||
|
||||
#if DMLC_USE_CXX11
|
||||
template<typename K, typename V>
|
||||
struct Handler<std::unordered_map<K, V> >
|
||||
: public CollectionHandler<std::unordered_map<K, V> > {
|
||||
};
|
||||
|
||||
template<typename K, typename V>
|
||||
struct Handler<std::unordered_multimap<K, V> >
|
||||
: public CollectionHandler<std::unordered_multimap<K, V> > {
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
struct Handler<std::unordered_set<T> >
|
||||
: public CollectionHandler<std::unordered_set<T> > {
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
struct Handler<std::unordered_multiset<T> >
|
||||
: public CollectionHandler<std::unordered_multiset<T> > {
|
||||
};
|
||||
#endif
|
||||
//! \endcond
|
||||
} // namespace serializer
|
||||
} // namespace dmlc
|
||||
#endif // DMLC_SERIALIZER_H_
|
||||
@ -1,171 +0,0 @@
|
||||
/*!
|
||||
* Copyright (c) 2015 by Contributors
|
||||
* \file type_traits.h
|
||||
* \brief type traits information header
|
||||
*/
|
||||
#ifndef DMLC_TYPE_TRAITS_H_
|
||||
#define DMLC_TYPE_TRAITS_H_
|
||||
|
||||
#include "./base.h"
|
||||
#if DMLC_USE_CXX11
|
||||
#include <type_traits>
|
||||
#endif // DMLC_USE_CXX11
|
||||
#include <string>
|
||||
|
||||
namespace dmlc {
|
||||
/*!
|
||||
* \brief whether a type is pod type
|
||||
* \tparam T the type to query
|
||||
*/
|
||||
template<typename T>
|
||||
struct is_pod {
|
||||
#if DMLC_USE_CXX11
|
||||
/*! \brief the value of the traits */
|
||||
static const bool value = std::is_pod<T>::value;
|
||||
#else
|
||||
/*! \brief the value of the traits */
|
||||
static const bool value = false;
|
||||
#endif // DMLC_USE_CXX11
|
||||
};
|
||||
|
||||
|
||||
/*!
|
||||
* \brief whether a type is integer type
|
||||
* \tparam T the type to query
|
||||
*/
|
||||
template<typename T>
|
||||
struct is_integral {
|
||||
#if DMLC_USE_CXX11
|
||||
/*! \brief the value of the traits */
|
||||
static const bool value = std::is_integral<T>::value;
|
||||
#else
|
||||
/*! \brief the value of the traits */
|
||||
static const bool value = false;
|
||||
#endif // DMLC_USE_CXX11
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief whether a type is floating point type
|
||||
* \tparam T the type to query
|
||||
*/
|
||||
template<typename T>
|
||||
struct is_floating_point {
|
||||
#if DMLC_USE_CXX11
|
||||
/*! \brief the value of the traits */
|
||||
static const bool value = std::is_floating_point<T>::value;
|
||||
#else
|
||||
/*! \brief the value of the traits */
|
||||
static const bool value = false;
|
||||
#endif // DMLC_USE_CXX11
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief whether a type is arithemetic type
|
||||
* \tparam T the type to query
|
||||
*/
|
||||
template<typename T>
|
||||
struct is_arithmetic {
|
||||
#if DMLC_USE_CXX11
|
||||
/*! \brief the value of the traits */
|
||||
static const bool value = std::is_arithmetic<T>::value;
|
||||
#else
|
||||
/*! \brief the value of the traits */
|
||||
static const bool value = (dmlc::is_integral<T>::value ||
|
||||
dmlc::is_floating_point<T>::value);
|
||||
#endif // DMLC_USE_CXX11
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief the string representation of type name
|
||||
* \tparam T the type to query
|
||||
* \return a const string of typename.
|
||||
*/
|
||||
template<typename T>
|
||||
inline const char* type_name() {
|
||||
return "";
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief whether a type have save/load function
|
||||
* \tparam T the type to query
|
||||
*/
|
||||
template<typename T>
|
||||
struct has_saveload {
|
||||
/*! \brief the value of the traits */
|
||||
static const bool value = false;
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief template to select type based on condition
|
||||
* For example, IfThenElseType<true, int, float>::Type will give int
|
||||
* \tparam cond the condition
|
||||
* \tparam Then the typename to be returned if cond is true
|
||||
* \tparam The typename to be returned if cond is false
|
||||
*/
|
||||
template<bool cond, typename Then, typename Else>
|
||||
struct IfThenElseType;
|
||||
|
||||
/*! \brief macro to quickly declare traits information */
|
||||
#define DMLC_DECLARE_TRAITS(Trait, Type, Value) \
|
||||
template<> \
|
||||
struct Trait<Type> { \
|
||||
static const bool value = Value; \
|
||||
}
|
||||
|
||||
/*! \brief macro to quickly declare traits information */
|
||||
#define DMLC_DECLARE_TYPE_NAME(Type, Name) \
|
||||
template<> \
|
||||
inline const char* type_name<Type>() { \
|
||||
return Name; \
|
||||
}
|
||||
|
||||
//! \cond Doxygen_Suppress
|
||||
// declare special traits when C++11 is not available
|
||||
#if DMLC_USE_CXX11 == 0
|
||||
DMLC_DECLARE_TRAITS(is_pod, char, true);
|
||||
DMLC_DECLARE_TRAITS(is_pod, int8_t, true);
|
||||
DMLC_DECLARE_TRAITS(is_pod, int16_t, true);
|
||||
DMLC_DECLARE_TRAITS(is_pod, int32_t, true);
|
||||
DMLC_DECLARE_TRAITS(is_pod, int64_t, true);
|
||||
DMLC_DECLARE_TRAITS(is_pod, uint8_t, true);
|
||||
DMLC_DECLARE_TRAITS(is_pod, uint16_t, true);
|
||||
DMLC_DECLARE_TRAITS(is_pod, uint32_t, true);
|
||||
DMLC_DECLARE_TRAITS(is_pod, uint64_t, true);
|
||||
DMLC_DECLARE_TRAITS(is_pod, float, true);
|
||||
DMLC_DECLARE_TRAITS(is_pod, double, true);
|
||||
|
||||
DMLC_DECLARE_TRAITS(is_integral, char, true);
|
||||
DMLC_DECLARE_TRAITS(is_integral, int8_t, true);
|
||||
DMLC_DECLARE_TRAITS(is_integral, int16_t, true);
|
||||
DMLC_DECLARE_TRAITS(is_integral, int32_t, true);
|
||||
DMLC_DECLARE_TRAITS(is_integral, int64_t, true);
|
||||
DMLC_DECLARE_TRAITS(is_integral, uint8_t, true);
|
||||
DMLC_DECLARE_TRAITS(is_integral, uint16_t, true);
|
||||
DMLC_DECLARE_TRAITS(is_integral, uint32_t, true);
|
||||
DMLC_DECLARE_TRAITS(is_integral, uint64_t, true);
|
||||
|
||||
DMLC_DECLARE_TRAITS(is_floating_point, float, true);
|
||||
DMLC_DECLARE_TRAITS(is_floating_point, double, true);
|
||||
|
||||
#endif // DMLC_USE_CXX11
|
||||
|
||||
DMLC_DECLARE_TYPE_NAME(float, "float");
|
||||
DMLC_DECLARE_TYPE_NAME(double, "double");
|
||||
DMLC_DECLARE_TYPE_NAME(int, "int");
|
||||
DMLC_DECLARE_TYPE_NAME(uint32_t, "int (non-negative)");
|
||||
DMLC_DECLARE_TYPE_NAME(uint64_t, "long (non-negative)");
|
||||
DMLC_DECLARE_TYPE_NAME(std::string, "string");
|
||||
DMLC_DECLARE_TYPE_NAME(bool, "boolean");
|
||||
|
||||
template<typename Then, typename Else>
|
||||
struct IfThenElseType<true, Then, Else> {
|
||||
typedef Then Type;
|
||||
};
|
||||
|
||||
template<typename Then, typename Else>
|
||||
struct IfThenElseType<false, Then, Else> {
|
||||
typedef Else Type;
|
||||
};
|
||||
//! \endcond
|
||||
} // namespace dmlc
|
||||
#endif // DMLC_TYPE_TRAITS_H_
|
||||
@ -9,7 +9,7 @@
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include "./internal/utils.h"
|
||||
#include "../dmlc/io.h"
|
||||
#include "../../dmlc-core/include/dmlc/io.h"
|
||||
|
||||
namespace rabit {
|
||||
/*!
|
||||
|
||||
@ -25,3 +25,9 @@ if [ ${TASK} == "test" ]; then
|
||||
../scripts/travis_runtest.sh || exit -1
|
||||
fi
|
||||
|
||||
if [ ${TASK} == "cmake-build" ]; then
|
||||
mkdir build
|
||||
cd build
|
||||
cmake ..
|
||||
make all || exit -1
|
||||
fi
|
||||
@ -50,12 +50,13 @@ void AllreduceRobust::Shutdown(void) {
|
||||
// execute check ack step, load happens here
|
||||
utils::Assert(RecoverExec(NULL, 0, ActionSummary::kCheckAck, ActionSummary::kSpecialOp),
|
||||
"Shutdown: check ack must return true");
|
||||
#ifdef __APPLE__
|
||||
// In OSX, one worker shutdowns and closes sockets while rest still run kCheckAck
|
||||
// This cause rest workers checkandrecover and hang inf, https://github.com/dmlc/xgboost/pull/3818
|
||||
|
||||
// one worker shutdowns and closes sockets while rest still run kCheckAck,
|
||||
// seems has something to do with time-wait state in tcp connection,
|
||||
// this cause rest workers checkandrecover and hang inf,
|
||||
// https://github.com/dmlc/xgboost/pull/3818
|
||||
// TODO(Chen Qin): a fundamental fix for this
|
||||
sleep(2);
|
||||
#endif // __APPLE__
|
||||
sleep(1);
|
||||
AllreduceBase::Shutdown();
|
||||
}
|
||||
/*!
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
export MPICXX = mpicxx
|
||||
export LDFLAGS= -L../lib -pthread -lm
|
||||
export CFLAGS = -Wall -O3 -msse2 -Wno-unknown-pragmas -fPIC -I../include -std=c++11
|
||||
export CFLAGS = -Wall -O3 -msse2 -Wno-unknown-pragmas -fPIC -I../include -I ../dmlc-core/include -std=c++11
|
||||
|
||||
OS := $(shell uname)
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user