Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

lint and travis

Browse Source
This commit is contained in:
tqchen 2015-07-03 15:15:11 -07:00
parent ceedf4ea96
commit 3cc49ad0e8
27 changed files with 423 additions and 296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -34,3 +34,4 @@
*tmp* *tmp*
*.rabit *.rabit
*.mock *.mock
dmlc-core

48
.travis.yml Normal file
View File

@ -0,0 +1,48 @@
# disable sudo to use container based build
sudo: false
# Use Build Matrix to do lint and build seperately
env:
matrix:
- TASK=lint LINT_LANG=cpp
- TASK=lint LINT_LANG=python
- TASK=doc
- TASK=build CXX=g++
# dependent apt packages
addons:
apt:
packages:
- doxygen
- wget
- git
- libcurl4-openssl-dev
- unzip
before_install:
- git clone https://github.com/dmlc/dmlc-core
- export TRAVIS=dmlc-core/scripts/travis/
- source ${TRAVIS}/travis_setup_env.sh
install:
- pip install cpplint pylint --user `whoami`
script: scripts/travis_script.sh
before_cache:
- ${TRAVIS}/travis_before_cache.sh
cache:
directories:
- ${HOME}/.cache/usr
notifications:
# Emails are sent to the committer's git-configured email address by default,
email:
on_success: change
on_failure: always

14
Makefile
View File

@ -13,6 +13,10 @@ ifeq ($(WITH_FPIC), 1)
CFLAGS += -fPIC CFLAGS += -fPIC
endif endif
ifndef LINT_LANG
LINT_LANG="all"
endif
# build path # build path
BPATH=. BPATH=.
# objectives that makes up rabit library # objectives that makes up rabit library
@ -22,7 +26,9 @@ OBJ= $(BPATH)/allreduce_base.o $(BPATH)/allreduce_robust.o $(BPATH)/engine.o $(B
SLIB= wrapper/librabit_wrapper.so wrapper/librabit_wrapper_mock.so wrapper/librabit_wrapper_mpi.so SLIB= wrapper/librabit_wrapper.so wrapper/librabit_wrapper_mock.so wrapper/librabit_wrapper_mpi.so
ALIB= lib/librabit.a lib/librabit_mpi.a lib/librabit_empty.a lib/librabit_mock.a lib/librabit_base.a ALIB= lib/librabit.a lib/librabit_mpi.a lib/librabit_empty.a lib/librabit_mock.a lib/librabit_base.a
HEADERS=src/*.h include/*.h include/rabit/*.h HEADERS=src/*.h include/*.h include/rabit/*.h
.PHONY: clean all install mpi python DMLC=dmlc-core
.PHONY: clean all install mpi python lint doc
all: lib/librabit.a lib/librabit_mock.a wrapper/librabit_wrapper.so wrapper/librabit_wrapper_mock.so lib/librabit_base.a all: lib/librabit.a lib/librabit_mock.a wrapper/librabit_wrapper.so wrapper/librabit_wrapper_mock.so lib/librabit_base.a
mpi: lib/librabit_mpi.a wrapper/librabit_wrapper_mpi.so mpi: lib/librabit_mpi.a wrapper/librabit_wrapper_mpi.so
@ -59,6 +65,12 @@ $(ALIB):
$(SLIB) : $(SLIB) :
$(CXX) $(CFLAGS) -shared -o $@ $(filter %.cpp %.o %.c %.cc %.a, $^) $(LDFLAGS) $(CXX) $(CFLAGS) -shared -o $@ $(filter %.cpp %.o %.c %.cc %.a, $^) $(LDFLAGS)
lint:
$(DMLC)/scripts/lint.py rabit $(LINT_LANG) src include wrapper
doc:
cd include; doxygen ../doc/Doxyfile; cd -
clean: clean:
$(RM) $(OBJ) $(MPIOBJ) $(ALIB) $(MPIALIB) *~ src/*~ include/*~ include/*/*~ wrapper/*~ $(RM) $(OBJ) $(MPIOBJ) $(ALIB) $(MPIALIB) *~ src/*~ include/*~ include/*/*~ wrapper/*~

10
include/dmlc/io.h
View File

@ -14,6 +14,7 @@
// include uint64_t only to make io standalone // include uint64_t only to make io standalone
#ifdef _MSC_VER #ifdef _MSC_VER
/*! \brief uint64 */
typedef unsigned __int64 uint64_t; typedef unsigned __int64 uint64_t;
#else #else
#include <inttypes.h> #include <inttypes.h>
@ -24,7 +25,7 @@ namespace dmlc {
/*! /*!
* \brief interface of stream I/O for serialization * \brief interface of stream I/O for serialization
*/ */
class Stream { class Stream { // NOLINT(*)
public: public:
/*! /*!
* \brief reads data from a stream * \brief reads data from a stream
@ -192,9 +193,10 @@ class InputSplit {
* List of possible types: "text", "recordio" * List of possible types: "text", "recordio"
* - "text": * - "text":
* text file, each line is treated as a record * text file, each line is treated as a record
* input split will split on \n or \r * input split will split on '\\n' or '\\r'
* - "recordio": * - "recordio":
* binary recordio file, see recordio.h * binary recordio file, see recordio.h
* \return a new input split
* \sa InputSplit::Type * \sa InputSplit::Type
*/ */
static InputSplit* Create(const char *uri, static InputSplit* Create(const char *uri,
@ -224,7 +226,7 @@ class ostream : public std::basic_ostream<char> {
* \param buffer_size internal streambuf size * \param buffer_size internal streambuf size
*/ */
explicit ostream(Stream *stream, explicit ostream(Stream *stream,
size_t buffer_size = 1 << 10) size_t buffer_size = (1 << 10))
: std::basic_ostream<char>(NULL), buf_(buffer_size) { : std::basic_ostream<char>(NULL), buf_(buffer_size) {
this->set_stream(stream); this->set_stream(stream);
} }
@ -287,7 +289,7 @@ class istream : public std::basic_istream<char> {
* \param buffer_size internal buffer size * \param buffer_size internal buffer size
*/ */
explicit istream(Stream *stream, explicit istream(Stream *stream,
size_t buffer_size = 1 << 10) size_t buffer_size = (1 << 10))
: std::basic_istream<char>(NULL), buf_(buffer_size) { : std::basic_istream<char>(NULL), buf_(buffer_size) {
this->set_stream(stream); this->set_stream(stream);
} }

22
include/rabit.h
View File

@ -8,12 +8,18 @@
* rabit.h and serializable.h is all what the user needs to use the rabit interface * rabit.h and serializable.h is all what the user needs to use the rabit interface
* \author Tianqi Chen, Ignacio Cano, Tianyi Zhou * \author Tianqi Chen, Ignacio Cano, Tianyi Zhou
*/ */
#ifndef RABIT_RABIT_H_ #ifndef RABIT_RABIT_H_ // NOLINT(*)
#define RABIT_RABIT_H_ #define RABIT_RABIT_H_ // NOLINT(*)
#include <string> #include <string>
#include <vector> #include <vector>
// whether or not use c++11 support
#ifndef DMLC_USE_CXX11
#define DMLC_USE_CXX11 (defined(__GXX_EXPERIMENTAL_CXX0X__) ||\
__cplusplus >= 201103L || defined(_MSC_VER))
#endif
// optionally support of lambda functions in C++11, if available // optionally support of lambda functions in C++11, if available
#if __cplusplus >= 201103L #if DMLC_USE_CXX11
#include <functional> #include <functional>
#endif // C++11 #endif // C++11
// contains definition of Serializable // contains definition of Serializable
@ -135,7 +141,7 @@ inline void Allreduce(DType *sendrecvbuf, size_t count,
void (*prepare_fun)(void *arg) = NULL, void (*prepare_fun)(void *arg) = NULL,
void *prepare_arg = NULL); void *prepare_arg = NULL);
// C++11 support for lambda prepare function // C++11 support for lambda prepare function
#if __cplusplus >= 201103L #if DMLC_USE_CXX11
/*! /*!
* \brief performs in-place Allreduce, on sendrecvbuf * \brief performs in-place Allreduce, on sendrecvbuf
* with a prepare function specified by a lambda function * with a prepare function specified by a lambda function
@ -242,7 +248,7 @@ class ReduceHandle;
* \tparam freduce the customized reduction function * \tparam freduce the customized reduction function
* DType must be a struct, with no pointer * DType must be a struct, with no pointer
*/ */
template<typename DType, void (*freduce)(DType &dst, const DType &src)> template<typename DType, void (*freduce)(DType &dst, const DType &src)> // NOLINT(*)
class Reducer { class Reducer {
public: public:
Reducer(void); Reducer(void);
@ -258,7 +264,7 @@ class Reducer {
inline void Allreduce(DType *sendrecvbuf, size_t count, inline void Allreduce(DType *sendrecvbuf, size_t count,
void (*prepare_fun)(void *arg) = NULL, void (*prepare_fun)(void *arg) = NULL,
void *prepare_arg = NULL); void *prepare_arg = NULL);
#if __cplusplus >= 201103L #if DMLC_USE_CXX11
/*! /*!
* \brief customized in-place all reduce operation, with lambda function as preprocessor * \brief customized in-place all reduce operation, with lambda function as preprocessor
* \param sendrecvbuf pointer to the array of objects to be reduced * \param sendrecvbuf pointer to the array of objects to be reduced
@ -303,7 +309,7 @@ class SerializeReducer {
void (*prepare_fun)(void *arg) = NULL, void (*prepare_fun)(void *arg) = NULL,
void *prepare_arg = NULL); void *prepare_arg = NULL);
// C++11 support for lambda prepare function // C++11 support for lambda prepare function
#if __cplusplus >= 201103L #if DMLC_USE_CXX11
/*! /*!
* \brief customized in-place all reduce operation, with lambda function as preprocessor * \brief customized in-place all reduce operation, with lambda function as preprocessor
* \param sendrecvobj pointer to the array of objects to be reduced * \param sendrecvobj pointer to the array of objects to be reduced
@ -326,4 +332,4 @@ class SerializeReducer {
} // namespace rabit } // namespace rabit
// implementation of template functions // implementation of template functions
#include "./rabit/rabit-inl.h" #include "./rabit/rabit-inl.h"
#endif // RABIT_RABIT_H_ #endif // RABIT_RABIT_H_ // NOLINT(*)

8
include/rabit/io.h
View File

@ -4,8 +4,8 @@
* \brief utilities with different serializable implementations * \brief utilities with different serializable implementations
* \author Tianqi Chen * \author Tianqi Chen
*/ */
#ifndef RABIT_UTILS_IO_H_ #ifndef RABIT_IO_H_
#define RABIT_UTILS_IO_H_ #define RABIT_IO_H_
#include <cstdio> #include <cstdio>
#include <vector> #include <vector>
#include <cstring> #include <cstring>
@ -51,6 +51,7 @@ struct MemoryFixSizeBuffer : public SeekStream {
virtual bool AtEnd(void) const { virtual bool AtEnd(void) const {
return curr_ptr_ == buffer_size_; return curr_ptr_ == buffer_size_;
} }
private: private:
/*! \brief in memory buffer */ /*! \brief in memory buffer */
char *p_buffer_; char *p_buffer_;
@ -93,6 +94,7 @@ struct MemoryBufferStream : public SeekStream {
virtual bool AtEnd(void) const { virtual bool AtEnd(void) const {
return curr_ptr_ == p_buffer_->length(); return curr_ptr_ == p_buffer_->length();
} }
private: private:
/*! \brief in memory buffer */ /*! \brief in memory buffer */
std::string *p_buffer_; std::string *p_buffer_;
@ -101,4 +103,4 @@ struct MemoryBufferStream : public SeekStream {
}; // class MemoryBufferStream }; // class MemoryBufferStream
} // namespace utils } // namespace utils
} // namespace rabit } // namespace rabit
#endif // RABIT_UTILS_IO_H_ #endif // RABIT_IO_H_

71
include/rabit/rabit-inl.h
View File

@ -1,12 +1,15 @@
/*! /*!
* Copyright by Contributors
* \file rabit-inl.h * \file rabit-inl.h
* \brief implementation of inline template function for rabit interface * \brief implementation of inline template function for rabit interface
* *
* \author Tianqi Chen * \author Tianqi Chen
*/ */
#ifndef RABIT_RABIT_INL_H #ifndef RABIT_RABIT_INL_H_
#define RABIT_RABIT_INL_H #define RABIT_RABIT_INL_H_
// use engine for implementation // use engine for implementation
#include <vector>
#include <string>
#include "./io.h" #include "./io.h"
#include "./utils.h" #include "./utils.h"
#include "../rabit.h" #include "../rabit.h"
@ -30,15 +33,15 @@ inline DataType GetType<int>(void) {
return kInt; return kInt;
} }
template<> template<>
inline DataType GetType<unsigned int>(void) { inline DataType GetType<unsigned int>(void) { // NOLINT(*)
return kUInt; return kUInt;
} }
template<> template<>
inline DataType GetType<long>(void) { inline DataType GetType<long>(void) { // NOLINT(*)
return kLong; return kLong;
} }
template<> template<>
inline DataType GetType<unsigned long>(void) { inline DataType GetType<unsigned long>(void) { // NOLINT(*)
return kULong; return kULong;
} }
template<> template<>
@ -50,54 +53,54 @@ inline DataType GetType<double>(void) {
return kDouble; return kDouble;
} }
template<> template<>
inline DataType GetType<long long>(void) { inline DataType GetType<long long>(void) { // NOLINT(*)
return kLongLong; return kLongLong;
} }
template<> template<>
inline DataType GetType<unsigned long long>(void) { inline DataType GetType<unsigned long long>(void) { // NOLINT(*)
return kULongLong; return kULongLong;
} }
} // namespace mpi } // namespace mpi
} // namespace engine } // namespace engine
namespace op { namespace op {
struct Max { struct Max {
const static engine::mpi::OpType kType = engine::mpi::kMax; static const engine::mpi::OpType kType = engine::mpi::kMax;
template<typename DType> template<typename DType>
inline static void Reduce(DType &dst, const DType &src) { inline static void Reduce(DType &dst, const DType &src) { // NOLINT(*)
if (dst < src) dst = src; if (dst < src) dst = src;
} }
}; };
struct Min { struct Min {
const static engine::mpi::OpType kType = engine::mpi::kMin; static const engine::mpi::OpType kType = engine::mpi::kMin;
template<typename DType> template<typename DType>
inline static void Reduce(DType &dst, const DType &src) { inline static void Reduce(DType &dst, const DType &src) { // NOLINT(*)
if (dst > src) dst = src; if (dst > src) dst = src;
} }
}; };
struct Sum { struct Sum {
const static engine::mpi::OpType kType = engine::mpi::kSum; static const engine::mpi::OpType kType = engine::mpi::kSum;
template<typename DType> template<typename DType>
inline static void Reduce(DType &dst, const DType &src) { inline static void Reduce(DType &dst, const DType &src) { // NOLINT(*)
dst += src; dst += src;
} }
}; };
struct BitOR { struct BitOR {
const static engine::mpi::OpType kType = engine::mpi::kBitwiseOR; static const engine::mpi::OpType kType = engine::mpi::kBitwiseOR;
template<typename DType> template<typename DType>
inline static void Reduce(DType &dst, const DType &src) { inline static void Reduce(DType &dst, const DType &src) { // NOLINT(*)
dst |= src; dst |= src;
} }
}; };
template<typename OP, typename DType> template<typename OP, typename DType>
inline void Reducer(const void *src_, void *dst_, int len, const MPI::Datatype &dtype) { inline void Reducer(const void *src_, void *dst_, int len, const MPI::Datatype &dtype) {
const DType *src = (const DType*)src_; const DType *src = (const DType*)src_;
DType *dst = (DType*)dst_; DType *dst = (DType*)dst_; // NOLINT(*)
for (int i = 0; i < len; ++i) { for (int i = 0; i < len; ++i) {
OP::Reduce(dst[i], src[i]); OP::Reduce(dst[i], src[i]);
} }
} }
} // namespace op } // namespace op
// intialize the rabit engine // intialize the rabit engine
inline void Init(int argc, char *argv[]) { inline void Init(int argc, char *argv[]) {
@ -154,21 +157,21 @@ template<typename OP, typename DType>
inline void Allreduce(DType *sendrecvbuf, size_t count, inline void Allreduce(DType *sendrecvbuf, size_t count,
void (*prepare_fun)(void *arg), void (*prepare_fun)(void *arg),
void *prepare_arg) { void *prepare_arg) {
engine::Allreduce_(sendrecvbuf, sizeof(DType), count, op::Reducer<OP,DType>, engine::Allreduce_(sendrecvbuf, sizeof(DType), count, op::Reducer<OP, DType>,
engine::mpi::GetType<DType>(), OP::kType, prepare_fun, prepare_arg); engine::mpi::GetType<DType>(), OP::kType, prepare_fun, prepare_arg);
} }
// C++11 support for lambda prepare function // C++11 support for lambda prepare function
#if __cplusplus >= 201103L #if DMLC_USE_CXX11
inline void InvokeLambda_(void *fun) { inline void InvokeLambda_(void *fun) {
(*static_cast<std::function<void()>*>(fun))(); (*static_cast<std::function<void()>*>(fun))();
} }
template<typename OP, typename DType> template<typename OP, typename DType>
inline void Allreduce(DType *sendrecvbuf, size_t count, std::function<void()> prepare_fun) { inline void Allreduce(DType *sendrecvbuf, size_t count, std::function<void()> prepare_fun) {
engine::Allreduce_(sendrecvbuf, sizeof(DType), count, op::Reducer<OP,DType>, engine::Allreduce_(sendrecvbuf, sizeof(DType), count, op::Reducer<OP, DType>,
engine::mpi::GetType<DType>(), OP::kType, InvokeLambda_, &prepare_fun); engine::mpi::GetType<DType>(), OP::kType, InvokeLambda_, &prepare_fun);
} }
#endif // C++11 #endif // C++11
// print message to the tracker // print message to the tracker
inline void TrackerPrint(const std::string &msg) { inline void TrackerPrint(const std::string &msg) {
@ -223,15 +226,16 @@ inline void ReducerSafe_(const void *src_, void *dst_, int len_, const MPI::Data
} }
} }
// function to perform reduction for Reducer // function to perform reduction for Reducer
template<typename DType, void (*freduce)(DType &dst, const DType &src)> template<typename DType, void (*freduce)(DType &dst, const DType &src)> // NOLINT(*)
inline void ReducerAlign_(const void *src_, void *dst_, int len_, const MPI::Datatype &dtype) { inline void ReducerAlign_(const void *src_, void *dst_,
int len_, const MPI::Datatype &dtype) {
const DType *psrc = reinterpret_cast<const DType*>(src_); const DType *psrc = reinterpret_cast<const DType*>(src_);
DType *pdst = reinterpret_cast<DType*>(dst_); DType *pdst = reinterpret_cast<DType*>(dst_);
for (int i = 0; i < len_; ++i) { for (int i = 0; i < len_; ++i) {
freduce(pdst[i], psrc[i]); freduce(pdst[i], psrc[i]);
} }
} }
template<typename DType, void (*freduce)(DType &dst, const DType &src)> template<typename DType, void (*freduce)(DType &dst, const DType &src)> // NOLINT(*)
inline Reducer<DType, freduce>::Reducer(void) { inline Reducer<DType, freduce>::Reducer(void) {
// it is safe to directly use handle for aligned data types // it is safe to directly use handle for aligned data types
if (sizeof(DType) == 8 || sizeof(DType) == 4 || sizeof(DType) == 1) { if (sizeof(DType) == 8 || sizeof(DType) == 4 || sizeof(DType) == 1) {
@ -240,7 +244,7 @@ inline Reducer<DType, freduce>::Reducer(void) {
this->handle_.Init(ReducerSafe_<DType, freduce>, sizeof(DType)); this->handle_.Init(ReducerSafe_<DType, freduce>, sizeof(DType));
} }
} }
template<typename DType, void (*freduce)(DType &dst, const DType &src)> template<typename DType, void (*freduce)(DType &dst, const DType &src)> // NOLINT(*)
inline void Reducer<DType, freduce>::Allreduce(DType *sendrecvbuf, size_t count, inline void Reducer<DType, freduce>::Allreduce(DType *sendrecvbuf, size_t count,
void (*prepare_fun)(void *arg), void (*prepare_fun)(void *arg),
void *prepare_arg) { void *prepare_arg) {
@ -248,13 +252,14 @@ inline void Reducer<DType, freduce>::Allreduce(DType *sendrecvbuf, size_t count,
} }
// function to perform reduction for SerializeReducer // function to perform reduction for SerializeReducer
template<typename DType> template<typename DType>
inline void SerializeReducerFunc_(const void *src_, void *dst_, int len_, const MPI::Datatype &dtype) { inline void SerializeReducerFunc_(const void *src_, void *dst_,
int len_, const MPI::Datatype &dtype) {
int nbytes = engine::ReduceHandle::TypeSize(dtype); int nbytes = engine::ReduceHandle::TypeSize(dtype);
// temp space // temp space
DType tsrc, tdst; DType tsrc, tdst;
for (int i = 0; i < len_; ++i) { for (int i = 0; i < len_; ++i) {
utils::MemoryFixSizeBuffer fsrc((char*)(src_) + i * nbytes, nbytes); utils::MemoryFixSizeBuffer fsrc((char*)(src_) + i * nbytes, nbytes); // NOLINT(*)
utils::MemoryFixSizeBuffer fdst((char*)(dst_) + i * nbytes, nbytes); utils::MemoryFixSizeBuffer fdst((char*)(dst_) + i * nbytes, nbytes); // NOLINT(*)
tsrc.Load(fsrc); tsrc.Load(fsrc);
tdst.Load(fdst); tdst.Load(fdst);
// govern const check // govern const check
@ -306,8 +311,8 @@ inline void SerializeReducer<DType>::Allreduce(DType *sendrecvobj,
} }
} }
#if __cplusplus >= 201103L #if DMLC_USE_CXX11
template<typename DType, void (*freduce)(DType &dst, const DType &src)> template<typename DType, void (*freduce)(DType &dst, const DType &src)> // NOLINT(*)g
inline void Reducer<DType, freduce>::Allreduce(DType *sendrecvbuf, size_t count, inline void Reducer<DType, freduce>::Allreduce(DType *sendrecvbuf, size_t count,
std::function<void()> prepare_fun) { std::function<void()> prepare_fun) {
this->Allreduce(sendrecvbuf, count, InvokeLambda_, &prepare_fun); this->Allreduce(sendrecvbuf, count, InvokeLambda_, &prepare_fun);
@ -320,4 +325,4 @@ inline void SerializeReducer<DType>::Allreduce(DType *sendrecvobj,
} }
#endif #endif
} // namespace rabit } // namespace rabit
#endif #endif // RABIT_RABIT_INL_H_

3
include/rabit/timer.h
View File

@ -1,4 +1,5 @@
/*! /*!
* Copyright by Contributors
* \file timer.h * \file timer.h
* \brief This file defines the utils for timing * \brief This file defines the utils for timing
* \author Tianqi Chen, Nacho, Tianyi * \author Tianqi Chen, Nacho, Tianyi
@ -18,7 +19,6 @@ namespace utils {
* \brief return time in seconds, not cross platform, avoid to use this in most places * \brief return time in seconds, not cross platform, avoid to use this in most places
*/ */
inline double GetTime(void) { inline double GetTime(void) {
// TODO: use c++11 chrono when c++11 was available
#ifdef __MACH__ #ifdef __MACH__
clock_serv_t cclock; clock_serv_t cclock;
mach_timespec_t mts; mach_timespec_t mts;
@ -32,7 +32,6 @@ inline double GetTime(void) {
utils::Check(clock_gettime(CLOCK_REALTIME, &ts) == 0, "failed to get time"); utils::Check(clock_gettime(CLOCK_REALTIME, &ts) == 0, "failed to get time");
return static_cast<double>(ts.tv_sec) + static_cast<double>(ts.tv_nsec) * 1e-9; return static_cast<double>(ts.tv_sec) + static_cast<double>(ts.tv_nsec) * 1e-9;
#else #else
// TODO: add MSVC macro, and MSVC timer
return static_cast<double>(time(NULL)); return static_cast<double>(time(NULL));
#endif #endif
#endif #endif

8
include/rabit/utils.h
View File

@ -27,7 +27,7 @@
#else #else
#ifdef _FILE_OFFSET_BITS #ifdef _FILE_OFFSET_BITS
#if _FILE_OFFSET_BITS == 32 #if _FILE_OFFSET_BITS == 32
#pragma message ("Warning: FILE OFFSET BITS defined to be 32 bit") #pragma message("Warning: FILE OFFSET BITS defined to be 32 bit")
#endif #endif
#endif #endif
@ -163,7 +163,7 @@ inline std::FILE *FopenCheck(const char *fname, const char *flag) {
// easy utils that can be directly accessed in xgboost // easy utils that can be directly accessed in xgboost
/*! \brief get the beginning address of a vector */ /*! \brief get the beginning address of a vector */
template<typename T> template<typename T>
inline T *BeginPtr(std::vector<T> &vec) { inline T *BeginPtr(std::vector<T> &vec) { // NOLINT(*)
if (vec.size() == 0) { if (vec.size() == 0) {
return NULL; return NULL;
} else { } else {
@ -172,14 +172,14 @@ inline T *BeginPtr(std::vector<T> &vec) {
} }
/*! \brief get the beginning address of a vector */ /*! \brief get the beginning address of a vector */
template<typename T> template<typename T>
inline const T *BeginPtr(const std::vector<T> &vec) { inline const T *BeginPtr(const std::vector<T> &vec) { // NOLINT(*)
if (vec.size() == 0) { if (vec.size() == 0) {
return NULL; return NULL;
} else { } else {
return &vec[0]; return &vec[0];
} }
} }
inline char* BeginPtr(std::string &str) { inline char* BeginPtr(std::string &str) { // NOLINT(*)
if (str.length() == 0) return NULL; if (str.length() == 0) return NULL;
return &str[0]; return &str[0];
} }

6
include/rabit_serializable.h
View File

@ -4,8 +4,8 @@
* \brief defines serializable interface of rabit * \brief defines serializable interface of rabit
* \author Tianqi Chen * \author Tianqi Chen
*/ */
#ifndef RABIT_RABIT_SERIALIZABLE_H_ #ifndef RABIT_SERIALIZABLE_H_
#define RABIT_RABIT_SERIALIZABLE_H_ #define RABIT_SERIALIZABLE_H_
#include <vector> #include <vector>
#include <string> #include <string>
#include "./rabit/utils.h" #include "./rabit/utils.h"
@ -24,4 +24,4 @@ typedef dmlc::Stream Stream;
typedef dmlc::Serializable Serializable; typedef dmlc::Serializable Serializable;
} // namespace rabit } // namespace rabit
#endif // RABIT_RABIT_SERIALIZABLE_H_ #endif // RABIT_SERIALIZABLE_H_

8
scripts/travis_runtest.sh Executable file
View File

@ -0,0 +1,8 @@
#!/bin/bash
make -f test.mk model_recover_10_10k || exit -1
make -f test.mk model_recover_10_10k_die_same || exit -1
make -f test.mk local_recover_10_10k || exit -1
make -f test.mk pylocal_recover_10_10k || exit -1
make -f test.mk lazy_recover_10_10k_die_hard || exit -1
make -f test.mk lazy_recover_10_10k_die_same || exit -1
make -f test.mk ringallreduce_10_10k || exit -1

22
scripts/travis_script.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
# main script of travis
if [ ${TASK} == "lint" ]; then
make lint || exit -1
fi
if [ ${TASK} == "doc" ]; then
make doc 2>log.txt
(cat log.txt|grep warning) && exit -1
fi
if [ ${TASK} == "build" ]; then
make all || exit -1
fi
if [ ${TASK} == "test" ]; then
cd test
make all || exit -1
./travis_runtest.sh || exit -1
fi

9
src/allreduce_base.cc
View File

@ -94,7 +94,8 @@ void AllreduceBase::Init(void) {
} }
} }
if (dmlc_role != "worker") { if (dmlc_role != "worker") {
fprintf(stderr, "Rabit Module currently only work with dmlc worker, quit this program by exit 0\n"); fprintf(stderr, "Rabit Module currently only work with dmlc worker"\
", quit this program by exit 0\n");
exit(0); exit(0);
} }
// clear the setting before start reconnection // clear the setting before start reconnection
@ -134,7 +135,7 @@ void AllreduceBase::TrackerPrint(const std::string &msg) {
// util to parse data with unit suffix // util to parse data with unit suffix
inline size_t ParseUnit(const char *name, const char *val) { inline size_t ParseUnit(const char *name, const char *val) {
char unit; char unit;
unsigned long amt; unsigned long amt; // NOLINT(*)
int n = sscanf(val, "%lu%c", &amt, &unit); int n = sscanf(val, "%lu%c", &amt, &unit);
size_t amount = amt; size_t amount = amt;
if (n == 2) { if (n == 2) {
@ -440,7 +441,7 @@ AllreduceBase::TryAllreduceTree(void *sendrecvbuf_,
selecter.WatchRead(links[i].sock); selecter.WatchRead(links[i].sock);
} }
// size_write <= size_read // size_write <= size_read
if (links[i].size_write != total_size){ if (links[i].size_write != total_size) {
if (links[i].size_write < size_down_in) { if (links[i].size_write < size_down_in) {
selecter.WatchWrite(links[i].sock); selecter.WatchWrite(links[i].sock);
} }
@ -477,7 +478,7 @@ AllreduceBase::TryAllreduceTree(void *sendrecvbuf_,
size_t max_reduce = total_size; size_t max_reduce = total_size;
for (int i = 0; i < nlink; ++i) { for (int i = 0; i < nlink; ++i) {
if (i != parent_index) { if (i != parent_index) {
max_reduce= std::min(max_reduce, links[i].size_read); max_reduce = std::min(max_reduce, links[i].size_read);
utils::Assert(buffer_size == 0 || buffer_size == links[i].buffer_size, utils::Assert(buffer_size == 0 || buffer_size == links[i].buffer_size,
"buffer size inconsistent"); "buffer size inconsistent");
buffer_size = links[i].buffer_size; buffer_size = links[i].buffer_size;

10
src/allreduce_base.h
View File

@ -225,7 +225,7 @@ class AllreduceBase : public IEngine {
ReturnTypeEnum value; ReturnTypeEnum value;
// constructor // constructor
ReturnType() {} ReturnType() {}
ReturnType(ReturnTypeEnum value) : value(value){} ReturnType(ReturnTypeEnum value) : value(value) {} // NOLINT(*)
inline bool operator==(const ReturnTypeEnum &v) const { inline bool operator==(const ReturnTypeEnum &v) const {
return value == v; return value == v;
} }
@ -235,11 +235,11 @@ class AllreduceBase : public IEngine {
}; };
/*! \brief translate errno to return type */ /*! \brief translate errno to return type */
inline static ReturnType Errno2Return() { inline static ReturnType Errno2Return() {
int errsv = utils::Socket::GetLastError(); int errsv = utils::Socket::GetLastError();
if (errsv == EAGAIN || errsv == EWOULDBLOCK || errsv == 0) return kSuccess; if (errsv == EAGAIN || errsv == EWOULDBLOCK || errsv == 0) return kSuccess;
#ifdef _WIN32 #ifdef _WIN32
if (errsv == WSAEWOULDBLOCK) return kSuccess; if (errsv == WSAEWOULDBLOCK) return kSuccess;
if (errsv == WSAECONNRESET) return kConnReset; if (errsv == WSAECONNRESET) return kConnReset;
#endif #endif
if (errsv == ECONNRESET) return kConnReset; if (errsv == ECONNRESET) return kConnReset;
return kSockError; return kSockError;
@ -522,4 +522,4 @@ class AllreduceBase : public IEngine {
}; };
} // namespace engine } // namespace engine
} // namespace rabit } // namespace rabit
#endif // RABIT_ALLREDUCE_BASE_H #endif // RABIT_ALLREDUCE_BASE_H_

4
src/allreduce_mock.h
View File

@ -1,4 +1,5 @@
/*! /*!
* Copyright by Contributors
* \file allreduce_mock.h * \file allreduce_mock.h
* \brief Mock test module of AllReduce engine, * \brief Mock test module of AllReduce engine,
* insert failures in certain call point, to test if the engine is robust to failure * insert failures in certain call point, to test if the engine is robust to failure
@ -100,6 +101,7 @@ class AllreduceMock : public AllreduceRobust {
this->Verify(MockKey(rank, version_number, seq_counter, num_trial), "LazyCheckPoint"); this->Verify(MockKey(rank, version_number, seq_counter, num_trial), "LazyCheckPoint");
AllreduceRobust::LazyCheckPoint(global_model); AllreduceRobust::LazyCheckPoint(global_model);
} }
protected: protected:
// force checkpoint to local // force checkpoint to local
int force_local; int force_local;
@ -173,4 +175,4 @@ class AllreduceMock : public AllreduceRobust {
}; };
} // namespace engine } // namespace engine
} // namespace rabit } // namespace rabit
#endif // RABIT_ALLREDUCE_MOCK_H_ #endif // RABIT_ALLREDUCE_MOCK_H_

20
src/allreduce_robust-inl.h
View File

@ -5,8 +5,8 @@
* *
* \author Tianqi Chen * \author Tianqi Chen
*/ */
#ifndef RABIT_ENGINE_ROBUST_INL_H_ #ifndef RABIT_ALLREDUCE_ROBUST_INL_H_
#define RABIT_ENGINE_ROBUST_INL_H_ #define RABIT_ALLREDUCE_ROBUST_INL_H_
#include <vector> #include <vector>
namespace rabit { namespace rabit {
@ -35,7 +35,7 @@ inline AllreduceRobust::ReturnType
AllreduceRobust::MsgPassing(const NodeType &node_value, AllreduceRobust::MsgPassing(const NodeType &node_value,
std::vector<EdgeType> *p_edge_in, std::vector<EdgeType> *p_edge_in,
std::vector<EdgeType> *p_edge_out, std::vector<EdgeType> *p_edge_out,
EdgeType (*func) EdgeType(*func)
(const NodeType &node_value, (const NodeType &node_value,
const std::vector<EdgeType> &edge_in, const std::vector<EdgeType> &edge_in,
size_t out_index)) { size_t out_index)) {
@ -80,8 +80,16 @@ AllreduceRobust::MsgPassing(const NodeType &node_value,
selecter.WatchRead(links[i].sock); selecter.WatchRead(links[i].sock);
} }
break; break;
case 1: if (i == parent_index) selecter.WatchWrite(links[i].sock); break; case 1:
case 2: if (i == parent_index) selecter.WatchRead(links[i].sock); break; if (i == parent_index) {
selecter.WatchWrite(links[i].sock);
}
break;
case 2:
if (i == parent_index) {
selecter.WatchRead(links[i].sock);
}
break;
case 3: case 3:
if (i != parent_index && links[i].size_write != sizeof(EdgeType)) { if (i != parent_index && links[i].size_write != sizeof(EdgeType)) {
selecter.WatchWrite(links[i].sock); selecter.WatchWrite(links[i].sock);
@ -158,4 +166,4 @@ AllreduceRobust::MsgPassing(const NodeType &node_value,
} }
} // namespace engine } // namespace engine
} // namespace rabit } // namespace rabit
#endif // RABIT_ENGINE_ROBUST_INL_H_ #endif // RABIT_ALLREDUCE_ROBUST_INL_H_

2
src/allreduce_robust.cc
View File

@ -224,7 +224,7 @@ void AllreduceRobust::LocalModelCheck(bool with_local) {
num_local_replica = 0; num_local_replica = 0;
} }
} else { } else {
utils::Check(use_local_model == int(with_local), utils::Check(use_local_model == static_cast<int>(with_local),
"Can only call Checkpoint/LoadCheckPoint always with"\ "Can only call Checkpoint/LoadCheckPoint always with"\
"or without local_model, but not mixed case"); "or without local_model, but not mixed case");
} }

3
src/allreduce_robust.h
View File

@ -287,6 +287,7 @@ class AllreduceRobust : public AllreduceBase {
if (seqno_.size() == 0) return -1; if (seqno_.size() == 0) return -1;
return seqno_.back(); return seqno_.back();
} }
private: private:
// sequence number of each // sequence number of each
std::vector<int> seqno_; std::vector<int> seqno_;
@ -509,7 +510,7 @@ o * the input state must exactly one saved state(local state of current node)
inline ReturnType MsgPassing(const NodeType &node_value, inline ReturnType MsgPassing(const NodeType &node_value,
std::vector<EdgeType> *p_edge_in, std::vector<EdgeType> *p_edge_in,
std::vector<EdgeType> *p_edge_out, std::vector<EdgeType> *p_edge_out,
EdgeType (*func) EdgeType(*func)
(const NodeType &node_value, (const NodeType &node_value,
const std::vector<EdgeType> &edge_in, const std::vector<EdgeType> &edge_in,
size_t out_index)); size_t out_index));

4
src/engine_mpi.cc
View File

@ -166,7 +166,7 @@ void ReduceHandle::Init(IEngine::ReduceFunction redfunc, size_t type_nbytes) {
if (type_nbytes != 0) { if (type_nbytes != 0) {
MPI::Datatype *dtype = new MPI::Datatype(); MPI::Datatype *dtype = new MPI::Datatype();
if (type_nbytes % 8 == 0) { if (type_nbytes % 8 == 0) {
*dtype = MPI::LONG.Create_contiguous(type_nbytes / sizeof(long)); *dtype = MPI::LONG.Create_contiguous(type_nbytes / sizeof(long)); // NOLINT(*)
} else if (type_nbytes % 4 == 0) { } else if (type_nbytes % 4 == 0) {
*dtype = MPI::INT.Create_contiguous(type_nbytes / sizeof(int)); *dtype = MPI::INT.Create_contiguous(type_nbytes / sizeof(int));
} else { } else {
@ -195,7 +195,7 @@ void ReduceHandle::Allreduce(void *sendrecvbuf,
dtype->Free(); dtype->Free();
} }
if (type_nbytes % 8 == 0) { if (type_nbytes % 8 == 0) {
*dtype = MPI::LONG.Create_contiguous(type_nbytes / sizeof(long)); *dtype = MPI::LONG.Create_contiguous(type_nbytes / sizeof(long)); // NOLINT(*)
} else if (type_nbytes % 4 == 0) { } else if (type_nbytes % 4 == 0) {
*dtype = MPI::INT.Create_contiguous(type_nbytes / sizeof(int)); *dtype = MPI::INT.Create_contiguous(type_nbytes / sizeof(int));
} else { } else {

17
src/socket.h
View File

@ -188,11 +188,11 @@ class Socket {
return port; return port;
} }
#if defined(_WIN32) #if defined(_WIN32)
if (WSAGetLastError() != WSAEADDRINUSE) { if (WSAGetLastError() != WSAEADDRINUSE) {
Socket::Error("TryBindHost"); Socket::Error("TryBindHost");
} }
#else #else
if (errno != EADDRINUSE) { if (errno != EADDRINUSE) {
Socket::Error("TryBindHost"); Socket::Error("TryBindHost");
} }
#endif #endif
@ -264,7 +264,8 @@ class TCPSocket : public Socket{
*/ */
inline void SetKeepAlive(bool keepalive) { inline void SetKeepAlive(bool keepalive) {
int opt = static_cast<int>(keepalive); int opt = static_cast<int>(keepalive);
if (setsockopt(sockfd, SOL_SOCKET, SO_KEEPALIVE, reinterpret_cast<char*>(&opt), sizeof(opt)) < 0) { if (setsockopt(sockfd, SOL_SOCKET, SO_KEEPALIVE,
reinterpret_cast<char*>(&opt), sizeof(opt)) < 0) {
Socket::Error("SetKeepAlive"); Socket::Error("SetKeepAlive");
} }
} }
@ -299,7 +300,7 @@ class TCPSocket : public Socket{
*/ */
inline int AtMark(void) const { inline int AtMark(void) const {
#ifdef _WIN32 #ifdef _WIN32
unsigned long atmark; unsigned long atmark; // NOLINT(*)
if (ioctlsocket(sockfd, SIOCATMARK, &atmark) != NO_ERROR) return -1; if (ioctlsocket(sockfd, SIOCATMARK, &atmark) != NO_ERROR) return -1;
#else #else
int atmark; int atmark;
@ -473,7 +474,7 @@ struct SelectHelper {
* \param timeout the timeout counter, can be 0, which means wait until the event happen * \param timeout the timeout counter, can be 0, which means wait until the event happen
* \return 1 if success, 0 if timeout, and -1 if error occurs * \return 1 if success, 0 if timeout, and -1 if error occurs
*/ */
inline static int WaitExcept(SOCKET fd, long timeout = 0) { inline static int WaitExcept(SOCKET fd, long timeout = 0) { // NOLINT(*)
fd_set wait_set; fd_set wait_set;
FD_ZERO(&wait_set); FD_ZERO(&wait_set);
FD_SET(fd, &wait_set); FD_SET(fd, &wait_set);
@ -489,7 +490,7 @@ struct SelectHelper {
* \return number of active descriptors selected, * \return number of active descriptors selected,
* return -1 if error occurs * return -1 if error occurs
*/ */
inline int Select(long timeout = 0) { inline int Select(long timeout = 0) { // NOLINT(*)
int ret = Select_(static_cast<int>(maxfd + 1), int ret = Select_(static_cast<int>(maxfd + 1),
&read_set, &write_set, &except_set, timeout); &read_set, &write_set, &except_set, timeout);
if (ret == -1) { if (ret == -1) {
@ -500,7 +501,7 @@ struct SelectHelper {
private: private:
inline static int Select_(int maxfd, fd_set *rfds, inline static int Select_(int maxfd, fd_set *rfds,
fd_set *wfds, fd_set *efds, long timeout) { fd_set *wfds, fd_set *efds, long timeout) { // NOLINT(*)
#if !defined(_WIN32) #if !defined(_WIN32)
utils::Assert(maxfd < FD_SETSIZE, "maxdf must be smaller than FDSETSIZE"); utils::Assert(maxfd < FD_SETSIZE, "maxdf must be smaller than FDSETSIZE");
#endif #endif

2
test/test.mk
View File

@ -1,7 +1,7 @@
# this is a makefile used to show testcases of rabit # this is a makefile used to show testcases of rabit
.PHONY: all .PHONY: all
all: all: model_recover_10_10k model_recover_10_10k_die_same
# this experiment test recovery with actually process exit, use keepalive to keep program alive # this experiment test recovery with actually process exit, use keepalive to keep program alive
model_recover_10_10k: model_recover_10_10k:

55
wrapper/rabit.py
View File

@ -3,6 +3,7 @@ Python interface for rabit
Reliable Allreduce and Broadcast Library Reliable Allreduce and Broadcast Library
Author: Tianqi Chen Author: Tianqi Chen
""" """
# pylint: disable=unused-argument,invalid-name,global-statement,dangerous-default-value,
import cPickle as pickle import cPickle as pickle
import ctypes import ctypes
import os import os
@ -17,10 +18,12 @@ else:
rbtlib = None rbtlib = None
# load in xgboost library # load in xgboost library
def loadlib__(lib = 'standard'): def loadlib__(lib='standard'):
"""Load rabit library"""
global rbtlib global rbtlib
if rbtlib != None: if rbtlib != None:
warnings.Warn('rabit.int call was ignored because it has already been initialized', level = 2) warnings.warn('rabit.int call was ignored because it has'\
' already been initialized', level=2)
return return
if lib == 'standard': if lib == 'standard':
rbtlib = ctypes.cdll.LoadLibrary(WRAPPER_PATH % '') rbtlib = ctypes.cdll.LoadLibrary(WRAPPER_PATH % '')
@ -35,6 +38,7 @@ def loadlib__(lib = 'standard'):
rbtlib.RabitVersionNumber.restype = ctypes.c_int rbtlib.RabitVersionNumber.restype = ctypes.c_int
def unloadlib__(): def unloadlib__():
"""Unload rabit library"""
global rbtlib global rbtlib
del rbtlib del rbtlib
rbtlib = None rbtlib = None
@ -51,7 +55,7 @@ def check_err__():
""" """
return return
def init(args = sys.argv, lib = 'standard'): def init(args=sys.argv, lib='standard'):
""" """
intialize the rabit module, call this once before using anything intialize the rabit module, call this once before using anything
Arguments: Arguments:
@ -145,7 +149,7 @@ def broadcast(data, root):
length = ctypes.c_ulong() length = ctypes.c_ulong()
if root == rank: if root == rank:
assert data is not None, 'need to pass in data when broadcasting' assert data is not None, 'need to pass in data when broadcasting'
s = pickle.dumps(data, protocol = pickle.HIGHEST_PROTOCOL) s = pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL)
length.value = len(s) length.value = len(s)
# run first broadcast # run first broadcast
rbtlib.RabitBroadcast(ctypes.byref(length), rbtlib.RabitBroadcast(ctypes.byref(length),
@ -179,7 +183,7 @@ DTYPE_ENUM__ = {
np.dtype('float64') : 7 np.dtype('float64') : 7
} }
def allreduce(data, op, prepare_fun = None): def allreduce(data, op, prepare_fun=None):
""" """
perform allreduce, return the result, this function is not thread-safe perform allreduce, return the result, this function is not thread-safe
Arguments: Arguments:
@ -190,7 +194,8 @@ def allreduce(data, op, prepare_fun = None):
prepare_fun: lambda data prepare_fun: lambda data
Lazy preprocessing function, if it is not None, prepare_fun(data) Lazy preprocessing function, if it is not None, prepare_fun(data)
will be called by the function before performing allreduce, to intialize the data will be called by the function before performing allreduce, to intialize the data
If the result of Allreduce can be recovered directly, then prepare_fun will NOT be called If the result of Allreduce can be recovered directly,
then prepare_fun will NOT be called
Returns: Returns:
the result of allreduce, have same shape as data the result of allreduce, have same shape as data
""" """
@ -206,12 +211,13 @@ def allreduce(data, op, prepare_fun = None):
buf.size, DTYPE_ENUM__[buf.dtype], buf.size, DTYPE_ENUM__[buf.dtype],
op, None, None) op, None, None)
else: else:
PFUNC = ctypes.CFUNCTYPE(None, ctypes.c_void_p) func_ptr = ctypes.CFUNCTYPE(None, ctypes.c_void_p)
def pfunc(args): def pfunc(args):
"""prepare function."""
prepare_fun(data) prepare_fun(data)
rbtlib.RabitAllreduce(buf.ctypes.data_as(ctypes.c_void_p), rbtlib.RabitAllreduce(buf.ctypes.data_as(ctypes.c_void_p),
buf.size, DTYPE_ENUM__[buf.dtype], buf.size, DTYPE_ENUM__[buf.dtype],
op, PFUNC(pfunc), None) op, func_ptr(pfunc), None)
check_err__() check_err__()
return buf return buf
@ -229,7 +235,7 @@ def load_model__(ptr, length):
data = (ctypes.c_char * length).from_address(ctypes.addressof(ptr.contents)) data = (ctypes.c_char * length).from_address(ctypes.addressof(ptr.contents))
return pickle.loads(data.raw) return pickle.loads(data.raw)
def load_checkpoint(with_local = False): def load_checkpoint(with_local=False):
""" """
load latest check point load latest check point
Arguments: Arguments:
@ -241,34 +247,34 @@ def load_checkpoint(with_local = False):
if returned version == 0, this means no model has been CheckPointed if returned version == 0, this means no model has been CheckPointed
and global_model, local_model returned will be None and global_model, local_model returned will be None
""" """
gp = ctypes.POINTER(ctypes.c_char)() gptr = ctypes.POINTER(ctypes.c_char)()
global_len = ctypes.c_ulong() global_len = ctypes.c_ulong()
if with_local: if with_local:
lp = ctypes.POINTER(ctypes.c_char)() lptr = ctypes.POINTER(ctypes.c_char)()
local_len = ctypes.c_ulong() local_len = ctypes.c_ulong()
version = rbtlib.RabitLoadCheckPoint( version = rbtlib.RabitLoadCheckPoint(
ctypes.byref(gp), ctypes.byref(gptr),
ctypes.byref(global_len), ctypes.byref(global_len),
ctypes.byref(lp), ctypes.byref(lptr),
ctypes.byref(local_len)) ctypes.byref(local_len))
check_err__() check_err__()
if version == 0: if version == 0:
return (version, None, None) return (version, None, None)
return (version, return (version,
load_model__(gp, global_len.value), load_model__(gptr, global_len.value),
load_model__(lp, local_len.value)) load_model__(lptr, local_len.value))
else: else:
version = rbtlib.RabitLoadCheckPoint( version = rbtlib.RabitLoadCheckPoint(
ctypes.byref(gp), ctypes.byref(gptr),
ctypes.byref(global_len), ctypes.byref(global_len),
None, None) None, None)
check_err__() check_err__()
if version == 0: if version == 0:
return (version, None) return (version, None)
return (version, return (version,
load_model__(gp, global_len.value)) load_model__(gptr, global_len.value))
def checkpoint(global_model, local_model = None): def checkpoint(global_model, local_model=None):
""" """
checkpoint the model, meaning we finished a stage of execution checkpoint the model, meaning we finished a stage of execution
every time we call check point, there is a version number which will increase by one every time we call check point, there is a version number which will increase by one
@ -285,16 +291,17 @@ def checkpoint(global_model, local_model = None):
while global_model do not need explicit replication. while global_model do not need explicit replication.
It is recommended to use global_model if possible It is recommended to use global_model if possible
""" """
sg = pickle.dumps(global_model) sglobal = pickle.dumps(global_model)
if local_model is None: if local_model is None:
rbtlib.RabitCheckPoint(sg, len(sg), None, 0) rbtlib.RabitCheckPoint(sglobal, len(sglobal), None, 0)
check_err__() check_err__()
del sg; del sglobal
else: else:
sl = pickle.dumps(local_model) slocal = pickle.dumps(local_model)
rbtlib.RabitCheckPoint(sg, len(sg), sl, len(sl)) rbtlib.RabitCheckPoint(sglobal, len(sglobal), slocal, len(slocal))
check_err__() check_err__()
del sl; del sg; del slocal
del sglobal
def version_number(): def version_number():
""" """

7
wrapper/rabit_wrapper.cc
View File

@ -1,3 +1,4 @@
// Copyright by Contributors
// implementations in ctypes // implementations in ctypes
#define _CRT_SECURE_NO_WARNINGS #define _CRT_SECURE_NO_WARNINGS
#define _CRT_SECURE_NO_DEPRECATE #define _CRT_SECURE_NO_DEPRECATE
@ -60,12 +61,12 @@ inline void Allreduce_(void *sendrecvbuf_,
return; return;
case kLong: case kLong:
rabit::Allreduce<OP> rabit::Allreduce<OP>
(static_cast<long*>(sendrecvbuf_), (static_cast<long*>(sendrecvbuf_), // NOLINT(*)
count, prepare_fun, prepare_arg); count, prepare_fun, prepare_arg);
return; return;
case kULong: case kULong:
rabit::Allreduce<OP> rabit::Allreduce<OP>
(static_cast<unsigned long*>(sendrecvbuf_), (static_cast<unsigned long*>(sendrecvbuf_), // NOLINT(*)
count, prepare_fun, prepare_arg); count, prepare_fun, prepare_arg);
return; return;
case kFloat: case kFloat:
@ -179,7 +180,7 @@ extern "C" {
if (s.length() > max_len) { if (s.length() > max_len) {
s.resize(max_len - 1); s.resize(max_len - 1);
} }
strcpy(out_name, s.c_str()); strcpy(out_name, s.c_str()); // NOLINT(*)
*out_len = static_cast<rbt_ulong>(s.length()); *out_len = static_cast<rbt_ulong>(s.length());
} }
void RabitBroadcast(void *sendrecv_data, void RabitBroadcast(void *sendrecv_data,

9
wrapper/rabit_wrapper.h
View File

@ -1,18 +1,19 @@
#ifndef RABIT_WRAPPER_H_
#define RABIT_WRAPPER_H_
/*! /*!
* Copyright by Contributors
* \file rabit_wrapper.h * \file rabit_wrapper.h
* \author Tianqi Chen * \author Tianqi Chen
* \brief a C style wrapper of rabit * \brief a C style wrapper of rabit
* can be used to create wrapper of other languages * can be used to create wrapper of other languages
*/ */
#ifndef RABIT_WRAPPER_H_
#define RABIT_WRAPPER_H_
#ifdef _MSC_VER #ifdef _MSC_VER
#define RABIT_DLL __declspec(dllexport) #define RABIT_DLL __declspec(dllexport)
#else #else
#define RABIT_DLL #define RABIT_DLL
#endif #endif
// manually define unsign long // manually define unsign long
typedef unsigned long rbt_ulong; typedef unsigned long rbt_ulong; // NOLINT(*)
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -122,4 +123,4 @@ extern "C" {
#ifdef __cplusplus #ifdef __cplusplus
} // C } // C
#endif #endif
#endif // XGBOOST_WRAPPER_H_ #endif // RABIT_WRAPPER_H_