lint and travis
This commit is contained in:
tqchen
parent
ceedf4ea96
commit
3cc49ad0e8
1
.gitignore
vendored
1
.gitignore
vendored
@ -34,3 +34,4 @@
|
||||
*tmp*
|
||||
*.rabit
|
||||
*.mock
|
||||
dmlc-core
|
||||
|
||||
48
.travis.yml
Normal file
48
.travis.yml
Normal file
@ -0,0 +1,48 @@
|
||||
# disable sudo to use container based build
|
||||
sudo: false
|
||||
|
||||
# Use Build Matrix to do lint and build seperately
|
||||
env:
|
||||
matrix:
|
||||
- TASK=lint LINT_LANG=cpp
|
||||
- TASK=lint LINT_LANG=python
|
||||
- TASK=doc
|
||||
- TASK=build CXX=g++
|
||||
|
||||
# dependent apt packages
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
- doxygen
|
||||
- wget
|
||||
- git
|
||||
- libcurl4-openssl-dev
|
||||
- unzip
|
||||
|
||||
before_install:
|
||||
- git clone https://github.com/dmlc/dmlc-core
|
||||
- export TRAVIS=dmlc-core/scripts/travis/
|
||||
- source ${TRAVIS}/travis_setup_env.sh
|
||||
|
||||
install:
|
||||
- pip install cpplint pylint --user `whoami`
|
||||
|
||||
script: scripts/travis_script.sh
|
||||
|
||||
|
||||
before_cache:
|
||||
- ${TRAVIS}/travis_before_cache.sh
|
||||
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- ${HOME}/.cache/usr
|
||||
|
||||
|
||||
notifications:
|
||||
# Emails are sent to the committer's git-configured email address by default,
|
||||
email:
|
||||
on_success: change
|
||||
on_failure: always
|
||||
|
||||
|
||||
14
Makefile
14
Makefile
@ -13,6 +13,10 @@ ifeq ($(WITH_FPIC), 1)
|
||||
CFLAGS += -fPIC
|
||||
endif
|
||||
|
||||
ifndef LINT_LANG
|
||||
LINT_LANG="all"
|
||||
endif
|
||||
|
||||
# build path
|
||||
BPATH=.
|
||||
# objectives that makes up rabit library
|
||||
@ -22,7 +26,9 @@ OBJ= $(BPATH)/allreduce_base.o $(BPATH)/allreduce_robust.o $(BPATH)/engine.o $(B
|
||||
SLIB= wrapper/librabit_wrapper.so wrapper/librabit_wrapper_mock.so wrapper/librabit_wrapper_mpi.so
|
||||
ALIB= lib/librabit.a lib/librabit_mpi.a lib/librabit_empty.a lib/librabit_mock.a lib/librabit_base.a
|
||||
HEADERS=src/*.h include/*.h include/rabit/*.h
|
||||
.PHONY: clean all install mpi python
|
||||
DMLC=dmlc-core
|
||||
|
||||
.PHONY: clean all install mpi python lint doc
|
||||
|
||||
all: lib/librabit.a lib/librabit_mock.a wrapper/librabit_wrapper.so wrapper/librabit_wrapper_mock.so lib/librabit_base.a
|
||||
mpi: lib/librabit_mpi.a wrapper/librabit_wrapper_mpi.so
|
||||
@ -59,6 +65,12 @@ $(ALIB):
|
||||
$(SLIB) :
|
||||
$(CXX) $(CFLAGS) -shared -o $@ $(filter %.cpp %.o %.c %.cc %.a, $^) $(LDFLAGS)
|
||||
|
||||
lint:
|
||||
$(DMLC)/scripts/lint.py rabit $(LINT_LANG) src include wrapper
|
||||
|
||||
doc:
|
||||
cd include; doxygen ../doc/Doxyfile; cd -
|
||||
|
||||
clean:
|
||||
$(RM) $(OBJ) $(MPIOBJ) $(ALIB) $(MPIALIB) *~ src/*~ include/*~ include/*/*~ wrapper/*~
|
||||
|
||||
|
||||
@ -14,6 +14,7 @@
|
||||
|
||||
// include uint64_t only to make io standalone
|
||||
#ifdef _MSC_VER
|
||||
/*! \brief uint64 */
|
||||
typedef unsigned __int64 uint64_t;
|
||||
#else
|
||||
#include <inttypes.h>
|
||||
@ -24,7 +25,7 @@ namespace dmlc {
|
||||
/*!
|
||||
* \brief interface of stream I/O for serialization
|
||||
*/
|
||||
class Stream {
|
||||
class Stream { // NOLINT(*)
|
||||
public:
|
||||
/*!
|
||||
* \brief reads data from a stream
|
||||
@ -192,9 +193,10 @@ class InputSplit {
|
||||
* List of possible types: "text", "recordio"
|
||||
* - "text":
|
||||
* text file, each line is treated as a record
|
||||
* input split will split on \n or \r
|
||||
* input split will split on '\\n' or '\\r'
|
||||
* - "recordio":
|
||||
* binary recordio file, see recordio.h
|
||||
* \return a new input split
|
||||
* \sa InputSplit::Type
|
||||
*/
|
||||
static InputSplit* Create(const char *uri,
|
||||
@ -224,7 +226,7 @@ class ostream : public std::basic_ostream<char> {
|
||||
* \param buffer_size internal streambuf size
|
||||
*/
|
||||
explicit ostream(Stream *stream,
|
||||
size_t buffer_size = 1 << 10)
|
||||
size_t buffer_size = (1 << 10))
|
||||
: std::basic_ostream<char>(NULL), buf_(buffer_size) {
|
||||
this->set_stream(stream);
|
||||
}
|
||||
@ -287,7 +289,7 @@ class istream : public std::basic_istream<char> {
|
||||
* \param buffer_size internal buffer size
|
||||
*/
|
||||
explicit istream(Stream *stream,
|
||||
size_t buffer_size = 1 << 10)
|
||||
size_t buffer_size = (1 << 10))
|
||||
: std::basic_istream<char>(NULL), buf_(buffer_size) {
|
||||
this->set_stream(stream);
|
||||
}
|
||||
|
||||
@ -8,12 +8,18 @@
|
||||
* rabit.h and serializable.h is all what the user needs to use the rabit interface
|
||||
* \author Tianqi Chen, Ignacio Cano, Tianyi Zhou
|
||||
*/
|
||||
#ifndef RABIT_RABIT_H_
|
||||
#define RABIT_RABIT_H_
|
||||
#ifndef RABIT_RABIT_H_ // NOLINT(*)
|
||||
#define RABIT_RABIT_H_ // NOLINT(*)
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// whether or not use c++11 support
|
||||
#ifndef DMLC_USE_CXX11
|
||||
#define DMLC_USE_CXX11 (defined(__GXX_EXPERIMENTAL_CXX0X__) ||\
|
||||
__cplusplus >= 201103L || defined(_MSC_VER))
|
||||
#endif
|
||||
// optionally support of lambda functions in C++11, if available
|
||||
#if __cplusplus >= 201103L
|
||||
#if DMLC_USE_CXX11
|
||||
#include <functional>
|
||||
#endif // C++11
|
||||
// contains definition of Serializable
|
||||
@ -135,7 +141,7 @@ inline void Allreduce(DType *sendrecvbuf, size_t count,
|
||||
void (*prepare_fun)(void *arg) = NULL,
|
||||
void *prepare_arg = NULL);
|
||||
// C++11 support for lambda prepare function
|
||||
#if __cplusplus >= 201103L
|
||||
#if DMLC_USE_CXX11
|
||||
/*!
|
||||
* \brief performs in-place Allreduce, on sendrecvbuf
|
||||
* with a prepare function specified by a lambda function
|
||||
@ -242,7 +248,7 @@ class ReduceHandle;
|
||||
* \tparam freduce the customized reduction function
|
||||
* DType must be a struct, with no pointer
|
||||
*/
|
||||
template<typename DType, void (*freduce)(DType &dst, const DType &src)>
|
||||
template<typename DType, void (*freduce)(DType &dst, const DType &src)> // NOLINT(*)
|
||||
class Reducer {
|
||||
public:
|
||||
Reducer(void);
|
||||
@ -258,7 +264,7 @@ class Reducer {
|
||||
inline void Allreduce(DType *sendrecvbuf, size_t count,
|
||||
void (*prepare_fun)(void *arg) = NULL,
|
||||
void *prepare_arg = NULL);
|
||||
#if __cplusplus >= 201103L
|
||||
#if DMLC_USE_CXX11
|
||||
/*!
|
||||
* \brief customized in-place all reduce operation, with lambda function as preprocessor
|
||||
* \param sendrecvbuf pointer to the array of objects to be reduced
|
||||
@ -303,7 +309,7 @@ class SerializeReducer {
|
||||
void (*prepare_fun)(void *arg) = NULL,
|
||||
void *prepare_arg = NULL);
|
||||
// C++11 support for lambda prepare function
|
||||
#if __cplusplus >= 201103L
|
||||
#if DMLC_USE_CXX11
|
||||
/*!
|
||||
* \brief customized in-place all reduce operation, with lambda function as preprocessor
|
||||
* \param sendrecvobj pointer to the array of objects to be reduced
|
||||
@ -326,4 +332,4 @@ class SerializeReducer {
|
||||
} // namespace rabit
|
||||
// implementation of template functions
|
||||
#include "./rabit/rabit-inl.h"
|
||||
#endif // RABIT_RABIT_H_
|
||||
#endif // RABIT_RABIT_H_ // NOLINT(*)
|
||||
|
||||
@ -4,8 +4,8 @@
|
||||
* \brief utilities with different serializable implementations
|
||||
* \author Tianqi Chen
|
||||
*/
|
||||
#ifndef RABIT_UTILS_IO_H_
|
||||
#define RABIT_UTILS_IO_H_
|
||||
#ifndef RABIT_IO_H_
|
||||
#define RABIT_IO_H_
|
||||
#include <cstdio>
|
||||
#include <vector>
|
||||
#include <cstring>
|
||||
@ -51,6 +51,7 @@ struct MemoryFixSizeBuffer : public SeekStream {
|
||||
virtual bool AtEnd(void) const {
|
||||
return curr_ptr_ == buffer_size_;
|
||||
}
|
||||
|
||||
private:
|
||||
/*! \brief in memory buffer */
|
||||
char *p_buffer_;
|
||||
@ -93,6 +94,7 @@ struct MemoryBufferStream : public SeekStream {
|
||||
virtual bool AtEnd(void) const {
|
||||
return curr_ptr_ == p_buffer_->length();
|
||||
}
|
||||
|
||||
private:
|
||||
/*! \brief in memory buffer */
|
||||
std::string *p_buffer_;
|
||||
@ -101,4 +103,4 @@ struct MemoryBufferStream : public SeekStream {
|
||||
}; // class MemoryBufferStream
|
||||
} // namespace utils
|
||||
} // namespace rabit
|
||||
#endif // RABIT_UTILS_IO_H_
|
||||
#endif // RABIT_IO_H_
|
||||
|
||||
@ -1,12 +1,15 @@
|
||||
/*!
|
||||
* Copyright by Contributors
|
||||
* \file rabit-inl.h
|
||||
* \brief implementation of inline template function for rabit interface
|
||||
*
|
||||
* \author Tianqi Chen
|
||||
*/
|
||||
#ifndef RABIT_RABIT_INL_H
|
||||
#define RABIT_RABIT_INL_H
|
||||
#ifndef RABIT_RABIT_INL_H_
|
||||
#define RABIT_RABIT_INL_H_
|
||||
// use engine for implementation
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include "./io.h"
|
||||
#include "./utils.h"
|
||||
#include "../rabit.h"
|
||||
@ -30,15 +33,15 @@ inline DataType GetType<int>(void) {
|
||||
return kInt;
|
||||
}
|
||||
template<>
|
||||
inline DataType GetType<unsigned int>(void) {
|
||||
inline DataType GetType<unsigned int>(void) { // NOLINT(*)
|
||||
return kUInt;
|
||||
}
|
||||
template<>
|
||||
inline DataType GetType<long>(void) {
|
||||
inline DataType GetType<long>(void) { // NOLINT(*)
|
||||
return kLong;
|
||||
}
|
||||
template<>
|
||||
inline DataType GetType<unsigned long>(void) {
|
||||
inline DataType GetType<unsigned long>(void) { // NOLINT(*)
|
||||
return kULong;
|
||||
}
|
||||
template<>
|
||||
@ -50,11 +53,11 @@ inline DataType GetType<double>(void) {
|
||||
return kDouble;
|
||||
}
|
||||
template<>
|
||||
inline DataType GetType<long long>(void) {
|
||||
inline DataType GetType<long long>(void) { // NOLINT(*)
|
||||
return kLongLong;
|
||||
}
|
||||
template<>
|
||||
inline DataType GetType<unsigned long long>(void) {
|
||||
inline DataType GetType<unsigned long long>(void) { // NOLINT(*)
|
||||
return kULongLong;
|
||||
}
|
||||
} // namespace mpi
|
||||
@ -62,37 +65,37 @@ inline DataType GetType<unsigned long long>(void) {
|
||||
|
||||
namespace op {
|
||||
struct Max {
|
||||
const static engine::mpi::OpType kType = engine::mpi::kMax;
|
||||
static const engine::mpi::OpType kType = engine::mpi::kMax;
|
||||
template<typename DType>
|
||||
inline static void Reduce(DType &dst, const DType &src) {
|
||||
inline static void Reduce(DType &dst, const DType &src) { // NOLINT(*)
|
||||
if (dst < src) dst = src;
|
||||
}
|
||||
};
|
||||
struct Min {
|
||||
const static engine::mpi::OpType kType = engine::mpi::kMin;
|
||||
static const engine::mpi::OpType kType = engine::mpi::kMin;
|
||||
template<typename DType>
|
||||
inline static void Reduce(DType &dst, const DType &src) {
|
||||
inline static void Reduce(DType &dst, const DType &src) { // NOLINT(*)
|
||||
if (dst > src) dst = src;
|
||||
}
|
||||
};
|
||||
struct Sum {
|
||||
const static engine::mpi::OpType kType = engine::mpi::kSum;
|
||||
static const engine::mpi::OpType kType = engine::mpi::kSum;
|
||||
template<typename DType>
|
||||
inline static void Reduce(DType &dst, const DType &src) {
|
||||
inline static void Reduce(DType &dst, const DType &src) { // NOLINT(*)
|
||||
dst += src;
|
||||
}
|
||||
};
|
||||
struct BitOR {
|
||||
const static engine::mpi::OpType kType = engine::mpi::kBitwiseOR;
|
||||
static const engine::mpi::OpType kType = engine::mpi::kBitwiseOR;
|
||||
template<typename DType>
|
||||
inline static void Reduce(DType &dst, const DType &src) {
|
||||
inline static void Reduce(DType &dst, const DType &src) { // NOLINT(*)
|
||||
dst |= src;
|
||||
}
|
||||
};
|
||||
template<typename OP, typename DType>
|
||||
inline void Reducer(const void *src_, void *dst_, int len, const MPI::Datatype &dtype) {
|
||||
const DType *src = (const DType*)src_;
|
||||
DType *dst = (DType*)dst_;
|
||||
DType *dst = (DType*)dst_; // NOLINT(*)
|
||||
for (int i = 0; i < len; ++i) {
|
||||
OP::Reduce(dst[i], src[i]);
|
||||
}
|
||||
@ -154,18 +157,18 @@ template<typename OP, typename DType>
|
||||
inline void Allreduce(DType *sendrecvbuf, size_t count,
|
||||
void (*prepare_fun)(void *arg),
|
||||
void *prepare_arg) {
|
||||
engine::Allreduce_(sendrecvbuf, sizeof(DType), count, op::Reducer<OP,DType>,
|
||||
engine::Allreduce_(sendrecvbuf, sizeof(DType), count, op::Reducer<OP, DType>,
|
||||
engine::mpi::GetType<DType>(), OP::kType, prepare_fun, prepare_arg);
|
||||
}
|
||||
|
||||
// C++11 support for lambda prepare function
|
||||
#if __cplusplus >= 201103L
|
||||
#if DMLC_USE_CXX11
|
||||
inline void InvokeLambda_(void *fun) {
|
||||
(*static_cast<std::function<void()>*>(fun))();
|
||||
}
|
||||
template<typename OP, typename DType>
|
||||
inline void Allreduce(DType *sendrecvbuf, size_t count, std::function<void()> prepare_fun) {
|
||||
engine::Allreduce_(sendrecvbuf, sizeof(DType), count, op::Reducer<OP,DType>,
|
||||
engine::Allreduce_(sendrecvbuf, sizeof(DType), count, op::Reducer<OP, DType>,
|
||||
engine::mpi::GetType<DType>(), OP::kType, InvokeLambda_, &prepare_fun);
|
||||
}
|
||||
#endif // C++11
|
||||
@ -223,15 +226,16 @@ inline void ReducerSafe_(const void *src_, void *dst_, int len_, const MPI::Data
|
||||
}
|
||||
}
|
||||
// function to perform reduction for Reducer
|
||||
template<typename DType, void (*freduce)(DType &dst, const DType &src)>
|
||||
inline void ReducerAlign_(const void *src_, void *dst_, int len_, const MPI::Datatype &dtype) {
|
||||
template<typename DType, void (*freduce)(DType &dst, const DType &src)> // NOLINT(*)
|
||||
inline void ReducerAlign_(const void *src_, void *dst_,
|
||||
int len_, const MPI::Datatype &dtype) {
|
||||
const DType *psrc = reinterpret_cast<const DType*>(src_);
|
||||
DType *pdst = reinterpret_cast<DType*>(dst_);
|
||||
for (int i = 0; i < len_; ++i) {
|
||||
freduce(pdst[i], psrc[i]);
|
||||
}
|
||||
}
|
||||
template<typename DType, void (*freduce)(DType &dst, const DType &src)>
|
||||
template<typename DType, void (*freduce)(DType &dst, const DType &src)> // NOLINT(*)
|
||||
inline Reducer<DType, freduce>::Reducer(void) {
|
||||
// it is safe to directly use handle for aligned data types
|
||||
if (sizeof(DType) == 8 || sizeof(DType) == 4 || sizeof(DType) == 1) {
|
||||
@ -240,7 +244,7 @@ inline Reducer<DType, freduce>::Reducer(void) {
|
||||
this->handle_.Init(ReducerSafe_<DType, freduce>, sizeof(DType));
|
||||
}
|
||||
}
|
||||
template<typename DType, void (*freduce)(DType &dst, const DType &src)>
|
||||
template<typename DType, void (*freduce)(DType &dst, const DType &src)> // NOLINT(*)
|
||||
inline void Reducer<DType, freduce>::Allreduce(DType *sendrecvbuf, size_t count,
|
||||
void (*prepare_fun)(void *arg),
|
||||
void *prepare_arg) {
|
||||
@ -248,13 +252,14 @@ inline void Reducer<DType, freduce>::Allreduce(DType *sendrecvbuf, size_t count,
|
||||
}
|
||||
// function to perform reduction for SerializeReducer
|
||||
template<typename DType>
|
||||
inline void SerializeReducerFunc_(const void *src_, void *dst_, int len_, const MPI::Datatype &dtype) {
|
||||
inline void SerializeReducerFunc_(const void *src_, void *dst_,
|
||||
int len_, const MPI::Datatype &dtype) {
|
||||
int nbytes = engine::ReduceHandle::TypeSize(dtype);
|
||||
// temp space
|
||||
DType tsrc, tdst;
|
||||
for (int i = 0; i < len_; ++i) {
|
||||
utils::MemoryFixSizeBuffer fsrc((char*)(src_) + i * nbytes, nbytes);
|
||||
utils::MemoryFixSizeBuffer fdst((char*)(dst_) + i * nbytes, nbytes);
|
||||
utils::MemoryFixSizeBuffer fsrc((char*)(src_) + i * nbytes, nbytes); // NOLINT(*)
|
||||
utils::MemoryFixSizeBuffer fdst((char*)(dst_) + i * nbytes, nbytes); // NOLINT(*)
|
||||
tsrc.Load(fsrc);
|
||||
tdst.Load(fdst);
|
||||
// govern const check
|
||||
@ -306,8 +311,8 @@ inline void SerializeReducer<DType>::Allreduce(DType *sendrecvobj,
|
||||
}
|
||||
}
|
||||
|
||||
#if __cplusplus >= 201103L
|
||||
template<typename DType, void (*freduce)(DType &dst, const DType &src)>
|
||||
#if DMLC_USE_CXX11
|
||||
template<typename DType, void (*freduce)(DType &dst, const DType &src)> // NOLINT(*)g
|
||||
inline void Reducer<DType, freduce>::Allreduce(DType *sendrecvbuf, size_t count,
|
||||
std::function<void()> prepare_fun) {
|
||||
this->Allreduce(sendrecvbuf, count, InvokeLambda_, &prepare_fun);
|
||||
@ -320,4 +325,4 @@ inline void SerializeReducer<DType>::Allreduce(DType *sendrecvobj,
|
||||
}
|
||||
#endif
|
||||
} // namespace rabit
|
||||
#endif
|
||||
#endif // RABIT_RABIT_INL_H_
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
/*!
|
||||
* Copyright by Contributors
|
||||
* \file timer.h
|
||||
* \brief This file defines the utils for timing
|
||||
* \author Tianqi Chen, Nacho, Tianyi
|
||||
@ -18,7 +19,6 @@ namespace utils {
|
||||
* \brief return time in seconds, not cross platform, avoid to use this in most places
|
||||
*/
|
||||
inline double GetTime(void) {
|
||||
// TODO: use c++11 chrono when c++11 was available
|
||||
#ifdef __MACH__
|
||||
clock_serv_t cclock;
|
||||
mach_timespec_t mts;
|
||||
@ -32,7 +32,6 @@ inline double GetTime(void) {
|
||||
utils::Check(clock_gettime(CLOCK_REALTIME, &ts) == 0, "failed to get time");
|
||||
return static_cast<double>(ts.tv_sec) + static_cast<double>(ts.tv_nsec) * 1e-9;
|
||||
#else
|
||||
// TODO: add MSVC macro, and MSVC timer
|
||||
return static_cast<double>(time(NULL));
|
||||
#endif
|
||||
#endif
|
||||
|
||||
@ -27,7 +27,7 @@
|
||||
#else
|
||||
#ifdef _FILE_OFFSET_BITS
|
||||
#if _FILE_OFFSET_BITS == 32
|
||||
#pragma message ("Warning: FILE OFFSET BITS defined to be 32 bit")
|
||||
#pragma message("Warning: FILE OFFSET BITS defined to be 32 bit")
|
||||
#endif
|
||||
#endif
|
||||
|
||||
@ -163,7 +163,7 @@ inline std::FILE *FopenCheck(const char *fname, const char *flag) {
|
||||
// easy utils that can be directly accessed in xgboost
|
||||
/*! \brief get the beginning address of a vector */
|
||||
template<typename T>
|
||||
inline T *BeginPtr(std::vector<T> &vec) {
|
||||
inline T *BeginPtr(std::vector<T> &vec) { // NOLINT(*)
|
||||
if (vec.size() == 0) {
|
||||
return NULL;
|
||||
} else {
|
||||
@ -172,14 +172,14 @@ inline T *BeginPtr(std::vector<T> &vec) {
|
||||
}
|
||||
/*! \brief get the beginning address of a vector */
|
||||
template<typename T>
|
||||
inline const T *BeginPtr(const std::vector<T> &vec) {
|
||||
inline const T *BeginPtr(const std::vector<T> &vec) { // NOLINT(*)
|
||||
if (vec.size() == 0) {
|
||||
return NULL;
|
||||
} else {
|
||||
return &vec[0];
|
||||
}
|
||||
}
|
||||
inline char* BeginPtr(std::string &str) {
|
||||
inline char* BeginPtr(std::string &str) { // NOLINT(*)
|
||||
if (str.length() == 0) return NULL;
|
||||
return &str[0];
|
||||
}
|
||||
|
||||
@ -4,8 +4,8 @@
|
||||
* \brief defines serializable interface of rabit
|
||||
* \author Tianqi Chen
|
||||
*/
|
||||
#ifndef RABIT_RABIT_SERIALIZABLE_H_
|
||||
#define RABIT_RABIT_SERIALIZABLE_H_
|
||||
#ifndef RABIT_SERIALIZABLE_H_
|
||||
#define RABIT_SERIALIZABLE_H_
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include "./rabit/utils.h"
|
||||
@ -24,4 +24,4 @@ typedef dmlc::Stream Stream;
|
||||
typedef dmlc::Serializable Serializable;
|
||||
|
||||
} // namespace rabit
|
||||
#endif // RABIT_RABIT_SERIALIZABLE_H_
|
||||
#endif // RABIT_SERIALIZABLE_H_
|
||||
|
||||
8
scripts/travis_runtest.sh
Executable file
8
scripts/travis_runtest.sh
Executable file
@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
make -f test.mk model_recover_10_10k || exit -1
|
||||
make -f test.mk model_recover_10_10k_die_same || exit -1
|
||||
make -f test.mk local_recover_10_10k || exit -1
|
||||
make -f test.mk pylocal_recover_10_10k || exit -1
|
||||
make -f test.mk lazy_recover_10_10k_die_hard || exit -1
|
||||
make -f test.mk lazy_recover_10_10k_die_same || exit -1
|
||||
make -f test.mk ringallreduce_10_10k || exit -1
|
||||
22
scripts/travis_script.sh
Executable file
22
scripts/travis_script.sh
Executable file
@ -0,0 +1,22 @@
|
||||
#!/bin/bash
|
||||
|
||||
# main script of travis
|
||||
if [ ${TASK} == "lint" ]; then
|
||||
make lint || exit -1
|
||||
fi
|
||||
|
||||
if [ ${TASK} == "doc" ]; then
|
||||
make doc 2>log.txt
|
||||
(cat log.txt|grep warning) && exit -1
|
||||
fi
|
||||
|
||||
if [ ${TASK} == "build" ]; then
|
||||
make all || exit -1
|
||||
fi
|
||||
|
||||
if [ ${TASK} == "test" ]; then
|
||||
cd test
|
||||
make all || exit -1
|
||||
./travis_runtest.sh || exit -1
|
||||
fi
|
||||
|
||||
@ -94,7 +94,8 @@ void AllreduceBase::Init(void) {
|
||||
}
|
||||
}
|
||||
if (dmlc_role != "worker") {
|
||||
fprintf(stderr, "Rabit Module currently only work with dmlc worker, quit this program by exit 0\n");
|
||||
fprintf(stderr, "Rabit Module currently only work with dmlc worker"\
|
||||
", quit this program by exit 0\n");
|
||||
exit(0);
|
||||
}
|
||||
// clear the setting before start reconnection
|
||||
@ -134,7 +135,7 @@ void AllreduceBase::TrackerPrint(const std::string &msg) {
|
||||
// util to parse data with unit suffix
|
||||
inline size_t ParseUnit(const char *name, const char *val) {
|
||||
char unit;
|
||||
unsigned long amt;
|
||||
unsigned long amt; // NOLINT(*)
|
||||
int n = sscanf(val, "%lu%c", &amt, &unit);
|
||||
size_t amount = amt;
|
||||
if (n == 2) {
|
||||
@ -440,7 +441,7 @@ AllreduceBase::TryAllreduceTree(void *sendrecvbuf_,
|
||||
selecter.WatchRead(links[i].sock);
|
||||
}
|
||||
// size_write <= size_read
|
||||
if (links[i].size_write != total_size){
|
||||
if (links[i].size_write != total_size) {
|
||||
if (links[i].size_write < size_down_in) {
|
||||
selecter.WatchWrite(links[i].sock);
|
||||
}
|
||||
@ -477,7 +478,7 @@ AllreduceBase::TryAllreduceTree(void *sendrecvbuf_,
|
||||
size_t max_reduce = total_size;
|
||||
for (int i = 0; i < nlink; ++i) {
|
||||
if (i != parent_index) {
|
||||
max_reduce= std::min(max_reduce, links[i].size_read);
|
||||
max_reduce = std::min(max_reduce, links[i].size_read);
|
||||
utils::Assert(buffer_size == 0 || buffer_size == links[i].buffer_size,
|
||||
"buffer size inconsistent");
|
||||
buffer_size = links[i].buffer_size;
|
||||
|
||||
@ -225,7 +225,7 @@ class AllreduceBase : public IEngine {
|
||||
ReturnTypeEnum value;
|
||||
// constructor
|
||||
ReturnType() {}
|
||||
ReturnType(ReturnTypeEnum value) : value(value){}
|
||||
ReturnType(ReturnTypeEnum value) : value(value) {} // NOLINT(*)
|
||||
inline bool operator==(const ReturnTypeEnum &v) const {
|
||||
return value == v;
|
||||
}
|
||||
@ -522,4 +522,4 @@ class AllreduceBase : public IEngine {
|
||||
};
|
||||
} // namespace engine
|
||||
} // namespace rabit
|
||||
#endif // RABIT_ALLREDUCE_BASE_H
|
||||
#endif // RABIT_ALLREDUCE_BASE_H_
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
/*!
|
||||
* Copyright by Contributors
|
||||
* \file allreduce_mock.h
|
||||
* \brief Mock test module of AllReduce engine,
|
||||
* insert failures in certain call point, to test if the engine is robust to failure
|
||||
@ -100,6 +101,7 @@ class AllreduceMock : public AllreduceRobust {
|
||||
this->Verify(MockKey(rank, version_number, seq_counter, num_trial), "LazyCheckPoint");
|
||||
AllreduceRobust::LazyCheckPoint(global_model);
|
||||
}
|
||||
|
||||
protected:
|
||||
// force checkpoint to local
|
||||
int force_local;
|
||||
|
||||
@ -5,8 +5,8 @@
|
||||
*
|
||||
* \author Tianqi Chen
|
||||
*/
|
||||
#ifndef RABIT_ENGINE_ROBUST_INL_H_
|
||||
#define RABIT_ENGINE_ROBUST_INL_H_
|
||||
#ifndef RABIT_ALLREDUCE_ROBUST_INL_H_
|
||||
#define RABIT_ALLREDUCE_ROBUST_INL_H_
|
||||
#include <vector>
|
||||
|
||||
namespace rabit {
|
||||
@ -35,7 +35,7 @@ inline AllreduceRobust::ReturnType
|
||||
AllreduceRobust::MsgPassing(const NodeType &node_value,
|
||||
std::vector<EdgeType> *p_edge_in,
|
||||
std::vector<EdgeType> *p_edge_out,
|
||||
EdgeType (*func)
|
||||
EdgeType(*func)
|
||||
(const NodeType &node_value,
|
||||
const std::vector<EdgeType> &edge_in,
|
||||
size_t out_index)) {
|
||||
@ -80,8 +80,16 @@ AllreduceRobust::MsgPassing(const NodeType &node_value,
|
||||
selecter.WatchRead(links[i].sock);
|
||||
}
|
||||
break;
|
||||
case 1: if (i == parent_index) selecter.WatchWrite(links[i].sock); break;
|
||||
case 2: if (i == parent_index) selecter.WatchRead(links[i].sock); break;
|
||||
case 1:
|
||||
if (i == parent_index) {
|
||||
selecter.WatchWrite(links[i].sock);
|
||||
}
|
||||
break;
|
||||
case 2:
|
||||
if (i == parent_index) {
|
||||
selecter.WatchRead(links[i].sock);
|
||||
}
|
||||
break;
|
||||
case 3:
|
||||
if (i != parent_index && links[i].size_write != sizeof(EdgeType)) {
|
||||
selecter.WatchWrite(links[i].sock);
|
||||
@ -158,4 +166,4 @@ AllreduceRobust::MsgPassing(const NodeType &node_value,
|
||||
}
|
||||
} // namespace engine
|
||||
} // namespace rabit
|
||||
#endif // RABIT_ENGINE_ROBUST_INL_H_
|
||||
#endif // RABIT_ALLREDUCE_ROBUST_INL_H_
|
||||
|
||||
@ -224,7 +224,7 @@ void AllreduceRobust::LocalModelCheck(bool with_local) {
|
||||
num_local_replica = 0;
|
||||
}
|
||||
} else {
|
||||
utils::Check(use_local_model == int(with_local),
|
||||
utils::Check(use_local_model == static_cast<int>(with_local),
|
||||
"Can only call Checkpoint/LoadCheckPoint always with"\
|
||||
"or without local_model, but not mixed case");
|
||||
}
|
||||
|
||||
@ -287,6 +287,7 @@ class AllreduceRobust : public AllreduceBase {
|
||||
if (seqno_.size() == 0) return -1;
|
||||
return seqno_.back();
|
||||
}
|
||||
|
||||
private:
|
||||
// sequence number of each
|
||||
std::vector<int> seqno_;
|
||||
@ -509,7 +510,7 @@ o * the input state must exactly one saved state(local state of current node)
|
||||
inline ReturnType MsgPassing(const NodeType &node_value,
|
||||
std::vector<EdgeType> *p_edge_in,
|
||||
std::vector<EdgeType> *p_edge_out,
|
||||
EdgeType (*func)
|
||||
EdgeType(*func)
|
||||
(const NodeType &node_value,
|
||||
const std::vector<EdgeType> &edge_in,
|
||||
size_t out_index));
|
||||
|
||||
@ -166,7 +166,7 @@ void ReduceHandle::Init(IEngine::ReduceFunction redfunc, size_t type_nbytes) {
|
||||
if (type_nbytes != 0) {
|
||||
MPI::Datatype *dtype = new MPI::Datatype();
|
||||
if (type_nbytes % 8 == 0) {
|
||||
*dtype = MPI::LONG.Create_contiguous(type_nbytes / sizeof(long));
|
||||
*dtype = MPI::LONG.Create_contiguous(type_nbytes / sizeof(long)); // NOLINT(*)
|
||||
} else if (type_nbytes % 4 == 0) {
|
||||
*dtype = MPI::INT.Create_contiguous(type_nbytes / sizeof(int));
|
||||
} else {
|
||||
@ -195,7 +195,7 @@ void ReduceHandle::Allreduce(void *sendrecvbuf,
|
||||
dtype->Free();
|
||||
}
|
||||
if (type_nbytes % 8 == 0) {
|
||||
*dtype = MPI::LONG.Create_contiguous(type_nbytes / sizeof(long));
|
||||
*dtype = MPI::LONG.Create_contiguous(type_nbytes / sizeof(long)); // NOLINT(*)
|
||||
} else if (type_nbytes % 4 == 0) {
|
||||
*dtype = MPI::INT.Create_contiguous(type_nbytes / sizeof(int));
|
||||
} else {
|
||||
|
||||
11
src/socket.h
11
src/socket.h
@ -264,7 +264,8 @@ class TCPSocket : public Socket{
|
||||
*/
|
||||
inline void SetKeepAlive(bool keepalive) {
|
||||
int opt = static_cast<int>(keepalive);
|
||||
if (setsockopt(sockfd, SOL_SOCKET, SO_KEEPALIVE, reinterpret_cast<char*>(&opt), sizeof(opt)) < 0) {
|
||||
if (setsockopt(sockfd, SOL_SOCKET, SO_KEEPALIVE,
|
||||
reinterpret_cast<char*>(&opt), sizeof(opt)) < 0) {
|
||||
Socket::Error("SetKeepAlive");
|
||||
}
|
||||
}
|
||||
@ -299,7 +300,7 @@ class TCPSocket : public Socket{
|
||||
*/
|
||||
inline int AtMark(void) const {
|
||||
#ifdef _WIN32
|
||||
unsigned long atmark;
|
||||
unsigned long atmark; // NOLINT(*)
|
||||
if (ioctlsocket(sockfd, SIOCATMARK, &atmark) != NO_ERROR) return -1;
|
||||
#else
|
||||
int atmark;
|
||||
@ -473,7 +474,7 @@ struct SelectHelper {
|
||||
* \param timeout the timeout counter, can be 0, which means wait until the event happen
|
||||
* \return 1 if success, 0 if timeout, and -1 if error occurs
|
||||
*/
|
||||
inline static int WaitExcept(SOCKET fd, long timeout = 0) {
|
||||
inline static int WaitExcept(SOCKET fd, long timeout = 0) { // NOLINT(*)
|
||||
fd_set wait_set;
|
||||
FD_ZERO(&wait_set);
|
||||
FD_SET(fd, &wait_set);
|
||||
@ -489,7 +490,7 @@ struct SelectHelper {
|
||||
* \return number of active descriptors selected,
|
||||
* return -1 if error occurs
|
||||
*/
|
||||
inline int Select(long timeout = 0) {
|
||||
inline int Select(long timeout = 0) { // NOLINT(*)
|
||||
int ret = Select_(static_cast<int>(maxfd + 1),
|
||||
&read_set, &write_set, &except_set, timeout);
|
||||
if (ret == -1) {
|
||||
@ -500,7 +501,7 @@ struct SelectHelper {
|
||||
|
||||
private:
|
||||
inline static int Select_(int maxfd, fd_set *rfds,
|
||||
fd_set *wfds, fd_set *efds, long timeout) {
|
||||
fd_set *wfds, fd_set *efds, long timeout) { // NOLINT(*)
|
||||
#if !defined(_WIN32)
|
||||
utils::Assert(maxfd < FD_SETSIZE, "maxdf must be smaller than FDSETSIZE");
|
||||
#endif
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
# this is a makefile used to show testcases of rabit
|
||||
.PHONY: all
|
||||
|
||||
all:
|
||||
all: model_recover_10_10k model_recover_10_10k_die_same
|
||||
|
||||
# this experiment test recovery with actually process exit, use keepalive to keep program alive
|
||||
model_recover_10_10k:
|
||||
|
||||
@ -3,6 +3,7 @@ Python interface for rabit
|
||||
Reliable Allreduce and Broadcast Library
|
||||
Author: Tianqi Chen
|
||||
"""
|
||||
# pylint: disable=unused-argument,invalid-name,global-statement,dangerous-default-value,
|
||||
import cPickle as pickle
|
||||
import ctypes
|
||||
import os
|
||||
@ -17,10 +18,12 @@ else:
|
||||
rbtlib = None
|
||||
|
||||
# load in xgboost library
|
||||
def loadlib__(lib = 'standard'):
|
||||
def loadlib__(lib='standard'):
|
||||
"""Load rabit library"""
|
||||
global rbtlib
|
||||
if rbtlib != None:
|
||||
warnings.Warn('rabit.int call was ignored because it has already been initialized', level = 2)
|
||||
warnings.warn('rabit.int call was ignored because it has'\
|
||||
' already been initialized', level=2)
|
||||
return
|
||||
if lib == 'standard':
|
||||
rbtlib = ctypes.cdll.LoadLibrary(WRAPPER_PATH % '')
|
||||
@ -35,6 +38,7 @@ def loadlib__(lib = 'standard'):
|
||||
rbtlib.RabitVersionNumber.restype = ctypes.c_int
|
||||
|
||||
def unloadlib__():
|
||||
"""Unload rabit library"""
|
||||
global rbtlib
|
||||
del rbtlib
|
||||
rbtlib = None
|
||||
@ -51,7 +55,7 @@ def check_err__():
|
||||
"""
|
||||
return
|
||||
|
||||
def init(args = sys.argv, lib = 'standard'):
|
||||
def init(args=sys.argv, lib='standard'):
|
||||
"""
|
||||
intialize the rabit module, call this once before using anything
|
||||
Arguments:
|
||||
@ -145,7 +149,7 @@ def broadcast(data, root):
|
||||
length = ctypes.c_ulong()
|
||||
if root == rank:
|
||||
assert data is not None, 'need to pass in data when broadcasting'
|
||||
s = pickle.dumps(data, protocol = pickle.HIGHEST_PROTOCOL)
|
||||
s = pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL)
|
||||
length.value = len(s)
|
||||
# run first broadcast
|
||||
rbtlib.RabitBroadcast(ctypes.byref(length),
|
||||
@ -179,7 +183,7 @@ DTYPE_ENUM__ = {
|
||||
np.dtype('float64') : 7
|
||||
}
|
||||
|
||||
def allreduce(data, op, prepare_fun = None):
|
||||
def allreduce(data, op, prepare_fun=None):
|
||||
"""
|
||||
perform allreduce, return the result, this function is not thread-safe
|
||||
Arguments:
|
||||
@ -190,7 +194,8 @@ def allreduce(data, op, prepare_fun = None):
|
||||
prepare_fun: lambda data
|
||||
Lazy preprocessing function, if it is not None, prepare_fun(data)
|
||||
will be called by the function before performing allreduce, to intialize the data
|
||||
If the result of Allreduce can be recovered directly, then prepare_fun will NOT be called
|
||||
If the result of Allreduce can be recovered directly,
|
||||
then prepare_fun will NOT be called
|
||||
Returns:
|
||||
the result of allreduce, have same shape as data
|
||||
"""
|
||||
@ -206,12 +211,13 @@ def allreduce(data, op, prepare_fun = None):
|
||||
buf.size, DTYPE_ENUM__[buf.dtype],
|
||||
op, None, None)
|
||||
else:
|
||||
PFUNC = ctypes.CFUNCTYPE(None, ctypes.c_void_p)
|
||||
func_ptr = ctypes.CFUNCTYPE(None, ctypes.c_void_p)
|
||||
def pfunc(args):
|
||||
"""prepare function."""
|
||||
prepare_fun(data)
|
||||
rbtlib.RabitAllreduce(buf.ctypes.data_as(ctypes.c_void_p),
|
||||
buf.size, DTYPE_ENUM__[buf.dtype],
|
||||
op, PFUNC(pfunc), None)
|
||||
op, func_ptr(pfunc), None)
|
||||
check_err__()
|
||||
return buf
|
||||
|
||||
@ -229,7 +235,7 @@ def load_model__(ptr, length):
|
||||
data = (ctypes.c_char * length).from_address(ctypes.addressof(ptr.contents))
|
||||
return pickle.loads(data.raw)
|
||||
|
||||
def load_checkpoint(with_local = False):
|
||||
def load_checkpoint(with_local=False):
|
||||
"""
|
||||
load latest check point
|
||||
Arguments:
|
||||
@ -241,34 +247,34 @@ def load_checkpoint(with_local = False):
|
||||
if returned version == 0, this means no model has been CheckPointed
|
||||
and global_model, local_model returned will be None
|
||||
"""
|
||||
gp = ctypes.POINTER(ctypes.c_char)()
|
||||
gptr = ctypes.POINTER(ctypes.c_char)()
|
||||
global_len = ctypes.c_ulong()
|
||||
if with_local:
|
||||
lp = ctypes.POINTER(ctypes.c_char)()
|
||||
lptr = ctypes.POINTER(ctypes.c_char)()
|
||||
local_len = ctypes.c_ulong()
|
||||
version = rbtlib.RabitLoadCheckPoint(
|
||||
ctypes.byref(gp),
|
||||
ctypes.byref(gptr),
|
||||
ctypes.byref(global_len),
|
||||
ctypes.byref(lp),
|
||||
ctypes.byref(lptr),
|
||||
ctypes.byref(local_len))
|
||||
check_err__()
|
||||
if version == 0:
|
||||
return (version, None, None)
|
||||
return (version,
|
||||
load_model__(gp, global_len.value),
|
||||
load_model__(lp, local_len.value))
|
||||
load_model__(gptr, global_len.value),
|
||||
load_model__(lptr, local_len.value))
|
||||
else:
|
||||
version = rbtlib.RabitLoadCheckPoint(
|
||||
ctypes.byref(gp),
|
||||
ctypes.byref(gptr),
|
||||
ctypes.byref(global_len),
|
||||
None, None)
|
||||
check_err__()
|
||||
if version == 0:
|
||||
return (version, None)
|
||||
return (version,
|
||||
load_model__(gp, global_len.value))
|
||||
load_model__(gptr, global_len.value))
|
||||
|
||||
def checkpoint(global_model, local_model = None):
|
||||
def checkpoint(global_model, local_model=None):
|
||||
"""
|
||||
checkpoint the model, meaning we finished a stage of execution
|
||||
every time we call check point, there is a version number which will increase by one
|
||||
@ -285,16 +291,17 @@ def checkpoint(global_model, local_model = None):
|
||||
while global_model do not need explicit replication.
|
||||
It is recommended to use global_model if possible
|
||||
"""
|
||||
sg = pickle.dumps(global_model)
|
||||
sglobal = pickle.dumps(global_model)
|
||||
if local_model is None:
|
||||
rbtlib.RabitCheckPoint(sg, len(sg), None, 0)
|
||||
rbtlib.RabitCheckPoint(sglobal, len(sglobal), None, 0)
|
||||
check_err__()
|
||||
del sg;
|
||||
del sglobal
|
||||
else:
|
||||
sl = pickle.dumps(local_model)
|
||||
rbtlib.RabitCheckPoint(sg, len(sg), sl, len(sl))
|
||||
slocal = pickle.dumps(local_model)
|
||||
rbtlib.RabitCheckPoint(sglobal, len(sglobal), slocal, len(slocal))
|
||||
check_err__()
|
||||
del sl; del sg;
|
||||
del slocal
|
||||
del sglobal
|
||||
|
||||
def version_number():
|
||||
"""
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
// Copyright by Contributors
|
||||
// implementations in ctypes
|
||||
#define _CRT_SECURE_NO_WARNINGS
|
||||
#define _CRT_SECURE_NO_DEPRECATE
|
||||
@ -60,12 +61,12 @@ inline void Allreduce_(void *sendrecvbuf_,
|
||||
return;
|
||||
case kLong:
|
||||
rabit::Allreduce<OP>
|
||||
(static_cast<long*>(sendrecvbuf_),
|
||||
(static_cast<long*>(sendrecvbuf_), // NOLINT(*)
|
||||
count, prepare_fun, prepare_arg);
|
||||
return;
|
||||
case kULong:
|
||||
rabit::Allreduce<OP>
|
||||
(static_cast<unsigned long*>(sendrecvbuf_),
|
||||
(static_cast<unsigned long*>(sendrecvbuf_), // NOLINT(*)
|
||||
count, prepare_fun, prepare_arg);
|
||||
return;
|
||||
case kFloat:
|
||||
@ -179,7 +180,7 @@ extern "C" {
|
||||
if (s.length() > max_len) {
|
||||
s.resize(max_len - 1);
|
||||
}
|
||||
strcpy(out_name, s.c_str());
|
||||
strcpy(out_name, s.c_str()); // NOLINT(*)
|
||||
*out_len = static_cast<rbt_ulong>(s.length());
|
||||
}
|
||||
void RabitBroadcast(void *sendrecv_data,
|
||||
|
||||
@ -1,18 +1,19 @@
|
||||
#ifndef RABIT_WRAPPER_H_
|
||||
#define RABIT_WRAPPER_H_
|
||||
/*!
|
||||
* Copyright by Contributors
|
||||
* \file rabit_wrapper.h
|
||||
* \author Tianqi Chen
|
||||
* \brief a C style wrapper of rabit
|
||||
* can be used to create wrapper of other languages
|
||||
*/
|
||||
#ifndef RABIT_WRAPPER_H_
|
||||
#define RABIT_WRAPPER_H_
|
||||
#ifdef _MSC_VER
|
||||
#define RABIT_DLL __declspec(dllexport)
|
||||
#else
|
||||
#define RABIT_DLL
|
||||
#endif
|
||||
// manually define unsign long
|
||||
typedef unsigned long rbt_ulong;
|
||||
typedef unsigned long rbt_ulong; // NOLINT(*)
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
@ -122,4 +123,4 @@ extern "C" {
|
||||
#ifdef __cplusplus
|
||||
} // C
|
||||
#endif
|
||||
#endif // XGBOOST_WRAPPER_H_
|
||||
#endif // RABIT_WRAPPER_H_
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user