Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Squashed 'subtree/rabit/' changes from fa99857..e81a11d

Browse Source 
e81a11d Merge pull request #25 from daiyl0320/master
35c3b37 add retry mechanism to ConnectTracker and modify Listen backlog to 128 in rabit_traker.py
c71ed6f try deply doxygen
62e5647 try deply doxygen
732f1c6 try
2fa6e02 ok
0537665 minor
7b59dcb minor
5934950 new doc
f538187 ok
44b6049 new doc
387339b add more
9d4397a chg
2879a48 chg
30e3110 ok
9ff0301 add link translation
6b629c2 k
32e1955 ok
8f4839d fix
93137b2 ok
7eeeb79 reload recommonmark
a8f00cc minor
19b0f01 ok
dd01184 minor
c1cdc19 minor
fcf0f43 try rst
cbc21ae try
62ddfa7 tiny
aefc05c final change
2aee9b4 minor
fe4e7c2 ok
8001983 change to subtitle
5ca33e4 ok
88f7d24 update guide
29d43ab add code
fe8bb3b minor hack for readthedocs
229c71d Merge branch 'master' of ssh://github.com/dmlc/rabit
7424218 ok
d1d45bb Update README.md
1e8813f Update README.md
1ccc990 Update README.md
0323e06 remove readme
679a835 remove theme
7ea5b7c remove numpydoc to napoleon
b73e2be Merge branch 'master' of ssh://github.com/dmlc/rabit
1742283 ok
1838e25 Update python-requirements.txt
bc4e957 ok
fba6fc2 ok
0251101 ok
d50b905 ok
d4f2509 ok
cdf401a ok
fef0ef2 new doc
cef360d ok
c125d2a ok
270a49e add requirments
744f901 get the basic doc
1cb5cad Merge branch 'master' of ssh://github.com/dmlc/rabit
8cc07ba minor
d74f126 Update .travis.yml
52b3dcd Update .travis.yml
099581b Update .travis.yml
1258046 Update .travis.yml
7addac9 Update Makefile
0ea7adf Update .travis.yml
f858856 Update travis_script.sh
d8eac4a Update README.md
3cc49ad lint and travis
ceedf4e fix
fd8920c fix win32
8bbed35 modify
9520b90 Merge pull request #14 from dmlc/hjk41
df14bb1 fix type
f441dc7 replace tab with blankspace
2467942 remove unnecessary include
181ef47 defined long long and ulonglong
1582180 use int32_t to define int and int64_t to define long. in VC long is 32bit
e0b7da0 fix

git-subtree-dir: subtree/rabit
git-subtree-split: e81a11dd7e
This commit is contained in:
tqchen
2015-10-20 19:37:47 -07:00
parent 7258f3353c
commit a16289b204
44 changed files with 1195 additions and 532 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
include/dmlc/io.h
View File

@@ -14,6 +14,7 @@
// include uint64_t only to make io standalone
#ifdef _MSC_VER
/*! \brief uint64 */
typedef unsigned __int64 uint64_t;
#else
#include <inttypes.h>
@@ -24,7 +25,7 @@ namespace dmlc {
/*!
* \brief interface of stream I/O for serialization
*/
class Stream {
class Stream { // NOLINT(*)
public:
/*!
* \brief reads data from a stream
@@ -71,7 +72,7 @@ class Stream {
/*!
* \brief writes a string
* \param str the string to be written/serialized
*/
*/
inline void Write(const std::string &str);
/*!
* \brief loads a string
@@ -94,7 +95,7 @@ class SeekStream: public Stream {
* \brief generic factory function
* create an SeekStream for read only,
* the stream will close the underlying files upon deletion
* error will be reported and the system will exit when create failed
* error will be reported and the system will exit when create failed
* \param uri the uri of the input currently we support
* hdfs://, s3://, and file:// by default file:// will be used
* \param allow_null whether NULL can be returned, or directly report error
@@ -107,12 +108,12 @@ class SeekStream: public Stream {
/*! \brief interface for serializable objects */
class Serializable {
public:
/*!
/*!
* \brief load the model from a stream
* \param fi stream where to load the model from
*/
virtual void Load(Stream *fi) = 0;
/*!
/*!
* \brief saves the model to a stream
* \param fo stream where to save the model to
*/
@@ -123,7 +124,7 @@ class Serializable {
* \brief input split creates that allows reading
* of records from split of data,
* independent part that covers all the dataset
*
*
* see InputSplit::Create for definition of record
*/
class InputSplit {
@@ -141,7 +142,7 @@ class InputSplit {
* this is a hint so may not be enforced,
* but InputSplit will try adjust its internal buffer
* size to the hinted value
* \param chunk_size the chunk size
* \param chunk_size the chunk size
*/
virtual void HintChunkSize(size_t chunk_size) {}
/*! \brief reset the position of InputSplit to beginning */
@@ -150,7 +151,7 @@ class InputSplit {
* \brief get the next record, the returning value
* is valid until next call to NextRecord or NextChunk
* caller can modify the memory content of out_rec
*
*
* For text, out_rec contains a single line
* For recordio, out_rec contains one record content(with header striped)
*
@@ -161,11 +162,11 @@ class InputSplit {
*/
virtual bool NextRecord(Blob *out_rec) = 0;
/*!
* \brief get a chunk of memory that can contain multiple records,
* \brief get a chunk of memory that can contain multiple records,
* the caller needs to parse the content of the resulting chunk,
* for text file, out_chunk can contain data of multiple lines
* for recordio, out_chunk can contain multiple records(including headers)
*
*
* This function ensures there won't be partial record in the chunk
* caller can modify the memory content of out_chunk,
* the memory is valid until next call to NextRecord or NextChunk
@@ -192,9 +193,10 @@ class InputSplit {
* List of possible types: "text", "recordio"
* - "text":
* text file, each line is treated as a record
* input split will split on \n or \r
* input split will split on '\\n' or '\\r'
* - "recordio":
* binary recordio file, see recordio.h
* \return a new input split
* \sa InputSplit::Type
*/
static InputSplit* Create(const char *uri,
@@ -224,7 +226,7 @@ class ostream : public std::basic_ostream<char> {
* \param buffer_size internal streambuf size
*/
explicit ostream(Stream *stream,
size_t buffer_size = 1 << 10)
size_t buffer_size = (1 << 10))
: std::basic_ostream<char>(NULL), buf_(buffer_size) {
this->set_stream(stream);
}
@@ -240,7 +242,7 @@ class ostream : public std::basic_ostream<char> {
buf_.set_stream(stream);
this->rdbuf(&buf_);
}
private:
// internal streambuf
class OutBuf : public std::streambuf {
@@ -251,7 +253,7 @@ class ostream : public std::basic_ostream<char> {
}
// set stream to the buffer
inline void set_stream(Stream *stream);
private:
/*! \brief internal stream by StreamBuf */
Stream *stream_;
@@ -287,7 +289,7 @@ class istream : public std::basic_istream<char> {
* \param buffer_size internal buffer size
*/
explicit istream(Stream *stream,
size_t buffer_size = 1 << 10)
size_t buffer_size = (1 << 10))
: std::basic_istream<char>(NULL), buf_(buffer_size) {
this->set_stream(stream);
}
@@ -325,7 +327,7 @@ class istream : public std::basic_istream<char> {
Stream *stream_;
/*! \brief how many bytes we read so far */
size_t bytes_read_;
/*! \brief internal buffer */
/*! \brief internal buffer */
std::vector<char> buffer_;
// override underflow
inline int_type underflow();
@@ -402,7 +404,7 @@ inline int ostream::OutBuf::overflow(int c) {
// implementations for istream
inline void istream::InBuf::set_stream(Stream *stream) {
stream_ = stream;
this->setg(&buffer_[0], &buffer_[0], &buffer_[0]);
this->setg(&buffer_[0], &buffer_[0], &buffer_[0]);
}
inline int istream::InBuf::underflow() {
char *bhead = &buffer_[0];

121
include/rabit.h
View File

@@ -8,12 +8,18 @@
* rabit.h and serializable.h is all what the user needs to use the rabit interface
* \author Tianqi Chen, Ignacio Cano, Tianyi Zhou
*/
#ifndef RABIT_RABIT_H_
#define RABIT_RABIT_H_
#ifndef RABIT_RABIT_H_ // NOLINT(*)
#define RABIT_RABIT_H_ // NOLINT(*)
#include <string>
#include <vector>
// whether or not use c++11 support
#ifndef DMLC_USE_CXX11
#define DMLC_USE_CXX11 (defined(__GXX_EXPERIMENTAL_CXX0X__) ||\
__cplusplus >= 201103L || defined(_MSC_VER))
#endif
// optionally support of lambda functions in C++11, if available
#if __cplusplus >= 201103L
#if DMLC_USE_CXX11
#include <functional>
#endif // C++11
// contains definition of Serializable
@@ -56,8 +62,8 @@ struct BitOR;
* \param argv the array of input arguments
*/
inline void Init(int argc, char *argv[]);
/*!
* \brief finalizes the rabit engine, call this function after you finished with all the jobs
/*!
* \brief finalizes the rabit engine, call this function after you finished with all the jobs
*/
inline void Finalize(void);
/*! \brief gets rank of the current process */
@@ -71,7 +77,7 @@ inline bool IsDistributed(void);
inline std::string GetProcessorName(void);
/*!
* \brief prints the msg to the tracker,
* this function can be used to communicate progress information to
* this function can be used to communicate progress information to
* the user who monitors the tracker
* \param msg the message to be printed
*/
@@ -89,7 +95,7 @@ inline void TrackerPrintf(const char *fmt, ...);
/*!
* \brief broadcasts a memory region to every node from the root
*
* Example: int a = 1; Broadcast(&a, sizeof(a), root);
* Example: int a = 1; Broadcast(&a, sizeof(a), root);
* \param sendrecv_data the pointer to the send/receive buffer,
* \param size the data size
* \param root the process root
@@ -113,48 +119,54 @@ inline void Broadcast(std::vector<DType> *sendrecv_data, int root);
*/
inline void Broadcast(std::string *sendrecv_data, int root);
/*!
* \brief performs in-place Allreduce on sendrecvbuf
* \brief performs in-place Allreduce on sendrecvbuf
* this function is NOT thread-safe
*
* Example Usage: the following code does an Allreduce and outputs the sum as the result
* vector<int> data(10);
* ...
* Allreduce<op::Sum>(&data[0], data.size());
* ...
* \code{.cpp}
* vector<int> data(10);
* ...
* Allreduce<op::Sum>(&data[0], data.size());
* ...
* \endcode
*
* \param sendrecvbuf buffer for both sending and receiving data
* \param count number of elements to be reduced
* \param prepare_fun Lazy preprocessing function, if it is not NULL, prepare_fun(prepare_arg)
* will be called by the function before performing Allreduce in order to initialize the data in sendrecvbuf.
* If the result of Allreduce can be recovered directly, then prepare_func will NOT be called
* \param prepare_arg argument used to pass into the lazy preprocessing function
* \tparam OP see namespace op, reduce operator
* \param prepare_arg argument used to pass into the lazy preprocessing function
* \tparam OP see namespace op, reduce operator
* \tparam DType data type
*/
template<typename OP, typename DType>
inline void Allreduce(DType *sendrecvbuf, size_t count,
void (*prepare_fun)(void *arg) = NULL,
void (*prepare_fun)(void *) = NULL,
void *prepare_arg = NULL);
// C++11 support for lambda prepare function
#if __cplusplus >= 201103L
#if DMLC_USE_CXX11
/*!
* \brief performs in-place Allreduce, on sendrecvbuf
* with a prepare function specified by a lambda function
*
* Example Usage: the following code does an Allreduce and outputs the sum as the result
* vector<int> data(10);
* ...
* Allreduce<op::Sum>(&data[0], data.size(), [&]() {
* for (int i = 0; i < 10; ++i) {
* data[i] = i;
* }
* });
* Example Usage:
* \code{.cpp}
* // the following code does an Allreduce and outputs the sum as the result
* vector<int> data(10);
* ...
* Allreduce<op::Sum>(&data[0], data.size(), [&]() {
* for (int i = 0; i < 10; ++i) {
* data[i] = i;
* }
* });
* ...
* \endcode
* \param sendrecvbuf buffer for both sending and receiving data
* \param count number of elements to be reduced
* \param prepare_fun Lazy lambda preprocessing function, prepare_fun() will be invoked
* by the function before performing Allreduce in order to initialize the data in sendrecvbuf.
* If the result of Allreduce can be recovered directly, then prepare_func will NOT be called
* \tparam OP see namespace op, reduce operator
* \tparam OP see namespace op, reduce operator
* \tparam DType data type
*/
template<typename OP, typename DType>
@@ -168,19 +180,20 @@ inline void Allreduce(DType *sendrecvbuf, size_t count,
* is the same in every node
* \param local_model pointer to the local model that is specific to the current node/rank
* this can be NULL when no local model is needed
*
*
* \return the version number of the check point loaded
* if returned version == 0, this means no model has been CheckPointed
* the p_model is not touched, users should do the necessary initialization by themselves
*
* Common usage example:
* int iter = rabit::LoadCheckPoint(&model);
* if (iter == 0) model.InitParameters();
* for (i = iter; i < max_iter; ++i) {
* do many things, include allreduce
* rabit::CheckPoint(model);
* }
*
* \code{.cpp}
* // Example usage code of LoadCheckPoint
* int iter = rabit::LoadCheckPoint(&model);
* if (iter == 0) model.InitParameters();
* for (i = iter; i < max_iter; ++i) {
* // do many things, include allreduce
* rabit::CheckPoint(model);
* }
* \endcode
* \sa CheckPoint, VersionNumber
*/
inline int LoadCheckPoint(Serializable *global_model,
@@ -188,7 +201,7 @@ inline int LoadCheckPoint(Serializable *global_model,
/*!
* \brief checkpoints the model, meaning a stage of execution has finished.
* every time we call check point, a version number will be increased by one
*
*
* \param global_model pointer to the globally shared model/state
* when calling this function, the caller needs to guarantee that the global_model
* is the same in every node
@@ -204,16 +217,16 @@ inline void CheckPoint(const Serializable *global_model,
/*!
* \brief This function can be used to replace CheckPoint for global_model only,
* when certain condition is met (see detailed explanation).
*
*
* This is a "lazy" checkpoint such that only the pointer to the global_model is
* remembered and no memory copy is taken. To use this function, the user MUST ensure that:
* The global_model must remain unchanged until the last call of Allreduce/Broadcast in the current version finishes.
* In other words, the global_model model can be changed only between the last call of
* In other words, the global_model model can be changed only between the last call of
* Allreduce/Broadcast and LazyCheckPoint, both in the same version
*
*
* For example, suppose the calling sequence is:
* LazyCheckPoint, code1, Allreduce, code2, Broadcast, code3, LazyCheckPoint/(or can be CheckPoint)
*
*
* Then the user MUST only change the global_model in code3.
*
* The use of LazyCheckPoint instead of CheckPoint will improve the efficiency of the program.
@@ -235,36 +248,36 @@ namespace engine {
class ReduceHandle;
} // namespace engine
/*!
* \brief template class to make customized reduce and all reduce easy
* Do not use reducer directly in the function you call Finalize,
* \brief template class to make customized reduce and all reduce easy
* Do not use reducer directly in the function you call Finalize,
* because the destructor can execute after Finalize
* \tparam DType data type that to be reduced
* \tparam freduce the customized reduction function
* DType must be a struct, with no pointer
*/
template<typename DType, void (*freduce)(DType &dst, const DType &src)>
template<typename DType, void (*freduce)(DType &dst, const DType &src)> // NOLINT(*)
class Reducer {
public:
Reducer(void);
/*!
* \brief customized in-place all reduce operation
* \brief customized in-place all reduce operation
* \param sendrecvbuf the in place send-recv buffer
* \param count number of elements to be reduced
* \param prepare_fun Lazy preprocessing function, if it is not NULL, prepare_fun(prepare_arg)
* will be called by the function before performing Allreduce, to initialize the data in sendrecvbuf.
* If the result of Allreduce can be recovered directly, then prepare_func will NOT be called
* \param prepare_arg argument used to pass into the lazy preprocessing function
* \param prepare_arg argument used to pass into the lazy preprocessing function
*/
inline void Allreduce(DType *sendrecvbuf, size_t count,
void (*prepare_fun)(void *arg) = NULL,
void (*prepare_fun)(void *) = NULL,
void *prepare_arg = NULL);
#if __cplusplus >= 201103L
#if DMLC_USE_CXX11
/*!
* \brief customized in-place all reduce operation, with lambda function as preprocessor
* \param sendrecvbuf pointer to the array of objects to be reduced
* \param count number of elements to be reduced
* \param prepare_fun lambda function executed to prepare the data, if necessary
*/
*/
inline void Allreduce(DType *sendrecvbuf, size_t count,
std::function<void()> prepare_fun);
#endif
@@ -278,7 +291,7 @@ class Reducer {
* this class defines complex reducer handles all the data structure that can be
* serialized/deserialized into fixed size buffer
* Do not use reducer directly in the function you call Finalize, because the destructor can execute after Finalize
*
*
* \tparam DType data type that to be reduced, DType must contain the following functions:
* \tparam freduce the customized reduction function
* (1) Save(IStream &fs) (2) Load(IStream &fs) (3) Reduce(const DType &src, size_t max_nbyte)
@@ -288,7 +301,7 @@ class SerializeReducer {
public:
SerializeReducer(void);
/*!
* \brief customized in-place all reduce operation
* \brief customized in-place all reduce operation
* \param sendrecvobj pointer to the array of objects to be reduced
* \param max_nbyte maximum amount of memory needed to serialize each object
* this includes budget limit for intermediate and final result
@@ -296,14 +309,14 @@ class SerializeReducer {
* \param prepare_fun Lazy preprocessing function, if it is not NULL, prepare_fun(prepare_arg)
* will be called by the function before performing Allreduce, to initialize the data in sendrecvbuf.
* If the result of Allreduce can be recovered directly, then the prepare_func will NOT be called
* \param prepare_arg argument used to pass into the lazy preprocessing function
* \param prepare_arg argument used to pass into the lazy preprocessing function
*/
inline void Allreduce(DType *sendrecvobj,
size_t max_nbyte, size_t count,
void (*prepare_fun)(void *arg) = NULL,
void (*prepare_fun)(void *) = NULL,
void *prepare_arg = NULL);
// C++11 support for lambda prepare function
#if __cplusplus >= 201103L
#if DMLC_USE_CXX11
/*!
* \brief customized in-place all reduce operation, with lambda function as preprocessor
* \param sendrecvobj pointer to the array of objects to be reduced
@@ -311,7 +324,7 @@ class SerializeReducer {
* this includes budget limit for intermediate and final result
* \param count number of elements to be reduced
* \param prepare_fun lambda function executed to prepare the data, if necessary
*/
*/
inline void Allreduce(DType *sendrecvobj,
size_t max_nbyte, size_t count,
std::function<void()> prepare_fun);
@@ -326,4 +339,4 @@ class SerializeReducer {
} // namespace rabit
// implementation of template functions
#include "./rabit/rabit-inl.h"
#endif // RABIT_RABIT_H_
#endif // RABIT_RABIT_H_ // NOLINT(*)

4
include/rabit/engine.h
View File

@@ -183,7 +183,9 @@ enum DataType {
kLong = 4,
kULong = 5,
kFloat = 6,
kDouble = 7
kDouble = 7,
kLongLong = 8,
kULongLong = 9
};
} // namespace mpi
/*!

8
include/rabit/io.h
View File

@@ -4,8 +4,8 @@
* \brief utilities with different serializable implementations
* \author Tianqi Chen
*/
#ifndef RABIT_UTILS_IO_H_
#define RABIT_UTILS_IO_H_
#ifndef RABIT_IO_H_
#define RABIT_IO_H_
#include <cstdio>
#include <vector>
#include <cstring>
@@ -51,6 +51,7 @@ struct MemoryFixSizeBuffer : public SeekStream {
virtual bool AtEnd(void) const {
return curr_ptr_ == buffer_size_;
}
private:
/*! \brief in memory buffer */
char *p_buffer_;
@@ -93,6 +94,7 @@ struct MemoryBufferStream : public SeekStream {
virtual bool AtEnd(void) const {
return curr_ptr_ == p_buffer_->length();
}
private:
/*! \brief in memory buffer */
std::string *p_buffer_;
@@ -101,4 +103,4 @@ struct MemoryBufferStream : public SeekStream {
}; // class MemoryBufferStream
} // namespace utils
} // namespace rabit
#endif // RABIT_UTILS_IO_H_
#endif // RABIT_IO_H_

77
include/rabit/rabit-inl.h
View File

@@ -1,12 +1,15 @@
/*!
* Copyright by Contributors
* \file rabit-inl.h
* \brief implementation of inline template function for rabit interface
*
* \author Tianqi Chen
*/
#ifndef RABIT_RABIT_INL_H
#define RABIT_RABIT_INL_H
#ifndef RABIT_RABIT_INL_H_
#define RABIT_RABIT_INL_H_
// use engine for implementation
#include <vector>
#include <string>
#include "./io.h"
#include "./utils.h"
#include "../rabit.h"
@@ -30,15 +33,15 @@ inline DataType GetType<int>(void) {
return kInt;
}
template<>
inline DataType GetType<unsigned>(void) {
inline DataType GetType<unsigned int>(void) { // NOLINT(*)
return kUInt;
}
template<>
inline DataType GetType<long>(void) {
inline DataType GetType<long>(void) { // NOLINT(*)
return kLong;
}
template<>
inline DataType GetType<unsigned long>(void) {
inline DataType GetType<unsigned long>(void) { // NOLINT(*)
return kULong;
}
template<>
@@ -49,47 +52,55 @@ template<>
inline DataType GetType<double>(void) {
return kDouble;
}
template<>
inline DataType GetType<long long>(void) { // NOLINT(*)
return kLongLong;
}
template<>
inline DataType GetType<unsigned long long>(void) { // NOLINT(*)
return kULongLong;
}
} // namespace mpi
} // namespace engine
namespace op {
struct Max {
const static engine::mpi::OpType kType = engine::mpi::kMax;
static const engine::mpi::OpType kType = engine::mpi::kMax;
template<typename DType>
inline static void Reduce(DType &dst, const DType &src) {
inline static void Reduce(DType &dst, const DType &src) { // NOLINT(*)
if (dst < src) dst = src;
}
};
struct Min {
const static engine::mpi::OpType kType = engine::mpi::kMin;
static const engine::mpi::OpType kType = engine::mpi::kMin;
template<typename DType>
inline static void Reduce(DType &dst, const DType &src) {
inline static void Reduce(DType &dst, const DType &src) { // NOLINT(*)
if (dst > src) dst = src;
}
};
struct Sum {
const static engine::mpi::OpType kType = engine::mpi::kSum;
static const engine::mpi::OpType kType = engine::mpi::kSum;
template<typename DType>
inline static void Reduce(DType &dst, const DType &src) {
inline static void Reduce(DType &dst, const DType &src) { // NOLINT(*)
dst += src;
}
};
struct BitOR {
const static engine::mpi::OpType kType = engine::mpi::kBitwiseOR;
static const engine::mpi::OpType kType = engine::mpi::kBitwiseOR;
template<typename DType>
inline static void Reduce(DType &dst, const DType &src) {
inline static void Reduce(DType &dst, const DType &src) { // NOLINT(*)
dst |= src;
}
};
template<typename OP, typename DType>
inline void Reducer(const void *src_, void *dst_, int len, const MPI::Datatype &dtype) {
const DType *src = (const DType*)src_;
DType *dst = (DType*)dst_;
DType *dst = (DType*)dst_; // NOLINT(*)
for (int i = 0; i < len; ++i) {
OP::Reduce(dst[i], src[i]);
}
}
} // namespace op
} // namespace op
// intialize the rabit engine
inline void Init(int argc, char *argv[]) {
@@ -144,23 +155,23 @@ inline void Broadcast(std::string *sendrecv_data, int root) {
// perform inplace Allreduce
template<typename OP, typename DType>
inline void Allreduce(DType *sendrecvbuf, size_t count,
void (*prepare_fun)(void *arg),
void (*prepare_fun)(void *arg),
void *prepare_arg) {
engine::Allreduce_(sendrecvbuf, sizeof(DType), count, op::Reducer<OP,DType>,
engine::Allreduce_(sendrecvbuf, sizeof(DType), count, op::Reducer<OP, DType>,
engine::mpi::GetType<DType>(), OP::kType, prepare_fun, prepare_arg);
}
// C++11 support for lambda prepare function
#if __cplusplus >= 201103L
#if DMLC_USE_CXX11
inline void InvokeLambda_(void *fun) {
(*static_cast<std::function<void()>*>(fun))();
}
template<typename OP, typename DType>
inline void Allreduce(DType *sendrecvbuf, size_t count, std::function<void()> prepare_fun) {
engine::Allreduce_(sendrecvbuf, sizeof(DType), count, op::Reducer<OP,DType>,
engine::Allreduce_(sendrecvbuf, sizeof(DType), count, op::Reducer<OP, DType>,
engine::mpi::GetType<DType>(), OP::kType, InvokeLambda_, &prepare_fun);
}
#endif // C++11
#endif // C++11
// print message to the tracker
inline void TrackerPrint(const std::string &msg) {
@@ -215,15 +226,16 @@ inline void ReducerSafe_(const void *src_, void *dst_, int len_, const MPI::Data
}
}
// function to perform reduction for Reducer
template<typename DType, void (*freduce)(DType &dst, const DType &src)>
inline void ReducerAlign_(const void *src_, void *dst_, int len_, const MPI::Datatype &dtype) {
template<typename DType, void (*freduce)(DType &dst, const DType &src)> // NOLINT(*)
inline void ReducerAlign_(const void *src_, void *dst_,
int len_, const MPI::Datatype &dtype) {
const DType *psrc = reinterpret_cast<const DType*>(src_);
DType *pdst = reinterpret_cast<DType*>(dst_);
for (int i = 0; i < len_; ++i) {
freduce(pdst[i], psrc[i]);
}
}
template<typename DType, void (*freduce)(DType &dst, const DType &src)>
template<typename DType, void (*freduce)(DType &dst, const DType &src)> // NOLINT(*)
inline Reducer<DType, freduce>::Reducer(void) {
// it is safe to directly use handle for aligned data types
if (sizeof(DType) == 8 || sizeof(DType) == 4 || sizeof(DType) == 1) {
@@ -232,7 +244,7 @@ inline Reducer<DType, freduce>::Reducer(void) {
this->handle_.Init(ReducerSafe_<DType, freduce>, sizeof(DType));
}
}
template<typename DType, void (*freduce)(DType &dst, const DType &src)>
template<typename DType, void (*freduce)(DType &dst, const DType &src)> // NOLINT(*)
inline void Reducer<DType, freduce>::Allreduce(DType *sendrecvbuf, size_t count,
void (*prepare_fun)(void *arg),
void *prepare_arg) {
@@ -240,13 +252,14 @@ inline void Reducer<DType, freduce>::Allreduce(DType *sendrecvbuf, size_t count,
}
// function to perform reduction for SerializeReducer
template<typename DType>
inline void SerializeReducerFunc_(const void *src_, void *dst_, int len_, const MPI::Datatype &dtype) {
inline void SerializeReducerFunc_(const void *src_, void *dst_,
int len_, const MPI::Datatype &dtype) {
int nbytes = engine::ReduceHandle::TypeSize(dtype);
// temp space
DType tsrc, tdst;
for (int i = 0; i < len_; ++i) {
utils::MemoryFixSizeBuffer fsrc((char*)(src_) + i * nbytes, nbytes);
utils::MemoryFixSizeBuffer fdst((char*)(dst_) + i * nbytes, nbytes);
utils::MemoryFixSizeBuffer fsrc((char*)(src_) + i * nbytes, nbytes); // NOLINT(*)
utils::MemoryFixSizeBuffer fdst((char*)(dst_) + i * nbytes, nbytes); // NOLINT(*)
tsrc.Load(fsrc);
tdst.Load(fdst);
// govern const check
@@ -288,8 +301,8 @@ inline void SerializeReducer<DType>::Allreduce(DType *sendrecvobj,
// setup closure
SerializeReduceClosure<DType> c;
c.sendrecvobj = sendrecvobj; c.max_nbyte = max_nbyte; c.count = count;
c.prepare_fun = prepare_fun; c.prepare_arg = prepare_arg; c.p_buffer = &buffer_;
// invoke here
c.prepare_fun = prepare_fun; c.prepare_arg = prepare_arg; c.p_buffer = &buffer_;
// invoke here
handle_.Allreduce(BeginPtr(buffer_), max_nbyte, count,
SerializeReduceClosure<DType>::Invoke, &c);
for (size_t i = 0; i < count; ++i) {
@@ -298,8 +311,8 @@ inline void SerializeReducer<DType>::Allreduce(DType *sendrecvobj,
}
}
#if __cplusplus >= 201103L
template<typename DType, void (*freduce)(DType &dst, const DType &src)>
#if DMLC_USE_CXX11
template<typename DType, void (*freduce)(DType &dst, const DType &src)> // NOLINT(*)g
inline void Reducer<DType, freduce>::Allreduce(DType *sendrecvbuf, size_t count,
std::function<void()> prepare_fun) {
this->Allreduce(sendrecvbuf, count, InvokeLambda_, &prepare_fun);
@@ -312,4 +325,4 @@ inline void SerializeReducer<DType>::Allreduce(DType *sendrecvobj,
}
#endif
} // namespace rabit
#endif
#endif // RABIT_RABIT_INL_H_

3
include/rabit/timer.h
View File

@@ -1,4 +1,5 @@
/*!
* Copyright by Contributors
* \file timer.h
* \brief This file defines the utils for timing
* \author Tianqi Chen, Nacho, Tianyi
@@ -18,7 +19,6 @@ namespace utils {
* \brief return time in seconds, not cross platform, avoid to use this in most places
*/
inline double GetTime(void) {
// TODO: use c++11 chrono when c++11 was available
#ifdef __MACH__
clock_serv_t cclock;
mach_timespec_t mts;
@@ -32,7 +32,6 @@ inline double GetTime(void) {
utils::Check(clock_gettime(CLOCK_REALTIME, &ts) == 0, "failed to get time");
return static_cast<double>(ts.tv_sec) + static_cast<double>(ts.tv_nsec) * 1e-9;
#else
// TODO: add MSVC macro, and MSVC timer
return static_cast<double>(time(NULL));
#endif
#endif

16
include/rabit/utils.h
View File

@@ -27,7 +27,7 @@
#else
#ifdef _FILE_OFFSET_BITS
#if _FILE_OFFSET_BITS == 32
#pragma message ("Warning: FILE OFFSET BITS defined to be 32 bit")
#pragma message("Warning: FILE OFFSET BITS defined to be 32 bit")
#endif
#endif
@@ -59,17 +59,17 @@ namespace utils {
const int kPrintBuffer = 1 << 12;
#ifndef RABIT_CUSTOMIZE_MSG_
/*!
/*!
* \brief handling of Assert error, caused by inappropriate input
* \param msg error message
* \param msg error message
*/
inline void HandleAssertError(const char *msg) {
fprintf(stderr, "AssertError:%s\n", msg);
exit(-1);
}
/*!
/*!
* \brief handling of Check error, caused by inappropriate input
* \param msg error message
* \param msg error message
*/
inline void HandleCheckError(const char *msg) {
fprintf(stderr, "%s\n", msg);
@@ -163,7 +163,7 @@ inline std::FILE *FopenCheck(const char *fname, const char *flag) {
// easy utils that can be directly accessed in xgboost
/*! \brief get the beginning address of a vector */
template<typename T>
inline T *BeginPtr(std::vector<T> &vec) {
inline T *BeginPtr(std::vector<T> &vec) { // NOLINT(*)
if (vec.size() == 0) {
return NULL;
} else {
@@ -172,14 +172,14 @@ inline T *BeginPtr(std::vector<T> &vec) {
}
/*! \brief get the beginning address of a vector */
template<typename T>
inline const T *BeginPtr(const std::vector<T> &vec) {
inline const T *BeginPtr(const std::vector<T> &vec) { // NOLINT(*)
if (vec.size() == 0) {
return NULL;
} else {
return &vec[0];
}
}
inline char* BeginPtr(std::string &str) {
inline char* BeginPtr(std::string &str) { // NOLINT(*)
if (str.length() == 0) return NULL;
return &str[0];
}

14
include/rabit_serializable.h
View File

@@ -4,8 +4,8 @@
* \brief defines serializable interface of rabit
* \author Tianqi Chen
*/
#ifndef RABIT_RABIT_SERIALIZABLE_H_
#define RABIT_RABIT_SERIALIZABLE_H_
#ifndef RABIT_SERIALIZABLE_H_
#define RABIT_SERIALIZABLE_H_
#include <vector>
#include <string>
#include "./rabit/utils.h"
@@ -13,15 +13,15 @@
namespace rabit {
/*!
* \brief defines stream used in rabit
* see definition of Stream in dmlc/io.h
* \brief defines stream used in rabit
* see definition of Stream in dmlc/io.h
*/
typedef dmlc::Stream Stream;
/*!
* \brief defines serializable objects used in rabit
* see definition of Serializable in dmlc/io.h
* \brief defines serializable objects used in rabit
* see definition of Serializable in dmlc/io.h
*/
typedef dmlc::Serializable Serializable;
} // namespace rabit
#endif // RABIT_RABIT_SERIALIZABLE_H_
#endif // RABIT_SERIALIZABLE_H_