Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

lint and travis

Browse Source
This commit is contained in:
tqchen
2015-07-03 15:15:11 -07:00
parent ceedf4ea96
commit 3cc49ad0e8
27 changed files with 423 additions and 296 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
wrapper/rabit.py
View File

@@ -3,6 +3,7 @@ Python interface for rabit
Reliable Allreduce and Broadcast Library
Author: Tianqi Chen
"""
# pylint: disable=unused-argument,invalid-name,global-statement,dangerous-default-value,
import cPickle as pickle
import ctypes
import os
@@ -17,10 +18,12 @@ else:
rbtlib = None
# load in xgboost library
def loadlib__(lib = 'standard'):
def loadlib__(lib='standard'):
"""Load rabit library"""
global rbtlib
if rbtlib != None:
warnings.Warn('rabit.int call was ignored because it has already been initialized', level = 2)
warnings.warn('rabit.int call was ignored because it has'\
' already been initialized', level=2)
return
if lib == 'standard':
rbtlib = ctypes.cdll.LoadLibrary(WRAPPER_PATH % '')
@@ -35,6 +38,7 @@ def loadlib__(lib = 'standard'):
rbtlib.RabitVersionNumber.restype = ctypes.c_int
def unloadlib__():
"""Unload rabit library"""
global rbtlib
del rbtlib
rbtlib = None
@@ -45,13 +49,13 @@ MIN = 1
SUM = 2
BITOR = 3
def check_err__():
def check_err__():
"""
reserved function used to check error
reserved function used to check error
"""
return
def init(args = sys.argv, lib = 'standard'):
def init(args=sys.argv, lib='standard'):
"""
intialize the rabit module, call this once before using anything
Arguments:
@@ -69,7 +73,7 @@ def init(args = sys.argv, lib = 'standard'):
def finalize():
"""
finalize the rabit engine, call this function after you finished all jobs
finalize the rabit engine, call this function after you finished all jobs
"""
rbtlib.RabitFinalize()
check_err__()
@@ -132,7 +136,7 @@ def broadcast(data, root):
print '@node[%d] after-broadcast: s=\"%s\"' % (rank, str(s))
rabit.finalize()
```
Arguments:
data: anytype that can be pickled
input data, if current rank does not equal root, this can be None
@@ -145,12 +149,12 @@ def broadcast(data, root):
length = ctypes.c_ulong()
if root == rank:
assert data is not None, 'need to pass in data when broadcasting'
s = pickle.dumps(data, protocol = pickle.HIGHEST_PROTOCOL)
s = pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL)
length.value = len(s)
# run first broadcast
rbtlib.RabitBroadcast(ctypes.byref(length),
ctypes.sizeof(ctypes.c_ulong),
root)
root)
check_err__()
if root != rank:
dptr = (ctypes.c_char * length.value)()
@@ -179,18 +183,19 @@ DTYPE_ENUM__ = {
np.dtype('float64') : 7
}
def allreduce(data, op, prepare_fun = None):
def allreduce(data, op, prepare_fun=None):
"""
perform allreduce, return the result, this function is not thread-safe
Arguments:
data: numpy ndarray
input data
input data
op: int
reduction operators, can be MIN, MAX, SUM, BITOR
prepare_fun: lambda data
Lazy preprocessing function, if it is not None, prepare_fun(data)
will be called by the function before performing allreduce, to intialize the data
If the result of Allreduce can be recovered directly, then prepare_fun will NOT be called
If the result of Allreduce can be recovered directly,
then prepare_fun will NOT be called
Returns:
the result of allreduce, have same shape as data
"""
@@ -206,12 +211,13 @@ def allreduce(data, op, prepare_fun = None):
buf.size, DTYPE_ENUM__[buf.dtype],
op, None, None)
else:
PFUNC = ctypes.CFUNCTYPE(None, ctypes.c_void_p)
func_ptr = ctypes.CFUNCTYPE(None, ctypes.c_void_p)
def pfunc(args):
"""prepare function."""
prepare_fun(data)
rbtlib.RabitAllreduce(buf.ctypes.data_as(ctypes.c_void_p),
buf.size, DTYPE_ENUM__[buf.dtype],
op, PFUNC(pfunc), None)
op, func_ptr(pfunc), None)
check_err__()
return buf
@@ -229,49 +235,49 @@ def load_model__(ptr, length):
data = (ctypes.c_char * length).from_address(ctypes.addressof(ptr.contents))
return pickle.loads(data.raw)
def load_checkpoint(with_local = False):
def load_checkpoint(with_local=False):
"""
load latest check point
Arguments:
with_local: boolean [default = False]
whether the checkpoint contains local model
Returns:
Returns:
if with_local: return (version, gobal_model, local_model)
else return (version, gobal_model)
if returned version == 0, this means no model has been CheckPointed
and global_model, local_model returned will be None
"""
gp = ctypes.POINTER(ctypes.c_char)()
gptr = ctypes.POINTER(ctypes.c_char)()
global_len = ctypes.c_ulong()
if with_local:
lp = ctypes.POINTER(ctypes.c_char)()
lptr = ctypes.POINTER(ctypes.c_char)()
local_len = ctypes.c_ulong()
version = rbtlib.RabitLoadCheckPoint(
ctypes.byref(gp),
ctypes.byref(gptr),
ctypes.byref(global_len),
ctypes.byref(lp),
ctypes.byref(lptr),
ctypes.byref(local_len))
check_err__()
if version == 0:
return (version, None, None)
return (version,
load_model__(gp, global_len.value),
load_model__(lp, local_len.value))
load_model__(gptr, global_len.value),
load_model__(lptr, local_len.value))
else:
version = rbtlib.RabitLoadCheckPoint(
ctypes.byref(gp),
ctypes.byref(gptr),
ctypes.byref(global_len),
None, None)
check_err__()
if version == 0:
return (version, None)
return (version,
load_model__(gp, global_len.value))
def checkpoint(global_model, local_model = None):
load_model__(gptr, global_len.value))
def checkpoint(global_model, local_model=None):
"""
checkpoint the model, meaning we finished a stage of execution
every time we call check point, there is a version number which will increase by one
every time we call check point, there is a version number which will increase by one
Arguments:
global_model: anytype that can be pickled
@@ -285,16 +291,17 @@ def checkpoint(global_model, local_model = None):
while global_model do not need explicit replication.
It is recommended to use global_model if possible
"""
sg = pickle.dumps(global_model)
sglobal = pickle.dumps(global_model)
if local_model is None:
rbtlib.RabitCheckPoint(sg, len(sg), None, 0)
rbtlib.RabitCheckPoint(sglobal, len(sglobal), None, 0)
check_err__()
del sg;
del sglobal
else:
sl = pickle.dumps(local_model)
rbtlib.RabitCheckPoint(sg, len(sg), sl, len(sl))
slocal = pickle.dumps(local_model)
rbtlib.RabitCheckPoint(sglobal, len(sglobal), slocal, len(slocal))
check_err__()
del sl; del sg;
del slocal
del sglobal
def version_number():
"""

13
wrapper/rabit_wrapper.cc
View File

@@ -1,3 +1,4 @@
// Copyright by Contributors
// implementations in ctypes
#define _CRT_SECURE_NO_WARNINGS
#define _CRT_SECURE_NO_DEPRECATE
@@ -28,7 +29,7 @@ struct FHelper<op::BitOR, DType> {
void (*prepare_fun)(void *arg),
void *prepare_arg) {
utils::Error("DataType does not support bitwise or operation");
}
}
};
template<typename OP>
inline void Allreduce_(void *sendrecvbuf_,
@@ -60,12 +61,12 @@ inline void Allreduce_(void *sendrecvbuf_,
return;
case kLong:
rabit::Allreduce<OP>
(static_cast<long*>(sendrecvbuf_),
(static_cast<long*>(sendrecvbuf_), // NOLINT(*)
count, prepare_fun, prepare_arg);
return;
case kULong:
rabit::Allreduce<OP>
(static_cast<unsigned long*>(sendrecvbuf_),
(static_cast<unsigned long*>(sendrecvbuf_), // NOLINT(*)
count, prepare_fun, prepare_arg);
return;
case kFloat:
@@ -135,7 +136,7 @@ struct ReadWrapper : public Serializable {
}
virtual void Save(Stream *fo) const {
utils::Error("not implemented");
}
}
};
struct WriteWrapper : public Serializable {
const char *data;
@@ -179,7 +180,7 @@ extern "C" {
if (s.length() > max_len) {
s.resize(max_len - 1);
}
strcpy(out_name, s.c_str());
strcpy(out_name, s.c_str()); // NOLINT(*)
*out_len = static_cast<rbt_ulong>(s.length());
}
void RabitBroadcast(void *sendrecv_data,
@@ -218,7 +219,7 @@ extern "C" {
*out_local_model = BeginPtr(local_buffer);
*out_local_len = static_cast<rbt_ulong>(local_buffer.length());
}
return version;
return version;
}
void RabitCheckPoint(const char *global_model,
rbt_ulong global_len,

27
wrapper/rabit_wrapper.h
View File

@@ -1,18 +1,19 @@
#ifndef RABIT_WRAPPER_H_
#define RABIT_WRAPPER_H_
/*!
* Copyright by Contributors
* \file rabit_wrapper.h
* \author Tianqi Chen
* \brief a C style wrapper of rabit
* can be used to create wrapper of other languages
*/
#ifndef RABIT_WRAPPER_H_
#define RABIT_WRAPPER_H_
#ifdef _MSC_VER
#define RABIT_DLL __declspec(dllexport)
#else
#define RABIT_DLL
#endif
// manually define unsign long
typedef unsigned long rbt_ulong;
typedef unsigned long rbt_ulong; // NOLINT(*)
#ifdef __cplusplus
extern "C" {
@@ -23,8 +24,8 @@ extern "C" {
* \param argv the array of input arguments
*/
RABIT_DLL void RabitInit(int argc, char *argv[]);
/*!
* \brief finalize the rabit engine, call this function after you finished all jobs
/*!
* \brief finalize the rabit engine, call this function after you finished all jobs
*/
RABIT_DLL void RabitFinalize(void);
/*! \brief get rank of current process */
@@ -37,9 +38,9 @@ extern "C" {
* the user who monitors the tracker
* \param msg the message to be printed
*/
RABIT_DLL void RabitTrackerPrint(const char *msg);
RABIT_DLL void RabitTrackerPrint(const char *msg);
/*!
* \brief get name of processor
* \brief get name of processor
* \param out_name hold output string
* \param out_len hold length of output string
* \param max_len maximum buffer length of input
@@ -50,7 +51,7 @@ extern "C" {
/*!
* \brief broadcast an memory region to all others from root
*
* Example: int a = 1; Broadcast(&a, sizeof(a), root);
* Example: int a = 1; Broadcast(&a, sizeof(a), root);
* \param sendrecv_data the pointer to send or recive buffer,
* \param size the size of the data
* \param root the root of process
@@ -58,7 +59,7 @@ extern "C" {
RABIT_DLL void RabitBroadcast(void *sendrecv_data,
rbt_ulong size, int root);
/*!
* \brief perform in-place allreduce, on sendrecvbuf
* \brief perform in-place allreduce, on sendrecvbuf
* this function is NOT thread-safe
*
* Example Usage: the following code gives sum of the result
@@ -81,14 +82,14 @@ extern "C" {
int enum_op,
void (*prepare_fun)(void *arg),
void *prepare_arg);
/*!
* \brief load latest check point
* \param out_global_model hold output of serialized global_model
* \param out_global_len the output length of serialized global model
* \param out_local_model hold output of serialized local_model, can be NULL
* \param out_local_len the output length of serialized local model, can be NULL
*
*
* \return the version number of check point loaded
* if returned version == 0, this means no model has been CheckPointed
* nothing will be touched
@@ -100,7 +101,7 @@ extern "C" {
/*!
* \brief checkpoint the model, meaning we finished a stage of execution
* every time we call check point, there is a version number which will increase by one
*
*
* \param global_model hold content of serialized global_model
* \param global_len the content length of serialized global model
* \param local_model hold content of serialized local_model, can be NULL
@@ -122,4 +123,4 @@ extern "C" {
#ifdef __cplusplus
} // C
#endif
#endif // XGBOOST_WRAPPER_H_
#endif // RABIT_WRAPPER_H_