* support run rabit tests as xgboost subproject using xgboost/dmlc-core * support tracker config set/get * remove redudant printf * remove redudant printf * add c++0x declaration * log allreduce/broadcast caller, engine should track caller stack for investigation * tracker support binary config format * Revert "tracker support binary config format" This reverts commit 2a28e5e2b55c200cb621af8d19f17ab1bc62503b. * remove caller, prototype fetch allreduce/broadcast results from resbuf * store cached allreduce/broadcast seq_no to tracker * allow restore all caches from other nodes * try new rabit collective cache, todo: recv_link seems down * link up cache restore with main recovery * cleanup load cache state * update cache api * pass test.mk * have a working tests * try to unify check into actionsummary * more logging to debug distributed hist three method issue * update rabit interface to support caller signature matching * splite seq_counter from cur_cache_seq to different variables * still see issue with inf loop * support debug print caller as well as allreduce op * cleanup * remove get/set cache from model_recover, adding recover in loadcheckpoint * clarify rabit cache strategy, cache is set only by successful collective call involving all nodes with unique cache key. if all nodes call getcache at same time, we keep rabit run collective call. If some nodes call getcache while others not, we backfill cache from those nodes with most entries * revert caller logs * fix lint error * fix engine mpi signature * support getcache by ref * allow result buffer presiet to filestream * add loging * try fix checkpoint failure recovery case * use int64_t to avoid overflow caused seq fault * try avoid int overflow * try fix checkpoint failure recovery case * try avoid seqno overflow to negative by offseting specifial flag value adding cache seq no to checkpoint/load checkpoint/check point ack to avoid confusion from cache recovery * fix cache seq assert error * remove loging, handle edge case * add extensive log to checkpoint state with different seq no * fix lint errors * clean up comments before merge back to master * add logs to allreduce/broadcast/checkpoint * use unsinged int 32 and give seq no larger range * address remove allreduce dropseq code segment * using caller signature to filter bootstrapallreduces * remove get/set cache from empty * apply signature to reducer * apply signature to broadcast * add key to broadcat log * fix broadcast signature * fix default _line value for non linux system * adding comments, remove sleep(1) * fix osx build issue * try fix mpi * fix doc * fix engine_empty api * logging, adding more logs, restore immutable assertion * print unsinged int with ud * fix lint * rename seqtype to kSeq and KCache indicating it's usage apply kDiffSeq check to load_cache routine * comment allreduce/broadcast log * allow tests run on arm * enable flag to turn on / off cache * add log info alert if user choose to enable rabit bootstrap cache * add rabit_debug setting so user can use config to turn on * log flags when user turn on rabit_debug * force rabit restart if tracker assign -1 rank * use OPENMP to vecotrize reducer * address comment * Revert "address comment" This reverts commit 1dc61f33e7357dad8fa65528abeb81db92c5f9ed. * fix checkpoint size print 0 * per feedback, remove DISABLEOPEMP, address race condition * - remove openmp from this pr - update name from cache to boostrapcache * add default value of signature macros * remove openmp from cmake file * Update src/allreduce_robust.cc Co-Authored-By: Philip Hyunsu Cho <chohyu01@cs.washington.edu> * Update src/allreduce_robust.cc Co-Authored-By: Philip Hyunsu Cho <chohyu01@cs.washington.edu> * run test with cmake * remove openmp * fix cmake based tests * use cmake test fix darwin .dylib issue * move around rabit_signature definition due to windows build * misc, add c++ check in CMakeFile * per feedback * resolve CMake file * update rabit version
140 lines
4.2 KiB
C++
140 lines
4.2 KiB
C++
/*!
|
|
* Copyright (c) 2014 by Contributors
|
|
* \file engine_empty.cc
|
|
* \brief this file provides a dummy implementation of engine that does nothing
|
|
* this file provides a way to fall back to single node program without causing too many dependencies
|
|
* This is usually NOT needed, use engine_mpi or engine for real distributed version
|
|
* \author Tianqi Chen
|
|
*/
|
|
#define _CRT_SECURE_NO_WARNINGS
|
|
#define _CRT_SECURE_NO_DEPRECATE
|
|
#define NOMINMAX
|
|
|
|
#include "../include/rabit/internal/engine.h"
|
|
|
|
namespace rabit {
|
|
|
|
namespace utils {
|
|
bool STOP_PROCESS_ON_ERROR = true;
|
|
}
|
|
|
|
namespace engine {
|
|
/*! \brief EmptyEngine */
|
|
class EmptyEngine : public IEngine {
|
|
public:
|
|
EmptyEngine(void) {
|
|
version_number = 0;
|
|
}
|
|
virtual void Allreduce(void *sendrecvbuf_,
|
|
size_t type_nbytes,
|
|
size_t count,
|
|
ReduceFunction reducer,
|
|
PreprocFunction prepare_fun,
|
|
void *prepare_arg,
|
|
bool is_bootstrap,
|
|
const char* _file,
|
|
const int _line,
|
|
const char* _caller) {
|
|
utils::Error("EmptyEngine:: Allreduce is not supported,"\
|
|
"use Allreduce_ instead");
|
|
}
|
|
virtual void Broadcast(void *sendrecvbuf_, size_t size, int root,
|
|
bool is_bootstrap, const char* _file,
|
|
const int _line, const char* _caller) {
|
|
}
|
|
virtual void InitAfterException(void) {
|
|
utils::Error("EmptyEngine is not fault tolerant");
|
|
}
|
|
virtual int LoadCheckPoint(Serializable *global_model,
|
|
Serializable *local_model = NULL) {
|
|
return 0;
|
|
}
|
|
virtual void CheckPoint(const Serializable *global_model,
|
|
const Serializable *local_model = NULL) {
|
|
version_number += 1;
|
|
}
|
|
virtual void LazyCheckPoint(const Serializable *global_model) {
|
|
version_number += 1;
|
|
}
|
|
virtual int VersionNumber(void) const {
|
|
return version_number;
|
|
}
|
|
/*! \brief get rank of current node */
|
|
virtual int GetRank(void) const {
|
|
return 0;
|
|
}
|
|
/*! \brief get total number of */
|
|
virtual int GetWorldSize(void) const {
|
|
return 1;
|
|
}
|
|
/*! \brief whether it is distributed */
|
|
virtual bool IsDistributed(void) const {
|
|
return false;
|
|
}
|
|
/*! \brief get the host name of current node */
|
|
virtual std::string GetHost(void) const {
|
|
return std::string("");
|
|
}
|
|
virtual void TrackerPrint(const std::string &msg) {
|
|
// simply print information into the tracker
|
|
utils::Printf("%s", msg.c_str());
|
|
}
|
|
|
|
private:
|
|
int version_number;
|
|
};
|
|
|
|
// singleton sync manager
|
|
EmptyEngine manager;
|
|
|
|
/*! \brief intiialize the synchronization module */
|
|
bool Init(int argc, char *argv[]) {
|
|
return true;
|
|
}
|
|
/*! \brief finalize syncrhonization module */
|
|
bool Finalize(void) {
|
|
return true;
|
|
}
|
|
|
|
/*! \brief singleton method to get engine */
|
|
IEngine *GetEngine(void) {
|
|
return &manager;
|
|
}
|
|
// perform in-place allreduce, on sendrecvbuf
|
|
void Allreduce_(void *sendrecvbuf,
|
|
size_t type_nbytes,
|
|
size_t count,
|
|
IEngine::ReduceFunction red,
|
|
mpi::DataType dtype,
|
|
mpi::OpType op,
|
|
IEngine::PreprocFunction prepare_fun,
|
|
void *prepare_arg,
|
|
bool is_bootstrap,
|
|
const char* _file,
|
|
const int _line,
|
|
const char* _caller) {
|
|
if (prepare_fun != NULL) prepare_fun(prepare_arg);
|
|
}
|
|
|
|
// code for reduce handle
|
|
ReduceHandle::ReduceHandle(void) : handle_(NULL), htype_(NULL) {
|
|
}
|
|
ReduceHandle::~ReduceHandle(void) {}
|
|
|
|
int ReduceHandle::TypeSize(const MPI::Datatype &dtype) {
|
|
return 0;
|
|
}
|
|
void ReduceHandle::Init(IEngine::ReduceFunction redfunc, size_t type_nbytes) {}
|
|
void ReduceHandle::Allreduce(void *sendrecvbuf,
|
|
size_t type_nbytes, size_t count,
|
|
IEngine::PreprocFunction prepare_fun,
|
|
void *prepare_arg,
|
|
bool is_bootstrap,
|
|
const char* _file,
|
|
const int _line,
|
|
const char* _caller) {
|
|
if (prepare_fun != NULL) prepare_fun(prepare_arg);
|
|
}
|
|
} // namespace engine
|
|
} // namespace rabit
|