add mock engine

Makefile

@@ -7,8 +7,8 @@ export CFLAGS = -Wall -O3 -msse2  -Wno-unknown-pragmas -fPIC -Iinclude
 BPATH=lib
 # objectives that makes up rabit library
 MPIOBJ= $(BPATH)/engine_mpi.o
-OBJ= $(BPATH)/allreduce_base.o $(BPATH)/allreduce_robust.o $(BPATH)/engine.o $(BPATH)/engine_empty.o
-ALIB= lib/librabit.a lib/librabit_mpi.a lib/librabit_empty.a
+OBJ= $(BPATH)/allreduce_base.o $(BPATH)/allreduce_robust.o $(BPATH)/engine.o $(BPATH)/engine_empty.o $(BPATH)/engine_mock.o
+ALIB= lib/librabit.a lib/librabit_mpi.a lib/librabit_empty.a lib/librabit_mock.a
 HEADERS=src/*.h include/*.h include/rabit/*.h
 .PHONY: clean all
 
@@ -19,8 +19,10 @@ $(BPATH)/engine.o: src/engine.cc $(HEADERS)
 $(BPATH)/allreduce_robust.o: src/allreduce_robust.cc $(HEADERS)
 $(BPATH)/engine_mpi.o: src/engine_mpi.cc $(HEADERS)
 $(BPATH)/engine_empty.o: src/engine_empty.cc $(HEADERS)
+$(BPATH)/engine_mock.o: src/engine_mock.cc $(HEADERS)
 
 lib/librabit.a: $(BPATH)/allreduce_base.o $(BPATH)/allreduce_robust.o $(BPATH)/engine.o
+lib/librabit_mock.a: $(BPATH)/allreduce_base.o $(BPATH)/allreduce_robust.o $(BPATH)/engine_mock.o
 lib/librabit_empty.a: $(BPATH)/engine_empty.o
 lib/librabit_mpi.a: $(MPIOBJ)
 

src/allreduce_mock.h

@@ -0,0 +1,94 @@
+/*!
+ * \file allreduce_mock.h
+ * \brief Mock test module of AllReduce engine,
+ * insert failures in certain call point, to test if the engine is robust to failure
+ * 
+ * \author Ignacio Cano, Tianqi Chen
+ */
+#ifndef RABIT_ALLREDUCE_MOCK_H
+#define RABIT_ALLREDUCE_MOCK_H
+#include <vector>
+#include <rabit/engine.h>
+#include <map>
+#include "./allreduce_robust.h"
+
+namespace rabit {
+namespace engine {
+class AllreduceMock : public AllreduceRobust {
+ public:
+  // constructor
+  AllreduceMock(void) {
+    num_trial = 0;
+  }
+  // destructor
+  virtual ~AllreduceMock(void) {}
+  virtual void SetParam(const char *name, const char *val) {
+    AllreduceRobust::SetParam(name, val);
+    // additional parameters
+    if (!strcmp(name, "rabit_num_trial")) num_trial = atoi(val);
+    if (!strcmp(name, "mock")) {
+      MockKey k;
+      utils::Check(sscanf(val, "%d,%d,%d,%d",
+                          &k.rank, &k.version, &k.seqno, &k.ntrial) == 4,
+                   "invalid mock parameter");
+      mock_map[k] = 1;
+    }
+  }
+  virtual void Allreduce(void *sendrecvbuf_,
+                         size_t type_nbytes,
+                         size_t count,
+                         ReduceFunction reducer,
+                         PreprocFunction prepare_fun,
+                         void *prepare_arg) {
+    this->Verify(MockKey(rank, version_number, seq_counter, num_trial));
+    AllreduceRobust::Allreduce(sendrecvbuf_, type_nbytes,
+                               count, reducer, prepare_fun, prepare_arg);
+  }
+  virtual void Broadcast(void *sendrecvbuf_, size_t total_size, int root) {
+    this->Verify(MockKey(rank, version_number, seq_counter, num_trial));
+    AllreduceRobust::Broadcast(sendrecvbuf_, total_size, root);
+  }
+  virtual void CheckPoint(const ISerializable *global_model,
+                          const ISerializable *local_model) {
+    this->Verify(MockKey(rank, version_number, seq_counter, num_trial));
+    AllreduceRobust::CheckPoint(global_model, local_model);
+  }
+  
+ private:
+  // key to identify the mock stage
+  struct MockKey {
+    int rank;
+    int version;
+    int seqno;
+    int ntrial;
+    MockKey(void) {}
+    MockKey(int rank, int version, int seqno, int ntrial) 
+        : rank(rank), version(version), seqno(seqno), ntrial(ntrial) {}
+    inline bool operator==(const MockKey &b) const {
+      return rank == b.rank && 
+          version == b.version &&
+          seqno == b.seqno &&
+          ntrial == b.ntrial;
+    }
+    inline bool operator<(const MockKey &b) const {
+      if (rank != b.rank) return rank < b.rank;
+      if (version != b.version) return version < b.version;
+      if (seqno != b.seqno) return seqno < b.seqno;
+      return ntrial < b.ntrial;
+    }
+  };
+  // number of failure trials
+  int num_trial;
+  // record all mock actions
+  std::map<MockKey, int> mock_map;
+  // used to generate all kinds of exceptions
+  inline void Verify(const MockKey &key) {
+    if (mock_map.count(key) != 0) {
+      num_trial += 1;
+      utils::Error("[%d]@@@Hit Mock Error", rank);
+    }
+  }
+};
+}  // namespace engine
+}  // namespace rabit
+#endif // RABIT_ALLREDUCE_MOCK_H

src/engine.cc

@@ -16,7 +16,11 @@
 namespace rabit {
 namespace engine {
 // singleton sync manager
+#ifndef RABIT_USE_MOCK
 AllreduceRobust manager;
+#else
+AllreduceMock manager;
+#endif
 
 /*! \brief intiialize the synchronization module */
 void Init(int argc, char *argv[]) {

src/engine_mock.cc

@@ -0,0 +1,15 @@
+/*!
+ * \file engine_mock.cc
+ * \brief this is an engine implementation that will 
+ * insert failures in certain call point, to test if the engine is robust to failure
+ * \author Tianqi Chen
+ */
+// define use MOCK, os we will use mock Manager
+#define _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_DEPRECATE
+#define NOMINMAX
+// switch engine to AllreduceMock
+#define RABIT_USE_MOCK
+#include "./allreduce_mock.h"
+#include "./engine.cc"
+