cleanup testcases

src/allreduce_base.cc

@@ -26,7 +26,7 @@ AllreduceBase::AllreduceBase(void) {
   hadoop_mode = 0;
   version_number = 0;
   task_id = "NULL";
-  this->SetParam("reduce_buffer", "256MB");
+  this->SetParam("rabit_reduce_buffer", "256MB");
 }
 
 // initialization function
@@ -38,8 +38,8 @@ void AllreduceBase::Init(void) {
       utils::Check(task_id != NULL, "hadoop_mode is set but cannot find mapred_task_id");
     }
     if (task_id != NULL) {
-      this->SetParam("task_id", task_id);
+      this->SetParam("rabit_task_id", task_id);
-      this->SetParam("hadoop_mode", "1");
+      this->SetParam("rabit_hadoop_mode", "1");
     }
   }  
   // start socket
@@ -83,9 +83,9 @@ void AllreduceBase::Shutdown(void) {
 void AllreduceBase::SetParam(const char *name, const char *val) {
   if (!strcmp(name, "rabit_tracker_uri")) tracker_uri = val;
   if (!strcmp(name, "rabit_tracker_port")) tracker_port = atoi(val);
-  if (!strcmp(name, "task_id")) task_id = val;
+  if (!strcmp(name, "rabit_task_id")) task_id = val;
-  if (!strcmp(name, "hadoop_mode")) hadoop_mode = atoi(val);
+  if (!strcmp(name, "rabit_hadoop_mode")) hadoop_mode = atoi(val);
-  if (!strcmp(name, "reduce_buffer")) {
+  if (!strcmp(name, "rabit_reduce_buffer")) {
     char unit;
     unsigned long amount;
     if (sscanf(val, "%lu%c", &amount, &unit) == 2) {

src/allreduce_robust.cc

@@ -17,10 +17,10 @@
 namespace rabit {
 namespace engine {
 AllreduceRobust::AllreduceRobust(void) {
-  result_buffer_round = 1;
   num_local_replica = 0;
   seq_counter = 0;
   local_chkpt_version = 0;
+  result_buffer_round = 1;
 }
 /*! \brief shutdown the engine */
 void AllreduceRobust::Shutdown(void) {
@@ -42,11 +42,11 @@ void AllreduceRobust::Shutdown(void) {
  */
 void AllreduceRobust::SetParam(const char *name, const char *val) {
   AllreduceBase::SetParam(name, val);
-  if (!strcmp(name, "result_buffer_round")) result_buffer_round = atoi(val);
+  if (!strcmp(name, "rabit_buffer_round")) result_buffer_round = atoi(val);
-  if (!strcmp(name, "result_replicate")) {
+  if (!strcmp(name, "rabit_global_replica")) {
     result_buffer_round = std::max(world_size / atoi(val), 1);
   }
-  if (!strcmp(name, "num_local_replica")) {
+  if (!strcmp(name, "rabit_local_replica")) {
     num_local_replica = atoi(val);
   }
 }

submit_hadoop.py

@@ -17,7 +17,6 @@ parser.add_argument('-hs', '--hadoop_streaming_jar', required=True)
 parser.add_argument('-i', '--input', required=True)
 parser.add_argument('-o', '--output', required=True)
 parser.add_argument('-m', '--mapper', required=True)
-#parser.add_argument('-r', '--reducer', required=False)
 parser.add_argument('-k', '--nclusters', required=True, type=int)
 parser.add_argument('-itr', '--iterations', required=True, type=int)
 args = parser.parse_args()

submit_mpi.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python
 """
-This is an example script to create a customized job submit
+This is an example script to create a customized job submit with mpi
 script using rabit engine
 """
 import sys
@@ -34,6 +34,7 @@ def mpi_submit(nslave, args):
 if __name__ == '__main__':
     if len(sys.argv) < 2:
         print 'Usage: <nslave> <machine_file> <cmd>'
+        print 'if <machine_file> == local, we will run using local mode'
         exit(0)        
     # call submit, with nslave, the commands to run each job and submit function
     tracker.submit(int(sys.argv[1]), sys.argv[2:], fun_submit= mpi_submit)

test/Makefile

@@ -5,33 +5,30 @@ export LDFLAGS= -pthread -lm  -lrt
 export CFLAGS = -Wall -O3 -msse2  -Wno-unknown-pragmas -fPIC -I../src
 
 # specify tensor path
-BIN = test_allreduce test_recover test_model_recover speed_test test_local_recover
+BIN = speed_test test_model_recover test_local_recover
 # objectives that makes up rabit library
 RABIT_OBJ = allreduce_base.o allreduce_robust.o engine.o
 MPIOBJ = engine_mpi.o
 
-OBJ = $(RABIT_OBJ) test_allreduce.o test_recover.o test_model_recover.o speed_test.o test_local_recover.o
+OBJ = $(RABIT_OBJ) speed_test.o test_model_recover.o test_local_recover.o
-MPIBIN = test_allreduce.mpi speed_test.mpi
+MPIBIN = speed_test.mpi
 .PHONY: clean all
 
 all: $(BIN) $(MPIBIN)
-
+# the rabit library
 allreduce_base.o: ../src/allreduce_base.cc ../src/*.h
 engine.o: ../src/engine.cc ../src/*.h
-allreduce_robust.o: ../src/allreduce_robust.cc ../src/*.h
 engine_mpi.o: ../src/engine_mpi.cc
-test_allreduce.o: test_allreduce.cpp ../src/*.h
+allreduce_robust.o: ../src/allreduce_robust.cc ../src/*.h
+
+# programs 
 speed_test.o: speed_test.cpp ../src/*.h
-test_recover.o: test_recover.cpp ../src/*.h 
 test_model_recover.o: test_model_recover.cpp ../src/*.h
 test_local_recover.o: test_local_recover.cpp ../src/*.h
 
 # we can link against MPI version to get use MPI
-test_allreduce: test_allreduce.o  $(RABIT_OBJ)
-test_allreduce.mpi: test_allreduce.o $(MPIOBJ)
 speed_test: speed_test.o  $(RABIT_OBJ)
 speed_test.mpi: speed_test.o $(MPIOBJ)
-test_recover: test_recover.o  $(RABIT_OBJ)
 test_model_recover: test_model_recover.o  $(RABIT_OBJ)
 test_local_recover: test_local_recover.o  $(RABIT_OBJ)
 
@@ -48,4 +45,4 @@ $(MPIOBJ) :
 	$(MPICXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c %.cc, $^) )
 
 clean:
-	$(RM) $(OBJ) $(BIN) $(MPIBIN) *~ ../src/*~
+	$(RM) $(OBJ) $(BIN) $(MPIBIN) $(MPIOBJ) *~ ../src/*~

test/README.md

@@ -0,0 +1,18 @@
+Testcases of Rabit
+====
+This folder contains internal testcases to test correctness and efficiency of rabit API
+
+The example running scripts for testcases are given by test.mk
+* type ```make -f test.mk testcasename``` to run certain testcase
+
+
+Helper Scripts
+====
+* test.mk contains Makefile documentation of all testcases
+* keepalive.sh helper bash to restart a program when it dies abnormally
+
+List of Programs
+====
+* speed_test: test the running speed of rabit API
+* test_local_recover: test recovery of local state when error happens
+* test_model_recover: test recovery of global state when error happens

test/keepalive.sh

@@ -6,9 +6,9 @@ then
     exit -1
 fi
 nrep=0
-echo ./$@ task_id=$OMPI_COMM_WORLD_RANK
+echo ./$@ rabit_task_id=$OMPI_COMM_WORLD_RANK
-until ./$@ task_id=$OMPI_COMM_WORLD_RANK repeat=$nrep; do
+until ./$@ rabit_task_id=$OMPI_COMM_WORLD_RANK repeat=$nrep; do
     sleep 1
     nrep=$((nrep+1))
-    echo ./$@ job_id=$OMPI_COMM_WORLD_RANK repeat=$nrep
+    echo ./$@ rabit_task_id=$OMPI_COMM_WORLD_RANK repeat=$nrep
 done

test/mock.h

@@ -11,6 +11,9 @@
 #include <sstream>
 #include <fstream>
 
+struct MockException {
+};
+
 namespace rabit {
 /*! \brief namespace of mock */
 namespace test {

test/speed_runner.py

@@ -31,4 +31,4 @@ def main():
     sys.stderr.flush()
 
 if __name__ == "__main__":
-  main()
+  main()

test/speed_test.cpp

@@ -1,3 +1,4 @@
+// This program is used to test the speed of rabit API
 #include <rabit.h>
 #include <utils.h>
 #include <timer.h>
@@ -12,8 +13,6 @@ double max_tdiff, sum_tdiff, bcast_tdiff, tot_tdiff;
 
 inline void TestMax(size_t n) {
   int rank = rabit::GetRank();
-  //int nproc = rabit::GetWorldSize();
-
   std::vector<float> ndata(n);
   for (size_t i = 0; i < ndata.size(); ++i) {
     ndata[i] = (i * (rank+1)) % 111;
@@ -25,7 +24,6 @@ inline void TestMax(size_t n) {
 
 inline void TestSum(size_t n) {
   int rank = rabit::GetRank();
-  //int nproc = rabit::GetWorldSize();
   const int z = 131;
   std::vector<float> ndata(n);
   for (size_t i = 0; i < ndata.size(); ++i) {

test/test.mk

@@ -0,0 +1,20 @@
+ifndef $(nslave)
+	nslave=2
+endif
+ifndef $(ndata)
+	ndata=10
+endif
+
+# this is a makefile used to show testcases of rabit
+.PHONY: model_recover local_recover speed
+
+
+local_recover:
+	../submit_mpi.py $(nslave) local test_local_recover $(ndata) rabit_local_replica=1
+
+local_recover_10_10k:
+	../submit_mpi.py 10 local test_local_recover 10000 rabit_local_replica=1
+
+# this experiment test recovery with actually process exit, use keepalive to keep program alive
+model_recover_10_10k:
+	../submit_mpi.py 10 local keepalive.sh test_model_recover 10000

test/test_allreduce.cpp

@@ -1,90 +0,0 @@
-#include <rabit.h>
-#include <utils.h>
-#include <cstdio>
-#include <cstdlib>
-#include <cmath>
-#include <mock.h>
-
-using namespace rabit;
-
-inline void TestMax(test::Mock &mock, size_t n) {
-  int rank = rabit::GetRank();
-  int nproc = rabit::GetWorldSize();
-  
-  std::vector<float> ndata(n);
-  for (size_t i = 0; i < ndata.size(); ++i) {
-    ndata[i] = (i * (rank+1)) % 111;
-  }
-  mock.Allreduce<op::Max>(&ndata[0], ndata.size());  
-  for (size_t i = 0; i < ndata.size(); ++i) {
-    float rmax = (i * 1) % 111;
-    for (int r = 0; r < nproc; ++r) {
-      rmax = std::max(rmax, (float)((i * (r+1)) % 111));
-    }
-    utils::Check(rmax == ndata[i], "[%d] TestMax check failure", rank);
-  }
-}
-
-inline void TestSum(test::Mock &mock, size_t n) {
-  int rank = rabit::GetRank();
-  int nproc = rabit::GetWorldSize();
-  const int z = 131;
-
-  std::vector<float> ndata(n);
-  for (size_t i = 0; i < ndata.size(); ++i) {
-    ndata[i] = (i * (rank+1)) % z;
-  }
-  mock.Allreduce<op::Sum>(&ndata[0], ndata.size());  
-  for (size_t i = 0; i < ndata.size(); ++i) {
-    float rsum = 0.0f;
-    for (int r = 0; r < nproc; ++r) {
-      rsum += (float)((i * (r+1)) % z);
-    }
-    utils::Check(fabsf(rsum - ndata[i]) < 1e-5 ,
-                 "[%d] TestSum check failure, local=%g, allreduce=%g", rank, rsum, ndata[i]);
-  }
-}
-
-inline void TestBcast(test::Mock &mock, size_t n, int root) {
-  int rank = rabit::GetRank();
-  std::string s; s.resize(n);      
-  for (size_t i = 0; i < n; ++i) {
-    s[i] = char(i % 126 + 1);
-  }
-  std::string res;
-  if (root == rank) {
-    res = s;
-    mock.Broadcast(&res, root);
-  } else {
-    mock.Broadcast(&res, root);
-  }
-  utils::Check(res == s, "[%d] TestBcast fail", rank);
-}
-
-int main(int argc, char *argv[]) {
-  if (argc < 3) {
-    printf("Usage: <ndata> <config>\n");
-    return 0;
-  }
-  int n = atoi(argv[1]);
-  rabit::Init(argc, argv);
-  int rank = rabit::GetRank();
-  int nproc = rabit::GetWorldSize();
-  std::string name = rabit::GetProcessorName();
-
-  test::Mock mock(rank, argv[2], argv[3]);
-
-  utils::LogPrintf("[%d] start at %s\n", rank, name.c_str());
-  TestMax(mock, n);
-  utils::LogPrintf("[%d] !!!TestMax pass\n", rank);
-  TestSum(mock, n);
-  utils::LogPrintf("[%d] !!!TestSum pass\n", rank);
-  int step = std::max(nproc / 3, 1);
-  for (int i = 0; i < nproc; i += step) {    
-    TestBcast(mock, n, i);
-  }
-  utils::LogPrintf("[%d] !!!TestBcast pass\n", rank);
-  rabit::Finalize();
-  printf("[%d] all check pass\n", rank);
-  return 0;
-}

test/test_local_recover.cpp

@@ -5,12 +5,28 @@
 #include <cstdio>
 #include <cstdlib>
 #include <cmath>
-#include <mock.h>
+#include "./mock.h"
 
 using namespace rabit;
-
+namespace rabit {
-struct MockException {
+namespace test {
-};
+inline void CallBegin(const char *fun, int ntrial, int iter) {
+  int rank = rabit::GetRank();
+  if (!strcmp(fun, "Allreduce::Sum")) {
+    if (ntrial == iter && rank == 0) throw MockException(); 
+  }
+  if (!strcmp(fun, "Allreduce::Max")) {
+    if (ntrial == iter && rank == 3) throw MockException(); 
+  }
+}
+inline void CallEnd(const char *fun, int ntrial, int iter) {
+  int rank = rabit::GetRank();
+  if (!strcmp(fun, "Allreduce::Bcast")) {
+    if (ntrial == iter && rand() % 10 == rank) throw MockException(); 
+  }
+}
+}
+}
 
 // dummy model
 class Model : public rabit::utils::ISerializable {
@@ -31,7 +47,7 @@ class Model : public rabit::utils::ISerializable {
   }
 };
 
-inline void TestMax(test::Mock &mock, Model *model, Model *local, int ntrial, int iter) {
+inline void TestMax(Model *model, Model *local, int ntrial, int iter) {
   int rank = rabit::GetRank();
   int nproc = rabit::GetWorldSize();
   const int z = iter + 111;
@@ -39,11 +55,11 @@ inline void TestMax(test::Mock &mock, Model *model, Model *local, int ntrial, in
   std::vector<float> ndata(model->data.size());
   for (size_t i = 0; i < ndata.size(); ++i) {
     ndata[i] = (i * (rank+1)) % z  + local->data[i];
-  }
+  }  
-  mock.Allreduce<op::Max>(&ndata[0], ndata.size());  
+  test::CallBegin("Allreduce::Max", ntrial, iter);
-  if (ntrial == iter && rank == 1) {
+  rabit::Allreduce<op::Max>(&ndata[0], ndata.size());  
-    throw MockException();
+  test::CallEnd("Allreduce::Max", ntrial, iter);
-  }
+
   for (size_t i = 0; i < ndata.size(); ++i) {
     float rmax = (i * 1) % z + model->data[i];
     for (int r = 0; r < nproc; ++r) {
@@ -58,7 +74,7 @@ inline void TestMax(test::Mock &mock, Model *model, Model *local, int ntrial, in
   }
 }
 
-inline void TestSum(test::Mock &mock, Model *model, Model *local, int ntrial, int iter) {
+inline void TestSum(Model *model, Model *local, int ntrial, int iter) {
   int rank = rabit::GetRank();
   int nproc = rabit::GetWorldSize();
   const int z = 131 + iter;
@@ -67,11 +83,10 @@ inline void TestSum(test::Mock &mock, Model *model, Model *local, int ntrial, in
   for (size_t i = 0; i < ndata.size(); ++i) {
     ndata[i] = (i * (rank+1)) % z + local->data[i];
   }
-  if (ntrial == iter && rank == 0) {
+  test::CallBegin("Allreduce::Sum", ntrial, iter);
-    throw MockException();
+  Allreduce<op::Sum>(&ndata[0], ndata.size());
-  }
+  test::CallEnd("Allreduce::Sum", ntrial, iter);
-  mock.Allreduce<op::Sum>(&ndata[0], ndata.size());
+  
-
   for (size_t i = 0; i < ndata.size(); ++i) {
     float rsum = 0.0f;
     for (int r = 0; r < nproc; ++r) {
@@ -86,7 +101,7 @@ inline void TestSum(test::Mock &mock, Model *model, Model *local, int ntrial, in
   }
 }
 
-inline void TestBcast(test::Mock &mock, size_t n, int root, int ntrial) {
+inline void TestBcast(size_t n, int root, int ntrial, int iter) {
   int rank = rabit::GetRank();
   std::string s; s.resize(n);      
   for (size_t i = 0; i < n; ++i) {
@@ -95,16 +110,20 @@ inline void TestBcast(test::Mock &mock, size_t n, int root, int ntrial) {
   std::string res;
   if (root == rank) {
     res = s;
-    mock.Broadcast(&res, root);
+    test::CallBegin("Broadcast", ntrial, iter);
+    rabit::Broadcast(&res, root);
+    test::CallBegin("Broadcast", ntrial, iter);
   } else {
-    mock.Broadcast(&res, root);
+    test::CallBegin("Broadcast", ntrial, iter);
+    rabit::Broadcast(&res, root);
+    test::CallEnd("Broadcast", ntrial, iter);
   }
   utils::Check(res == s, "[%d] TestBcast fail", rank);
 }
 
 int main(int argc, char *argv[]) {
   if (argc < 3) {
-    printf("Usage: <ndata> <config>\n");
+    printf("Usage: <ndata>\n");
     return 0;
   }
   int n = atoi(argv[1]);
@@ -112,7 +131,6 @@ int main(int argc, char *argv[]) {
   int rank = rabit::GetRank();
   int nproc = rabit::GetWorldSize();
   std::string name = rabit::GetProcessorName();
-  test::Mock mock(rank, argv[2], argv[3]);
   Model model, local;  
   srand(0);
   int ntrial = 0;
@@ -131,14 +149,14 @@ int main(int argc, char *argv[]) {
         utils::LogPrintf("[%d] reload-trail=%d, init iter=%d\n", rank, ntrial, iter);
       }
       for (int r = iter; r < 3; ++r) { 
-        TestMax(mock, &model, &local, ntrial, r);
+        TestMax(&model, &local, ntrial, r);
         utils::LogPrintf("[%d] !!!TestMax pass, iter=%d\n",  rank, r);  
         int step = std::max(nproc / 3, 1);
         for (int i = 0; i < nproc; i += step) {
-          //TestBcast(mock, n, i, ntrial);
+          TestBcast(n, i, ntrial, r);
         }
-        //utils::LogPrintf("[%d] !!!TestBcast pass, iter=%d\n", rank, r);
+        utils::LogPrintf("[%d] !!!TestBcast pass, iter=%d\n", rank, r);
-        TestSum(mock, &model, &local, ntrial, r);
+        TestSum(&model, &local, ntrial, r);
         utils::LogPrintf("[%d] !!!TestSum pass, iter=%d\n", rank, r);
         rabit::CheckPoint(&model, &local);
         utils::LogPrintf("[%d] !!!CheckPont pass, iter=%d\n", rank, r);

test/test_model_recover.cpp

@@ -5,12 +5,28 @@
 #include <cstdio>
 #include <cstdlib>
 #include <cmath>
-#include <mock.h>
+#include "./mock.h"
 
 using namespace rabit;
-
+namespace rabit {
-struct MockException {
+namespace test {
-};
+inline void CallBegin(const char *fun, int ntrial, int iter) {
+  int rank = rabit::GetRank();
+  if (!strcmp(fun, "Allreduce::Sum")) {
+    if (ntrial == iter && rank == 0) exit(-1);
+  }
+  if (!strcmp(fun, "Allreduce::Max")) {
+    if (ntrial == iter && rank == 3) exit(-1);
+  }
+}
+inline void CallEnd(const char *fun, int ntrial, int iter) {
+  int rank = rabit::GetRank();
+  if (!strcmp(fun, "Allreduce::Bcast")) {
+    if (ntrial == iter && rand() % 10 == rank) exit(-1);
+  }
+}
+}
+}
 
 // dummy model
 class Model : public rabit::utils::ISerializable {
@@ -31,7 +47,7 @@ class Model : public rabit::utils::ISerializable {
   }
 };
 
-inline void TestMax(test::Mock &mock, Model *model, int ntrial, int iter) {
+inline void TestMax(Model *model, int ntrial, int iter) {
   int rank = rabit::GetRank();
   int nproc = rabit::GetWorldSize();
   const int z = iter + 111;
@@ -40,10 +56,10 @@ inline void TestMax(test::Mock &mock, Model *model, int ntrial, int iter) {
   for (size_t i = 0; i < ndata.size(); ++i) {
     ndata[i] = (i * (rank+1)) % z  + model->data[i];
   }
-  mock.Allreduce<op::Max>(&ndata[0], ndata.size());  
+  test::CallBegin("Allreduce::Max", ntrial, iter);
-  if (ntrial == 0 && rank == 3) {
+  rabit::Allreduce<op::Max>(&ndata[0], ndata.size());  
-    exit(-1);
+  test::CallEnd("Allreduce::Max", ntrial, iter);
-  }
+
   for (size_t i = 0; i < ndata.size(); ++i) {
     float rmax = (i * 1) % z + model->data[i];
     for (int r = 0; r < nproc; ++r) {
@@ -54,7 +70,7 @@ inline void TestMax(test::Mock &mock, Model *model, int ntrial, int iter) {
   model->data = ndata;
 }
 
-inline void TestSum(test::Mock &mock, Model *model, int ntrial, int iter) {
+inline void TestSum(Model *model, int ntrial, int iter) {
   int rank = rabit::GetRank();
   int nproc = rabit::GetWorldSize();
   const int z = 131 + iter;
@@ -63,11 +79,9 @@ inline void TestSum(test::Mock &mock, Model *model, int ntrial, int iter) {
   for (size_t i = 0; i < ndata.size(); ++i) {
     ndata[i] = (i * (rank+1)) % z + model->data[i];
   }
-  if (iter == 0 && ntrial==0 && rank == 0) {
+  test::CallBegin("Allreduce::Sum", ntrial, iter);
-    throw MockException();
+  Allreduce<op::Sum>(&ndata[0], ndata.size());
-  }
+  test::CallEnd("Allreduce::Sum", ntrial, iter);
-
-  mock.Allreduce<op::Sum>(&ndata[0], ndata.size());
 
   for (size_t i = 0; i < ndata.size(); ++i) {
     float rsum = model->data[i] * nproc;
@@ -80,7 +94,7 @@ inline void TestSum(test::Mock &mock, Model *model, int ntrial, int iter) {
   model->data = ndata;
 }
 
-inline void TestBcast(test::Mock &mock, size_t n, int root, int ntrial) {
+inline void TestBcast(size_t n, int root, int ntrial, int iter) {
   int rank = rabit::GetRank();
   std::string s; s.resize(n);      
   for (size_t i = 0; i < n; ++i) {
@@ -89,9 +103,13 @@ inline void TestBcast(test::Mock &mock, size_t n, int root, int ntrial) {
   std::string res;
   if (root == rank) {
     res = s;
-    mock.Broadcast(&res, root);
+    test::CallBegin("Broadcast", ntrial, iter);
+    rabit::Broadcast(&res, root);
+    test::CallBegin("Broadcast", ntrial, iter);
   } else {
-    mock.Broadcast(&res, root);
+    test::CallBegin("Broadcast", ntrial, iter);
+    rabit::Broadcast(&res, root);
+    test::CallEnd("Broadcast", ntrial, iter);
   }
   utils::Check(res == s, "[%d] TestBcast fail", rank);
 }
@@ -106,7 +124,6 @@ int main(int argc, char *argv[]) {
   int rank = rabit::GetRank();
   int nproc = rabit::GetWorldSize();
   std::string name = rabit::GetProcessorName();
-  test::Mock mock(rank, argv[2], argv[3]);
   Model model;  
   srand(0);
   int ntrial = 0;
@@ -124,14 +141,14 @@ int main(int argc, char *argv[]) {
         utils::LogPrintf("[%d] reload-trail=%d, init iter=%d\n", rank, ntrial, iter);
       }
       for (int r = iter; r < 3; ++r) { 
-        TestMax(mock, &model, ntrial, r);
+        TestMax(&model, ntrial, r);
         utils::LogPrintf("[%d] !!!TestMax pass, iter=%d\n",  rank, r);  
-        //int step = std::max(nproc / 3, 1);
+        int step = std::max(nproc / 3, 1);
-        //for (int i = 0; i < nproc; i += step) {
+        for (int i = 0; i < nproc; i += step) {
-          //TestBcast(mock, n, i, ntrial);
+          TestBcast(n, i, ntrial, r);
-          //}
+        }
-        //utils::LogPrintf("[%d] !!!TestBcast pass, iter=%d\n", rank, r);
+        utils::LogPrintf("[%d] !!!TestBcast pass, iter=%d\n", rank, r);
-        TestSum(mock, &model, ntrial, r);
+        TestSum(&model, ntrial, r);
         utils::LogPrintf("[%d] !!!TestSum pass, iter=%d\n", rank, r);
         rabit::CheckPoint(&model);
         utils::LogPrintf("[%d] !!!CheckPont pass, iter=%d\n", rank, r);

test/test_recover.cpp

@@ -1,125 +0,0 @@
-#include <rabit.h>
-#include <utils.h>
-#include <cstdio>
-#include <cstdlib>
-#include <cmath>
-#include <mock.h>
-
-using namespace rabit;
-
-struct MockException {
-};
-
-inline void TestMax(test::Mock &mock, size_t n, int ntrial) {
-  int rank = rabit::GetRank();
-  int nproc = rabit::GetWorldSize();
-  
-  std::vector<float> ndata(n);
-  for (size_t i = 0; i < ndata.size(); ++i) {
-    ndata[i] = (i * (rank+1)) % 111;
-  }
-  mock.Allreduce<op::Max>(&ndata[0], ndata.size());  
-  if (ntrial == 0 && rank == 15) throw MockException();
-  for (size_t i = 0; i < ndata.size(); ++i) {
-    float rmax = (i * 1) % 111;
-    for (int r = 0; r < nproc; ++r) {
-      rmax = std::max(rmax, (float)((i * (r+1)) % 111));
-    }
-    utils::Check(rmax == ndata[i], "[%d] TestMax check failure", rank);
-  }
-}
-
-inline void TestSum(test::Mock &mock, size_t n, int ntrial) {
-  int rank = rabit::GetRank();
-  int nproc = rabit::GetWorldSize();
-  const int z = 131;
-
-  std::vector<float> ndata(n);
-  for (size_t i = 0; i < ndata.size(); ++i) {
-    ndata[i] = (i * (rank+1)) % z;
-  }
-  mock.Allreduce<op::Sum>(&ndata[0], ndata.size());
-
-  if (ntrial == 0 && rank == 0) throw MockException();
-
-  for (size_t i = 0; i < ndata.size(); ++i) {
-    float rsum = 0.0f;
-    for (int r = 0; r < nproc; ++r) {
-      rsum += (float)((i * (r+1)) % z);
-    }
-    utils::Check(fabsf(rsum - ndata[i]) < 1e-5 ,
-                 "[%d] TestSum check failure, local=%g, allreduce=%g", rank, rsum, ndata[i]);
-  }
-}
-
-inline void TestBcast(test::Mock &mock, size_t n, int root, int ntrial) {
-  int rank = rabit::GetRank();
-  std::string s; s.resize(n);      
-  for (size_t i = 0; i < n; ++i) {
-    s[i] = char(i % 126 + 1);
-  }
-  std::string res;
-  if (root == rank) {
-    res = s;
-    mock.Broadcast(&res, root);
-  } else {
-    mock.Broadcast(&res, root);
-  }
-  utils::Check(res == s, "[%d] TestBcast fail", rank);
-}
-// dummy model
-class Model : public rabit::utils::ISerializable {
- public:
-  // load from stream
-  virtual void Load(rabit::utils::IStream &fi) {
-    // do nothing
-  }
-  /*! \brief save the model to the stream */
-  virtual void Save(rabit::utils::IStream &fo) const {
-    // do nothing
-  }
-  virtual void InitModel(void) {
-    // do nothing
-  }
-};
-
-int main(int argc, char *argv[]) {
-  if (argc < 3) {
-    printf("Usage: <ndata> <config>\n");
-    return 0;
-  }
-  int n = atoi(argv[1]);
-  rabit::Init(argc, argv);
-  int rank = rabit::GetRank();
-  int nproc = rabit::GetWorldSize();
-  std::string name = rabit::GetProcessorName();
-  test::Mock mock(rank, argv[2], argv[3]);
-  Model model;  
-  srand(0);
-  int ntrial = 0;
-  while (true) {
-    try {
-      if (rabit::LoadCheckPoint(&model) == 0) {
-        model.InitModel();
-      }      
-      utils::LogPrintf("[%d/%d] start at %s\n", rank, ntrial, name.c_str());
-      TestMax(mock, n, ntrial);
-      utils::LogPrintf("[%d/%d] !!!TestMax pass\n", rank, ntrial);  
-      TestSum(mock, n, ntrial);
-      utils::LogPrintf("[%d/%d] !!!TestSum pass\n", rank, ntrial);
-      int step = std::max(nproc / 3, 1);
-      for (int i = 0; i < nproc; i += step) {
-        TestBcast(mock, n, i, ntrial);
-      }
-      utils::LogPrintf("[%d] !!!TestBcast pass\n", rank);
-      // reach here
-      break;
-    } catch (MockException &e) {
-      rabit::engine::GetEngine()->InitAfterException();
-      ++ntrial;
-    }
-  }
-  rabit::Finalize();
-  printf("[%d] all check pass\n", rank);
-  return 0;
-}

test/testcase0.conf

@@ -1 +0,0 @@
-# Test Case 0 -> nothing fails

test/testcase1.conf

@@ -1,9 +0,0 @@
-# Test Case example config
-# You configure which methods should fail
-# Format <round>_<rank> = <operation>
-# <operation> can be one of the following = allreduce, broadcast, loadcheckpoint, checkpoint
-
-1_0 = allreduce
-1_1 = broadcast
-
-2_2 = allreduce