execute it like ./test.sh 4 4000 testcase0.conf to obtain a successful execution

updating mock. It now wraps the calls to sync and reads config from configuration file.
I believe it's better not to use the preprocessor directive, i.e. not to put any test code in the engine_tcp. I just call the mock in the test_allreduce file. It's a file purely for testing purposes, so it's fine to use the mock there.

test/Makefile

@@ -9,10 +9,6 @@ ifeq ($(no_omp),1)
 else 
 	CFLAGS += -fopenmp
 endif
-ifeq ($(test),1)
-	CFLAGS += -DTEST
-endif
-
 
 # specify tensor path
 BIN = test_allreduce

test/test.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
-if [ "$#" -ne 2 ];
+if [ "$#" -ne 3 ];
 then
-    echo "Usage <nslave> <ndata>"
+    echo "Usage <nslave> <ndata> <config>"
     exit -1
 fi
-../submit_job_tcp.py $1 test_allreduce $2
+../submit_job_tcp.py $1 test_allreduce $2 $3

test/test_allreduce.cpp

@@ -8,7 +8,7 @@
 
 using namespace sync;
 
-inline void TestMax(size_t n) {
+inline void TestMax(test::Mock &mock, size_t n) {
   int rank = sync::GetRank();
   int nproc = sync::GetWorldSize();
   
@@ -16,7 +16,7 @@ inline void TestMax(size_t n) {
   for (size_t i = 0; i < ndata.size(); ++i) {
     ndata[i] = (i * (rank+1)) % 111;
   }
-  sync::AllReduce<op::Max>(&ndata[0], ndata.size());  
+  mock.AllReduce<op::Max>(&ndata[0], ndata.size());  
   for (size_t i = 0; i < ndata.size(); ++i) {
     float rmax = (i * 1) % 111;
     for (int r = 0; r < nproc; ++r) {
@@ -26,7 +26,7 @@ inline void TestMax(size_t n) {
   }
 }
 
-inline void TestSum(size_t n) {
+inline void TestSum(test::Mock &mock, size_t n) {
   int rank = sync::GetRank();
   int nproc = sync::GetWorldSize();
   const int z = 131;
@@ -35,7 +35,7 @@ inline void TestSum(size_t n) {
   for (size_t i = 0; i < ndata.size(); ++i) {
     ndata[i] = (i * (rank+1)) % z;
   }
-  sync::AllReduce<op::Sum>(&ndata[0], ndata.size());  
+  mock.AllReduce<op::Sum>(&ndata[0], ndata.size());  
   for (size_t i = 0; i < ndata.size(); ++i) {
     float rsum = 0.0f;
     for (int r = 0; r < nproc; ++r) {
@@ -46,7 +46,7 @@ inline void TestSum(size_t n) {
   }
 }
 
-inline void TestBcast(size_t n, int root) {
+inline void TestBcast(test::Mock &mock, size_t n, int root) {
   int rank = sync::GetRank();
   std::string s; s.resize(n);      
   for (size_t i = 0; i < n; ++i) {
@@ -55,31 +55,16 @@ inline void TestBcast(size_t n, int root) {
   std::string res;
   if (root == rank) {
     res = s;
-    sync::Bcast(&res, root);
+    mock.Broadcast(&res, root);
   } else {
-    sync::Bcast(&res, root);
+    mock.Broadcast(&res, root);
   }
   utils::Check(res == s, "[%d] TestBcast fail", rank);
 }
 
-// ugly stuff, just to see if it works. To be removed
-inline void Record(test::Mock& mock, const int rank) {
-  switch(rank) {
-    case 0:
-      mock.OnAllReduce(0, false);
-      break;
-    case 1: 
-      mock.OnAllReduce(1, false);
-      break;
-    case 2:
-      mock.OnAllReduce(2, true);
-      break;
-  }
-}
-
 int main(int argc, char *argv[]) {
-  if (argc < 2) {
-    printf("Usage: <ndata>\n");
+  if (argc < 3) {
+    printf("Usage: <ndata> <config>\n");
     return 0;
   }
   int n = atoi(argv[1]);
@@ -87,17 +72,12 @@ int main(int argc, char *argv[]) {
   int rank = sync::GetRank();
   std::string name = sync::GetProcessorName();
 
-  #ifdef TEST
-    test::Mock mock;
-    Record(mock, rank);
-    mock.Replay();
-    sync::SetMock(mock);
-  #endif
+  test::Mock mock(rank, argv[2]);
 
   printf("[%d] start at %s\n", rank, name.c_str());
-  TestMax(n);
+  TestMax(mock, n);
   printf("[%d] TestMax pass\n", rank);
-  TestSum(n);
+  TestSum(mock, n);
   printf("[%d] TestSum pass\n", rank);
   sync::Finalize();
   printf("[%d] all check pass\n", rank);

test/testcase0.conf

@@ -0,0 +1 @@
+# Test Case 0 -> nothing fails

test/testcase1.conf

@@ -0,0 +1,12 @@
+# Test Case example config
+# You configure which methods should fail
+# Format <round>_<rank> = <operation>
+# <operation> can be one of the following = allreduce, broadcast, loadcheckpoint, checkpoint
+
+1_0 = allreduce
+1_1 = broadcast
+1_2 = loadcheckpoint
+1_3 = checkpoint
+
+2_0 = allreduce
+2_2 = checkpoint