Merge rabit

rabit/test/.gitignore

@@ -0,0 +1,3 @@
+*.mpi
+*_test
+*_recover

rabit/test/Makefile

@@ -0,0 +1,77 @@
+RABIT_BUILD_DMLC = 0
+
+ifeq ($(RABIT_BUILD_DMLC),1)
+    DMLC=../dmlc-core
+else
+    DMLC=../../dmlc-core
+endif
+
+MPICXX=../mpich/bin/mpicxx
+export LDFLAGS=  -L../lib -pthread -lm
+export CFLAGS = -Wall -O3 -Wno-unknown-pragmas
+
+export CC = gcc
+export CXX = g++
+
+
+#----------------------------
+# Settings for power and arm arch
+#----------------------------
+ARCH := $(shell uname -a)
+ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64))
+	CFLAGS += -march=native
+else
+	CFLAGS += -msse2
+endif
+
+ifndef WITH_FPIC
+	WITH_FPIC = 1
+endif
+ifeq ($(WITH_FPIC), 1)
+	CFLAGS += -fPIC
+endif
+
+CFLAGS += -I../include -I $(DMLC)/include -std=c++11
+
+# specify tensor path
+BIN = speed_test model_recover local_recover lazy_recover
+OBJ = $(RABIT_OBJ) speed_test.o model_recover.o local_recover.o lazy_recover.o
+MPIBIN = speed_test.mpi
+.PHONY: clean all lib mpi
+
+.PHONY: lib all
+
+all: $(BIN)
+
+lib:
+	cd ..;make clean;make;cd -
+
+.PHONY: mpi
+mpi:
+	cd ..;make mpi;cd -
+
+# programs
+speed_test.o: speed_test.cc ../include/rabit/*.h lib mpi
+model_recover.o: model_recover.cc ../include/rabit/*.h lib
+local_recover.o: local_recover.cc ../include/rabit/*.h lib
+lazy_recover.o: lazy_recover.cc ../include/rabit/*.h lib
+
+# we can link against MPI version to get use MPI
+speed_test: speed_test.o  $(RABIT_OBJ)
+speed_test.mpi: speed_test.o $(MPIOBJ)
+model_recover: model_recover.o  $(RABIT_OBJ)
+local_recover: local_recover.o  $(RABIT_OBJ)
+lazy_recover: lazy_recover.o  $(RABIT_OBJ)
+
+$(BIN) :
+	$(CXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c %.cc, $^) ../lib/librabit_mock.a $(LDFLAGS)
+
+$(OBJ) :
+	$(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c %.cc, $^) )
+
+$(MPIBIN) :
+	$(MPICXX) $(CFLAGS) -I../mpich/include -shared -o $@ $(filter %.cpp %.o %.c %.cc, $^) \
+    ../lib/librabit_mpi.so $(LDFLAGS) -L../mpich/lib -Wl,-rpath,../mpich/lib -lmpi
+
+clean:
+	$(RM) $(OBJ) $(BIN) $(MPIBIN) $(MPIOBJ) *~ ../src/*~

rabit/test/README.md

@@ -0,0 +1,18 @@
+Testcases of Rabit
+====
+This folder contains internal testcases to test correctness and efficiency of rabit API
+
+The example running scripts for testcases are given by test.mk
+* type ```make -f test.mk testcasename``` to run certain testcase
+
+
+Helper Scripts
+====
+* test.mk contains Makefile documentation of all testcases
+* keepalive.sh helper bash to restart a program when it dies abnormally
+
+List of Programs
+====
+* speed_test: test the running speed of rabit API
+* test_local_recover: test recovery of local state when error happens
+* test_model_recover: test recovery of global state when error happens

rabit/test/cpp/CMakeLists.txt

@@ -0,0 +1,32 @@
+find_package(GTest REQUIRED)
+
+add_executable(
+  unit_tests
+  test_io.cc
+  test_utils.cc
+  allreduce_robust_test.cc
+  allreduce_base_test.cc
+  allreduce_mock_test.cc
+  test_main.cpp)
+
+target_link_libraries(
+  unit_tests PRIVATE
+  GTest::GTest GTest::Main
+  rabit_base rabit_mock rabit)
+
+target_include_directories(unit_tests PUBLIC
+  "$<BUILD_INTERFACE:${rabit_SOURCE_DIR}/include>"
+  "$<BUILD_INTERFACE:${DMLC_ROOT}/include>")
+
+set_target_properties(unit_tests
+  PROPERTIES
+  CXX_STANDARD 11
+  CXX_STANDARD_REQUIRED ON
+  RUNTIME_OUTPUT_DIRECTORY ${rabit_BINARY_DIR}
+  RUNTIME_OUTPUT_DIRECTORY_DEBUG ${rabit_BINARY_DIR}
+  RUNTIME_OUTPUT_DIRECTORY_RELEASE ${rabit_BINARY_DIR})
+
+add_test(
+  NAME TestRabitLib
+  COMMAND unit_tests
+  WORKING_DIRECTORY ${rabit_BINARY_DIR})

rabit/test/cpp/README.md

@@ -0,0 +1 @@
+Unittests for Rabit

rabit/test/cpp/allreduce_base_test.cc

@@ -0,0 +1,66 @@
+#define RABIT_CXXTESTDEFS_H
+#include <gtest/gtest.h>
+
+#include <string>
+#include <iostream>
+#include "../../src/allreduce_base.h"
+
+TEST(allreduce_base, init_task)
+{
+  rabit::engine::AllreduceBase base;
+
+  std::string rabit_task_id = "rabit_task_id=1";
+  char cmd[rabit_task_id.size()+1];
+  std::copy(rabit_task_id.begin(), rabit_task_id.end(), cmd);
+  cmd[rabit_task_id.size()] = '\0';
+
+  char* argv[] = {cmd};
+  base.Init(1, argv);
+  EXPECT_EQ(base.task_id, "1");
+}
+
+TEST(allreduce_base, init_with_cache_on)
+{
+  rabit::engine::AllreduceBase base;
+
+  std::string rabit_task_id = "rabit_task_id=1";
+  char cmd[rabit_task_id.size()+1];
+  std::copy(rabit_task_id.begin(), rabit_task_id.end(), cmd);
+  cmd[rabit_task_id.size()] = '\0';
+
+  std::string rabit_bootstrap_cache = "rabit_bootstrap_cache=1";
+  char cmd2[rabit_bootstrap_cache.size()+1];
+  std::copy(rabit_bootstrap_cache.begin(), rabit_bootstrap_cache.end(), cmd2);
+  cmd2[rabit_bootstrap_cache.size()] = '\0';
+
+  std::string rabit_debug = "rabit_debug=1";
+  char cmd3[rabit_debug.size()+1];
+  std::copy(rabit_debug.begin(), rabit_debug.end(), cmd3);
+  cmd3[rabit_debug.size()] = '\0';
+
+  char* argv[] = {cmd, cmd2, cmd3};
+  base.Init(3, argv);
+  EXPECT_EQ(base.task_id, "1");
+  EXPECT_EQ(base.rabit_bootstrap_cache, 1);
+  EXPECT_EQ(base.rabit_debug, 1);
+}
+
+TEST(allreduce_base, init_with_ring_reduce)
+{
+  rabit::engine::AllreduceBase base;
+
+  std::string rabit_task_id = "rabit_task_id=1";
+  char cmd[rabit_task_id.size()+1];
+  std::copy(rabit_task_id.begin(), rabit_task_id.end(), cmd);
+  cmd[rabit_task_id.size()] = '\0';
+
+  std::string rabit_reduce_ring_mincount = "rabit_reduce_ring_mincount=1";
+  char cmd2[rabit_reduce_ring_mincount.size()+1];
+  std::copy(rabit_reduce_ring_mincount.begin(), rabit_reduce_ring_mincount.end(), cmd2);
+  cmd2[rabit_reduce_ring_mincount.size()] = '\0';
+
+  char* argv[] = {cmd, cmd2};
+  base.Init(2, argv);
+  EXPECT_EQ(base.task_id, "1");
+  EXPECT_EQ(base.reduce_ring_mincount, 1);
+}

rabit/test/cpp/allreduce_base_test.cpp

@@ -0,0 +1,66 @@
+#define RABIT_CXXTESTDEFS_H
+#include <gtest/gtest.h>
+
+#include <string>
+#include <iostream>
+#include "../../src/allreduce_base.h"
+
+TEST(allreduce_base, init_task)
+{
+  rabit::engine::AllreduceBase base;
+
+  std::string rabit_task_id = "rabit_task_id=1";
+  char cmd[rabit_task_id.size()+1];
+  std::copy(rabit_task_id.begin(), rabit_task_id.end(), cmd);
+  cmd[rabit_task_id.size()] = '\0';
+
+  char* argv[] = {cmd};
+  base.Init(1, argv);
+  EXPECT_EQ(base.task_id, "1");
+}
+
+TEST(allreduce_base, init_with_cache_on)
+{
+  rabit::engine::AllreduceBase base;
+
+  std::string rabit_task_id = "rabit_task_id=1";
+  char cmd[rabit_task_id.size()+1];
+  std::copy(rabit_task_id.begin(), rabit_task_id.end(), cmd);
+  cmd[rabit_task_id.size()] = '\0';
+
+  std::string rabit_bootstrap_cache = "rabit_bootstrap_cache=1";
+  char cmd2[rabit_bootstrap_cache.size()+1];
+  std::copy(rabit_bootstrap_cache.begin(), rabit_bootstrap_cache.end(), cmd2);
+  cmd2[rabit_bootstrap_cache.size()] = '\0';
+
+  std::string rabit_debug = "rabit_debug=1";
+  char cmd3[rabit_debug.size()+1];
+  std::copy(rabit_debug.begin(), rabit_debug.end(), cmd3);
+  cmd3[rabit_debug.size()] = '\0';
+
+  char* argv[] = {cmd, cmd2, cmd3};
+  base.Init(3, argv);
+  EXPECT_EQ(base.task_id, "1");
+  EXPECT_EQ(base.rabit_bootstrap_cache, 1);
+  EXPECT_EQ(base.rabit_debug, 1);
+}
+
+TEST(allreduce_base, init_with_ring_reduce)
+{
+  rabit::engine::AllreduceBase base;
+
+  std::string rabit_task_id = "rabit_task_id=1";
+  char cmd[rabit_task_id.size()+1];
+  std::copy(rabit_task_id.begin(), rabit_task_id.end(), cmd);
+  cmd[rabit_task_id.size()] = '\0';
+
+  std::string rabit_reduce_ring_mincount = "rabit_reduce_ring_mincount=1";
+  char cmd2[rabit_reduce_ring_mincount.size()+1];
+  std::copy(rabit_reduce_ring_mincount.begin(), rabit_reduce_ring_mincount.end(), cmd2);
+  cmd2[rabit_reduce_ring_mincount.size()] = '\0';
+
+  char* argv[] = {cmd, cmd2};
+  base.Init(2, argv);
+  EXPECT_EQ(base.task_id, "1");
+  EXPECT_EQ(base.reduce_ring_mincount, 1);
+}

rabit/test/cpp/allreduce_mock_test.cc

@@ -0,0 +1,36 @@
+#define RABIT_CXXTESTDEFS_H
+#include <gtest/gtest.h>
+
+#include <string>
+#include <iostream>
+#include "../../src/allreduce_mock.h"
+
+TEST(allreduce_mock, mock_allreduce)
+{
+  rabit::engine::AllreduceMock m;
+
+  std::string mock_str = "mock=0,0,0,0";
+  char cmd[mock_str.size()+1];
+  std::copy(mock_str.begin(), mock_str.end(), cmd);
+  cmd[mock_str.size()] = '\0';
+
+  char* argv[] = {cmd};
+  m.Init(1, argv);
+  m.rank = 0;
+  EXPECT_THROW(m.Allreduce(nullptr,0,0,nullptr,nullptr,nullptr), dmlc::Error);
+}
+
+TEST(allreduce_mock, mock_broadcast)
+{
+  rabit::engine::AllreduceMock m;
+  std::string mock_str = "mock=0,1,2,0";
+  char cmd[mock_str.size()+1];
+  std::copy(mock_str.begin(), mock_str.end(), cmd);
+  cmd[mock_str.size()] = '\0';
+  char* argv[] = {cmd};
+  m.Init(1, argv);
+  m.rank = 0;
+  m.version_number=1;
+  m.seq_counter=2;
+  EXPECT_THROW(m.Broadcast(nullptr,0,0), dmlc::Error);
+}

rabit/test/cpp/allreduce_mock_test.cpp

@@ -0,0 +1,52 @@
+#define RABIT_CXXTESTDEFS_H
+#include <gtest/gtest.h>
+
+#include <string>
+#include <iostream>
+#include <dmlc/logging.h>
+#include "../../src/allreduce_mock.h"
+
+TEST(allreduce_mock, mock_allreduce)
+{
+  rabit::engine::AllreduceMock m;
+
+  std::string mock_str = "mock=0,0,0,0";
+  char cmd[mock_str.size()+1];
+  std::copy(mock_str.begin(), mock_str.end(), cmd);
+  cmd[mock_str.size()] = '\0';
+
+  char* argv[] = {cmd};
+  m.Init(1, argv);
+  m.rank = 0;
+  EXPECT_THROW({m.Allreduce(nullptr,0,0,nullptr,nullptr,nullptr);}, dmlc::Error);
+}
+
+TEST(allreduce_mock, mock_broadcast)
+{
+  rabit::engine::AllreduceMock m;
+  std::string mock_str = "mock=0,1,2,0";
+  char cmd[mock_str.size()+1];
+  std::copy(mock_str.begin(), mock_str.end(), cmd);
+  cmd[mock_str.size()] = '\0';
+  char* argv[] = {cmd};
+  m.Init(1, argv);
+  m.rank = 0;
+  m.version_number=1;
+  m.seq_counter=2;
+  EXPECT_THROW({m.Broadcast(nullptr,0,0);}, dmlc::Error);
+}
+
+TEST(allreduce_mock, mock_gather)
+{
+  rabit::engine::AllreduceMock m;
+  std::string mock_str = "mock=3,13,22,0";
+  char cmd[mock_str.size()+1];
+  std::copy(mock_str.begin(), mock_str.end(), cmd);
+  cmd[mock_str.size()] = '\0';
+  char* argv[] = {cmd};
+  m.Init(1, argv);
+  m.rank = 3;
+  m.version_number=13;
+  m.seq_counter=22;
+  EXPECT_THROW({m.Allgather(nullptr,0,0,0,0);}, dmlc::Error);
+}

rabit/test/cpp/allreduce_robust_test.cc

@@ -0,0 +1,233 @@
+#define RABIT_CXXTESTDEFS_H
+#include <gtest/gtest.h>
+
+#include <chrono>
+#include <string>
+#include <iostream>
+#include "../../src/allreduce_robust.h"
+
+inline void mockerr(const char *fmt, ...) {EXPECT_STRCASEEQ(fmt, "[%d] exit due to time out %d s\n");}
+inline void mockassert(bool val, const char *fmt, ...) {}
+rabit::engine::AllreduceRobust::ReturnType err_type(rabit::engine::AllreduceRobust::ReturnTypeEnum::kSockError);
+rabit::engine::AllreduceRobust::ReturnType succ_type(rabit::engine::AllreduceRobust::ReturnTypeEnum::kSuccess);
+
+TEST(allreduce_robust, sync_error_timeout)
+{
+  rabit::engine::AllreduceRobust m;
+
+  std::string rabit_timeout = "rabit_timeout=1";
+  char cmd[rabit_timeout.size()+1];
+  std::copy(rabit_timeout.begin(), rabit_timeout.end(), cmd);
+  cmd[rabit_timeout.size()] = '\0';
+
+  std::string rabit_timeout_sec = "rabit_timeout_sec=1";
+  char cmd1[rabit_timeout_sec.size()+1];
+  std::copy(rabit_timeout_sec.begin(), rabit_timeout_sec.end(), cmd1);
+  cmd1[rabit_timeout_sec.size()] = '\0';
+
+  char* argv[] = {cmd,cmd1};
+  m.Init(2, argv);
+  m.rank = 0;
+  m.rabit_bootstrap_cache = 1;
+  m._error = mockerr;
+  m._assert = mockassert;
+  EXPECT_EQ(m.CheckAndRecover(err_type), false);
+  std::this_thread::sleep_for(std::chrono::milliseconds(1500));
+  EXPECT_EQ(m.rabit_timeout_task.get(), false);
+}
+
+TEST(allreduce_robust, sync_error_reset)
+{
+  rabit::engine::AllreduceRobust m;
+
+  std::string rabit_timeout = "rabit_timeout=1";
+  char cmd[rabit_timeout.size()+1];
+  std::copy(rabit_timeout.begin(), rabit_timeout.end(), cmd);
+  cmd[rabit_timeout.size()] = '\0';
+
+  std::string rabit_timeout_sec = "rabit_timeout_sec=1";
+  char cmd1[rabit_timeout_sec.size()+1];
+  std::copy(rabit_timeout_sec.begin(), rabit_timeout_sec.end(), cmd1);
+  cmd1[rabit_timeout_sec.size()] = '\0';
+
+  std::string rabit_debug = "rabit_debug=1";
+  char cmd2[rabit_debug.size()+1];
+  std::copy(rabit_debug.begin(), rabit_debug.end(), cmd2);
+  cmd2[rabit_debug.size()] = '\0';
+
+  char* argv[] = {cmd, cmd1,cmd2};
+  m.Init(3, argv);
+  m.rank = 0;
+  m._assert = mockassert;
+  EXPECT_EQ(m.CheckAndRecover(err_type), false);
+  std::this_thread::sleep_for(std::chrono::milliseconds(100));
+  EXPECT_EQ(m.CheckAndRecover(succ_type), true);
+  EXPECT_EQ(m.rabit_timeout_task.get(), true);
+  m.Shutdown();
+}
+
+TEST(allreduce_robust, sync_success_error_timeout)
+{
+  rabit::engine::AllreduceRobust m;
+
+  std::string rabit_timeout = "rabit_timeout=1";
+  char cmd[rabit_timeout.size()+1];
+  std::copy(rabit_timeout.begin(), rabit_timeout.end(), cmd);
+  cmd[rabit_timeout.size()] = '\0';
+
+  std::string rabit_timeout_sec = "rabit_timeout_sec=1";
+  char cmd1[rabit_timeout_sec.size()+1];
+  std::copy(rabit_timeout_sec.begin(), rabit_timeout_sec.end(), cmd1);
+  cmd1[rabit_timeout_sec.size()] = '\0';
+
+  std::string rabit_debug = "rabit_debug=1";
+  char cmd2[rabit_debug.size()+1];
+  std::copy(rabit_debug.begin(), rabit_debug.end(), cmd2);
+  cmd2[rabit_debug.size()] = '\0';
+
+  char* argv[] = {cmd, cmd1,cmd2};
+  m.Init(3, argv);
+  m.rank = 0;
+  m.rabit_bootstrap_cache = 1;
+  m._assert = mockassert;
+  m._error = mockerr;
+  EXPECT_EQ(m.CheckAndRecover(succ_type), true);
+  std::this_thread::sleep_for(std::chrono::milliseconds(100));
+  EXPECT_EQ(m.CheckAndRecover(err_type), false);
+  std::this_thread::sleep_for(std::chrono::milliseconds(1500));
+  EXPECT_EQ(m.rabit_timeout_task.get(), false);
+}
+
+TEST(allreduce_robust, sync_success_error_success)
+{
+  rabit::engine::AllreduceRobust m;
+
+  std::string rabit_timeout = "rabit_timeout=1";
+  char cmd[rabit_timeout.size()+1];
+  std::copy(rabit_timeout.begin(), rabit_timeout.end(), cmd);
+  cmd[rabit_timeout.size()] = '\0';
+
+  std::string rabit_timeout_sec = "rabit_timeout_sec=1";
+  char cmd1[rabit_timeout_sec.size()+1];
+  std::copy(rabit_timeout_sec.begin(), rabit_timeout_sec.end(), cmd1);
+  cmd1[rabit_timeout_sec.size()] = '\0';
+
+  std::string rabit_debug = "rabit_debug=1";
+  char cmd2[rabit_debug.size()+1];
+  std::copy(rabit_debug.begin(), rabit_debug.end(), cmd2);
+  cmd2[rabit_debug.size()] = '\0';
+
+  char* argv[] = {cmd, cmd1,cmd2};
+  m.Init(3, argv);
+  m.rank = 0;
+  m.rabit_bootstrap_cache = 1;
+  m._assert = mockassert;
+  EXPECT_EQ(m.CheckAndRecover(succ_type), true);
+  std::this_thread::sleep_for(std::chrono::milliseconds(10));
+
+  EXPECT_EQ(m.CheckAndRecover(err_type), false);
+  std::this_thread::sleep_for(std::chrono::milliseconds(10));
+  EXPECT_EQ(m.CheckAndRecover(succ_type), true);
+  std::this_thread::sleep_for(std::chrono::milliseconds(1100));
+  EXPECT_EQ(m.rabit_timeout_task.get(), true);
+  m.Shutdown();
+}
+
+TEST(allreduce_robust, sync_error_no_reset_timeout)
+{
+  rabit::engine::AllreduceRobust m;
+
+  std::string rabit_timeout = "rabit_timeout=1";
+  char cmd[rabit_timeout.size()+1];
+  std::copy(rabit_timeout.begin(), rabit_timeout.end(), cmd);
+  cmd[rabit_timeout.size()] = '\0';
+
+  std::string rabit_timeout_sec = "rabit_timeout_sec=1";
+  char cmd1[rabit_timeout_sec.size()+1];
+  std::copy(rabit_timeout_sec.begin(), rabit_timeout_sec.end(), cmd1);
+  cmd1[rabit_timeout_sec.size()] = '\0';
+
+  std::string rabit_debug = "rabit_debug=1";
+  char cmd2[rabit_debug.size()+1];
+  std::copy(rabit_debug.begin(), rabit_debug.end(), cmd2);
+  cmd2[rabit_debug.size()] = '\0';
+
+  char* argv[] = {cmd, cmd1,cmd2};
+  m.Init(3, argv);
+  m.rank = 0;
+  m.rabit_bootstrap_cache = 1;
+  m._assert = mockassert;
+  m._error = mockerr;
+  auto start = std::chrono::system_clock::now();
+
+  EXPECT_EQ(m.CheckAndRecover(err_type), false);
+  std::this_thread::sleep_for(std::chrono::milliseconds(1100));
+
+  EXPECT_EQ(m.CheckAndRecover(err_type), false);
+
+  m.rabit_timeout_task.wait();
+  auto end = std::chrono::system_clock::now();
+  std::chrono::duration<double> diff = end-start;
+
+  EXPECT_EQ(m.rabit_timeout_task.get(), false);
+  // expect second error don't overwrite/reset timeout task
+  EXPECT_LT(diff.count(), 2);
+}
+
+TEST(allreduce_robust, no_timeout_shut_down)
+{
+  rabit::engine::AllreduceRobust m;
+
+  std::string rabit_timeout = "rabit_timeout=1";
+  char cmd[rabit_timeout.size()+1];
+  std::copy(rabit_timeout.begin(), rabit_timeout.end(), cmd);
+  cmd[rabit_timeout.size()] = '\0';
+
+  std::string rabit_timeout_sec = "rabit_timeout_sec=1";
+  char cmd1[rabit_timeout_sec.size()+1];
+  std::copy(rabit_timeout_sec.begin(), rabit_timeout_sec.end(), cmd1);
+  cmd1[rabit_timeout_sec.size()] = '\0';
+
+  std::string rabit_debug = "rabit_debug=1";
+  char cmd2[rabit_debug.size()+1];
+  std::copy(rabit_debug.begin(), rabit_debug.end(), cmd2);
+  cmd2[rabit_debug.size()] = '\0';
+
+  char* argv[] = {cmd, cmd1,cmd2};
+  m.Init(3, argv);
+  m.rank = 0;
+
+  EXPECT_EQ(m.CheckAndRecover(succ_type), true);
+  std::this_thread::sleep_for(std::chrono::milliseconds(10));
+  m.Shutdown();
+}
+
+TEST(allreduce_robust, shut_down_before_timeout)
+{
+  rabit::engine::AllreduceRobust m;
+
+  std::string rabit_timeout = "rabit_timeout=1";
+  char cmd[rabit_timeout.size()+1];
+  std::copy(rabit_timeout.begin(), rabit_timeout.end(), cmd);
+  cmd[rabit_timeout.size()] = '\0';
+
+  std::string rabit_timeout_sec = "rabit_timeout_sec=1";
+  char cmd1[rabit_timeout_sec.size()+1];
+  std::copy(rabit_timeout_sec.begin(), rabit_timeout_sec.end(), cmd1);
+  cmd1[rabit_timeout_sec.size()] = '\0';
+
+  std::string rabit_debug = "rabit_debug=1";
+  char cmd2[rabit_debug.size()+1];
+  std::copy(rabit_debug.begin(), rabit_debug.end(), cmd2);
+  cmd2[rabit_debug.size()] = '\0';
+
+  char* argv[] = {cmd, cmd1,cmd2};
+  m.Init(3, argv);
+  m.rank = 0;
+  rabit::engine::AllreduceRobust::LinkRecord a;
+  m.err_link = &a;
+  
+  EXPECT_EQ(m.CheckAndRecover(err_type), false);
+  std::this_thread::sleep_for(std::chrono::milliseconds(10));
+  m.Shutdown();
+}

rabit/test/cpp/test_io.cc

@@ -0,0 +1,18 @@
+/*!
+ *  Copyright (c) 2019 by Contributors
+ */
+#include <gtest/gtest.h>
+#include <rabit/internal/io.h>
+
+#include <vector>
+
+namespace rabit {
+TEST(MemoryFixSizeBuffer, Seek) {
+  size_t constexpr kSize { 64 };
+  std::vector<int32_t> memory( kSize );
+  utils::MemoryFixSizeBuffer buf(memory.data(), memory.size());
+  buf.Seek(utils::MemoryFixSizeBuffer::SeekEnd);
+  size_t end = buf.Tell();
+  ASSERT_EQ(end, kSize);
+}
+}  // namespace rabit

rabit/test/cpp/test_main.cpp

@@ -0,0 +1,8 @@
+#include "gtest/gtest.h"
+
+int main(int argc, char** argv)
+{
+    ::testing::InitGoogleTest(&argc, argv);
+    ::testing::FLAGS_gtest_death_test_style = "threadsafe";
+    return RUN_ALL_TESTS();
+}

rabit/test/cpp/test_utils.cc

@@ -0,0 +1,6 @@
+#include <gtest/gtest.h>
+#include <rabit/internal/utils.h>
+
+TEST(Utils, Assert) {
+  EXPECT_THROW({rabit::utils::Assert(false, "foo");}, dmlc::Error);
+}

rabit/test/lazy_recover.cc

@@ -0,0 +1,125 @@
+// this is a test case to test whether rabit can recover model when
+// facing an exception
+#include <rabit/rabit.h>
+#include <cstdio>
+#include <cstdlib>
+#include <cmath>
+using namespace rabit;
+
+// dummy model
+class Model : public rabit::Serializable {
+ public:
+  // iterations
+  std::vector<float> data;
+  // load from stream
+  virtual void Load(rabit::Stream *fi) {
+    fi->Read(&data);
+  }
+  /*! \brief save the model to the stream */
+  virtual void Save(rabit::Stream *fo) const {
+    fo->Write(data);
+  }
+  virtual void InitModel(size_t n) {
+    data.clear();
+    data.resize(n, 1.0f);
+  }
+};
+
+inline void TestMax(Model *model, int ntrial, int iter) {
+  int rank = rabit::GetRank();
+  int nproc = rabit::GetWorldSize();
+  const int z = iter + 111;
+
+  std::vector<float> ndata(model->data.size());
+  for (size_t i = 0; i < ndata.size(); ++i) {
+    ndata[i] = (i * (rank+1)) % z  + model->data[i];
+  }
+  rabit::Allreduce<op::Max>(&ndata[0], ndata.size());
+
+  for (size_t i = 0; i < ndata.size(); ++i) {
+    float rmax = (i * 1) % z + model->data[i];
+    for (int r = 0; r < nproc; ++r) {
+      rmax = std::max(rmax, (float)((i * (r+1)) % z) + model->data[i]);
+    }
+    utils::Check(rmax == ndata[i], "[%d] TestMax check failurem i=%lu, rmax=%f, ndata=%f", rank, i, rmax, ndata[i]);
+  }
+}
+
+inline void TestSum(Model *model, int ntrial, int iter) {
+  int rank = rabit::GetRank();
+  int nproc = rabit::GetWorldSize();
+  const int z = 131 + iter;
+
+  std::vector<float> ndata(model->data.size());
+  for (size_t i = 0; i < ndata.size(); ++i) {
+    ndata[i] = (i * (rank+1)) % z + model->data[i];
+  }
+  Allreduce<op::Sum>(&ndata[0], ndata.size());
+
+  for (size_t i = 0; i < ndata.size(); ++i) {
+    float rsum = model->data[i] * nproc;
+    for (int r = 0; r < nproc; ++r) {
+      rsum += (float)((i * (r+1)) % z);
+    }
+    utils::Check(fabsf(rsum - ndata[i]) < 1e-5 ,
+                 "[%d] TestSum check failure, local=%g, allreduce=%g", rank, rsum, ndata[i]);
+  }
+  model->data = ndata;
+}
+
+inline void TestBcast(size_t n, int root, int ntrial, int iter) {
+  int rank = rabit::GetRank();
+  std::string s; s.resize(n);
+  for (size_t i = 0; i < n; ++i) {
+    s[i] = char(i % 126 + 1);
+  }
+  std::string res;
+  if (root == rank) {
+    res = s;
+    rabit::Broadcast(&res, root);
+  } else {
+    rabit::Broadcast(&res, root);
+  }
+  utils::Check(res == s, "[%d] TestBcast fail", rank);
+}
+
+int main(int argc, char *argv[]) {
+  if (argc < 3) {
+    printf("Usage: <ndata> <config>\n");
+    return 0;
+  }
+  int n = atoi(argv[1]);
+  rabit::Init(argc, argv);
+  int rank = rabit::GetRank();
+  int nproc = rabit::GetWorldSize();
+  std::string name = rabit::GetProcessorName();
+  Model model;
+  srand(0);
+  int ntrial = 0;
+  for (int i = 1; i < argc; ++i) {
+    int n;
+    if (sscanf(argv[i], "rabit_num_trial=%d", &n) == 1) ntrial = n;
+  }
+  int iter = rabit::LoadCheckPoint(&model);
+  if (iter == 0) {
+    model.InitModel(n);
+    printf("[%d] reload-trail=%d, init iter=%d\n", rank, ntrial, iter);
+  } else {
+    printf("[%d] reload-trail=%d, init iter=%d\n", rank, ntrial, iter);
+  }
+  for (int r = iter; r < 3; ++r) {
+    TestMax(&model, ntrial, r);
+    printf("[%d] !!!TestMax pass, iter=%d\n",  rank, r);
+    int step = std::max(nproc / 3, 1);
+    for (int i = 0; i < nproc; i += step) {
+      TestBcast(n, i, ntrial, r);
+    }
+    printf("[%d] !!!TestBcast pass, iter=%d\n", rank, r);
+    TestSum(&model, ntrial, r);
+    printf("[%d] !!!TestSum pass, iter=%d\n", rank, r);
+    rabit::LazyCheckPoint(&model);
+    printf("[%d] !!!CheckPoint pass, iter=%d\n", rank, r);
+  }
+  rabit::Finalize();
+  return 0;
+}

rabit/test/local_recover.cc

@@ -0,0 +1,137 @@
+// this is a test case to test whether rabit can recover model when
+// facing an exception
+#include <rabit/rabit.h>
+#include <cstdio>
+#include <cstdlib>
+#include <cmath>
+
+using namespace rabit;
+
+// dummy model
+class Model : public rabit::Serializable {
+ public:
+  // iterations
+  std::vector<float> data;
+  // load from stream
+  virtual void Load(rabit::Stream *fi) {
+    fi->Read(&data);
+  }
+  /*! \brief save the model to the stream */
+  virtual void Save(rabit::Stream *fo) const {
+    fo->Write(data);
+  }
+  virtual void InitModel(size_t n, float v) {
+    data.clear();
+    data.resize(n, v);
+  }
+};
+
+inline void TestMax(Model *model, Model *local, int ntrial, int iter) {
+  int rank = rabit::GetRank();
+  int nproc = rabit::GetWorldSize();
+  const int z = iter + 111;
+  std::vector<float> ndata(model->data.size());
+  rabit::Allreduce<op::Max>(&ndata[0], ndata.size(),
+                            [&]() {
+                              // use lambda expression to prepare the data
+                              for (size_t i = 0; i < ndata.size(); ++i) {
+                                ndata[i] = (i * (rank+1)) % z  + local->data[i];
+                              }
+                            });
+
+  for (size_t i = 0; i < ndata.size(); ++i) {
+    float rmax = (i * 1) % z + model->data[i];
+    for (int r = 0; r < nproc; ++r) {
+      rmax = std::max(rmax, (float)((i * (r+1)) % z) + model->data[i] + r);
+    }
+    utils::Check(rmax == ndata[i], "[%d] TestMax check failure", rank);
+  }
+  model->data = ndata;
+  local->data = ndata;
+  for (size_t i = 0; i < ndata.size(); ++i) {
+    local->data[i] = ndata[i] + rank;
+  }
+}
+
+inline void TestSum(Model *model, Model *local, int ntrial, int iter) {
+  int rank = rabit::GetRank();
+  int nproc = rabit::GetWorldSize();
+  const int z = 131 + iter;
+
+  std::vector<float> ndata(model->data.size());
+  for (size_t i = 0; i < ndata.size(); ++i) {
+    ndata[i] = (i * (rank+1)) % z + local->data[i];
+  }
+  Allreduce<op::Sum>(&ndata[0], ndata.size());
+
+  for (size_t i = 0; i < ndata.size(); ++i) {
+    float rsum = 0.0f;
+    for (int r = 0; r < nproc; ++r) {
+      rsum += (float)((i * (r+1)) % z) + model->data[i] + r;
+    }
+    utils::Check(fabsf(rsum - ndata[i]) < 1e-5 ,
+                 "[%d] TestSum check failure, local=%g, allreduce=%g", rank, rsum, ndata[i]);
+  }
+  model->data = ndata;
+  for (size_t i = 0; i < ndata.size(); ++i) {
+    local->data[i] = ndata[i] + rank;
+  }
+}
+
+inline void TestBcast(size_t n, int root, int ntrial, int iter) {
+  int rank = rabit::GetRank();
+  std::string s; s.resize(n);
+  for (size_t i = 0; i < n; ++i) {
+    s[i] = char(i % 126 + 1);
+  }
+  std::string res;
+  if (root == rank) {
+    res = s;
+    rabit::Broadcast(&res, root);
+  } else {
+    rabit::Broadcast(&res, root);
+  }
+  utils::Check(res == s, "[%d] TestBcast fail", rank);
+}
+
+int main(int argc, char *argv[]) {
+  if (argc < 3) {
+    printf("Usage: <ndata>\n");
+    return 0;
+  }
+  int n = atoi(argv[1]);
+  rabit::Init(argc, argv);
+  int rank = rabit::GetRank();
+  int nproc = rabit::GetWorldSize();
+  std::string name = rabit::GetProcessorName();
+  Model model, local;
+  srand(0);
+  int ntrial = 0;
+  for (int i = 1; i < argc; ++i) {
+    int n;
+    if (sscanf(argv[i], "repeat=%d", &n) == 1) ntrial = n;
+  }
+  int iter = rabit::LoadCheckPoint(&model, &local);
+  if (iter == 0) {
+    model.InitModel(n, 1.0f);
+    local.InitModel(n, 1.0f + rank);
+    printf("[%d] reload-trail=%d, init iter=%d\n", rank, ntrial, iter);
+  } else {
+    printf("[%d] reload-trail=%d, init iter=%d\n", rank, ntrial, iter);
+  }
+  for (int r = iter; r < 3; ++r) {
+    TestMax(&model, &local, ntrial, r);
+    printf("[%d] !!!TestMax pass, iter=%d\n",  rank, r);
+    int step = std::max(nproc / 3, 1);
+    for (int i = 0; i < nproc; i += step) {
+      TestBcast(n, i, ntrial, r);
+    }
+    printf("[%d] !!!TestBcast pass, iter=%d\n", rank, r);
+    TestSum(&model, &local, ntrial, r);
+    printf("[%d] !!!TestSum pass, iter=%d\n", rank, r);
+    rabit::CheckPoint(&model, &local);
+    printf("[%d] !!!CheckPoint pass, iter=%d\n", rank, r);
+  }
+  rabit::Finalize();
+  return 0;
+}

rabit/test/local_recover.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+from __future__ import print_function
+from builtins import range
+
+import sys
+sys.path.append('../python')
+
+import rabit
+import numpy as np
+
+rabit.init(lib='mock')
+rank = rabit.get_rank()
+n = 10
+nround = 3
+data = np.ones(n) * rank
+
+version, model, local = rabit.load_checkpoint(True)
+if version == 0:
+    model = np.zeros(n)
+    local = np.ones(n)
+else:
+    print('[%d] restart from version %d' % (rank, version))
+
+for i in range(version, nround):
+    res = rabit.allreduce(data + model+local, rabit.SUM)
+    print('[%d] iter=%d: %s' % (rank, i, str(res)))
+    model = res
+    local[:] = i
+    rabit.checkpoint(model, local)
+
+rabit.finalize()

rabit/test/model_recover.cc

@@ -0,0 +1,157 @@
+// this is a test case to test whether rabit can recover model when
+// facing an exception
+#include <rabit/rabit.h>
+#include <cstdio>
+#include <cstdlib>
+#include <cmath>
+
+using namespace rabit;
+
+// dummy model
+class Model : public rabit::Serializable {
+ public:
+  // iterations
+  std::vector<float> data;
+  // load from stream
+  virtual void Load(rabit::Stream *fi) {
+    fi->Read(&data);
+  }
+  /*! \brief save the model to the stream */
+  virtual void Save(rabit::Stream *fo) const {
+    fo->Write(data);
+  }
+  virtual void InitModel(size_t n) {
+    data.clear();
+    data.resize(n, 1.0f);
+  }
+};
+
+inline void TestMax(Model *model, int iter) {
+  int rank = rabit::GetRank();
+  int nproc = rabit::GetWorldSize();
+  const int z = iter + 111;
+
+  std::vector<float> ndata(model->data.size());
+  for (size_t i = 0; i < ndata.size(); ++i) {
+    ndata[i] = (i * (rank+1)) % z  + model->data[i];
+  }
+  rabit::Allreduce<op::Max>(&ndata[0], ndata.size());
+
+  for (size_t i = 0; i < ndata.size(); ++i) {
+    float rmax = (i * 1) % z + model->data[i];
+    for (int r = 0; r < nproc; ++r) {
+      rmax = std::max(rmax, (float)((i * (r+1)) % z) + model->data[i]);
+    }
+    utils::Check(rmax == ndata[i], "[%d] TestMax check failurem i=%lu, rmax=%f, ndata=%f", rank, i, rmax, ndata[i]);
+  }
+  model->data = ndata;
+}
+
+inline void TestSum(Model *model, int iter) {
+  int rank = rabit::GetRank();
+  int nproc = rabit::GetWorldSize();
+  const int z = 131 + iter;
+
+  std::vector<float> ndata(model->data.size());
+  for (size_t i = 0; i < ndata.size(); ++i) {
+    ndata[i] = (i * (rank+1)) % z + model->data[i];
+  }
+  Allreduce<op::Sum>(&ndata[0], ndata.size());
+
+  for (size_t i = 0; i < ndata.size(); ++i) {
+    float rsum = model->data[i] * nproc;
+    for (int r = 0; r < nproc; ++r) {
+      rsum += (float)((i * (r+1)) % z);
+    }
+    utils::Check(fabsf(rsum - ndata[i]) < 1e-5 ,
+                 "[%d] TestSum check failure, local=%g, allreduce=%g", rank, rsum, ndata[i]);
+  }
+  model->data = ndata;
+}
+
+inline void TestAllgather(Model *model, int iter) {
+  int rank = rabit::GetRank();
+  int nproc = rabit::GetWorldSize();
+  const int z = 131 + iter;
+
+  std::vector<float> ndata(model->data.size() * nproc);
+  size_t beginSlice = rank * model->data.size();
+  for (size_t i = 0; i < model->data.size(); ++i) {
+    ndata[beginSlice + i] = (i * (rank+1)) % z + model->data[i];
+  }
+  Allgather(&ndata[0], ndata.size(), beginSlice,
+  model->data.size(), model->data.size());
+
+  for (size_t i = 0; i < ndata.size(); ++i) {
+    int curRank = i / model->data.size();
+    int remainder = i % model->data.size();
+    float data = (remainder * (curRank+1)) % z + model->data[remainder];
+    utils::Check(fabsf(data - ndata[i]) < 1e-5 ,
+                 "[%d] TestAllgather check failure, local=%g, allgatherring=%g", rank, data, ndata[i]);
+  }
+  model->data = ndata;
+}
+
+inline void TestBcast(size_t n, int root) {
+  int rank = rabit::GetRank();
+  std::string s; s.resize(n);
+  for (size_t i = 0; i < n; ++i) {
+    s[i] = char(i % 126 + 1);
+  }
+  std::string res;
+  if (root == rank) {
+    res = s;
+  }
+  rabit::Broadcast(&res, root);
+
+  utils::Check(res == s, "[%d] TestBcast fail", rank);
+}
+
+int main(int argc, char *argv[]) {
+  if (argc < 3) {
+    printf("Usage: <ndata> <config>\n");
+    return 0;
+  }
+  int n = atoi(argv[1]);
+  rabit::Init(argc, argv);
+  int rank = rabit::GetRank();
+  int nproc = rabit::GetWorldSize();
+  std::string name = rabit::GetProcessorName();
+
+  int max_rank = rank;
+  rabit::Allreduce<op::Max>(&max_rank, 1);
+  utils::Check(max_rank == nproc - 1, "max rank is world size-1");
+
+  Model model;
+  srand(0);
+  int ntrial = 0;
+  for (int i = 1; i < argc; ++i) {
+    int n;
+    if (sscanf(argv[i], "rabit_num_trial=%d", &n) == 1) ntrial = n;
+  }
+  int iter = rabit::LoadCheckPoint(&model);
+  if (iter == 0) {
+    model.InitModel(n);
+  }
+  printf("[%d] reload-trail=%d, init iter=%d\n", rank, ntrial, iter);
+
+  for (int r = iter; r < 3; ++r) {
+    TestMax(&model, r);
+    printf("[%d] !!!TestMax pass, iter=%d\n",  rank, r);
+    int step = std::max(nproc / 3, 1);
+    for (int i = 0; i < nproc; i += step) {
+      TestBcast(n, i);
+    }
+    printf("[%d] !!!TestBcast pass, iter=%d\n", rank, r);
+
+    TestSum(&model, r);
+    printf("[%d] !!!TestSum pass, iter=%d\n", rank, r);
+    TestAllgather(&model, r);
+    printf("[%d] !!!TestAllgather pass, iter=%d\n", rank, r);
+    rabit::CheckPoint(&model);
+    printf("[%d] !!!Checkpoint pass, iter=%d\n", rank, r);
+  }
+  rabit::Finalize();
+  return 0;
+}
+

rabit/test/speed_runner.py

@@ -0,0 +1,34 @@
+import os
+import argparse
+import sys
+
+def main():
+  parser = argparse.ArgumentParser(description='TODO')
+  parser.add_argument('-ho', '--host_dir', required=True)
+  parser.add_argument('-s', '--submit_script', required=True)
+  parser.add_argument('-rex', '--rabit_exec', required=True)
+  parser.add_argument('-mpi', '--mpi_exec', required=True)
+  args = parser.parse_args()
+
+  ndata = [10**4, 10**5, 10**6, 10**7]
+  nrepeat = [10**4, 10**3, 10**2, 10]
+
+  machines = [2,4,8,16,31]
+
+  executables = [args.rabit_exec, args.mpi_exec]
+
+  for executable in executables:
+    sys.stderr.write('Executable %s' % executable)
+    sys.stderr.flush()
+    for i, data in enumerate(ndata):
+      for machine in machines:
+        host_file = os.path.join(args.host_dir, 'hosts%d' % machine)
+        cmd = 'python %s %d %s %s %d %d' % (args.submit_script, machine, host_file, executable, data, nrepeat[i])
+        sys.stderr.write('data=%d, repeat=%d, machine=%d\n' % (data, nrepeat[i], machine))
+        sys.stderr.flush()
+        os.system(cmd)
+    sys.stderr.write('\n')
+    sys.stderr.flush()
+
+if __name__ == "__main__":
+  main()

rabit/test/speed_test.cc

@@ -0,0 +1,99 @@
+// This program is used to test the speed of rabit API
+#include <rabit/rabit.h>
+#include <rabit/internal/timer.h>
+#include <cstdio>
+#include <cstdlib>
+#include <cmath>
+#include <time.h>
+
+using namespace rabit;
+
+double max_tdiff, sum_tdiff, bcast_tdiff, tot_tdiff;
+
+inline void TestMax(size_t n) {
+  int rank = rabit::GetRank();
+  std::vector<float> ndata(n);
+  for (size_t i = 0; i < ndata.size(); ++i) {
+    ndata[i] = (i * (rank+1)) % 111;
+  }
+  double tstart = utils::GetTime();
+  rabit::Allreduce<op::Max>(&ndata[0], ndata.size());
+  max_tdiff += utils::GetTime() - tstart;
+}
+
+inline void TestSum(size_t n) {
+  int rank = rabit::GetRank();
+  const int z = 131;
+  std::vector<float> ndata(n);
+  for (size_t i = 0; i < ndata.size(); ++i) {
+    ndata[i] = (i * (rank+1)) % z;
+  }
+  double tstart = utils::GetTime();
+  rabit::Allreduce<op::Sum>(&ndata[0], ndata.size());
+  sum_tdiff += utils::GetTime() - tstart;
+}
+
+inline void TestBcast(size_t n, int root) {
+  int rank = rabit::GetRank();
+  std::string s; s.resize(n);
+  for (size_t i = 0; i < n; ++i) {
+    s[i] = char(i % 126 + 1);
+  }
+  std::string res;
+  res.resize(n);
+  if (root == rank) {
+    res = s;
+  }
+  double tstart = utils::GetTime();
+  rabit::Broadcast(&res[0], res.length(), root);
+  bcast_tdiff += utils::GetTime() - tstart;
+}
+
+inline void PrintStats(const char *name, double tdiff, int n, int nrep, size_t size) {
+  int nproc = rabit::GetWorldSize();
+  double tsum = tdiff;
+  rabit::Allreduce<op::Sum>(&tsum, 1);
+  double tavg = tsum / nproc;
+  double tsqr = tdiff - tavg;
+  tsqr *= tsqr;
+  rabit::Allreduce<op::Sum>(&tsqr, 1);
+  double tstd = sqrt(tsqr / nproc);
+  if (rabit::GetRank() == 0) {
+    rabit::TrackerPrintf("%s: mean=%g, std=%g sec\n", name, tavg, tstd);
+    double ndata = n;
+    ndata *= nrep * size;
+    if (n != 0) {
+      rabit::TrackerPrintf("%s-speed: %g MB/sec\n", name, (ndata / tavg) / 1024 / 1024 );
+    }
+  }
+}
+
+int main(int argc, char *argv[]) {
+  if (argc < 3) {
+    printf("Usage: <ndata> <nrepeat>\n");
+    return 0;
+  }
+  srand(0);
+  int n = atoi(argv[1]);
+  int nrep = atoi(argv[2]);
+  utils::Check(nrep >= 1, "need to at least repeat running once");
+  rabit::Init(argc, argv);
+  //int rank = rabit::GetRank();
+  int nproc = rabit::GetWorldSize();
+  std::string name = rabit::GetProcessorName();
+  max_tdiff = sum_tdiff = bcast_tdiff = 0;
+  double tstart = utils::GetTime();
+  for (int i = 0; i < nrep; ++i) {
+    TestMax(n);
+    TestSum(n);
+    TestBcast(n, rand() % nproc);
+  }
+  tot_tdiff = utils::GetTime() - tstart;
+  // use allreduce to get the sum and std of time
+  PrintStats("max_tdiff", max_tdiff, n, nrep, sizeof(float));
+  PrintStats("sum_tdiff", sum_tdiff, n, nrep, sizeof(float));
+  PrintStats("bcast_tdiff", bcast_tdiff, n, nrep, sizeof(char));
+  PrintStats("tot_tdiff", tot_tdiff, 0, nrep, sizeof(float));
+  rabit::Finalize();
+  return 0;
+}

rabit/test/test.mk

@@ -0,0 +1,37 @@
+RABIT_BUILD_DMLC = 0
+
+ifeq ($(RABIT_BUILD_DMLC),1)
+    DMLC=../dmlc-core
+else
+    DMLC=../../dmlc-core
+endif
+
+# this is a makefile used to show testcases of rabit
+.PHONY: all
+
+all: model_recover_10_10k  model_recover_10_10k_die_same model_recover_10_10k_die_hard local_recover_10_10k lazy_recover_10_10k_die_hard lazy_recover_10_10k_die_same ringallreduce_10_10k pylocal_recover_10_10k
+
+# this experiment test recovery with actually process exit, use keepalive to keep program alive
+model_recover_10_10k:
+	python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 model_recover 10000 mock=0,0,1,0 mock=1,1,1,0 rabit_bootstrap_cache=true rabit_debug=true rabit_reduce_ring_mincount=1 rabit_timeout=true rabit_timeout_sec=5
+
+model_recover_10_10k_die_same:
+	python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 model_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 rabit_bootstrap_cache=1
+
+model_recover_10_10k_die_hard:
+	python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 model_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=1,1,1,1 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=8,1,2,0 mock=4,1,3,0 rabit_bootstrap_cache=1
+
+local_recover_10_10k:
+	python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 local_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=1,1,1,1
+
+pylocal_recover_10_10k:
+	python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 local_recover.py 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=1,1,1,1
+
+lazy_recover_10_10k_die_hard:
+	python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 lazy_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=1,1,1,1 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=8,1,2,0 mock=4,1,3,0
+
+lazy_recover_10_10k_die_same:
+	python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 lazy_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0
+
+ringallreduce_10_10k:
+	python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 model_recover 10000 rabit_reduce_ring_mincount=10