diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f6c5eaa..c9c68c362 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,9 +23,11 @@ write_version()
 set_default_configuration_release()
 
 #-- Options
+## User options
 option(BUILD_C_DOC "Build documentation for C APIs using Doxygen." OFF)
 option(USE_OPENMP "Build with OpenMP support." ON)
 option(BUILD_STATIC_LIB "Build static library" OFF)
+option(RABIT_BUILD_MPI "Build MPI" OFF)
 ## Bindings
 option(JVM_BINDINGS "Build JVM bindings" OFF)
 option(R_LIB "Build shared library for R package" OFF)
@@ -61,7 +63,7 @@ address, leak, undefined and thread.")
 option(PLUGIN_LZ4 "Build lz4 plugin" OFF)
 option(PLUGIN_DENSE_PARSER "Build dense parser plugin" OFF)
 option(PLUGIN_RMM "Build with RAPIDS Memory Manager (RMM)" OFF)
-## TODO: 1. Add check if DPC++ compiler is used for building 
+## TODO: 1. Add check if DPC++ compiler is used for building
 option(PLUGIN_UPDATER_ONEAPI "DPC++ updater" OFF)
 option(ADD_PKGCONFIG "Add xgboost.pc into system." ON)
 
@@ -160,9 +162,6 @@ endif (ENABLE_ALL_WARNINGS)
 target_link_libraries(objxgboost PUBLIC dmlc)
 
 # rabit
-set(RABIT_BUILD_DMLC OFF)
-set(DMLC_ROOT ${xgboost_SOURCE_DIR}/dmlc-core)
-set(RABIT_WITH_R_LIB ${R_LIB})
 add_subdirectory(rabit)
 
 if (RABIT_MOCK)
@@ -285,7 +284,7 @@ install(DIRECTORY ${xgboost_SOURCE_DIR}/include/xgboost
 #
 # https://github.com/dmlc/xgboost/issues/6085
 if (BUILD_STATIC_LIB)
-  set(INSTALL_TARGETS xgboost runxgboost objxgboost)
+  set(INSTALL_TARGETS xgboost runxgboost objxgboost rabit)
 else (BUILD_STATIC_LIB)
   set(INSTALL_TARGETS xgboost runxgboost)
 endif (BUILD_STATIC_LIB)
diff --git a/Jenkinsfile b/Jenkinsfile
index 54c8b9565..a42c0f03b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -152,14 +152,14 @@ def BuildCPU() {
       # We want to make sure that we use the configured header build/dmlc/build_config.h instead of include/dmlc/build_config_default.h.
       # See discussion at https://github.com/dmlc/xgboost/issues/5510
     ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh -DPLUGIN_LZ4=ON -DPLUGIN_DENSE_PARSER=ON
-    ${dockerRun} ${container_type} ${docker_binary} build/testxgboost
+    ${dockerRun} ${container_type} ${docker_binary} ctest
     """
     // Sanitizer test
     def docker_extra_params = "CI_DOCKER_EXTRA_PARAMS_INIT='-e ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer -e ASAN_OPTIONS=symbolize=1 -e UBSAN_OPTIONS=print_stacktrace=1:log_path=ubsan_error.log --cap-add SYS_PTRACE'"
     sh """
     ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh -DUSE_SANITIZER=ON -DENABLED_SANITIZERS="address;leak;undefined" \
       -DCMAKE_BUILD_TYPE=Debug -DSANITIZER_PATH=/usr/lib/x86_64-linux-gnu/
-    ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} build/testxgboost
+    ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} ctest
     """
 
     stash name: 'xgboost_cli', includes: 'xgboost'
@@ -193,7 +193,7 @@ def BuildCPUNonOmp() {
     """
     echo "Running Non-OpenMP C++ test..."
     sh """
-    ${dockerRun} ${container_type} ${docker_binary} build/testxgboost
+    ${dockerRun} ${container_type} ${docker_binary} ctest
     """
     deleteDir()
   }
diff --git a/rabit/CMakeLists.txt b/rabit/CMakeLists.txt
index 541b990ed..4cc135c48 100644
--- a/rabit/CMakeLists.txt
+++ b/rabit/CMakeLists.txt
@@ -1,47 +1,28 @@
 cmake_minimum_required(VERSION 3.3)
 
-project(rabit VERSION 0.3.0 LANGUAGES CXX)
-
-if ((${CMAKE_VERSION} VERSION_GREATER 3.13) OR (${CMAKE_VERSION} VERSION_EQUAL 3.13))
-  # This allows user to specify `RABIT_BUILD_DMLC` and others as CMake variable.
-  cmake_policy(SET CMP0077 NEW)
-endif ((${CMAKE_VERSION} VERSION_GREATER 3.13) OR (${CMAKE_VERSION} VERSION_EQUAL 3.13))
-
-option(RABIT_BUILD_TESTS "Build rabit tests" OFF)
-option(RABIT_BUILD_MPI "Build MPI" OFF)
-option(RABIT_BUILD_DMLC "Include DMLC_CORE in build" OFF)
-option(RABIT_WITH_R_LIB "Fit the strict environment of R" OFF)
-
-option(DMLC_ROOT "Specify root of external dmlc core.")
-# by default point to xgboost/dmlc-core
-set(DMLC_ROOT ${CMAKE_CURRENT_LIST_DIR}/../dmlc-core)
-
-# moved from xgboost build
 if(R_LIB OR MINGW OR WIN32)
   add_library(rabit src/engine_empty.cc src/c_api.cc)
   set(rabit_libs rabit)
   set_target_properties(rabit
-          PROPERTIES CXX_STANDARD 11
-          CXX_STANDARD_REQUIRED ON
-          POSITION_INDEPENDENT_CODE ON)
+    PROPERTIES CXX_STANDARD 14
+    CXX_STANDARD_REQUIRED ON
+    POSITION_INDEPENDENT_CODE ON)
 else()
   find_package(Threads REQUIRED)
-  add_library(rabit_empty src/engine_empty.cc src/c_api.cc)
-  add_library(rabit_base src/allreduce_base.cc src/engine_base.cc src/c_api.cc)
 
   add_library(rabit src/allreduce_base.cc src/allreduce_robust.cc src/engine.cc src/c_api.cc)
   add_library(rabit_mock_static src/allreduce_base.cc src/allreduce_robust.cc src/engine_mock.cc src/c_api.cc)
   add_library(rabit_mock SHARED src/allreduce_base.cc src/allreduce_robust.cc src/engine_mock.cc src/c_api.cc)
-  target_link_libraries(rabit Threads::Threads)
-  target_link_libraries(rabit_mock_static Threads::Threads)
-  target_link_libraries(rabit_mock Threads::Threads)
+  target_link_libraries(rabit Threads::Threads dmlc)
+  target_link_libraries(rabit_mock_static Threads::Threads dmlc)
+  target_link_libraries(rabit_mock Threads::Threads dmlc)
 
-  set(rabit_libs rabit rabit_base rabit_empty rabit_mock rabit_mock_static)
-  set_target_properties(rabit rabit_base rabit_empty rabit_mock rabit_mock_static
-    PROPERTIES CXX_STANDARD 11
+  set(rabit_libs rabit rabit_mock rabit_mock_static)
+  set_target_properties(rabit rabit_mock rabit_mock_static
+    PROPERTIES CXX_STANDARD 14
     CXX_STANDARD_REQUIRED ON
     POSITION_INDEPENDENT_CODE ON)
-ENDIF(R_LIB OR MINGW OR WIN32)
+endif(R_LIB OR MINGW OR WIN32)
 
 if(RABIT_BUILD_MPI)
   find_package(MPI REQUIRED)
@@ -55,9 +36,6 @@ endif()
 
 # place binaries and libraries according to GNU standards
 include(GNUInstallDirs)
-set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR})
-set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR})
-set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR})
 
 # we use this to get code coverage
 if ((CMAKE_CONFIGURATION_TYPES STREQUAL "Debug") AND (CMAKE_CXX_COMPILER_ID MATCHES GNU))
@@ -68,23 +46,14 @@ if ((CMAKE_CONFIGURATION_TYPES STREQUAL "Debug") AND (CMAKE_CXX_COMPILER_ID MATC
     endforeach()
 endif((CMAKE_CONFIGURATION_TYPES STREQUAL "Debug") AND (CMAKE_CXX_COMPILER_ID MATCHES GNU))
 
-if(RABIT_BUILD_DMLC)
-  set(DMLC_ROOT ${CMAKE_CURRENT_LIST_DIR}/dmlc-core)
-endif()
-
-if(DMLC_ROOT)
-  message("DMLC_ROOT point to " ${DMLC_ROOT})
-endif(DMLC_ROOT)
-
 foreach(lib ${rabit_libs})
   target_include_directories(${lib} PUBLIC
-          "$<BUILD_INTERFACE:${rabit_SOURCE_DIR}/include>"
-          "$<BUILD_INTERFACE:${DMLC_ROOT}/include>")
+          "$<BUILD_INTERFACE:${xgboost_SOURCE_DIR}/rabit/include>"
+          "$<BUILD_INTERFACE:${xgboost_SOURCE_DIR}/dmlc-core/include>")
 endforeach()
 
-if (RABIT_BUILD_TESTS)
+if (GOOGLE_TEST AND (NOT WIN32))
   enable_testing()
-  add_subdirectory(${rabit_SOURCE_DIR}/test/cpp)
 
   # rabit mock based integration tests
   list(REMOVE_ITEM rabit_libs "rabit_mock_static") # remove here to avoid installing it
@@ -93,90 +62,21 @@ if (RABIT_BUILD_TESTS)
   foreach(test ${tests})
     add_executable(${test} test/${test}.cc)
     target_link_libraries(${test} rabit_mock_static)
-    set_target_properties(${test}  PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON)
-    install(TARGETS ${test} DESTINATION test) # Why are we installing these??
+    set_target_properties(${test}  PROPERTIES CXX_STANDARD 14 CXX_STANDARD_REQUIRED ON)
+    add_test(NAME ${test} COMMAND ${test} WORKING_DIRECTORY ${xgboost_BINARY_DIR})
   endforeach()
 
   if(RABIT_BUILD_MPI)
     add_executable(speed_test_mpi test/speed_test.cc)
     target_link_libraries(speed_test_mpi rabit_mpi)
-    install(TARGETS speed_test_mpi DESTINATION test)
-  endif()
-endif (RABIT_BUILD_TESTS)
-
-# Installation (https://github.com/forexample/package-example) {
-
-# Layout. This works for all platforms:
-#   * <prefix>/lib/cmake/<PROJECT-NAME>
-#   * <prefix>/lib/
-#   * <prefix>/include/
-set(CMAKE_INSTALL_PREFIX "${rabit_SOURCE_DIR}")
-set(config_install_dir "lib/cmake/${PROJECT_NAME}")
-set(include_install_dir "include")
-
-set(generated_dir "${CMAKE_CURRENT_BINARY_DIR}/generated")
-
-# Configuration
-set(version_config "${generated_dir}/${PROJECT_NAME}ConfigVersion.cmake")
-set(project_config "${generated_dir}/${PROJECT_NAME}Config.cmake")
-set(TARGETS_EXPORT_NAME "${PROJECT_NAME}Targets")
-set(namespace "${PROJECT_NAME}::")
-
-# Include module with fuction 'write_basic_package_version_file'
-include(CMakePackageConfigHelpers)
-
-# Configure '<PROJECT-NAME>ConfigVersion.cmake'
-# Use:
-#   * PROJECT_VERSION
-write_basic_package_version_file(
-    "${version_config}" COMPATIBILITY SameMajorVersion
-)
-
-# Configure '<PROJECT-NAME>Config.cmake'
-# Use variables:
-#   * TARGETS_EXPORT_NAME
-#   * PROJECT_NAME
-configure_package_config_file(
-    "cmake/Config.cmake.in"
-    "${project_config}"
-    INSTALL_DESTINATION "${config_install_dir}"
-)
-
-# Targets:
-#   * <prefix>/lib/librabit.a
-#   * <prefix>/lib/librabit_base
-#   * <prefix>/lib/librabit_empty
-#   * header location after install: <prefix>/include/rabit/rabit.h
-#   * headers can be included by C++ code `#include <rabit/rabit.h>`
-install(
-    TARGETS ${rabit_libs}
-    EXPORT "${TARGETS_EXPORT_NAME}"
-    LIBRARY DESTINATION "lib"
-    ARCHIVE DESTINATION "lib"
-    RUNTIME DESTINATION "bin"
-    INCLUDES DESTINATION "${include_install_dir}"
-)
+    add_test(NAME speed_test_mpi COMMAND speed_test_mpi WORKING_DIRECTORY ${xgboost_BINARY_DIR})
+  endif(RABIT_BUILD_MPI)
+endif (GOOGLE_TEST AND (NOT WIN32))
 
 # Headers:
+set(include_install_dir "include")
 install(
-    DIRECTORY "include/"
-    DESTINATION "${include_install_dir}"
-    FILES_MATCHING PATTERN "*.h"
-)
-
-# Config
-#   * <prefix>/lib/cmake/rabit/rabitConfig.cmake
-#   * <prefix>/lib/cmake/rabit/rabitConfigVersion.cmake
-install(
-    FILES "${project_config}" "${version_config}"
-    DESTINATION "${config_install_dir}"
-)
-
-# Config
-#   * <prefix>/lib/cmake/Foo/FooTargets.cmake
-install(
-    EXPORT "${TARGETS_EXPORT_NAME}"
-    NAMESPACE "${namespace}"
-    DESTINATION "${config_install_dir}"
-)
-# }
+  DIRECTORY "include/"
+  DESTINATION "${include_install_dir}"
+  FILES_MATCHING PATTERN "*.h"
+  )
diff --git a/rabit/test/cpp/CMakeLists.txt b/rabit/test/cpp/CMakeLists.txt
deleted file mode 100644
index 9216d080c..000000000
--- a/rabit/test/cpp/CMakeLists.txt
+++ /dev/null
@@ -1,32 +0,0 @@
-find_package(GTest REQUIRED)
-
-add_executable(
-  unit_tests
-  test_io.cc
-  test_utils.cc
-  allreduce_robust_test.cc
-  allreduce_base_test.cc
-  allreduce_mock_test.cc
-  test_main.cpp)
-
-target_link_libraries(
-  unit_tests PRIVATE
-  GTest::GTest GTest::Main
-  rabit_base rabit_mock rabit)
-
-target_include_directories(unit_tests PUBLIC
-  "$<BUILD_INTERFACE:${rabit_SOURCE_DIR}/include>"
-  "$<BUILD_INTERFACE:${DMLC_ROOT}/include>")
-
-set_target_properties(unit_tests
-  PROPERTIES
-  CXX_STANDARD 11
-  CXX_STANDARD_REQUIRED ON
-  RUNTIME_OUTPUT_DIRECTORY ${rabit_BINARY_DIR}
-  RUNTIME_OUTPUT_DIRECTORY_DEBUG ${rabit_BINARY_DIR}
-  RUNTIME_OUTPUT_DIRECTORY_RELEASE ${rabit_BINARY_DIR})
-
-add_test(
-  NAME TestRabitLib
-  COMMAND unit_tests
-  WORKING_DIRECTORY ${rabit_BINARY_DIR})
diff --git a/rabit/test/cpp/README.md b/rabit/test/cpp/README.md
deleted file mode 100644
index 9962980c2..000000000
--- a/rabit/test/cpp/README.md
+++ /dev/null
@@ -1 +0,0 @@
-Unittests for Rabit
diff --git a/rabit/test/cpp/allreduce_base_test.cpp b/rabit/test/cpp/allreduce_base_test.cpp
deleted file mode 100644
index 65a3dd50b..000000000
--- a/rabit/test/cpp/allreduce_base_test.cpp
+++ /dev/null
@@ -1,66 +0,0 @@
-#define RABIT_CXXTESTDEFS_H
-#include <gtest/gtest.h>
-
-#include <string>
-#include <iostream>
-#include "../../src/allreduce_base.h"
-
-TEST(allreduce_base, init_task)
-{
-  rabit::engine::AllreduceBase base;
-
-  std::string rabit_task_id = "rabit_task_id=1";
-  char cmd[rabit_task_id.size()+1];
-  std::copy(rabit_task_id.begin(), rabit_task_id.end(), cmd);
-  cmd[rabit_task_id.size()] = '\0';
-
-  char* argv[] = {cmd};
-  base.Init(1, argv);
-  EXPECT_EQ(base.task_id, "1");
-}
-
-TEST(allreduce_base, init_with_cache_on)
-{
-  rabit::engine::AllreduceBase base;
-
-  std::string rabit_task_id = "rabit_task_id=1";
-  char cmd[rabit_task_id.size()+1];
-  std::copy(rabit_task_id.begin(), rabit_task_id.end(), cmd);
-  cmd[rabit_task_id.size()] = '\0';
-
-  std::string rabit_bootstrap_cache = "rabit_bootstrap_cache=1";
-  char cmd2[rabit_bootstrap_cache.size()+1];
-  std::copy(rabit_bootstrap_cache.begin(), rabit_bootstrap_cache.end(), cmd2);
-  cmd2[rabit_bootstrap_cache.size()] = '\0';
-
-  std::string rabit_debug = "rabit_debug=1";
-  char cmd3[rabit_debug.size()+1];
-  std::copy(rabit_debug.begin(), rabit_debug.end(), cmd3);
-  cmd3[rabit_debug.size()] = '\0';
-
-  char* argv[] = {cmd, cmd2, cmd3};
-  base.Init(3, argv);
-  EXPECT_EQ(base.task_id, "1");
-  EXPECT_EQ(base.rabit_bootstrap_cache, 1);
-  EXPECT_EQ(base.rabit_debug, 1);
-}
-
-TEST(allreduce_base, init_with_ring_reduce)
-{
-  rabit::engine::AllreduceBase base;
-
-  std::string rabit_task_id = "rabit_task_id=1";
-  char cmd[rabit_task_id.size()+1];
-  std::copy(rabit_task_id.begin(), rabit_task_id.end(), cmd);
-  cmd[rabit_task_id.size()] = '\0';
-
-  std::string rabit_reduce_ring_mincount = "rabit_reduce_ring_mincount=1";
-  char cmd2[rabit_reduce_ring_mincount.size()+1];
-  std::copy(rabit_reduce_ring_mincount.begin(), rabit_reduce_ring_mincount.end(), cmd2);
-  cmd2[rabit_reduce_ring_mincount.size()] = '\0';
-
-  char* argv[] = {cmd, cmd2};
-  base.Init(2, argv);
-  EXPECT_EQ(base.task_id, "1");
-  EXPECT_EQ(base.reduce_ring_mincount, 1);
-}
diff --git a/rabit/test/cpp/allreduce_mock_test.cc b/rabit/test/cpp/allreduce_mock_test.cc
deleted file mode 100644
index 5d03dc71a..000000000
--- a/rabit/test/cpp/allreduce_mock_test.cc
+++ /dev/null
@@ -1,36 +0,0 @@
-#define RABIT_CXXTESTDEFS_H
-#include <gtest/gtest.h>
-
-#include <string>
-#include <iostream>
-#include "../../src/allreduce_mock.h"
-
-TEST(allreduce_mock, mock_allreduce)
-{
-  rabit::engine::AllreduceMock m;
-
-  std::string mock_str = "mock=0,0,0,0";
-  char cmd[mock_str.size()+1];
-  std::copy(mock_str.begin(), mock_str.end(), cmd);
-  cmd[mock_str.size()] = '\0';
-
-  char* argv[] = {cmd};
-  m.Init(1, argv);
-  m.rank = 0;
-  EXPECT_THROW(m.Allreduce(nullptr,0,0,nullptr,nullptr,nullptr), dmlc::Error);
-}
-
-TEST(allreduce_mock, mock_broadcast)
-{
-  rabit::engine::AllreduceMock m;
-  std::string mock_str = "mock=0,1,2,0";
-  char cmd[mock_str.size()+1];
-  std::copy(mock_str.begin(), mock_str.end(), cmd);
-  cmd[mock_str.size()] = '\0';
-  char* argv[] = {cmd};
-  m.Init(1, argv);
-  m.rank = 0;
-  m.version_number=1;
-  m.seq_counter=2;
-  EXPECT_THROW(m.Broadcast(nullptr,0,0), dmlc::Error);
-}
diff --git a/rabit/test/cpp/test_main.cpp b/rabit/test/cpp/test_main.cpp
deleted file mode 100644
index 6eb025ac9..000000000
--- a/rabit/test/cpp/test_main.cpp
+++ /dev/null
@@ -1,8 +0,0 @@
-#include "gtest/gtest.h"
-
-int main(int argc, char** argv)
-{
-    ::testing::InitGoogleTest(&argc, argv);
-    ::testing::FLAGS_gtest_death_test_style = "threadsafe";
-    return RUN_ALL_TESTS();
-}
diff --git a/rabit/test/cpp/allreduce_base_test.cc b/tests/cpp/rabit/allreduce_base_test.cc
similarity index 85%
rename from rabit/test/cpp/allreduce_base_test.cc
rename to tests/cpp/rabit/allreduce_base_test.cc
index 65a3dd50b..8983e9aa6 100644
--- a/rabit/test/cpp/allreduce_base_test.cc
+++ b/tests/cpp/rabit/allreduce_base_test.cc
@@ -1,11 +1,12 @@
 #define RABIT_CXXTESTDEFS_H
+#if !defined(_WIN32)
 #include <gtest/gtest.h>
 
 #include <string>
 #include <iostream>
-#include "../../src/allreduce_base.h"
+#include "../../../rabit/src/allreduce_base.h"
 
-TEST(allreduce_base, init_task)
+TEST(AllreduceBase, InitTask)
 {
   rabit::engine::AllreduceBase base;
 
@@ -19,7 +20,7 @@ TEST(allreduce_base, init_task)
   EXPECT_EQ(base.task_id, "1");
 }
 
-TEST(allreduce_base, init_with_cache_on)
+TEST(AllreduceBase, InitWithCacheOn)
 {
   rabit::engine::AllreduceBase base;
 
@@ -41,11 +42,11 @@ TEST(allreduce_base, init_with_cache_on)
   char* argv[] = {cmd, cmd2, cmd3};
   base.Init(3, argv);
   EXPECT_EQ(base.task_id, "1");
-  EXPECT_EQ(base.rabit_bootstrap_cache, 1);
+  EXPECT_TRUE(base.rabit_bootstrap_cache);
   EXPECT_EQ(base.rabit_debug, 1);
 }
 
-TEST(allreduce_base, init_with_ring_reduce)
+TEST(AllreduceBase, InitWithRingReduce)
 {
   rabit::engine::AllreduceBase base;
 
@@ -62,5 +63,6 @@ TEST(allreduce_base, init_with_ring_reduce)
   char* argv[] = {cmd, cmd2};
   base.Init(2, argv);
   EXPECT_EQ(base.task_id, "1");
-  EXPECT_EQ(base.reduce_ring_mincount, 1);
+  EXPECT_EQ(base.reduce_ring_mincount, 1ul);
 }
+#endif  // !defined(_WIN32)
diff --git a/rabit/test/cpp/allreduce_mock_test.cpp b/tests/cpp/rabit/allreduce_mock_test.cc
similarity index 74%
rename from rabit/test/cpp/allreduce_mock_test.cpp
rename to tests/cpp/rabit/allreduce_mock_test.cc
index 5d7a1b16f..8d6f519c5 100644
--- a/rabit/test/cpp/allreduce_mock_test.cpp
+++ b/tests/cpp/rabit/allreduce_mock_test.cc
@@ -1,12 +1,12 @@
 #define RABIT_CXXTESTDEFS_H
+#if !defined(_WIN32)
 #include <gtest/gtest.h>
 
 #include <string>
 #include <iostream>
-#include <dmlc/logging.h>
-#include "../../src/allreduce_mock.h"
+#include "../../../rabit/src/allreduce_mock.h"
 
-TEST(allreduce_mock, mock_allreduce)
+TEST(AllreduceMock, MockAllreduce)
 {
   rabit::engine::AllreduceMock m;
 
@@ -18,10 +18,10 @@ TEST(allreduce_mock, mock_allreduce)
   char* argv[] = {cmd};
   m.Init(1, argv);
   m.rank = 0;
-  EXPECT_THROW({m.Allreduce(nullptr,0,0,nullptr,nullptr,nullptr);}, dmlc::Error);
+  EXPECT_THROW(m.Allreduce(nullptr,0,0,nullptr,nullptr,nullptr), dmlc::Error);
 }
 
-TEST(allreduce_mock, mock_broadcast)
+TEST(AllreduceMock, MockBroadcast)
 {
   rabit::engine::AllreduceMock m;
   std::string mock_str = "mock=0,1,2,0";
@@ -33,10 +33,10 @@ TEST(allreduce_mock, mock_broadcast)
   m.rank = 0;
   m.version_number=1;
   m.seq_counter=2;
-  EXPECT_THROW({m.Broadcast(nullptr,0,0);}, dmlc::Error);
+  EXPECT_THROW(m.Broadcast(nullptr,0,0), dmlc::Error);
 }
 
-TEST(allreduce_mock, mock_gather)
+TEST(AllreduceMock, MockGather)
 {
   rabit::engine::AllreduceMock m;
   std::string mock_str = "mock=3,13,22,0";
@@ -50,3 +50,4 @@ TEST(allreduce_mock, mock_gather)
   m.seq_counter=22;
   EXPECT_THROW({m.Allgather(nullptr,0,0,0,0);}, dmlc::Error);
 }
+#endif  // !defined(_WIN32)
diff --git a/rabit/test/cpp/allreduce_robust_test.cc b/tests/cpp/rabit/allreduce_robust_test.cc
similarity index 85%
rename from rabit/test/cpp/allreduce_robust_test.cc
rename to tests/cpp/rabit/allreduce_robust_test.cc
index c86f76271..02e19c4b6 100644
--- a/rabit/test/cpp/allreduce_robust_test.cc
+++ b/tests/cpp/rabit/allreduce_robust_test.cc
@@ -1,17 +1,18 @@
 #define RABIT_CXXTESTDEFS_H
+#if !defined(_WIN32)
 #include <gtest/gtest.h>
 
 #include <chrono>
 #include <string>
 #include <iostream>
-#include "../../src/allreduce_robust.h"
+#include "../../../rabit/src/allreduce_robust.h"
 
-inline void mockerr(const char *fmt, ...) {EXPECT_STRCASEEQ(fmt, "[%d] exit due to time out %d s\n");}
-inline void mockassert(bool val, const char *fmt, ...) {}
+inline void MockErr(const char *fmt, ...) {EXPECT_STRCASEEQ(fmt, "[%d] exit due to time out %d s\n");}
+inline void MockAssert(bool val, const char *fmt, ...) {}
 rabit::engine::AllreduceRobust::ReturnType err_type(rabit::engine::AllreduceRobust::ReturnTypeEnum::kSockError);
 rabit::engine::AllreduceRobust::ReturnType succ_type(rabit::engine::AllreduceRobust::ReturnTypeEnum::kSuccess);
 
-TEST(allreduce_robust, sync_error_timeout)
+TEST(AllreduceRobust, SyncErrorTimeout)
 {
   rabit::engine::AllreduceRobust m;
 
@@ -28,15 +29,15 @@ TEST(allreduce_robust, sync_error_timeout)
   char* argv[] = {cmd,cmd1};
   m.Init(2, argv);
   m.rank = 0;
-  m.rabit_bootstrap_cache = 1;
-  m._error = mockerr;
-  m._assert = mockassert;
+  m.rabit_bootstrap_cache = true;
+  m.error_ = MockErr;
+  m.assert_ = MockAssert;
   EXPECT_EQ(m.CheckAndRecover(err_type), false);
   std::this_thread::sleep_for(std::chrono::milliseconds(1500));
-  EXPECT_EQ(m.rabit_timeout_task.get(), false);
+  EXPECT_EQ(m.rabit_timeout_task_.get(), false);
 }
 
-TEST(allreduce_robust, sync_error_reset)
+TEST(AllreduceRobust, SyncErrorReset)
 {
   rabit::engine::AllreduceRobust m;
 
@@ -58,15 +59,15 @@ TEST(allreduce_robust, sync_error_reset)
   char* argv[] = {cmd, cmd1,cmd2};
   m.Init(3, argv);
   m.rank = 0;
-  m._assert = mockassert;
+  m.assert_ = MockAssert;
   EXPECT_EQ(m.CheckAndRecover(err_type), false);
   std::this_thread::sleep_for(std::chrono::milliseconds(100));
   EXPECT_EQ(m.CheckAndRecover(succ_type), true);
-  EXPECT_EQ(m.rabit_timeout_task.get(), true);
+  EXPECT_EQ(m.rabit_timeout_task_.get(), true);
   m.Shutdown();
 }
 
-TEST(allreduce_robust, sync_success_error_timeout)
+TEST(AllreduceRobust, SyncSuccessErrorTimeout)
 {
   rabit::engine::AllreduceRobust m;
 
@@ -88,17 +89,17 @@ TEST(allreduce_robust, sync_success_error_timeout)
   char* argv[] = {cmd, cmd1,cmd2};
   m.Init(3, argv);
   m.rank = 0;
-  m.rabit_bootstrap_cache = 1;
-  m._assert = mockassert;
-  m._error = mockerr;
+  m.rabit_bootstrap_cache = true;
+  m.assert_ = MockAssert;
+  m.error_ = MockErr;
   EXPECT_EQ(m.CheckAndRecover(succ_type), true);
   std::this_thread::sleep_for(std::chrono::milliseconds(100));
   EXPECT_EQ(m.CheckAndRecover(err_type), false);
   std::this_thread::sleep_for(std::chrono::milliseconds(1500));
-  EXPECT_EQ(m.rabit_timeout_task.get(), false);
+  EXPECT_EQ(m.rabit_timeout_task_.get(), false);
 }
 
-TEST(allreduce_robust, sync_success_error_success)
+TEST(AllreduceRobust, SyncSuccessErrorSuccess)
 {
   rabit::engine::AllreduceRobust m;
 
@@ -120,8 +121,8 @@ TEST(allreduce_robust, sync_success_error_success)
   char* argv[] = {cmd, cmd1,cmd2};
   m.Init(3, argv);
   m.rank = 0;
-  m.rabit_bootstrap_cache = 1;
-  m._assert = mockassert;
+  m.rabit_bootstrap_cache = true;
+  m.assert_ = MockAssert;
   EXPECT_EQ(m.CheckAndRecover(succ_type), true);
   std::this_thread::sleep_for(std::chrono::milliseconds(10));
 
@@ -129,11 +130,11 @@ TEST(allreduce_robust, sync_success_error_success)
   std::this_thread::sleep_for(std::chrono::milliseconds(10));
   EXPECT_EQ(m.CheckAndRecover(succ_type), true);
   std::this_thread::sleep_for(std::chrono::milliseconds(1100));
-  EXPECT_EQ(m.rabit_timeout_task.get(), true);
+  EXPECT_EQ(m.rabit_timeout_task_.get(), true);
   m.Shutdown();
 }
 
-TEST(allreduce_robust, sync_error_no_reset_timeout)
+TEST(AllreduceRobust, SyncErrorNoResetTimeout)
 {
   rabit::engine::AllreduceRobust m;
 
@@ -155,9 +156,9 @@ TEST(allreduce_robust, sync_error_no_reset_timeout)
   char* argv[] = {cmd, cmd1,cmd2};
   m.Init(3, argv);
   m.rank = 0;
-  m.rabit_bootstrap_cache = 1;
-  m._assert = mockassert;
-  m._error = mockerr;
+  m.rabit_bootstrap_cache = true;
+  m.assert_ = MockAssert;
+  m.error_ = MockErr;
   auto start = std::chrono::system_clock::now();
 
   EXPECT_EQ(m.CheckAndRecover(err_type), false);
@@ -165,16 +166,16 @@ TEST(allreduce_robust, sync_error_no_reset_timeout)
 
   EXPECT_EQ(m.CheckAndRecover(err_type), false);
 
-  m.rabit_timeout_task.wait();
+  m.rabit_timeout_task_.wait();
   auto end = std::chrono::system_clock::now();
   std::chrono::duration<double> diff = end-start;
 
-  EXPECT_EQ(m.rabit_timeout_task.get(), false);
+  EXPECT_EQ(m.rabit_timeout_task_.get(), false);
   // expect second error don't overwrite/reset timeout task
   EXPECT_LT(diff.count(), 2);
 }
 
-TEST(allreduce_robust, no_timeout_shut_down)
+TEST(AllreduceRobust, NoTimeoutShutDown)
 {
   rabit::engine::AllreduceRobust m;
 
@@ -202,7 +203,7 @@ TEST(allreduce_robust, no_timeout_shut_down)
   m.Shutdown();
 }
 
-TEST(allreduce_robust, shut_down_before_timeout)
+TEST(AllreduceRobust, ShutDownBeforeTimeout)
 {
   rabit::engine::AllreduceRobust m;
 
@@ -226,8 +227,9 @@ TEST(allreduce_robust, shut_down_before_timeout)
   m.rank = 0;
   rabit::engine::AllreduceRobust::LinkRecord a;
   m.err_link = &a;
-  
+
   EXPECT_EQ(m.CheckAndRecover(err_type), false);
   std::this_thread::sleep_for(std::chrono::milliseconds(10));
   m.Shutdown();
-}
\ No newline at end of file
+}
+#endif  // !defined(_WIN32)
diff --git a/rabit/test/cpp/test_io.cc b/tests/cpp/rabit/test_io.cc
similarity index 88%
rename from rabit/test/cpp/test_io.cc
rename to tests/cpp/rabit/test_io.cc
index 0e4b70b1b..f76ee7dda 100644
--- a/rabit/test/cpp/test_io.cc
+++ b/tests/cpp/rabit/test_io.cc
@@ -11,7 +11,7 @@ TEST(MemoryFixSizeBuffer, Seek) {
   size_t constexpr kSize { 64 };
   std::vector<int32_t> memory( kSize );
   utils::MemoryFixSizeBuffer buf(memory.data(), memory.size());
-  buf.Seek(utils::MemoryFixSizeBuffer::SeekEnd);
+  buf.Seek(utils::MemoryFixSizeBuffer::kSeekEnd);
   size_t end = buf.Tell();
   ASSERT_EQ(end, kSize);
 }
diff --git a/rabit/test/cpp/test_utils.cc b/tests/cpp/rabit/test_utils.cc
similarity index 100%
rename from rabit/test/cpp/test_utils.cc
rename to tests/cpp/rabit/test_utils.cc