Clean up cmake script and code includes (#106)
* Clean up CMake scripts and related include paths. * Add unittests.
This commit is contained in:
parent
e92641887b
commit
ddcc2d85da
12
.travis.yml
12
.travis.yml
@ -38,6 +38,8 @@ addons:
|
|||||||
- openssh-server
|
- openssh-server
|
||||||
- python3
|
- python3
|
||||||
- python3-setuptools
|
- python3-setuptools
|
||||||
|
- python3-pip
|
||||||
|
- tree
|
||||||
homebrew:
|
homebrew:
|
||||||
packages:
|
packages:
|
||||||
- gcc49
|
- gcc49
|
||||||
@ -45,18 +47,13 @@ addons:
|
|||||||
- libgit2
|
- libgit2
|
||||||
- python3
|
- python3
|
||||||
update: true
|
update: true
|
||||||
|
|
||||||
before_install:
|
before_install:
|
||||||
- git clone https://github.com/dmlc/dmlc-core
|
- git clone https://github.com/dmlc/dmlc-core
|
||||||
- export TRAVIS=dmlc-core/scripts/travis/
|
- export TRAVIS=dmlc-core/scripts/travis/
|
||||||
- source ${TRAVIS}/travis_setup_env.sh
|
- source ${TRAVIS}/travis_setup_env.sh
|
||||||
- ${TRAVIS}/travis_osx_install.sh
|
- ${TRAVIS}/travis_osx_install.sh
|
||||||
|
- source ./scripts/travis_setup.sh
|
||||||
install:
|
|
||||||
- if [[ ${TRAVIS_OS_NAME} == "linux" ]]; then sudo apt-get install python3-pip; fi
|
|
||||||
- if [[ ${TRAVIS_OS_NAME} == "osx" ]]; then brew install python3; fi
|
|
||||||
- pip3 install cpplint pylint urllib3 numpy
|
|
||||||
- pip3 install websocket-client kubernetes
|
|
||||||
|
|
||||||
script: scripts/travis_script.sh
|
script: scripts/travis_script.sh
|
||||||
|
|
||||||
@ -78,4 +75,3 @@ notifications:
|
|||||||
email:
|
email:
|
||||||
on_success: change
|
on_success: change
|
||||||
on_failure: always
|
on_failure: always
|
||||||
|
|
||||||
|
|||||||
@ -1,25 +1,25 @@
|
|||||||
cmake_minimum_required(VERSION 3.3)
|
cmake_minimum_required(VERSION 3.3)
|
||||||
|
|
||||||
project(rabit VERSION 0.3.0)
|
project(rabit VERSION 0.3.0 LANGUAGES CXX)
|
||||||
|
|
||||||
include(CheckCXXCompilerFlag)
|
|
||||||
CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11)
|
|
||||||
if(COMPILER_SUPPORTS_CXX11)
|
|
||||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
|
|
||||||
else()
|
|
||||||
message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.")
|
|
||||||
endif()
|
|
||||||
|
|
||||||
option(RABIT_BUILD_TESTS "Build rabit tests" OFF)
|
option(RABIT_BUILD_TESTS "Build rabit tests" OFF)
|
||||||
option(RABIT_BUILD_MPI "Build MPI" OFF)
|
option(RABIT_BUILD_MPI "Build MPI" OFF)
|
||||||
option(RABIT_BUILD_DMLC "Include DMLC_CORE in build" OFF)
|
option(RABIT_BUILD_DMLC "Include DMLC_CORE in build" OFF)
|
||||||
|
option(RABIT_WITH_R_LIB "Fit the strict environment of R" OFF)
|
||||||
|
|
||||||
|
option(DMLC_ROOT "Specify root of external dmlc core.")
|
||||||
|
# by default point to xgboost/dmlc-core
|
||||||
|
set(DMLC_ROOT ${CMAKE_CURRENT_LIST_DIR}/../dmlc-core)
|
||||||
|
|
||||||
# moved from xgboost build
|
# moved from xgboost build
|
||||||
if(R_LIB OR MINGW OR WIN32)
|
if(R_LIB OR MINGW OR WIN32)
|
||||||
add_library(rabit_base src/allreduce_base.cc src/engine_base.cc src/c_api.cc)
|
add_library(rabit src/engine_empty.cc src/c_api.cc)
|
||||||
set(rabit_libs rabit)
|
set(rabit_libs rabit)
|
||||||
set_target_properties(rabit PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON)
|
set_target_properties(rabit
|
||||||
ELSE()
|
PROPERTIES CXX_STANDARD 11
|
||||||
|
CXX_STANDARD_REQUIRED ON
|
||||||
|
POSITION_INDEPENDENT_CODE ON)
|
||||||
|
else()
|
||||||
add_library(rabit src/allreduce_base.cc src/allreduce_robust.cc src/engine.cc src/c_api.cc)
|
add_library(rabit src/allreduce_base.cc src/allreduce_robust.cc src/engine.cc src/c_api.cc)
|
||||||
add_library(rabit_base src/allreduce_base.cc src/engine_base.cc src/c_api.cc)
|
add_library(rabit_base src/allreduce_base.cc src/engine_base.cc src/c_api.cc)
|
||||||
add_library(rabit_empty src/engine_empty.cc src/c_api.cc)
|
add_library(rabit_empty src/engine_empty.cc src/c_api.cc)
|
||||||
@ -27,36 +27,56 @@ ELSE()
|
|||||||
add_library(rabit_mock SHARED src/allreduce_base.cc src/allreduce_robust.cc src/engine_mock.cc src/c_api.cc)
|
add_library(rabit_mock SHARED src/allreduce_base.cc src/allreduce_robust.cc src/engine_mock.cc src/c_api.cc)
|
||||||
|
|
||||||
set(rabit_libs rabit rabit_base rabit_empty rabit_mock rabit_mock_static)
|
set(rabit_libs rabit rabit_base rabit_empty rabit_mock rabit_mock_static)
|
||||||
set_target_properties(rabit rabit_base rabit_empty rabit_mock rabit_mock_static PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON)
|
set_target_properties(rabit rabit_base rabit_empty rabit_mock rabit_mock_static
|
||||||
|
PROPERTIES CXX_STANDARD 11
|
||||||
|
CXX_STANDARD_REQUIRED ON
|
||||||
|
POSITION_INDEPENDENT_CODE ON)
|
||||||
ENDIF(R_LIB OR MINGW OR WIN32)
|
ENDIF(R_LIB OR MINGW OR WIN32)
|
||||||
|
|
||||||
if(RABIT_BUILD_MPI)
|
if(RABIT_BUILD_MPI)
|
||||||
find_package(MPI REQUIRED)
|
find_package(MPI REQUIRED)
|
||||||
|
if (NOT MPI_CXX_FOUND)
|
||||||
|
message(FATAL_ERROR "CXX Interface for MPI is required for building MPI backend.")
|
||||||
|
endif (NOT MPI_CXX_FOUND)
|
||||||
add_library(rabit_mpi src/engine_mpi.cc ${MPI_INCLUDE_PATH})
|
add_library(rabit_mpi src/engine_mpi.cc ${MPI_INCLUDE_PATH})
|
||||||
target_link_libraries(rabit_mpi ${MPI_CXX_LIBRARIES})
|
target_link_libraries(rabit_mpi ${MPI_CXX_LIBRARIES})
|
||||||
list(APPEND rabit_libs rabit_mpi)
|
list(APPEND rabit_libs rabit_mpi)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
|
# place binaries and libraries according to GNU standards
|
||||||
|
include(GNUInstallDirs)
|
||||||
|
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR})
|
||||||
|
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR})
|
||||||
|
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR})
|
||||||
|
|
||||||
|
# we use this to get code coverage
|
||||||
|
if ((CMAKE_CONFIGURATION_TYPES STREQUAL "Debug") AND (CMAKE_CXX_COMPILER_ID MATCHES GNU))
|
||||||
|
foreach(lib ${rabit_libs})
|
||||||
|
target_compile_options(${lib}
|
||||||
|
-fprofile-arcs
|
||||||
|
-ftest-coverage)
|
||||||
|
endforeach()
|
||||||
|
endif((CMAKE_CONFIGURATION_TYPES STREQUAL "Debug") AND (CMAKE_CXX_COMPILER_ID MATCHES GNU))
|
||||||
|
|
||||||
if(RABIT_BUILD_DMLC)
|
if(RABIT_BUILD_DMLC)
|
||||||
foreach(lib ${rabit_libs})
|
set(DMLC_ROOT ${CMAKE_CURRENT_LIST_DIR}/dmlc-core)
|
||||||
#include "./internal/utils.h"
|
|
||||||
target_include_directories(${lib} PUBLIC
|
|
||||||
"$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include>"
|
|
||||||
"$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/dmlc-core/include>"
|
|
||||||
)
|
|
||||||
endforeach()
|
|
||||||
else()
|
|
||||||
foreach(lib ${rabit_libs})
|
|
||||||
#include "./internal/utils.h"
|
|
||||||
target_include_directories(${lib} PUBLIC
|
|
||||||
"$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include>"
|
|
||||||
"$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/../dmlc_core/include>"
|
|
||||||
)
|
|
||||||
endforeach()
|
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
if(RABIT_BUILD_TESTS)
|
if(DMLC_ROOT)
|
||||||
|
message("DMLC_ROOT point to " ${DMLC_ROOT})
|
||||||
|
endif(DMLC_ROOT)
|
||||||
|
|
||||||
|
foreach(lib ${rabit_libs})
|
||||||
|
target_include_directories(${lib} PUBLIC
|
||||||
|
"$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include>"
|
||||||
|
"$<BUILD_INTERFACE:${DMLC_ROOT}/include>")
|
||||||
|
endforeach()
|
||||||
|
|
||||||
|
if (RABIT_BUILD_TESTS)
|
||||||
|
enable_testing()
|
||||||
|
add_subdirectory(${rabit_SOURCE_DIR}/test/cpp)
|
||||||
|
|
||||||
|
# rabit mock based integration tests
|
||||||
list(REMOVE_ITEM rabit_libs "rabit_mock_static") # remove here to avoid installing it
|
list(REMOVE_ITEM rabit_libs "rabit_mock_static") # remove here to avoid installing it
|
||||||
set(tests lazy_recover local_recover model_recover)
|
set(tests lazy_recover local_recover model_recover)
|
||||||
|
|
||||||
@ -64,14 +84,15 @@ if(RABIT_BUILD_TESTS)
|
|||||||
add_executable(${test} test/${test}.cc)
|
add_executable(${test} test/${test}.cc)
|
||||||
target_link_libraries(${test} rabit_mock_static)
|
target_link_libraries(${test} rabit_mock_static)
|
||||||
set_target_properties(${test} PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON)
|
set_target_properties(${test} PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON)
|
||||||
install(TARGETS ${test} DESTINATION test)
|
install(TARGETS ${test} DESTINATION test) # Why are we installing these??
|
||||||
endforeach()
|
endforeach()
|
||||||
|
|
||||||
if(RABIT_BUILD_MPI)
|
if(RABIT_BUILD_MPI)
|
||||||
add_executable(speed_test_mpi test/speed_test.cc)
|
add_executable(speed_test_mpi test/speed_test.cc)
|
||||||
target_link_libraries(speed_test_mpi rabit_mpi)
|
target_link_libraries(speed_test_mpi rabit_mpi)
|
||||||
install(TARGETS speed_test_mpi DESTINATION test)
|
install(TARGETS speed_test_mpi DESTINATION test)
|
||||||
endif()
|
endif()
|
||||||
endif()
|
endif (RABIT_BUILD_TESTS)
|
||||||
|
|
||||||
# Installation (https://github.com/forexample/package-example) {
|
# Installation (https://github.com/forexample/package-example) {
|
||||||
|
|
||||||
@ -79,7 +100,7 @@ endif()
|
|||||||
# * <prefix>/lib/cmake/<PROJECT-NAME>
|
# * <prefix>/lib/cmake/<PROJECT-NAME>
|
||||||
# * <prefix>/lib/
|
# * <prefix>/lib/
|
||||||
# * <prefix>/include/
|
# * <prefix>/include/
|
||||||
set(CMAKE_INSTALL_PREFIX "../")
|
set(CMAKE_INSTALL_PREFIX "${rabit_SOURCE_DIR}")
|
||||||
set(config_install_dir "lib/cmake/${PROJECT_NAME}")
|
set(config_install_dir "lib/cmake/${PROJECT_NAME}")
|
||||||
set(include_install_dir "include")
|
set(include_install_dir "include")
|
||||||
|
|
||||||
|
|||||||
20
cmake/googletest-download.cmake
Normal file
20
cmake/googletest-download.cmake
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
# code copied from https://crascit.com/2015/07/25/cmake-gtest/
|
||||||
|
cmake_minimum_required(VERSION 3.5 FATAL_ERROR)
|
||||||
|
|
||||||
|
project(googletest-download NONE)
|
||||||
|
|
||||||
|
include(ExternalProject)
|
||||||
|
|
||||||
|
ExternalProject_Add(
|
||||||
|
googletest
|
||||||
|
SOURCE_DIR "@GOOGLETEST_DOWNLOAD_ROOT@/googletest-src"
|
||||||
|
BINARY_DIR "@GOOGLETEST_DOWNLOAD_ROOT@/googletest-build"
|
||||||
|
GIT_REPOSITORY
|
||||||
|
https://github.com/google/googletest.git
|
||||||
|
GIT_TAG
|
||||||
|
release-1.8.0
|
||||||
|
CONFIGURE_COMMAND ""
|
||||||
|
BUILD_COMMAND ""
|
||||||
|
INSTALL_COMMAND ""
|
||||||
|
TEST_COMMAND ""
|
||||||
|
)
|
||||||
32
cmake/googletest.cmake
Normal file
32
cmake/googletest.cmake
Normal file
@ -0,0 +1,32 @@
|
|||||||
|
# the following code to fetch googletest
|
||||||
|
# is inspired by and adapted after https://crascit.com/2015/07/25/cmake-gtest/
|
||||||
|
# download and unpack googletest at configure time
|
||||||
|
|
||||||
|
macro(fetch_googletest _download_module_path _download_root)
|
||||||
|
set(GOOGLETEST_DOWNLOAD_ROOT ${_download_root})
|
||||||
|
configure_file(
|
||||||
|
${_download_module_path}/googletest-download.cmake
|
||||||
|
${_download_root}/CMakeLists.txt
|
||||||
|
@ONLY
|
||||||
|
)
|
||||||
|
unset(GOOGLETEST_DOWNLOAD_ROOT)
|
||||||
|
|
||||||
|
execute_process(
|
||||||
|
COMMAND
|
||||||
|
"${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
|
||||||
|
WORKING_DIRECTORY
|
||||||
|
${_download_root}
|
||||||
|
)
|
||||||
|
execute_process(
|
||||||
|
COMMAND
|
||||||
|
"${CMAKE_COMMAND}" --build .
|
||||||
|
WORKING_DIRECTORY
|
||||||
|
${_download_root}
|
||||||
|
)
|
||||||
|
|
||||||
|
# adds the targers: gtest, gtest_main, gmock, gmock_main
|
||||||
|
add_subdirectory(
|
||||||
|
${_download_root}/googletest-src
|
||||||
|
${_download_root}/googletest-build
|
||||||
|
)
|
||||||
|
endmacro()
|
||||||
@ -7,7 +7,7 @@
|
|||||||
#ifndef RABIT_INTERNAL_ENGINE_H_
|
#ifndef RABIT_INTERNAL_ENGINE_H_
|
||||||
#define RABIT_INTERNAL_ENGINE_H_
|
#define RABIT_INTERNAL_ENGINE_H_
|
||||||
#include <string>
|
#include <string>
|
||||||
#include "../serializable.h"
|
#include "rabit/serializable.h"
|
||||||
|
|
||||||
#if (defined(__GNUC__) && !defined(__clang__))
|
#if (defined(__GNUC__) && !defined(__clang__))
|
||||||
#define _FILE __builtin_FILE()
|
#define _FILE __builtin_FILE()
|
||||||
|
|||||||
@ -11,8 +11,8 @@
|
|||||||
#include <cstring>
|
#include <cstring>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include "./utils.h"
|
#include "rabit/internal/utils.h"
|
||||||
#include "../serializable.h"
|
#include "rabit/serializable.h"
|
||||||
|
|
||||||
namespace rabit {
|
namespace rabit {
|
||||||
namespace utils {
|
namespace utils {
|
||||||
|
|||||||
@ -10,9 +10,9 @@
|
|||||||
// use engine for implementation
|
// use engine for implementation
|
||||||
#include <vector>
|
#include <vector>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include "./io.h"
|
#include "rabit/internal/io.h"
|
||||||
#include "./utils.h"
|
#include "rabit/internal/utils.h"
|
||||||
#include "../rabit.h"
|
#include "rabit/rabit.h"
|
||||||
|
|
||||||
namespace rabit {
|
namespace rabit {
|
||||||
namespace engine {
|
namespace engine {
|
||||||
|
|||||||
@ -4,8 +4,8 @@
|
|||||||
* \brief this file aims to provide a wrapper of sockets
|
* \brief this file aims to provide a wrapper of sockets
|
||||||
* \author Tianqi Chen
|
* \author Tianqi Chen
|
||||||
*/
|
*/
|
||||||
#ifndef RABIT_SOCKET_H_
|
#ifndef RABIT_INTERNAL_SOCKET_H_
|
||||||
#define RABIT_SOCKET_H_
|
#define RABIT_INTERNAL_SOCKET_H_
|
||||||
#if defined(_WIN32)
|
#if defined(_WIN32)
|
||||||
#include <winsock2.h>
|
#include <winsock2.h>
|
||||||
#include <ws2tcpip.h>
|
#include <ws2tcpip.h>
|
||||||
@ -26,7 +26,7 @@
|
|||||||
#include <cstring>
|
#include <cstring>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
#include <unordered_map>
|
#include <unordered_map>
|
||||||
#include "../include/rabit/internal/utils.h"
|
#include "utils.h"
|
||||||
|
|
||||||
#if defined(_WIN32) || defined(__MINGW32__)
|
#if defined(_WIN32) || defined(__MINGW32__)
|
||||||
typedef int ssize_t;
|
typedef int ssize_t;
|
||||||
@ -533,4 +533,4 @@ struct PollHelper {
|
|||||||
};
|
};
|
||||||
} // namespace utils
|
} // namespace utils
|
||||||
} // namespace rabit
|
} // namespace rabit
|
||||||
#endif // RABIT_SOCKET_H_
|
#endif // RABIT_INTERNAL_SOCKET_H_
|
||||||
@ -3,8 +3,8 @@
|
|||||||
* \file thread_local.h
|
* \file thread_local.h
|
||||||
* \brief Common utility for thread local storage.
|
* \brief Common utility for thread local storage.
|
||||||
*/
|
*/
|
||||||
#ifndef RABIT_THREAD_LOCAL_H_
|
#ifndef RABIT_INTERNAL_THREAD_LOCAL_H_
|
||||||
#define RABIT_THREAD_LOCAL_H_
|
#define RABIT_INTERNAL_THREAD_LOCAL_H_
|
||||||
|
|
||||||
#include "../include/dmlc/base.h"
|
#include "../include/dmlc/base.h"
|
||||||
|
|
||||||
@ -84,4 +84,4 @@ class ThreadLocalStore {
|
|||||||
std::vector<T*> data_;
|
std::vector<T*> data_;
|
||||||
};
|
};
|
||||||
} // namespace rabit
|
} // namespace rabit
|
||||||
#endif // RABIT_THREAD_LOCAL_H_
|
#endif // RABIT_INTERNAL_THREAD_LOCAL_H_
|
||||||
@ -8,10 +8,10 @@
|
|||||||
#define RABIT_SERIALIZABLE_H_
|
#define RABIT_SERIALIZABLE_H_
|
||||||
#include <vector>
|
#include <vector>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include "./internal/utils.h"
|
#include "rabit/internal/utils.h"
|
||||||
|
|
||||||
#ifndef DMLC_IO_H_
|
#ifndef DMLC_IO_H_
|
||||||
#include "../../dmlc-core/include/dmlc/io.h"
|
#include "dmlc/io.h"
|
||||||
#endif // DMLC_IO_H_
|
#endif // DMLC_IO_H_
|
||||||
|
|
||||||
namespace rabit {
|
namespace rabit {
|
||||||
|
|||||||
@ -24,9 +24,12 @@ fi
|
|||||||
if [ ${TASK} == "cmake-test" ]; then
|
if [ ${TASK} == "cmake-test" ]; then
|
||||||
mkdir build
|
mkdir build
|
||||||
cd build
|
cd build
|
||||||
cmake -DRABIT_BUILD_TESTS=ON -DRABIT_BUILD_DMLC=ON ..
|
cmake -DRABIT_BUILD_TESTS=ON -DRABIT_BUILD_DMLC=ON -DGTEST_ROOT=${HOME}/.local ..
|
||||||
|
#unit tests
|
||||||
|
make
|
||||||
|
make test
|
||||||
make install || exit -1
|
make install || exit -1
|
||||||
cd ../test
|
cd ../test
|
||||||
../scripts/travis_runtest.sh || exit -1
|
../scripts/travis_runtest.sh || exit -1
|
||||||
rm -rf ../build
|
rm -rf ../build
|
||||||
fi
|
fi
|
||||||
|
|||||||
35
scripts/travis_setup.sh
Executable file
35
scripts/travis_setup.sh
Executable file
@ -0,0 +1,35 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
echo "Testing on: ${TRAVIS_OS_NAME}, Home directory: ${HOME}"
|
||||||
|
|
||||||
|
pip3 install cpplint pylint urllib3 numpy cpplint
|
||||||
|
pip3 install websocket-client kubernetes
|
||||||
|
|
||||||
|
|
||||||
|
# Install googletest under home directory
|
||||||
|
GTEST_VERSION=1.8.1
|
||||||
|
GTEST_RELEASE=release-${GTEST_VERSION}.tar.gz
|
||||||
|
GTEST_TAR_BALL=googletest_${GTEST_RELEASE}
|
||||||
|
|
||||||
|
wget https://github.com/google/googletest/archive/${GTEST_RELEASE} -O ${GTEST_TAR_BALL}
|
||||||
|
echo "152b849610d91a9dfa1401293f43230c2e0c33f8 ${GTEST_TAR_BALL}" | sha1sum -c
|
||||||
|
tar -xf ${GTEST_TAR_BALL}
|
||||||
|
pushd .
|
||||||
|
|
||||||
|
cd googletest-release-${GTEST_VERSION}
|
||||||
|
mkdir build
|
||||||
|
cd build
|
||||||
|
echo "Installing to ${HOME}/.local"
|
||||||
|
cmake .. -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=${HOME}/.local
|
||||||
|
make -j$(nproc)
|
||||||
|
make install
|
||||||
|
|
||||||
|
popd
|
||||||
|
|
||||||
|
if [ ${TRAVIS_OS_NAME} == "linux" ]; then
|
||||||
|
sudo apt-get install python3-pip tree
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ${TRAVIS_OS_NAME} == "osx" ]; then
|
||||||
|
brew install python3
|
||||||
|
fi
|
||||||
17
src/CMakeLists.txt
Normal file
17
src/CMakeLists.txt
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
option(DMLC_ROOT "Specify root of external dmlc core.")
|
||||||
|
|
||||||
|
add_library(allreduce_base "")
|
||||||
|
|
||||||
|
target_sources(
|
||||||
|
allreduce_base
|
||||||
|
PRIVATE
|
||||||
|
allreduce_base.cc
|
||||||
|
PUBLIC
|
||||||
|
${CMAKE_CURRENT_LIST_DIR}/allreduce_base.h
|
||||||
|
)
|
||||||
|
|
||||||
|
target_include_directories(
|
||||||
|
allreduce_base
|
||||||
|
PUBLIC
|
||||||
|
${DMLC_ROOT}/include
|
||||||
|
${CMAKE_CURRENT_LIST_DIR}/../../include)
|
||||||
@ -9,9 +9,8 @@
|
|||||||
#define _CRT_SECURE_NO_DEPRECATE
|
#define _CRT_SECURE_NO_DEPRECATE
|
||||||
#define NOMINMAX
|
#define NOMINMAX
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <cstdlib>
|
|
||||||
#include <cstring>
|
#include <cstring>
|
||||||
#include "./allreduce_base.h"
|
#include "allreduce_base.h"
|
||||||
|
|
||||||
namespace rabit {
|
namespace rabit {
|
||||||
|
|
||||||
@ -39,16 +38,8 @@ AllreduceBase::AllreduceBase(void) {
|
|||||||
err_link = NULL;
|
err_link = NULL;
|
||||||
dmlc_role = "worker";
|
dmlc_role = "worker";
|
||||||
this->SetParam("rabit_reduce_buffer", "256MB");
|
this->SetParam("rabit_reduce_buffer", "256MB");
|
||||||
// setup possible enviroment variable of intrest
|
// setup possible enviroment variable of interest
|
||||||
env_vars.push_back("rabit_task_id");
|
// include dmlc support direct variables
|
||||||
env_vars.push_back("rabit_num_trial");
|
|
||||||
env_vars.push_back("rabit_reduce_buffer");
|
|
||||||
env_vars.push_back("rabit_reduce_ring_mincount");
|
|
||||||
env_vars.push_back("rabit_tracker_uri");
|
|
||||||
env_vars.push_back("rabit_tracker_port");
|
|
||||||
env_vars.push_back("rabit_bootstrap_cache");
|
|
||||||
env_vars.push_back("rabit_debug");
|
|
||||||
// also include dmlc support direct variables
|
|
||||||
env_vars.push_back("DMLC_TASK_ID");
|
env_vars.push_back("DMLC_TASK_ID");
|
||||||
env_vars.push_back("DMLC_ROLE");
|
env_vars.push_back("DMLC_ROLE");
|
||||||
env_vars.push_back("DMLC_NUM_ATTEMPT");
|
env_vars.push_back("DMLC_NUM_ATTEMPT");
|
||||||
@ -199,7 +190,8 @@ void AllreduceBase::SetParam(const char *name, const char *val) {
|
|||||||
if (!strcmp(name, "rabit_world_size")) world_size = atoi(val);
|
if (!strcmp(name, "rabit_world_size")) world_size = atoi(val);
|
||||||
if (!strcmp(name, "rabit_hadoop_mode")) hadoop_mode = atoi(val);
|
if (!strcmp(name, "rabit_hadoop_mode")) hadoop_mode = atoi(val);
|
||||||
if (!strcmp(name, "rabit_reduce_ring_mincount")) {
|
if (!strcmp(name, "rabit_reduce_ring_mincount")) {
|
||||||
reduce_ring_mincount = ParseUnit(name, val);
|
reduce_ring_mincount = atoi(val);
|
||||||
|
utils::Assert(reduce_ring_mincount > 0, "rabit_reduce_ring_mincount should be greater than 0");
|
||||||
}
|
}
|
||||||
if (!strcmp(name, "rabit_reduce_buffer")) {
|
if (!strcmp(name, "rabit_reduce_buffer")) {
|
||||||
reduce_buffer_size = (ParseUnit(name, val) + 7) >> 3;
|
reduce_buffer_size = (ParseUnit(name, val) + 7) >> 3;
|
||||||
|
|||||||
@ -15,9 +15,15 @@
|
|||||||
#include <vector>
|
#include <vector>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include "../include/rabit/internal/utils.h"
|
#include "rabit/internal/utils.h"
|
||||||
#include "../include/rabit/internal/engine.h"
|
#include "rabit/internal/engine.h"
|
||||||
#include "./socket.h"
|
#include "rabit/internal/socket.h"
|
||||||
|
|
||||||
|
#ifdef RABIT_CXXTESTDEFS_H
|
||||||
|
#define private public
|
||||||
|
#define protected public
|
||||||
|
#endif // RABIT_CXXTESTDEFS_H
|
||||||
|
|
||||||
|
|
||||||
namespace MPI {
|
namespace MPI {
|
||||||
// MPI data type to be compatible with existing MPI interface
|
// MPI data type to be compatible with existing MPI interface
|
||||||
|
|||||||
@ -11,9 +11,9 @@
|
|||||||
#include <vector>
|
#include <vector>
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <sstream>
|
#include <sstream>
|
||||||
#include "../include/rabit/internal/engine.h"
|
#include "rabit/internal/engine.h"
|
||||||
#include "../include/rabit/internal/timer.h"
|
#include "rabit/internal/timer.h"
|
||||||
#include "./allreduce_robust.h"
|
#include "allreduce_robust.h"
|
||||||
|
|
||||||
namespace rabit {
|
namespace rabit {
|
||||||
namespace engine {
|
namespace engine {
|
||||||
|
|||||||
@ -10,12 +10,12 @@
|
|||||||
#define NOMINMAX
|
#define NOMINMAX
|
||||||
#include <limits>
|
#include <limits>
|
||||||
#include <utility>
|
#include <utility>
|
||||||
#include "../include/rabit/internal/io.h"
|
#include "rabit/internal/io.h"
|
||||||
#include "../include/rabit/internal/timer.h"
|
#include "rabit/internal/timer.h"
|
||||||
#include "../include/rabit/internal/utils.h"
|
#include "rabit/internal/utils.h"
|
||||||
#include "../include/rabit/internal/engine.h"
|
#include "rabit/internal/engine.h"
|
||||||
#include "../include/rabit/internal/rabit-inl.h"
|
#include "rabit/internal/rabit-inl.h"
|
||||||
#include "./allreduce_robust.h"
|
#include "allreduce_robust.h"
|
||||||
|
|
||||||
namespace rabit {
|
namespace rabit {
|
||||||
namespace engine {
|
namespace engine {
|
||||||
@ -870,7 +870,7 @@ AllreduceRobust::TryRecoverData(RecoverType role,
|
|||||||
/*!
|
/*!
|
||||||
* \brief try to fetch allreduce/broadcast results from rest of nodes
|
* \brief try to fetch allreduce/broadcast results from rest of nodes
|
||||||
* as collaberative function called by all nodes, only requester node
|
* as collaberative function called by all nodes, only requester node
|
||||||
* will pass seqno to rest of nodes and reconstruct/backfill sendrecvbuf_
|
* will pass seqno to rest of nodes and reconstruct/backfill sendrecvbuf_
|
||||||
* of specific seqno from other nodes.
|
* of specific seqno from other nodes.
|
||||||
*/
|
*/
|
||||||
AllreduceRobust::ReturnType AllreduceRobust::TryRestoreCache(bool requester,
|
AllreduceRobust::ReturnType AllreduceRobust::TryRestoreCache(bool requester,
|
||||||
|
|||||||
@ -13,8 +13,8 @@
|
|||||||
#include <vector>
|
#include <vector>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include "../include/rabit/internal/engine.h"
|
#include "rabit/internal/engine.h"
|
||||||
#include "./allreduce_base.h"
|
#include "allreduce_base.h"
|
||||||
|
|
||||||
namespace rabit {
|
namespace rabit {
|
||||||
namespace engine {
|
namespace engine {
|
||||||
|
|||||||
@ -5,8 +5,8 @@
|
|||||||
|
|
||||||
#include <cstring>
|
#include <cstring>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include "../include/rabit/rabit.h"
|
#include "rabit/rabit.h"
|
||||||
#include "../include/rabit/c_api.h"
|
#include "rabit/c_api.h"
|
||||||
|
|
||||||
namespace rabit {
|
namespace rabit {
|
||||||
namespace c_api {
|
namespace c_api {
|
||||||
|
|||||||
@ -11,10 +11,10 @@
|
|||||||
#define NOMINMAX
|
#define NOMINMAX
|
||||||
|
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include "../include/rabit/internal/engine.h"
|
#include "rabit/internal/engine.h"
|
||||||
#include "./allreduce_base.h"
|
#include "allreduce_base.h"
|
||||||
#include "./allreduce_robust.h"
|
#include "allreduce_robust.h"
|
||||||
#include "./thread_local.h"
|
#include "rabit/internal/thread_local.h"
|
||||||
|
|
||||||
namespace rabit {
|
namespace rabit {
|
||||||
namespace engine {
|
namespace engine {
|
||||||
|
|||||||
@ -11,5 +11,5 @@
|
|||||||
#define NOMINMAX
|
#define NOMINMAX
|
||||||
// switch engine to AllreduceMock
|
// switch engine to AllreduceMock
|
||||||
#define RABIT_USE_BASE
|
#define RABIT_USE_BASE
|
||||||
#include "./engine.cc"
|
#include "engine.cc"
|
||||||
|
|
||||||
|
|||||||
@ -10,7 +10,7 @@
|
|||||||
#define _CRT_SECURE_NO_DEPRECATE
|
#define _CRT_SECURE_NO_DEPRECATE
|
||||||
#define NOMINMAX
|
#define NOMINMAX
|
||||||
|
|
||||||
#include "../include/rabit/internal/engine.h"
|
#include "rabit/internal/engine.h"
|
||||||
|
|
||||||
namespace rabit {
|
namespace rabit {
|
||||||
|
|
||||||
|
|||||||
@ -11,6 +11,6 @@
|
|||||||
#define NOMINMAX
|
#define NOMINMAX
|
||||||
// switch engine to AllreduceMock
|
// switch engine to AllreduceMock
|
||||||
#define RABIT_USE_MOCK
|
#define RABIT_USE_MOCK
|
||||||
#include "./allreduce_mock.h"
|
#include "allreduce_mock.h"
|
||||||
#include "./engine.cc"
|
#include "engine.cc"
|
||||||
|
|
||||||
|
|||||||
@ -11,8 +11,8 @@
|
|||||||
#define NOMINMAX
|
#define NOMINMAX
|
||||||
#include <mpi.h>
|
#include <mpi.h>
|
||||||
#include <cstdio>
|
#include <cstdio>
|
||||||
#include "../include/rabit/internal/engine.h"
|
#include "rabit/internal/engine.h"
|
||||||
#include "../include/rabit/internal/utils.h"
|
#include "rabit/internal/utils.h"
|
||||||
|
|
||||||
namespace rabit {
|
namespace rabit {
|
||||||
|
|
||||||
|
|||||||
28
test/cpp/CMakeLists.txt
Normal file
28
test/cpp/CMakeLists.txt
Normal file
@ -0,0 +1,28 @@
|
|||||||
|
find_package(GTest REQUIRED)
|
||||||
|
|
||||||
|
add_executable(
|
||||||
|
unit_tests
|
||||||
|
allreduce_base_test.cpp
|
||||||
|
test_main.cpp)
|
||||||
|
|
||||||
|
target_link_libraries(
|
||||||
|
unit_tests
|
||||||
|
GTest::GTest GTest::Main
|
||||||
|
rabit_base)
|
||||||
|
|
||||||
|
target_include_directories(unit_tests PUBLIC
|
||||||
|
"$<BUILD_INTERFACE:${rabit_SOURCE_DIR}/include>"
|
||||||
|
"$<BUILD_INTERFACE:${DMLC_ROOT}/include>")
|
||||||
|
|
||||||
|
set_target_properties(unit_tests
|
||||||
|
PROPERTIES
|
||||||
|
CXX_STANDARD 11
|
||||||
|
CXX_STANDARD_REQUIRED ON
|
||||||
|
RUNTIME_OUTPUT_DIRECTORY ${rabit_BINARY_DIR}
|
||||||
|
RUNTIME_OUTPUT_DIRECTORY_DEBUG ${rabit_BINARY_DIR}
|
||||||
|
RUNTIME_OUTPUT_DIRECTORY_RELEASE ${rabit_BINARY_DIR})
|
||||||
|
|
||||||
|
add_test(
|
||||||
|
NAME TestRabitLib
|
||||||
|
COMMAND unit_tests
|
||||||
|
WORKING_DIRECTORY ${rabit_BINARY_DIR})
|
||||||
1
test/cpp/README.md
Normal file
1
test/cpp/README.md
Normal file
@ -0,0 +1 @@
|
|||||||
|
Unittests for Rabit
|
||||||
66
test/cpp/allreduce_base_test.cpp
Normal file
66
test/cpp/allreduce_base_test.cpp
Normal file
@ -0,0 +1,66 @@
|
|||||||
|
#define RABIT_CXXTESTDEFS_H
|
||||||
|
#include <gtest/gtest.h>
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
#include <iostream>
|
||||||
|
#include "../../src/allreduce_base.h"
|
||||||
|
|
||||||
|
TEST(allreduce_base, init_task)
|
||||||
|
{
|
||||||
|
rabit::engine::AllreduceBase base;
|
||||||
|
|
||||||
|
std::string rabit_task_id = "rabit_task_id=1";
|
||||||
|
char cmd[rabit_task_id.size()+1];
|
||||||
|
std::copy(rabit_task_id.begin(), rabit_task_id.end(), cmd);
|
||||||
|
cmd[rabit_task_id.size()] = '\0';
|
||||||
|
|
||||||
|
char* argv[] = {cmd};
|
||||||
|
base.Init(1, argv);
|
||||||
|
EXPECT_EQ(base.task_id, "1");
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(allreduce_base, init_with_cache_on)
|
||||||
|
{
|
||||||
|
rabit::engine::AllreduceBase base;
|
||||||
|
|
||||||
|
std::string rabit_task_id = "rabit_task_id=1";
|
||||||
|
char cmd[rabit_task_id.size()+1];
|
||||||
|
std::copy(rabit_task_id.begin(), rabit_task_id.end(), cmd);
|
||||||
|
cmd[rabit_task_id.size()] = '\0';
|
||||||
|
|
||||||
|
std::string rabit_bootstrap_cache = "rabit_bootstrap_cache=1";
|
||||||
|
char cmd2[rabit_bootstrap_cache.size()+1];
|
||||||
|
std::copy(rabit_bootstrap_cache.begin(), rabit_bootstrap_cache.end(), cmd2);
|
||||||
|
cmd2[rabit_bootstrap_cache.size()] = '\0';
|
||||||
|
|
||||||
|
std::string rabit_debug = "rabit_debug=1";
|
||||||
|
char cmd3[rabit_debug.size()+1];
|
||||||
|
std::copy(rabit_debug.begin(), rabit_debug.end(), cmd3);
|
||||||
|
cmd3[rabit_debug.size()] = '\0';
|
||||||
|
|
||||||
|
char* argv[] = {cmd, cmd2, cmd3};
|
||||||
|
base.Init(3, argv);
|
||||||
|
EXPECT_EQ(base.task_id, "1");
|
||||||
|
EXPECT_EQ(base.rabit_bootstrap_cache, 1);
|
||||||
|
EXPECT_EQ(base.rabit_debug, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(allreduce_base, init_with_ring_reduce)
|
||||||
|
{
|
||||||
|
rabit::engine::AllreduceBase base;
|
||||||
|
|
||||||
|
std::string rabit_task_id = "rabit_task_id=1";
|
||||||
|
char cmd[rabit_task_id.size()+1];
|
||||||
|
std::copy(rabit_task_id.begin(), rabit_task_id.end(), cmd);
|
||||||
|
cmd[rabit_task_id.size()] = '\0';
|
||||||
|
|
||||||
|
std::string rabit_reduce_ring_mincount = "rabit_reduce_ring_mincount=1";
|
||||||
|
char cmd2[rabit_reduce_ring_mincount.size()+1];
|
||||||
|
std::copy(rabit_reduce_ring_mincount.begin(), rabit_reduce_ring_mincount.end(), cmd2);
|
||||||
|
cmd2[rabit_reduce_ring_mincount.size()] = '\0';
|
||||||
|
|
||||||
|
char* argv[] = {cmd, cmd2};
|
||||||
|
base.Init(2, argv);
|
||||||
|
EXPECT_EQ(base.task_id, "1");
|
||||||
|
EXPECT_EQ(base.reduce_ring_mincount, 1);
|
||||||
|
}
|
||||||
7
test/cpp/test_main.cpp
Normal file
7
test/cpp/test_main.cpp
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
#include "gtest/gtest.h"
|
||||||
|
|
||||||
|
int main(int argc, char** argv)
|
||||||
|
{
|
||||||
|
::testing::InitGoogleTest(&argc, argv);
|
||||||
|
return RUN_ALL_TESTS();
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user