From 207f0587111a951f5cd6c609ade001587f242bd1 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Tue, 16 Apr 2019 01:08:12 +0800 Subject: [PATCH] Refactor CMake scripts. (#4323) * Refactor CMake scripts. * Remove CMake CUDA wrapper. * Bump CMake version for CUDA. * Use CMake to handle Doxygen. * Split up CMakeList. * Export install target. * Use modern CMake. * Remove build.sh * Workaround for gpu_hist test. * Use cmake 3.12. * Revert machine.conf. * Move CLI test to gpu. * Small cleanup. * Support using XGBoost as submodule. * Fix windows * Fix cpp tests on Windows * Remove duplicated find_package. --- .travis.yml | 4 +- CMakeLists.txt | 470 ++++++------------ R-package/CMakeLists.txt | 34 ++ R-package/src/xgboost_custom.cc | 2 +- build.sh | 51 -- cmake/Doc.cmake | 16 + cmake/Utils.cmake | 41 +- cmake/build_config.h.in | 11 - cmake/modules/FindNccl.cmake | 9 +- cmake/xgboost-config.cmake.in | 5 + demo/c-api/README.md | 30 ++ doc/{Doxyfile => Doxyfile.in} | 6 +- include/xgboost/build_config.h | 4 + include/xgboost/c_api.h | 3 - jvm-packages/CMakeLists.txt | 22 + .../xgboost4j/src/native/xgboost4j.cpp | 1 + src/CMakeLists.txt | 107 ++++ src/c_api/c_api.cc | 3 + src/tree/updater_gpu_hist.cu | 28 +- tests/ci_build/Dockerfile.clang_tidy | 4 +- tests/ci_build/Dockerfile.gpu | 4 +- tests/ci_build/build_via_cmake.sh | 2 +- tests/ci_build/test_gpu.sh | 6 +- tests/ci_build/tidy.py | 61 ++- tests/cli/machine.conf.in | 13 + tests/cpp/CMakeLists.txt | 56 +++ tests/cpp/common/test_transform_range.cu | 4 +- tests/travis/run_test.sh | 10 +- 28 files changed, 578 insertions(+), 429 deletions(-) create mode 100644 R-package/CMakeLists.txt delete mode 100755 build.sh create mode 100644 cmake/Doc.cmake delete mode 100644 cmake/build_config.h.in create mode 100644 cmake/xgboost-config.cmake.in create mode 100644 demo/c-api/README.md rename doc/{Doxyfile => Doxyfile.in} (99%) create mode 100644 jvm-packages/CMakeLists.txt create mode 100644 src/CMakeLists.txt create mode 100644 tests/cli/machine.conf.in create mode 100644 tests/cpp/CMakeLists.txt diff --git a/.travis.yml b/.travis.yml index dc38d6298..645504631 100644 --- a/.travis.yml +++ b/.travis.yml @@ -62,8 +62,8 @@ addons: - libcurl4-openssl-dev - unzip - graphviz - - gcc-4.8 - - g++-4.8 + - gcc-5 + - g++-5 - gcc-7 - g++-7 homebrew: diff --git a/CMakeLists.txt b/CMakeLists.txt index af3611ba2..65614d0c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,359 +1,205 @@ -cmake_minimum_required (VERSION 3.2) -project(xgboost) +cmake_minimum_required(VERSION 3.2) +project(xgboost LANGUAGES CXX C VERSION 0.82) include(cmake/Utils.cmake) list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/modules") - -# Options -## OPENMP -option(USE_OPENMP "Build with OpenMP acceleration" ON) -if (USE_OPENMP) - find_package(OpenMP) -endif() +cmake_policy(SET CMP0022 NEW) set_default_configuration_release() msvc_use_static_runtime() -# Options -## GPUs -option(USE_CUDA "Build with GPU acceleration" OFF) -option(USE_NVTX "Build with cuda profiling annotations. Developers only." OFF) -option(USE_NCCL "Build with multiple GPUs support" OFF) -set(GPU_COMPUTE_VER "" CACHE STRING - "Space separated list of compute versions to be built against, e.g. '35 61'") -set(NVTX_HEADER_DIR "" CACHE PATH - "Path to the stand-alone nvtx header") - +#-- Options +option(BUILD_C_DOC "Build documentation for C APIs using Doxygen." OFF) +option(USE_OPENMP "Build with OpenMP support." ON) ## Bindings option(JVM_BINDINGS "Build JVM bindings" OFF) option(R_LIB "Build shared library for R package" OFF) - -## Devs +## Dev +option(GOOGLE_TEST "Build google tests" OFF) +option(USE_NVTX "Build with cuda profiling annotations. Developers only." OFF) +set(NVTX_HEADER_DIR "" CACHE PATH "Path to the stand-alone nvtx header") +## CUDA +option(USE_CUDA "Build with GPU acceleration" OFF) +option(USE_NCCL "Build with NCCL to enable multi-GPU support." OFF) +set(GPU_COMPUTE_VER "" CACHE STRING + "Semicolon separated list of compute versions to be built against, e.g. '35;61'") +## Sanitizers option(USE_SANITIZER "Use santizer flags" OFF) option(SANITIZER_PATH "Path to sanitizes.") set(ENABLED_SANITIZERS "address" "leak" CACHE STRING "Semicolon separated list of sanitizer names. E.g 'address;leak'. Supported sanitizers are address, leak and thread.") -option(GOOGLE_TEST "Build google tests" OFF) - -# Plugins +## Plugins option(PLUGIN_LZ4 "Build lz4 plugin" OFF) option(PLUGIN_DENSE_PARSER "Build dense parser plugin" OFF) -# Deprecation warning -if(USE_AVX) +## Deprecation warning +if (USE_AVX) message(WARNING "The option 'USE_AVX' is deprecated as experimental AVX features have been removed from xgboost.") -endif() - -# Compiler flags -set(CMAKE_CXX_STANDARD 11) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -if(OpenMP_CXX_FOUND OR OPENMP_FOUND) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") -endif() -set(CMAKE_POSITION_INDEPENDENT_CODE ON) -if(MSVC) - # Multithreaded compilation - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP") -else() - # Correct error for GCC 5 and cuda - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_MWAITXINTRIN_H_INCLUDED -D_FORCE_INLINES") - # Performance - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -funroll-loops") -endif() -if(WIN32 AND MINGW) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libstdc++") -endif() - -# Check existence of software pre-fetching -include(CheckCXXSourceCompiles) -check_cxx_source_compiles(" -#include -int main() { - char data = 0; - const char* address = &data; - _mm_prefetch(address, _MM_HINT_NTA); - return 0; -} -" XGBOOST_MM_PREFETCH_PRESENT) -check_cxx_source_compiles(" -int main() { - char data = 0; - const char* address = &data; - __builtin_prefetch(address, 0, 0); - return 0; -} -" XGBOOST_BUILTIN_PREFETCH_PRESENT) +endif (USE_AVX) # Sanitizer -if(USE_SANITIZER) +if (USE_SANITIZER) include(cmake/Sanitizer.cmake) enable_sanitizers("${ENABLED_SANITIZERS}") -endif(USE_SANITIZER) +endif (USE_SANITIZER) + +if (USE_CUDA) + cmake_minimum_required(VERSION 3.12) + # `export CXX=' is ignored by CMake CUDA. + set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER}) + message(STATUS "Configured CUDA host compiler: ${CMAKE_CUDA_HOST_COMPILER}") + + enable_language(CUDA) + set(GEN_CODE "") + format_gencode_flags("${GPU_COMPUTE_VER}" GEN_CODE) + message(STATUS "CUDA GEN_CODE: ${GEN_CODE}") +endif (USE_CUDA) # dmlc-core -add_subdirectory(dmlc-core) -set(LINK_LIBRARIES dmlc rabit) - -# enable custom logging -add_definitions(-DDMLC_LOG_CUSTOMIZE=1) - -# compiled code customizations for R package -if(R_LIB) - add_definitions( - -DXGBOOST_STRICT_R_MODE=1 - -DXGBOOST_CUSTOMIZE_GLOBAL_PRNG=1 - -DDMLC_LOG_BEFORE_THROW=0 - -DDMLC_DISABLE_STDIN=1 - -DDMLC_LOG_CUSTOMIZE=1 - -DRABIT_CUSTOMIZE_MSG_ - -DRABIT_STRICT_CXX98_ - ) -endif() - -# Gather source files -include_directories ( - ${PROJECT_SOURCE_DIR}/include - ${PROJECT_SOURCE_DIR}/dmlc-core/include - ${PROJECT_SOURCE_DIR}/rabit/include -) - -# Generate configurable header -set(CMAKE_LOCAL "${PROJECT_SOURCE_DIR}/cmake") -set(INCLUDE_ROOT "${PROJECT_SOURCE_DIR}/include") -message(STATUS "${CMAKE_LOCAL}/build_config.h.in -> ${INCLUDE_ROOT}/xgboost/build_config.h") -configure_file("${CMAKE_LOCAL}/build_config.h.in" "${INCLUDE_ROOT}/xgboost/build_config.h") - -file(GLOB_RECURSE SOURCES - src/*.cc - src/*.h - include/*.h -) - -# Only add main function for executable target -list(REMOVE_ITEM SOURCES ${PROJECT_SOURCE_DIR}/src/cli_main.cc) - -file(GLOB_RECURSE CUDA_SOURCES - src/*.cu - src/*.cuh -) - -# Add plugins to source files -if(PLUGIN_LZ4) - list(APPEND SOURCES plugin/lz4/sparse_page_lz4_format.cc) - link_libraries(lz4) -endif() -if(PLUGIN_DENSE_PARSER) - list(APPEND SOURCES plugin/dense_parser/dense_libsvm.cc) -endif() +add_subdirectory(${PROJECT_SOURCE_DIR}/dmlc-core) +set_target_properties(dmlc PROPERTIES + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON) +list(APPEND LINKED_LIBRARIES_PRIVATE dmlc) # rabit -# TODO: Use CMakeLists.txt from rabit. -set(RABIT_SOURCES +# full rabit doesn't build on windows, so we can't import it as subdirectory +if(MINGW OR R_LIB) + set(RABIT_SOURCES + rabit/src/engine_empty.cc + rabit/src/c_api.cc) +else () + set(RABIT_SOURCES rabit/src/allreduce_base.cc rabit/src/allreduce_robust.cc rabit/src/engine.cc - rabit/src/c_api.cc -) -set(RABIT_EMPTY_SOURCES - rabit/src/engine_empty.cc - rabit/src/c_api.cc -) + rabit/src/c_api.cc) +endif (MINGW OR R_LIB) +add_library(rabit STATIC ${RABIT_SOURCES}) +target_include_directories(rabit PRIVATE + $ + $) +set_target_properties(rabit + PROPERTIES + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON) +list(APPEND LINKED_LIBRARIES_PRIVATE rabit) -if(MINGW OR R_LIB) - # build a dummy rabit library - add_library(rabit STATIC ${RABIT_EMPTY_SOURCES}) -else() - add_library(rabit STATIC ${RABIT_SOURCES}) -endif() +# Exports some R specific definitions and objects +if (R_LIB) + add_subdirectory(${PROJECT_SOURCE_DIR}/R-package) +endif (R_LIB) -if (GENERATE_COMPILATION_DATABASE) - set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -endif (GENERATE_COMPILATION_DATABASE) +# core xgboost +add_subdirectory(${PROJECT_SOURCE_DIR}/src) +set(XGBOOST_OBJ_SOURCES "${XGBOOST_OBJ_SOURCES};$") -if(USE_CUDA AND (NOT GENERATE_COMPILATION_DATABASE)) - find_package(CUDA 8.0 REQUIRED) - cmake_minimum_required(VERSION 3.5) +#-- Shared library +add_library(xgboost SHARED ${XGBOOST_OBJ_SOURCES} ${PLUGINS_SOURCES}) +target_include_directories(xgboost + INTERFACE + $ + $) +target_link_libraries(xgboost PRIVATE ${LINKED_LIBRARIES_PRIVATE}) - add_definitions(-DXGBOOST_USE_CUDA) +# This creates its own shared library `xgboost4j'. +if (JVM_BINDINGS) + add_subdirectory(${PROJECT_SOURCE_DIR}/jvm-packages) +endif (JVM_BINDINGS) +#-- End shared library - include_directories(cub) +#-- CLI for xgboost +add_executable(runxgboost ${PROJECT_SOURCE_DIR}/src/cli_main.cc ${XGBOOST_OBJ_SOURCES}) +# For cli_main.cc only +target_include_directories(runxgboost + PRIVATE + ${PROJECT_SOURCE_DIR}/include + ${PROJECT_SOURCE_DIR}/dmlc-core/include + ${PROJECT_SOURCE_DIR}/rabit/include) +target_link_libraries(runxgboost PRIVATE ${LINKED_LIBRARIES_PRIVATE}) +set_target_properties( + runxgboost PROPERTIES + OUTPUT_NAME xgboost + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON) +#-- End CLI for xgboost - if(USE_NCCL) - find_package(Nccl REQUIRED) - cuda_include_directories(${NCCL_INCLUDE_DIR}) - add_definitions(-DXGBOOST_USE_NCCL) - endif() +set_output_directory(runxgboost ${PROJECT_SOURCE_DIR}) +set_output_directory(xgboost ${PROJECT_SOURCE_DIR}/lib) - if(USE_NVTX) - cuda_include_directories("${NVTX_HEADER_DIR}") - add_definitions(-DXGBOOST_USE_NVTX) - endif() - - set(GENCODE_FLAGS "") - format_gencode_flags("${GPU_COMPUTE_VER}" GENCODE_FLAGS) - message("cuda architecture flags: ${GENCODE_FLAGS}") - - set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS};--expt-extended-lambda;--expt-relaxed-constexpr;${GENCODE_FLAGS};-lineinfo;") - if(NOT MSVC) - set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS};-Xcompiler -fPIC; -Xcompiler -Werror; -std=c++11") - endif() - - cuda_add_library(gpuxgboost ${CUDA_SOURCES} STATIC) - - if(USE_NCCL) - link_directories(${NCCL_LIBRARY}) - target_link_libraries(gpuxgboost ${NCCL_LIB_NAME}) - endif() - - list(APPEND LINK_LIBRARIES gpuxgboost) - -elseif (USE_CUDA AND GENERATE_COMPILATION_DATABASE) - # Enable CUDA language to generate a compilation database. - cmake_minimum_required(VERSION 3.8) - - find_package(CUDA 8.0 REQUIRED) - enable_language(CUDA) - set(CMAKE_CUDA_COMPILER clang++) - set(CUDA_SEPARABLE_COMPILATION ON) - if (NOT CLANG_CUDA_GENCODE) - set(CLANG_CUDA_GENCODE "--cuda-gpu-arch=sm_35") - endif (NOT CLANG_CUDA_GENCODE) - set(CMAKE_CUDA_FLAGS " -Wno-deprecated ${CLANG_CUDA_GENCODE} -fPIC ${GENCODE} -std=c++11 -x cuda") - message(STATUS "CMAKE_CUDA_FLAGS: ${CMAKE_CUDA_FLAGS}") - - add_library(gpuxgboost STATIC ${CUDA_SOURCES}) - - if(USE_NCCL) - find_package(Nccl REQUIRED) - target_include_directories(gpuxgboost PUBLIC ${NCCL_INCLUDE_DIR}) - target_compile_definitions(gpuxgboost PUBLIC -DXGBOOST_USE_NCCL) - target_link_libraries(gpuxgboost PUBLIC ${NCCL_LIB_NAME}) - endif() - - target_compile_definitions(gpuxgboost PUBLIC -DXGBOOST_USE_CUDA) - # A hack for CMake to make arguments valid for clang++ - string(REPLACE "-x cu" "-x cuda" CMAKE_CUDA_COMPILE_PTX_COMPILATION - ${CMAKE_CUDA_COMPILE_PTX_COMPILATION}) - string(REPLACE "-x cu" "-x cuda" CMAKE_CUDA_COMPILE_WHOLE_COMPILATION - ${CMAKE_CUDA_COMPILE_WHOLE_COMPILATION}) - string(REPLACE "-x cu" "-x cuda" CMAKE_CUDA_COMPILE_SEPARABLE_COMPILATION - ${CMAKE_CUDA_COMPILE_SEPARABLE_COMPILATION}) - target_include_directories(gpuxgboost PUBLIC cub) -endif() - - -# flags and sources for R-package -if(R_LIB) - file(GLOB_RECURSE R_SOURCES - R-package/src/*.h - R-package/src/*.c - R-package/src/*.cc - ) - list(APPEND SOURCES ${R_SOURCES}) -endif() - -add_library(objxgboost OBJECT ${SOURCES}) - -# building shared library for R package -if(R_LIB) - find_package(LibR REQUIRED) - - list(APPEND LINK_LIBRARIES "${LIBR_CORE_LIBRARY}") - MESSAGE(STATUS "LIBR_CORE_LIBRARY " ${LIBR_CORE_LIBRARY}) - - # Shared library target for the R package - add_library(xgboost SHARED $) - include_directories(xgboost - "${LIBR_INCLUDE_DIRS}" - "${PROJECT_SOURCE_DIR}" - ) - - target_link_libraries(xgboost ${LINK_LIBRARIES}) - # R uses no lib prefix in shared library names of its packages +#-- Installing XGBoost +if (R_LIB) set_target_properties(xgboost PROPERTIES PREFIX "") - if(APPLE) + if (APPLE) set_target_properties(xgboost PROPERTIES SUFFIX ".so") - endif() - + endif (APPLE) setup_rpackage_install_target(xgboost ${CMAKE_CURRENT_BINARY_DIR}) - # use a dummy location for any other remaining installs set(CMAKE_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/dummy_inst") +endif (R_LIB) +if (MINGW) + set_target_properties(xgboost PROPERTIES PREFIX "") +endif (MINGW) - # main targets: shared library & exe -else() - # Executable - add_executable(runxgboost $ src/cli_main.cc) - set_target_properties(runxgboost PROPERTIES - OUTPUT_NAME xgboost - ) - set_output_directory(runxgboost ${PROJECT_SOURCE_DIR}) - target_link_libraries(runxgboost ${LINK_LIBRARIES}) +if (BUILD_C_DOC) + include(cmake/Doc.cmake) + run_doxygen() +endif (BUILD_C_DOC) - # Shared library - add_library(xgboost SHARED $) - target_link_libraries(xgboost ${LINK_LIBRARIES}) - set_output_directory(xgboost ${PROJECT_SOURCE_DIR}/lib) - if(MINGW) - # remove the 'lib' prefix to conform to windows convention for shared library names - set_target_properties(xgboost PROPERTIES PREFIX "") - endif() +include(GNUInstallDirs) +# Exposing only C APIs. +install(FILES + "${PROJECT_SOURCE_DIR}/include/xgboost/c_api.h" + DESTINATION + include/xgboost/) - #Ensure these two targets do not build simultaneously, as they produce outputs with conflicting names - add_dependencies(xgboost runxgboost) -endif() +install(TARGETS xgboost runxgboost + EXPORT XGBoostTargets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + INCLUDES DESTINATION ${LIBLEGACY_INCLUDE_DIRS}) +install(EXPORT XGBoostTargets + FILE XGBoostTargets.cmake + NAMESPACE xgboost:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/xgboost) -# JVM -if(JVM_BINDINGS) - find_package(JNI QUIET REQUIRED) +include(CMakePackageConfigHelpers) +configure_package_config_file( + ${CMAKE_CURRENT_LIST_DIR}/cmake/xgboost-config.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/cmake/xgboost-config.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/xgboost) +write_basic_package_version_file( + ${CMAKE_BINARY_DIR}/cmake/xgboost-config-version.cmake + VERSION ${XGBOOST_VERSION} + COMPATIBILITY AnyNewerVersion) +install( + FILES + ${CMAKE_BINARY_DIR}/cmake/xgboost-config.cmake + ${CMAKE_BINARY_DIR}/cmake/xgboost-config-version.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/xgboost) - add_library(xgboost4j SHARED - $ - jvm-packages/xgboost4j/src/native/xgboost4j.cpp) - target_include_directories(xgboost4j - PRIVATE ${JNI_INCLUDE_DIRS} - PRIVATE jvm-packages/xgboost4j/src/native) - target_link_libraries(xgboost4j - ${LINK_LIBRARIES} - ${JAVA_JVM_LIBRARY}) - set_output_directory(xgboost4j ${PROJECT_SOURCE_DIR}/lib) -endif() - - -# Test -if(GOOGLE_TEST) +#-- Test +if (GOOGLE_TEST) enable_testing() - find_package(GTest REQUIRED) + # Unittests. + add_subdirectory(${PROJECT_SOURCE_DIR}/tests/cpp) + add_test( + NAME TestXGBoostLib + COMMAND testxgboost + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) - file(GLOB_RECURSE TEST_SOURCES "tests/cpp/*.cc") - auto_source_group("${TEST_SOURCES}") - - if(USE_CUDA AND (NOT GENERATE_COMPILATION_DATABASE)) - file(GLOB_RECURSE CUDA_TEST_SOURCES "tests/cpp/*.cu") - cuda_include_directories(${GTEST_INCLUDE_DIRS}) - cuda_compile(CUDA_TEST_OBJS ${CUDA_TEST_SOURCES}) - elseif (USE_CUDA AND GENERATE_COMPILATION_DATABASE) - file(GLOB_RECURSE CUDA_TEST_SOURCES "tests/cpp/*.cu") - else() - set(CUDA_TEST_OBJS "") - endif() - - if (USE_CUDA AND GENERATE_COMPILATION_DATABASE) - add_executable(testxgboost ${TEST_SOURCES} ${CUDA_TEST_SOURCES} - $) - target_include_directories(testxgboost PRIVATE cub) - else () - add_executable(testxgboost ${TEST_SOURCES} ${CUDA_TEST_OBJS} - $) - endif () - - set_output_directory(testxgboost ${CMAKE_BINARY_DIR}) - target_include_directories(testxgboost - PRIVATE ${GTEST_INCLUDE_DIRS}) - target_link_libraries(testxgboost ${GTEST_LIBRARIES} ${LINK_LIBRARIES}) - - add_test(TestXGBoost testxgboost) -endif() - - -# Group sources -auto_source_group("${SOURCES}") + # CLI tests + configure_file( + ${PROJECT_SOURCE_DIR}/tests/cli/machine.conf.in + ${PROJECT_BINARY_DIR}/tests/cli/machine.conf + @ONLY) + add_test( + NAME TestXGBoostCLI + COMMAND runxgboost ${PROJECT_BINARY_DIR}/tests/cli/machine.conf + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) + set_tests_properties(TestXGBoostCLI + PROPERTIES + PASS_REGULAR_EXPRESSION ".*test-rmse:0.087.*") +endif (GOOGLE_TEST) diff --git a/R-package/CMakeLists.txt b/R-package/CMakeLists.txt new file mode 100644 index 000000000..35971bb6f --- /dev/null +++ b/R-package/CMakeLists.txt @@ -0,0 +1,34 @@ +find_package(LibR REQUIRED) +message(STATUS "LIBR_CORE_LIBRARY " ${LIBR_CORE_LIBRARY}) + +file(GLOB_RECURSE R_SOURCES + ${CMAKE_CURRENT_LIST_DIR}/src/*.cc + ${CMAKE_CURRENT_LIST_DIR}/src/*.c) +# Use object library to expose symbols +add_library(xgboost-r OBJECT ${R_SOURCES}) + +set(R_DEFINITIONS + -DXGBOOST_STRICT_R_MODE=1 + -DXGBOOST_CUSTOMIZE_GLOBAL_PRNG=1 + -DDMLC_LOG_BEFORE_THROW=0 + -DDMLC_DISABLE_STDIN=1 + -DDMLC_LOG_CUSTOMIZE=1 + -DRABIT_CUSTOMIZE_MSG_ + -DRABIT_STRICT_CXX98_) +target_compile_definitions(xgboost-r + PRIVATE ${R_DEFINITIONS}) +target_include_directories(xgboost-r + PRIVATE + ${LIBR_INCLUDE_DIRS} + ${PROJECT_SOURCE_DIR}/include + ${PROJECT_SOURCE_DIR}/dmlc-core/include + ${PROJECT_SOURCE_DIR}/rabit/include) +set_target_properties( + xgboost-r PROPERTIES + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON) + +set(XGBOOST_DEFINITIONS ${R_DEFINITIONS} PARENT_SCOPE) +set(XGBOOST_OBJ_SOURCES $ PARENT_SCOPE) +set(LINKED_LIBRARIES_PRIVATE ${LINKED_LIBRARIES_PRIVATE} ${LIBR_CORE_LIBRARY} PARENT_SCOPE) diff --git a/R-package/src/xgboost_custom.cc b/R-package/src/xgboost_custom.cc index 8a169f2dc..2387e7212 100644 --- a/R-package/src/xgboost_custom.cc +++ b/R-package/src/xgboost_custom.cc @@ -3,7 +3,7 @@ // to change behavior of libxgboost #include -#include "src/common/random.h" +#include "../../src/common/random.h" #include "./xgboost_R.h" // redirect the messages to R's console. diff --git a/build.sh b/build.sh deleted file mode 100755 index 564a53b0f..000000000 --- a/build.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -# This is a simple script to make xgboost in MAC and Linux -# Basically, it first try to make with OpenMP, if fails, disable OpenMP and make it again. -# This will automatically make xgboost for MAC users who don't have OpenMP support. -# In most cases, type make will give what you want. - -# See additional instruction in doc/build.md -set -e - -if make; then - echo "Successfully build multi-thread xgboost" -else - - not_ready=0 - - if [[ ! -e ./rabit/Makefile ]]; then - echo "" - echo "Please init the rabit submodule:" - echo "git submodule update --init --recursive -- rabit" - not_ready=1 - fi - - if [[ ! -e ./dmlc-core/Makefile ]]; then - echo "" - echo "Please init the dmlc-core submodule:" - echo "git submodule update --init --recursive -- dmlc-core" - not_ready=1 - fi - - if [[ "${not_ready}" == "1" ]]; then - echo "" - echo "Please fix the errors above and retry the build, or reclone the repository with:" - echo "git clone --recursive https://github.com/dmlc/xgboost.git" - echo "" - exit 1 - fi - - - echo "-----------------------------" - echo "Building multi-thread xgboost failed" - echo "Start to build single-thread xgboost" - make clean_all - make config=make/minimum.mk - if [ $? -eq 0 ] ;then - echo "Successfully build single-thread xgboost" - echo "If you want multi-threaded version" - echo "See additional instructions in doc/build.md" - else - echo "Failed to build single-thread xgboost" - fi -fi diff --git a/cmake/Doc.cmake b/cmake/Doc.cmake new file mode 100644 index 000000000..0122be12c --- /dev/null +++ b/cmake/Doc.cmake @@ -0,0 +1,16 @@ +function (run_doxygen) + find_package(Doxygen REQUIRED) + + if (NOT DOXYGEN_DOT_FOUND) + message(FATAL_ERROR "Command `dot` not found. Please install graphviz.") + endif (NOT DOXYGEN_DOT_FOUND) + + configure_file( + ${PROJECT_SOURCE_DIR}/doc/Doxyfile.in + ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY) + add_custom_target( doc_doxygen ALL + COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Generate C APIs documentation." + VERBATIM) +endfunction (run_doxygen) diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake index 2bc9fb40b..ac3b2c36e 100644 --- a/cmake/Utils.cmake +++ b/cmake/Utils.cmake @@ -1,4 +1,3 @@ - # Automatically set source group based on folder function(auto_source_group SOURCES) @@ -29,25 +28,37 @@ function(msvc_use_static_runtime) set(${variable} "${${variable}}" PARENT_SCOPE) endif() endforeach() + set(variables + CMAKE_CUDA_FLAGS_DEBUG + CMAKE_CUDA_FLAGS_MINSIZEREL + CMAKE_CUDA_FLAGS_RELEASE + CMAKE_CUDA_FLAGS_RELWITHDEBINFO + ) + foreach(variable ${variables}) + if(${variable} MATCHES "-MD") + string(REGEX REPLACE "-MD" "-MT" ${variable} "${${variable}}") + set(${variable} "${${variable}}" PARENT_SCOPE) + endif() + endforeach() endif() endfunction(msvc_use_static_runtime) # Set output directory of target, ignoring debug or release function(set_output_directory target dir) - set_target_properties(${target} PROPERTIES + set_target_properties(${target} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${dir} RUNTIME_OUTPUT_DIRECTORY_DEBUG ${dir} RUNTIME_OUTPUT_DIRECTORY_RELEASE ${dir} - LIBRARY_OUTPUT_DIRECTORY ${dir} - LIBRARY_OUTPUT_DIRECTORY_DEBUG ${dir} - LIBRARY_OUTPUT_DIRECTORY_RELEASE ${dir} + LIBRARY_OUTPUT_DIRECTORY ${dir} + LIBRARY_OUTPUT_DIRECTORY_DEBUG ${dir} + LIBRARY_OUTPUT_DIRECTORY_RELEASE ${dir} ) endfunction(set_output_directory) # Set a default build type to release if none was specified function(set_default_configuration_release) if(CMAKE_CONFIGURATION_TYPES STREQUAL "Debug;Release;MinSizeRel;RelWithDebInfo") # multiconfig generator? - set(CMAKE_CONFIGURATION_TYPES Release CACHE STRING "" FORCE) + set(CMAKE_CONFIGURATION_TYPES Release CACHE STRING "" FORCE) elseif(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) message(STATUS "Setting build type to 'Release' as none was specified.") set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE ) @@ -58,9 +69,9 @@ endfunction(set_default_configuration_release) # Also generates PTX for the most recent architecture for forwards compatibility function(format_gencode_flags flags out) # Set up architecture flags - if(NOT flags) + if(NOT flags) if((CUDA_VERSION_MAJOR EQUAL 10) OR (CUDA_VERSION_MAJOR GREATER 10)) - set(flags "35;50;52;60;61;70;75") + set(flags "35;50;52;60;61;70;75") elseif(CUDA_VERSION_MAJOR EQUAL 9) set(flags "35;50;52;60;61;70") else() @@ -69,12 +80,12 @@ function(format_gencode_flags flags out) endif() # Generate SASS foreach(ver ${flags}) - set(${out} "${${out}}-gencode arch=compute_${ver},code=sm_${ver};") + set(${out} "${${out}}--generate-code=arch=compute_${ver},code=sm_${ver};") endforeach() # Generate PTX for last architecture list(GET flags -1 ver) - set(${out} "${${out}}-gencode arch=compute_${ver},code=compute_${ver};") - + set(${out} "${${out}}--generate-code=arch=compute_${ver},code=compute_${ver};") + set(${out} "${${out}}" PARENT_SCOPE) endfunction(format_gencode_flags flags) @@ -82,6 +93,10 @@ endfunction(format_gencode_flags flags) # if necessary, installs the main R package dependencies; # runs R CMD INSTALL. function(setup_rpackage_install_target rlib_target build_dir) + # backup cmake_install.cmake + install(CODE "file(COPY \"${build_dir}/R-package/cmake_install.cmake\" +DESTINATION \"${build_dir}/bak\")") + install(CODE "file(REMOVE_RECURSE \"${build_dir}/R-package\")") install( DIRECTORY "${PROJECT_SOURCE_DIR}/R-package" @@ -100,4 +115,8 @@ function(setup_rpackage_install_target rlib_target build_dir) install(CODE "execute_process(COMMAND \"${LIBR_EXECUTABLE}\" \"-q\" \"-e\" \"${XGB_DEPS_SCRIPT}\")") install(CODE "execute_process(COMMAND \"${LIBR_EXECUTABLE}\" CMD INSTALL\ \"--no-multiarch\" \"--build\" \"${build_dir}/R-package\")") + + # restore cmake_install.cmake + install(CODE "file(RENAME \"${build_dir}/bak/cmake_install.cmake\" + \"${build_dir}/R-package/cmake_install.cmake\")") endfunction(setup_rpackage_install_target) diff --git a/cmake/build_config.h.in b/cmake/build_config.h.in deleted file mode 100644 index 76195aeca..000000000 --- a/cmake/build_config.h.in +++ /dev/null @@ -1,11 +0,0 @@ -/*! - * Copyright 2019 by Contributors - * \file build_config.h - */ -#ifndef XGBOOST_BUILD_CONFIG_H_ -#define XGBOOST_BUILD_CONFIG_H_ - -#cmakedefine XGBOOST_MM_PREFETCH_PRESENT -#cmakedefine XGBOOST_BUILTIN_PREFETCH_PRESENT - -#endif // XGBOOST_BUILD_CONFIG_H_ diff --git a/cmake/modules/FindNccl.cmake b/cmake/modules/FindNccl.cmake index 2d39abf19..ed1e195a0 100644 --- a/cmake/modules/FindNccl.cmake +++ b/cmake/modules/FindNccl.cmake @@ -37,15 +37,11 @@ set(NCCL_LIB_NAME nccl_static) find_path(NCCL_INCLUDE_DIR NAMES nccl.h - PATHS $ENV{NCCL_ROOT}/include ${NCCL_ROOT}/include ${CUDA_INCLUDE_DIRS} /usr/include) + PATHS $ENV{NCCL_ROOT}/include ${NCCL_ROOT}/include) find_library(NCCL_LIBRARY NAMES ${NCCL_LIB_NAME} - PATHS $ENV{NCCL_ROOT}/lib ${NCCL_ROOT}/lib ${CUDA_INCLUDE_DIRS}/../lib /usr/lib) - -if (NCCL_INCLUDE_DIR AND NCCL_LIBRARY) - get_filename_component(NCCL_LIBRARY ${NCCL_LIBRARY} PATH) -endif () + PATHS $ENV{NCCL_ROOT}/lib/ ${NCCL_ROOT}/lib) include(FindPackageHandleStandardArgs) find_package_handle_standard_args(Nccl DEFAULT_MSG @@ -54,5 +50,4 @@ find_package_handle_standard_args(Nccl DEFAULT_MSG mark_as_advanced( NCCL_INCLUDE_DIR NCCL_LIBRARY - NCCL_LIB_NAME ) diff --git a/cmake/xgboost-config.cmake.in b/cmake/xgboost-config.cmake.in new file mode 100644 index 000000000..6a155f0eb --- /dev/null +++ b/cmake/xgboost-config.cmake.in @@ -0,0 +1,5 @@ +@PACKAGE_INIT@ + +if(NOT TARGET xgboost::xgboost) + include(${CMAKE_CURRENT_LIST_DIR}/XGBoostTargets.cmake) +endif() diff --git a/demo/c-api/README.md b/demo/c-api/README.md new file mode 100644 index 000000000..d94f4aca3 --- /dev/null +++ b/demo/c-api/README.md @@ -0,0 +1,30 @@ +C-APIs +=== + +**XGBoost** implements a C API originally designed for various language +bindings. For detailed reference, please check xgboost/c_api.h. Here is a +demonstration of using the API. + +# CMake +If you use **CMake** for your project, you can either install **XGBoost** +somewhere in your system and tell CMake to find it by calling +`find_package(xgboost)`, or put **XGBoost** inside your project's source tree +and call **CMake** command: `add_subdirectory(xgboost)`. To use +`find_package()`, put the following in your **CMakeLists.txt**: + +``` CMake +find_package(xgboost REQUIRED) +add_executable(api-demo c-api-demo.c) +target_link_libraries(api-demo xgboost::xgboost) +``` + +If you want to put XGBoost inside your project (like git submodule), use this +instead: +``` CMake +add_subdirectory(xgboost) +add_executable(api-demo c-api-demo.c) +target_link_libraries(api-demo xgboost) +``` + +# make +You can start by modifying the makefile in this directory to fit your need. \ No newline at end of file diff --git a/doc/Doxyfile b/doc/Doxyfile.in similarity index 99% rename from doc/Doxyfile rename to doc/Doxyfile.in index 7ec79dace..766034e4f 100644 --- a/doc/Doxyfile +++ b/doc/Doxyfile.in @@ -38,7 +38,7 @@ PROJECT_NAME = "xgboost" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = +PROJECT_NUMBER = @XGBOOST_VERSION@ # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -58,7 +58,7 @@ PROJECT_LOGO = # entered, it will be relative to the location where doxygen was started. If # left blank the current directory will be used. -OUTPUT_DIRECTORY = doc/doxygen +OUTPUT_DIRECTORY = @PROJECT_BINARY_DIR@/doc_doxygen # If the CREATE_SUBDIRS tag is set to YES, then doxygen will create 4096 sub- # directories (in 2 levels) under the output directory of each output format and @@ -753,7 +753,7 @@ WARN_LOGFILE = # spaces. # Note: If this tag is empty the current directory is searched. -INPUT = include src/common +INPUT = @PROJECT_SOURCE_DIR@/include @PROJECT_SOURCE_DIR@/src/common # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/include/xgboost/build_config.h b/include/xgboost/build_config.h index 14bb0785b..6d364a6ff 100644 --- a/include/xgboost/build_config.h +++ b/include/xgboost/build_config.h @@ -5,6 +5,8 @@ #ifndef XGBOOST_BUILD_CONFIG_H_ #define XGBOOST_BUILD_CONFIG_H_ +// These check are for Makefile. +#if !defined(XGBOOST_MM_PREFETCH_PRESENT) && !defined(XGBOOST_BUILTIN_PREFETCH_PRESENT) /* default logic for software pre-fetching */ #if (defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_AMD64))) || defined(__INTEL_COMPILER) // Enable _mm_prefetch for Intel compiler and MSVC+x86 @@ -15,4 +17,6 @@ #define XGBOOST_BUILTIN_PREFETCH_PRESENT #endif // GUARDS +#endif // !defined(XGBOOST_MM_PREFETCH_PRESENT) && !defined() + #endif // XGBOOST_BUILD_CONFIG_H_ diff --git a/include/xgboost/c_api.h b/include/xgboost/c_api.h index 166ba833a..3328aba88 100644 --- a/include/xgboost/c_api.h +++ b/include/xgboost/c_api.h @@ -17,9 +17,6 @@ #include #endif // __cplusplus -// XGBoost C API will include APIs in Rabit C API -#include - #if defined(_MSC_VER) || defined(_WIN32) #define XGB_DLL XGB_EXTERN_C __declspec(dllexport) #else diff --git a/jvm-packages/CMakeLists.txt b/jvm-packages/CMakeLists.txt new file mode 100644 index 000000000..65c0924cf --- /dev/null +++ b/jvm-packages/CMakeLists.txt @@ -0,0 +1,22 @@ +find_package(JNI REQUIRED) + +add_library(xgboost4j SHARED + ${PROJECT_SOURCE_DIR}/jvm-packages/xgboost4j/src/native/xgboost4j.cpp + ${XGBOOST_OBJ_SOURCES}) +target_include_directories(xgboost4j + PRIVATE + ${JNI_INCLUDE_DIRS} + ${PROJECT_SOURCE_DIR}/jvm-packages/xgboost4j/src/native + ${PROJECT_SOURCE_DIR}/include + ${PROJECT_SOURCE_DIR}/dmlc-core/include + ${PROJECT_SOURCE_DIR}/rabit/include) + +set_output_directory(xgboost4j ${PROJECT_SOURCE_DIR}/lib) +set_target_properties( + xgboost4j PROPERTIES + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON) +target_link_libraries(xgboost4j + PRIVATE + ${LINKED_LIBRARIES_PRIVATE} + ${JAVA_JVM_LIBRARY}) diff --git a/jvm-packages/xgboost4j/src/native/xgboost4j.cpp b/jvm-packages/xgboost4j/src/native/xgboost4j.cpp index 5e268093d..285793879 100644 --- a/jvm-packages/xgboost4j/src/native/xgboost4j.cpp +++ b/jvm-packages/xgboost4j/src/native/xgboost4j.cpp @@ -13,6 +13,7 @@ */ #include +#include #include #include #include diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 000000000..89c2d2f57 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,107 @@ +file(GLOB_RECURSE CPU_SOURCES *.cc) +list(REMOVE_ITEM CPU_SOURCES ${PROJECT_SOURCE_DIR}/src/cli_main.cc) + +include(CheckCXXSourceCompiles) +check_cxx_source_compiles(" +#include +int main() { + char data = 0; + const char* address = &data; + _mm_prefetch(address, _MM_HINT_NTA); + return 0; +} +" XGBOOST_MM_PREFETCH_PRESENT) +check_cxx_source_compiles(" +int main() { + char data = 0; + const char* address = &data; + __builtin_prefetch(address, 0, 0); + return 0; +} +" XGBOOST_BUILTIN_PREFETCH_PRESENT) + +# Add plugins to source files +if (PLUGIN_LZ4) + list(APPEND PLUGINS_SOURCES ${PROJECT_SOURCE_DIR}/plugin/lz4/sparse_page_lz4_format.cc) + list(APPEND SRC_LIBS lz4) +endif (PLUGIN_LZ4) +if (PLUGIN_DENSE_PARSER) + list(APPEND PLUGINS_SOURCES ${PROJECT_SOURCE_DIR}/plugin/dense_parser/dense_libsvm.cc) +endif (PLUGIN_DENSE_PARSER) + +#-- Object library +# Object library is necessary for jvm-package, which creates its own shared +# library. +if (USE_CUDA) + file(GLOB_RECURSE CUDA_SOURCES *.cu) + add_library(objxgboost OBJECT ${CPU_SOURCES} ${CUDA_SOURCES} ${PLUGINS_SOURCES}) + target_compile_definitions(objxgboost + PRIVATE -DXGBOOST_USE_CUDA=1) + target_include_directories(objxgboost PRIVATE ${PROJECT_SOURCE_DIR}/cub/) + target_compile_options(objxgboost PRIVATE + $<$:--expt-extended-lambda> + $<$:--expt-relaxed-constexpr> + $<$:-lineinfo> + $<$:--std=c++11> + $<$:${GEN_CODE}>) + + if (USE_NCCL) + find_package(Nccl REQUIRED) + target_include_directories(objxgboost PRIVATE ${NCCL_INCLUDE_DIR}) + target_compile_definitions(objxgboost PRIVATE -DXGBOOST_USE_NCCL=1) + list(APPEND SRC_LIBS ${NCCL_LIBRARY}) + endif (USE_NCCL) + + if (USE_NVTX) + target_include_directories(objxgboost PRIVATE "${NVTX_HEADER_DIR}") + target_compile_definitions(objxgboost PRIVATE -DXGBOOST_USE_NVTX=1) + endif (USE_NVTX) + + set_target_properties(objxgboost PROPERTIES + CUDA_SEPARABLE_COMPILATION OFF) +else (USE_CUDA) + add_library(objxgboost OBJECT ${CPU_SOURCES} ${PLUGINS_SOURCES}) +endif (USE_CUDA) + +target_include_directories(objxgboost + PRIVATE + ${PROJECT_SOURCE_DIR}/include + ${PROJECT_SOURCE_DIR}/dmlc-core/include + ${PROJECT_SOURCE_DIR}/rabit/include) +target_compile_options(objxgboost + PRIVATE + $<$,$>:/MP> + $<$:-funroll-loops>) +if (WIN32 AND MINGW) + target_compile_options(objxgboost PUBLIC -static-libstdc++) +endif (WIN32 AND MINGW) + +set_target_properties(objxgboost PROPERTIES + POSITION_INDEPENDENT_CODE ON + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON) +target_compile_definitions(objxgboost + PRIVATE + -DDMLC_LOG_CUSTOMIZE=1 # enable custom logging + $<$>:_MWAITXINTRIN_H_INCLUDED> + ${XGBOOST_DEFINITIONS}) +if (XGBOOST_MM_PREFETCH_PRESENT) + target_compile_definitions(objxgboost + PRIVATE + -DXGBOOST_MM_PREFETCH_PRESENT=1) +endif(XGBOOST_MM_PREFETCH_PRESENT) +if (XGBOOST_BUILTIN_PREFETCH_PRESENT) + target_compile_definitions(objxgboost + PRIVATE + -DXGBOOST_BUILTIN_PREFETCH_PRESENT=1) +endif (XGBOOST_BUILTIN_PREFETCH_PRESENT) + +if (USE_OPENMP) + find_package(OpenMP REQUIRED) + if (OpenMP_CXX_FOUND OR OPENMP_FOUND) + target_compile_options(objxgboost PRIVATE $<$:${OpenMP_CXX_FLAGS}>) + list(APPEND SRC_LIBS ${OpenMP_CXX_LIBRARIES}) + set(LINKED_LIBRARIES_PRIVATE "${LINKED_LIBRARIES_PRIVATE};${SRC_LIBS}" PARENT_SCOPE) + endif (OpenMP_CXX_FOUND OR OPENMP_FOUND) +endif (USE_OPENMP) +#-- End object library diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc index 2aee319c7..ac9c35c4c 100644 --- a/src/c_api/c_api.cc +++ b/src/c_api/c_api.cc @@ -4,8 +4,11 @@ #include #include #include + #include #include +#include + #include #include #include diff --git a/src/tree/updater_gpu_hist.cu b/src/tree/updater_gpu_hist.cu index e93ca9a35..4a27058fc 100644 --- a/src/tree/updater_gpu_hist.cu +++ b/src/tree/updater_gpu_hist.cu @@ -28,7 +28,9 @@ namespace xgboost { namespace tree { +#if !defined(GTEST_TEST) DMLC_REGISTRY_FILE_TAG(updater_gpu_hist); +#endif // !defined(GTEST_TEST) // training parameters specific to this algorithm struct GPUHistMakerTrainParam @@ -47,8 +49,9 @@ struct GPUHistMakerTrainParam "-1 to use all rows assignted to a GPU, and 0 to auto-deduce"); } }; - +#if !defined(GTEST_TEST) DMLC_REGISTER_PARAMETER(GPUHistMakerTrainParam); +#endif // !defined(GTEST_TEST) struct ExpandEntry { int nid; @@ -102,9 +105,10 @@ inline static bool LossGuide(ExpandEntry lhs, ExpandEntry rhs) { } // Find a gidx value for a given feature otherwise return -1 if not found -__device__ int BinarySearchRow(bst_uint begin, bst_uint end, - common::CompressedIterator data, - int const fidx_begin, int const fidx_end) { +__forceinline__ __device__ int BinarySearchRow( + bst_uint begin, bst_uint end, + common::CompressedIterator data, + int const fidx_begin, int const fidx_end) { bst_uint previous_middle = UINT32_MAX; while (end != begin) { auto middle = begin + (end - begin) / 2; @@ -466,6 +470,7 @@ struct CalcWeightTrainParam { }; // Bin each input data entry, store the bin indices in compressed form. +template::type = 0> __global__ void CompressBinEllpackKernel( common::CompressedBufferWriter wr, common::CompressedByteT* __restrict__ buffer, // gidx_buffer @@ -559,11 +564,11 @@ struct IndicateLeftTransform { * segments. Based on a single pass of exclusive scan, uses iterators to * redirect inputs and outputs. */ -void SortPosition(dh::CubMemory* temp_memory, common::Span position, - common::Span position_out, common::Span ridx, - common::Span ridx_out, int left_nidx, - int right_nidx, int64_t* d_left_count, - cudaStream_t stream = nullptr) { +inline void SortPosition(dh::CubMemory* temp_memory, common::Span position, + common::Span position_out, common::Span ridx, + common::Span ridx_out, int left_nidx, + int right_nidx, int64_t* d_left_count, + cudaStream_t stream = nullptr) { auto d_position_out = position_out.data(); auto d_position_in = position.data(); auto d_ridx_out = ridx_out.data(); @@ -593,7 +598,7 @@ void SortPosition(dh::CubMemory* temp_memory, common::Span position, } /*! \brief Count how many rows are assigned to left node. */ -__device__ void CountLeft(int64_t* d_count, int val, int left_nidx) { +__forceinline__ __device__ void CountLeft(int64_t* d_count, int val, int left_nidx) { unsigned ballot = __ballot(val == left_nidx); if (threadIdx.x % 32 == 0) { atomicAdd(reinterpret_cast(d_count), // NOLINT @@ -1614,8 +1619,11 @@ class GPUHistMaker : public TreeUpdater { std::unique_ptr> double_maker_; }; +#if !defined(GTEST_TEST) XGBOOST_REGISTER_TREE_UPDATER(GPUHistMaker, "grow_gpu_hist") .describe("Grow tree with GPU.") .set_body([]() { return new GPUHistMaker(); }); +#endif // !defined(GTEST_TEST) + } // namespace tree } // namespace xgboost diff --git a/tests/ci_build/Dockerfile.clang_tidy b/tests/ci_build/Dockerfile.clang_tidy index 1d09fce96..ec68e2c66 100644 --- a/tests/ci_build/Dockerfile.clang_tidy +++ b/tests/ci_build/Dockerfile.clang_tidy @@ -7,7 +7,9 @@ ENV DEBIAN_FRONTEND noninteractive # Install all basic requirements RUN \ apt-get update && \ - apt-get install -y tar unzip wget git build-essential cmake python3 python3-pip llvm-7 clang-tidy-7 clang-7 + apt-get install -y tar unzip wget git build-essential python3 python3-pip llvm-7 clang-tidy-7 clang-7 && \ + wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \ + bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr # Set default clang-tidy version RUN \ diff --git a/tests/ci_build/Dockerfile.gpu b/tests/ci_build/Dockerfile.gpu index a97b4043c..1bf705558 100644 --- a/tests/ci_build/Dockerfile.gpu +++ b/tests/ci_build/Dockerfile.gpu @@ -15,8 +15,8 @@ RUN \ wget https://repo.continuum.io/miniconda/Miniconda2-4.3.27-Linux-x86_64.sh && \ bash Miniconda2-4.3.27-Linux-x86_64.sh -b -p /opt/python && \ # CMake - wget -nv -nc https://cmake.org/files/v3.6/cmake-3.6.0-Linux-x86_64.sh --no-check-certificate && \ - bash cmake-3.6.0-Linux-x86_64.sh --skip-license --prefix=/usr + wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \ + bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr # NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html) RUN \ diff --git a/tests/ci_build/build_via_cmake.sh b/tests/ci_build/build_via_cmake.sh index f418a9ff7..46bdd152c 100755 --- a/tests/ci_build/build_via_cmake.sh +++ b/tests/ci_build/build_via_cmake.sh @@ -14,7 +14,7 @@ rm -rf release-1.7.0.zip* rm -rf build mkdir build cd build -cmake .. "$@" -DGOOGLE_TEST=ON -DGTEST_ROOT=$PWD/../gtest +cmake .. "$@" -DGOOGLE_TEST=ON -DGTEST_ROOT=$PWD/../gtest -DCMAKE_VERBOSE_MAKEFILE=ON make clean make -j cd .. diff --git a/tests/ci_build/test_gpu.sh b/tests/ci_build/test_gpu.sh index d82485b22..0347a1538 100755 --- a/tests/ci_build/test_gpu.sh +++ b/tests/ci_build/test_gpu.sh @@ -5,4 +5,8 @@ cd python-package python setup.py install --user cd .. pytest -v -s --fulltrace -m "(not mgpu) and (not slow)" tests/python-gpu -./build/testxgboost --gtest_filter=-*.MGPU_* +pushd . +cd build +./testxgboost --gtest_filter=-*.MGPU_* +ctest --output-on-failure --tests-regex "TestXGBoostCLI" +popd diff --git a/tests/ci_build/tidy.py b/tests/ci_build/tidy.py index a9b175da0..c6cb5ccf0 100755 --- a/tests/ci_build/tidy.py +++ b/tests/ci_build/tidy.py @@ -60,7 +60,7 @@ class ClangTidy(object): '''Run CMake to generate compilation database.''' os.mkdir(self.cdb_path) os.chdir(self.cdb_path) - cmake_args = ['cmake', '..', '-DGENERATE_COMPILATION_DATABASE=ON', + cmake_args = ['cmake', '..', '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', '-DGOOGLE_TEST=ON', '-DGTEST_ROOT={}'.format( self.gtest_path)] if self.cuda_lint: @@ -68,23 +68,64 @@ class ClangTidy(object): subprocess.run(cmake_args) os.chdir(self.root_path) + def convert_nvcc_command_to_clang(self, command): + '''Convert nvcc flags to corresponding clang flags.''' + components = command.split() + compiler: str = components[0] + if compiler.find('nvcc') != -1: + compiler = 'clang++' + components[0] = compiler + # check each component in a command + converted_components = [compiler] + + for i in range(len(components)): + if components[i] == '-lineinfo': + continue + elif components[i] == '-fuse-ld=gold': + continue + elif components[i] == '-rdynamic': + continue + elif (components[i] == '-x' and + components[i+1] == 'cu'): + # -x cu -> -x cuda + converted_components.append('-x') + converted_components.append('cuda') + components[i+1] = '' + continue + elif components[i].find('-Xcompiler') != -1: + continue + elif components[i].find('--expt') != -1: + continue + elif components[i].find('-ccbin') != -1: + continue + elif components[i].find('--generate-code') != -1: + keyword = 'code=sm' + pos = components[i].find(keyword) + capability = components[i][pos + len(keyword) + 1: + pos + len(keyword) + 3] + if pos != -1: + converted_components.append( + '--cuda-gpu-arch=sm_' + capability) + elif components[i].find('--std=c++11') != -1: + converted_components.append('-std=c++11') + else: + converted_components.append(components[i]) + + command = '' + for c in converted_components: + command = command + ' ' + c + command = command.strip() + return command + def _configure_flags(self, path, command): common_args = ['clang-tidy', "-header-filter='(xgboost\\/src|xgboost\\/include)'", '-config='+self.clang_tidy] common_args.append(path) common_args.append('--') + command = self.convert_nvcc_command_to_clang(command) command = command.split()[1:] # remove clang/c++/g++ - # filter out not used flags - if '-fuse-ld=gold' in command: - command.remove('-fuse-ld=gold') - if '-rdynamic' in command: - command.remove('-rdynamic') - if '-Xcompiler=-fPIC' in command: - command.remove('-Xcompiler=-fPIC') - if '-Xcompiler=-fPIE' in command: - command.remove('-Xcompiler=-fPIE') if '-c' in command: index = command.index('-c') del command[index+1] diff --git a/tests/cli/machine.conf.in b/tests/cli/machine.conf.in new file mode 100644 index 000000000..acac74f93 --- /dev/null +++ b/tests/cli/machine.conf.in @@ -0,0 +1,13 @@ +# Originally an example in demo/regression/ +booster = gbtree +objective = reg:linear +eta = 1.0 +gamma = 1.0 +seed = 0 +min_child_weight = 0 +max_depth = 3 + +num_round = 2 +save_period = 0 +data = "@PROJECT_SOURCE_DIR@/demo/data/agaricus.txt.train" +eval[test] = "@PROJECT_SOURCE_DIR@/demo/data/agaricus.txt.test" \ No newline at end of file diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt new file mode 100644 index 000000000..bd303917b --- /dev/null +++ b/tests/cpp/CMakeLists.txt @@ -0,0 +1,56 @@ +find_package(GTest REQUIRED) +file(GLOB_RECURSE TEST_SOURCES "*.cc") + +if (USE_CUDA) + file(GLOB_RECURSE CUDA_TEST_SOURCES "*.cu") + list(APPEND TEST_SOURCES ${CUDA_TEST_SOURCES}) +endif (USE_CUDA) +add_executable(testxgboost ${TEST_SOURCES} ${XGBOOST_OBJ_SOURCES}) + +if (USE_CUDA) + target_include_directories(testxgboost PRIVATE + ${PROJECT_SOURCE_DIR}/cub/) + target_compile_options(testxgboost PRIVATE + $<$:--expt-extended-lambda> + $<$:--expt-relaxed-constexpr> + $<$:-lineinfo> + $<$:--std=c++11> + $<$:${GEN_CODE}>) + target_compile_definitions(testxgboost + PRIVATE -DXGBOOST_USE_CUDA=1) + set_target_properties(testxgboost PROPERTIES + CUDA_SEPARABLE_COMPILATION OFF) + + if (USE_NCCL) + find_package(Nccl REQUIRED) + target_include_directories(testxgboost PRIVATE ${NCCL_INCLUDE_DIR}) + target_compile_definitions(testxgboost PRIVATE -DXGBOOST_USE_NCCL=1) + target_link_libraries(testxgboost PRIVATE ${NCCL_LIBRARY}) + endif (USE_NCCL) + + if (USE_NVTX) + target_include_directories(testxgboost PRIVATE "${NVTX_HEADER_DIR}") + target_compile_definitions(testxgboost PRIVATE -DXGBOOST_USE_NVTX=1) + endif (USE_NVTX) +endif (USE_CUDA) + +target_include_directories(testxgboost + PRIVATE + ${GTEST_INCLUDE_DIRS} + ${PROJECT_SOURCE_DIR}/include + ${PROJECT_SOURCE_DIR}/dmlc-core/include + ${PROJECT_SOURCE_DIR}/rabit/include) +set_target_properties( + testxgboost PROPERTIES + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON) +target_link_libraries(testxgboost + PRIVATE + ${GTEST_LIBRARIES} + ${LINKED_LIBRARIES_PRIVATE} + ${OpenMP_CXX_LIBRARIES}) +target_compile_definitions(testxgboost PRIVATE ${XGBOOST_DEFINITIONS}) +if (USE_OPENMP) + target_compile_options(testxgboost PRIVATE $<$:${OpenMP_CXX_FLAGS}>) +endif (USE_OPENMP) +set_output_directory(testxgboost ${PROJECT_BINARY_DIR}) diff --git a/tests/cpp/common/test_transform_range.cu b/tests/cpp/common/test_transform_range.cu index abd11e86b..45f1f3122 100644 --- a/tests/cpp/common/test_transform_range.cu +++ b/tests/cpp/common/test_transform_range.cu @@ -43,7 +43,7 @@ template struct TestTransformRangeGranular { const size_t granularity = 8; - TestTransformRangeGranular(const size_t granular) : granularity{granular} {} + explicit TestTransformRangeGranular(const size_t granular) : granularity{granular} {} void XGBOOST_DEVICE operator()(size_t _idx, Span _out, Span _in) { auto in_sub = _in.subspan(_idx * granularity, granularity); @@ -105,6 +105,6 @@ TEST(Transform, MGPU_SpecifiedGpuId) { ASSERT_TRUE(std::equal(h_sol.begin(), h_sol.end(), res.begin())); } -} // namespace xgboost } // namespace common +} // namespace xgboost #endif \ No newline at end of file diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index c30ef0399..325206bab 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -3,7 +3,12 @@ if [ ${TASK} == "lint" ]; then make lint || exit -1 echo "Check documentations..." - make doxygen 2>log.txt + + mkdir build_doc + cd build_doc + cmake .. -DBUILD_C_DOC=ON + make doc_doxygen 2> log.txt + (cat log.txt| grep -v ENABLE_PREPROCESSING |grep -v "unsupported tag") > logclean.txt echo "---------Error Log----------" cat logclean.txt @@ -11,6 +16,9 @@ if [ ${TASK} == "lint" ]; then (cat logclean.txt|grep warning) && exit -1 (cat logclean.txt|grep error) && exit -1 + cd - + rm -rf build_doc + exit 0 fi