Refactor CMake scripts. (#4323)

* Refactor CMake scripts. * Remove CMake CUDA wrapper. * Bump CMake version for CUDA. * Use CMake to handle Doxygen. * Split up CMakeList. * Export install target. * Use modern CMake. * Remove build.sh * Workaround for gpu_hist test. * Use cmake 3.12. * Revert machine.conf. * Move CLI test to gpu. * Small cleanup. * Support using XGBoost as submodule. * Fix windows * Fix cpp tests on Windows * Remove duplicated find_package.
2019-04-16 01:08:12 +08:00
parent 84d992babc
commit 207f058711
28 changed files with 578 additions and 429 deletions
--- a/tests/ci_build/Dockerfile.clang_tidy
+++ b/tests/ci_build/Dockerfile.clang_tidy
@@ -7,7 +7,9 @@ ENV DEBIAN_FRONTEND noninteractive
 # Install all basic requirements
 RUN \
    apt-get update && \
-    apt-get install -y tar unzip wget git build-essential cmake python3 python3-pip llvm-7 clang-tidy-7 clang-7
+    apt-get install -y tar unzip wget git build-essential python3 python3-pip llvm-7 clang-tidy-7 clang-7 && \
+    wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
+    bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr

 # Set default clang-tidy version
 RUN \
--- a/tests/ci_build/Dockerfile.gpu
+++ b/tests/ci_build/Dockerfile.gpu
@@ -15,8 +15,8 @@ RUN \
    wget https://repo.continuum.io/miniconda/Miniconda2-4.3.27-Linux-x86_64.sh && \
    bash Miniconda2-4.3.27-Linux-x86_64.sh -b -p /opt/python && \
    # CMake
-    wget -nv -nc https://cmake.org/files/v3.6/cmake-3.6.0-Linux-x86_64.sh --no-check-certificate && \
-    bash cmake-3.6.0-Linux-x86_64.sh --skip-license --prefix=/usr
+    wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
+    bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr

 # NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
 RUN \
--- a/tests/ci_build/build_via_cmake.sh
+++ b/tests/ci_build/build_via_cmake.sh
@@ -14,7 +14,7 @@ rm -rf release-1.7.0.zip*
 rm -rf build
 mkdir build
 cd build
-cmake .. "$@" -DGOOGLE_TEST=ON -DGTEST_ROOT=$PWD/../gtest
+cmake .. "$@" -DGOOGLE_TEST=ON -DGTEST_ROOT=$PWD/../gtest -DCMAKE_VERBOSE_MAKEFILE=ON
 make clean
 make -j
 cd ..
--- a/tests/ci_build/test_gpu.sh
+++ b/tests/ci_build/test_gpu.sh
@@ -5,4 +5,8 @@ cd python-package
 python setup.py install --user
 cd ..
 pytest -v -s --fulltrace -m "(not mgpu) and (not slow)" tests/python-gpu
-./build/testxgboost --gtest_filter=-*.MGPU_*
+pushd .
+cd build
+./testxgboost --gtest_filter=-*.MGPU_*
+ctest --output-on-failure --tests-regex "TestXGBoostCLI"
+popd
--- a/tests/ci_build/tidy.py
+++ b/tests/ci_build/tidy.py
@@ -60,7 +60,7 @@ class ClangTidy(object):
        '''Run CMake to generate compilation database.'''
        os.mkdir(self.cdb_path)
        os.chdir(self.cdb_path)
-        cmake_args = ['cmake', '..', '-DGENERATE_COMPILATION_DATABASE=ON',
+        cmake_args = ['cmake', '..', '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
                      '-DGOOGLE_TEST=ON', '-DGTEST_ROOT={}'.format(
                          self.gtest_path)]
        if self.cuda_lint:
@@ -68,23 +68,64 @@ class ClangTidy(object):
        subprocess.run(cmake_args)
        os.chdir(self.root_path)

+    def convert_nvcc_command_to_clang(self, command):
+        '''Convert nvcc flags to corresponding clang flags.'''
+        components = command.split()
+        compiler: str = components[0]
+        if compiler.find('nvcc') != -1:
+            compiler = 'clang++'
+            components[0] = compiler
+        # check each component in a command
+        converted_components = [compiler]
+
+        for i in range(len(components)):
+            if components[i] == '-lineinfo':
+                continue
+            elif components[i] == '-fuse-ld=gold':
+                continue
+            elif components[i] == '-rdynamic':
+                continue
+            elif (components[i] == '-x' and
+                  components[i+1] == 'cu'):
+                # -x cu -> -x cuda
+                converted_components.append('-x')
+                converted_components.append('cuda')
+                components[i+1] = ''
+                continue
+            elif components[i].find('-Xcompiler') != -1:
+                continue
+            elif components[i].find('--expt') != -1:
+                continue
+            elif components[i].find('-ccbin') != -1:
+                continue
+            elif components[i].find('--generate-code') != -1:
+                keyword = 'code=sm'
+                pos = components[i].find(keyword)
+                capability = components[i][pos + len(keyword) + 1:
+                                           pos + len(keyword) + 3]
+                if pos != -1:
+                    converted_components.append(
+                        '--cuda-gpu-arch=sm_' + capability)
+            elif components[i].find('--std=c++11') != -1:
+                converted_components.append('-std=c++11')
+            else:
+                converted_components.append(components[i])
+
+        command = ''
+        for c in converted_components:
+            command = command + ' ' + c
+        command = command.strip()
+        return command
+
    def _configure_flags(self, path, command):
        common_args = ['clang-tidy',
                       "-header-filter='(xgboost\\/src|xgboost\\/include)'",
                       '-config='+self.clang_tidy]
        common_args.append(path)
        common_args.append('--')
+        command = self.convert_nvcc_command_to_clang(command)

        command = command.split()[1:]  # remove clang/c++/g++
-        # filter out not used flags
-        if '-fuse-ld=gold' in command:
-            command.remove('-fuse-ld=gold')
-        if '-rdynamic' in command:
-            command.remove('-rdynamic')
-        if '-Xcompiler=-fPIC' in command:
-            command.remove('-Xcompiler=-fPIC')
-        if '-Xcompiler=-fPIE' in command:
-            command.remove('-Xcompiler=-fPIE')
        if '-c' in command:
            index = command.index('-c')
            del command[index+1]
--- a/tests/cli/machine.conf.in
+++ b/tests/cli/machine.conf.in
@@ -0,0 +1,13 @@
+# Originally an example in demo/regression/
+booster = gbtree
+objective = reg:linear
+eta = 1.0
+gamma = 1.0
+seed = 0
+min_child_weight = 0
+max_depth = 3
+
+num_round = 2
+save_period = 0
+data = "@PROJECT_SOURCE_DIR@/demo/data/agaricus.txt.train"
+eval[test] = "@PROJECT_SOURCE_DIR@/demo/data/agaricus.txt.test"
--- a/tests/cpp/CMakeLists.txt
+++ b/tests/cpp/CMakeLists.txt
@@ -0,0 +1,56 @@
+find_package(GTest REQUIRED)
+file(GLOB_RECURSE TEST_SOURCES "*.cc")
+
+if (USE_CUDA)
+  file(GLOB_RECURSE CUDA_TEST_SOURCES "*.cu")
+  list(APPEND TEST_SOURCES ${CUDA_TEST_SOURCES})
+endif (USE_CUDA)
+add_executable(testxgboost ${TEST_SOURCES} ${XGBOOST_OBJ_SOURCES})
+
+if (USE_CUDA)
+  target_include_directories(testxgboost PRIVATE
+    ${PROJECT_SOURCE_DIR}/cub/)
+  target_compile_options(testxgboost PRIVATE
+    $<$<COMPILE_LANGUAGE:CUDA>:--expt-extended-lambda>
+    $<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr>
+    $<$<COMPILE_LANGUAGE:CUDA>:-lineinfo>
+    $<$<COMPILE_LANGUAGE:CUDA>:--std=c++11>
+    $<$<COMPILE_LANGUAGE:CUDA>:${GEN_CODE}>)
+  target_compile_definitions(testxgboost
+    PRIVATE -DXGBOOST_USE_CUDA=1)
+  set_target_properties(testxgboost PROPERTIES
+    CUDA_SEPARABLE_COMPILATION OFF)
+
+  if (USE_NCCL)
+    find_package(Nccl REQUIRED)
+    target_include_directories(testxgboost PRIVATE ${NCCL_INCLUDE_DIR})
+    target_compile_definitions(testxgboost PRIVATE -DXGBOOST_USE_NCCL=1)
+    target_link_libraries(testxgboost PRIVATE ${NCCL_LIBRARY})
+  endif (USE_NCCL)
+
+  if (USE_NVTX)
+    target_include_directories(testxgboost PRIVATE "${NVTX_HEADER_DIR}")
+    target_compile_definitions(testxgboost PRIVATE -DXGBOOST_USE_NVTX=1)
+  endif (USE_NVTX)
+endif (USE_CUDA)
+
+target_include_directories(testxgboost
+  PRIVATE
+  ${GTEST_INCLUDE_DIRS}
+  ${PROJECT_SOURCE_DIR}/include
+  ${PROJECT_SOURCE_DIR}/dmlc-core/include
+  ${PROJECT_SOURCE_DIR}/rabit/include)
+set_target_properties(
+  testxgboost PROPERTIES
+  CXX_STANDARD 11
+  CXX_STANDARD_REQUIRED ON)
+target_link_libraries(testxgboost
+  PRIVATE
+  ${GTEST_LIBRARIES}
+  ${LINKED_LIBRARIES_PRIVATE}
+  ${OpenMP_CXX_LIBRARIES})
+target_compile_definitions(testxgboost PRIVATE ${XGBOOST_DEFINITIONS})
+if (USE_OPENMP)
+  target_compile_options(testxgboost PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${OpenMP_CXX_FLAGS}>)
+endif (USE_OPENMP)
+set_output_directory(testxgboost ${PROJECT_BINARY_DIR})
--- a/tests/cpp/common/test_transform_range.cu
+++ b/tests/cpp/common/test_transform_range.cu
@@ -43,7 +43,7 @@ template <typename T>
 struct TestTransformRangeGranular {
  const size_t granularity = 8;

-  TestTransformRangeGranular(const size_t granular) : granularity{granular} {}
+  explicit TestTransformRangeGranular(const size_t granular) : granularity{granular} {}
  void XGBOOST_DEVICE operator()(size_t _idx,
                                 Span<bst_float> _out, Span<const bst_float> _in) {
    auto in_sub = _in.subspan(_idx * granularity, granularity);
@@ -105,6 +105,6 @@ TEST(Transform, MGPU_SpecifiedGpuId) {
  ASSERT_TRUE(std::equal(h_sol.begin(), h_sol.end(), res.begin()));
 }

-}  // namespace xgboost
 }  // namespace common
+}  // namespace xgboost
 #endif
--- a/tests/travis/run_test.sh
+++ b/tests/travis/run_test.sh
@@ -3,7 +3,12 @@
 if [ ${TASK} == "lint" ]; then
    make lint || exit -1
    echo "Check documentations..."
-    make doxygen 2>log.txt
+
+    mkdir build_doc
+    cd build_doc
+    cmake .. -DBUILD_C_DOC=ON
+    make doc_doxygen 2> log.txt
+
    (cat log.txt| grep -v ENABLE_PREPROCESSING |grep -v "unsupported tag") > logclean.txt
    echo "---------Error Log----------"
    cat logclean.txt
@@ -11,6 +16,9 @@ if [ ${TASK} == "lint" ]; then
    (cat logclean.txt|grep warning) && exit -1
    (cat logclean.txt|grep error) && exit -1

+    cd -
+    rm -rf build_doc
+
    exit 0
 fi