Ensure that configured dmlc/build_config.h is picked up by Rabit and XGBoost (#5514)

* Ensure that configured header (build_config.h) from dmlc-core is picked up by Rabit and XGBoost

* Check which Rabit target is being used

* Use CMake 3.13 in all Jenkins tests

* Upgrade CMake in Travis CI

* Install CMake using Kitware installer

* Remove existing CMake (3.12.4)

CMakeLists.txt

@@ -1,8 +1,9 @@
-cmake_minimum_required(VERSION 3.12)
+cmake_minimum_required(VERSION 3.13)
 project(xgboost LANGUAGES CXX C VERSION 1.1.0)
 include(cmake/Utils.cmake)
 list(APPEND CMAKE_MODULE_PATH "${xgboost_SOURCE_DIR}/cmake/modules")
 cmake_policy(SET CMP0022 NEW)
+cmake_policy(SET CMP0079 NEW)
 
 if ((${CMAKE_VERSION} VERSION_GREATER 3.13) OR (${CMAKE_VERSION} VERSION_EQUAL 3.13))
   cmake_policy(SET CMP0077 NEW)
@@ -123,6 +124,13 @@ if (RABIT_MOCK)
 else()
   list(APPEND LINKED_LIBRARIES_PRIVATE rabit)
 endif(RABIT_MOCK)
+foreach(lib rabit rabit_base rabit_empty rabit_mock rabit_mock_static)
+  # Explicitly link dmlc to rabit, so that configured header (build_config.h)
+  # from dmlc is correctly applied to rabit.
+  if (TARGET ${lib})
+    target_link_libraries(${lib} dmlc)
+  endif (TARGET ${lib})
+endforeach()
 
 # Exports some R specific definitions and objects
 if (R_LIB)
@@ -132,6 +140,7 @@ endif (R_LIB)
 # core xgboost
 add_subdirectory(${xgboost_SOURCE_DIR}/plugin)
 add_subdirectory(${xgboost_SOURCE_DIR}/src)
+target_link_libraries(objxgboost PUBLIC dmlc)
 set(XGBOOST_OBJ_SOURCES "${XGBOOST_OBJ_SOURCES};$<TARGET_OBJECTS:objxgboost>")
 
 #-- library

Jenkinsfile

@@ -88,7 +88,6 @@ pipeline {
             'test-jvm-jdk8': { CrossTestJVMwithJDK(jdk_version: '8', spark_version: '2.4.3') },
             'test-jvm-jdk11': { CrossTestJVMwithJDK(jdk_version: '11') },
             'test-jvm-jdk12': { CrossTestJVMwithJDK(jdk_version: '12') },
-            'test-r-3.4.4': { TestR(use_r35: false) },
             'test-r-3.5.3': { TestR(use_r35: true) }
           ])
         }
@@ -176,17 +175,21 @@ def BuildCPU() {
     def container_type = "cpu"
     def docker_binary = "docker"
     sh """
+    ${dockerRun} ${container_type} ${docker_binary} rm -fv dmlc-core/include/dmlc/build_config_default.h
+      # This step is not necessary, but here we include it, to ensure that DMLC_CORE_USE_CMAKE flag is correctly propagated
+      # We want to make sure that we use the configured header build/dmlc/build_config.h instead of include/dmlc/build_config_default.h.
+      # See discussion at https://github.com/dmlc/xgboost/issues/5510
     ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh
     ${dockerRun} ${container_type} ${docker_binary} build/testxgboost
     """
     // Sanitizer test
     def docker_extra_params = "CI_DOCKER_EXTRA_PARAMS_INIT='-e ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer -e ASAN_OPTIONS=symbolize=1 -e UBSAN_OPTIONS=print_stacktrace=1:log_path=ubsan_error.log --cap-add SYS_PTRACE'"
-    def docker_args = "--build-arg CMAKE_VERSION=3.12"
     sh """
-    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_SANITIZER=ON -DENABLED_SANITIZERS="address;leak;undefined" \
+    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh -DUSE_SANITIZER=ON -DENABLED_SANITIZERS="address;leak;undefined" \
       -DCMAKE_BUILD_TYPE=Debug -DSANITIZER_PATH=/usr/lib/x86_64-linux-gnu/
     ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} build/testxgboost
     """
+
     deleteDir()
   }
 }

dmlc-core

@@ -1 +1 @@
-Subproject commit 981b1c32f91668e669ee376856f92f36cfd2a351
+Subproject commit 5df8305fe699d3b503d10c60a231ab0223142407

tests/ci_build/Dockerfile.clang_tidy

@@ -13,8 +13,8 @@ RUN \
     add-apt-repository -u 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main' && \
     apt-get update && \
     apt-get install -y llvm-10 clang-tidy-10 clang-10 && \
-    wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
+    wget -nv -nc https://cmake.org/files/v3.13/cmake-3.13.0-Linux-x86_64.sh --no-check-certificate && \
-    bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr
+    bash cmake-3.13.0-Linux-x86_64.sh --skip-license --prefix=/usr
 
 # Set default clang-tidy version
 RUN \

tests/ci_build/Dockerfile.cpu

@@ -1,5 +1,4 @@
 FROM ubuntu:18.04
-ARG CMAKE_VERSION=3.12
 
 # Environment
 ENV DEBIAN_FRONTEND noninteractive
@@ -10,11 +9,11 @@ RUN \
     apt-get update && \
     apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libasan2 libidn11 && \
     # CMake
-    wget -nv -nc https://cmake.org/files/v$CMAKE_VERSION/cmake-$CMAKE_VERSION.0-Linux-x86_64.sh --no-check-certificate && \
+    wget -nv -nc https://cmake.org/files/v3.13/cmake-3.13.0-Linux-x86_64.sh --no-check-certificate && \
-    bash cmake-$CMAKE_VERSION.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
+    bash cmake-3.13.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
     # Python
-    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
+    wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
-    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python
+    bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/python
 
 ENV PATH=/opt/python/bin:$PATH
 

tests/ci_build/Dockerfile.gpu_build

@@ -20,8 +20,8 @@ RUN \
     wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
     bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \
     # CMake
-    wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
+    wget -nv -nc https://cmake.org/files/v3.13/cmake-3.13.0-Linux-x86_64.sh --no-check-certificate && \
-    bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr
+    bash cmake-3.13.0-Linux-x86_64.sh --skip-license --prefix=/usr
 
 # NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
 RUN \

tests/ci_build/Dockerfile.jvm

@@ -11,8 +11,8 @@ RUN \
     wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
     bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \
     # CMake
-    wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
+    wget -nv -nc https://cmake.org/files/v3.13/cmake-3.13.0-Linux-x86_64.sh --no-check-certificate && \
-    bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
+    bash cmake-3.13.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
     # Maven
     wget https://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \
     tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \

tests/ci_build/Dockerfile.rproject

@@ -15,8 +15,8 @@ RUN \
     fi && \
     apt-get install -y r-base r-base-core r-recommended && \
     # CMake
-    wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
+    wget -nv -nc https://cmake.org/files/v3.13/cmake-3.13.0-Linux-x86_64.sh --no-check-certificate && \
-    bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr
+    bash cmake-3.13.0-Linux-x86_64.sh --skip-license --prefix=/usr
 
 # Use 16 workers to compile R packages
 ENV MAKE 'make -j16'

tests/travis/setup.sh

@@ -19,3 +19,10 @@ fi
 if [ ${TASK} == "cmake_test" ] && [ ${TRAVIS_OS_NAME} == "osx" ]; then
     sudo softwareupdate -i "Command Line Tools (macOS High Sierra version 10.13) for Xcode-9.3"
 fi
+
+if [ ${TASK} == "python_sdist_test" ] && [ ${TRAVIS_OS_NAME} == "linux" ]; then
+    wget https://github.com/Kitware/CMake/releases/download/v3.17.1/cmake-3.17.1-Linux-x86_64.sh
+    sudo bash cmake-3.17.1-Linux-x86_64.sh --prefix=/usr/local --skip-license
+    sudo rm -rf /usr/local/cmake-3.12.4  # Remove existing CMake
+    cmake --version
+fi