[CI] Refactor Jenkins CI pipeline + migrate all Linux tests to Jenkins (#4401)

* All Linux tests are now in Jenkins CI
* Tests are now de-coupled from builds. We can now build XGBoost with one version of CUDA/JDK and test it with another version of CUDA/JDK
* Builds (compilation) are significantly faster because 1) They use C5 instances with faster CPU cores; and 2) build environment setup is cached using Docker containers

.travis.yml

@@ -3,7 +3,6 @@ sudo: required
 
 # Enabling test on Linux and OS X
 os:
-  - linux
   - osx
 
 osx_image: xcode9.3
@@ -11,61 +10,15 @@ osx_image: xcode9.3
 # Use Build Matrix to do lint and build seperately
 env:
   matrix:
-    # code lint
-    - TASK=lint
-    # r package test
-    - TASK=r_test
     # python package test
     - TASK=python_test
-    - TASK=python_lightweight_test
     # java package test
     - TASK=java_test
     # cmake test
-    - TASK=cmake_test
+    # - TASK=cmake_test
-    # c++ test
-    - TASK=cpp_test
-    # distributed test
-    - TASK=distributed_test
-    # address sanitizer test
-    - TASK=sanitizer_test
-
-matrix:
-  exclude:
-    - os: osx
-      env: TASK=lint
-    - os: osx
-      env: TASK=cmake_test
-    - os: linux
-      env: TASK=r_test
-    - os: osx
-      env: TASK=python_lightweight_test
-    - os: osx
-      env: TASK=cpp_test
-    - os: osx
-      env: TASK=distributed_test
-    - os: osx
-      env: TASK=sanitizer_test
 
 # dependent apt packages
 addons:
-  apt:
-    sources:
-      - llvm-toolchain-trusty-5.0
-      - ubuntu-toolchain-r-test
-      - george-edison55-precise-backports
-    packages:
-      - clang
-      - clang-tidy-5.0
-      - cmake-data
-      - doxygen
-      - wget
-      - libcurl4-openssl-dev
-      - unzip
-      - graphviz
-      - gcc-5
-      - g++-5
-      - gcc-7
-      - g++-7
   homebrew:
     packages:
       - gcc@7

Jenkinsfile

@@ -3,22 +3,8 @@
 // Jenkins pipeline
 // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
 
-import groovy.transform.Field
-
-/* Unrestricted tasks: tasks that do NOT generate artifacts */
-
 // Command to run command inside a docker container
-def dockerRun = 'tests/ci_build/ci_build.sh'
+dockerRun = 'tests/ci_build/ci_build.sh'
-// Utility functions
-@Field
-def utils
-
-def buildMatrix = [
-    [ "enabled": true,  "os" : "linux", "withGpu": true, "withNccl": true,  "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "10.0", "multiGpu": true],
-    [ "enabled": true,  "os" : "linux", "withGpu": true, "withNccl": true,  "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "9.2" ],
-    [ "enabled": true,  "os" : "linux", "withGpu": true, "withNccl": true,  "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "8.0" ],
-    [ "enabled": true,  "os" : "linux", "withGpu": true, "withNccl": false, "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "8.0" ],
-]
 
 pipeline {
   // Each stage specify its own agent
@@ -34,98 +20,308 @@ pipeline {
     timestamps()
     timeout(time: 120, unit: 'MINUTES')
     buildDiscarder(logRotator(numToKeepStr: '10'))
+    preserveStashes()
   }
 
   // Build stages
   stages {
-        stage('Jenkins: Get sources') {
+    stage('Get sources') {
-            agent {
+      agent { label 'linux && cpu' }
-                label 'unrestricted'
-            }
       steps {
         script {
-                    utils = load('tests/ci_build/jenkins_tools.Groovy')
+          checkoutSrcs()
-                    utils.checkoutSrcs()
         }
-                stash name: 'srcs', excludes: '.git/'
+        stash name: 'srcs'
-                milestone label: 'Sources ready', ordinal: 1
+        milestone ordinal: 1
       }
     }
-        stage('Jenkins: Build & Test') {
+    stage('Formatting Check') {
+      agent none
       steps {
         script {
-                    parallel (buildMatrix.findAll{it['enabled']}.collectEntries{ c ->
+          parallel ([
-                        def buildName = utils.getBuildName(c)
+            'clang-tidy': { ClangTidy() },
-                        utils.buildFactory(buildName, c, false, this.&buildPlatformCmake)
+            'lint': { Lint() },
-                    } + [ "clang-tidy" : { buildClangTidyJob() } ])
+            'sphinx-doc': { SphinxDoc() },
+            'doxygen': { Doxygen() }
+          ])
         }
+        milestone ordinal: 2
+      }
+    }
+    stage('Build') {
+      agent none
+      steps {
+        script {
+          parallel ([
+            'build-cpu': { BuildCPU() },
+            'build-gpu-cuda8.0': { BuildCUDA(cuda_version: '8.0') },
+            'build-gpu-cuda9.2': { BuildCUDA(cuda_version: '9.2') },
+            'build-gpu-cuda10.0': { BuildCUDA(cuda_version: '10.0') },
+            'build-jvm-packages': { BuildJVMPackages(spark_version: '2.4.1') },
+            'build-jvm-doc': { BuildJVMDoc() }
+          ])
+        }
+        milestone ordinal: 3
+      }
+    }
+    stage('Test') {
+      agent none
+      steps {
+        script {
+          parallel ([
+            'test-python-cpu': { TestPythonCPU() },
+            'test-python-gpu-cuda8.0': { TestPythonGPU(cuda_version: '8.0') },
+            'test-python-gpu-cuda9.2': { TestPythonGPU(cuda_version: '9.2') },
+            'test-python-gpu-cuda10.0': { TestPythonGPU(cuda_version: '10.0') },
+            'test-python-mgpu-cuda10.0': { TestPythonGPU(cuda_version: '10.0', multi_gpu: true) },
+            'test-cpp-gpu': { TestCppGPU(cuda_version: '10.0') },
+            'test-cpp-mgpu': { TestCppGPU(cuda_version: '10.0', multi_gpu: true) },
+            'test-jvm-jdk8': { CrossTestJVMwithJDK(jdk_version: '8') },
+            'test-jvm-jdk11': { CrossTestJVMwithJDK(jdk_version: '11') },
+            'test-jvm-jdk12': { CrossTestJVMwithJDK(jdk_version: '12') },
+            'test-r-3.4.4': { TestR(use_r35: false) },
+            'test-r-3.5.3': { TestR(use_r35: true) }
+          ])
+        }
+        milestone ordinal: 4
       }
     }
   }
 }
 
-/**
+// check out source code from git
- * Build platform and test it via cmake.
+def checkoutSrcs() {
- */
+  retry(5) {
-def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) {
+    try {
-    def opts = utils.cmakeOptions(conf)
+      timeout(time: 2, unit: 'MINUTES') {
-    // Destination dir for artifacts
+        checkout scm
-    def distDir = "dist/${buildName}"
+        sh 'git submodule update --init'
-    def dockerArgs = ""
-    if (conf["withGpu"]) {
-        dockerArgs = "--build-arg CUDA_VERSION=" + conf["cudaVersion"]
-    }
-    def test_suite = conf["withGpu"] ? (conf["multiGpu"] ? "mgpu" : "gpu") : "cpu"
-    // Build node - this is returned result
-    retry(1) {
-        node(nodeReq) {
-            unstash name: 'srcs'
-            echo """
-            |===== XGBoost CMake build =====
-            |  dockerTarget: ${dockerTarget}
-            |  cmakeOpts   : ${opts}
-            |=========================
-            """.stripMargin('|')
-            // Invoke command inside docker
-            sh """
-            ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/build_via_cmake.sh ${opts}
-            ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/test_${test_suite}.sh
-            """
-            if (!conf["multiGpu"]) {
-                sh """
-                ${dockerRun} ${dockerTarget} ${dockerArgs} bash -c "cd python-package; rm -f dist/*; python setup.py bdist_wheel --universal"
-                rm -rf "${distDir}"; mkdir -p "${distDir}/py"
-                cp xgboost "${distDir}"
-                cp -r python-package/dist "${distDir}/py"
-                # Test the wheel for compatibility on a barebones CPU container
-                ${dockerRun} release ${dockerArgs} bash -c " \
-                    pip install --user python-package/dist/xgboost-*-none-any.whl && \
-		    pytest -v --fulltrace -s tests/python"
-                # Test the wheel for compatibility on CUDA 10.0 container
-                ${dockerRun} gpu --build-arg CUDA_VERSION=10.0 bash -c " \
-                    pip install --user python-package/dist/xgboost-*-none-any.whl && \
-		    pytest -v -s --fulltrace -m '(not mgpu) and (not slow)' tests/python-gpu"
-                """
       }
+    } catch (exc) {
+      deleteDir()
+      error "Failed to fetch source codes"
     }
   }
 }
 
-/**
+def ClangTidy() {
- * Run a clang-tidy job on a GPU machine
+  node('linux && cpu') {
- */
-def buildClangTidyJob() {
-    def nodeReq = "linux && gpu && unrestricted"
-    node(nodeReq) {
     unstash name: 'srcs'
     echo "Running clang-tidy job..."
-        // Invoke command inside docker
+    def container_type = "clang_tidy"
-        // Install Google Test and Python yaml
+    def docker_binary = "docker"
-        dockerTarget = "clang_tidy"
+    def dockerArgs = "--build-arg CUDA_VERSION=9.2"
-        dockerArgs = "--build-arg CUDA_VERSION=9.2"
     sh """
-        ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/clang_tidy.sh
+    ${dockerRun} ${container_type} ${docker_binary} ${dockerArgs} tests/ci_build/clang_tidy.sh
     """
+    deleteDir()
   }
 }
 
+def Lint() {
+  node('linux && cpu') {
+    unstash name: 'srcs'
+    echo "Running lint..."
+    def container_type = "cpu"
+    def docker_binary = "docker"
+    sh """
+    ${dockerRun} ${container_type} ${docker_binary} make lint
+    """
+    deleteDir()
+  }
+}
+
+def SphinxDoc() {
+  node('linux && cpu') {
+    unstash name: 'srcs'
+    echo "Running sphinx-doc..."
+    def container_type = "cpu"
+    def docker_binary = "docker"
+    def docker_extra_params = "CI_DOCKER_EXTRA_PARAMS_INIT='-e SPHINX_GIT_BRANCH=${BRANCH_NAME}'"
+    sh """#!/bin/bash
+    ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} make -C doc html
+    """
+    deleteDir()
+  }
+}
+
+def Doxygen() {
+  node('linux && cpu') {
+    unstash name: 'srcs'
+    echo "Running doxygen..."
+    def container_type = "cpu"
+    def docker_binary = "docker"
+    sh """
+    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/doxygen.sh
+    """
+    deleteDir()
+  }
+}
+
+def BuildCPU() {
+  node('linux && cpu') {
+    unstash name: 'srcs'
+    echo "Build CPU"
+    def container_type = "cpu"
+    def docker_binary = "docker"
+    sh """
+    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh
+    ${dockerRun} ${container_type} ${docker_binary} build/testxgboost
+    """
+    // Sanitizer test
+    def docker_extra_params = "CI_DOCKER_EXTRA_PARAMS_INIT='-e ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer -e ASAN_OPTIONS=symbolize=1 --cap-add SYS_PTRACE'"
+    sh """
+    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh -DUSE_SANITIZER=ON -DENABLED_SANITIZERS="address" \
+      -DCMAKE_BUILD_TYPE=Debug -DSANITIZER_PATH=/usr/lib/x86_64-linux-gnu/
+    ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} build/testxgboost
+    """
+    deleteDir()
+  }
+}
+
+def BuildCUDA(args) {
+  node('linux && cpu') {
+    unstash name: 'srcs'
+    echo "Build with CUDA ${args.cuda_version}"
+    def container_type = "gpu_build"
+    def docker_binary = "docker"
+    def docker_args = "--build-arg CUDA_VERSION=${args.cuda_version}"
+    sh """
+    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_CUDA=ON -DUSE_NCCL=ON -DOPEN_MP:BOOL=ON
+    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal"
+    """
+    // Only stash wheel for CUDA 8.0 target
+    if (args.cuda_version == '8.0') {
+      echo 'Stashing Python wheel...'
+      stash name: 'xgboost_whl', includes: 'python-package/dist/*.whl'
+      archiveArtifacts artifacts: "python-package/dist/*.whl", allowEmptyArchive: true
+      echo 'Stashing C++ test executable (testxgboost)...'
+      stash name: 'xgboost_cpp_tests', includes: 'build/testxgboost'
+    }
+    deleteDir()
+  }
+}
+
+def BuildJVMPackages(args) {
+  node('linux && cpu') {
+    unstash name: 'srcs'
+    echo "Build XGBoost4J-Spark with Spark ${args.spark_version}"
+    def container_type = "jvm"
+    def docker_binary = "docker"
+    // Use only 4 CPU cores
+    def docker_extra_params = "CI_DOCKER_EXTRA_PARAMS_INIT='--cpuset-cpus 0-3'"
+    sh """
+    ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_jvm_packages.sh
+    """
+    echo 'Stashing XGBoost4J JAR...'
+    stash name: 'xgboost4j_jar', includes: 'jvm-packages/xgboost4j/target/*.jar,jvm-packages/xgboost4j-spark/target/*.jar,jvm-packages/xgboost4j-example/target/*.jar'
+    deleteDir()
+  }
+}
+
+def BuildJVMDoc() {
+  node('linux && cpu') {
+    unstash name: 'srcs'
+    echo "Building JVM doc..."
+    def container_type = "jvm"
+    def docker_binary = "docker"
+    sh """
+    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_jvm_doc.sh ${BRANCH_NAME}
+    """
+    archiveArtifacts artifacts: "jvm-packages/${BRANCH_NAME}.tar.bz2", allowEmptyArchive: true
+    echo 'Uploading doc...'
+    s3Upload file: "jvm-packages/${BRANCH_NAME}.tar.bz2", bucket: 'xgboost-docs', acl: 'PublicRead', path: "${BRANCH_NAME}.tar.bz2"
+    deleteDir()
+  }
+}
+
+def TestPythonCPU() {
+  node('linux && cpu') {
+    unstash name: 'xgboost_whl'
+    unstash name: 'srcs'
+    echo "Test Python CPU"
+    def container_type = "cpu"
+    def docker_binary = "docker"
+    sh """
+    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/test_python.sh cpu
+    """
+    deleteDir()
+  }
+}
+
+def TestPythonGPU(args) {
+  nodeReq = (args.multi_gpu) ? 'linux && mgpu' : 'linux && gpu'
+  node(nodeReq) {
+    unstash name: 'xgboost_whl'
+    unstash name: 'srcs'
+    echo "Test Python GPU: CUDA ${args.cuda_version}"
+    def container_type = "gpu"
+    def docker_binary = "nvidia-docker"
+    def docker_args = "--build-arg CUDA_VERSION=${args.cuda_version}"
+    if (args.multi_gpu) {
+      echo "Using multiple GPUs"
+      sh """
+      ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/test_python.sh mgpu
+      """
+    } else {
+      echo "Using a single GPU"
+      sh """
+      ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/test_python.sh gpu
+      """
+    }
+    deleteDir()
+  }
+}
+
+def TestCppGPU(args) {
+  nodeReq = (args.multi_gpu) ? 'linux && mgpu' : 'linux && gpu'
+  node(nodeReq) {
+    unstash name: 'xgboost_cpp_tests'
+    unstash name: 'srcs'
+    echo "Test C++, CUDA ${args.cuda_version}"
+    def container_type = "gpu"
+    def docker_binary = "nvidia-docker"
+    def docker_args = "--build-arg CUDA_VERSION=${args.cuda_version}"
+    if (args.multi_gpu) {
+      echo "Using multiple GPUs"
+      sh "${dockerRun} ${container_type} ${docker_binary} ${docker_args} build/testxgboost --gtest_filter=*.MGPU_*"
+    } else {
+      echo "Using a single GPU"
+      sh "${dockerRun} ${container_type} ${docker_binary} ${docker_args} build/testxgboost --gtest_filter=-*.MGPU_*"
+    }
+    deleteDir()
+  }
+}
+
+def CrossTestJVMwithJDK(args) {
+  node('linux && cpu') {
+    unstash name: 'xgboost4j_jar'
+    unstash name: 'srcs'
+    echo "Test XGBoost4J on a machine with JDK ${args.jdk_version}"
+    def container_type = "jvm_cross"
+    def docker_binary = "docker"
+    def docker_args = "--build-arg JDK_VERSION=${args.jdk_version}"
+    // Only run integration tests for JDK 8, as Spark doesn't support later JDKs yet
+    def docker_extra_params = (args.jdk_version == '8') ? "CI_DOCKER_EXTRA_PARAMS_INIT='-e RUN_INTEGRATION_TEST=1'" : ""
+    sh """
+    ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/test_jvm_cross.sh
+    """
+    deleteDir()
+  }
+}
+
+def TestR(args) {
+  node('linux && cpu') {
+    unstash name: 'srcs'
+    echo "Test R package"
+    def container_type = "rproject"
+    def docker_binary = "docker"
+    def use_r35_flag = (args.use_r35) ? "1" : "0"
+    def docker_args = "--build-arg USE_R35=${use_r35_flag}"
+    sh """
+    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_test_rpkg.sh
+    """
+    deleteDir()
+  }
+}

Jenkinsfile-restricted

@@ -1,124 +0,0 @@
-#!/usr/bin/groovy
-// -*- mode: groovy -*-
-// Jenkins pipeline
-// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
-
-import groovy.transform.Field
-
-/* Restricted tasks: tasks generating artifacts, such as binary wheels and
-                     documentation */
-
-// Command to run command inside a docker container
-def dockerRun = 'tests/ci_build/ci_build.sh'
-// Utility functions
-@Field
-def utils
-@Field
-def commit_id
-@Field
-def branch_name
-
-def buildMatrix = [
-    [ "enabled": true,  "os" : "linux", "withGpu": true, "withNccl": true,  "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "10.0" ],
-    [ "enabled": true,  "os" : "linux", "withGpu": true, "withNccl": true,  "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "9.2" ],
-    [ "enabled": true,  "os" : "linux", "withGpu": true, "withNccl": true,  "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "8.0" ],
-    [ "enabled": true,  "os" : "linux", "withGpu": true, "withNccl": false, "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "8.0" ],
-]
-
-pipeline {
-    // Each stage specify its own agent
-    agent none
-
-    // Setup common job properties
-    options {
-        ansiColor('xterm')
-        timestamps()
-        timeout(time: 120, unit: 'MINUTES')
-        buildDiscarder(logRotator(numToKeepStr: '10'))
-    }
-
-    // Build stages
-    stages {
-        stage('Jenkins: Get sources') {
-            agent {
-                label 'restricted'
-            }
-            steps {
-                script {
-                    utils = load('tests/ci_build/jenkins_tools.Groovy')
-                    utils.checkoutSrcs()
-                    commit_id = "${GIT_COMMIT}"
-                    branch_name = "${GIT_LOCAL_BRANCH}"
-                }
-                stash name: 'srcs', excludes: '.git/'
-                milestone label: 'Sources ready', ordinal: 1
-            }
-        }
-        stage('Jenkins: Build doc') {
-            steps {
-                script {
-                    retry(1) {
-                        node('linux && cpu && restricted') {
-                            unstash name: 'srcs'
-                            echo 'Building doc...'
-                            dir ('jvm-packages') {
-                                sh "bash ./build_doc.sh ${commit_id}"
-                                archiveArtifacts artifacts: "${commit_id}.tar.bz2", allowEmptyArchive: true
-                                echo 'Deploying doc...'
-                                withAWS(credentials:'xgboost-doc-bucket') {
-                                    s3Upload file: "${commit_id}.tar.bz2", bucket: 'xgboost-docs', acl: 'PublicRead', path: "${branch_name}.tar.bz2"
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        stage('Jenkins: Build artifacts') {
-            steps {
-                script {
-                    parallel (buildMatrix.findAll{it['enabled']}.collectEntries{ c ->
-                        def buildName = utils.getBuildName(c)
-                        utils.buildFactory(buildName, c, true, this.&buildPlatformCmake)
-                    })
-                }
-            }
-        }
-    }
-}
-
-/**
- * Build platform and test it via cmake.
- */
-def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) {
-    def opts = utils.cmakeOptions(conf)
-    // Destination dir for artifacts
-    def distDir = "dist/${buildName}"
-    def dockerArgs = ""
-    if(conf["withGpu"]){
-        dockerArgs = "--build-arg CUDA_VERSION=" + conf["cudaVersion"]
-    }
-    // Build node - this is returned result
-    retry(1) {
-        node(nodeReq) {
-            unstash name: 'srcs'
-            echo """
-            |===== XGBoost CMake build =====
-            |  dockerTarget: ${dockerTarget}
-            |  cmakeOpts   : ${opts}
-            |=========================
-            """.stripMargin('|')
-            // Invoke command inside docker
-            sh """
-            ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/build_via_cmake.sh ${opts}
-            ${dockerRun} ${dockerTarget} ${dockerArgs} bash -c "cd python-package; rm -f dist/*; python setup.py bdist_wheel --universal"
-            rm -rf "${distDir}"; mkdir -p "${distDir}/py"
-            cp xgboost "${distDir}"
-            cp -r lib "${distDir}"
-            cp -r python-package/dist "${distDir}/py"
-            """
-            archiveArtifacts artifacts: "${distDir}/**/*.*", allowEmptyArchive: true
-        }
-    }
-}

Makefile

@@ -176,7 +176,11 @@ rcpplint:
 	python3 dmlc-core/scripts/lint.py xgboost ${LINT_LANG} R-package/src
 
 lint: rcpplint
-	python3 dmlc-core/scripts/lint.py --pylint-rc ${PWD}/python-package/.pylintrc xgboost ${LINT_LANG} include src plugin python-package
+	python3 dmlc-core/scripts/lint.py --exclude_path python-package/xgboost/dmlc-core \
+	  python-package/xgboost/include python-package/xgboost/lib \
+	  python-package/xgboost/make python-package/xgboost/rabit \
+	  python-package/xgboost/src --pylint-rc ${PWD}/python-package/.pylintrc xgboost \
+	  ${LINT_LANG} include src plugin python-package
 
 pylint:
 	flake8 --ignore E501 python-package

doc/conf.py

@@ -22,6 +22,9 @@ import os, subprocess
 import shlex
 import guzzle_sphinx_theme
 
+git_branch = os.getenv('SPHINX_GIT_BRANCH', default=None)
+if git_branch is None:
+    # If SPHINX_GIT_BRANCH environment variable is not given, run git to determine branch name
     git_branch = [re.sub(r'origin/', '', x.lstrip(' ')) for x in str(git.branch('-r', '--contains', 'HEAD')).rstrip('\n').split('\n')]
     git_branch = [x for x in git_branch if 'HEAD' not in x]
 print('git_branch = {}'.format(git_branch[0]))

jvm-packages/xgboost4j-example/src/main/java/ml/dmlc/xgboost4j/java/example/ExternalMemory.java

@@ -32,8 +32,8 @@ public class ExternalMemory {
     //this is the only difference, add a # followed by a cache prefix name
     //several cache file with the prefix will be generated
     //currently only support convert from libsvm file
-    DMatrix trainMat = new DMatrix("../demo/data/agaricus.txt.train#dtrain.cache");
+    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train#dtrain.cache");
-    DMatrix testMat = new DMatrix("../demo/data/agaricus.txt.test#dtest.cache");
+    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test#dtest.cache");
 
     //specify parameters
     HashMap<String, Object> params = new HashMap<String, Object>();

jvm-packages/xgboost4j-tester/generate_pom.py

@@ -0,0 +1,208 @@
+import sys
+
+pom_template = """
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>ml.dmlc</groupId>
+  <artifactId>xgboost4j-tester</artifactId>
+  <version>1.0-SNAPSHOT</version>
+
+  <name>xgboost4j-tester</name>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <maven.compiler.source>{maven_compiler_source}</maven.compiler.source>
+    <maven.compiler.target>{maven_compiler_target}</maven.compiler.target>
+    <spark.version>{spark_version}</spark.version>
+    <scala.version>{scala_version}</scala.version>
+    <scala.binary.version>{scala_binary_version}</scala.binary.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.esotericsoftware.kryo</groupId>
+      <artifactId>kryo</artifactId>
+      <version>2.21</version>
+    </dependency>
+    <dependency>
+      <groupId>org.scala-lang</groupId>
+      <artifactId>scala-compiler</artifactId>
+      <version>${{scala.version}}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.scala-lang</groupId>
+      <artifactId>scala-reflect</artifactId>
+      <version>${{scala.version}}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.scala-lang</groupId>
+      <artifactId>scala-library</artifactId>
+      <version>${{scala.version}}</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-logging</groupId>
+      <artifactId>commons-logging</artifactId>
+      <version>1.2</version>
+    </dependency>
+    <dependency>
+      <groupId>com.typesafe.akka</groupId>
+      <artifactId>akka-actor_${{scala.binary.version}}</artifactId>
+      <version>2.3.11</version>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.typesafe.akka</groupId>
+      <artifactId>akka-testkit_${{scala.binary.version}}</artifactId>
+      <version>2.3.11</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.scalatest</groupId>
+      <artifactId>scalatest_${{scala.binary.version}}</artifactId>
+      <version>3.0.0</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <version>3.4</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-core_${{scala.binary.version}}</artifactId>
+      <version>${{spark.version}}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-sql_${{scala.binary.version}}</artifactId>
+      <version>${{spark.version}}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-mllib_${{scala.binary.version}}</artifactId>
+      <version>${{spark.version}}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.11</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>ml.dmlc</groupId>
+      <artifactId>xgboost4j</artifactId>
+      <version>{xgboost4j_version}</version>
+    </dependency>
+    <dependency>
+      <groupId>ml.dmlc</groupId>
+      <artifactId>xgboost4j</artifactId>
+      <version>{xgboost4j_version}</version>
+      <classifier>tests</classifier>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>ml.dmlc</groupId>
+      <artifactId>xgboost4j-spark</artifactId>
+      <version>{xgboost4j_version}</version>
+    </dependency>
+    <dependency>
+      <groupId>ml.dmlc</groupId>
+      <artifactId>xgboost4j-example</artifactId>
+      <version>{xgboost4j_version}</version>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle -->
+      <plugin>
+        <artifactId>maven-clean-plugin</artifactId>
+        <version>3.1.0</version>
+      </plugin>
+      <!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
+      <plugin>
+        <artifactId>maven-resources-plugin</artifactId>
+        <version>3.0.2</version>
+      </plugin>
+      <plugin>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>3.8.0</version>
+      </plugin>
+      <plugin>
+        <artifactId>maven-jar-plugin</artifactId>
+        <version>3.0.2</version>
+      </plugin>
+      <plugin>
+        <artifactId>maven-install-plugin</artifactId>
+        <version>2.5.2</version>
+      </plugin>
+      <plugin>
+        <artifactId>maven-deploy-plugin</artifactId>
+        <version>2.8.2</version>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <version>2.4</version>
+        <configuration>
+          <descriptorRefs>
+            <descriptorRef>jar-with-dependencies</descriptorRef>
+          </descriptorRefs>
+          <archive>
+            <manifest>
+              <mainClass>ml.dmlc.xgboost4j.tester.App</mainClass>
+            </manifest>
+          </archive>
+        </configuration>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle -->
+      <plugin>
+        <artifactId>maven-site-plugin</artifactId>
+        <version>3.7.1</version>
+      </plugin>
+      <plugin>
+        <artifactId>maven-project-info-reports-plugin</artifactId>
+        <version>3.0.0</version>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>2.22.1</version>
+        <configuration>
+          <dependenciesToScan>
+            <dependency>ml.dmlc:xgboost4j</dependency>
+          </dependenciesToScan>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>
+"""
+
+if __name__ == '__main__':
+  if len(sys.argv) != 7:
+    print('Usage: {} [xgboost4j version] [maven compiler source level] [maven compiler target level] [spark version] [scala version] [scala binary version]'.format(sys.argv[0]))
+    sys.exit(1)
+  with open('pom.xml', 'w') as f:
+    print(pom_template.format(xgboost4j_version=sys.argv[1],
+                              maven_compiler_source=sys.argv[2],
+                              maven_compiler_target=sys.argv[3],
+                              spark_version=sys.argv[4],
+                              scala_version=sys.argv[5],
+                              scala_binary_version=sys.argv[6]), file=f)

jvm-packages/xgboost4j-tester/get_iris.py

@@ -0,0 +1,10 @@
+from sklearn.datasets import load_iris
+import numpy as np
+import pandas
+
+X, y = load_iris(return_X_y=True)
+y = y.astype(np.int)
+df = pandas.DataFrame(data=X, columns=['sepal length', 'sepal width', 'petal length', 'petal width'])
+class_id_to_name = {0:'Iris-setosa', 1:'Iris-versicolor', 2:'Iris-virginica'}
+df['class'] = np.vectorize(class_id_to_name.get)(y)
+df.to_csv('./iris.csv', float_format='%.1f', header=False, index=False)

jvm-packages/xgboost4j-tester/src/main/java/ml/dmlc/xgboost4j/tester/App.java

@@ -0,0 +1,26 @@
+package ml.dmlc.xgboost4j.tester;
+
+import ml.dmlc.xgboost4j.java.example.*;
+
+import java.io.IOException;
+import ml.dmlc.xgboost4j.java.XGBoostError;
+
+public class App {
+  public static void main(String[] args) throws IOException, XGBoostError {
+    String[] args2 = new String[0];
+    System.out.println("BoostFromPrediction");
+    BoostFromPrediction.main(args2);
+    System.out.println("CrossValidation");
+    CrossValidation.main(args2);
+    System.out.println("CustomObjective");
+    CustomObjective.main(args2);
+    System.out.println("ExternalMemory");
+    ExternalMemory.main(args2);
+    System.out.println("GeneralizedLinearModel");
+    GeneralizedLinearModel.main(args2);
+    System.out.println("PredictFirstNtree");
+    PredictFirstNtree.main(args2);
+    System.out.println("PredictLeafIndices");
+    PredictLeafIndices.main(args2);
+  }
+}

jvm-packages/xgboost4j-tester/src/test/java/ml/dmlc/xgboost4j/tester/AppTest.java

@@ -0,0 +1,20 @@
+package ml.dmlc.xgboost4j.tester;
+
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+/**
+ * Unit test for simple App.
+ */
+public class AppTest 
+{
+    /**
+     * Rigorous Test :-)
+     */
+    @Test
+    public void shouldAnswerWithTrue()
+    {
+        assertTrue( true );
+    }
+}

jvm-packages/xgboost4j/pom.xml

@@ -71,6 +71,18 @@
                   </execution>
               </executions>
           </plugin>
+          <plugin>
+              <groupId>org.apache.maven.plugins</groupId>
+              <artifactId>maven-jar-plugin</artifactId>
+              <version>3.0.2</version>
+              <executions>
+                  <execution>
+                      <goals>
+                          <goal>test-jar</goal>
+                      </goals>
+                  </execution>
+              </executions>
+          </plugin>
           <plugin>
               <groupId>org.apache.maven.plugins</groupId>
               <artifactId>maven-resources-plugin</artifactId>

tests/ci_build/Dockerfile.clang_tidy

@@ -16,17 +16,6 @@ RUN \
     update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-7 100 && \
     update-alternatives --install /usr/bin/clang clang /usr/bin/clang-7 100
 
-# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
-RUN \
-    export CUDA_SHORT=`echo $CUDA_VERSION | egrep -o '[0-9]+\.[0-9]'` && \
-    if [ "${CUDA_SHORT}" != "10.0" ]; then \
-    wget https://developer.download.nvidia.com/compute/redist/nccl/v2.2/nccl_2.2.13-1%2Bcuda${CUDA_SHORT}_x86_64.txz && \
-    tar xf "nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz" && \
-    cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/include/nccl.h /usr/include && \
-    cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/lib/* /usr/lib && \
-    rm -f nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz && \
-    rm -r nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64; fi
-
 # Install Python packages
 RUN \
     pip3 install pyyaml

tests/ci_build/Dockerfile.cpu

@@ -0,0 +1,38 @@
+FROM ubuntu:18.04
+
+# Environment
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install all basic requirements
+RUN \
+    apt-get update && \
+    apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libasan2 && \
+    # CMake
+    wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
+    bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
+    # Python
+    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
+    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python
+
+ENV PATH=/opt/python/bin:$PATH
+
+ENV GOSU_VERSION 1.10
+
+# Install Python packages
+RUN \
+    pip install pyyaml cpplint pylint astroid sphinx numpy scipy pandas matplotlib sh recommonmark guzzle_sphinx_theme mock \
+                breathe matplotlib graphviz pytest scikit-learn wheel kubernetes urllib3 && \
+    pip install https://h2o-release.s3.amazonaws.com/datatable/stable/datatable-0.7.0/datatable-0.7.0-cp37-cp37m-linux_x86_64.whl
+
+# Install lightweight sudo (not bound to TTY)
+RUN set -ex; \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]

tests/ci_build/Dockerfile.gpu

@@ -1,47 +1,22 @@
 ARG CUDA_VERSION
-FROM nvidia/cuda:$CUDA_VERSION-devel-centos6
+FROM nvidia/cuda:$CUDA_VERSION-runtime-ubuntu16.04
 
 # Environment
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install all basic requirements
 RUN \
-    yum -y update && \
+    apt-get update && \
-    yum install -y tar unzip wget xz git centos-release-scl yum-utils && \
+    apt-get install -y wget unzip bzip2 libgomp1 && \
-    yum-config-manager --enable centos-sclo-rh-testing && \
-    yum -y update && \
-    yum install -y devtoolset-4-gcc devtoolset-4-binutils devtoolset-4-gcc-c++ && \
     # Python
-    wget https://repo.continuum.io/miniconda/Miniconda2-4.3.27-Linux-x86_64.sh && \
+    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
-    bash Miniconda2-4.3.27-Linux-x86_64.sh -b -p /opt/python && \
+    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python
-    # CMake
-    wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
-    bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr
-
-# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
-RUN \
-    export CUDA_SHORT=`echo $CUDA_VERSION | egrep -o '[0-9]+\.[0-9]'` && \
-    if [ "${CUDA_SHORT}" != "10.0" ]; then \
-    wget https://developer.download.nvidia.com/compute/redist/nccl/v2.2/nccl_2.2.13-1%2Bcuda${CUDA_SHORT}_x86_64.txz && \
-    tar xf "nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz" && \
-    cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/include/nccl.h /usr/include && \
-    cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/lib/* /usr/lib && \
-    rm -f nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz && \
-    rm -r nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64; else \
-    wget https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
-    rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
-    yum -y update && \
-    yum install -y libnccl-2.4.2-1+cuda10.0 libnccl-devel-2.4.2-1+cuda10.0 libnccl-static-2.4.2-1+cuda10.0 && \
-    rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm; fi
 
 ENV PATH=/opt/python/bin:$PATH
-ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
-ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
-ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
 
 # Install Python packages
 RUN \
-    pip install numpy pytest scipy scikit-learn wheel kubernetes urllib3==1.22
+    pip install numpy pytest scipy scikit-learn pandas matplotlib wheel kubernetes urllib3 graphviz
 
 ENV GOSU_VERSION 1.10
 

tests/ci_build/Dockerfile.gpu_build

@@ -0,0 +1,59 @@
+ARG CUDA_VERSION
+FROM nvidia/cuda:$CUDA_VERSION-devel-centos6
+
+# Environment
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install all basic requirements
+RUN \
+    yum -y update && \
+    yum install -y tar unzip wget xz git centos-release-scl yum-utils && \
+    yum-config-manager --enable centos-sclo-rh-testing && \
+    yum -y update && \
+    yum install -y devtoolset-4-gcc devtoolset-4-binutils devtoolset-4-gcc-c++ && \
+    # Python
+    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
+    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \
+    # CMake
+    wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
+    bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr
+
+# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
+RUN \
+    export CUDA_SHORT=`echo $CUDA_VERSION | egrep -o '[0-9]+\.[0-9]'` && \
+    if [ "${CUDA_SHORT}" != "10.0" ]; then \
+    wget https://developer.download.nvidia.com/compute/redist/nccl/v2.2/nccl_2.2.13-1%2Bcuda${CUDA_SHORT}_x86_64.txz && \
+    tar xf "nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz" && \
+    cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/include/nccl.h /usr/include && \
+    cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/lib/* /usr/lib && \
+    rm -f nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz && \
+    rm -r nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64; else \
+    wget https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
+    rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
+    yum -y update && \
+    yum install -y libnccl-2.4.2-1+cuda10.0 libnccl-devel-2.4.2-1+cuda10.0 libnccl-static-2.4.2-1+cuda10.0 && \
+    rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm; fi
+
+ENV PATH=/opt/python/bin:$PATH
+ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
+ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
+ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
+
+# Install Python packages
+RUN \
+    pip install numpy pytest scipy scikit-learn wheel kubernetes urllib3==1.22
+
+ENV GOSU_VERSION 1.10
+
+# Install lightweight sudo (not bound to TTY)
+RUN set -ex; \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]

tests/ci_build/Dockerfile.jvm

@@ -0,0 +1,43 @@
+FROM centos:6
+
+# Install all basic requirements
+RUN \
+    yum -y update && \
+    yum install -y tar unzip wget xz git centos-release-scl yum-utils java-1.8.0-openjdk-devel && \
+    yum-config-manager --enable centos-sclo-rh-testing && \
+    yum -y update && \
+    yum install -y devtoolset-4-gcc devtoolset-4-binutils devtoolset-4-gcc-c++ && \
+    # Python
+    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
+    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \
+    # CMake
+    wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
+    bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
+    # Maven
+    wget http://apache.osuosl.org/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \
+    tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \
+    ln -s /opt/apache-maven-3.6.1/ /opt/maven
+
+ENV PATH=/opt/python/bin:/opt/maven/bin:$PATH
+ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
+ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
+ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
+
+# Install Python packages
+RUN \
+    pip install numpy pytest scipy scikit-learn wheel kubernetes urllib3==1.22 awscli
+
+ENV GOSU_VERSION 1.10
+
+# Install lightweight sudo (not bound to TTY)
+RUN set -ex; \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]

tests/ci_build/Dockerfile.jvm_cross

@@ -0,0 +1,48 @@
+FROM ubuntu:19.04
+ARG JDK_VERSION=8
+
+# Environment
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install all basic requirements
+RUN \
+    apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:openjdk-r/ppa && \
+    apt-get update && \
+    apt-get install -y tar unzip wget openjdk-$JDK_VERSION-jdk libgomp1 && \
+    # Python
+    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
+    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \
+    # Maven
+    wget http://apache.osuosl.org/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \
+    tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \
+    ln -s /opt/apache-maven-3.6.1/ /opt/maven && \
+    # Spark
+    wget https://archive.apache.org/dist/spark/spark-2.4.1/spark-2.4.1-bin-hadoop2.7.tgz && \
+    tar xvf spark-2.4.1-bin-hadoop2.7.tgz -C /opt && \
+    ln -s /opt/spark-2.4.1-bin-hadoop2.7 /opt/spark
+
+ENV PATH=/opt/python/bin:/opt/spark/bin:/opt/maven/bin:$PATH
+
+# Install Python packages
+RUN \
+    pip install numpy scipy pandas scikit-learn
+
+ENV GOSU_VERSION 1.10
+
+# Install lightweight sudo (not bound to TTY)
+RUN set -ex; \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Set default JDK version
+RUN update-java-alternatives -v -s java-1.$JDK_VERSION.0-openjdk-amd64
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]

tests/ci_build/Dockerfile.release

@@ -1,8 +1,5 @@
 FROM centos:6
 
-# Environment
-ENV DEBIAN_FRONTEND noninteractive
-
 # Install all basic requirements
 RUN \
     yum -y update && \

tests/ci_build/Dockerfile.rproject

@@ -0,0 +1,44 @@
+FROM ubuntu:18.04
+ARG USE_R35=0
+
+# Environment
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install all basic requirements
+RUN \
+    apt-get update && \
+    apt-get install -y software-properties-common tar unzip wget git build-essential doxygen graphviz libcurl4-openssl-dev libssl-dev libxml2-dev && \
+    if [ $USE_R35 -eq 1 ]; then \
+      apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E084DAB9 && \
+      add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' && \
+      apt-get update; \
+    fi && \
+    apt-get install -y r-base r-base-core r-recommended && \
+    # CMake
+    wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
+    bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr
+
+# Use 16 workers to compile R packages
+ENV MAKE 'make -j16'
+
+# Install R packages
+RUN Rscript -e "install.packages( \
+        c('devtools', 'testthat', 'lintr', 'DiagrammeR', 'Ckmeans.1d.dp', 'vcd', 'data.table', 'stringi'), \
+        repos = 'http://cloud.r-project.org', \
+        dependencies = c('Depends', 'Imports', 'LinkingTo') \
+    )"
+
+ENV GOSU_VERSION 1.10
+
+# Install lightweight sudo (not bound to TTY)
+RUN set -ex; \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]

tests/ci_build/build_jvm_doc.sh

@@ -1,21 +1,27 @@
 #!/bin/bash
 
 if [ $# -ne 1 ]; then
-  echo "Usage: $0 [commit id]"
+  echo "Usage: $0 [branch name]"
   exit 1
 fi
 
 set -e
 set -x
 
-commit_id=$1
+# Initialize local Maven repository
+./tests/ci_build/initialize_maven.sh
+
+rm -rf build/
+cd jvm-packages
+
+branch_name=$1
 
 # Install JVM packages in local Maven repository
-mvn install -DskipTests
+mvn --no-transfer-progress install -DskipTests
 # Build Scaladocs
-mvn scala:doc -DskipTests
+mvn --no-transfer-progress scala:doc -DskipTests
 # Build Javadocs
-mvn javadoc:javadoc -DskipTests
+mvn --no-transfer-progress javadoc:javadoc -DskipTests
 
 # Package JVM docs in a tarball
 mkdir -p tmp/scaladocs
@@ -25,8 +31,8 @@ cp -rv xgboost4j-spark/target/site/scaladocs/ ./tmp/scaladocs/xgboost4j-spark/
 cp -rv xgboost4j-flink/target/site/scaladocs/ ./tmp/scaladocs/xgboost4j-flink/
 
 cd tmp
-tar cvjf ${commit_id}.tar.bz2 javadocs/ scaladocs/
+tar cvjf ${branch_name}.tar.bz2 javadocs/ scaladocs/
-mv ${commit_id}.tar.bz2 ..
+mv ${branch_name}.tar.bz2 ..
 cd ..
 rm -rfv tmp/
 

tests/ci_build/build_jvm_packages.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+set -x
+
+# Initialize local Maven repository
+./tests/ci_build/initialize_maven.sh
+
+rm -rf build/
+cd jvm-packages
+
+mvn --no-transfer-progress package -DskipTests
+
+set +x
+set +e

tests/ci_build/build_test_rpkg.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+set -x
+
+make Rpack
+cd xgboost/
+
+# Run tests
+echo "Building with R CMD build"
+R CMD build --no-build-vignettes --no-manual .
+
+echo "Running R tests"
+R_PACKAGE_TARBALL=$(ls -1t *.tar.gz | head -n 1)
+
+export _R_CHECK_TIMINGS_=0
+export _R_CHECK_FORCE_SUGGESTS_=false
+R CMD check \
+  ${R_PACKAGE_TARBALL} \
+  --no-vignettes \
+  --no-manual \
+  --as-cran \
+  --install-args=--build

tests/ci_build/build_via_cmake.sh

@@ -16,5 +16,5 @@ mkdir build
 cd build
 cmake .. "$@" -DGOOGLE_TEST=ON -DGTEST_ROOT=$PWD/../gtest -DCMAKE_VERBOSE_MAKEFILE=ON
 make clean
-make -j
+make -j$(nproc)
 cd ..

tests/ci_build/ci_build.sh

@@ -2,16 +2,21 @@
 #
 # Execute command within a docker container
 #
-# Usage: ci_build.sh <CONTAINER_TYPE> [--dockerfile <DOCKERFILE_PATH>] [-it]
+# Usage: ci_build.sh <CONTAINER_TYPE> <DOCKER_BINARY>
-#                    <COMMAND>
+#                    [--dockerfile <DOCKERFILE_PATH>] [-it]
+#                    [--build-arg <BUILD_ARG>] <COMMAND>
 #
 # CONTAINER_TYPE: Type of the docker container used the run the build: e.g.,
 #                 (cpu | gpu)
 #
+# DOCKER_BINARY: Command to invoke docker, e.g. (docker | nvidia-docker).
+#
 # DOCKERFILE_PATH: (Optional) Path to the Dockerfile used for docker build.  If
 #                  this optional value is not supplied (via the --dockerfile
 #                  flag), will use Dockerfile.CONTAINER_TYPE in default
 #
+# BUILD_ARG: (Optional) an argument to be passed to docker build
+#
 # COMMAND: Command to be executed in the docker container
 #
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -24,6 +29,10 @@ shift 1
 DOCKERFILE_PATH="${SCRIPT_DIR}/Dockerfile.${CONTAINER_TYPE}"
 DOCKER_CONTEXT_PATH="${SCRIPT_DIR}"
 
+# Get docker binary command (should be either docker or nvidia-docker)
+DOCKER_BINARY="$1"
+shift 1
+
 if [[ "$1" == "--dockerfile" ]]; then
     DOCKERFILE_PATH="$2"
     DOCKER_CONTEXT_PATH=$(dirname "${DOCKERFILE_PATH}")
@@ -32,6 +41,11 @@ if [[ "$1" == "--dockerfile" ]]; then
     shift 2
 fi
 
+if [[ -n "${CI_DOCKER_EXTRA_PARAMS_INIT}" ]]
+then
+    IFS=' ' read -r -a CI_DOCKER_EXTRA_PARAMS <<< "${CI_DOCKER_EXTRA_PARAMS_INIT}"
+fi
+
 if [[ "$1" == "-it" ]]; then
     CI_DOCKER_EXTRA_PARAMS+=('-it')
     shift 1
@@ -61,13 +75,6 @@ if [ "$#" -lt 1 ] || [ ! -e "${SCRIPT_DIR}/Dockerfile.${CONTAINER_TYPE}" ]; then
       exit 1
 fi
 
-# Use nvidia-docker if the container is GPU.
-if [[ "${CONTAINER_TYPE}" == *"gpu"* ]]; then
-    DOCKER_BINARY="nvidia-docker"
-else
-    DOCKER_BINARY="docker"
-fi
-
 # Helper function to traverse directories up until given file is found.
 function upsearch () {
     test / == "$PWD" && return || \
@@ -84,7 +91,9 @@ DOCKER_IMG_NAME="xgb-ci.${CONTAINER_TYPE}"
 
 # Append cuda version if available
 CUDA_VERSION=$(echo "${CI_DOCKER_BUILD_ARG}" | grep CUDA_VERSION | egrep -o '[0-9]*\.[0-9]*')
-DOCKER_IMG_NAME=$DOCKER_IMG_NAME$CUDA_VERSION 
+# Append jdk version if available
+JDK_VERSION=$(echo "${CI_DOCKER_BUILD_ARG}" | grep JDK_VERSION | egrep -o '[0-9]*')
+DOCKER_IMG_NAME=$DOCKER_IMG_NAME$CUDA_VERSION$JDK_VERSION
 
 # Under Jenkins matrix build, the build tag may contain characters such as
 # commas (,) and equal signs (=), which are not valid inside docker image names.
@@ -98,7 +107,7 @@ UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || e
 # MSYS, Git Bash, etc.
 MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "")
 
-if [[ -z "$UBUNTU_ON_WINDOWS" ]] && [[ -z "$MSYS" ]]; then
+if [[ -z "$UBUNTU_ON_WINDOWS" ]] && [[ -z "$MSYS" ]] && [[ ! "$OSTYPE" == "darwin"* ]]; then
     USER_IDS="-e CI_BUILD_UID=$( id -u ) -e CI_BUILD_GID=$( id -g ) -e CI_BUILD_USER=$( id -un ) -e CI_BUILD_GROUP=$( id -gn ) -e CI_BUILD_HOME=${WORKSPACE}"
 fi
 
@@ -181,6 +190,7 @@ echo "Running '${COMMAND[*]}' inside ${DOCKER_IMG_NAME}..."
 # By default we cleanup - remove the container once it finish running (--rm)
 # and share the PID namespace (--pid=host) so the process inside does not have
 # pid 1 and SIGKILL is propagated to the process inside (jenkins can kill it).
+set -x
 ${DOCKER_BINARY} run --rm --pid=host \
     -v "${WORKSPACE}":/workspace \
     -w /workspace \

tests/ci_build/doxygen.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+set -x
+
+rm -rf build
+mkdir build
+cd build
+cmake .. -DBUILD_C_DOC=ON
+make -j

tests/ci_build/initialize_maven.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+set -x
+
+if [ -z ${CI_BUILD_USER} ]
+then
+  echo 'Must be run inside Jenkins CI'
+  exit 1
+fi
+gosu root mkdir -p /cache
+gosu root chown ${CI_BUILD_USER}:${CI_BUILD_GROUP} /cache
+
+# Download cached Maven repository, to speed up build
+python3 -m awscli s3 cp s3://xgboost-ci-jenkins-artifacts/maven-repo-cache.tar.bz2 /cache/maven-repo-cache.tar.bz2 || true
+
+if [[ -f "/cache/maven-repo-cache.tar.bz2" ]]
+then
+  tar xvf /cache/maven-repo-cache.tar.bz2 -C ${HOME}
+fi

tests/ci_build/jenkins_tools.Groovy

@@ -6,20 +6,6 @@
 // Command to run command inside a docker container
 dockerRun = 'tests/ci_build/ci_build.sh'
 
-// initialize source codes
-def checkoutSrcs() {
-  retry(5) {
-    try {
-      timeout(time: 2, unit: 'MINUTES') {
-        checkout scm
-        sh 'git submodule update --init'
-      }
-    } catch (exc) {
-      deleteDir()
-      error "Failed to fetch source codes"
-    }
-  }
-}
 
 /**
  * Creates cmake and make builds

tests/ci_build/test_gpu.sh

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-cd python-package
-python setup.py install --user
-cd ..
-pytest -v -s --fulltrace -m "(not mgpu) and (not slow)" tests/python-gpu
-pushd .
-cd build
-./testxgboost --gtest_filter=-*.MGPU_*
-ctest --output-on-failure --tests-regex "TestXGBoostCLI"
-popd

tests/ci_build/test_jvm_cross.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -e
+set -x
+
+# Initialize local Maven repository
+./tests/ci_build/initialize_maven.sh
+
+# Get version number of XGBoost4J and other auxiliary information
+cd jvm-packages
+xgboost4j_version=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+maven_compiler_source=$(mvn help:evaluate -Dexpression=maven.compiler.source -q -DforceStdout)
+maven_compiler_target=$(mvn help:evaluate -Dexpression=maven.compiler.target -q -DforceStdout)
+spark_version=$(mvn help:evaluate -Dexpression=spark.version -q -DforceStdout)
+scala_version=$(mvn help:evaluate -Dexpression=scala.version -q -DforceStdout)
+scala_binary_version=$(mvn help:evaluate -Dexpression=scala.binary.version -q -DforceStdout)
+
+# Install XGBoost4J JAR into local Maven repository
+mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j -Dversion=${xgboost4j_version} -Dpackaging=jar
+mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j-${xgboost4j_version}-tests.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j -Dversion=${xgboost4j_version} -Dpackaging=test-jar -Dclassifier=tests
+mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-spark/target/xgboost4j-spark-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-spark -Dversion=${xgboost4j_version} -Dpackaging=jar
+mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-example/target/xgboost4j-example-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-example -Dversion=${xgboost4j_version} -Dpackaging=jar
+
+cd xgboost4j-tester
+# Generate pom.xml for XGBoost4J-tester, a dummy project to run XGBoost4J tests
+python3 ./generate_pom.py ${xgboost4j_version} ${maven_compiler_source} ${maven_compiler_target} ${spark_version} ${scala_version} ${scala_binary_version}
+# Run unit tests with XGBoost4J
+mvn --no-transfer-progress package
+
+# Run integration tests with XGBoost4J
+java -jar ./target/xgboost4j-tester-1.0-SNAPSHOT-jar-with-dependencies.jar
+
+# Run integration tests with XGBoost4J-Spark
+if [ ! -z "$RUN_INTEGRATION_TEST" ]
+then
+  python3 get_iris.py
+  spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkTraining --master 'local[8]' ./target/xgboost4j-tester-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv
+  # Disabled due to https://github.com/dmlc/xgboost/issues/4406
+  #spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkMLlibPipeline --master 'local[8]' ./target/xgboost4j-tester-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv ${PWD}/native_model ${PWD}/pipeline_model
+fi
+
+set +x
+set +e

tests/ci_build/test_mgpu.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-cd python-package
-python setup.py install --user
-cd ..
-pytest -v -s --fulltrace -m "(not slow) and mgpu" tests/python-gpu
-./build/testxgboost --gtest_filter=*.MGPU_*
-
-cd tests/distributed
-./runtests-gpu.sh

tests/ci_build/test_python.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+set -e
+set -x
+
+suite=$1
+
+# Install XGBoost Python package
+wheel_found=0
+for file in python-package/dist/*.whl
+do
+  pip install --user ${file}
+  wheel_found=1
+  break  # need just one
+done
+if [ "$wheel_found" -eq 0 ]
+then
+  pushd .
+  cd python-package
+  python setup.py install --user
+  popd
+fi
+
+# Run specified test suite
+case "$suite" in
+  gpu)
+    pytest -v -s --fulltrace -m "(not slow) and (not mgpu)" tests/python-gpu
+    ;;
+
+  mgpu)
+    pytest -v -s --fulltrace -m "(not slow) and mgpu" tests/python-gpu
+    cd tests/distributed
+    ./runtests-gpu.sh
+    ;;
+
+  cpu)
+    pytest -v -s --fulltrace tests/python
+    cd tests/distributed
+    ./runtests.sh
+    ;;
+
+  *)
+    echo "Usage: $0 {gpu|mgpu|cpu}"
+    exit 1
+    ;;
+esac

tests/distributed/runtests-gpu.sh

@@ -3,7 +3,6 @@
 rm -f *.model*
 
 export DMLC_SUBMIT_CLUSTER=local
-export PYTHONPATH=../../python-package
 submit="timeout 30 python ../../dmlc-core/tracker/dmlc-submit"
 
 echo -e "\n ====== 1. Basic distributed-gpu test with Python: 4 workers; 1 GPU per worker ====== \n"

tests/distributed/runtests.sh

@@ -1,9 +1,13 @@
 #!/bin/bash
 
+rm -f *.model*
+
+export DMLC_SUBMIT_CLUSTER=local
+
+submit="timeout 30 python ../../dmlc-core/tracker/dmlc-submit"
+
 echo "====== 1. Basic distributed test with Python ======"
-PYTHONPATH=../../python-package/ python ../../dmlc-core/tracker/dmlc-submit  --cluster=local --num-workers=3\
+$submit --cluster=local --num-workers=3 python test_basic.py
-  python test_basic.py
 
 echo "====== 2. Regression test for issue #3402 ======"
-PYTHONPATH=../../python-package/ python ../../dmlc-core/tracker/dmlc-submit  --cluster=local --num-workers=2 --worker-cores=1\
+$submit --cluster=local --num-workers=2 --worker-cores=1 python test_issue3402.py
-  python test_issue3402.py

tests/travis/run_test.sh

@@ -1,34 +1,8 @@
 #!/bin/bash
 
-if [ ${TASK} == "lint" ]; then
-    source activate python3
-    conda install numpy scipy
-    python -m pip install cpplint pylint astroid
-    make lint || exit -1
-    echo "Check documentations..."
-
-    mkdir build_doc
-    cd build_doc
-    cmake .. -DBUILD_C_DOC=ON
-    make doc_doxygen 2> log.txt
-
-    (cat log.txt| grep -v ENABLE_PREPROCESSING |grep -v "unsupported tag") > logclean.txt
-    echo "---------Error Log----------"
-    cat logclean.txt
-    echo "----------------------------"
-    (cat logclean.txt|grep warning) && exit -1
-    (cat logclean.txt|grep error) && exit -1
-
-    cd -
-    rm -rf build_doc
-
-    exit 0
-fi
-
 cp make/travis.mk config.mk
 make -f dmlc-core/scripts/packages.mk lz4
 
-
 if [ ${TRAVIS_OS_NAME} == "osx" ]; then
     echo 'USE_OPENMP=0' >> config.mk
     echo 'TMPVAR := $(XGB_PLUGINS)' >> config.mk
@@ -45,96 +19,10 @@ if [ ${TASK} == "python_test" ]; then
     python --version
     conda install numpy scipy pandas matplotlib scikit-learn
 
-    # Install data table from source
-    wget http://releases.llvm.org/5.0.2/clang+llvm-5.0.2-x86_64-linux-gnu-ubuntu-14.04.tar.xz
-    tar xf clang+llvm-5.0.2-x86_64-linux-gnu-ubuntu-14.04.tar.xz
-    export LLVM5=$(pwd)/clang+llvm-5.0.2-x86_64-linux-gnu-ubuntu-14.04
-    python -m pip install datatable --no-binary datatable
-
     python -m pip install graphviz pytest pytest-cov codecov
+    python -m pip install https://h2o-release.s3.amazonaws.com/datatable/stable/datatable-0.7.0/datatable-0.7.0-cp37-cp37m-linux_x86_64.whl
     python -m pytest -v --fulltrace -s tests/python --cov=python-package/xgboost || exit -1
     codecov
-
-    source activate python2
-    echo "-------------------------------"
-    python --version
-    conda install numpy scipy pandas matplotlib scikit-learn
-    python -m pip install graphviz pytest
-    python -m pytest -v --fulltrace -s tests/python || exit -1
-    exit 0
-fi
-
-if [ ${TASK} == "python_lightweight_test" ]; then
-    make all || exit -1
-
-    echo "-------------------------------"
-    source activate python3
-    python --version
-    conda install numpy scipy
-    python -m pip install graphviz pytest pytest-cov codecov
-    python -m pytest -v --fulltrace -s tests/python --cov=python-package/xgboost || exit -1
-    codecov
-
-    source activate python2
-    echo "-------------------------------"
-    python --version
-    conda install numpy scipy pytest
-    python -m pip install graphviz
-    python -m pip install flake8==3.4.1
-    python -m pytest -v --fulltrace -s tests/python || exit -1
-
-    flake8 --ignore E501 python-package || exit -1
-    flake8 --ignore E501 tests/python || exit -1
-    exit 0
-fi
-
-if [ ${TASK} == "r_test" ]; then
-    set -e
-
-    make Rpack
-    cd ./xgboost
-
-    # Install package deps
-    Rscript -e "install.packages( \
-        c('devtools', 'testthat', 'lintr') \
-        , repos = 'http://cloud.r-project.org' \
-        , dependencies = c('Depends', 'Imports', 'LinkingTo') \
-    )"
-
-    Rscript -e \
-        "devtools::install_deps( \
-            repos = 'http://cloud.r-project.org' \
-            , upgrade = 'never' \
-            , dependencies = c('Depends', 'Imports', 'LinkingTo') \
-        )"
-
-    # install suggested packages separately to avoid huge build times
-    Rscript -e "install.packages( \
-        c('DiagrammeR', 'Ckmeans.1d.dp', 'vcd') \
-        , repos = 'https://cloud.r-project.org' \
-        , dependencies = c('Depends', 'Imports', 'LinkingTo') \
-    )"
-
-    # Run tests
-    echo "Building with R CMD build"
-    R CMD build \
-        --no-build-vignettes \
-        --no-manual \
-        .
-
-    echo "Running R tests"
-    R_PACKAGE_TARBALL=$(ls -1t *.tar.gz | head -n 1)
-
-    export _R_CHECK_TIMINGS_=0
-    export _R_CHECK_FORCE_SUGGESTS_=false
-    R CMD check \
-        ${R_PACKAGE_TARBALL} \
-        --no-vignettes \
-        --no-manual \
-        --as-cran \
-        --install-args=--build
-
-    exit 0
 fi
 
 if [ ${TASK} == "java_test" ]; then
@@ -150,7 +38,7 @@ if [ ${TASK} == "cmake_test" ]; then
     wget -nc https://github.com/google/googletest/archive/release-1.7.0.zip
     unzip -n release-1.7.0.zip
     mv googletest-release-1.7.0 gtest && cd gtest
-    cmake . && make
+    CC=gcc-7 CXX=g++-7 cmake . && make
     mkdir lib && mv libgtest.a lib
     cd ..
     rm -rf release-1.7.0.zip
@@ -159,59 +47,9 @@ if [ ${TASK} == "cmake_test" ]; then
     rm -rf build
     mkdir build && cd build
     PLUGINS="-DPLUGIN_LZ4=ON -DPLUGIN_DENSE_PARSER=ON"
-    cmake .. -DGOOGLE_TEST=ON -DGTEST_ROOT=$PWD/../gtest/ ${PLUGINS}
+    CC=gcc-7 CXX=g++-7 cmake .. -DGOOGLE_TEST=ON -DGTEST_ROOT=$PWD/../gtest/ ${PLUGINS}
     make
     ./testxgboost
     cd ..
     rm -rf build
 fi
-
-if [ ${TASK} == "cpp_test" ]; then
-    set -e
-    make -f dmlc-core/scripts/packages.mk gtest
-    echo "TEST_COVER=1" >> config.mk
-    echo "GTEST_PATH="${CACHE_PREFIX} >> config.mk
-    make cover
-fi
-
-
-if [ ${TASK} == "distributed_test" ]; then
-    set -e
-    make all || exit -1
-    echo "-------------------------------"
-    source activate python3
-    python --version
-    conda install numpy scipy
-    python -m pip install kubernetes
-    cd tests/distributed
-    ./runtests.sh
-fi
-
-if [ ${TASK} == "sanitizer_test" ]; then
-    set -e
-    # Build gtest via cmake
-    wget -nc https://github.com/google/googletest/archive/release-1.7.0.zip
-    unzip -n release-1.7.0.zip
-    mv googletest-release-1.7.0 gtest && cd gtest
-    CC=gcc-7 CXX=g++-7 cmake -DCMAKE_CXX_FLAGS="-fuse-ld=gold" \
-      -DCMAKE_C_FLAGS="-fuse-ld=gold"
-    make
-    mkdir lib && mv libgtest.a lib
-    cd ..
-    rm -rf release-1.7.0.zip
-
-    mkdir build && cd build
-    CC=gcc-7 CXX=g++-7 cmake .. -DGOOGLE_TEST=ON -DGTEST_ROOT=$PWD/../gtest/ \
-      -DUSE_SANITIZER=ON -DENABLED_SANITIZERS="address" \
-      -DCMAKE_BUILD_TYPE=Debug \
-      -DSANITIZER_PATH=/usr/lib/x86_64-linux-gnu/ \
-      -DCMAKE_CXX_FLAGS="-fuse-ld=gold" \
-      -DCMAKE_C_FLAGS="-fuse-ld=gold"
-    make
-
-    export ASAN_SYMBOLIZER_PATH=$(which llvm-symbolizer)
-    ASAN_OPTIONS=symbolize=1 ./testxgboost
-    cd ..
-    rm -rf build
-    exit 0
-fi

tests/travis/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-if [ ${TASK} == "lint" ]; then
+if [ ${TASK} == "python_test" ]; then
     if [ ${TRAVIS_OS_NAME} == "osx" ]; then
         wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
     else
@@ -16,20 +16,6 @@ if [ ${TASK} == "lint" ]; then
     conda create -n python3 python=3.7
 fi
 
-
+if [ ${TASK} == "cmake_test" ] && [ ${TRAVIS_OS_NAME} == "osx" ]; then
-if [ ${TASK} == "python_test" ] || [ ${TASK} == "python_lightweight_test" ] || [ ${TASK} ==  "distributed_test" ]; then
+    sudo softwareupdate -i "Command Line Tools (macOS High Sierra version 10.13) for Xcode-9.3"
-    if [ ${TRAVIS_OS_NAME} == "osx" ]; then
-        wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
-    else
-        wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
-    fi
-    bash conda.sh -b -p $HOME/miniconda
-    export PATH="$HOME/miniconda/bin:$PATH"
-    hash -r
-    conda config --set always_yes yes --set changeps1 no
-    conda update -q conda
-    # Useful for debugging any issues with conda
-    conda info -a
-    conda create -n python3 python=3.7
-    conda create -n python2 python=2.7
 fi