[CI] Refactor Jenkins CI pipeline + migrate all Linux tests to Jenkins (#4401)

* All Linux tests are now in Jenkins CI * Tests are now de-coupled from builds. We can now build XGBoost with one version of CUDA/JDK and test it with another version of CUDA/JDK * Builds (compilation) are significantly faster because 1) They use C5 instances with faster CPU cores; and 2) build environment setup is cached using Docker containers
2019-04-26 18:39:12 -07:00
parent 995698b0cb
commit ea850ecd20
35 changed files with 1046 additions and 585 deletions
--- a/tests/ci_build/Dockerfile.clang_tidy
+++ b/tests/ci_build/Dockerfile.clang_tidy
@@ -16,17 +16,6 @@ RUN \
    update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-7 100 && \
    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-7 100

-# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
-RUN \
-    export CUDA_SHORT=`echo $CUDA_VERSION | egrep -o '[0-9]+\.[0-9]'` && \
-    if [ "${CUDA_SHORT}" != "10.0" ]; then \
-    wget https://developer.download.nvidia.com/compute/redist/nccl/v2.2/nccl_2.2.13-1%2Bcuda${CUDA_SHORT}_x86_64.txz && \
-    tar xf "nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz" && \
-    cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/include/nccl.h /usr/include && \
-    cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/lib/* /usr/lib && \
-    rm -f nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz && \
-    rm -r nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64; fi
-
 # Install Python packages
 RUN \
    pip3 install pyyaml
--- a/tests/ci_build/Dockerfile.cpu
+++ b/tests/ci_build/Dockerfile.cpu
@@ -0,0 +1,38 @@
+FROM ubuntu:18.04
+
+# Environment
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install all basic requirements
+RUN \
+    apt-get update && \
+    apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libasan2 && \
+    # CMake
+    wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
+    bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
+    # Python
+    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
+    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python
+
+ENV PATH=/opt/python/bin:$PATH
+
+ENV GOSU_VERSION 1.10
+
+# Install Python packages
+RUN \
+    pip install pyyaml cpplint pylint astroid sphinx numpy scipy pandas matplotlib sh recommonmark guzzle_sphinx_theme mock \
+                breathe matplotlib graphviz pytest scikit-learn wheel kubernetes urllib3 && \
+    pip install https://h2o-release.s3.amazonaws.com/datatable/stable/datatable-0.7.0/datatable-0.7.0-cp37-cp37m-linux_x86_64.whl
+
+# Install lightweight sudo (not bound to TTY)
+RUN set -ex; \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]
--- a/tests/ci_build/Dockerfile.gpu
+++ b/tests/ci_build/Dockerfile.gpu
@@ -1,47 +1,22 @@
 ARG CUDA_VERSION
-FROM nvidia/cuda:$CUDA_VERSION-devel-centos6
+FROM nvidia/cuda:$CUDA_VERSION-runtime-ubuntu16.04

 # Environment
 ENV DEBIAN_FRONTEND noninteractive

 # Install all basic requirements
 RUN \
-    yum -y update && \
-    yum install -y tar unzip wget xz git centos-release-scl yum-utils && \
-    yum-config-manager --enable centos-sclo-rh-testing && \
-    yum -y update && \
-    yum install -y devtoolset-4-gcc devtoolset-4-binutils devtoolset-4-gcc-c++ && \
+    apt-get update && \
+    apt-get install -y wget unzip bzip2 libgomp1 && \
    # Python
-    wget https://repo.continuum.io/miniconda/Miniconda2-4.3.27-Linux-x86_64.sh && \
-    bash Miniconda2-4.3.27-Linux-x86_64.sh -b -p /opt/python && \
-    # CMake
-    wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
-    bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr
-
-# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
-RUN \
-    export CUDA_SHORT=`echo $CUDA_VERSION | egrep -o '[0-9]+\.[0-9]'` && \
-    if [ "${CUDA_SHORT}" != "10.0" ]; then \
-    wget https://developer.download.nvidia.com/compute/redist/nccl/v2.2/nccl_2.2.13-1%2Bcuda${CUDA_SHORT}_x86_64.txz && \
-    tar xf "nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz" && \
-    cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/include/nccl.h /usr/include && \
-    cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/lib/* /usr/lib && \
-    rm -f nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz && \
-    rm -r nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64; else \
-    wget https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
-    rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
-    yum -y update && \
-    yum install -y libnccl-2.4.2-1+cuda10.0 libnccl-devel-2.4.2-1+cuda10.0 libnccl-static-2.4.2-1+cuda10.0 && \
-    rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm; fi
+    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
+    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python

 ENV PATH=/opt/python/bin:$PATH
-ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
-ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
-ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp

 # Install Python packages
 RUN \
-    pip install numpy pytest scipy scikit-learn wheel kubernetes urllib3==1.22
+    pip install numpy pytest scipy scikit-learn pandas matplotlib wheel kubernetes urllib3 graphviz

 ENV GOSU_VERSION 1.10

--- a/tests/ci_build/Dockerfile.gpu_build
+++ b/tests/ci_build/Dockerfile.gpu_build
@@ -0,0 +1,59 @@
+ARG CUDA_VERSION
+FROM nvidia/cuda:$CUDA_VERSION-devel-centos6
+
+# Environment
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install all basic requirements
+RUN \
+    yum -y update && \
+    yum install -y tar unzip wget xz git centos-release-scl yum-utils && \
+    yum-config-manager --enable centos-sclo-rh-testing && \
+    yum -y update && \
+    yum install -y devtoolset-4-gcc devtoolset-4-binutils devtoolset-4-gcc-c++ && \
+    # Python
+    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
+    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \
+    # CMake
+    wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
+    bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr
+
+# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
+RUN \
+    export CUDA_SHORT=`echo $CUDA_VERSION | egrep -o '[0-9]+\.[0-9]'` && \
+    if [ "${CUDA_SHORT}" != "10.0" ]; then \
+    wget https://developer.download.nvidia.com/compute/redist/nccl/v2.2/nccl_2.2.13-1%2Bcuda${CUDA_SHORT}_x86_64.txz && \
+    tar xf "nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz" && \
+    cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/include/nccl.h /usr/include && \
+    cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/lib/* /usr/lib && \
+    rm -f nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz && \
+    rm -r nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64; else \
+    wget https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
+    rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
+    yum -y update && \
+    yum install -y libnccl-2.4.2-1+cuda10.0 libnccl-devel-2.4.2-1+cuda10.0 libnccl-static-2.4.2-1+cuda10.0 && \
+    rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm; fi
+
+ENV PATH=/opt/python/bin:$PATH
+ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
+ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
+ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
+
+# Install Python packages
+RUN \
+    pip install numpy pytest scipy scikit-learn wheel kubernetes urllib3==1.22
+
+ENV GOSU_VERSION 1.10
+
+# Install lightweight sudo (not bound to TTY)
+RUN set -ex; \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]
--- a/tests/ci_build/Dockerfile.jvm
+++ b/tests/ci_build/Dockerfile.jvm
@@ -0,0 +1,43 @@
+FROM centos:6
+
+# Install all basic requirements
+RUN \
+    yum -y update && \
+    yum install -y tar unzip wget xz git centos-release-scl yum-utils java-1.8.0-openjdk-devel && \
+    yum-config-manager --enable centos-sclo-rh-testing && \
+    yum -y update && \
+    yum install -y devtoolset-4-gcc devtoolset-4-binutils devtoolset-4-gcc-c++ && \
+    # Python
+    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
+    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \
+    # CMake
+    wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
+    bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
+    # Maven
+    wget http://apache.osuosl.org/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \
+    tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \
+    ln -s /opt/apache-maven-3.6.1/ /opt/maven
+
+ENV PATH=/opt/python/bin:/opt/maven/bin:$PATH
+ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
+ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
+ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
+
+# Install Python packages
+RUN \
+    pip install numpy pytest scipy scikit-learn wheel kubernetes urllib3==1.22 awscli
+
+ENV GOSU_VERSION 1.10
+
+# Install lightweight sudo (not bound to TTY)
+RUN set -ex; \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]
--- a/tests/ci_build/Dockerfile.jvm_cross
+++ b/tests/ci_build/Dockerfile.jvm_cross
@@ -0,0 +1,48 @@
+FROM ubuntu:19.04
+ARG JDK_VERSION=8
+
+# Environment
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install all basic requirements
+RUN \
+    apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:openjdk-r/ppa && \
+    apt-get update && \
+    apt-get install -y tar unzip wget openjdk-$JDK_VERSION-jdk libgomp1 && \
+    # Python
+    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
+    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \
+    # Maven
+    wget http://apache.osuosl.org/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \
+    tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \
+    ln -s /opt/apache-maven-3.6.1/ /opt/maven && \
+    # Spark
+    wget https://archive.apache.org/dist/spark/spark-2.4.1/spark-2.4.1-bin-hadoop2.7.tgz && \
+    tar xvf spark-2.4.1-bin-hadoop2.7.tgz -C /opt && \
+    ln -s /opt/spark-2.4.1-bin-hadoop2.7 /opt/spark
+
+ENV PATH=/opt/python/bin:/opt/spark/bin:/opt/maven/bin:$PATH
+
+# Install Python packages
+RUN \
+    pip install numpy scipy pandas scikit-learn
+
+ENV GOSU_VERSION 1.10
+
+# Install lightweight sudo (not bound to TTY)
+RUN set -ex; \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Set default JDK version
+RUN update-java-alternatives -v -s java-1.$JDK_VERSION.0-openjdk-amd64
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]
--- a/tests/ci_build/Dockerfile.release
+++ b/tests/ci_build/Dockerfile.release
@@ -1,8 +1,5 @@
 FROM centos:6

-# Environment
-ENV DEBIAN_FRONTEND noninteractive
-
 # Install all basic requirements
 RUN \
    yum -y update && \
--- a/tests/ci_build/Dockerfile.rproject
+++ b/tests/ci_build/Dockerfile.rproject
@@ -0,0 +1,44 @@
+FROM ubuntu:18.04
+ARG USE_R35=0
+
+# Environment
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install all basic requirements
+RUN \
+    apt-get update && \
+    apt-get install -y software-properties-common tar unzip wget git build-essential doxygen graphviz libcurl4-openssl-dev libssl-dev libxml2-dev && \
+    if [ $USE_R35 -eq 1 ]; then \
+      apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E084DAB9 && \
+      add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' && \
+      apt-get update; \
+    fi && \
+    apt-get install -y r-base r-base-core r-recommended && \
+    # CMake
+    wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
+    bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr
+
+# Use 16 workers to compile R packages
+ENV MAKE 'make -j16'
+
+# Install R packages
+RUN Rscript -e "install.packages( \
+        c('devtools', 'testthat', 'lintr', 'DiagrammeR', 'Ckmeans.1d.dp', 'vcd', 'data.table', 'stringi'), \
+        repos = 'http://cloud.r-project.org', \
+        dependencies = c('Depends', 'Imports', 'LinkingTo') \
+    )"
+
+ENV GOSU_VERSION 1.10
+
+# Install lightweight sudo (not bound to TTY)
+RUN set -ex; \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]
--- a/tests/ci_build/build_jvm_doc.sh
+++ b/tests/ci_build/build_jvm_doc.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+if [ $# -ne 1 ]; then
+  echo "Usage: $0 [branch name]"
+  exit 1
+fi
+
+set -e
+set -x
+
+# Initialize local Maven repository
+./tests/ci_build/initialize_maven.sh
+
+rm -rf build/
+cd jvm-packages
+
+branch_name=$1
+
+# Install JVM packages in local Maven repository
+mvn --no-transfer-progress install -DskipTests
+# Build Scaladocs
+mvn --no-transfer-progress scala:doc -DskipTests
+# Build Javadocs
+mvn --no-transfer-progress javadoc:javadoc -DskipTests
+
+# Package JVM docs in a tarball
+mkdir -p tmp/scaladocs
+cp -rv xgboost4j/target/site/apidocs/ ./tmp/javadocs/
+cp -rv xgboost4j/target/site/scaladocs/ ./tmp/scaladocs/xgboost4j/
+cp -rv xgboost4j-spark/target/site/scaladocs/ ./tmp/scaladocs/xgboost4j-spark/
+cp -rv xgboost4j-flink/target/site/scaladocs/ ./tmp/scaladocs/xgboost4j-flink/
+
+cd tmp
+tar cvjf ${branch_name}.tar.bz2 javadocs/ scaladocs/
+mv ${branch_name}.tar.bz2 ..
+cd ..
+rm -rfv tmp/
+
+set +x
+set +e
--- a/tests/ci_build/build_jvm_packages.sh
+++ b/tests/ci_build/build_jvm_packages.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+set -x
+
+# Initialize local Maven repository
+./tests/ci_build/initialize_maven.sh
+
+rm -rf build/
+cd jvm-packages
+
+mvn --no-transfer-progress package -DskipTests
+
+set +x
+set +e
--- a/tests/ci_build/build_test_rpkg.sh
+++ b/tests/ci_build/build_test_rpkg.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+set -x
+
+make Rpack
+cd xgboost/
+
+# Run tests
+echo "Building with R CMD build"
+R CMD build --no-build-vignettes --no-manual .
+
+echo "Running R tests"
+R_PACKAGE_TARBALL=$(ls -1t *.tar.gz | head -n 1)
+
+export _R_CHECK_TIMINGS_=0
+export _R_CHECK_FORCE_SUGGESTS_=false
+R CMD check \
+  ${R_PACKAGE_TARBALL} \
+  --no-vignettes \
+  --no-manual \
+  --as-cran \
+  --install-args=--build
--- a/tests/ci_build/build_via_cmake.sh
+++ b/tests/ci_build/build_via_cmake.sh
@@ -16,5 +16,5 @@ mkdir build
 cd build
 cmake .. "$@" -DGOOGLE_TEST=ON -DGTEST_ROOT=$PWD/../gtest -DCMAKE_VERBOSE_MAKEFILE=ON
 make clean
-make -j
+make -j$(nproc)
 cd ..
--- a/tests/ci_build/ci_build.sh
+++ b/tests/ci_build/ci_build.sh
@@ -2,16 +2,21 @@
 #
 # Execute command within a docker container
 #
-# Usage: ci_build.sh <CONTAINER_TYPE> [--dockerfile <DOCKERFILE_PATH>] [-it]
-#                    <COMMAND>
+# Usage: ci_build.sh <CONTAINER_TYPE> <DOCKER_BINARY>
+#                    [--dockerfile <DOCKERFILE_PATH>] [-it]
+#                    [--build-arg <BUILD_ARG>] <COMMAND>
 #
 # CONTAINER_TYPE: Type of the docker container used the run the build: e.g.,
 #                 (cpu | gpu)
 #
+# DOCKER_BINARY: Command to invoke docker, e.g. (docker | nvidia-docker).
+#
 # DOCKERFILE_PATH: (Optional) Path to the Dockerfile used for docker build.  If
 #                  this optional value is not supplied (via the --dockerfile
 #                  flag), will use Dockerfile.CONTAINER_TYPE in default
 #
+# BUILD_ARG: (Optional) an argument to be passed to docker build
+#
 # COMMAND: Command to be executed in the docker container
 #
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -24,6 +29,10 @@ shift 1
 DOCKERFILE_PATH="${SCRIPT_DIR}/Dockerfile.${CONTAINER_TYPE}"
 DOCKER_CONTEXT_PATH="${SCRIPT_DIR}"

+# Get docker binary command (should be either docker or nvidia-docker)
+DOCKER_BINARY="$1"
+shift 1
+
 if [[ "$1" == "--dockerfile" ]]; then
    DOCKERFILE_PATH="$2"
    DOCKER_CONTEXT_PATH=$(dirname "${DOCKERFILE_PATH}")
@@ -32,6 +41,11 @@ if [[ "$1" == "--dockerfile" ]]; then
    shift 2
 fi

+if [[ -n "${CI_DOCKER_EXTRA_PARAMS_INIT}" ]]
+then
+    IFS=' ' read -r -a CI_DOCKER_EXTRA_PARAMS <<< "${CI_DOCKER_EXTRA_PARAMS_INIT}"
+fi
+
 if [[ "$1" == "-it" ]]; then
    CI_DOCKER_EXTRA_PARAMS+=('-it')
    shift 1
@@ -61,13 +75,6 @@ if [ "$#" -lt 1 ] || [ ! -e "${SCRIPT_DIR}/Dockerfile.${CONTAINER_TYPE}" ]; then
      exit 1
 fi

-# Use nvidia-docker if the container is GPU.
-if [[ "${CONTAINER_TYPE}" == *"gpu"* ]]; then
-    DOCKER_BINARY="nvidia-docker"
-else
-    DOCKER_BINARY="docker"
-fi
-
 # Helper function to traverse directories up until given file is found.
 function upsearch () {
    test / == "$PWD" && return || \
@@ -84,7 +91,9 @@ DOCKER_IMG_NAME="xgb-ci.${CONTAINER_TYPE}"

 # Append cuda version if available
 CUDA_VERSION=$(echo "${CI_DOCKER_BUILD_ARG}" | grep CUDA_VERSION | egrep -o '[0-9]*\.[0-9]*')
-DOCKER_IMG_NAME=$DOCKER_IMG_NAME$CUDA_VERSION 
+# Append jdk version if available
+JDK_VERSION=$(echo "${CI_DOCKER_BUILD_ARG}" | grep JDK_VERSION | egrep -o '[0-9]*')
+DOCKER_IMG_NAME=$DOCKER_IMG_NAME$CUDA_VERSION$JDK_VERSION

 # Under Jenkins matrix build, the build tag may contain characters such as
 # commas (,) and equal signs (=), which are not valid inside docker image names.
@@ -98,7 +107,7 @@ UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || e
 # MSYS, Git Bash, etc.
 MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "")

-if [[ -z "$UBUNTU_ON_WINDOWS" ]] && [[ -z "$MSYS" ]]; then
+if [[ -z "$UBUNTU_ON_WINDOWS" ]] && [[ -z "$MSYS" ]] && [[ ! "$OSTYPE" == "darwin"* ]]; then
    USER_IDS="-e CI_BUILD_UID=$( id -u ) -e CI_BUILD_GID=$( id -g ) -e CI_BUILD_USER=$( id -un ) -e CI_BUILD_GROUP=$( id -gn ) -e CI_BUILD_HOME=${WORKSPACE}"
 fi

@@ -181,6 +190,7 @@ echo "Running '${COMMAND[*]}' inside ${DOCKER_IMG_NAME}..."
 # By default we cleanup - remove the container once it finish running (--rm)
 # and share the PID namespace (--pid=host) so the process inside does not have
 # pid 1 and SIGKILL is propagated to the process inside (jenkins can kill it).
+set -x
 ${DOCKER_BINARY} run --rm --pid=host \
    -v "${WORKSPACE}":/workspace \
    -w /workspace \
--- a/tests/ci_build/doxygen.sh
+++ b/tests/ci_build/doxygen.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+set -x
+
+rm -rf build
+mkdir build
+cd build
+cmake .. -DBUILD_C_DOC=ON
+make -j
--- a/tests/ci_build/initialize_maven.sh
+++ b/tests/ci_build/initialize_maven.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+set -x
+
+if [ -z ${CI_BUILD_USER} ]
+then
+  echo 'Must be run inside Jenkins CI'
+  exit 1
+fi
+gosu root mkdir -p /cache
+gosu root chown ${CI_BUILD_USER}:${CI_BUILD_GROUP} /cache
+
+# Download cached Maven repository, to speed up build
+python3 -m awscli s3 cp s3://xgboost-ci-jenkins-artifacts/maven-repo-cache.tar.bz2 /cache/maven-repo-cache.tar.bz2 || true
+
+if [[ -f "/cache/maven-repo-cache.tar.bz2" ]]
+then
+  tar xvf /cache/maven-repo-cache.tar.bz2 -C ${HOME}
+fi
--- a/tests/ci_build/jenkins_tools.Groovy
+++ b/tests/ci_build/jenkins_tools.Groovy
@@ -6,20 +6,6 @@
 // Command to run command inside a docker container
 dockerRun = 'tests/ci_build/ci_build.sh'

-// initialize source codes
-def checkoutSrcs() {
-  retry(5) {
-    try {
-      timeout(time: 2, unit: 'MINUTES') {
-        checkout scm
-        sh 'git submodule update --init'
-      }
-    } catch (exc) {
-      deleteDir()
-      error "Failed to fetch source codes"
-    }
-  }
-}

 /**
 * Creates cmake and make builds
--- a/tests/ci_build/test_gpu.sh
+++ b/tests/ci_build/test_gpu.sh
@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-cd python-package
-python setup.py install --user
-cd ..
-pytest -v -s --fulltrace -m "(not mgpu) and (not slow)" tests/python-gpu
-pushd .
-cd build
-./testxgboost --gtest_filter=-*.MGPU_*
-ctest --output-on-failure --tests-regex "TestXGBoostCLI"
-popd
--- a/tests/ci_build/test_jvm_cross.sh
+++ b/tests/ci_build/test_jvm_cross.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -e
+set -x
+
+# Initialize local Maven repository
+./tests/ci_build/initialize_maven.sh
+
+# Get version number of XGBoost4J and other auxiliary information
+cd jvm-packages
+xgboost4j_version=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+maven_compiler_source=$(mvn help:evaluate -Dexpression=maven.compiler.source -q -DforceStdout)
+maven_compiler_target=$(mvn help:evaluate -Dexpression=maven.compiler.target -q -DforceStdout)
+spark_version=$(mvn help:evaluate -Dexpression=spark.version -q -DforceStdout)
+scala_version=$(mvn help:evaluate -Dexpression=scala.version -q -DforceStdout)
+scala_binary_version=$(mvn help:evaluate -Dexpression=scala.binary.version -q -DforceStdout)
+
+# Install XGBoost4J JAR into local Maven repository
+mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j -Dversion=${xgboost4j_version} -Dpackaging=jar
+mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j-${xgboost4j_version}-tests.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j -Dversion=${xgboost4j_version} -Dpackaging=test-jar -Dclassifier=tests
+mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-spark/target/xgboost4j-spark-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-spark -Dversion=${xgboost4j_version} -Dpackaging=jar
+mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-example/target/xgboost4j-example-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-example -Dversion=${xgboost4j_version} -Dpackaging=jar
+
+cd xgboost4j-tester
+# Generate pom.xml for XGBoost4J-tester, a dummy project to run XGBoost4J tests
+python3 ./generate_pom.py ${xgboost4j_version} ${maven_compiler_source} ${maven_compiler_target} ${spark_version} ${scala_version} ${scala_binary_version}
+# Run unit tests with XGBoost4J
+mvn --no-transfer-progress package
+
+# Run integration tests with XGBoost4J
+java -jar ./target/xgboost4j-tester-1.0-SNAPSHOT-jar-with-dependencies.jar
+
+# Run integration tests with XGBoost4J-Spark
+if [ ! -z "$RUN_INTEGRATION_TEST" ]
+then
+  python3 get_iris.py
+  spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkTraining --master 'local[8]' ./target/xgboost4j-tester-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv
+  # Disabled due to https://github.com/dmlc/xgboost/issues/4406
+  #spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkMLlibPipeline --master 'local[8]' ./target/xgboost4j-tester-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv ${PWD}/native_model ${PWD}/pipeline_model
+fi
+
+set +x
+set +e
--- a/tests/ci_build/test_mgpu.sh
+++ b/tests/ci_build/test_mgpu.sh
@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-cd python-package
-python setup.py install --user
-cd ..
-pytest -v -s --fulltrace -m "(not slow) and mgpu" tests/python-gpu
-./build/testxgboost --gtest_filter=*.MGPU_*
-
-cd tests/distributed
-./runtests-gpu.sh
--- a/tests/ci_build/test_python.sh
+++ b/tests/ci_build/test_python.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+set -e
+set -x
+
+suite=$1
+
+# Install XGBoost Python package
+wheel_found=0
+for file in python-package/dist/*.whl
+do
+  pip install --user ${file}
+  wheel_found=1
+  break  # need just one
+done
+if [ "$wheel_found" -eq 0 ]
+then
+  pushd .
+  cd python-package
+  python setup.py install --user
+  popd
+fi
+
+# Run specified test suite
+case "$suite" in
+  gpu)
+    pytest -v -s --fulltrace -m "(not slow) and (not mgpu)" tests/python-gpu
+    ;;
+
+  mgpu)
+    pytest -v -s --fulltrace -m "(not slow) and mgpu" tests/python-gpu
+    cd tests/distributed
+    ./runtests-gpu.sh
+    ;;
+
+  cpu)
+    pytest -v -s --fulltrace tests/python
+    cd tests/distributed
+    ./runtests.sh
+    ;;
+
+  *)
+    echo "Usage: $0 {gpu|mgpu|cpu}"
+    exit 1
+    ;;
+esac