Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CI] Refactor Jenkins CI pipeline + migrate all Linux tests to Jenkins (#4401)

Browse Source 
* All Linux tests are now in Jenkins CI
* Tests are now de-coupled from builds. We can now build XGBoost with one version of CUDA/JDK and test it with another version of CUDA/JDK
* Builds (compilation) are significantly faster because 1) They use C5 instances with faster CPU cores; and 2) build environment setup is cached using Docker containers
This commit is contained in:
Philip Hyunsu Cho
2019-04-26 18:39:12 -07:00
committed by GitHub
parent 995698b0cb
commit ea850ecd20
35 changed files with 1046 additions and 585 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
tests/ci_build/Dockerfile.clang_tidy
View File

@@ -16,17 +16,6 @@ RUN \
update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-7 100 && \
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-7 100
# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
RUN \
export CUDA_SHORT=`echo $CUDA_VERSION | egrep -o '[0-9]+\.[0-9]'` && \
if [ "${CUDA_SHORT}" != "10.0" ]; then \
wget https://developer.download.nvidia.com/compute/redist/nccl/v2.2/nccl_2.2.13-1%2Bcuda${CUDA_SHORT}_x86_64.txz && \
tar xf "nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz" && \
cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/include/nccl.h /usr/include && \
cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/lib/* /usr/lib && \
rm -f nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz && \
rm -r nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64; fi
# Install Python packages
RUN \
pip3 install pyyaml

38
tests/ci_build/Dockerfile.cpu Normal file
View File

@@ -0,0 +1,38 @@
FROM ubuntu:18.04
# Environment
ENV DEBIAN_FRONTEND noninteractive
# Install all basic requirements
RUN \
apt-get update && \
apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libasan2 && \
# CMake
wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
# Python
wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python
ENV PATH=/opt/python/bin:$PATH
ENV GOSU_VERSION 1.10
# Install Python packages
RUN \
pip install pyyaml cpplint pylint astroid sphinx numpy scipy pandas matplotlib sh recommonmark guzzle_sphinx_theme mock \
breathe matplotlib graphviz pytest scikit-learn wheel kubernetes urllib3 && \
pip install https://h2o-release.s3.amazonaws.com/datatable/stable/datatable-0.7.0/datatable-0.7.0-cp37-cp37m-linux_x86_64.whl
# Install lightweight sudo (not bound to TTY)
RUN set -ex; \
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
chmod +x /usr/local/bin/gosu && \
gosu nobody true
# Default entry-point to use if running locally
# It will preserve attributes of created files
COPY entrypoint.sh /scripts/
WORKDIR /workspace
ENTRYPOINT ["/scripts/entrypoint.sh"]

37
tests/ci_build/Dockerfile.gpu
View File

@@ -1,47 +1,22 @@
ARG CUDA_VERSION
FROM nvidia/cuda:$CUDA_VERSION-devel-centos6
FROM nvidia/cuda:$CUDA_VERSION-runtime-ubuntu16.04
# Environment
ENV DEBIAN_FRONTEND noninteractive
# Install all basic requirements
RUN \
yum -y update && \
yum install -y tar unzip wget xz git centos-release-scl yum-utils && \
yum-config-manager --enable centos-sclo-rh-testing && \
yum -y update && \
yum install -y devtoolset-4-gcc devtoolset-4-binutils devtoolset-4-gcc-c++ && \
apt-get update && \
apt-get install -y wget unzip bzip2 libgomp1 && \
# Python
wget https://repo.continuum.io/miniconda/Miniconda2-4.3.27-Linux-x86_64.sh && \
bash Miniconda2-4.3.27-Linux-x86_64.sh -b -p /opt/python && \
# CMake
wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr
# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
RUN \
export CUDA_SHORT=`echo $CUDA_VERSION | egrep -o '[0-9]+\.[0-9]'` && \
if [ "${CUDA_SHORT}" != "10.0" ]; then \
wget https://developer.download.nvidia.com/compute/redist/nccl/v2.2/nccl_2.2.13-1%2Bcuda${CUDA_SHORT}_x86_64.txz && \
tar xf "nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz" && \
cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/include/nccl.h /usr/include && \
cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/lib/* /usr/lib && \
rm -f nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz && \
rm -r nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64; else \
wget https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
yum -y update && \
yum install -y libnccl-2.4.2-1+cuda10.0 libnccl-devel-2.4.2-1+cuda10.0 libnccl-static-2.4.2-1+cuda10.0 && \
rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm; fi
wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python
ENV PATH=/opt/python/bin:$PATH
ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
# Install Python packages
RUN \
pip install numpy pytest scipy scikit-learn wheel kubernetes urllib3==1.22
pip install numpy pytest scipy scikit-learn pandas matplotlib wheel kubernetes urllib3 graphviz
ENV GOSU_VERSION 1.10

59
tests/ci_build/Dockerfile.gpu_build Normal file
View File

@@ -0,0 +1,59 @@
ARG CUDA_VERSION
FROM nvidia/cuda:$CUDA_VERSION-devel-centos6
# Environment
ENV DEBIAN_FRONTEND noninteractive
# Install all basic requirements
RUN \
yum -y update && \
yum install -y tar unzip wget xz git centos-release-scl yum-utils && \
yum-config-manager --enable centos-sclo-rh-testing && \
yum -y update && \
yum install -y devtoolset-4-gcc devtoolset-4-binutils devtoolset-4-gcc-c++ && \
# Python
wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \
# CMake
wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr
# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
RUN \
export CUDA_SHORT=`echo $CUDA_VERSION | egrep -o '[0-9]+\.[0-9]'` && \
if [ "${CUDA_SHORT}" != "10.0" ]; then \
wget https://developer.download.nvidia.com/compute/redist/nccl/v2.2/nccl_2.2.13-1%2Bcuda${CUDA_SHORT}_x86_64.txz && \
tar xf "nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz" && \
cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/include/nccl.h /usr/include && \
cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/lib/* /usr/lib && \
rm -f nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz && \
rm -r nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64; else \
wget https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
yum -y update && \
yum install -y libnccl-2.4.2-1+cuda10.0 libnccl-devel-2.4.2-1+cuda10.0 libnccl-static-2.4.2-1+cuda10.0 && \
rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm; fi
ENV PATH=/opt/python/bin:$PATH
ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
# Install Python packages
RUN \
pip install numpy pytest scipy scikit-learn wheel kubernetes urllib3==1.22
ENV GOSU_VERSION 1.10
# Install lightweight sudo (not bound to TTY)
RUN set -ex; \
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
chmod +x /usr/local/bin/gosu && \
gosu nobody true
# Default entry-point to use if running locally
# It will preserve attributes of created files
COPY entrypoint.sh /scripts/
WORKDIR /workspace
ENTRYPOINT ["/scripts/entrypoint.sh"]

43
tests/ci_build/Dockerfile.jvm Normal file
View File

@@ -0,0 +1,43 @@
FROM centos:6
# Install all basic requirements
RUN \
yum -y update && \
yum install -y tar unzip wget xz git centos-release-scl yum-utils java-1.8.0-openjdk-devel && \
yum-config-manager --enable centos-sclo-rh-testing && \
yum -y update && \
yum install -y devtoolset-4-gcc devtoolset-4-binutils devtoolset-4-gcc-c++ && \
# Python
wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \
# CMake
wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
# Maven
wget http://apache.osuosl.org/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \
tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \
ln -s /opt/apache-maven-3.6.1/ /opt/maven
ENV PATH=/opt/python/bin:/opt/maven/bin:$PATH
ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
# Install Python packages
RUN \
pip install numpy pytest scipy scikit-learn wheel kubernetes urllib3==1.22 awscli
ENV GOSU_VERSION 1.10
# Install lightweight sudo (not bound to TTY)
RUN set -ex; \
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
chmod +x /usr/local/bin/gosu && \
gosu nobody true
# Default entry-point to use if running locally
# It will preserve attributes of created files
COPY entrypoint.sh /scripts/
WORKDIR /workspace
ENTRYPOINT ["/scripts/entrypoint.sh"]

48
tests/ci_build/Dockerfile.jvm_cross Normal file
View File

@@ -0,0 +1,48 @@
FROM ubuntu:19.04
ARG JDK_VERSION=8
# Environment
ENV DEBIAN_FRONTEND noninteractive
# Install all basic requirements
RUN \
apt-get update && \
apt-get install -y software-properties-common && \
add-apt-repository ppa:openjdk-r/ppa && \
apt-get update && \
apt-get install -y tar unzip wget openjdk-$JDK_VERSION-jdk libgomp1 && \
# Python
wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \
# Maven
wget http://apache.osuosl.org/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \
tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \
ln -s /opt/apache-maven-3.6.1/ /opt/maven && \
# Spark
wget https://archive.apache.org/dist/spark/spark-2.4.1/spark-2.4.1-bin-hadoop2.7.tgz && \
tar xvf spark-2.4.1-bin-hadoop2.7.tgz -C /opt && \
ln -s /opt/spark-2.4.1-bin-hadoop2.7 /opt/spark
ENV PATH=/opt/python/bin:/opt/spark/bin:/opt/maven/bin:$PATH
# Install Python packages
RUN \
pip install numpy scipy pandas scikit-learn
ENV GOSU_VERSION 1.10
# Install lightweight sudo (not bound to TTY)
RUN set -ex; \
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
chmod +x /usr/local/bin/gosu && \
gosu nobody true
# Set default JDK version
RUN update-java-alternatives -v -s java-1.$JDK_VERSION.0-openjdk-amd64
# Default entry-point to use if running locally
# It will preserve attributes of created files
COPY entrypoint.sh /scripts/
WORKDIR /workspace
ENTRYPOINT ["/scripts/entrypoint.sh"]

3
tests/ci_build/Dockerfile.release
View File

@@ -1,8 +1,5 @@
FROM centos:6
# Environment
ENV DEBIAN_FRONTEND noninteractive
# Install all basic requirements
RUN \
yum -y update && \

44
tests/ci_build/Dockerfile.rproject Normal file
View File

@@ -0,0 +1,44 @@
FROM ubuntu:18.04
ARG USE_R35=0
# Environment
ENV DEBIAN_FRONTEND noninteractive
# Install all basic requirements
RUN \
apt-get update && \
apt-get install -y software-properties-common tar unzip wget git build-essential doxygen graphviz libcurl4-openssl-dev libssl-dev libxml2-dev && \
if [ $USE_R35 -eq 1 ]; then \
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E084DAB9 && \
add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' && \
apt-get update; \
fi && \
apt-get install -y r-base r-base-core r-recommended && \
# CMake
wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr
# Use 16 workers to compile R packages
ENV MAKE 'make -j16'
# Install R packages
RUN Rscript -e "install.packages( \
c('devtools', 'testthat', 'lintr', 'DiagrammeR', 'Ckmeans.1d.dp', 'vcd', 'data.table', 'stringi'), \
repos = 'http://cloud.r-project.org', \
dependencies = c('Depends', 'Imports', 'LinkingTo') \
)"
ENV GOSU_VERSION 1.10
# Install lightweight sudo (not bound to TTY)
RUN set -ex; \
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
chmod +x /usr/local/bin/gosu && \
gosu nobody true
# Default entry-point to use if running locally
# It will preserve attributes of created files
COPY entrypoint.sh /scripts/
WORKDIR /workspace
ENTRYPOINT ["/scripts/entrypoint.sh"]

40
tests/ci_build/build_jvm_doc.sh Executable file
View File

@@ -0,0 +1,40 @@
#!/bin/bash
if [ $# -ne 1 ]; then
echo "Usage: $0 [branch name]"
exit 1
fi
set -e
set -x
# Initialize local Maven repository
./tests/ci_build/initialize_maven.sh
rm -rf build/
cd jvm-packages
branch_name=$1
# Install JVM packages in local Maven repository
mvn --no-transfer-progress install -DskipTests
# Build Scaladocs
mvn --no-transfer-progress scala:doc -DskipTests
# Build Javadocs
mvn --no-transfer-progress javadoc:javadoc -DskipTests
# Package JVM docs in a tarball
mkdir -p tmp/scaladocs
cp -rv xgboost4j/target/site/apidocs/ ./tmp/javadocs/
cp -rv xgboost4j/target/site/scaladocs/ ./tmp/scaladocs/xgboost4j/
cp -rv xgboost4j-spark/target/site/scaladocs/ ./tmp/scaladocs/xgboost4j-spark/
cp -rv xgboost4j-flink/target/site/scaladocs/ ./tmp/scaladocs/xgboost4j-flink/
cd tmp
tar cvjf ${branch_name}.tar.bz2 javadocs/ scaladocs/
mv ${branch_name}.tar.bz2 ..
cd ..
rm -rfv tmp/
set +x
set +e

15
tests/ci_build/build_jvm_packages.sh Executable file
View File

@@ -0,0 +1,15 @@
#!/bin/bash
set -e
set -x
# Initialize local Maven repository
./tests/ci_build/initialize_maven.sh
rm -rf build/
cd jvm-packages
mvn --no-transfer-progress package -DskipTests
set +x
set +e

23
tests/ci_build/build_test_rpkg.sh Executable file
View File

@@ -0,0 +1,23 @@
#!/bin/bash
set -e
set -x
make Rpack
cd xgboost/
# Run tests
echo "Building with R CMD build"
R CMD build --no-build-vignettes --no-manual .
echo "Running R tests"
R_PACKAGE_TARBALL=$(ls -1t *.tar.gz | head -n 1)
export _R_CHECK_TIMINGS_=0
export _R_CHECK_FORCE_SUGGESTS_=false
R CMD check \
${R_PACKAGE_TARBALL} \
--no-vignettes \
--no-manual \
--as-cran \
--install-args=--build

2
tests/ci_build/build_via_cmake.sh
View File

@@ -16,5 +16,5 @@ mkdir build
cd build
cmake .. "$@" -DGOOGLE_TEST=ON -DGTEST_ROOT=$PWD/../gtest -DCMAKE_VERBOSE_MAKEFILE=ON
make clean
make -j
make -j$(nproc)
cd ..

32
tests/ci_build/ci_build.sh
View File

@@ -2,16 +2,21 @@
#
# Execute command within a docker container
#
# Usage: ci_build.sh <CONTAINER_TYPE> [--dockerfile <DOCKERFILE_PATH>] [-it]
# <COMMAND>
# Usage: ci_build.sh <CONTAINER_TYPE> <DOCKER_BINARY>
# [--dockerfile <DOCKERFILE_PATH>] [-it]
# [--build-arg <BUILD_ARG>] <COMMAND>
#
# CONTAINER_TYPE: Type of the docker container used the run the build: e.g.,
# (cpu | gpu)
#
# DOCKER_BINARY: Command to invoke docker, e.g. (docker | nvidia-docker).
#
# DOCKERFILE_PATH: (Optional) Path to the Dockerfile used for docker build. If
# this optional value is not supplied (via the --dockerfile
# flag), will use Dockerfile.CONTAINER_TYPE in default
#
# BUILD_ARG: (Optional) an argument to be passed to docker build
#
# COMMAND: Command to be executed in the docker container
#
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -24,6 +29,10 @@ shift 1
DOCKERFILE_PATH="${SCRIPT_DIR}/Dockerfile.${CONTAINER_TYPE}"
DOCKER_CONTEXT_PATH="${SCRIPT_DIR}"
# Get docker binary command (should be either docker or nvidia-docker)
DOCKER_BINARY="$1"
shift 1
if [[ "$1" == "--dockerfile" ]]; then
DOCKERFILE_PATH="$2"
DOCKER_CONTEXT_PATH=$(dirname "${DOCKERFILE_PATH}")
@@ -32,6 +41,11 @@ if [[ "$1" == "--dockerfile" ]]; then
shift 2
fi
if [[ -n "${CI_DOCKER_EXTRA_PARAMS_INIT}" ]]
then
IFS=' ' read -r -a CI_DOCKER_EXTRA_PARAMS <<< "${CI_DOCKER_EXTRA_PARAMS_INIT}"
fi
if [[ "$1" == "-it" ]]; then
CI_DOCKER_EXTRA_PARAMS+=('-it')
shift 1
@@ -61,13 +75,6 @@ if [ "$#" -lt 1 ] || [ ! -e "${SCRIPT_DIR}/Dockerfile.${CONTAINER_TYPE}" ]; then
exit 1
fi
# Use nvidia-docker if the container is GPU.
if [[ "${CONTAINER_TYPE}" == *"gpu"* ]]; then
DOCKER_BINARY="nvidia-docker"
else
DOCKER_BINARY="docker"
fi
# Helper function to traverse directories up until given file is found.
function upsearch () {
test / == "$PWD" && return || \
@@ -84,7 +91,9 @@ DOCKER_IMG_NAME="xgb-ci.${CONTAINER_TYPE}"
# Append cuda version if available
CUDA_VERSION=$(echo "${CI_DOCKER_BUILD_ARG}" | grep CUDA_VERSION | egrep -o '[0-9]*\.[0-9]*')
DOCKER_IMG_NAME=$DOCKER_IMG_NAME$CUDA_VERSION
# Append jdk version if available
JDK_VERSION=$(echo "${CI_DOCKER_BUILD_ARG}" | grep JDK_VERSION | egrep -o '[0-9]*')
DOCKER_IMG_NAME=$DOCKER_IMG_NAME$CUDA_VERSION$JDK_VERSION
# Under Jenkins matrix build, the build tag may contain characters such as
# commas (,) and equal signs (=), which are not valid inside docker image names.
@@ -98,7 +107,7 @@ UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || e
# MSYS, Git Bash, etc.
MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "")
if [[ -z "$UBUNTU_ON_WINDOWS" ]] && [[ -z "$MSYS" ]]; then
if [[ -z "$UBUNTU_ON_WINDOWS" ]] && [[ -z "$MSYS" ]] && [[ ! "$OSTYPE" == "darwin"* ]]; then
USER_IDS="-e CI_BUILD_UID=$( id -u ) -e CI_BUILD_GID=$( id -g ) -e CI_BUILD_USER=$( id -un ) -e CI_BUILD_GROUP=$( id -gn ) -e CI_BUILD_HOME=${WORKSPACE}"
fi
@@ -181,6 +190,7 @@ echo "Running '${COMMAND[*]}' inside ${DOCKER_IMG_NAME}..."
# By default we cleanup - remove the container once it finish running (--rm)
# and share the PID namespace (--pid=host) so the process inside does not have
# pid 1 and SIGKILL is propagated to the process inside (jenkins can kill it).
set -x
${DOCKER_BINARY} run --rm --pid=host \
-v "${WORKSPACE}":/workspace \
-w /workspace \

9
tests/ci_build/doxygen.sh Executable file
View File

@@ -0,0 +1,9 @@
#!/bin/bash
set -e
set -x
rm -rf build
mkdir build
cd build
cmake .. -DBUILD_C_DOC=ON
make -j

19
tests/ci_build/initialize_maven.sh Executable file
View File

@@ -0,0 +1,19 @@
#!/bin/bash
set -e
set -x
if [ -z ${CI_BUILD_USER} ]
then
echo 'Must be run inside Jenkins CI'
exit 1
fi
gosu root mkdir -p /cache
gosu root chown ${CI_BUILD_USER}:${CI_BUILD_GROUP} /cache
# Download cached Maven repository, to speed up build
python3 -m awscli s3 cp s3://xgboost-ci-jenkins-artifacts/maven-repo-cache.tar.bz2 /cache/maven-repo-cache.tar.bz2 || true
if [[ -f "/cache/maven-repo-cache.tar.bz2" ]]
then
tar xvf /cache/maven-repo-cache.tar.bz2 -C ${HOME}
fi

14
tests/ci_build/jenkins_tools.Groovy
View File

@@ -6,20 +6,6 @@
// Command to run command inside a docker container
dockerRun = 'tests/ci_build/ci_build.sh'
// initialize source codes
def checkoutSrcs() {
retry(5) {
try {
timeout(time: 2, unit: 'MINUTES') {
checkout scm
sh 'git submodule update --init'
}
} catch (exc) {
deleteDir()
error "Failed to fetch source codes"
}
}
}
/**
* Creates cmake and make builds

12
tests/ci_build/test_gpu.sh
View File

@@ -1,12 +0,0 @@
#!/usr/bin/env bash
set -e
cd python-package
python setup.py install --user
cd ..
pytest -v -s --fulltrace -m "(not mgpu) and (not slow)" tests/python-gpu
pushd .
cd build
./testxgboost --gtest_filter=-*.MGPU_*
ctest --output-on-failure --tests-regex "TestXGBoostCLI"
popd

43
tests/ci_build/test_jvm_cross.sh Executable file
View File

@@ -0,0 +1,43 @@
#!/bin/bash
set -e
set -x
# Initialize local Maven repository
./tests/ci_build/initialize_maven.sh
# Get version number of XGBoost4J and other auxiliary information
cd jvm-packages
xgboost4j_version=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
maven_compiler_source=$(mvn help:evaluate -Dexpression=maven.compiler.source -q -DforceStdout)
maven_compiler_target=$(mvn help:evaluate -Dexpression=maven.compiler.target -q -DforceStdout)
spark_version=$(mvn help:evaluate -Dexpression=spark.version -q -DforceStdout)
scala_version=$(mvn help:evaluate -Dexpression=scala.version -q -DforceStdout)
scala_binary_version=$(mvn help:evaluate -Dexpression=scala.binary.version -q -DforceStdout)
# Install XGBoost4J JAR into local Maven repository
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j -Dversion=${xgboost4j_version} -Dpackaging=jar
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j-${xgboost4j_version}-tests.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j -Dversion=${xgboost4j_version} -Dpackaging=test-jar -Dclassifier=tests
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-spark/target/xgboost4j-spark-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-spark -Dversion=${xgboost4j_version} -Dpackaging=jar
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-example/target/xgboost4j-example-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-example -Dversion=${xgboost4j_version} -Dpackaging=jar
cd xgboost4j-tester
# Generate pom.xml for XGBoost4J-tester, a dummy project to run XGBoost4J tests
python3 ./generate_pom.py ${xgboost4j_version} ${maven_compiler_source} ${maven_compiler_target} ${spark_version} ${scala_version} ${scala_binary_version}
# Run unit tests with XGBoost4J
mvn --no-transfer-progress package
# Run integration tests with XGBoost4J
java -jar ./target/xgboost4j-tester-1.0-SNAPSHOT-jar-with-dependencies.jar
# Run integration tests with XGBoost4J-Spark
if [ ! -z "$RUN_INTEGRATION_TEST" ]
then
python3 get_iris.py
spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkTraining --master 'local[8]' ./target/xgboost4j-tester-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv
# Disabled due to https://github.com/dmlc/xgboost/issues/4406
#spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkMLlibPipeline --master 'local[8]' ./target/xgboost4j-tester-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv ${PWD}/native_model ${PWD}/pipeline_model
fi
set +x
set +e

11
tests/ci_build/test_mgpu.sh
View File

@@ -1,11 +0,0 @@
#!/usr/bin/env bash
set -e
cd python-package
python setup.py install --user
cd ..
pytest -v -s --fulltrace -m "(not slow) and mgpu" tests/python-gpu
./build/testxgboost --gtest_filter=*.MGPU_*
cd tests/distributed
./runtests-gpu.sh

45
tests/ci_build/test_python.sh Executable file
View File

@@ -0,0 +1,45 @@
#!/bin/bash
set -e
set -x
suite=$1
# Install XGBoost Python package
wheel_found=0
for file in python-package/dist/*.whl
do
pip install --user ${file}
wheel_found=1
break # need just one
done
if [ "$wheel_found" -eq 0 ]
then
pushd .
cd python-package
python setup.py install --user
popd
fi
# Run specified test suite
case "$suite" in
gpu)
pytest -v -s --fulltrace -m "(not slow) and (not mgpu)" tests/python-gpu
;;
mgpu)
pytest -v -s --fulltrace -m "(not slow) and mgpu" tests/python-gpu
cd tests/distributed
./runtests-gpu.sh
;;
cpu)
pytest -v -s --fulltrace tests/python
cd tests/distributed
./runtests.sh
;;
*)
echo "Usage: $0 {gpu|mgpu|cpu}"
exit 1
;;
esac

1
tests/distributed/runtests-gpu.sh
View File

@@ -3,7 +3,6 @@
rm -f *.model*
export DMLC_SUBMIT_CLUSTER=local
export PYTHONPATH=../../python-package
submit="timeout 30 python ../../dmlc-core/tracker/dmlc-submit"
echo -e "\n ====== 1. Basic distributed-gpu test with Python: 4 workers; 1 GPU per worker ====== \n"

12
tests/distributed/runtests.sh
View File

@@ -1,9 +1,13 @@
#!/bin/bash
rm -f *.model*
export DMLC_SUBMIT_CLUSTER=local
submit="timeout 30 python ../../dmlc-core/tracker/dmlc-submit"
echo "====== 1. Basic distributed test with Python ======"
PYTHONPATH=../../python-package/ python ../../dmlc-core/tracker/dmlc-submit --cluster=local --num-workers=3\
python test_basic.py
$submit --cluster=local --num-workers=3 python test_basic.py
echo "====== 2. Regression test for issue #3402 ======"
PYTHONPATH=../../python-package/ python ../../dmlc-core/tracker/dmlc-submit --cluster=local --num-workers=2 --worker-cores=1\
python test_issue3402.py
$submit --cluster=local --num-workers=2 --worker-cores=1 python test_issue3402.py

168
tests/travis/run_test.sh
View File

@@ -1,34 +1,8 @@
#!/bin/bash
if [ ${TASK} == "lint" ]; then
source activate python3
conda install numpy scipy
python -m pip install cpplint pylint astroid
make lint || exit -1
echo "Check documentations..."
mkdir build_doc
cd build_doc
cmake .. -DBUILD_C_DOC=ON
make doc_doxygen 2> log.txt
(cat log.txt| grep -v ENABLE_PREPROCESSING |grep -v "unsupported tag") > logclean.txt
echo "---------Error Log----------"
cat logclean.txt
echo "----------------------------"
(cat logclean.txt|grep warning) && exit -1
(cat logclean.txt|grep error) && exit -1
cd -
rm -rf build_doc
exit 0
fi
cp make/travis.mk config.mk
make -f dmlc-core/scripts/packages.mk lz4
if [ ${TRAVIS_OS_NAME} == "osx" ]; then
echo 'USE_OPENMP=0' >> config.mk
echo 'TMPVAR := $(XGB_PLUGINS)' >> config.mk
@@ -45,96 +19,10 @@ if [ ${TASK} == "python_test" ]; then
python --version
conda install numpy scipy pandas matplotlib scikit-learn
# Install data table from source
wget http://releases.llvm.org/5.0.2/clang+llvm-5.0.2-x86_64-linux-gnu-ubuntu-14.04.tar.xz
tar xf clang+llvm-5.0.2-x86_64-linux-gnu-ubuntu-14.04.tar.xz
export LLVM5=$(pwd)/clang+llvm-5.0.2-x86_64-linux-gnu-ubuntu-14.04
python -m pip install datatable --no-binary datatable
python -m pip install graphviz pytest pytest-cov codecov
python -m pip install https://h2o-release.s3.amazonaws.com/datatable/stable/datatable-0.7.0/datatable-0.7.0-cp37-cp37m-linux_x86_64.whl
python -m pytest -v --fulltrace -s tests/python --cov=python-package/xgboost || exit -1
codecov
source activate python2
echo "-------------------------------"
python --version
conda install numpy scipy pandas matplotlib scikit-learn
python -m pip install graphviz pytest
python -m pytest -v --fulltrace -s tests/python || exit -1
exit 0
fi
if [ ${TASK} == "python_lightweight_test" ]; then
make all || exit -1
echo "-------------------------------"
source activate python3
python --version
conda install numpy scipy
python -m pip install graphviz pytest pytest-cov codecov
python -m pytest -v --fulltrace -s tests/python --cov=python-package/xgboost || exit -1
codecov
source activate python2
echo "-------------------------------"
python --version
conda install numpy scipy pytest
python -m pip install graphviz
python -m pip install flake8==3.4.1
python -m pytest -v --fulltrace -s tests/python || exit -1
flake8 --ignore E501 python-package || exit -1
flake8 --ignore E501 tests/python || exit -1
exit 0
fi
if [ ${TASK} == "r_test" ]; then
set -e
make Rpack
cd ./xgboost
# Install package deps
Rscript -e "install.packages( \
c('devtools', 'testthat', 'lintr') \
, repos = 'http://cloud.r-project.org' \
, dependencies = c('Depends', 'Imports', 'LinkingTo') \
)"
Rscript -e \
"devtools::install_deps( \
repos = 'http://cloud.r-project.org' \
, upgrade = 'never' \
, dependencies = c('Depends', 'Imports', 'LinkingTo') \
)"
# install suggested packages separately to avoid huge build times
Rscript -e "install.packages( \
c('DiagrammeR', 'Ckmeans.1d.dp', 'vcd') \
, repos = 'https://cloud.r-project.org' \
, dependencies = c('Depends', 'Imports', 'LinkingTo') \
)"
# Run tests
echo "Building with R CMD build"
R CMD build \
--no-build-vignettes \
--no-manual \
.
echo "Running R tests"
R_PACKAGE_TARBALL=$(ls -1t *.tar.gz | head -n 1)
export _R_CHECK_TIMINGS_=0
export _R_CHECK_FORCE_SUGGESTS_=false
R CMD check \
${R_PACKAGE_TARBALL} \
--no-vignettes \
--no-manual \
--as-cran \
--install-args=--build
exit 0
fi
if [ ${TASK} == "java_test" ]; then
@@ -150,7 +38,7 @@ if [ ${TASK} == "cmake_test" ]; then
wget -nc https://github.com/google/googletest/archive/release-1.7.0.zip
unzip -n release-1.7.0.zip
mv googletest-release-1.7.0 gtest && cd gtest
cmake . && make
CC=gcc-7 CXX=g++-7 cmake . && make
mkdir lib && mv libgtest.a lib
cd ..
rm -rf release-1.7.0.zip
@@ -159,59 +47,9 @@ if [ ${TASK} == "cmake_test" ]; then
rm -rf build
mkdir build && cd build
PLUGINS="-DPLUGIN_LZ4=ON -DPLUGIN_DENSE_PARSER=ON"
cmake .. -DGOOGLE_TEST=ON -DGTEST_ROOT=$PWD/../gtest/ ${PLUGINS}
CC=gcc-7 CXX=g++-7 cmake .. -DGOOGLE_TEST=ON -DGTEST_ROOT=$PWD/../gtest/ ${PLUGINS}
make
./testxgboost
cd ..
rm -rf build
fi
if [ ${TASK} == "cpp_test" ]; then
set -e
make -f dmlc-core/scripts/packages.mk gtest
echo "TEST_COVER=1" >> config.mk
echo "GTEST_PATH="${CACHE_PREFIX} >> config.mk
make cover
fi
if [ ${TASK} == "distributed_test" ]; then
set -e
make all || exit -1
echo "-------------------------------"
source activate python3
python --version
conda install numpy scipy
python -m pip install kubernetes
cd tests/distributed
./runtests.sh
fi
if [ ${TASK} == "sanitizer_test" ]; then
set -e
# Build gtest via cmake
wget -nc https://github.com/google/googletest/archive/release-1.7.0.zip
unzip -n release-1.7.0.zip
mv googletest-release-1.7.0 gtest && cd gtest
CC=gcc-7 CXX=g++-7 cmake -DCMAKE_CXX_FLAGS="-fuse-ld=gold" \
-DCMAKE_C_FLAGS="-fuse-ld=gold"
make
mkdir lib && mv libgtest.a lib
cd ..
rm -rf release-1.7.0.zip
mkdir build && cd build
CC=gcc-7 CXX=g++-7 cmake .. -DGOOGLE_TEST=ON -DGTEST_ROOT=$PWD/../gtest/ \
-DUSE_SANITIZER=ON -DENABLED_SANITIZERS="address" \
-DCMAKE_BUILD_TYPE=Debug \
-DSANITIZER_PATH=/usr/lib/x86_64-linux-gnu/ \
-DCMAKE_CXX_FLAGS="-fuse-ld=gold" \
-DCMAKE_C_FLAGS="-fuse-ld=gold"
make
export ASAN_SYMBOLIZER_PATH=$(which llvm-symbolizer)
ASAN_OPTIONS=symbolize=1 ./testxgboost
cd ..
rm -rf build
exit 0
fi

20
tests/travis/setup.sh
View File

@@ -1,6 +1,6 @@
#!/bin/bash
if [ ${TASK} == "lint" ]; then
if [ ${TASK} == "python_test" ]; then
if [ ${TRAVIS_OS_NAME} == "osx" ]; then
wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
else
@@ -16,20 +16,6 @@ if [ ${TASK} == "lint" ]; then
conda create -n python3 python=3.7
fi
if [ ${TASK} == "python_test" ] || [ ${TASK} == "python_lightweight_test" ] || [ ${TASK} == "distributed_test" ]; then
if [ ${TRAVIS_OS_NAME} == "osx" ]; then
wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
else
wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
fi
bash conda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
hash -r
conda config --set always_yes yes --set changeps1 no
conda update -q conda
# Useful for debugging any issues with conda
conda info -a
conda create -n python3 python=3.7
conda create -n python2 python=2.7
if [ ${TASK} == "cmake_test" ] && [ ${TRAVIS_OS_NAME} == "osx" ]; then
sudo softwareupdate -i "Command Line Tools (macOS High Sierra version 10.13) for Xcode-9.3"
fi