Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CI] Upgrade build image to CentOS 7 + GCC 8; require CUDA 10.1 and later (#7141)

Browse Source
This commit is contained in:
Philip Hyunsu Cho 2021-07-29 10:54:33 -07:00 committed by GitHub
parent dfdf0b08fc
commit f1a4a1ac95
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 156 additions and 274 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CMakeLists.txt
View File

@ -125,8 +125,8 @@ if (USE_CUDA)
message(STATUS "Configured CUDA host compiler: ${CMAKE_CUDA_HOST_COMPILER}") message(STATUS "Configured CUDA host compiler: ${CMAKE_CUDA_HOST_COMPILER}")
enable_language(CUDA) enable_language(CUDA)
if (${CMAKE_CUDA_COMPILER_VERSION} VERSION_LESS 10.0) if (${CMAKE_CUDA_COMPILER_VERSION} VERSION_LESS 10.1)
message(FATAL_ERROR "CUDA version must be at least 10.0!") message(FATAL_ERROR "CUDA version must be at least 10.1!")
endif() endif()
set(GEN_CODE "") set(GEN_CODE "")
format_gencode_flags("${GPU_COMPUTE_VER}" GEN_CODE) format_gencode_flags("${GPU_COMPUTE_VER}" GEN_CODE)

28
Jenkinsfile vendored
View File

@ -7,7 +7,7 @@
dockerRun = 'tests/ci_build/ci_build.sh' dockerRun = 'tests/ci_build/ci_build.sh'
// Which CUDA version to use when building reference distribution wheel // Which CUDA version to use when building reference distribution wheel
ref_cuda_ver = '10.0' ref_cuda_ver = '10.1'
import groovy.transform.Field import groovy.transform.Field
@ -58,13 +58,13 @@ pipeline {
'build-cpu': { BuildCPU() }, 'build-cpu': { BuildCPU() },
'build-cpu-arm64': { BuildCPUARM64() }, 'build-cpu-arm64': { BuildCPUARM64() },
'build-cpu-rabit-mock': { BuildCPUMock() }, 'build-cpu-rabit-mock': { BuildCPUMock() },
// Build reference, distribution-ready Python wheel with CUDA 10.0 // Build reference, distribution-ready Python wheel with CUDA 10.1
// using CentOS 6 image // using CentOS 7 image
'build-gpu-cuda10.0': { BuildCUDA(cuda_version: '10.0') }, 'build-gpu-cuda10.1': { BuildCUDA(cuda_version: '10.1') },
// The build-gpu-* builds below use Ubuntu image // The build-gpu-* builds below use Ubuntu image
'build-gpu-cuda11.0': { BuildCUDA(cuda_version: '11.0', build_rmm: true) }, 'build-gpu-cuda11.0': { BuildCUDA(cuda_version: '11.0', build_rmm: true) },
'build-gpu-rpkg': { BuildRPackageWithCUDA(cuda_version: '10.0') }, 'build-gpu-rpkg': { BuildRPackageWithCUDA(cuda_version: '10.1') },
'build-jvm-packages-gpu-cuda10.0': { BuildJVMPackagesWithCUDA(spark_version: '3.0.0', cuda_version: '10.0') }, 'build-jvm-packages-gpu-cuda10.1': { BuildJVMPackagesWithCUDA(spark_version: '3.0.0', cuda_version: '10.1') },
'build-jvm-packages': { BuildJVMPackages(spark_version: '3.0.0') }, 'build-jvm-packages': { BuildJVMPackages(spark_version: '3.0.0') },
'build-jvm-doc': { BuildJVMDoc() } 'build-jvm-doc': { BuildJVMDoc() }
]) ])
@ -79,9 +79,9 @@ pipeline {
'test-python-cpu': { TestPythonCPU() }, 'test-python-cpu': { TestPythonCPU() },
'test-python-cpu-arm64': { TestPythonCPUARM64() }, 'test-python-cpu-arm64': { TestPythonCPUARM64() },
// artifact_cuda_version doesn't apply to RMM tests; RMM tests will always match CUDA version between artifact and host env // artifact_cuda_version doesn't apply to RMM tests; RMM tests will always match CUDA version between artifact and host env
'test-python-gpu-cuda11.0-cross': { TestPythonGPU(artifact_cuda_version: '10.0', host_cuda_version: '11.0', test_rmm: true) }, 'test-python-gpu-cuda11.0-cross': { TestPythonGPU(artifact_cuda_version: '10.1', host_cuda_version: '11.0', test_rmm: true) },
'test-python-gpu-cuda11.0': { TestPythonGPU(artifact_cuda_version: '11.0', host_cuda_version: '11.0') }, 'test-python-gpu-cuda11.0': { TestPythonGPU(artifact_cuda_version: '11.0', host_cuda_version: '11.0') },
'test-python-mgpu-cuda11.0': { TestPythonGPU(artifact_cuda_version: '10.0', host_cuda_version: '11.0', multi_gpu: true, test_rmm: true) }, 'test-python-mgpu-cuda11.0': { TestPythonGPU(artifact_cuda_version: '10.1', host_cuda_version: '11.0', multi_gpu: true, test_rmm: true) },
'test-cpp-gpu-cuda11.0': { TestCppGPU(artifact_cuda_version: '11.0', host_cuda_version: '11.0', test_rmm: true) }, 'test-cpp-gpu-cuda11.0': { TestCppGPU(artifact_cuda_version: '11.0', host_cuda_version: '11.0', test_rmm: true) },
'test-jvm-jdk8': { CrossTestJVMwithJDK(jdk_version: '8', spark_version: '3.0.0') }, 'test-jvm-jdk8': { CrossTestJVMwithJDK(jdk_version: '8', spark_version: '3.0.0') },
'test-jvm-jdk11': { CrossTestJVMwithJDK(jdk_version: '11') }, 'test-jvm-jdk11': { CrossTestJVMwithJDK(jdk_version: '11') },
@ -119,7 +119,7 @@ def checkoutSrcs() {
} }
def GetCUDABuildContainerType(cuda_version) { def GetCUDABuildContainerType(cuda_version) {
return (cuda_version == ref_cuda_ver) ? 'gpu_build_centos6' : 'gpu_build' return (cuda_version == ref_cuda_ver) ? 'gpu_build_centos7' : 'gpu_build'
} }
def ClangTidy() { def ClangTidy() {
@ -218,7 +218,7 @@ def BuildCUDA(args) {
if (env.BRANCH_NAME != 'master' && !(env.BRANCH_NAME.startsWith('release'))) { if (env.BRANCH_NAME != 'master' && !(env.BRANCH_NAME.startsWith('release'))) {
arch_flag = "-DGPU_COMPUTE_VER=75" arch_flag = "-DGPU_COMPUTE_VER=75"
} }
def wheel_tag = "manylinux2010_x86_64" def wheel_tag = "manylinux2014_x86_64"
sh """ sh """
${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_CUDA=ON -DUSE_NCCL=ON -DOPEN_MP:BOOL=ON -DHIDE_CXX_SYMBOLS=ON ${arch_flag} ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_CUDA=ON -DUSE_NCCL=ON -DOPEN_MP:BOOL=ON -DHIDE_CXX_SYMBOLS=ON ${arch_flag}
${dockerRun} ${container_type} ${docker_binary} ${docker_args} bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal" ${dockerRun} ${container_type} ${docker_binary} ${docker_args} bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal"
@ -251,7 +251,7 @@ def BuildCUDA(args) {
rm -rf build/ rm -rf build/
${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh --conda-env=gpu_test -DUSE_CUDA=ON -DUSE_NCCL=ON -DPLUGIN_RMM=ON ${arch_flag} ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh --conda-env=gpu_test -DUSE_CUDA=ON -DUSE_NCCL=ON -DPLUGIN_RMM=ON ${arch_flag}
${dockerRun} ${container_type} ${docker_binary} ${docker_args} bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal" ${dockerRun} ${container_type} ${docker_binary} ${docker_args} bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal"
${dockerRun} ${container_type} ${docker_binary} ${docker_args} python tests/ci_build/rename_whl.py python-package/dist/*.whl ${commit_id} manylinux2010_x86_64 ${dockerRun} ${container_type} ${docker_binary} ${docker_args} python tests/ci_build/rename_whl.py python-package/dist/*.whl ${commit_id} manylinux2014_x86_64
""" """
echo 'Stashing Python wheel...' echo 'Stashing Python wheel...'
stash name: "xgboost_whl_rmm_cuda${args.cuda_version}", includes: 'python-package/dist/*.whl' stash name: "xgboost_whl_rmm_cuda${args.cuda_version}", includes: 'python-package/dist/*.whl'
@ -265,9 +265,9 @@ def BuildCUDA(args) {
def BuildRPackageWithCUDA(args) { def BuildRPackageWithCUDA(args) {
node('linux && cpu_build') { node('linux && cpu_build') {
unstash name: 'srcs' unstash name: 'srcs'
def container_type = 'gpu_build_r_centos6' def container_type = 'gpu_build_r_centos7'
def docker_binary = "docker" def docker_binary = "docker"
def docker_args = "--build-arg CUDA_VERSION_ARG=10.0" def docker_args = "--build-arg CUDA_VERSION_ARG=${args.cuda_version}"
if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) { if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) {
sh """ sh """
${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_r_pkg_with_cuda.sh ${commit_id} ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_r_pkg_with_cuda.sh ${commit_id}
@ -445,7 +445,7 @@ def DeployJVMPackages(args) {
if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) { if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) {
echo 'Deploying to xgboost-maven-repo S3 repo...' echo 'Deploying to xgboost-maven-repo S3 repo...'
sh """ sh """
${dockerRun} jvm_gpu_build docker --build-arg CUDA_VERSION_ARG=10.0 tests/ci_build/deploy_jvm_packages.sh ${args.spark_version} ${dockerRun} jvm_gpu_build docker --build-arg CUDA_VERSION_ARG=10.1 tests/ci_build/deploy_jvm_packages.sh ${args.spark_version}
""" """
} }
deleteDir() deleteDir()

2
dev/release-pypi.py
View File

@ -85,7 +85,7 @@ def main(args: argparse.Namespace) -> None:
rel = version.StrictVersion(args.release) rel = version.StrictVersion(args.release)
platforms = [ platforms = [
"win_amd64", "win_amd64",
"manylinux2010_x86_64", "manylinux2014_x86_64",
"manylinux2014_aarch64", "manylinux2014_aarch64",
"macosx_10_14_x86_64.macosx_10_15_x86_64.macosx_11_0_x86_64", "macosx_10_14_x86_64.macosx_10_15_x86_64.macosx_11_0_x86_64",
] ]

37
tests/ci_build/CentOS-Base.repo
View File

@ -1,37 +0,0 @@
[base]
name=CentOS-$releasever - Base
baseurl=http://vault.centos.org/centos/$releasever/os/$basearch/
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
#released updates
[updates]
name=CentOS-$releasever - Updates
baseurl=http://vault.centos.org/centos/$releasever/updates/$basearch/
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
#additional packages that may be useful
[extras]
name=CentOS-$releasever - Extras
baseurl=http://vault.centos.org/centos/$releasever/extras/$basearch/
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
#additional packages that extend functionality of existing packages
[centosplus]
name=CentOS-$releasever - Plus
mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=centosplus&infra=$infra
#baseurl=http://mirror.centos.org/centos/$releasever/centosplus/$basearch/
gpgcheck=1
enabled=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
#contrib - packages by Centos Users
[contrib]
name=CentOS-$releasever - Contrib
mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=contrib&infra=$infra
#baseurl=http://mirror.centos.org/centos/$releasever/contrib/$basearch/
gpgcheck=1
enabled=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6

2
tests/ci_build/Dockerfile.auditwheel_x86_64
View File

@ -1,4 +1,4 @@
FROM quay.io/pypa/manylinux2010_x86_64 FROM quay.io/pypa/manylinux2014_x86_64
# Install lightweight sudo (not bound to TTY) # Install lightweight sudo (not bound to TTY)
ENV GOSU_VERSION 1.10 ENV GOSU_VERSION 1.10

8
tests/ci_build/Dockerfile.cpu
View File

@ -7,7 +7,10 @@ SHELL ["/bin/bash", "-c"] # Use Bash as shell
# Install all basic requirements # Install all basic requirements
RUN \ RUN \
apt-get update && \ apt-get update && \
apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libasan2 libidn11 ninja-build && \ apt-get install -y software-properties-common && \
add-apt-repository ppa:ubuntu-toolchain-r/test && \
apt-get update && \
apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libasan2 libidn11 ninja-build gcc-8 g++-8 && \
# CMake # CMake
wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \ wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \
bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \ bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
@ -16,6 +19,9 @@ RUN \
bash Miniconda3.sh -b -p /opt/python bash Miniconda3.sh -b -p /opt/python
ENV PATH=/opt/python/bin:$PATH ENV PATH=/opt/python/bin:$PATH
ENV CC=gcc-8
ENV CXX=g++-8
ENV CPP=cpp-8
ENV GOSU_VERSION 1.10 ENV GOSU_VERSION 1.10

8
tests/ci_build/Dockerfile.gpu_build
View File

@ -9,7 +9,10 @@ SHELL ["/bin/bash", "-c"] # Use Bash as shell
# Install all basic requirements # Install all basic requirements
RUN \ RUN \
apt-get update && \ apt-get update && \
apt-get install -y tar unzip wget bzip2 libgomp1 git build-essential doxygen graphviz llvm libasan2 libidn11 ninja-build && \ apt-get install -y software-properties-common && \
add-apt-repository ppa:ubuntu-toolchain-r/test && \
apt-get update && \
apt-get install -y tar unzip wget bzip2 libgomp1 git build-essential doxygen graphviz llvm libasan2 libidn11 ninja-build gcc-8 g++-8 && \
# CMake # CMake
wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \ wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \
bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \ bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
@ -25,6 +28,9 @@ RUN \
apt-get install -y --allow-downgrades --allow-change-held-packages libnccl2=${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-dev=${NCCL_VERSION}+cuda${CUDA_SHORT} apt-get install -y --allow-downgrades --allow-change-held-packages libnccl2=${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-dev=${NCCL_VERSION}+cuda${CUDA_SHORT}
ENV PATH=/opt/python/bin:$PATH ENV PATH=/opt/python/bin:$PATH
ENV CC=gcc-8
ENV CXX=g++-8
ENV CPP=cpp-8
ENV GOSU_VERSION 1.10 ENV GOSU_VERSION 1.10

64
tests/ci_build/Dockerfile.gpu_build_centos6
View File

@ -1,64 +0,0 @@
ARG CUDA_VERSION_ARG
FROM nvidia/cuda:$CUDA_VERSION_ARG-devel-centos6
ARG CUDA_VERSION_ARG
# Environment
ENV DEBIAN_FRONTEND noninteractive
ENV DEVTOOLSET_URL_ROOT http://vault.centos.org/6.9/sclo/x86_64/rh/devtoolset-4/
COPY CentOS-Base.repo /etc/yum.repos.d/
# Install all basic requirements
RUN \
yum install -y epel-release && \
yum -y update && \
yum install -y tar unzip wget xz git patchelf && \
yum install -y $DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-5.3.1-6.1.el6.x86_64.rpm \
$DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-c++-5.3.1-6.1.el6.x86_64.rpm \
$DEVTOOLSET_URL_ROOT/devtoolset-4-binutils-2.25.1-8.el6.x86_64.rpm \
$DEVTOOLSET_URL_ROOT/devtoolset-4-runtime-4.1-3.sc1.el6.x86_64.rpm \
$DEVTOOLSET_URL_ROOT/devtoolset-4-libstdc++-devel-5.3.1-6.1.el6.x86_64.rpm && \
# Python
wget -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
bash Miniconda3.sh -b -p /opt/python && \
/opt/python/bin/python -m pip install auditwheel && \
# CMake
wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \
bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
# Ninja
mkdir -p /usr/local && \
cd /usr/local/ && \
wget -nv -nc https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz --no-check-certificate && \
tar xf v1.10.0.tar.gz && mv ninja-1.10.0 ninja && rm -v v1.10.0.tar.gz && \
cd ninja && \
/opt/python/bin/python ./configure.py --bootstrap
# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
RUN \
export CUDA_SHORT=`echo $CUDA_VERSION_ARG | grep -o -E '[0-9]+\.[0-9]'` && \
export NCCL_VERSION=2.4.8-1 && \
wget https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
yum -y update && \
yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT} && \
rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm;
ENV PATH=/opt/python/bin:/usr/local/ninja:$PATH
ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
ENV GOSU_VERSION 1.10
# Install lightweight sudo (not bound to TTY)
RUN set -ex; \
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
chmod +x /usr/local/bin/gosu && \
gosu nobody true
# Default entry-point to use if running locally
# It will preserve attributes of created files
COPY entrypoint.sh /scripts/
WORKDIR /workspace
ENTRYPOINT ["/scripts/entrypoint.sh"]

46
tests/ci_build/Dockerfile.gpu_build_centos7 Normal file
View File

@ -0,0 +1,46 @@
ARG CUDA_VERSION_ARG
FROM nvidia/cuda:$CUDA_VERSION_ARG-devel-centos7
ARG CUDA_VERSION_ARG
# Install all basic requirements
RUN \
yum install -y epel-release centos-release-scl && \
yum-config-manager --enable centos-sclo-rh-testing && \
yum -y update && \
yum install -y tar unzip wget xz git which ninja-build devtoolset-8-gcc devtoolset-8-binutils devtoolset-8-gcc-c++ && \
# Python
wget -nv -nc -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
bash Miniconda3.sh -b -p /opt/python && \
# CMake
wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \
bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr
# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
RUN \
export CUDA_SHORT=`echo $CUDA_VERSION_ARG | grep -o -E '[0-9]+\.[0-9]'` && \
export NCCL_VERSION=2.4.8-1 && \
wget -nv -nc https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
yum -y update && \
yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT} && \
rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm;
ENV PATH=/opt/python/bin:/usr/local/ninja:$PATH
ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc
ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/c++
ENV CPP=/opt/rh/devtoolset-8/root/usr/bin/cpp
ENV GOSU_VERSION 1.10
# Install lightweight sudo (not bound to TTY)
RUN set -ex; \
wget -nv -nc -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
chmod +x /usr/local/bin/gosu && \
gosu nobody true
# Default entry-point to use if running locally
# It will preserve attributes of created files
COPY entrypoint.sh /scripts/
WORKDIR /workspace
ENTRYPOINT ["/scripts/entrypoint.sh"]

112
tests/ci_build/Dockerfile.gpu_build_r_centos6
View File

@ -1,112 +0,0 @@
ARG CUDA_VERSION_ARG
FROM nvidia/cuda:$CUDA_VERSION_ARG-devel-centos6
ARG CUDA_VERSION_ARG
# Environment
ENV DEBIAN_FRONTEND noninteractive
ENV DEVTOOLSET_URL_ROOT http://vault.centos.org/6.9/sclo/x86_64/rh/devtoolset-4/
COPY CentOS-Base.repo /etc/yum.repos.d/
# Install all basic requirements
RUN \
yum install -y epel-release && \
yum -y update && \
yum install -y tar unzip wget xz git patchelf readline-devel libX11-devel libXt-devel \
xorg-x11-server-devel openssl-devel texlive-* && \
yum install -y $DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-5.3.1-6.1.el6.x86_64.rpm \
$DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-gfortran-5.3.1-6.1.el6.x86_64.rpm \
$DEVTOOLSET_URL_ROOT/devtoolset-4-libquadmath-devel-5.3.1-6.1.el6.x86_64.rpm \
$DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-c++-5.3.1-6.1.el6.x86_64.rpm \
$DEVTOOLSET_URL_ROOT/devtoolset-4-binutils-2.25.1-8.el6.x86_64.rpm \
$DEVTOOLSET_URL_ROOT/devtoolset-4-runtime-4.1-3.sc1.el6.x86_64.rpm \
$DEVTOOLSET_URL_ROOT/devtoolset-4-libstdc++-devel-5.3.1-6.1.el6.x86_64.rpm
ENV PATH=/opt/python/bin:/usr/local/ninja:/opt/software/packages/bin:/opt/R/3.3.0/bin:$PATH
ENV LD_LIBRARY_PATH=/opt/software/packages/lib:/opt/R/3.3.0/lib64:$LD_LIBRARY_PATH
ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
ENV F77=/opt/rh/devtoolset-4/root/usr/bin/gfortran
# A few packages have to be built from the source because CentOS 6 is a very old distribution and
# the system packages are not sufficiently up-to-date to build R 3.3.0. We'll want to update to
# CentOS 7 after the 1.4.0 release. Tracking issue: dmlc/xgboost#6791.
#
# Why choose an old Linux distro? This is so that the resulting xgboost.so is compatible with a
# wide range of Linux OSes currently in operation. See https://www.python.org/dev/peps/pep-0571/
RUN \
wget https://zlib.net/fossils/zlib-1.2.5.tar.gz && \
wget https://sourceware.org/pub/bzip2/bzip2-1.0.6.tar.gz && \
wget http://tukaani.org/xz/xz-5.2.2.tar.gz && \
wget https://ftp.pcre.org/pub/pcre/pcre-8.40.tar.gz && \
wget https://www.openssl.org/source/old/1.0.0/openssl-1.0.0k.tar.gz && \
wget --no-check-certificate https://curl.se/download/curl-7.47.1.tar.gz && \
tar xf zlib-1.2.5.tar.gz && \
tar xf bzip2-1.0.6.tar.gz && \
tar xf xz-5.2.2.tar.gz && \
tar xf pcre-8.40.tar.gz && \
tar xf openssl-1.0.0k.tar.gz && \
tar xf curl-7.47.1.tar.gz && \
cd zlib-1.2.5 && \
./configure --prefix=/opt/software/packages && \
make -j$(nproc) && \
make install && \
cd ../bzip2-1.0.6 && \
sed -i 's/CFLAGS=-Wall/CFLAGS=-fPIC -Wall/g' Makefile && \
make -f Makefile-libbz2_so && \
make clean && \
make -j$(nproc) && \
make -n install PREFIX=/opt/software/packages && \
make install PREFIX=/opt/software/packages && \
cd ../xz-5.2.2 && \
./configure --prefix=/opt/software/packages && \
make -j$(nproc) && \
make install && \
cd ../pcre-8.40 && \
./configure --enable-utf8 --prefix=/opt/software/packages && \
make -j$(nproc) && \
make install && \
cd ../curl-7.47.1 && \
./configure --prefix=/opt/software/packages --with-ssl && \
make -j$(nproc) && \
make install && \
export CFLAGS="-I/opt/software/packages/include" && \
export LDFLAGS="-L/opt/software/packages/lib" && \
cd .. && \
# R 3.3.0
wget -nv -nc https://cran.r-project.org/src/base/R-3/R-3.3.0.tar.gz && \
tar xf R-3.3.0.tar.gz && \
cd R-3.3.0 && \
./configure --prefix=/opt/R/3.3.0 --enable-R-shlib && \
make -j$(nproc) && \
make install && \
# Python
wget -nv -nc -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
bash Miniconda3.sh -b -p /opt/python && \
/opt/python/bin/python -m pip install auditwheel && \
# CMake
wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \
bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
# Ninja
mkdir -p /usr/local && \
cd /usr/local/ && \
wget -nv -nc https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz --no-check-certificate && \
tar xf v1.10.0.tar.gz && mv ninja-1.10.0 ninja && rm -v v1.10.0.tar.gz && \
cd ninja && \
/opt/python/bin/python ./configure.py --bootstrap
ENV GOSU_VERSION 1.10
# Install lightweight sudo (not bound to TTY)
RUN set -ex; \
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
chmod +x /usr/local/bin/gosu && \
gosu nobody true
# Default entry-point to use if running locally
# It will preserve attributes of created files
COPY entrypoint.sh /scripts/
WORKDIR /workspace
ENTRYPOINT ["/scripts/entrypoint.sh"]

53
tests/ci_build/Dockerfile.gpu_build_r_centos7 Normal file
View File

@ -0,0 +1,53 @@
ARG CUDA_VERSION_ARG
FROM nvidia/cuda:$CUDA_VERSION_ARG-devel-centos7
ARG CUDA_VERSION_ARG
# Install all basic requirements
RUN \
yum install -y epel-release centos-release-scl && \
yum-config-manager --enable centos-sclo-rh-testing && \
yum -y update && \
yum install -y tar unzip wget xz git which ninja-build readline-devel libX11-devel libXt-devel \
xorg-x11-server-devel openssl-devel zlib-devel bzip2-devel xz-devel \
pcre-devel libcurl-devel texlive-* \
devtoolset-8-gcc devtoolset-8-binutils devtoolset-8-gcc-c++ \
devtoolset-8-gcc-gfortran devtoolset-8-libquadmath-devel \
devtoolset-8-runtime devtoolset-8-libstdc++-devel
ENV PATH=/opt/python/bin:/usr/local/ninja:/opt/software/packages/bin:/opt/R/3.3.0/bin:$PATH
ENV LD_LIBRARY_PATH=/opt/software/packages/lib:/opt/R/3.3.0/lib64:$LD_LIBRARY_PATH
ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc
ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/c++
ENV CPP=/opt/rh/devtoolset-8/root/usr/bin/cpp
ENV F77=/opt/rh/devtoolset-8/root/usr/bin/gfortran
# R 3.3.0
RUN \
wget -nv -nc https://cran.r-project.org/src/base/R-3/R-3.3.0.tar.gz && \
tar xf R-3.3.0.tar.gz && \
cd R-3.3.0 && \
./configure --prefix=/opt/R/3.3.0 --enable-R-shlib && \
make -j$(nproc) && \
make install && \
# Python
wget -nv -nc -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
bash Miniconda3.sh -b -p /opt/python && \
/opt/python/bin/python -m pip install auditwheel && \
# CMake
wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \
bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr
ENV GOSU_VERSION 1.10
# Install lightweight sudo (not bound to TTY)
RUN set -ex; \
wget -nv -nc -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
chmod +x /usr/local/bin/gosu && \
gosu nobody true
# Default entry-point to use if running locally
# It will preserve attributes of created files
COPY entrypoint.sh /scripts/
WORKDIR /workspace
ENTRYPOINT ["/scripts/entrypoint.sh"]

32
tests/ci_build/Dockerfile.jvm
View File

@ -1,44 +1,38 @@
FROM centos:6 FROM centos:7
ENV DEVTOOLSET_URL_ROOT http://vault.centos.org/6.9/sclo/x86_64/rh/devtoolset-4/
COPY CentOS-Base.repo /etc/yum.repos.d/
# Install all basic requirements # Install all basic requirements
RUN \ RUN \
yum install -y epel-release && \ yum install -y epel-release centos-release-scl && \
yum-config-manager --enable centos-sclo-rh-testing && \
yum -y update && \ yum -y update && \
yum install -y tar unzip wget xz git java-1.8.0-openjdk-devel && \ yum install -y tar unzip make bzip2 wget xz git which ninja-build java-1.8.0-openjdk-devel \
yum install -y $DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-5.3.1-6.1.el6.x86_64.rpm \ devtoolset-8-gcc devtoolset-8-binutils devtoolset-8-gcc-c++ \
$DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-c++-5.3.1-6.1.el6.x86_64.rpm \ devtoolset-8-runtime devtoolset-8-libstdc++-devel && \
$DEVTOOLSET_URL_ROOT/devtoolset-4-binutils-2.25.1-8.el6.x86_64.rpm \
$DEVTOOLSET_URL_ROOT/devtoolset-4-runtime-4.1-3.sc1.el6.x86_64.rpm \
$DEVTOOLSET_URL_ROOT/devtoolset-4-libstdc++-devel-5.3.1-6.1.el6.x86_64.rpm && \
# Python # Python
wget -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ wget -nv -nc -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
bash Miniconda3.sh -b -p /opt/python && \ bash Miniconda3.sh -b -p /opt/python && \
# CMake # CMake
wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \ wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \
bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \ bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
# Maven # Maven
wget https://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \ wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \
tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \ tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \
ln -s /opt/apache-maven-3.6.1/ /opt/maven ln -s /opt/apache-maven-3.6.1/ /opt/maven
ENV PATH=/opt/python/bin:/opt/maven/bin:$PATH ENV PATH=/opt/python/bin:/opt/maven/bin:$PATH
ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc
ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++ ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/c++
ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp ENV CPP=/opt/rh/devtoolset-8/root/usr/bin/cpp
# Install Python packages # Install Python packages
RUN \ RUN \
pip install numpy pytest scipy scikit-learn wheel kubernetes urllib3==1.22 awscli pip install numpy pytest scipy scikit-learn wheel kubernetes awscli
ENV GOSU_VERSION 1.10 ENV GOSU_VERSION 1.10
# Install lightweight sudo (not bound to TTY) # Install lightweight sudo (not bound to TTY)
RUN set -ex; \ RUN set -ex; \
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \ wget -nv -nc -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
chmod +x /usr/local/bin/gosu && \ chmod +x /usr/local/bin/gosu && \
gosu nobody true gosu nobody true

34
tests/ci_build/Dockerfile.jvm_gpu_build
View File

@ -1,31 +1,21 @@
ARG CUDA_VERSION_ARG ARG CUDA_VERSION_ARG
FROM nvidia/cuda:$CUDA_VERSION_ARG-devel-centos6 FROM nvidia/cuda:$CUDA_VERSION_ARG-devel-centos7
ARG CUDA_VERSION_ARG ARG CUDA_VERSION_ARG
# Environment
ENV DEBIAN_FRONTEND noninteractive
ENV DEVTOOLSET_URL_ROOT http://vault.centos.org/6.9/sclo/x86_64/rh/devtoolset-4/
COPY CentOS-Base.repo /etc/yum.repos.d/
# Install all basic requirements # Install all basic requirements
RUN \ RUN \
yum install -y epel-release && \ yum install -y epel-release centos-release-scl && \
yum-config-manager --enable centos-sclo-rh-testing && \
yum -y update && \ yum -y update && \
yum install -y tar unzip wget xz git java-1.8.0-openjdk-devel && \ yum install -y tar unzip wget xz git which ninja-build java-1.8.0-openjdk-devel devtoolset-8-gcc devtoolset-8-binutils devtoolset-8-gcc-c++ && \
yum install -y $DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-5.3.1-6.1.el6.x86_64.rpm \
$DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-c++-5.3.1-6.1.el6.x86_64.rpm \
$DEVTOOLSET_URL_ROOT/devtoolset-4-binutils-2.25.1-8.el6.x86_64.rpm \
$DEVTOOLSET_URL_ROOT/devtoolset-4-runtime-4.1-3.sc1.el6.x86_64.rpm \
$DEVTOOLSET_URL_ROOT/devtoolset-4-libstdc++-devel-5.3.1-6.1.el6.x86_64.rpm && \
# Python # Python
wget -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ wget -nv -nc -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
bash Miniconda3.sh -b -p /opt/python && \ bash Miniconda3.sh -b -p /opt/python && \
# CMake # CMake
wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \ wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \
bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \ bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
# Maven # Maven
wget https://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \ wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \
tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \ tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \
ln -s /opt/apache-maven-3.6.1/ /opt/maven ln -s /opt/apache-maven-3.6.1/ /opt/maven
@ -33,26 +23,26 @@ RUN \
RUN \ RUN \
export CUDA_SHORT=`echo $CUDA_VERSION_ARG | grep -o -E '[0-9]+\.[0-9]'` && \ export CUDA_SHORT=`echo $CUDA_VERSION_ARG | grep -o -E '[0-9]+\.[0-9]'` && \
export NCCL_VERSION=2.4.8-1 && \ export NCCL_VERSION=2.4.8-1 && \
wget https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \ wget -nv -nc https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \ rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
yum -y update && \ yum -y update && \
yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT} && \ yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT} && \
rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm; rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm;
ENV PATH=/opt/python/bin:/opt/maven/bin:$PATH ENV PATH=/opt/python/bin:/opt/maven/bin:$PATH
ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc
ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++ ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/c++
ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp ENV CPP=/opt/rh/devtoolset-8/root/usr/bin/cpp
# Install Python packages # Install Python packages
RUN \ RUN \
pip install numpy pytest scipy scikit-learn wheel kubernetes urllib3==1.22 awscli pip install numpy pytest scipy scikit-learn wheel kubernetes awscli
ENV GOSU_VERSION 1.10 ENV GOSU_VERSION 1.10
# Install lightweight sudo (not bound to TTY) # Install lightweight sudo (not bound to TTY)
RUN set -ex; \ RUN set -ex; \
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \ wget -nv -nc -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
chmod +x /usr/local/bin/gosu && \ chmod +x /usr/local/bin/gosu && \
gosu nobody true gosu nobody true