[CI] Upgrade build image to CentOS 7 + GCC 8; require CUDA 10.1 and later (#7141)
This commit is contained in:
Philip Hyunsu Cho
GitHub
parent
dfdf0b08fc
commit
f1a4a1ac95
@ -125,8 +125,8 @@ if (USE_CUDA)
|
||||
message(STATUS "Configured CUDA host compiler: ${CMAKE_CUDA_HOST_COMPILER}")
|
||||
|
||||
enable_language(CUDA)
|
||||
if (${CMAKE_CUDA_COMPILER_VERSION} VERSION_LESS 10.0)
|
||||
message(FATAL_ERROR "CUDA version must be at least 10.0!")
|
||||
if (${CMAKE_CUDA_COMPILER_VERSION} VERSION_LESS 10.1)
|
||||
message(FATAL_ERROR "CUDA version must be at least 10.1!")
|
||||
endif()
|
||||
set(GEN_CODE "")
|
||||
format_gencode_flags("${GPU_COMPUTE_VER}" GEN_CODE)
|
||||
|
||||
28
Jenkinsfile
vendored
28
Jenkinsfile
vendored
@ -7,7 +7,7 @@
|
||||
dockerRun = 'tests/ci_build/ci_build.sh'
|
||||
|
||||
// Which CUDA version to use when building reference distribution wheel
|
||||
ref_cuda_ver = '10.0'
|
||||
ref_cuda_ver = '10.1'
|
||||
|
||||
import groovy.transform.Field
|
||||
|
||||
@ -58,13 +58,13 @@ pipeline {
|
||||
'build-cpu': { BuildCPU() },
|
||||
'build-cpu-arm64': { BuildCPUARM64() },
|
||||
'build-cpu-rabit-mock': { BuildCPUMock() },
|
||||
// Build reference, distribution-ready Python wheel with CUDA 10.0
|
||||
// using CentOS 6 image
|
||||
'build-gpu-cuda10.0': { BuildCUDA(cuda_version: '10.0') },
|
||||
// Build reference, distribution-ready Python wheel with CUDA 10.1
|
||||
// using CentOS 7 image
|
||||
'build-gpu-cuda10.1': { BuildCUDA(cuda_version: '10.1') },
|
||||
// The build-gpu-* builds below use Ubuntu image
|
||||
'build-gpu-cuda11.0': { BuildCUDA(cuda_version: '11.0', build_rmm: true) },
|
||||
'build-gpu-rpkg': { BuildRPackageWithCUDA(cuda_version: '10.0') },
|
||||
'build-jvm-packages-gpu-cuda10.0': { BuildJVMPackagesWithCUDA(spark_version: '3.0.0', cuda_version: '10.0') },
|
||||
'build-gpu-rpkg': { BuildRPackageWithCUDA(cuda_version: '10.1') },
|
||||
'build-jvm-packages-gpu-cuda10.1': { BuildJVMPackagesWithCUDA(spark_version: '3.0.0', cuda_version: '10.1') },
|
||||
'build-jvm-packages': { BuildJVMPackages(spark_version: '3.0.0') },
|
||||
'build-jvm-doc': { BuildJVMDoc() }
|
||||
])
|
||||
@ -79,9 +79,9 @@ pipeline {
|
||||
'test-python-cpu': { TestPythonCPU() },
|
||||
'test-python-cpu-arm64': { TestPythonCPUARM64() },
|
||||
// artifact_cuda_version doesn't apply to RMM tests; RMM tests will always match CUDA version between artifact and host env
|
||||
'test-python-gpu-cuda11.0-cross': { TestPythonGPU(artifact_cuda_version: '10.0', host_cuda_version: '11.0', test_rmm: true) },
|
||||
'test-python-gpu-cuda11.0-cross': { TestPythonGPU(artifact_cuda_version: '10.1', host_cuda_version: '11.0', test_rmm: true) },
|
||||
'test-python-gpu-cuda11.0': { TestPythonGPU(artifact_cuda_version: '11.0', host_cuda_version: '11.0') },
|
||||
'test-python-mgpu-cuda11.0': { TestPythonGPU(artifact_cuda_version: '10.0', host_cuda_version: '11.0', multi_gpu: true, test_rmm: true) },
|
||||
'test-python-mgpu-cuda11.0': { TestPythonGPU(artifact_cuda_version: '10.1', host_cuda_version: '11.0', multi_gpu: true, test_rmm: true) },
|
||||
'test-cpp-gpu-cuda11.0': { TestCppGPU(artifact_cuda_version: '11.0', host_cuda_version: '11.0', test_rmm: true) },
|
||||
'test-jvm-jdk8': { CrossTestJVMwithJDK(jdk_version: '8', spark_version: '3.0.0') },
|
||||
'test-jvm-jdk11': { CrossTestJVMwithJDK(jdk_version: '11') },
|
||||
@ -119,7 +119,7 @@ def checkoutSrcs() {
|
||||
}
|
||||
|
||||
def GetCUDABuildContainerType(cuda_version) {
|
||||
return (cuda_version == ref_cuda_ver) ? 'gpu_build_centos6' : 'gpu_build'
|
||||
return (cuda_version == ref_cuda_ver) ? 'gpu_build_centos7' : 'gpu_build'
|
||||
}
|
||||
|
||||
def ClangTidy() {
|
||||
@ -218,7 +218,7 @@ def BuildCUDA(args) {
|
||||
if (env.BRANCH_NAME != 'master' && !(env.BRANCH_NAME.startsWith('release'))) {
|
||||
arch_flag = "-DGPU_COMPUTE_VER=75"
|
||||
}
|
||||
def wheel_tag = "manylinux2010_x86_64"
|
||||
def wheel_tag = "manylinux2014_x86_64"
|
||||
sh """
|
||||
${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_CUDA=ON -DUSE_NCCL=ON -DOPEN_MP:BOOL=ON -DHIDE_CXX_SYMBOLS=ON ${arch_flag}
|
||||
${dockerRun} ${container_type} ${docker_binary} ${docker_args} bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal"
|
||||
@ -251,7 +251,7 @@ def BuildCUDA(args) {
|
||||
rm -rf build/
|
||||
${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh --conda-env=gpu_test -DUSE_CUDA=ON -DUSE_NCCL=ON -DPLUGIN_RMM=ON ${arch_flag}
|
||||
${dockerRun} ${container_type} ${docker_binary} ${docker_args} bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal"
|
||||
${dockerRun} ${container_type} ${docker_binary} ${docker_args} python tests/ci_build/rename_whl.py python-package/dist/*.whl ${commit_id} manylinux2010_x86_64
|
||||
${dockerRun} ${container_type} ${docker_binary} ${docker_args} python tests/ci_build/rename_whl.py python-package/dist/*.whl ${commit_id} manylinux2014_x86_64
|
||||
"""
|
||||
echo 'Stashing Python wheel...'
|
||||
stash name: "xgboost_whl_rmm_cuda${args.cuda_version}", includes: 'python-package/dist/*.whl'
|
||||
@ -265,9 +265,9 @@ def BuildCUDA(args) {
|
||||
def BuildRPackageWithCUDA(args) {
|
||||
node('linux && cpu_build') {
|
||||
unstash name: 'srcs'
|
||||
def container_type = 'gpu_build_r_centos6'
|
||||
def container_type = 'gpu_build_r_centos7'
|
||||
def docker_binary = "docker"
|
||||
def docker_args = "--build-arg CUDA_VERSION_ARG=10.0"
|
||||
def docker_args = "--build-arg CUDA_VERSION_ARG=${args.cuda_version}"
|
||||
if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) {
|
||||
sh """
|
||||
${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_r_pkg_with_cuda.sh ${commit_id}
|
||||
@ -445,7 +445,7 @@ def DeployJVMPackages(args) {
|
||||
if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) {
|
||||
echo 'Deploying to xgboost-maven-repo S3 repo...'
|
||||
sh """
|
||||
${dockerRun} jvm_gpu_build docker --build-arg CUDA_VERSION_ARG=10.0 tests/ci_build/deploy_jvm_packages.sh ${args.spark_version}
|
||||
${dockerRun} jvm_gpu_build docker --build-arg CUDA_VERSION_ARG=10.1 tests/ci_build/deploy_jvm_packages.sh ${args.spark_version}
|
||||
"""
|
||||
}
|
||||
deleteDir()
|
||||
|
||||
@ -85,7 +85,7 @@ def main(args: argparse.Namespace) -> None:
|
||||
rel = version.StrictVersion(args.release)
|
||||
platforms = [
|
||||
"win_amd64",
|
||||
"manylinux2010_x86_64",
|
||||
"manylinux2014_x86_64",
|
||||
"manylinux2014_aarch64",
|
||||
"macosx_10_14_x86_64.macosx_10_15_x86_64.macosx_11_0_x86_64",
|
||||
]
|
||||
|
||||
@ -1,37 +0,0 @@
|
||||
[base]
|
||||
name=CentOS-$releasever - Base
|
||||
baseurl=http://vault.centos.org/centos/$releasever/os/$basearch/
|
||||
gpgcheck=1
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
|
||||
|
||||
#released updates
|
||||
[updates]
|
||||
name=CentOS-$releasever - Updates
|
||||
baseurl=http://vault.centos.org/centos/$releasever/updates/$basearch/
|
||||
gpgcheck=1
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
|
||||
|
||||
#additional packages that may be useful
|
||||
[extras]
|
||||
name=CentOS-$releasever - Extras
|
||||
baseurl=http://vault.centos.org/centos/$releasever/extras/$basearch/
|
||||
gpgcheck=1
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
|
||||
|
||||
#additional packages that extend functionality of existing packages
|
||||
[centosplus]
|
||||
name=CentOS-$releasever - Plus
|
||||
mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=centosplus&infra=$infra
|
||||
#baseurl=http://mirror.centos.org/centos/$releasever/centosplus/$basearch/
|
||||
gpgcheck=1
|
||||
enabled=0
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
|
||||
|
||||
#contrib - packages by Centos Users
|
||||
[contrib]
|
||||
name=CentOS-$releasever - Contrib
|
||||
mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=contrib&infra=$infra
|
||||
#baseurl=http://mirror.centos.org/centos/$releasever/contrib/$basearch/
|
||||
gpgcheck=1
|
||||
enabled=0
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
|
||||
@ -1,4 +1,4 @@
|
||||
FROM quay.io/pypa/manylinux2010_x86_64
|
||||
FROM quay.io/pypa/manylinux2014_x86_64
|
||||
|
||||
# Install lightweight sudo (not bound to TTY)
|
||||
ENV GOSU_VERSION 1.10
|
||||
|
||||
@ -7,7 +7,10 @@ SHELL ["/bin/bash", "-c"] # Use Bash as shell
|
||||
# Install all basic requirements
|
||||
RUN \
|
||||
apt-get update && \
|
||||
apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libasan2 libidn11 ninja-build && \
|
||||
apt-get install -y software-properties-common && \
|
||||
add-apt-repository ppa:ubuntu-toolchain-r/test && \
|
||||
apt-get update && \
|
||||
apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libasan2 libidn11 ninja-build gcc-8 g++-8 && \
|
||||
# CMake
|
||||
wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \
|
||||
bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
|
||||
@ -16,6 +19,9 @@ RUN \
|
||||
bash Miniconda3.sh -b -p /opt/python
|
||||
|
||||
ENV PATH=/opt/python/bin:$PATH
|
||||
ENV CC=gcc-8
|
||||
ENV CXX=g++-8
|
||||
ENV CPP=cpp-8
|
||||
|
||||
ENV GOSU_VERSION 1.10
|
||||
|
||||
|
||||
@ -9,7 +9,10 @@ SHELL ["/bin/bash", "-c"] # Use Bash as shell
|
||||
# Install all basic requirements
|
||||
RUN \
|
||||
apt-get update && \
|
||||
apt-get install -y tar unzip wget bzip2 libgomp1 git build-essential doxygen graphviz llvm libasan2 libidn11 ninja-build && \
|
||||
apt-get install -y software-properties-common && \
|
||||
add-apt-repository ppa:ubuntu-toolchain-r/test && \
|
||||
apt-get update && \
|
||||
apt-get install -y tar unzip wget bzip2 libgomp1 git build-essential doxygen graphviz llvm libasan2 libidn11 ninja-build gcc-8 g++-8 && \
|
||||
# CMake
|
||||
wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \
|
||||
bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
|
||||
@ -25,6 +28,9 @@ RUN \
|
||||
apt-get install -y --allow-downgrades --allow-change-held-packages libnccl2=${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-dev=${NCCL_VERSION}+cuda${CUDA_SHORT}
|
||||
|
||||
ENV PATH=/opt/python/bin:$PATH
|
||||
ENV CC=gcc-8
|
||||
ENV CXX=g++-8
|
||||
ENV CPP=cpp-8
|
||||
|
||||
ENV GOSU_VERSION 1.10
|
||||
|
||||
|
||||
@ -1,64 +0,0 @@
|
||||
ARG CUDA_VERSION_ARG
|
||||
FROM nvidia/cuda:$CUDA_VERSION_ARG-devel-centos6
|
||||
ARG CUDA_VERSION_ARG
|
||||
|
||||
# Environment
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
ENV DEVTOOLSET_URL_ROOT http://vault.centos.org/6.9/sclo/x86_64/rh/devtoolset-4/
|
||||
|
||||
COPY CentOS-Base.repo /etc/yum.repos.d/
|
||||
|
||||
# Install all basic requirements
|
||||
RUN \
|
||||
yum install -y epel-release && \
|
||||
yum -y update && \
|
||||
yum install -y tar unzip wget xz git patchelf && \
|
||||
yum install -y $DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-5.3.1-6.1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-c++-5.3.1-6.1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-binutils-2.25.1-8.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-runtime-4.1-3.sc1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-libstdc++-devel-5.3.1-6.1.el6.x86_64.rpm && \
|
||||
# Python
|
||||
wget -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
|
||||
bash Miniconda3.sh -b -p /opt/python && \
|
||||
/opt/python/bin/python -m pip install auditwheel && \
|
||||
# CMake
|
||||
wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \
|
||||
bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
|
||||
# Ninja
|
||||
mkdir -p /usr/local && \
|
||||
cd /usr/local/ && \
|
||||
wget -nv -nc https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz --no-check-certificate && \
|
||||
tar xf v1.10.0.tar.gz && mv ninja-1.10.0 ninja && rm -v v1.10.0.tar.gz && \
|
||||
cd ninja && \
|
||||
/opt/python/bin/python ./configure.py --bootstrap
|
||||
|
||||
# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
|
||||
RUN \
|
||||
export CUDA_SHORT=`echo $CUDA_VERSION_ARG | grep -o -E '[0-9]+\.[0-9]'` && \
|
||||
export NCCL_VERSION=2.4.8-1 && \
|
||||
wget https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
|
||||
rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
|
||||
yum -y update && \
|
||||
yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT} && \
|
||||
rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm;
|
||||
|
||||
ENV PATH=/opt/python/bin:/usr/local/ninja:$PATH
|
||||
ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
|
||||
|
||||
ENV GOSU_VERSION 1.10
|
||||
|
||||
# Install lightweight sudo (not bound to TTY)
|
||||
RUN set -ex; \
|
||||
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
|
||||
chmod +x /usr/local/bin/gosu && \
|
||||
gosu nobody true
|
||||
|
||||
# Default entry-point to use if running locally
|
||||
# It will preserve attributes of created files
|
||||
COPY entrypoint.sh /scripts/
|
||||
|
||||
WORKDIR /workspace
|
||||
ENTRYPOINT ["/scripts/entrypoint.sh"]
|
||||
46
tests/ci_build/Dockerfile.gpu_build_centos7
Normal file
46
tests/ci_build/Dockerfile.gpu_build_centos7
Normal file
@ -0,0 +1,46 @@
|
||||
ARG CUDA_VERSION_ARG
|
||||
FROM nvidia/cuda:$CUDA_VERSION_ARG-devel-centos7
|
||||
ARG CUDA_VERSION_ARG
|
||||
|
||||
# Install all basic requirements
|
||||
RUN \
|
||||
yum install -y epel-release centos-release-scl && \
|
||||
yum-config-manager --enable centos-sclo-rh-testing && \
|
||||
yum -y update && \
|
||||
yum install -y tar unzip wget xz git which ninja-build devtoolset-8-gcc devtoolset-8-binutils devtoolset-8-gcc-c++ && \
|
||||
# Python
|
||||
wget -nv -nc -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
|
||||
bash Miniconda3.sh -b -p /opt/python && \
|
||||
# CMake
|
||||
wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \
|
||||
bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr
|
||||
|
||||
# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
|
||||
RUN \
|
||||
export CUDA_SHORT=`echo $CUDA_VERSION_ARG | grep -o -E '[0-9]+\.[0-9]'` && \
|
||||
export NCCL_VERSION=2.4.8-1 && \
|
||||
wget -nv -nc https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
|
||||
rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
|
||||
yum -y update && \
|
||||
yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT} && \
|
||||
rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm;
|
||||
|
||||
ENV PATH=/opt/python/bin:/usr/local/ninja:$PATH
|
||||
ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/devtoolset-8/root/usr/bin/cpp
|
||||
|
||||
ENV GOSU_VERSION 1.10
|
||||
|
||||
# Install lightweight sudo (not bound to TTY)
|
||||
RUN set -ex; \
|
||||
wget -nv -nc -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
|
||||
chmod +x /usr/local/bin/gosu && \
|
||||
gosu nobody true
|
||||
|
||||
# Default entry-point to use if running locally
|
||||
# It will preserve attributes of created files
|
||||
COPY entrypoint.sh /scripts/
|
||||
|
||||
WORKDIR /workspace
|
||||
ENTRYPOINT ["/scripts/entrypoint.sh"]
|
||||
@ -1,112 +0,0 @@
|
||||
ARG CUDA_VERSION_ARG
|
||||
FROM nvidia/cuda:$CUDA_VERSION_ARG-devel-centos6
|
||||
ARG CUDA_VERSION_ARG
|
||||
|
||||
# Environment
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
ENV DEVTOOLSET_URL_ROOT http://vault.centos.org/6.9/sclo/x86_64/rh/devtoolset-4/
|
||||
|
||||
COPY CentOS-Base.repo /etc/yum.repos.d/
|
||||
|
||||
# Install all basic requirements
|
||||
RUN \
|
||||
yum install -y epel-release && \
|
||||
yum -y update && \
|
||||
yum install -y tar unzip wget xz git patchelf readline-devel libX11-devel libXt-devel \
|
||||
xorg-x11-server-devel openssl-devel texlive-* && \
|
||||
yum install -y $DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-5.3.1-6.1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-gfortran-5.3.1-6.1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-libquadmath-devel-5.3.1-6.1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-c++-5.3.1-6.1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-binutils-2.25.1-8.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-runtime-4.1-3.sc1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-libstdc++-devel-5.3.1-6.1.el6.x86_64.rpm
|
||||
|
||||
ENV PATH=/opt/python/bin:/usr/local/ninja:/opt/software/packages/bin:/opt/R/3.3.0/bin:$PATH
|
||||
ENV LD_LIBRARY_PATH=/opt/software/packages/lib:/opt/R/3.3.0/lib64:$LD_LIBRARY_PATH
|
||||
ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
|
||||
ENV F77=/opt/rh/devtoolset-4/root/usr/bin/gfortran
|
||||
|
||||
# A few packages have to be built from the source because CentOS 6 is a very old distribution and
|
||||
# the system packages are not sufficiently up-to-date to build R 3.3.0. We'll want to update to
|
||||
# CentOS 7 after the 1.4.0 release. Tracking issue: dmlc/xgboost#6791.
|
||||
#
|
||||
# Why choose an old Linux distro? This is so that the resulting xgboost.so is compatible with a
|
||||
# wide range of Linux OSes currently in operation. See https://www.python.org/dev/peps/pep-0571/
|
||||
RUN \
|
||||
wget https://zlib.net/fossils/zlib-1.2.5.tar.gz && \
|
||||
wget https://sourceware.org/pub/bzip2/bzip2-1.0.6.tar.gz && \
|
||||
wget http://tukaani.org/xz/xz-5.2.2.tar.gz && \
|
||||
wget https://ftp.pcre.org/pub/pcre/pcre-8.40.tar.gz && \
|
||||
wget https://www.openssl.org/source/old/1.0.0/openssl-1.0.0k.tar.gz && \
|
||||
wget --no-check-certificate https://curl.se/download/curl-7.47.1.tar.gz && \
|
||||
tar xf zlib-1.2.5.tar.gz && \
|
||||
tar xf bzip2-1.0.6.tar.gz && \
|
||||
tar xf xz-5.2.2.tar.gz && \
|
||||
tar xf pcre-8.40.tar.gz && \
|
||||
tar xf openssl-1.0.0k.tar.gz && \
|
||||
tar xf curl-7.47.1.tar.gz && \
|
||||
cd zlib-1.2.5 && \
|
||||
./configure --prefix=/opt/software/packages && \
|
||||
make -j$(nproc) && \
|
||||
make install && \
|
||||
cd ../bzip2-1.0.6 && \
|
||||
sed -i 's/CFLAGS=-Wall/CFLAGS=-fPIC -Wall/g' Makefile && \
|
||||
make -f Makefile-libbz2_so && \
|
||||
make clean && \
|
||||
make -j$(nproc) && \
|
||||
make -n install PREFIX=/opt/software/packages && \
|
||||
make install PREFIX=/opt/software/packages && \
|
||||
cd ../xz-5.2.2 && \
|
||||
./configure --prefix=/opt/software/packages && \
|
||||
make -j$(nproc) && \
|
||||
make install && \
|
||||
cd ../pcre-8.40 && \
|
||||
./configure --enable-utf8 --prefix=/opt/software/packages && \
|
||||
make -j$(nproc) && \
|
||||
make install && \
|
||||
cd ../curl-7.47.1 && \
|
||||
./configure --prefix=/opt/software/packages --with-ssl && \
|
||||
make -j$(nproc) && \
|
||||
make install && \
|
||||
export CFLAGS="-I/opt/software/packages/include" && \
|
||||
export LDFLAGS="-L/opt/software/packages/lib" && \
|
||||
cd .. && \
|
||||
# R 3.3.0
|
||||
wget -nv -nc https://cran.r-project.org/src/base/R-3/R-3.3.0.tar.gz && \
|
||||
tar xf R-3.3.0.tar.gz && \
|
||||
cd R-3.3.0 && \
|
||||
./configure --prefix=/opt/R/3.3.0 --enable-R-shlib && \
|
||||
make -j$(nproc) && \
|
||||
make install && \
|
||||
# Python
|
||||
wget -nv -nc -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
|
||||
bash Miniconda3.sh -b -p /opt/python && \
|
||||
/opt/python/bin/python -m pip install auditwheel && \
|
||||
# CMake
|
||||
wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \
|
||||
bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
|
||||
# Ninja
|
||||
mkdir -p /usr/local && \
|
||||
cd /usr/local/ && \
|
||||
wget -nv -nc https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz --no-check-certificate && \
|
||||
tar xf v1.10.0.tar.gz && mv ninja-1.10.0 ninja && rm -v v1.10.0.tar.gz && \
|
||||
cd ninja && \
|
||||
/opt/python/bin/python ./configure.py --bootstrap
|
||||
|
||||
ENV GOSU_VERSION 1.10
|
||||
|
||||
# Install lightweight sudo (not bound to TTY)
|
||||
RUN set -ex; \
|
||||
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
|
||||
chmod +x /usr/local/bin/gosu && \
|
||||
gosu nobody true
|
||||
|
||||
# Default entry-point to use if running locally
|
||||
# It will preserve attributes of created files
|
||||
COPY entrypoint.sh /scripts/
|
||||
|
||||
WORKDIR /workspace
|
||||
ENTRYPOINT ["/scripts/entrypoint.sh"]
|
||||
53
tests/ci_build/Dockerfile.gpu_build_r_centos7
Normal file
53
tests/ci_build/Dockerfile.gpu_build_r_centos7
Normal file
@ -0,0 +1,53 @@
|
||||
ARG CUDA_VERSION_ARG
|
||||
FROM nvidia/cuda:$CUDA_VERSION_ARG-devel-centos7
|
||||
ARG CUDA_VERSION_ARG
|
||||
|
||||
# Install all basic requirements
|
||||
RUN \
|
||||
yum install -y epel-release centos-release-scl && \
|
||||
yum-config-manager --enable centos-sclo-rh-testing && \
|
||||
yum -y update && \
|
||||
yum install -y tar unzip wget xz git which ninja-build readline-devel libX11-devel libXt-devel \
|
||||
xorg-x11-server-devel openssl-devel zlib-devel bzip2-devel xz-devel \
|
||||
pcre-devel libcurl-devel texlive-* \
|
||||
devtoolset-8-gcc devtoolset-8-binutils devtoolset-8-gcc-c++ \
|
||||
devtoolset-8-gcc-gfortran devtoolset-8-libquadmath-devel \
|
||||
devtoolset-8-runtime devtoolset-8-libstdc++-devel
|
||||
|
||||
ENV PATH=/opt/python/bin:/usr/local/ninja:/opt/software/packages/bin:/opt/R/3.3.0/bin:$PATH
|
||||
ENV LD_LIBRARY_PATH=/opt/software/packages/lib:/opt/R/3.3.0/lib64:$LD_LIBRARY_PATH
|
||||
ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/devtoolset-8/root/usr/bin/cpp
|
||||
ENV F77=/opt/rh/devtoolset-8/root/usr/bin/gfortran
|
||||
|
||||
# R 3.3.0
|
||||
RUN \
|
||||
wget -nv -nc https://cran.r-project.org/src/base/R-3/R-3.3.0.tar.gz && \
|
||||
tar xf R-3.3.0.tar.gz && \
|
||||
cd R-3.3.0 && \
|
||||
./configure --prefix=/opt/R/3.3.0 --enable-R-shlib && \
|
||||
make -j$(nproc) && \
|
||||
make install && \
|
||||
# Python
|
||||
wget -nv -nc -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
|
||||
bash Miniconda3.sh -b -p /opt/python && \
|
||||
/opt/python/bin/python -m pip install auditwheel && \
|
||||
# CMake
|
||||
wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \
|
||||
bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr
|
||||
|
||||
ENV GOSU_VERSION 1.10
|
||||
|
||||
# Install lightweight sudo (not bound to TTY)
|
||||
RUN set -ex; \
|
||||
wget -nv -nc -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
|
||||
chmod +x /usr/local/bin/gosu && \
|
||||
gosu nobody true
|
||||
|
||||
# Default entry-point to use if running locally
|
||||
# It will preserve attributes of created files
|
||||
COPY entrypoint.sh /scripts/
|
||||
|
||||
WORKDIR /workspace
|
||||
ENTRYPOINT ["/scripts/entrypoint.sh"]
|
||||
@ -1,44 +1,38 @@
|
||||
FROM centos:6
|
||||
|
||||
ENV DEVTOOLSET_URL_ROOT http://vault.centos.org/6.9/sclo/x86_64/rh/devtoolset-4/
|
||||
|
||||
COPY CentOS-Base.repo /etc/yum.repos.d/
|
||||
FROM centos:7
|
||||
|
||||
# Install all basic requirements
|
||||
RUN \
|
||||
yum install -y epel-release && \
|
||||
yum install -y epel-release centos-release-scl && \
|
||||
yum-config-manager --enable centos-sclo-rh-testing && \
|
||||
yum -y update && \
|
||||
yum install -y tar unzip wget xz git java-1.8.0-openjdk-devel && \
|
||||
yum install -y $DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-5.3.1-6.1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-c++-5.3.1-6.1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-binutils-2.25.1-8.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-runtime-4.1-3.sc1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-libstdc++-devel-5.3.1-6.1.el6.x86_64.rpm && \
|
||||
yum install -y tar unzip make bzip2 wget xz git which ninja-build java-1.8.0-openjdk-devel \
|
||||
devtoolset-8-gcc devtoolset-8-binutils devtoolset-8-gcc-c++ \
|
||||
devtoolset-8-runtime devtoolset-8-libstdc++-devel && \
|
||||
# Python
|
||||
wget -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
|
||||
wget -nv -nc -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
|
||||
bash Miniconda3.sh -b -p /opt/python && \
|
||||
# CMake
|
||||
wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \
|
||||
bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
|
||||
# Maven
|
||||
wget https://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \
|
||||
wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \
|
||||
tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \
|
||||
ln -s /opt/apache-maven-3.6.1/ /opt/maven
|
||||
|
||||
ENV PATH=/opt/python/bin:/opt/maven/bin:$PATH
|
||||
ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
|
||||
ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/devtoolset-8/root/usr/bin/cpp
|
||||
|
||||
# Install Python packages
|
||||
RUN \
|
||||
pip install numpy pytest scipy scikit-learn wheel kubernetes urllib3==1.22 awscli
|
||||
pip install numpy pytest scipy scikit-learn wheel kubernetes awscli
|
||||
|
||||
ENV GOSU_VERSION 1.10
|
||||
|
||||
# Install lightweight sudo (not bound to TTY)
|
||||
RUN set -ex; \
|
||||
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
|
||||
wget -nv -nc -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
|
||||
chmod +x /usr/local/bin/gosu && \
|
||||
gosu nobody true
|
||||
|
||||
|
||||
@ -1,31 +1,21 @@
|
||||
ARG CUDA_VERSION_ARG
|
||||
FROM nvidia/cuda:$CUDA_VERSION_ARG-devel-centos6
|
||||
FROM nvidia/cuda:$CUDA_VERSION_ARG-devel-centos7
|
||||
ARG CUDA_VERSION_ARG
|
||||
|
||||
# Environment
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
ENV DEVTOOLSET_URL_ROOT http://vault.centos.org/6.9/sclo/x86_64/rh/devtoolset-4/
|
||||
|
||||
COPY CentOS-Base.repo /etc/yum.repos.d/
|
||||
|
||||
# Install all basic requirements
|
||||
RUN \
|
||||
yum install -y epel-release && \
|
||||
yum install -y epel-release centos-release-scl && \
|
||||
yum-config-manager --enable centos-sclo-rh-testing && \
|
||||
yum -y update && \
|
||||
yum install -y tar unzip wget xz git java-1.8.0-openjdk-devel && \
|
||||
yum install -y $DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-5.3.1-6.1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-c++-5.3.1-6.1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-binutils-2.25.1-8.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-runtime-4.1-3.sc1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-libstdc++-devel-5.3.1-6.1.el6.x86_64.rpm && \
|
||||
yum install -y tar unzip wget xz git which ninja-build java-1.8.0-openjdk-devel devtoolset-8-gcc devtoolset-8-binutils devtoolset-8-gcc-c++ && \
|
||||
# Python
|
||||
wget -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
|
||||
wget -nv -nc -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
|
||||
bash Miniconda3.sh -b -p /opt/python && \
|
||||
# CMake
|
||||
wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \
|
||||
bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
|
||||
# Maven
|
||||
wget https://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \
|
||||
wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \
|
||||
tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \
|
||||
ln -s /opt/apache-maven-3.6.1/ /opt/maven
|
||||
|
||||
@ -33,26 +23,26 @@ RUN \
|
||||
RUN \
|
||||
export CUDA_SHORT=`echo $CUDA_VERSION_ARG | grep -o -E '[0-9]+\.[0-9]'` && \
|
||||
export NCCL_VERSION=2.4.8-1 && \
|
||||
wget https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
|
||||
wget -nv -nc https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
|
||||
rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
|
||||
yum -y update && \
|
||||
yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT} && \
|
||||
rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm;
|
||||
|
||||
ENV PATH=/opt/python/bin:/opt/maven/bin:$PATH
|
||||
ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
|
||||
ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/devtoolset-8/root/usr/bin/cpp
|
||||
|
||||
# Install Python packages
|
||||
RUN \
|
||||
pip install numpy pytest scipy scikit-learn wheel kubernetes urllib3==1.22 awscli
|
||||
pip install numpy pytest scipy scikit-learn wheel kubernetes awscli
|
||||
|
||||
ENV GOSU_VERSION 1.10
|
||||
|
||||
# Install lightweight sudo (not bound to TTY)
|
||||
RUN set -ex; \
|
||||
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
|
||||
wget -nv -nc -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
|
||||
chmod +x /usr/local/bin/gosu && \
|
||||
gosu nobody true
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user