[CI] Migrate to rockylinux8 / manylinux_2_28_x86_64 (#10399)
* [CI] Migrate to rockylinux8 / manylinux_2_28_x86_64 * Scrub all references to CentOS 7 * Fix * Remove use of yum * Use gcc-10 in cpu * Temporarily disable -Werror * Use GCC 9 for now * Roll back gRPC * Scrub all references to manylinux2014_x86_64 * Revise rename_whl.py to handle no-op rename * Change JDK_VERSION back to 8 * Reviewer's comment * Use GCC 10 * Use Spark 3.5.1, same as in pom.xml * Fix JAR install
This commit is contained in:
Philip Hyunsu Cho
committed by
GitHub
GitHub
parent
320e7c2041
commit
bc3747bdce
@@ -1,22 +1,21 @@
|
||||
FROM quay.io/pypa/manylinux2014_aarch64
|
||||
FROM quay.io/pypa/manylinux_2_28_aarch64
|
||||
|
||||
SHELL ["/bin/bash", "-c"] # Use Bash as shell
|
||||
|
||||
# Install all basic requirements
|
||||
RUN \
|
||||
yum update -y && \
|
||||
yum install -y tar unzip wget xz git centos-release-scl-rh yum-utils && \
|
||||
yum-config-manager --enable centos-sclo-rh-testing && \
|
||||
yum update -y && \
|
||||
yum install -y devtoolset-9 && \
|
||||
dnf -y update && \
|
||||
dnf -y install dnf-plugins-core && \
|
||||
dnf config-manager --set-enabled powertools && \
|
||||
dnf install -y tar unzip wget xz git which ninja-build gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \
|
||||
# Python
|
||||
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-aarch64.sh && \
|
||||
bash conda.sh -b -p /opt/mambaforge
|
||||
bash conda.sh -b -p /opt/miniforge
|
||||
|
||||
ENV PATH=/opt/mambaforge/bin:$PATH
|
||||
ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/devtoolset-9/root/usr/bin/cpp
|
||||
ENV PATH=/opt/miniforge/bin:$PATH
|
||||
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
|
||||
ENV GOSU_VERSION 1.10
|
||||
|
||||
# Create new Conda environment
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM quay.io/pypa/manylinux2014_x86_64
|
||||
FROM quay.io/pypa/manylinux_2_28_x86_64
|
||||
|
||||
# Install lightweight sudo (not bound to TTY)
|
||||
ENV GOSU_VERSION 1.10
|
||||
|
||||
@@ -10,15 +10,15 @@ RUN \
|
||||
apt-get install -y software-properties-common && \
|
||||
add-apt-repository ppa:ubuntu-toolchain-r/test && \
|
||||
apt-get update && \
|
||||
apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libidn12 cmake ninja-build gcc-9 g++-9 openjdk-8-jdk-headless && \
|
||||
apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libidn12 cmake ninja-build gcc-10 g++-10 openjdk-8-jdk-headless && \
|
||||
# Python
|
||||
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
|
||||
bash conda.sh -b -p /opt/mambaforge
|
||||
bash conda.sh -b -p /opt/miniforge
|
||||
|
||||
ENV PATH=/opt/mambaforge/bin:$PATH
|
||||
ENV CC=gcc-9
|
||||
ENV CXX=g++-9
|
||||
ENV CPP=cpp-9
|
||||
ENV PATH=/opt/miniforge/bin:$PATH
|
||||
ENV CC=gcc-10
|
||||
ENV CXX=g++-10
|
||||
ENV CPP=cpp-10
|
||||
|
||||
ENV GOSU_VERSION 1.10
|
||||
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
|
||||
|
||||
@@ -15,9 +15,9 @@ RUN \
|
||||
apt-get install -y wget unzip bzip2 libgomp1 build-essential openjdk-8-jdk-headless && \
|
||||
# Python
|
||||
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
|
||||
bash conda.sh -b -p /opt/mambaforge
|
||||
bash conda.sh -b -p /opt/miniforge
|
||||
|
||||
ENV PATH=/opt/mambaforge/bin:$PATH
|
||||
ENV PATH=/opt/miniforge/bin:$PATH
|
||||
|
||||
# Create new Conda environment with cuDF, Dask, and cuPy
|
||||
RUN \
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
ARG CUDA_VERSION_ARG
|
||||
FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-centos7
|
||||
FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-rockylinux8
|
||||
ARG CUDA_VERSION_ARG
|
||||
ARG R_VERSION_ARG
|
||||
|
||||
@@ -7,23 +7,23 @@ ARG R_VERSION_ARG
|
||||
RUN \
|
||||
curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/D42D0685.pub | sed '/^Version/d' \
|
||||
> /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA && \
|
||||
yum install -y epel-release centos-release-scl && \
|
||||
yum-config-manager --enable centos-sclo-rh-testing && \
|
||||
yum -y update && \
|
||||
yum install -y tar unzip wget xz git which ninja-build readline-devel libX11-devel libXt-devel \
|
||||
dnf -y update && \
|
||||
dnf -y install dnf-plugins-core && \
|
||||
dnf config-manager --set-enabled powertools && \
|
||||
dnf install -y tar unzip wget xz git which ninja-build readline-devel libX11-devel libXt-devel \
|
||||
xorg-x11-server-devel openssl-devel zlib-devel bzip2-devel xz-devel \
|
||||
pcre2-devel libcurl-devel texlive-* \
|
||||
devtoolset-9-gcc devtoolset-9-binutils devtoolset-9-gcc-c++ \
|
||||
devtoolset-9-gcc-gfortran devtoolset-9-libquadmath-devel \
|
||||
devtoolset-9-runtime devtoolset-9-libstdc++-devel
|
||||
gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ \
|
||||
gcc-toolset-10-gcc-gfortran gcc-toolset-10-libquadmath-devel \
|
||||
gcc-toolset-10-runtime gcc-toolset-10-libstdc++-devel
|
||||
|
||||
ENV PATH=/opt/mambaforge/bin:/usr/local/ninja:/opt/software/packages/bin:/opt/R/$R_VERSION_ARG/bin:$PATH
|
||||
ENV PATH=/opt/miniforge/bin:/usr/local/ninja:/opt/software/packages/bin:/opt/R/$R_VERSION_ARG/bin:$PATH
|
||||
ENV LD_LIBRARY_PATH=/opt/software/packages/lib:/opt/R/$R_VERSION_ARG/lib64:$LD_LIBRARY_PATH
|
||||
ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/devtoolset-9/root/usr/bin/cpp
|
||||
ENV F77=/opt/rh/devtoolset-9/root/usr/bin/gfortran
|
||||
ENV FC=/opt/rh/devtoolset-9/root/usr/bin/gfortran
|
||||
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
|
||||
ENV F77=/opt/rh/gcc-toolset-10/root/usr/bin/gfortran
|
||||
ENV FC=/opt/rh/gcc-toolset-10/root/usr/bin/gfortran
|
||||
|
||||
RUN \
|
||||
wget -nv -nc https://cran.r-project.org/src/base/R-4/R-$R_VERSION_ARG.tar.gz && \
|
||||
@@ -36,11 +36,11 @@ RUN \
|
||||
run \
|
||||
# Python
|
||||
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
|
||||
bash conda.sh -b -p /opt/mambaforge && \
|
||||
/opt/mambaforge/bin/python -m pip install auditwheel awscli && \
|
||||
bash conda.sh -b -p /opt/miniforge && \
|
||||
/opt/miniforge/bin/python -m pip install auditwheel awscli && \
|
||||
# CMake
|
||||
wget -nv -nc https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.sh --no-check-certificate && \
|
||||
bash cmake-3.18.0-Linux-x86_64.sh --skip-license --prefix=/usr
|
||||
wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
|
||||
bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr
|
||||
|
||||
ENV GOSU_VERSION 1.10
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
ARG CUDA_VERSION_ARG
|
||||
FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-centos7
|
||||
FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-rockylinux8
|
||||
ARG CUDA_VERSION_ARG
|
||||
ARG NCCL_VERSION_ARG
|
||||
ARG RAPIDS_VERSION_ARG
|
||||
@@ -8,39 +8,38 @@ ARG RAPIDS_VERSION_ARG
|
||||
RUN \
|
||||
curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/D42D0685.pub | sed '/^Version/d' \
|
||||
> /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA && \
|
||||
yum install -y epel-release centos-release-scl && \
|
||||
yum-config-manager --enable centos-sclo-rh-testing && \
|
||||
yum -y update && \
|
||||
yum install -y tar unzip wget xz git which ninja-build devtoolset-9-gcc devtoolset-9-binutils devtoolset-9-gcc-c++ && \
|
||||
dnf -y update && \
|
||||
dnf -y install dnf-plugins-core && \
|
||||
dnf config-manager --set-enabled powertools && \
|
||||
dnf install -y tar unzip wget xz git which ninja-build gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \
|
||||
# Python
|
||||
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
|
||||
bash conda.sh -b -p /opt/mambaforge && \
|
||||
/opt/mambaforge/bin/python -m pip install awscli && \
|
||||
bash conda.sh -b -p /opt/miniforge && \
|
||||
/opt/miniforge/bin/python -m pip install awscli && \
|
||||
# CMake
|
||||
wget -nv -nc https://cmake.org/files/v3.26/cmake-3.26.4-linux-x86_64.sh --no-check-certificate && \
|
||||
bash cmake-3.26.4-linux-x86_64.sh --skip-license --prefix=/usr
|
||||
wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
|
||||
bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr
|
||||
|
||||
# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
|
||||
RUN \
|
||||
export CUDA_SHORT=`echo $CUDA_VERSION_ARG | grep -o -E '[0-9]+\.[0-9]'` && \
|
||||
export NCCL_VERSION=$NCCL_VERSION_ARG && \
|
||||
wget -nv -nc https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
|
||||
rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
|
||||
yum -y update && \
|
||||
yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} && \
|
||||
rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm;
|
||||
dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo && \
|
||||
dnf -y update && \
|
||||
dnf install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT}
|
||||
|
||||
ENV PATH=/opt/mambaforge/bin:/usr/local/ninja:$PATH
|
||||
ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/devtoolset-9/root/usr/bin/cpp
|
||||
ENV CUDAHOSTCXX=/opt/rh/devtoolset-9/root/usr/bin/c++
|
||||
ENV PATH=/opt/miniforge/bin:/usr/local/ninja:$PATH
|
||||
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
|
||||
ENV CUDAHOSTCXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
|
||||
|
||||
ENV GOSU_VERSION 1.10
|
||||
|
||||
# Install RMM
|
||||
# Install RMM (Patch out -Werror)
|
||||
RUN git clone -b v${RAPIDS_VERSION_ARG}.00 https://github.com/rapidsai/rmm.git --recurse-submodules --depth 1 && \
|
||||
pushd rmm && \
|
||||
find . -name CMakeLists.txt -print0 | xargs -0 sed -i 's/-Werror//g' && \
|
||||
mkdir build && \
|
||||
pushd build && \
|
||||
cmake .. -GNinja -DCMAKE_INSTALL_PREFIX=/opt/rmm -DCUDA_STATIC_RUNTIME=ON && \
|
||||
@@ -18,9 +18,9 @@ RUN \
|
||||
apt-get install -y wget unzip bzip2 libgomp1 build-essential openjdk-8-jdk-headless && \
|
||||
# Python
|
||||
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
|
||||
bash conda.sh -b -p /opt/mambaforge
|
||||
bash conda.sh -b -p /opt/miniforge
|
||||
|
||||
ENV PATH=/opt/mambaforge/bin:$PATH
|
||||
ENV PATH=/opt/miniforge/bin:$PATH
|
||||
|
||||
# Create new Conda environment with dev versions of cuDF, Dask, and cuPy
|
||||
RUN \
|
||||
|
||||
@@ -1,32 +1,31 @@
|
||||
FROM centos:7
|
||||
FROM rockylinux:8
|
||||
|
||||
# Install all basic requirements
|
||||
RUN \
|
||||
yum install -y epel-release centos-release-scl && \
|
||||
yum-config-manager --enable centos-sclo-rh-testing && \
|
||||
yum -y update && \
|
||||
yum install -y tar unzip make bzip2 wget xz git which ninja-build java-1.8.0-openjdk-devel \
|
||||
devtoolset-9-gcc devtoolset-9-binutils devtoolset-9-gcc-c++ \
|
||||
devtoolset-9-runtime devtoolset-9-libstdc++-devel && \
|
||||
dnf -y update && \
|
||||
dnf -y install dnf-plugins-core && \
|
||||
dnf config-manager --set-enabled powertools && \
|
||||
dnf install -y tar unzip make bzip2 wget xz git which ninja-build java-1.8.0-openjdk-devel \
|
||||
gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ \
|
||||
gcc-toolset-10-runtime gcc-toolset-10-libstdc++-devel && \
|
||||
# Python
|
||||
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
|
||||
bash conda.sh -b -p /opt/mambaforge && \
|
||||
bash conda.sh -b -p /opt/miniforge && \
|
||||
# CMake
|
||||
wget -nv -nc https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.sh --no-check-certificate && \
|
||||
bash cmake-3.18.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
|
||||
wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
|
||||
bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr && \
|
||||
# Maven
|
||||
wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz && \
|
||||
tar xvf apache-maven-3.6.3-bin.tar.gz -C /opt && \
|
||||
ln -s /opt/apache-maven-3.6.3/ /opt/maven
|
||||
wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.9.7/binaries/apache-maven-3.9.7-bin.tar.gz && \
|
||||
tar xvf apache-maven-3.9.7-bin.tar.gz -C /opt && \
|
||||
ln -s /opt/apache-maven-3.9.7/ /opt/maven
|
||||
|
||||
ENV PATH=/opt/mambaforge/bin:/opt/maven/bin:$PATH
|
||||
ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/devtoolset-9/root/usr/bin/cpp
|
||||
ENV PATH=/opt/miniforge/bin:/opt/maven/bin:$PATH
|
||||
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
|
||||
|
||||
# Install Python packages
|
||||
RUN \
|
||||
pip install numpy pytest scipy scikit-learn wheel kubernetes awscli
|
||||
RUN pip install numpy pytest scipy scikit-learn wheel kubernetes awscli
|
||||
|
||||
ENV GOSU_VERSION 1.10
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
FROM ubuntu:18.04
|
||||
FROM ubuntu:22.04
|
||||
ARG JDK_VERSION=8
|
||||
ARG SPARK_VERSION=3.4.0
|
||||
ARG SPARK_VERSION=3.5.1
|
||||
|
||||
# Environment
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
@@ -14,12 +14,12 @@ RUN \
|
||||
apt-get install -y tar unzip wget openjdk-$JDK_VERSION-jdk libgomp1 && \
|
||||
# Python
|
||||
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
|
||||
bash conda.sh -b -p /opt/mambaforge && \
|
||||
/opt/mambaforge/bin/pip install awscli && \
|
||||
bash conda.sh -b -p /opt/miniforge && \
|
||||
/opt/miniforge/bin/pip install awscli && \
|
||||
# Maven
|
||||
wget -nv https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz && \
|
||||
tar xvf apache-maven-3.6.3-bin.tar.gz -C /opt && \
|
||||
ln -s /opt/apache-maven-3.6.3/ /opt/maven && \
|
||||
wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.9.7/binaries/apache-maven-3.9.7-bin.tar.gz && \
|
||||
tar xvf apache-maven-3.9.7-bin.tar.gz -C /opt && \
|
||||
ln -s /opt/apache-maven-3.9.7/ /opt/maven && \
|
||||
# Spark with scala 2.12
|
||||
mkdir -p /opt/spark-scala-2.12 && \
|
||||
wget -nv https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop3.tgz && \
|
||||
@@ -29,11 +29,10 @@ RUN \
|
||||
wget -nv https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop3-scala2.13.tgz && \
|
||||
tar xvf spark-$SPARK_VERSION-bin-hadoop3-scala2.13.tgz --strip-components=1 -C /opt/spark-scala-2.13
|
||||
|
||||
ENV PATH=/opt/mambaforge/bin:/opt/spark/bin:/opt/maven/bin:$PATH
|
||||
ENV PATH=/opt/miniforge/bin:/opt/spark/bin:/opt/maven/bin:$PATH
|
||||
|
||||
# Install Python packages
|
||||
RUN \
|
||||
pip install numpy scipy pandas scikit-learn
|
||||
RUN pip install numpy scipy pandas scikit-learn
|
||||
|
||||
ENV GOSU_VERSION 1.10
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
ARG CUDA_VERSION_ARG
|
||||
FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-centos7
|
||||
FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-rockylinux8
|
||||
ARG CUDA_VERSION_ARG
|
||||
ARG NCCL_VERSION_ARG
|
||||
|
||||
@@ -7,37 +7,36 @@ ARG NCCL_VERSION_ARG
|
||||
RUN \
|
||||
curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/D42D0685.pub | sed '/^Version/d' \
|
||||
> /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA && \
|
||||
yum install -y epel-release centos-release-scl && \
|
||||
yum-config-manager --enable centos-sclo-rh-testing && \
|
||||
yum -y update && \
|
||||
yum install -y tar unzip wget xz git which ninja-build java-1.8.0-openjdk-devel devtoolset-9-gcc devtoolset-9-binutils devtoolset-9-gcc-c++ && \
|
||||
dnf -y update && \
|
||||
dnf -y install dnf-plugins-core && \
|
||||
dnf config-manager --set-enabled powertools && \
|
||||
dnf install -y tar unzip wget xz git which ninja-build java-1.8.0-openjdk-devel gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \
|
||||
# Python
|
||||
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
|
||||
bash conda.sh -b -p /opt/mambaforge && \
|
||||
bash conda.sh -b -p /opt/miniforge && \
|
||||
# CMake
|
||||
wget -nv -nc https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.sh --no-check-certificate && \
|
||||
bash cmake-3.18.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
|
||||
wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
|
||||
bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr && \
|
||||
# Maven
|
||||
wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz && \
|
||||
tar xvf apache-maven-3.6.3-bin.tar.gz -C /opt && \
|
||||
ln -s /opt/apache-maven-3.6.3/ /opt/maven
|
||||
wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.9.7/binaries/apache-maven-3.9.7-bin.tar.gz && \
|
||||
tar xvf apache-maven-3.9.7-bin.tar.gz -C /opt && \
|
||||
ln -s /opt/apache-maven-3.9.7/ /opt/maven
|
||||
|
||||
# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
|
||||
RUN \
|
||||
export CUDA_SHORT=`echo $CUDA_VERSION_ARG | grep -o -E '[0-9]+\.[0-9]'` && \
|
||||
export NCCL_VERSION=$NCCL_VERSION_ARG && \
|
||||
yum-config-manager --add-repo http://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo && \
|
||||
yum -y update && \
|
||||
yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT}
|
||||
dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo && \
|
||||
dnf -y update && \
|
||||
dnf install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT}
|
||||
|
||||
ENV PATH=/opt/mambaforge/bin:/opt/maven/bin:$PATH
|
||||
ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/devtoolset-9/root/usr/bin/cpp
|
||||
ENV PATH=/opt/miniforge/bin:/opt/maven/bin:$PATH
|
||||
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
|
||||
|
||||
# Install Python packages
|
||||
RUN \
|
||||
pip install numpy pytest scipy scikit-learn wheel kubernetes awscli
|
||||
RUN pip install numpy pytest scipy scikit-learn wheel kubernetes awscli
|
||||
|
||||
ENV GOSU_VERSION 1.10
|
||||
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
FROM s390x/ubuntu:20.04
|
||||
|
||||
# Environment
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
SHELL ["/bin/bash", "-c"] # Use Bash as shell
|
||||
|
||||
# Install all basic requirements
|
||||
RUN \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends tar unzip wget git build-essential ninja-build \
|
||||
cmake time python3 python3-pip python3-numpy python3-scipy python3-sklearn r-base && \
|
||||
python3 -m pip install pytest hypothesis
|
||||
|
||||
ENV GOSU_VERSION 1.10
|
||||
|
||||
# Install lightweight sudo (not bound to TTY)
|
||||
RUN set -ex; \
|
||||
wget -nv -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
|
||||
chmod +x /usr/local/bin/gosu && \
|
||||
gosu nobody true
|
||||
|
||||
# Default entry-point to use if running locally
|
||||
# It will preserve attributes of created files
|
||||
COPY entrypoint.sh /scripts/
|
||||
|
||||
WORKDIR /workspace
|
||||
ENTRYPOINT ["/scripts/entrypoint.sh"]
|
||||
@@ -11,7 +11,7 @@ then
|
||||
# Workaround for file permission error
|
||||
if [[ -n $CI_BUILD_UID ]]
|
||||
then
|
||||
gosu root chown -R "${CI_BUILD_UID}:${CI_BUILD_GID}" /opt/mambaforge/envs
|
||||
gosu root chown -R "${CI_BUILD_UID}:${CI_BUILD_GID}" /opt/miniforge/envs
|
||||
fi
|
||||
|
||||
source activate ${conda_env}
|
||||
@@ -24,7 +24,9 @@ fi
|
||||
rm -rf build
|
||||
mkdir build
|
||||
cd build
|
||||
cmake .. ${cmake_args} -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_ALL_WARNINGS=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=ON -GNinja ${cmake_prefix_flag} -DHIDE_CXX_SYMBOLS=ON -DBUILD_DEPRECATED_CLI=ON
|
||||
# Disable CMAKE_COMPILE_WARNING_AS_ERROR option temporarily until
|
||||
# https://github.com/dmlc/xgboost/issues/10400 is fixed
|
||||
cmake .. ${cmake_args} -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_ALL_WARNINGS=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF -GNinja ${cmake_prefix_flag} -DHIDE_CXX_SYMBOLS=ON -DBUILD_DEPRECATED_CLI=ON
|
||||
ninja clean
|
||||
time ninja -v
|
||||
cd ..
|
||||
|
||||
@@ -49,7 +49,7 @@ if __name__ == "__main__":
|
||||
"--platform-tag",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Platform tag (e.g. manylinux2014_x86_64)",
|
||||
help="Platform tag (e.g. manylinux_2_28_x86_64)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--meta-path", type=str, required=True, help="Directory to place meta.json"
|
||||
|
||||
@@ -26,9 +26,12 @@ def main(args):
|
||||
)
|
||||
new_wheel_path = wheel_dir / new_wheel_name
|
||||
print(f"Renaming {wheel_name} to {new_wheel_name}...")
|
||||
if new_wheel_path.is_file():
|
||||
new_wheel_path.unlink()
|
||||
wheel_path.rename(new_wheel_path)
|
||||
if new_wheel_name == wheel_name:
|
||||
print("Skipping, as the old name is identical to the new name.")
|
||||
else:
|
||||
if new_wheel_path.is_file():
|
||||
new_wheel_path.unlink()
|
||||
wheel_path.rename(new_wheel_path)
|
||||
|
||||
filesize = new_wheel_path.stat().st_size / 1024 / 1024 # MiB
|
||||
print(f"Wheel size: {filesize:.2f} MiB")
|
||||
@@ -53,7 +56,7 @@ if __name__ == "__main__":
|
||||
"--platform-tag",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Platform tag (e.g. manylinux2014_x86_64)",
|
||||
help="Platform tag (e.g. manylinux_2_28_x86_64)",
|
||||
)
|
||||
parsed_args = parser.parse_args()
|
||||
main(parsed_args)
|
||||
|
||||
@@ -27,10 +27,10 @@ for scala_binary_version in "2.12" "2.13"; do
|
||||
scala_version=$(mvn help:evaluate -Dexpression=scala.version -q -DforceStdout)
|
||||
|
||||
# Install XGBoost4J JAR into local Maven repository
|
||||
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar
|
||||
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}-tests.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=test-jar -Dclassifier=tests
|
||||
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-spark/target/xgboost4j-spark_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-spark_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar
|
||||
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-example/target/xgboost4j-example_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-example_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar
|
||||
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar -DgeneratePom=true
|
||||
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}-tests.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=test-jar -Dclassifier=tests -DgeneratePom=true
|
||||
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-spark/target/xgboost4j-spark_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-spark_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar -DgeneratePom=true
|
||||
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-example/target/xgboost4j-example_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-example_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar -DgeneratePom=true
|
||||
|
||||
cd xgboost4j-tester
|
||||
# Generate pom.xml for XGBoost4J-tester, a dummy project to run XGBoost4J tests
|
||||
|
||||
Reference in New Issue
Block a user