[CI] Migrate to rockylinux8 / manylinux_2_28_x86_64 (#10399)

* [CI] Migrate to rockylinux8 / manylinux_2_28_x86_64 * Scrub all references to CentOS 7 * Fix * Remove use of yum * Use gcc-10 in cpu * Temporarily disable -Werror * Use GCC 9 for now * Roll back gRPC * Scrub all references to manylinux2014_x86_64 * Revise rename_whl.py to handle no-op rename * Change JDK_VERSION back to 8 * Reviewer's comment * Use GCC 10 * Use Spark 3.5.1, same as in pom.xml * Fix JAR install
2024-06-17 12:07:49 -07:00 · 2024-06-17 12:07:49 -07:00 · bc3747bdce
commit bc3747bdce
parent 320e7c2041
27 changed files with 136 additions and 171 deletions
--- a/dev/release-artifacts.py
+++ b/dev/release-artifacts.py
@ -123,8 +123,8 @@ def download_py_packages(
 ) -> None:
    platforms = [
        "win_amd64",
-        "manylinux2014_x86_64",
-        "manylinux2014_aarch64",
+        "manylinux_2_28_x86_64",
+        "manylinux_2_28_aarch64",
        "macosx_10_15_x86_64.macosx_11_0_x86_64.macosx_12_0_x86_64",
        "macosx_12_0_arm64",
    ]
--- a/python-package/xgboost/testing/init.py
+++ b/python-package/xgboost/testing/init.py
@ -195,12 +195,6 @@ def skip_win() -> PytestSkip:
    return {"reason": "Unsupported platform.", "condition": is_windows()}


-def skip_s390x() -> PytestSkip:
-    condition = platform.machine() == "s390x"
-    reason = "Known to fail on s390x"
-    return {"condition": condition, "reason": reason}
-
-
 class IteratorForTest(xgb.core.DataIter):
    """Iterator for testing streaming DMatrix. (external memory, quantile)"""

@ -523,7 +517,6 @@ def make_sparse_regression(

    """
    if not hasattr(np.random, "default_rng"):
-        # old version of numpy on s390x
        rng = np.random.RandomState(1994)
        X = sparse.random(
            m=n_samples,
--- a/tests/buildkite/build-containers.sh
+++ b/tests/buildkite/build-containers.sh
@ -20,7 +20,7 @@ case "${container}" in
  cpu)
    ;;

-  gpu|gpu_build_centos7)
+  gpu|gpu_build_rockylinux8)
    BUILD_ARGS="$BUILD_ARGS --build-arg CUDA_VERSION_ARG=$CUDA_VERSION"
    BUILD_ARGS="$BUILD_ARGS --build-arg NCCL_VERSION_ARG=$NCCL_VERSION"
    BUILD_ARGS="$BUILD_ARGS --build-arg RAPIDS_VERSION_ARG=$RAPIDS_VERSION"
--- a/tests/buildkite/build-cpu-arm64.sh
+++ b/tests/buildkite/build-cpu-arm64.sh
@ -2,7 +2,7 @@

 set -euo pipefail

-WHEEL_TAG=manylinux2014_aarch64
+WHEEL_TAG=manylinux_2_28_aarch64

 echo "--- Build CPU code targeting ARM64"

@ -24,7 +24,7 @@ $command_wrapper python tests/ci_build/rename_whl.py  \
  --commit-hash ${BUILDKITE_COMMIT}  \
  --platform-tag ${WHEEL_TAG}

-echo "--- Audit binary wheel to ensure it's compliant with manylinux2014 standard"
+echo "--- Audit binary wheel to ensure it's compliant with ${WHEEL_TAG} standard"
 $command_wrapper auditwheel repair --plat ${WHEEL_TAG} python-package/dist/*.whl
 $command_wrapper python tests/ci_build/rename_whl.py  \
  --wheel-path wheelhouse/*.whl  \
--- a/tests/buildkite/build-cuda-with-rmm.sh
+++ b/tests/buildkite/build-cuda-with-rmm.sh
@ -2,7 +2,7 @@

 set -euo pipefail

-WHEEL_TAG=manylinux2014_x86_64
+WHEEL_TAG=manylinux_2_28_x86_64

 source tests/buildkite/conftest.sh

@ -15,7 +15,7 @@ else
  arch_flag=""
 fi

-command_wrapper="tests/ci_build/ci_build.sh gpu_build_centos7 --build-arg "`
+command_wrapper="tests/ci_build/ci_build.sh gpu_build_rockylinux8 --build-arg "`
                `"CUDA_VERSION_ARG=$CUDA_VERSION --build-arg "`
                `"NCCL_VERSION_ARG=$NCCL_VERSION --build-arg "`
                `"RAPIDS_VERSION_ARG=$RAPIDS_VERSION"
@ -41,7 +41,7 @@ $command_wrapper python tests/ci_build/rename_whl.py  \
  --commit-hash ${BUILDKITE_COMMIT}  \
  --platform-tag ${WHEEL_TAG}

-echo "--- Audit binary wheel to ensure it's compliant with manylinux2014 standard"
+echo "--- Audit binary wheel to ensure it's compliant with ${WHEEL_TAG} standard"
 tests/ci_build/ci_build.sh auditwheel_x86_64 auditwheel repair \
  --plat ${WHEEL_TAG} python-package/dist/*.whl
 $command_wrapper python tests/ci_build/rename_whl.py  \
--- a/tests/buildkite/build-cuda.sh
+++ b/tests/buildkite/build-cuda.sh
@ -2,7 +2,7 @@

 set -euo pipefail

-WHEEL_TAG=manylinux2014_x86_64
+WHEEL_TAG=manylinux_2_28_x86_64

 source tests/buildkite/conftest.sh

@ -15,7 +15,7 @@ else
  arch_flag=""
 fi

-command_wrapper="tests/ci_build/ci_build.sh gpu_build_centos7 --build-arg "`
+command_wrapper="tests/ci_build/ci_build.sh gpu_build_rockylinux8 --build-arg "`
                `"CUDA_VERSION_ARG=$CUDA_VERSION --build-arg "`
                `"NCCL_VERSION_ARG=$NCCL_VERSION --build-arg "`
                `"RAPIDS_VERSION_ARG=$RAPIDS_VERSION"
@ -40,7 +40,7 @@ $command_wrapper python tests/ci_build/rename_whl.py  \
  --commit-hash ${BUILDKITE_COMMIT}  \
  --platform-tag ${WHEEL_TAG}

-echo "--- Audit binary wheel to ensure it's compliant with manylinux2014 standard"
+echo "--- Audit binary wheel to ensure it's compliant with ${WHEEL_TAG} standard"
 tests/ci_build/ci_build.sh auditwheel_x86_64 auditwheel repair \
  --plat ${WHEEL_TAG} python-package/dist/*.whl
 $command_wrapper python tests/ci_build/rename_whl.py  \
--- a/tests/buildkite/build-gpu-rpkg.sh
+++ b/tests/buildkite/build-gpu-rpkg.sh
@ -6,7 +6,7 @@ source tests/buildkite/conftest.sh

 echo "--- Build XGBoost R package with CUDA"

-tests/ci_build/ci_build.sh gpu_build_r_centos7 \
+tests/ci_build/ci_build.sh gpu_build_r_rockylinux8 \
 			   --build-arg CUDA_VERSION_ARG=${CUDA_VERSION} \
 			   --build-arg R_VERSION_ARG=${R_VERSION} \
 			   tests/ci_build/build_r_pkg_with_cuda.sh \
--- a/tests/buildkite/conftest.sh
+++ b/tests/buildkite/conftest.sh
@ -26,7 +26,7 @@ CUDA_VERSION=11.8.0
 NCCL_VERSION=2.16.5-1
 RAPIDS_VERSION=24.04
 DEV_RAPIDS_VERSION=24.06
-SPARK_VERSION=3.4.0
+SPARK_VERSION=3.5.1
 JDK_VERSION=8
 R_VERSION=4.3.2

--- a/tests/buildkite/pipeline-mgpu.yml
+++ b/tests/buildkite/pipeline-mgpu.yml
@ -17,7 +17,7 @@ steps:
  - label: ":docker: Build containers"
    commands:
      - "tests/buildkite/build-containers.sh gpu"
-      - "tests/buildkite/build-containers.sh gpu_build_centos7"
+      - "tests/buildkite/build-containers.sh gpu_build_rockylinux8"
      - "tests/buildkite/build-containers.sh jvm_gpu_build"
    key: build-containers
    agents:
--- a/tests/buildkite/pipeline-nightly.yml
+++ b/tests/buildkite/pipeline-nightly.yml
@ -12,7 +12,7 @@ steps:
  #### -------- CONTAINER BUILD --------
  - label: ":docker: Build containers"
    commands:
-      - "tests/buildkite/build-containers.sh gpu_build_centos7"
+      - "tests/buildkite/build-containers.sh gpu_build_rockylinux8"
      - "tests/buildkite/build-containers.sh gpu_dev_ver"
    key: build-containers
    agents:
--- a/tests/buildkite/pipeline.yml
+++ b/tests/buildkite/pipeline.yml
@ -15,7 +15,7 @@ steps:
    commands:
      - "tests/buildkite/build-containers.sh cpu"
      - "tests/buildkite/build-containers.sh gpu"
-      - "tests/buildkite/build-containers.sh gpu_build_centos7"
+      - "tests/buildkite/build-containers.sh gpu_build_rockylinux8"
    key: build-containers
    agents:
      queue: linux-amd64-cpu
--- a/tests/ci_build/Dockerfile.aarch64
+++ b/tests/ci_build/Dockerfile.aarch64
@ -1,22 +1,21 @@
-FROM quay.io/pypa/manylinux2014_aarch64
+FROM quay.io/pypa/manylinux_2_28_aarch64

 SHELL ["/bin/bash", "-c"]   # Use Bash as shell

 # Install all basic requirements
 RUN \
-    yum update -y && \
-    yum install -y tar unzip wget xz git centos-release-scl-rh yum-utils && \
-    yum-config-manager --enable centos-sclo-rh-testing && \
-    yum update -y && \
-    yum install -y devtoolset-9 && \
+    dnf -y update && \
+    dnf -y install dnf-plugins-core && \
+    dnf config-manager --set-enabled powertools && \
+    dnf install -y tar unzip wget xz git which ninja-build gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \
    # Python
    wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-aarch64.sh && \
-    bash conda.sh -b -p /opt/mambaforge
+    bash conda.sh -b -p /opt/miniforge

-ENV PATH=/opt/mambaforge/bin:$PATH
-ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc
-ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/c++
-ENV CPP=/opt/rh/devtoolset-9/root/usr/bin/cpp
+ENV PATH=/opt/miniforge/bin:$PATH
+ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
+ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
+ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
 ENV GOSU_VERSION 1.10

 # Create new Conda environment
--- a/tests/ci_build/Dockerfile.auditwheel_x86_64
+++ b/tests/ci_build/Dockerfile.auditwheel_x86_64
@ -1,4 +1,4 @@
-FROM quay.io/pypa/manylinux2014_x86_64
+FROM quay.io/pypa/manylinux_2_28_x86_64

 # Install lightweight sudo (not bound to TTY)
 ENV GOSU_VERSION 1.10
--- a/tests/ci_build/Dockerfile.cpu
+++ b/tests/ci_build/Dockerfile.cpu
@ -10,15 +10,15 @@ RUN \
    apt-get install -y software-properties-common && \
    add-apt-repository ppa:ubuntu-toolchain-r/test && \
    apt-get update && \
-    apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libidn12 cmake ninja-build gcc-9 g++-9 openjdk-8-jdk-headless && \
+    apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libidn12 cmake ninja-build gcc-10 g++-10 openjdk-8-jdk-headless && \
    # Python
    wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
-    bash conda.sh -b -p /opt/mambaforge
+    bash conda.sh -b -p /opt/miniforge

-ENV PATH=/opt/mambaforge/bin:$PATH
-ENV CC=gcc-9
-ENV CXX=g++-9
-ENV CPP=cpp-9
+ENV PATH=/opt/miniforge/bin:$PATH
+ENV CC=gcc-10
+ENV CXX=g++-10
+ENV CPP=cpp-10

 ENV GOSU_VERSION 1.10
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
--- a/tests/ci_build/Dockerfile.gpu
+++ b/tests/ci_build/Dockerfile.gpu
@ -15,9 +15,9 @@ RUN \
    apt-get install -y wget unzip bzip2 libgomp1 build-essential openjdk-8-jdk-headless && \
    # Python
    wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
-    bash conda.sh -b -p /opt/mambaforge
+    bash conda.sh -b -p /opt/miniforge

-ENV PATH=/opt/mambaforge/bin:$PATH
+ENV PATH=/opt/miniforge/bin:$PATH

 # Create new Conda environment with cuDF, Dask, and cuPy
 RUN \
--- a/tests/ci_build/Dockerfile.gpu_build_r_rockylinux8
+++ b/tests/ci_build/Dockerfile.gpu_build_r_rockylinux8
@ -1,5 +1,5 @@
 ARG CUDA_VERSION_ARG
-FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-centos7
+FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-rockylinux8
 ARG CUDA_VERSION_ARG
 ARG R_VERSION_ARG

@ -7,23 +7,23 @@ ARG R_VERSION_ARG
 RUN \
    curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/D42D0685.pub | sed '/^Version/d' \
        > /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA && \
-    yum install -y epel-release centos-release-scl && \
-    yum-config-manager --enable centos-sclo-rh-testing && \
-    yum -y update && \
-    yum install -y tar unzip wget xz git which ninja-build readline-devel libX11-devel libXt-devel \
+    dnf -y update && \
+    dnf -y install dnf-plugins-core && \
+    dnf config-manager --set-enabled powertools && \
+    dnf install -y tar unzip wget xz git which ninja-build readline-devel libX11-devel libXt-devel \
                   xorg-x11-server-devel openssl-devel zlib-devel bzip2-devel xz-devel \
                   pcre2-devel libcurl-devel texlive-* \
-                   devtoolset-9-gcc devtoolset-9-binutils devtoolset-9-gcc-c++ \
-                   devtoolset-9-gcc-gfortran devtoolset-9-libquadmath-devel \
-                   devtoolset-9-runtime devtoolset-9-libstdc++-devel
+                   gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ \
+                   gcc-toolset-10-gcc-gfortran gcc-toolset-10-libquadmath-devel \
+                   gcc-toolset-10-runtime gcc-toolset-10-libstdc++-devel

-ENV PATH=/opt/mambaforge/bin:/usr/local/ninja:/opt/software/packages/bin:/opt/R/$R_VERSION_ARG/bin:$PATH
+ENV PATH=/opt/miniforge/bin:/usr/local/ninja:/opt/software/packages/bin:/opt/R/$R_VERSION_ARG/bin:$PATH
 ENV LD_LIBRARY_PATH=/opt/software/packages/lib:/opt/R/$R_VERSION_ARG/lib64:$LD_LIBRARY_PATH
-ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc
-ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/c++
-ENV CPP=/opt/rh/devtoolset-9/root/usr/bin/cpp
-ENV F77=/opt/rh/devtoolset-9/root/usr/bin/gfortran
-ENV FC=/opt/rh/devtoolset-9/root/usr/bin/gfortran
+ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
+ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
+ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
+ENV F77=/opt/rh/gcc-toolset-10/root/usr/bin/gfortran
+ENV FC=/opt/rh/gcc-toolset-10/root/usr/bin/gfortran

 RUN \
    wget -nv -nc https://cran.r-project.org/src/base/R-4/R-$R_VERSION_ARG.tar.gz  && \
@ -36,11 +36,11 @@ RUN \
 run \
    # Python
    wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
-    bash conda.sh -b -p /opt/mambaforge && \
-    /opt/mambaforge/bin/python -m pip install auditwheel awscli && \
+    bash conda.sh -b -p /opt/miniforge && \
+    /opt/miniforge/bin/python -m pip install auditwheel awscli && \
    # CMake
-    wget -nv -nc https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.sh --no-check-certificate && \
-    bash cmake-3.18.0-Linux-x86_64.sh --skip-license --prefix=/usr
+    wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
+    bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr

 ENV GOSU_VERSION 1.10

--- a/tests/ci_build/Dockerfile.gpu_build_rockylinux8
+++ b/tests/ci_build/Dockerfile.gpu_build_rockylinux8
@ -1,5 +1,5 @@
 ARG CUDA_VERSION_ARG
-FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-centos7
+FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-rockylinux8
 ARG CUDA_VERSION_ARG
 ARG NCCL_VERSION_ARG
 ARG RAPIDS_VERSION_ARG
@ -8,39 +8,38 @@ ARG RAPIDS_VERSION_ARG
 RUN \
    curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/D42D0685.pub | sed '/^Version/d' \
        > /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA && \
-    yum install -y epel-release centos-release-scl && \
-    yum-config-manager --enable centos-sclo-rh-testing && \
-    yum -y update && \
-    yum install -y tar unzip wget xz git which ninja-build devtoolset-9-gcc devtoolset-9-binutils devtoolset-9-gcc-c++ && \
+    dnf -y update && \
+    dnf -y install dnf-plugins-core && \
+    dnf config-manager --set-enabled powertools && \
+    dnf install -y tar unzip wget xz git which ninja-build gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \
    # Python
    wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
-    bash conda.sh -b -p /opt/mambaforge && \
-    /opt/mambaforge/bin/python -m pip install awscli && \
+    bash conda.sh -b -p /opt/miniforge && \
+    /opt/miniforge/bin/python -m pip install awscli && \
    # CMake
-    wget -nv -nc https://cmake.org/files/v3.26/cmake-3.26.4-linux-x86_64.sh --no-check-certificate && \
-    bash cmake-3.26.4-linux-x86_64.sh --skip-license --prefix=/usr
+    wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
+    bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr

 # NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
 RUN \
    export CUDA_SHORT=`echo $CUDA_VERSION_ARG | grep -o -E '[0-9]+\.[0-9]'` && \
    export NCCL_VERSION=$NCCL_VERSION_ARG && \
-    wget -nv -nc https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
-    rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
-    yum -y update && \
-    yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} && \
-    rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm;
+    dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo && \
+    dnf -y update && \
+    dnf install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT}

-ENV PATH=/opt/mambaforge/bin:/usr/local/ninja:$PATH
-ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc
-ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/c++
-ENV CPP=/opt/rh/devtoolset-9/root/usr/bin/cpp
-ENV CUDAHOSTCXX=/opt/rh/devtoolset-9/root/usr/bin/c++
+ENV PATH=/opt/miniforge/bin:/usr/local/ninja:$PATH
+ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
+ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
+ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
+ENV CUDAHOSTCXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++

 ENV GOSU_VERSION 1.10

-# Install RMM
+# Install RMM (Patch out -Werror)
 RUN git clone -b v${RAPIDS_VERSION_ARG}.00 https://github.com/rapidsai/rmm.git --recurse-submodules --depth 1 && \
    pushd rmm && \
+    find . -name CMakeLists.txt -print0 | xargs -0 sed -i 's/-Werror//g' && \
    mkdir build && \
    pushd build && \
    cmake .. -GNinja -DCMAKE_INSTALL_PREFIX=/opt/rmm -DCUDA_STATIC_RUNTIME=ON && \
--- a/tests/ci_build/Dockerfile.gpu_dev_ver
+++ b/tests/ci_build/Dockerfile.gpu_dev_ver
@ -18,9 +18,9 @@ RUN \
    apt-get install -y wget unzip bzip2 libgomp1 build-essential openjdk-8-jdk-headless && \
    # Python
    wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
-    bash conda.sh -b -p /opt/mambaforge
+    bash conda.sh -b -p /opt/miniforge

-ENV PATH=/opt/mambaforge/bin:$PATH
+ENV PATH=/opt/miniforge/bin:$PATH

 # Create new Conda environment with dev versions of cuDF, Dask, and cuPy
 RUN \
--- a/tests/ci_build/Dockerfile.jvm
+++ b/tests/ci_build/Dockerfile.jvm
@ -1,32 +1,31 @@
-FROM centos:7
+FROM rockylinux:8

 # Install all basic requirements
 RUN \
-    yum install -y epel-release centos-release-scl && \
-    yum-config-manager --enable centos-sclo-rh-testing && \
-    yum -y update && \
-    yum install -y tar unzip make bzip2 wget xz git which ninja-build java-1.8.0-openjdk-devel \
-                   devtoolset-9-gcc devtoolset-9-binutils devtoolset-9-gcc-c++ \
-                   devtoolset-9-runtime devtoolset-9-libstdc++-devel && \
+    dnf -y update && \
+    dnf -y install dnf-plugins-core && \
+    dnf config-manager --set-enabled powertools && \
+    dnf install -y tar unzip make bzip2 wget xz git which ninja-build java-1.8.0-openjdk-devel \
+                   gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ \
+                   gcc-toolset-10-runtime gcc-toolset-10-libstdc++-devel && \
    # Python
    wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
-    bash conda.sh -b -p /opt/mambaforge && \
+    bash conda.sh -b -p /opt/miniforge && \
    # CMake
-    wget -nv -nc https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.sh --no-check-certificate && \
-    bash cmake-3.18.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
+    wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
+    bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr && \
    # Maven
-    wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz && \
-    tar xvf apache-maven-3.6.3-bin.tar.gz -C /opt && \
-    ln -s /opt/apache-maven-3.6.3/ /opt/maven
+    wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.9.7/binaries/apache-maven-3.9.7-bin.tar.gz && \
+    tar xvf apache-maven-3.9.7-bin.tar.gz -C /opt && \
+    ln -s /opt/apache-maven-3.9.7/ /opt/maven

-ENV PATH=/opt/mambaforge/bin:/opt/maven/bin:$PATH
-ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc
-ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/c++
-ENV CPP=/opt/rh/devtoolset-9/root/usr/bin/cpp
+ENV PATH=/opt/miniforge/bin:/opt/maven/bin:$PATH
+ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
+ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
+ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp

 # Install Python packages
-RUN \
-    pip install numpy pytest scipy scikit-learn wheel kubernetes awscli
+RUN pip install numpy pytest scipy scikit-learn wheel kubernetes awscli

 ENV GOSU_VERSION 1.10

--- a/tests/ci_build/Dockerfile.jvm_cross
+++ b/tests/ci_build/Dockerfile.jvm_cross
@ -1,6 +1,6 @@
-FROM ubuntu:18.04
+FROM ubuntu:22.04
 ARG JDK_VERSION=8
-ARG SPARK_VERSION=3.4.0
+ARG SPARK_VERSION=3.5.1

 # Environment
 ENV DEBIAN_FRONTEND noninteractive
@ -14,12 +14,12 @@ RUN \
    apt-get install -y tar unzip wget openjdk-$JDK_VERSION-jdk libgomp1 && \
    # Python
    wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
-    bash conda.sh -b -p /opt/mambaforge && \
-    /opt/mambaforge/bin/pip install awscli && \
+    bash conda.sh -b -p /opt/miniforge && \
+    /opt/miniforge/bin/pip install awscli && \
    # Maven
-    wget -nv https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz && \
-    tar xvf apache-maven-3.6.3-bin.tar.gz -C /opt && \
-    ln -s /opt/apache-maven-3.6.3/ /opt/maven && \
+    wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.9.7/binaries/apache-maven-3.9.7-bin.tar.gz && \
+    tar xvf apache-maven-3.9.7-bin.tar.gz -C /opt && \
+    ln -s /opt/apache-maven-3.9.7/ /opt/maven && \
    # Spark with scala 2.12
    mkdir -p /opt/spark-scala-2.12 && \
    wget -nv https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop3.tgz && \
@ -29,11 +29,10 @@ RUN \
    wget -nv https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop3-scala2.13.tgz && \
    tar xvf spark-$SPARK_VERSION-bin-hadoop3-scala2.13.tgz --strip-components=1 -C /opt/spark-scala-2.13

-ENV PATH=/opt/mambaforge/bin:/opt/spark/bin:/opt/maven/bin:$PATH
+ENV PATH=/opt/miniforge/bin:/opt/spark/bin:/opt/maven/bin:$PATH

 # Install Python packages
-RUN \
-    pip install numpy scipy pandas scikit-learn
+RUN pip install numpy scipy pandas scikit-learn

 ENV GOSU_VERSION 1.10

--- a/tests/ci_build/Dockerfile.jvm_gpu_build
+++ b/tests/ci_build/Dockerfile.jvm_gpu_build
@ -1,5 +1,5 @@
 ARG CUDA_VERSION_ARG
-FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-centos7
+FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-rockylinux8
 ARG CUDA_VERSION_ARG
 ARG NCCL_VERSION_ARG

@ -7,37 +7,36 @@ ARG NCCL_VERSION_ARG
 RUN \
    curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/D42D0685.pub | sed '/^Version/d' \
        > /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA && \
-    yum install -y epel-release centos-release-scl && \
-    yum-config-manager --enable centos-sclo-rh-testing && \
-    yum -y update && \
-    yum install -y tar unzip wget xz git which ninja-build java-1.8.0-openjdk-devel devtoolset-9-gcc devtoolset-9-binutils devtoolset-9-gcc-c++ && \
+    dnf -y update && \
+    dnf -y install dnf-plugins-core && \
+    dnf config-manager --set-enabled powertools && \
+    dnf install -y tar unzip wget xz git which ninja-build java-1.8.0-openjdk-devel gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \
    # Python
    wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
-    bash conda.sh -b -p /opt/mambaforge && \
+    bash conda.sh -b -p /opt/miniforge && \
    # CMake
-    wget -nv -nc https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.sh --no-check-certificate && \
-    bash cmake-3.18.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
+    wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
+    bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr && \
    # Maven
-    wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz && \
-    tar xvf apache-maven-3.6.3-bin.tar.gz -C /opt && \
-    ln -s /opt/apache-maven-3.6.3/ /opt/maven
+    wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.9.7/binaries/apache-maven-3.9.7-bin.tar.gz && \
+    tar xvf apache-maven-3.9.7-bin.tar.gz -C /opt && \
+    ln -s /opt/apache-maven-3.9.7/ /opt/maven

 # NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
 RUN \
    export CUDA_SHORT=`echo $CUDA_VERSION_ARG | grep -o -E '[0-9]+\.[0-9]'` && \
    export NCCL_VERSION=$NCCL_VERSION_ARG && \
-    yum-config-manager --add-repo http://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo && \
-    yum -y update && \
-    yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT}
+    dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo && \
+    dnf -y update && \
+    dnf install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT}

-ENV PATH=/opt/mambaforge/bin:/opt/maven/bin:$PATH
-ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc
-ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/c++
-ENV CPP=/opt/rh/devtoolset-9/root/usr/bin/cpp
+ENV PATH=/opt/miniforge/bin:/opt/maven/bin:$PATH
+ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
+ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
+ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp

 # Install Python packages
-RUN \
-    pip install numpy pytest scipy scikit-learn wheel kubernetes awscli
+RUN pip install numpy pytest scipy scikit-learn wheel kubernetes awscli

 ENV GOSU_VERSION 1.10

--- a/tests/ci_build/Dockerfile.s390x
+++ b/tests/ci_build/Dockerfile.s390x
@ -1,27 +0,0 @@
-FROM s390x/ubuntu:20.04
-
-# Environment
-ENV DEBIAN_FRONTEND noninteractive
-SHELL ["/bin/bash", "-c"]   # Use Bash as shell
-
-# Install all basic requirements
-RUN \
-    apt-get update && \
-    apt-get install -y --no-install-recommends tar unzip wget git build-essential ninja-build \
-      cmake time python3 python3-pip python3-numpy python3-scipy python3-sklearn r-base && \
-    python3 -m pip install pytest hypothesis
-
-ENV GOSU_VERSION 1.10
-
-# Install lightweight sudo (not bound to TTY)
-RUN set -ex; \
-    wget -nv -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
-    chmod +x /usr/local/bin/gosu && \
-    gosu nobody true
-
-# Default entry-point to use if running locally
-# It will preserve attributes of created files
-COPY entrypoint.sh /scripts/
-
-WORKDIR /workspace
-ENTRYPOINT ["/scripts/entrypoint.sh"]
--- a/tests/ci_build/build_via_cmake.sh
+++ b/tests/ci_build/build_via_cmake.sh
@ -11,7 +11,7 @@ then
  # Workaround for file permission error
  if [[ -n $CI_BUILD_UID ]]
  then
-    gosu root chown -R "${CI_BUILD_UID}:${CI_BUILD_GID}" /opt/mambaforge/envs
+    gosu root chown -R "${CI_BUILD_UID}:${CI_BUILD_GID}" /opt/miniforge/envs
  fi

  source activate ${conda_env}
@ -24,7 +24,9 @@ fi
 rm -rf build
 mkdir build
 cd build
-cmake .. ${cmake_args} -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_ALL_WARNINGS=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=ON -GNinja ${cmake_prefix_flag} -DHIDE_CXX_SYMBOLS=ON -DBUILD_DEPRECATED_CLI=ON
+# Disable CMAKE_COMPILE_WARNING_AS_ERROR option temporarily until
+# https://github.com/dmlc/xgboost/issues/10400 is fixed
+cmake .. ${cmake_args} -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_ALL_WARNINGS=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF -GNinja ${cmake_prefix_flag} -DHIDE_CXX_SYMBOLS=ON -DBUILD_DEPRECATED_CLI=ON
 ninja clean
 time ninja -v
 cd ..
--- a/tests/ci_build/format_wheel_meta.py
+++ b/tests/ci_build/format_wheel_meta.py
@ -49,7 +49,7 @@ if __name__ == "__main__":
        "--platform-tag",
        type=str,
        required=True,
-        help="Platform tag (e.g. manylinux2014_x86_64)",
+        help="Platform tag (e.g. manylinux_2_28_x86_64)",
    )
    parser.add_argument(
        "--meta-path", type=str, required=True, help="Directory to place meta.json"
--- a/tests/ci_build/rename_whl.py
+++ b/tests/ci_build/rename_whl.py
@ -26,9 +26,12 @@ def main(args):
    )
    new_wheel_path = wheel_dir / new_wheel_name
    print(f"Renaming {wheel_name} to {new_wheel_name}...")
-    if new_wheel_path.is_file():
-        new_wheel_path.unlink()
-    wheel_path.rename(new_wheel_path)
+    if new_wheel_name == wheel_name:
+        print("Skipping, as the old name is identical to the new name.")
+    else:
+        if new_wheel_path.is_file():
+            new_wheel_path.unlink()
+        wheel_path.rename(new_wheel_path)

    filesize = new_wheel_path.stat().st_size / 1024 / 1024  # MiB
    print(f"Wheel size: {filesize:.2f} MiB")
@ -53,7 +56,7 @@ if __name__ == "__main__":
        "--platform-tag",
        type=str,
        required=True,
-        help="Platform tag (e.g. manylinux2014_x86_64)",
+        help="Platform tag (e.g. manylinux_2_28_x86_64)",
    )
    parsed_args = parser.parse_args()
    main(parsed_args)
--- a/tests/ci_build/test_jvm_cross.sh
+++ b/tests/ci_build/test_jvm_cross.sh
@ -27,10 +27,10 @@ for scala_binary_version in "2.12" "2.13"; do
  scala_version=$(mvn help:evaluate -Dexpression=scala.version -q -DforceStdout)

  # Install XGBoost4J JAR into local Maven repository
-  mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar
-  mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}-tests.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=test-jar -Dclassifier=tests
-  mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-spark/target/xgboost4j-spark_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-spark_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar
-  mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-example/target/xgboost4j-example_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-example_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar
+  mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar -DgeneratePom=true
+  mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}-tests.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=test-jar -Dclassifier=tests -DgeneratePom=true
+  mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-spark/target/xgboost4j-spark_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-spark_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar -DgeneratePom=true
+  mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-example/target/xgboost4j-example_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-example_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar -DgeneratePom=true

  cd xgboost4j-tester
  # Generate pom.xml for XGBoost4J-tester, a dummy project to run XGBoost4J tests
--- a/tests/python/test_basic.py
+++ b/tests/python/test_basic.py
@ -244,7 +244,6 @@ class TestBasic:
        assert isinstance(cv, dict)
        assert len(cv) == (4)

-    @pytest.mark.skipif(**tm.skip_s390x())
    def test_cv_explicit_fold_indices_labels(self):
        params = {"max_depth": 2, "eta": 1, "objective": "reg:squarederror"}
        N = 100