[CI] Migrate to rockylinux8 / manylinux_2_28_x86_64 (#10399)

* [CI] Migrate to rockylinux8 / manylinux_2_28_x86_64

* Scrub all references to CentOS 7

* Fix

* Remove use of yum

* Use gcc-10 in cpu

* Temporarily disable -Werror

* Use GCC 9 for now

* Roll back gRPC

* Scrub all references to manylinux2014_x86_64

* Revise rename_whl.py to handle no-op rename

* Change JDK_VERSION back to 8

* Reviewer's comment

* Use GCC 10

* Use Spark 3.5.1, same as in pom.xml

* Fix JAR install

dev/release-artifacts.py

@@ -123,8 +123,8 @@ def download_py_packages(
 ) -> None:
     platforms = [
         "win_amd64",
-        "manylinux2014_x86_64",
+        "manylinux_2_28_x86_64",
-        "manylinux2014_aarch64",
+        "manylinux_2_28_aarch64",
         "macosx_10_15_x86_64.macosx_11_0_x86_64.macosx_12_0_x86_64",
         "macosx_12_0_arm64",
     ]

python-package/xgboost/testing/__init__.py

@@ -195,12 +195,6 @@ def skip_win() -> PytestSkip:
     return {"reason": "Unsupported platform.", "condition": is_windows()}
 
 
-def skip_s390x() -> PytestSkip:
-    condition = platform.machine() == "s390x"
-    reason = "Known to fail on s390x"
-    return {"condition": condition, "reason": reason}
-
-
 class IteratorForTest(xgb.core.DataIter):
     """Iterator for testing streaming DMatrix. (external memory, quantile)"""
 
@@ -523,7 +517,6 @@ def make_sparse_regression(
 
     """
     if not hasattr(np.random, "default_rng"):
-        # old version of numpy on s390x
         rng = np.random.RandomState(1994)
         X = sparse.random(
             m=n_samples,

tests/buildkite/build-containers.sh

@@ -20,7 +20,7 @@ case "${container}" in
   cpu)
     ;;
 
-  gpu|gpu_build_centos7)
+  gpu|gpu_build_rockylinux8)
     BUILD_ARGS="$BUILD_ARGS --build-arg CUDA_VERSION_ARG=$CUDA_VERSION"
     BUILD_ARGS="$BUILD_ARGS --build-arg NCCL_VERSION_ARG=$NCCL_VERSION"
     BUILD_ARGS="$BUILD_ARGS --build-arg RAPIDS_VERSION_ARG=$RAPIDS_VERSION"

tests/buildkite/build-cpu-arm64.sh

@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-WHEEL_TAG=manylinux2014_aarch64
+WHEEL_TAG=manylinux_2_28_aarch64
 
 echo "--- Build CPU code targeting ARM64"
 
@@ -24,7 +24,7 @@ $command_wrapper python tests/ci_build/rename_whl.py  \
   --commit-hash ${BUILDKITE_COMMIT}  \
   --platform-tag ${WHEEL_TAG}
 
-echo "--- Audit binary wheel to ensure it's compliant with manylinux2014 standard"
+echo "--- Audit binary wheel to ensure it's compliant with ${WHEEL_TAG} standard"
 $command_wrapper auditwheel repair --plat ${WHEEL_TAG} python-package/dist/*.whl
 $command_wrapper python tests/ci_build/rename_whl.py  \
   --wheel-path wheelhouse/*.whl  \

tests/buildkite/build-cuda-with-rmm.sh

@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-WHEEL_TAG=manylinux2014_x86_64
+WHEEL_TAG=manylinux_2_28_x86_64
 
 source tests/buildkite/conftest.sh
 
@@ -15,7 +15,7 @@ else
   arch_flag=""
 fi
 
-command_wrapper="tests/ci_build/ci_build.sh gpu_build_centos7 --build-arg "`
+command_wrapper="tests/ci_build/ci_build.sh gpu_build_rockylinux8 --build-arg "`
                 `"CUDA_VERSION_ARG=$CUDA_VERSION --build-arg "`
                 `"NCCL_VERSION_ARG=$NCCL_VERSION --build-arg "`
                 `"RAPIDS_VERSION_ARG=$RAPIDS_VERSION"
@@ -41,7 +41,7 @@ $command_wrapper python tests/ci_build/rename_whl.py  \
   --commit-hash ${BUILDKITE_COMMIT}  \
   --platform-tag ${WHEEL_TAG}
 
-echo "--- Audit binary wheel to ensure it's compliant with manylinux2014 standard"
+echo "--- Audit binary wheel to ensure it's compliant with ${WHEEL_TAG} standard"
 tests/ci_build/ci_build.sh auditwheel_x86_64 auditwheel repair \
   --plat ${WHEEL_TAG} python-package/dist/*.whl
 $command_wrapper python tests/ci_build/rename_whl.py  \

tests/buildkite/build-cuda.sh

@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-WHEEL_TAG=manylinux2014_x86_64
+WHEEL_TAG=manylinux_2_28_x86_64
 
 source tests/buildkite/conftest.sh
 
@@ -15,7 +15,7 @@ else
   arch_flag=""
 fi
 
-command_wrapper="tests/ci_build/ci_build.sh gpu_build_centos7 --build-arg "`
+command_wrapper="tests/ci_build/ci_build.sh gpu_build_rockylinux8 --build-arg "`
                 `"CUDA_VERSION_ARG=$CUDA_VERSION --build-arg "`
                 `"NCCL_VERSION_ARG=$NCCL_VERSION --build-arg "`
                 `"RAPIDS_VERSION_ARG=$RAPIDS_VERSION"
@@ -40,7 +40,7 @@ $command_wrapper python tests/ci_build/rename_whl.py  \
   --commit-hash ${BUILDKITE_COMMIT}  \
   --platform-tag ${WHEEL_TAG}
 
-echo "--- Audit binary wheel to ensure it's compliant with manylinux2014 standard"
+echo "--- Audit binary wheel to ensure it's compliant with ${WHEEL_TAG} standard"
 tests/ci_build/ci_build.sh auditwheel_x86_64 auditwheel repair \
   --plat ${WHEEL_TAG} python-package/dist/*.whl
 $command_wrapper python tests/ci_build/rename_whl.py  \

tests/buildkite/build-gpu-rpkg.sh

@@ -6,7 +6,7 @@ source tests/buildkite/conftest.sh
 
 echo "--- Build XGBoost R package with CUDA"
 
-tests/ci_build/ci_build.sh gpu_build_r_centos7 \
+tests/ci_build/ci_build.sh gpu_build_r_rockylinux8 \
 			   --build-arg CUDA_VERSION_ARG=${CUDA_VERSION} \
 			   --build-arg R_VERSION_ARG=${R_VERSION} \
 			   tests/ci_build/build_r_pkg_with_cuda.sh \

tests/buildkite/conftest.sh

@@ -26,7 +26,7 @@ CUDA_VERSION=11.8.0
 NCCL_VERSION=2.16.5-1
 RAPIDS_VERSION=24.04
 DEV_RAPIDS_VERSION=24.06
-SPARK_VERSION=3.4.0
+SPARK_VERSION=3.5.1
 JDK_VERSION=8
 R_VERSION=4.3.2
 

tests/buildkite/pipeline-mgpu.yml

@@ -17,7 +17,7 @@ steps:
   - label: ":docker: Build containers"
     commands:
       - "tests/buildkite/build-containers.sh gpu"
-      - "tests/buildkite/build-containers.sh gpu_build_centos7"
+      - "tests/buildkite/build-containers.sh gpu_build_rockylinux8"
       - "tests/buildkite/build-containers.sh jvm_gpu_build"
     key: build-containers
     agents:

tests/buildkite/pipeline-nightly.yml

@@ -12,7 +12,7 @@ steps:
   #### -------- CONTAINER BUILD --------
   - label: ":docker: Build containers"
     commands:
-      - "tests/buildkite/build-containers.sh gpu_build_centos7"
+      - "tests/buildkite/build-containers.sh gpu_build_rockylinux8"
       - "tests/buildkite/build-containers.sh gpu_dev_ver"
     key: build-containers
     agents:

tests/buildkite/pipeline.yml

@@ -15,7 +15,7 @@ steps:
     commands:
       - "tests/buildkite/build-containers.sh cpu"
       - "tests/buildkite/build-containers.sh gpu"
-      - "tests/buildkite/build-containers.sh gpu_build_centos7"
+      - "tests/buildkite/build-containers.sh gpu_build_rockylinux8"
     key: build-containers
     agents:
       queue: linux-amd64-cpu

tests/ci_build/Dockerfile.aarch64

@@ -1,22 +1,21 @@
-FROM quay.io/pypa/manylinux2014_aarch64
+FROM quay.io/pypa/manylinux_2_28_aarch64
 
 SHELL ["/bin/bash", "-c"]   # Use Bash as shell
 
 # Install all basic requirements
 RUN \
-    yum update -y && \
+    dnf -y update && \
-    yum install -y tar unzip wget xz git centos-release-scl-rh yum-utils && \
+    dnf -y install dnf-plugins-core && \
-    yum-config-manager --enable centos-sclo-rh-testing && \
+    dnf config-manager --set-enabled powertools && \
-    yum update -y && \
+    dnf install -y tar unzip wget xz git which ninja-build gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \
-    yum install -y devtoolset-9 && \
     # Python
     wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-aarch64.sh && \
-    bash conda.sh -b -p /opt/mambaforge
+    bash conda.sh -b -p /opt/miniforge
 
-ENV PATH=/opt/mambaforge/bin:$PATH
+ENV PATH=/opt/miniforge/bin:$PATH
-ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc
+ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
-ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/c++
+ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
-ENV CPP=/opt/rh/devtoolset-9/root/usr/bin/cpp
+ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
 ENV GOSU_VERSION 1.10
 
 # Create new Conda environment

tests/ci_build/Dockerfile.auditwheel_x86_64

@@ -1,4 +1,4 @@
-FROM quay.io/pypa/manylinux2014_x86_64
+FROM quay.io/pypa/manylinux_2_28_x86_64
 
 # Install lightweight sudo (not bound to TTY)
 ENV GOSU_VERSION 1.10

tests/ci_build/Dockerfile.cpu

@@ -10,15 +10,15 @@ RUN \
     apt-get install -y software-properties-common && \
     add-apt-repository ppa:ubuntu-toolchain-r/test && \
     apt-get update && \
-    apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libidn12 cmake ninja-build gcc-9 g++-9 openjdk-8-jdk-headless && \
+    apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libidn12 cmake ninja-build gcc-10 g++-10 openjdk-8-jdk-headless && \
     # Python
     wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
-    bash conda.sh -b -p /opt/mambaforge
+    bash conda.sh -b -p /opt/miniforge
 
-ENV PATH=/opt/mambaforge/bin:$PATH
+ENV PATH=/opt/miniforge/bin:$PATH
-ENV CC=gcc-9
+ENV CC=gcc-10
-ENV CXX=g++-9
+ENV CXX=g++-10
-ENV CPP=cpp-9
+ENV CPP=cpp-10
 
 ENV GOSU_VERSION 1.10
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/

tests/ci_build/Dockerfile.gpu

@@ -15,9 +15,9 @@ RUN \
     apt-get install -y wget unzip bzip2 libgomp1 build-essential openjdk-8-jdk-headless && \
     # Python
     wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
-    bash conda.sh -b -p /opt/mambaforge
+    bash conda.sh -b -p /opt/miniforge
 
-ENV PATH=/opt/mambaforge/bin:$PATH
+ENV PATH=/opt/miniforge/bin:$PATH
 
 # Create new Conda environment with cuDF, Dask, and cuPy
 RUN \

tests/ci_build/Dockerfile.gpu_build_r_rockylinux8

@@ -1,5 +1,5 @@
 ARG CUDA_VERSION_ARG
-FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-centos7
+FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-rockylinux8
 ARG CUDA_VERSION_ARG
 ARG R_VERSION_ARG
 
@@ -7,23 +7,23 @@ ARG R_VERSION_ARG
 RUN \
     curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/D42D0685.pub | sed '/^Version/d' \
         > /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA && \
-    yum install -y epel-release centos-release-scl && \
+    dnf -y update && \
-    yum-config-manager --enable centos-sclo-rh-testing && \
+    dnf -y install dnf-plugins-core && \
-    yum -y update && \
+    dnf config-manager --set-enabled powertools && \
-    yum install -y tar unzip wget xz git which ninja-build readline-devel libX11-devel libXt-devel \
+    dnf install -y tar unzip wget xz git which ninja-build readline-devel libX11-devel libXt-devel \
                    xorg-x11-server-devel openssl-devel zlib-devel bzip2-devel xz-devel \
                    pcre2-devel libcurl-devel texlive-* \
-                   devtoolset-9-gcc devtoolset-9-binutils devtoolset-9-gcc-c++ \
+                   gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ \
-                   devtoolset-9-gcc-gfortran devtoolset-9-libquadmath-devel \
+                   gcc-toolset-10-gcc-gfortran gcc-toolset-10-libquadmath-devel \
-                   devtoolset-9-runtime devtoolset-9-libstdc++-devel
+                   gcc-toolset-10-runtime gcc-toolset-10-libstdc++-devel
 
-ENV PATH=/opt/mambaforge/bin:/usr/local/ninja:/opt/software/packages/bin:/opt/R/$R_VERSION_ARG/bin:$PATH
+ENV PATH=/opt/miniforge/bin:/usr/local/ninja:/opt/software/packages/bin:/opt/R/$R_VERSION_ARG/bin:$PATH
 ENV LD_LIBRARY_PATH=/opt/software/packages/lib:/opt/R/$R_VERSION_ARG/lib64:$LD_LIBRARY_PATH
-ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc
+ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
-ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/c++
+ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
-ENV CPP=/opt/rh/devtoolset-9/root/usr/bin/cpp
+ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
-ENV F77=/opt/rh/devtoolset-9/root/usr/bin/gfortran
+ENV F77=/opt/rh/gcc-toolset-10/root/usr/bin/gfortran
-ENV FC=/opt/rh/devtoolset-9/root/usr/bin/gfortran
+ENV FC=/opt/rh/gcc-toolset-10/root/usr/bin/gfortran
 
 RUN \
     wget -nv -nc https://cran.r-project.org/src/base/R-4/R-$R_VERSION_ARG.tar.gz  && \
@@ -36,11 +36,11 @@ RUN \
 run \
     # Python
     wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
-    bash conda.sh -b -p /opt/mambaforge && \
+    bash conda.sh -b -p /opt/miniforge && \
-    /opt/mambaforge/bin/python -m pip install auditwheel awscli && \
+    /opt/miniforge/bin/python -m pip install auditwheel awscli && \
     # CMake
-    wget -nv -nc https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.sh --no-check-certificate && \
+    wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
-    bash cmake-3.18.0-Linux-x86_64.sh --skip-license --prefix=/usr
+    bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr
 
 ENV GOSU_VERSION 1.10
 

tests/ci_build/Dockerfile.gpu_build_rockylinux8

@@ -1,5 +1,5 @@
 ARG CUDA_VERSION_ARG
-FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-centos7
+FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-rockylinux8
 ARG CUDA_VERSION_ARG
 ARG NCCL_VERSION_ARG
 ARG RAPIDS_VERSION_ARG
@@ -8,39 +8,38 @@ ARG RAPIDS_VERSION_ARG
 RUN \
     curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/D42D0685.pub | sed '/^Version/d' \
         > /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA && \
-    yum install -y epel-release centos-release-scl && \
+    dnf -y update && \
-    yum-config-manager --enable centos-sclo-rh-testing && \
+    dnf -y install dnf-plugins-core && \
-    yum -y update && \
+    dnf config-manager --set-enabled powertools && \
-    yum install -y tar unzip wget xz git which ninja-build devtoolset-9-gcc devtoolset-9-binutils devtoolset-9-gcc-c++ && \
+    dnf install -y tar unzip wget xz git which ninja-build gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \
     # Python
     wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
-    bash conda.sh -b -p /opt/mambaforge && \
+    bash conda.sh -b -p /opt/miniforge && \
-    /opt/mambaforge/bin/python -m pip install awscli && \
+    /opt/miniforge/bin/python -m pip install awscli && \
     # CMake
-    wget -nv -nc https://cmake.org/files/v3.26/cmake-3.26.4-linux-x86_64.sh --no-check-certificate && \
+    wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
-    bash cmake-3.26.4-linux-x86_64.sh --skip-license --prefix=/usr
+    bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr
 
 # NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
 RUN \
     export CUDA_SHORT=`echo $CUDA_VERSION_ARG | grep -o -E '[0-9]+\.[0-9]'` && \
     export NCCL_VERSION=$NCCL_VERSION_ARG && \
-    wget -nv -nc https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
+    dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo && \
-    rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
+    dnf -y update && \
-    yum -y update && \
+    dnf install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT}
-    yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} && \
-    rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm;
 
-ENV PATH=/opt/mambaforge/bin:/usr/local/ninja:$PATH
+ENV PATH=/opt/miniforge/bin:/usr/local/ninja:$PATH
-ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc
+ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
-ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/c++
+ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
-ENV CPP=/opt/rh/devtoolset-9/root/usr/bin/cpp
+ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
-ENV CUDAHOSTCXX=/opt/rh/devtoolset-9/root/usr/bin/c++
+ENV CUDAHOSTCXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
 
 ENV GOSU_VERSION 1.10
 
-# Install RMM
+# Install RMM (Patch out -Werror)
 RUN git clone -b v${RAPIDS_VERSION_ARG}.00 https://github.com/rapidsai/rmm.git --recurse-submodules --depth 1 && \
     pushd rmm && \
+    find . -name CMakeLists.txt -print0 | xargs -0 sed -i 's/-Werror//g' && \
     mkdir build && \
     pushd build && \
     cmake .. -GNinja -DCMAKE_INSTALL_PREFIX=/opt/rmm -DCUDA_STATIC_RUNTIME=ON && \

tests/ci_build/Dockerfile.gpu_dev_ver

@@ -18,9 +18,9 @@ RUN \
     apt-get install -y wget unzip bzip2 libgomp1 build-essential openjdk-8-jdk-headless && \
     # Python
     wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
-    bash conda.sh -b -p /opt/mambaforge
+    bash conda.sh -b -p /opt/miniforge
 
-ENV PATH=/opt/mambaforge/bin:$PATH
+ENV PATH=/opt/miniforge/bin:$PATH
 
 # Create new Conda environment with dev versions of cuDF, Dask, and cuPy
 RUN \

tests/ci_build/Dockerfile.jvm

@@ -1,32 +1,31 @@
-FROM centos:7
+FROM rockylinux:8
 
 # Install all basic requirements
 RUN \
-    yum install -y epel-release centos-release-scl && \
+    dnf -y update && \
-    yum-config-manager --enable centos-sclo-rh-testing && \
+    dnf -y install dnf-plugins-core && \
-    yum -y update && \
+    dnf config-manager --set-enabled powertools && \
-    yum install -y tar unzip make bzip2 wget xz git which ninja-build java-1.8.0-openjdk-devel \
+    dnf install -y tar unzip make bzip2 wget xz git which ninja-build java-1.8.0-openjdk-devel \
-                   devtoolset-9-gcc devtoolset-9-binutils devtoolset-9-gcc-c++ \
+                   gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ \
-                   devtoolset-9-runtime devtoolset-9-libstdc++-devel && \
+                   gcc-toolset-10-runtime gcc-toolset-10-libstdc++-devel && \
     # Python
     wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
-    bash conda.sh -b -p /opt/mambaforge && \
+    bash conda.sh -b -p /opt/miniforge && \
     # CMake
-    wget -nv -nc https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.sh --no-check-certificate && \
+    wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
-    bash cmake-3.18.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
+    bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr && \
     # Maven
-    wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz && \
+    wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.9.7/binaries/apache-maven-3.9.7-bin.tar.gz && \
-    tar xvf apache-maven-3.6.3-bin.tar.gz -C /opt && \
+    tar xvf apache-maven-3.9.7-bin.tar.gz -C /opt && \
-    ln -s /opt/apache-maven-3.6.3/ /opt/maven
+    ln -s /opt/apache-maven-3.9.7/ /opt/maven
 
-ENV PATH=/opt/mambaforge/bin:/opt/maven/bin:$PATH
+ENV PATH=/opt/miniforge/bin:/opt/maven/bin:$PATH
-ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc
+ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
-ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/c++
+ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
-ENV CPP=/opt/rh/devtoolset-9/root/usr/bin/cpp
+ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
 
 # Install Python packages
-RUN \
+RUN pip install numpy pytest scipy scikit-learn wheel kubernetes awscli
-    pip install numpy pytest scipy scikit-learn wheel kubernetes awscli
 
 ENV GOSU_VERSION 1.10
 

tests/ci_build/Dockerfile.jvm_cross

@@ -1,6 +1,6 @@
-FROM ubuntu:18.04
+FROM ubuntu:22.04
 ARG JDK_VERSION=8
-ARG SPARK_VERSION=3.4.0
+ARG SPARK_VERSION=3.5.1
 
 # Environment
 ENV DEBIAN_FRONTEND noninteractive
@@ -14,12 +14,12 @@ RUN \
     apt-get install -y tar unzip wget openjdk-$JDK_VERSION-jdk libgomp1 && \
     # Python
     wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
-    bash conda.sh -b -p /opt/mambaforge && \
+    bash conda.sh -b -p /opt/miniforge && \
-    /opt/mambaforge/bin/pip install awscli && \
+    /opt/miniforge/bin/pip install awscli && \
     # Maven
-    wget -nv https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz && \
+    wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.9.7/binaries/apache-maven-3.9.7-bin.tar.gz && \
-    tar xvf apache-maven-3.6.3-bin.tar.gz -C /opt && \
+    tar xvf apache-maven-3.9.7-bin.tar.gz -C /opt && \
-    ln -s /opt/apache-maven-3.6.3/ /opt/maven && \
+    ln -s /opt/apache-maven-3.9.7/ /opt/maven && \
     # Spark with scala 2.12
     mkdir -p /opt/spark-scala-2.12 && \
     wget -nv https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop3.tgz && \
@@ -29,11 +29,10 @@ RUN \
     wget -nv https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop3-scala2.13.tgz && \
     tar xvf spark-$SPARK_VERSION-bin-hadoop3-scala2.13.tgz --strip-components=1 -C /opt/spark-scala-2.13
 
-ENV PATH=/opt/mambaforge/bin:/opt/spark/bin:/opt/maven/bin:$PATH
+ENV PATH=/opt/miniforge/bin:/opt/spark/bin:/opt/maven/bin:$PATH
 
 # Install Python packages
-RUN \
+RUN pip install numpy scipy pandas scikit-learn
-    pip install numpy scipy pandas scikit-learn
 
 ENV GOSU_VERSION 1.10
 

tests/ci_build/Dockerfile.jvm_gpu_build

@@ -1,5 +1,5 @@
 ARG CUDA_VERSION_ARG
-FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-centos7
+FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-rockylinux8
 ARG CUDA_VERSION_ARG
 ARG NCCL_VERSION_ARG
 
@@ -7,37 +7,36 @@ ARG NCCL_VERSION_ARG
 RUN \
     curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/D42D0685.pub | sed '/^Version/d' \
         > /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA && \
-    yum install -y epel-release centos-release-scl && \
+    dnf -y update && \
-    yum-config-manager --enable centos-sclo-rh-testing && \
+    dnf -y install dnf-plugins-core && \
-    yum -y update && \
+    dnf config-manager --set-enabled powertools && \
-    yum install -y tar unzip wget xz git which ninja-build java-1.8.0-openjdk-devel devtoolset-9-gcc devtoolset-9-binutils devtoolset-9-gcc-c++ && \
+    dnf install -y tar unzip wget xz git which ninja-build java-1.8.0-openjdk-devel gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \
     # Python
     wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
-    bash conda.sh -b -p /opt/mambaforge && \
+    bash conda.sh -b -p /opt/miniforge && \
     # CMake
-    wget -nv -nc https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.sh --no-check-certificate && \
+    wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
-    bash cmake-3.18.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
+    bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr && \
     # Maven
-    wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz && \
+    wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.9.7/binaries/apache-maven-3.9.7-bin.tar.gz && \
-    tar xvf apache-maven-3.6.3-bin.tar.gz -C /opt && \
+    tar xvf apache-maven-3.9.7-bin.tar.gz -C /opt && \
-    ln -s /opt/apache-maven-3.6.3/ /opt/maven
+    ln -s /opt/apache-maven-3.9.7/ /opt/maven
 
 # NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
 RUN \
     export CUDA_SHORT=`echo $CUDA_VERSION_ARG | grep -o -E '[0-9]+\.[0-9]'` && \
     export NCCL_VERSION=$NCCL_VERSION_ARG && \
-    yum-config-manager --add-repo http://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo && \
+    dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo && \
-    yum -y update && \
+    dnf -y update && \
-    yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT}
+    dnf install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT}
 
-ENV PATH=/opt/mambaforge/bin:/opt/maven/bin:$PATH
+ENV PATH=/opt/miniforge/bin:/opt/maven/bin:$PATH
-ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc
+ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
-ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/c++
+ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
-ENV CPP=/opt/rh/devtoolset-9/root/usr/bin/cpp
+ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
 
 # Install Python packages
-RUN \
+RUN pip install numpy pytest scipy scikit-learn wheel kubernetes awscli
-    pip install numpy pytest scipy scikit-learn wheel kubernetes awscli
 
 ENV GOSU_VERSION 1.10
 

tests/ci_build/Dockerfile.s390x

@@ -1,27 +0,0 @@
-FROM s390x/ubuntu:20.04
-
-# Environment
-ENV DEBIAN_FRONTEND noninteractive
-SHELL ["/bin/bash", "-c"]   # Use Bash as shell
-
-# Install all basic requirements
-RUN \
-    apt-get update && \
-    apt-get install -y --no-install-recommends tar unzip wget git build-essential ninja-build \
-      cmake time python3 python3-pip python3-numpy python3-scipy python3-sklearn r-base && \
-    python3 -m pip install pytest hypothesis
-
-ENV GOSU_VERSION 1.10
-
-# Install lightweight sudo (not bound to TTY)
-RUN set -ex; \
-    wget -nv -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
-    chmod +x /usr/local/bin/gosu && \
-    gosu nobody true
-
-# Default entry-point to use if running locally
-# It will preserve attributes of created files
-COPY entrypoint.sh /scripts/
-
-WORKDIR /workspace
-ENTRYPOINT ["/scripts/entrypoint.sh"]

tests/ci_build/build_via_cmake.sh

@@ -11,7 +11,7 @@ then
   # Workaround for file permission error
   if [[ -n $CI_BUILD_UID ]]
   then
-    gosu root chown -R "${CI_BUILD_UID}:${CI_BUILD_GID}" /opt/mambaforge/envs
+    gosu root chown -R "${CI_BUILD_UID}:${CI_BUILD_GID}" /opt/miniforge/envs
   fi
 
   source activate ${conda_env}
@@ -24,7 +24,9 @@ fi
 rm -rf build
 mkdir build
 cd build
-cmake .. ${cmake_args} -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_ALL_WARNINGS=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=ON -GNinja ${cmake_prefix_flag} -DHIDE_CXX_SYMBOLS=ON -DBUILD_DEPRECATED_CLI=ON
+# Disable CMAKE_COMPILE_WARNING_AS_ERROR option temporarily until
+# https://github.com/dmlc/xgboost/issues/10400 is fixed
+cmake .. ${cmake_args} -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_ALL_WARNINGS=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF -GNinja ${cmake_prefix_flag} -DHIDE_CXX_SYMBOLS=ON -DBUILD_DEPRECATED_CLI=ON
 ninja clean
 time ninja -v
 cd ..

tests/ci_build/format_wheel_meta.py

@@ -49,7 +49,7 @@ if __name__ == "__main__":
         "--platform-tag",
         type=str,
         required=True,
-        help="Platform tag (e.g. manylinux2014_x86_64)",
+        help="Platform tag (e.g. manylinux_2_28_x86_64)",
     )
     parser.add_argument(
         "--meta-path", type=str, required=True, help="Directory to place meta.json"

tests/ci_build/rename_whl.py

@@ -26,9 +26,12 @@ def main(args):
     )
     new_wheel_path = wheel_dir / new_wheel_name
     print(f"Renaming {wheel_name} to {new_wheel_name}...")
-    if new_wheel_path.is_file():
+    if new_wheel_name == wheel_name:
-        new_wheel_path.unlink()
+        print("Skipping, as the old name is identical to the new name.")
-    wheel_path.rename(new_wheel_path)
+    else:
+        if new_wheel_path.is_file():
+            new_wheel_path.unlink()
+        wheel_path.rename(new_wheel_path)
 
     filesize = new_wheel_path.stat().st_size / 1024 / 1024  # MiB
     print(f"Wheel size: {filesize:.2f} MiB")
@@ -53,7 +56,7 @@ if __name__ == "__main__":
         "--platform-tag",
         type=str,
         required=True,
-        help="Platform tag (e.g. manylinux2014_x86_64)",
+        help="Platform tag (e.g. manylinux_2_28_x86_64)",
     )
     parsed_args = parser.parse_args()
     main(parsed_args)

tests/ci_build/test_jvm_cross.sh

@@ -27,10 +27,10 @@ for scala_binary_version in "2.12" "2.13"; do
   scala_version=$(mvn help:evaluate -Dexpression=scala.version -q -DforceStdout)
 
   # Install XGBoost4J JAR into local Maven repository
-  mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar
+  mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar -DgeneratePom=true
-  mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}-tests.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=test-jar -Dclassifier=tests
+  mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}-tests.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=test-jar -Dclassifier=tests -DgeneratePom=true
-  mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-spark/target/xgboost4j-spark_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-spark_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar
+  mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-spark/target/xgboost4j-spark_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-spark_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar -DgeneratePom=true
-  mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-example/target/xgboost4j-example_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-example_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar
+  mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-example/target/xgboost4j-example_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-example_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar -DgeneratePom=true
 
   cd xgboost4j-tester
   # Generate pom.xml for XGBoost4J-tester, a dummy project to run XGBoost4J tests

tests/python/test_basic.py

@@ -244,7 +244,6 @@ class TestBasic:
         assert isinstance(cv, dict)
         assert len(cv) == (4)
 
-    @pytest.mark.skipif(**tm.skip_s390x())
     def test_cv_explicit_fold_indices_labels(self):
         params = {"max_depth": 2, "eta": 1, "objective": "reg:squarederror"}
         N = 100