[CI] Set up pipeline to build manylinux2014 wheels (#10478)

tests/buildkite/build-cuda-with-rmm.sh

@@ -42,7 +42,7 @@ $command_wrapper python tests/ci_build/rename_whl.py  \
   --platform-tag ${WHEEL_TAG}
 
 echo "--- Audit binary wheel to ensure it's compliant with ${WHEEL_TAG} standard"
-tests/ci_build/ci_build.sh auditwheel_x86_64 auditwheel repair \
+tests/ci_build/ci_build.sh manylinux_2_28_x86_64 auditwheel repair \
   --plat ${WHEEL_TAG} python-package/dist/*.whl
 $command_wrapper python tests/ci_build/rename_whl.py  \
   --wheel-path wheelhouse/*.whl  \
@@ -50,7 +50,7 @@ $command_wrapper python tests/ci_build/rename_whl.py  \
   --platform-tag ${WHEEL_TAG}
 mv -v wheelhouse/*.whl python-package/dist/
 # Make sure that libgomp.so is vendored in the wheel
-tests/ci_build/ci_build.sh auditwheel_x86_64 bash -c \
+tests/ci_build/ci_build.sh manylinux_2_28_x86_64 bash -c \
   "unzip -l python-package/dist/*.whl | grep libgomp  || exit -1"
 
 echo "--- Upload Python wheel"

tests/buildkite/build-cuda.sh

@@ -41,7 +41,7 @@ $command_wrapper python tests/ci_build/rename_whl.py  \
   --platform-tag ${WHEEL_TAG}
 
 echo "--- Audit binary wheel to ensure it's compliant with ${WHEEL_TAG} standard"
-tests/ci_build/ci_build.sh auditwheel_x86_64 auditwheel repair \
+tests/ci_build/ci_build.sh manylinux_2_28_x86_64 auditwheel repair \
   --plat ${WHEEL_TAG} python-package/dist/*.whl
 $command_wrapper python tests/ci_build/rename_whl.py  \
   --wheel-path wheelhouse/*.whl  \
@@ -49,7 +49,7 @@ $command_wrapper python tests/ci_build/rename_whl.py  \
   --platform-tag ${WHEEL_TAG}
 mv -v wheelhouse/*.whl python-package/dist/
 # Make sure that libgomp.so is vendored in the wheel
-tests/ci_build/ci_build.sh auditwheel_x86_64 bash -c \
+tests/ci_build/ci_build.sh manylinux_2_28_x86_64 bash -c \
   "unzip -l python-package/dist/*.whl | grep libgomp  || exit -1"
 
 echo "--- Upload Python wheel"

tests/buildkite/build-manylinux2014-aarch64.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source tests/buildkite/conftest.sh
+
+WHEEL_TAG=manylinux2014_aarch64
+command_wrapper="tests/ci_build/ci_build.sh manylinux2014_aarch64"
+python_bin="/opt/python/cp310-cp310/bin/python"
+
+echo "--- Build binary wheel for ${WHEEL_TAG}"
+# Patch to add warning about manylinux2014 variant
+patch -p0 < tests/buildkite/manylinux2014_warning.patch
+$command_wrapper bash -c \
+  "cd python-package && ${python_bin} -m pip wheel --no-deps -vvv . --wheel-dir dist/"
+git checkout python-package/xgboost/core.py  # discard the patch
+
+$command_wrapper auditwheel repair --plat ${WHEEL_TAG} python-package/dist/*.whl
+$command_wrapper ${python_bin} tests/ci_build/rename_whl.py  \
+  --wheel-path wheelhouse/*.whl  \
+  --commit-hash ${BUILDKITE_COMMIT}  \
+  --platform-tag ${WHEEL_TAG}
+rm -rf python-package/dist/
+mkdir python-package/dist/
+mv -v wheelhouse/*.whl python-package/dist/
+
+echo "--- Upload Python wheel"
+buildkite-agent artifact upload python-package/dist/*.whl
+$command_wrapper bash -c "${python_bin} -m pip install awscli"
+if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
+then
+  ${python_bin} -m awscli s3 \
+    cp python-package/dist/*.whl s3://xgboost-nightly-builds/${BRANCH_NAME}/ \
+    --acl public-read --no-progress
+fi

tests/buildkite/build-manylinux2014-x86_64.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source tests/buildkite/conftest.sh
+
+WHEEL_TAG=manylinux2014_x86_64
+command_wrapper="tests/ci_build/ci_build.sh manylinux2014_x86_64"
+python_bin="/opt/python/cp310-cp310/bin/python"
+
+echo "--- Build binary wheel for ${WHEEL_TAG}"
+# Patch to add warning about manylinux2014 variant
+patch -p0 < tests/buildkite/manylinux2014_warning.patch
+$command_wrapper bash -c \
+  "cd python-package && ${python_bin} -m pip wheel --no-deps -vvv . --wheel-dir dist/"
+git checkout python-package/xgboost/core.py  # discard the patch
+
+$command_wrapper auditwheel repair --plat ${WHEEL_TAG} python-package/dist/*.whl
+$command_wrapper ${python_bin} tests/ci_build/rename_whl.py  \
+  --wheel-path wheelhouse/*.whl  \
+  --commit-hash ${BUILDKITE_COMMIT}  \
+  --platform-tag ${WHEEL_TAG}
+rm -rf python-package/dist/
+mkdir python-package/dist/
+mv -v wheelhouse/*.whl python-package/dist/
+
+echo "--- Upload Python wheel"
+buildkite-agent artifact upload python-package/dist/*.whl
+$command_wrapper bash -c "${python_bin} -m pip install awscli"
+if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
+then
+  ${python_bin} -m awscli s3 \
+    cp python-package/dist/*.whl s3://xgboost-nightly-builds/${BRANCH_NAME}/ \
+    --acl public-read --no-progress
+fi

tests/buildkite/manylinux2014_warning.patch

@@ -0,0 +1,37 @@
+diff --git python-package/pyproject.toml python-package/pyproject.toml
+index a273d8c13..dee49686a 100644
+--- python-package/pyproject.toml
++++ python-package/pyproject.toml
+@@ -30,8 +30,7 @@ classifiers = [
+ ]
+ dependencies = [
+     "numpy",
+-    "scipy",
+-    "nvidia-nccl-cu12 ; platform_system == 'Linux' and platform_machine != 'aarch64'"
++    "scipy"
+ ]
+ 
+ [project.urls]
+diff --git python-package/xgboost/core.py python-package/xgboost/core.py
+index e8bc735e6..030972ef2 100644
+--- python-package/xgboost/core.py
++++ python-package/xgboost/core.py
+@@ -262,6 +262,18 @@ Likely cause:
+         )
+         raise ValueError(msg)
+ 
++    warnings.warn(
++        "Your system has an old version of glibc (< 2.28). We will stop supporting "
++        "Linux distros with glibc older than 2.28 after **May 31, 2025**. "
++        "Please upgrade to a recent Linux distro (with glibc 2.28+) to use "
++        "future versions of XGBoost.\n"
++        "Note: You have installed the 'manylinux2014' variant of XGBoost. Certain "
++        "features such as GPU algorithms or federated learning are not available. "
++        "To use these features, please upgrade to a recent Linux distro with glibc "
++        "2.28+, and install the 'manylinux_2_28' variant.",
++        FutureWarning
++    )
++
+     return lib
+ 
+ 

tests/buildkite/pipeline.yml

@@ -26,18 +26,17 @@ steps:
     key: run-clang-tidy
     agents:
       queue: linux-amd64-cpu
-  - wait
   - label: ":console: Build CPU"
     command: "tests/buildkite/build-cpu.sh"
     key: build-cpu
     agents:
       queue: linux-amd64-cpu
-  - label: ":console: Build CPU ARM64"
+  - label: ":console: Build CPU ARM64 + manylinux_2_28_aarch64 wheel"
     command: "tests/buildkite/build-cpu-arm64.sh"
     key: build-cpu-arm64
     agents:
       queue: linux-arm64-cpu
-  - label: ":console: Build CUDA"
+  - label: ":console: Build CUDA + manylinux_2_28_x86_64 wheel"
     command: "tests/buildkite/build-cuda.sh"
     key: build-cuda
     agents:
@@ -62,6 +61,16 @@ steps:
     key: build-jvm-doc
     agents:
       queue: linux-amd64-cpu
+  - label: ":console: Build manylinux2014_x86_64 wheel"
+    command: "tests/buildkite/build-manylinux2014-x86_64.sh"
+    key: build-manylinux2014-x86_64
+    agents:
+      queue: linux-amd64-cpu
+  - label: ":console: Build manylinux2014_aarch64 wheel"
+    command: "tests/buildkite/build-manylinux2014-aarch64.sh"
+    key: build-manylinux2014-aarch64
+    agents:
+      queue: linux-arm64-cpu
   - wait
   #### -------- TEST --------
   - label: ":console: Test Python package, CPU"

tests/ci_build/Dockerfile.manylinux2014_aarch64

@@ -0,0 +1,15 @@
+FROM quay.io/pypa/manylinux2014_aarch64
+
+# Install lightweight sudo (not bound to TTY)
+ENV GOSU_VERSION 1.10
+RUN set -ex; \
+    curl -o /usr/local/bin/gosu -L "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-arm64" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]

tests/ci_build/Dockerfile.manylinux2014_x86_64

@@ -0,0 +1,15 @@
+FROM quay.io/pypa/manylinux2014_x86_64
+
+# Install lightweight sudo (not bound to TTY)
+ENV GOSU_VERSION 1.10
+RUN set -ex; \
+    curl -o /usr/local/bin/gosu -L "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]