[CI] Clean up Python wheel build pipeline (#7626)
* [CI] Always upload artifacts to [branch_name]/ * [CI] Move detailed setup inside build_python_wheels.sh * Fix typo
This commit is contained in:
Philip Hyunsu Cho
GitHub
parent
f6e6d0b2c0
commit
34a238ca98
21
.github/workflows/python_wheels.yml
vendored
21
.github/workflows/python_wheels.yml
vendored
@ -10,13 +10,9 @@ jobs:
|
||||
matrix:
|
||||
include:
|
||||
- os: macos-latest
|
||||
python: 37
|
||||
platform_id: macosx_x86_64
|
||||
wheel_tag: macosx_10_15_x86_64.macosx_11_0_x86_64.macosx_12_0_x86_64
|
||||
- os: macos-latest
|
||||
python: 38
|
||||
platform_id: macosx_arm64
|
||||
wheel_tag: macosx_12_0_arm64
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
@ -25,23 +21,8 @@ jobs:
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: '3.9'
|
||||
- name: Set env var for ARM64
|
||||
shell: bash
|
||||
run: echo "::set-output name=value::CIBW_TARGET_OSX_ARM64=1"
|
||||
id: arm64_flag
|
||||
if: matrix.platform_id == 'macosx_arm64'
|
||||
- name: Build wheels
|
||||
env:
|
||||
CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }}
|
||||
CIBW_ARCHS: all
|
||||
CIBW_ENVIRONMENT: ${{ steps.arm64_flag.outputs.value }}
|
||||
CIBW_TEST_SKIP: "*-macosx_arm64"
|
||||
CIBW_BUILD_VERBOSITY: 3
|
||||
run: bash tests/ci_build/build_python_wheels.sh
|
||||
|
||||
- name: Rename Python wheel
|
||||
run: |
|
||||
python tests/ci_build/rename_whl.py wheelhouse/*.whl ${{ github.sha }} ${{ matrix.wheel_tag }}
|
||||
run: bash tests/ci_build/build_python_wheels.sh ${{ matrix.platform_id }} ${{ github.sha }}
|
||||
- name: Extract branch name
|
||||
shell: bash
|
||||
run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
|
||||
|
||||
6
Jenkinsfile
vendored
6
Jenkinsfile
vendored
@ -179,7 +179,7 @@ def BuildCPUARM64() {
|
||||
stash name: "xgboost_whl_arm64_cpu", includes: 'python-package/dist/*.whl'
|
||||
if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) {
|
||||
echo 'Uploading Python wheel...'
|
||||
path = ("${BRANCH_NAME}" == 'master') ? '' : "${BRANCH_NAME}/"
|
||||
path = "${BRANCH_NAME}/"
|
||||
s3Upload bucket: 'xgboost-nightly-builds', path: path, acl: 'PublicRead', workingDir: 'python-package/dist', includePathPattern:'**/*.whl'
|
||||
}
|
||||
stash name: 'xgboost_cli_arm64', includes: 'xgboost'
|
||||
@ -232,7 +232,7 @@ def BuildCUDA(args) {
|
||||
stash name: "xgboost_whl_cuda${args.cuda_version}", includes: 'python-package/dist/*.whl'
|
||||
if (args.cuda_version == ref_cuda_ver && (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release'))) {
|
||||
echo 'Uploading Python wheel...'
|
||||
path = ("${BRANCH_NAME}" == 'master') ? '' : "${BRANCH_NAME}/"
|
||||
path = "${BRANCH_NAME}/"
|
||||
s3Upload bucket: 'xgboost-nightly-builds', path: path, acl: 'PublicRead', workingDir: 'python-package/dist', includePathPattern:'**/*.whl'
|
||||
}
|
||||
echo 'Stashing C++ test executable (testxgboost)...'
|
||||
@ -268,7 +268,7 @@ def BuildRPackageWithCUDA(args) {
|
||||
${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_r_pkg_with_cuda.sh ${commit_id}
|
||||
"""
|
||||
echo 'Uploading R tarball...'
|
||||
path = ("${BRANCH_NAME}" == 'master') ? '' : "${BRANCH_NAME}/"
|
||||
path = "${BRANCH_NAME}/"
|
||||
s3Upload bucket: 'xgboost-nightly-builds', path: path, acl: 'PublicRead', includePathPattern:'xgboost_r_gpu_linux_*.tar.gz'
|
||||
}
|
||||
deleteDir()
|
||||
|
||||
@ -107,7 +107,7 @@ def BuildWin64() {
|
||||
stash name: 'xgboost_whl', includes: 'python-package/dist/*.whl'
|
||||
if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) {
|
||||
echo 'Uploading Python wheel...'
|
||||
path = ("${BRANCH_NAME}" == 'master') ? '' : "${BRANCH_NAME}/"
|
||||
path = "${BRANCH_NAME}/"
|
||||
s3Upload bucket: 'xgboost-nightly-builds', path: path, acl: 'PublicRead', workingDir: 'python-package/dist', includePathPattern:'**/*.whl'
|
||||
}
|
||||
echo 'Stashing C++ test executable (testxgboost)...'
|
||||
@ -127,7 +127,7 @@ def BuildRPackageWithCUDAWin64() {
|
||||
bash tests/ci_build/build_r_pkg_with_cuda_win64.sh ${commit_id}
|
||||
"""
|
||||
echo 'Uploading R tarball...'
|
||||
path = ("${BRANCH_NAME}" == 'master') ? '' : "${BRANCH_NAME}/"
|
||||
path = "${BRANCH_NAME}/"
|
||||
s3Upload bucket: 'xgboost-nightly-builds', path: path, acl: 'PublicRead', includePathPattern:'xgboost_r_gpu_win64_*.tar.gz'
|
||||
}
|
||||
deleteDir()
|
||||
|
||||
@ -3,36 +3,66 @@
|
||||
set -e
|
||||
set -x
|
||||
|
||||
if [[ $# -ne 2 ]]; then
|
||||
echo "Usage: $0 [platform_id] [commit ID]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
platform_id=$1
|
||||
commit_id=$2
|
||||
|
||||
# Bundle libomp 11.1.0 when targeting MacOS.
|
||||
# This is a workaround in order to prevent segfaults when running inside a Conda environment.
|
||||
# See https://github.com/dmlc/xgboost/issues/7039#issuecomment-1025125003 for more context.
|
||||
# The workaround is also used by the scikit-learn project.
|
||||
if [[ "$RUNNER_OS" == "macOS" ]]; then
|
||||
if [[ "$platform_id" == macosx_* ]]; then
|
||||
# Make sure to use a libomp version binary compatible with the oldest
|
||||
# supported version of the macos SDK as libomp will be vendored into the
|
||||
# XGBoost wheels for MacOS.
|
||||
|
||||
if [[ "$CIBW_BUILD" == *-macosx_arm64 ]]; then
|
||||
if [[ "$platform_id" == macosx_arm64 ]]; then
|
||||
# MacOS, Apple Silicon
|
||||
# arm64 builds must cross compile because CI is on x64
|
||||
# cibuildwheel will take care of cross-compilation.
|
||||
wheel_tag=macosx_12_0_arm64
|
||||
cpython_ver=38
|
||||
setup_env_var='CIBW_TARGET_OSX_ARM64=1' # extra flag to be passed to setup.py
|
||||
export PYTHON_CROSSENV=1
|
||||
export MACOSX_DEPLOYMENT_TARGET=12.0
|
||||
OPENMP_URL="https://anaconda.org/conda-forge/llvm-openmp/11.1.0/download/osx-arm64/llvm-openmp-11.1.0-hf3c4609_1.tar.bz2"
|
||||
else
|
||||
elif [[ "$platform_id" == macosx_x86_64 ]]; then
|
||||
# MacOS, Intel
|
||||
wheel_tag=macosx_10_15_x86_64.macosx_11_0_x86_64.macosx_12_0_x86_64
|
||||
cpython_ver=37
|
||||
export MACOSX_DEPLOYMENT_TARGET=10.13
|
||||
OPENMP_URL="https://anaconda.org/conda-forge/llvm-openmp/11.1.0/download/osx-64/llvm-openmp-11.1.0-hda6cdc1_1.tar.bz2"
|
||||
else
|
||||
echo "Platform not supported: $platform_id"
|
||||
exit 3
|
||||
fi
|
||||
# Set up environment variables to configure cibuildwheel
|
||||
export CIBW_BUILD=cp${cpython_ver}-${platform_id}
|
||||
export CIBW_ARCHS=all
|
||||
export CIBW_ENVIRONMENT=${setup_env_var}
|
||||
export CIBW_TEST_SKIP='*-macosx_arm64'
|
||||
export CIBW_BUILD_VERBOSITY=3
|
||||
|
||||
sudo conda create -n build $OPENMP_URL
|
||||
PREFIX="/usr/local/miniconda/envs/build"
|
||||
|
||||
# Set up build flags for cibuildwheel
|
||||
# This is needed to bundle libomp lib we downloaded earlier
|
||||
export CC=/usr/bin/clang
|
||||
export CXX=/usr/bin/clang++
|
||||
export CPPFLAGS="$CPPFLAGS -Xpreprocessor -fopenmp"
|
||||
export CFLAGS="$CFLAGS -I$PREFIX/include"
|
||||
export CXXFLAGS="$CXXFLAGS -I$PREFIX/include"
|
||||
export LDFLAGS="$LDFLAGS -Wl,-rpath,$PREFIX/lib -L$PREFIX/lib -lomp"
|
||||
else
|
||||
echo "Platform not supported: $platform_id"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
python -m pip install cibuildwheel
|
||||
python -m cibuildwheel python-package --output-dir wheelhouse
|
||||
python tests/ci_build/rename_whl.py wheelhouse/*.whl ${commit_id} ${wheel_tag}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user