Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CI] add script to generate meta info and upload to s3 (#10295)

Browse Source 
* [CI] add script to generate meta info and upload to s3

* Write Python script to generate meta.json

* Update other pipelines

* Add wheel_name field

* Add description

---------

Co-authored-by: Hyunsu Cho <phcho@nvidia.com>
This commit is contained in:
Bobby Wang
2024-05-25 01:03:28 +08:00
committed by GitHub
parent 5086decb0c
commit 9def441e9a
7 changed files with 147 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
tests/buildkite/build-cpu-arm64.sh
View File

@@ -19,13 +19,17 @@ $command_wrapper bash -c "cd build && ctest --extra-verbose"
echo "--- Build binary wheel"
$command_wrapper bash -c \
"cd python-package && rm -rf dist/* && pip wheel --no-deps -v . --wheel-dir dist/"
$command_wrapper python tests/ci_build/rename_whl.py python-package/dist/*.whl \
${BUILDKITE_COMMIT} ${WHEEL_TAG}
$command_wrapper python tests/ci_build/rename_whl.py \
--wheel-path python-package/dist/*.whl \
--commit-hash ${BUILDKITE_COMMIT} \
--platform-tag ${WHEEL_TAG}
echo "--- Audit binary wheel to ensure it's compliant with manylinux2014 standard"
$command_wrapper auditwheel repair --plat ${WHEEL_TAG} python-package/dist/*.whl
$command_wrapper python tests/ci_build/rename_whl.py wheelhouse/*.whl \
${BUILDKITE_COMMIT} ${WHEEL_TAG}
$command_wrapper python tests/ci_build/rename_whl.py \
--wheel-path wheelhouse/*.whl \
--commit-hash ${BUILDKITE_COMMIT} \
--platform-tag ${WHEEL_TAG}
mv -v wheelhouse/*.whl python-package/dist/
# Make sure that libgomp.so is vendored in the wheel
$command_wrapper bash -c \

12
tests/buildkite/build-cuda-with-rmm.sh
View File

@@ -36,14 +36,18 @@ $command_wrapper tests/ci_build/build_via_cmake.sh \
echo "--- Build binary wheel"
$command_wrapper bash -c \
"cd python-package && rm -rf dist/* && pip wheel --no-deps -v . --wheel-dir dist/"
$command_wrapper python tests/ci_build/rename_whl.py python-package/dist/*.whl \
${BUILDKITE_COMMIT} ${WHEEL_TAG}
$command_wrapper python tests/ci_build/rename_whl.py \
--wheel-path python-package/dist/*.whl \
--commit-hash ${BUILDKITE_COMMIT} \
--platform-tag ${WHEEL_TAG}
echo "--- Audit binary wheel to ensure it's compliant with manylinux2014 standard"
tests/ci_build/ci_build.sh auditwheel_x86_64 auditwheel repair \
--plat ${WHEEL_TAG} python-package/dist/*.whl
$command_wrapper python tests/ci_build/rename_whl.py wheelhouse/*.whl \
${BUILDKITE_COMMIT} ${WHEEL_TAG}
$command_wrapper python tests/ci_build/rename_whl.py \
--wheel-path wheelhouse/*.whl \
--commit-hash ${BUILDKITE_COMMIT} \
--platform-tag ${WHEEL_TAG}
mv -v wheelhouse/*.whl python-package/dist/
# Make sure that libgomp.so is vendored in the wheel
tests/ci_build/ci_build.sh auditwheel_x86_64 bash -c \

21
tests/buildkite/build-cuda.sh
View File

@@ -35,14 +35,18 @@ $command_wrapper tests/ci_build/build_via_cmake.sh \
echo "--- Build binary wheel"
$command_wrapper bash -c \
"cd python-package && rm -rf dist/* && pip wheel --no-deps -v . --wheel-dir dist/"
$command_wrapper python tests/ci_build/rename_whl.py python-package/dist/*.whl \
${BUILDKITE_COMMIT} ${WHEEL_TAG}
$command_wrapper python tests/ci_build/rename_whl.py \
--wheel-path python-package/dist/*.whl \
--commit-hash ${BUILDKITE_COMMIT} \
--platform-tag ${WHEEL_TAG}
echo "--- Audit binary wheel to ensure it's compliant with manylinux2014 standard"
tests/ci_build/ci_build.sh auditwheel_x86_64 auditwheel repair \
--plat ${WHEEL_TAG} python-package/dist/*.whl
$command_wrapper python tests/ci_build/rename_whl.py wheelhouse/*.whl \
${BUILDKITE_COMMIT} ${WHEEL_TAG}
$command_wrapper python tests/ci_build/rename_whl.py \
--wheel-path wheelhouse/*.whl \
--commit-hash ${BUILDKITE_COMMIT} \
--platform-tag ${WHEEL_TAG}
mv -v wheelhouse/*.whl python-package/dist/
# Make sure that libgomp.so is vendored in the wheel
tests/ci_build/ci_build.sh auditwheel_x86_64 bash -c \
@@ -54,6 +58,15 @@ if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
then
aws s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/${BRANCH_NAME}/ \
--acl public-read --no-progress
# Generate the meta info which includes xgboost version and the commit info
$command_wrapper python tests/ci_build/format_wheel_meta.py \
--wheel-path python-package/dist/*.whl \
--commit-hash ${BUILDKITE_COMMIT} \
--platform-tag ${WHEEL_TAG} \
--meta-path python-package/dist/
aws s3 cp python-package/dist/meta.json s3://xgboost-nightly-builds/${BRANCH_NAME}/ \
--acl public-read --no-progress
fi
echo "-- Stash C++ test executable (testxgboost)"
buildkite-agent artifact upload build/testxgboost

7
tests/buildkite/build-win64-gpu.ps1
View File

@@ -29,7 +29,10 @@ conda activate
& pip wheel --no-deps -v . --wheel-dir dist/
Get-ChildItem . -Filter dist/*.whl |
Foreach-Object {
& python ../tests/ci_build/rename_whl.py $_.FullName $Env:BUILDKITE_COMMIT win_amd64
& python ../tests/ci_build/rename_whl.py `
--wheel-path $_.FullName `
--commit-hash $Env:BUILDKITE_COMMIT `
--platform-tag win_amd64
if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
}
@@ -44,7 +47,7 @@ if ( $is_release_branch -eq 1 ) {
Get-ChildItem . -Filter python-package/dist/*.whl |
Foreach-Object {
& aws s3 cp python-package/dist/$_ s3://xgboost-nightly-builds/$Env:BUILDKITE_BRANCH/ `
--acl public-read --no-progress
--acl public-read --no-progress
if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
}
}