Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

More CI improvements (#8313)

Browse Source 
* Reduce clutter in log of Python test

* Set up BuildKite test analytics

* Add separate step for building containers

* Enable incremental update of CI stack; custom agent IAM policy
This commit is contained in:
Philip Hyunsu Cho
2022-10-06 06:33:46 -08:00
committed by GitHub
parent bc7a6ec603
commit 50ff8a2623
11 changed files with 230 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests/ci_build/Dockerfile.gpu
View File

@@ -25,7 +25,9 @@ RUN \
python=3.9 cudf=$RAPIDS_VERSION_ARG* rmm=$RAPIDS_VERSION_ARG* cudatoolkit=$CUDA_VERSION_ARG \
dask dask-cuda=$RAPIDS_VERSION_ARG* dask-cudf=$RAPIDS_VERSION_ARG* cupy \
numpy pytest scipy scikit-learn pandas matplotlib wheel python-kubernetes urllib3 graphviz hypothesis \
pyspark cloudpickle cuda-python=11.7.0 && mamba clean --all
pyspark cloudpickle cuda-python=11.7.0 && \
mamba clean --all && \
conda run --no-capture-output -n gpu_test pip install buildkite-test-collector
ENV GOSU_VERSION 1.10
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/

6
tests/ci_build/Dockerfile.rmm
View File

@@ -27,8 +27,10 @@ ENV PATH=/opt/python/bin:$PATH
# Create new Conda environment with RMM
RUN \
conda create -n gpu_test -c rapidsai-nightly -c rapidsai -c nvidia -c conda-forge -c defaults \
python=3.9 rmm=$RAPIDS_VERSION_ARG* cudatoolkit=$CUDA_VERSION_ARG cmake
conda install -c conda-forge mamba && \
mamba create -n gpu_test -c rapidsai-nightly -c rapidsai -c nvidia -c conda-forge -c defaults \
python=3.9 rmm=$RAPIDS_VERSION_ARG* cudatoolkit=$CUDA_VERSION_ARG cmake && \
mamba clean --all
ENV GOSU_VERSION 1.10

9
tests/ci_build/test_python.sh
View File

@@ -1,6 +1,5 @@
#!/bin/bash
set -e
set -x
if [ "$#" -lt 1 ]
then
@@ -54,39 +53,47 @@ function uninstall_xgboost {
case "$suite" in
gpu)
source activate gpu_test
set -x
install_xgboost
setup_pyspark_envs
pytest -v -s -rxXs --fulltrace --durations=0 -m "not mgpu" ${args} tests/python-gpu
unset_pyspark_envs
uninstall_xgboost
set +x
;;
mgpu)
source activate gpu_test
set -x
install_xgboost
setup_pyspark_envs
pytest -v -s -rxXs --fulltrace --durations=0 -m "mgpu" ${args} tests/python-gpu
unset_pyspark_envs
uninstall_xgboost
set +x
;;
cpu)
source activate cpu_test
set -x
install_xgboost
export RAY_OBJECT_STORE_ALLOW_SLOW_STORAGE=1
setup_pyspark_envs
pytest -v -s -rxXs --fulltrace --durations=0 ${args} tests/python
unset_pyspark_envs
uninstall_xgboost
set +x
;;
cpu-arm64)
source activate aarch64_test
set -x
install_xgboost
setup_pyspark_envs
pytest -v -s -rxXs --fulltrace --durations=0 ${args} tests/python/test_basic.py tests/python/test_basic_models.py tests/python/test_model_compatibility.py
unset_pyspark_envs
uninstall_xgboost
set +x
;;
*)