From ac9136ee4956e05e0b20429222de930b33135e86 Mon Sep 17 00:00:00 2001 From: Philip Hyunsu Cho Date: Sat, 18 Jul 2020 03:30:40 -0700 Subject: [PATCH] Further improvements and savings in Jenkins pipeline (#5904) * Publish artifacts only on the master and release branches * Build CUDA only for Compute Capability 7.5 when building PRs * Run all Windows jobs in a single worker image * Build nightly XGBoost4J SNAPSHOT JARs with Scala 2.12 only * Show skipped Python tests on Windows * Make Graphviz optional for Python tests * Add back C++ tests * Unstash xgboost_cpp_tests * Fix label to CUDA 10.1 * Install cuPy for CUDA 10.1 * Install jsonschema * Address reviewer's feedback --- Jenkinsfile | 25 +++++++--- Jenkinsfile-win64 | 72 ++++++++++++--------------- tests/ci_build/deploy_jvm_packages.sh | 11 ---- tests/ci_build/win64_conda_env.yml | 18 +++++++ tests/python-gpu/test_gpu_demos.py | 3 ++ tests/python/test_plotting.py | 2 +- tests/python/test_with_sklearn.py | 1 + tests/python/testing.py | 20 ++++++++ 8 files changed, 93 insertions(+), 59 deletions(-) create mode 100644 tests/ci_build/win64_conda_env.yml diff --git a/Jenkinsfile b/Jenkinsfile index f65e2d6d7..a827b1a61 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -173,8 +173,10 @@ def Doxygen() { sh """ ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/doxygen.sh ${BRANCH_NAME} """ - echo 'Uploading doc...' - s3Upload file: "build/${BRANCH_NAME}.tar.bz2", bucket: 'xgboost-docs', acl: 'PublicRead', path: "doxygen/${BRANCH_NAME}.tar.bz2" + if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) { + echo 'Uploading doc...' + s3Upload file: "build/${BRANCH_NAME}.tar.bz2", bucket: 'xgboost-docs', acl: 'PublicRead', path: "doxygen/${BRANCH_NAME}.tar.bz2" + } deleteDir() } } @@ -245,8 +247,12 @@ def BuildCUDA(args) { def container_type = "gpu_build" def docker_binary = "docker" def docker_args = "--build-arg CUDA_VERSION=${args.cuda_version}" + def arch_flag = "" + if (env.BRANCH_NAME != 'master' && !(env.BRANCH_NAME.startsWith('release'))) { + arch_flag = "-DGPU_COMPUTE_VER=75" + } sh """ - ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_CUDA=ON -DUSE_NCCL=ON -DOPEN_MP:BOOL=ON -DHIDE_CXX_SYMBOLS=ON + ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_CUDA=ON -DUSE_NCCL=ON -DOPEN_MP:BOOL=ON -DHIDE_CXX_SYMBOLS=ON ${arch_flag} ${dockerRun} ${container_type} ${docker_binary} ${docker_args} bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal" ${dockerRun} ${container_type} ${docker_binary} ${docker_args} python3 tests/ci_build/rename_whl.py python-package/dist/*.whl ${commit_id} manylinux2010_x86_64 """ @@ -254,8 +260,11 @@ def BuildCUDA(args) { if (args.cuda_version == '10.0') { echo 'Stashing Python wheel...' stash name: 'xgboost_whl_cuda10', includes: 'python-package/dist/*.whl' - path = ("${BRANCH_NAME}" == 'master') ? '' : "${BRANCH_NAME}/" - s3Upload bucket: 'xgboost-nightly-builds', path: path, acl: 'PublicRead', workingDir: 'python-package/dist', includePathPattern:'**/*.whl' + if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) { + echo 'Uploading Python wheel...' + path = ("${BRANCH_NAME}" == 'master') ? '' : "${BRANCH_NAME}/" + s3Upload bucket: 'xgboost-nightly-builds', path: path, acl: 'PublicRead', workingDir: 'python-package/dist', includePathPattern:'**/*.whl' + } echo 'Stashing C++ test executable (testxgboost)...' stash name: 'xgboost_cpp_tests', includes: 'build/testxgboost' } @@ -289,8 +298,10 @@ def BuildJVMDoc() { sh """ ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_jvm_doc.sh ${BRANCH_NAME} """ - echo 'Uploading doc...' - s3Upload file: "jvm-packages/${BRANCH_NAME}.tar.bz2", bucket: 'xgboost-docs', acl: 'PublicRead', path: "${BRANCH_NAME}.tar.bz2" + if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) { + echo 'Uploading doc...' + s3Upload file: "jvm-packages/${BRANCH_NAME}.tar.bz2", bucket: 'xgboost-docs', acl: 'PublicRead', path: "${BRANCH_NAME}.tar.bz2" + } deleteDir() } } diff --git a/Jenkinsfile-win64 b/Jenkinsfile-win64 index 60ea44352..7c9654f6e 100644 --- a/Jenkinsfile-win64 +++ b/Jenkinsfile-win64 @@ -10,6 +10,15 @@ def commit_id // necessary to pass a variable from one stage to another pipeline { agent none + + // Setup common job properties + options { + timestamps() + timeout(time: 240, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '10')) + preserveStashes() + } + // Build stages stages { stage('Jenkins Win64: Initialize') { @@ -29,7 +38,7 @@ pipeline { steps { script { parallel ([ - 'build-win64-cuda10.0': { BuildWin64() } + 'build-win64-cuda10.1': { BuildWin64() } ]) } milestone ordinal: 2 @@ -40,8 +49,7 @@ pipeline { steps { script { parallel ([ - 'test-win64-cpu': { TestWin64CPU() }, - 'test-win64-gpu-cuda10.1': { TestWin64GPU(cuda_target: 'cuda10_1') } + 'test-win64-cuda10.1': { TestWin64() }, ]) } milestone ordinal: 3 @@ -66,14 +74,18 @@ def checkoutSrcs() { } def BuildWin64() { - node('win64 && build && cuda10') { + node('win64 && cuda10_unified') { unstash name: 'srcs' echo "Building XGBoost for Windows AMD64 target..." bat "nvcc --version" + def arch_flag = "" + if (env.BRANCH_NAME != 'master' && !(env.BRANCH_NAME.startsWith('release'))) { + arch_flag = "-DGPU_COMPUTE_VER=75" + } bat """ mkdir build cd build - cmake .. -G"Visual Studio 15 2017 Win64" -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON + cmake .. -G"Visual Studio 15 2017 Win64" -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON ${arch_flag} """ bat """ cd build @@ -91,8 +103,11 @@ def BuildWin64() { """ echo 'Stashing Python wheel...' stash name: 'xgboost_whl', includes: 'python-package/dist/*.whl' - path = ("${BRANCH_NAME}" == 'master') ? '' : "${BRANCH_NAME}/" - s3Upload bucket: 'xgboost-nightly-builds', path: path, acl: 'PublicRead', workingDir: 'python-package/dist', includePathPattern:'**/*.whl' + if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) { + echo 'Uploading Python wheel...' + path = ("${BRANCH_NAME}" == 'master') ? '' : "${BRANCH_NAME}/" + s3Upload bucket: 'xgboost-nightly-builds', path: path, acl: 'PublicRead', workingDir: 'python-package/dist', includePathPattern:'**/*.whl' + } echo 'Stashing C++ test executable (testxgboost)...' stash name: 'xgboost_cpp_tests', includes: 'build/testxgboost.exe' stash name: 'xgboost_cli', includes: 'xgboost.exe' @@ -100,52 +115,29 @@ def BuildWin64() { } } -def TestWin64CPU() { - node('win64 && cpu') { +def TestWin64() { + node('win64 && cuda10_unified') { unstash name: 'srcs' unstash name: 'xgboost_whl' unstash name: 'xgboost_cli' - echo "Test Win64 CPU" - echo "Installing Python wheel..." - bat "conda activate && (python -m pip uninstall -y xgboost || cd .)" - bat """ - conda activate && for /R %%i in (python-package\\dist\\*.whl) DO python -m pip install "%%i" - """ - echo "Installing Python dependencies..." - bat """ - conda activate && conda install -y hypothesis && conda upgrade scikit-learn pandas numpy hypothesis - """ - echo "Running Python tests..." - bat "conda activate && python -m pytest -v -s --fulltrace tests\\python" - bat "conda activate && python -m pip uninstall -y xgboost" - deleteDir() - } -} - -def TestWin64GPU(args) { - node("win64 && gpu && ${args.cuda_target}") { - unstash name: 'srcs' - unstash name: 'xgboost_whl' unstash name: 'xgboost_cpp_tests' - echo "Test Win64 GPU (${args.cuda_target})" + echo "Test Win64" bat "nvcc --version" echo "Running C++ tests..." bat "build\\testxgboost.exe" - echo "Installing Python wheel..." - bat "conda activate && (python -m pip uninstall -y xgboost || cd .)" - bat """ - conda activate && for /R %%i in (python-package\\dist\\*.whl) DO python -m pip install "%%i" - """ echo "Installing Python dependencies..." - def cuda_short_ver = args.cuda_target.replaceAll('_', '') + def env_name = 'win64_' + UUID.randomUUID().toString().replaceAll('-', '') + bat "conda env create -n ${env_name} --file=tests/ci_build/win64_conda_env.yml" + echo "Installing Python wheel..." bat """ - conda activate && conda install -y hypothesis && conda upgrade scikit-learn pandas numpy hypothesis && python -m pip install cupy-${cuda_short_ver} + conda activate ${env_name} && for /R %%i in (python-package\\dist\\*.whl) DO python -m pip install "%%i" """ echo "Running Python tests..." + bat "conda activate ${env_name} && python -m pytest -v -s -rxXs --fulltrace tests\\python" bat """ - conda activate && python -m pytest -v -s --fulltrace -m "(not slow) and (not mgpu)" tests\\python-gpu + conda activate ${env_name} && python -m pytest -v -s -rxXs --fulltrace -m "(not slow) and (not mgpu)" tests\\python-gpu """ - bat "conda activate && python -m pip uninstall -y xgboost" + bat "conda env remove --name ${env_name}" deleteDir() } } diff --git a/tests/ci_build/deploy_jvm_packages.sh b/tests/ci_build/deploy_jvm_packages.sh index dff2c6649..950cfeb38 100755 --- a/tests/ci_build/deploy_jvm_packages.sh +++ b/tests/ci_build/deploy_jvm_packages.sh @@ -20,16 +20,5 @@ cd jvm-packages # Deploy to S3 bucket xgboost-maven-repo mvn --no-transfer-progress package deploy -P release-to-s3 -Dspark.version=${spark_version} -DskipTests -# Compile XGBoost4J with Scala 2.11 too -mvn clean -# Rename artifactId of all XGBoost4J packages with suffix _2.11 -sed -i -e 's/xgboost\(.*\)_[0-9\.]\+/xgboost\1_2.11/' $(find . -name pom.xml) -# Modify scala.version and scala.binary.version fields -sed -i -e 's/[0-9\.]\+/2.11.12/' $(find . -name pom.xml) -sed -i -e 's/[0-9\.]\+/2.11/' $(find . -name pom.xml) - -# Re-build and deploy -mvn --no-transfer-progress package deploy -P release-to-s3 -Dspark.version=${spark_version} -DskipTests - set +x set +e diff --git a/tests/ci_build/win64_conda_env.yml b/tests/ci_build/win64_conda_env.yml new file mode 100644 index 000000000..ad84aebc3 --- /dev/null +++ b/tests/ci_build/win64_conda_env.yml @@ -0,0 +1,18 @@ +name: win64_env +channels: +- conda-forge +dependencies: +- python=3.7 +- numpy +- scipy +- matplotlib +- scikit-learn +- pandas +- pytest +- python-graphviz +- boto3 +- hypothesis +- jsonschema +- pip +- pip: + - cupy-cuda101 diff --git a/tests/python-gpu/test_gpu_demos.py b/tests/python-gpu/test_gpu_demos.py index 03c4a6279..a3a9aaff5 100644 --- a/tests/python-gpu/test_gpu_demos.py +++ b/tests/python-gpu/test_gpu_demos.py @@ -1,10 +1,13 @@ import os import subprocess import sys +import pytest sys.path.append("tests/python") +import testing as tm import test_demos as td # noqa +@pytest.mark.skipif(**tm.no_cupy()) def test_data_iterator(): script = os.path.join(td.PYTHON_DEMO_DIR, 'data_iterator.py') cmd = ['python', script] diff --git a/tests/python/test_plotting.py b/tests/python/test_plotting.py index a5a62f8b9..18b0b83c7 100644 --- a/tests/python/test_plotting.py +++ b/tests/python/test_plotting.py @@ -15,7 +15,7 @@ except ImportError: pass -pytestmark = pytest.mark.skipif(**tm.no_matplotlib()) +pytestmark = pytest.mark.skipif(**tm.no_multiple(tm.no_matplotlib(), tm.no_graphviz())) dpath = 'demo/data/' diff --git a/tests/python/test_with_sklearn.py b/tests/python/test_with_sklearn.py index 0bb5b3ada..06742d4ca 100644 --- a/tests/python/test_with_sklearn.py +++ b/tests/python/test_with_sklearn.py @@ -437,6 +437,7 @@ def test_sklearn_api_gblinear(): @pytest.mark.skipif(**tm.no_matplotlib()) +@pytest.mark.skipif(**tm.no_graphviz()) def test_sklearn_plotting(): from sklearn.datasets import load_iris diff --git a/tests/python/testing.py b/tests/python/testing.py index f8d2431bf..7ae202f01 100644 --- a/tests/python/testing.py +++ b/tests/python/testing.py @@ -98,6 +98,26 @@ def no_json_schema(): return {'condition': True, 'reason': reason} +def no_graphviz(): + reason = 'graphviz is not installed' + try: + import graphviz # noqa + return {'condition': False, 'reason': reason} + except ImportError: + return {'condition': True, 'reason': reason} + + +def no_multiple(*args): + condition = False + reason = '' + for arg in args: + condition = (condition or arg['condition']) + if arg['condition']: + reason = arg['reason'] + break + return {'condition': condition, 'reason': reason} + + # Contains a dataset in numpy format as well as the relevant objective and metric class TestDataset: def __init__(self, name, get_dataset, objective, metric