From 29d6a5e2b8ce3ae3cc5db4a25ea69fc866818a11 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Thu, 27 May 2021 04:49:45 +0800 Subject: [PATCH] [CI] Move appveyor tests to action (#6986) * Drop support for VS14, use VS15 instead. * Drop support for mingw. * Remove debug build. * Split up jvm tests. * Split up Python tests. --- .github/workflows/jvm_tests.yml | 74 +++++++++++++ .github/workflows/main.yml | 109 -------------------- .github/workflows/python_tests.yml | 91 ++++++++++++++++ appveyor.yml | 71 ------------- tests/ci_build/conda_env/win64_cpu_test.yml | 21 ++++ 5 files changed, 186 insertions(+), 180 deletions(-) create mode 100644 .github/workflows/jvm_tests.yml create mode 100644 .github/workflows/python_tests.yml delete mode 100644 appveyor.yml create mode 100644 tests/ci_build/conda_env/win64_cpu_test.yml diff --git a/.github/workflows/jvm_tests.yml b/.github/workflows/jvm_tests.yml new file mode 100644 index 000000000..314bc1286 --- /dev/null +++ b/.github/workflows/jvm_tests.yml @@ -0,0 +1,74 @@ +name: XGBoost-JVM-Tests + +on: [push, pull_request] + +jobs: + test-with-jvm: + name: Test JVM on OS ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [windows-latest, ubuntu-latest] + + steps: + - uses: actions/checkout@v2 + with: + submodules: 'true' + + - uses: actions/setup-python@v2 + with: + python-version: '3.8' + architecture: 'x64' + + - uses: actions/setup-java@v1 + with: + java-version: 1.8 + + - name: Install Python packages + run: | + python -m pip install wheel setuptools + python -m pip install awscli + + - name: Cache Maven packages + uses: actions/cache@v2 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 + + - name: Test XGBoost4J + run: | + cd jvm-packages + mvn test -B -pl :xgboost4j_2.12 + + - name: Extract branch name + shell: bash + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + id: extract_branch + if: | + (github.ref == 'refs/heads/master' || contains(github.ref, 'refs/heads/release_')) && + matrix.os == 'windows-latest' + + - name: Publish artifact xgboost4j.dll to S3 + run: | + cd lib/ + Rename-Item -Path xgboost4j.dll -NewName xgboost4j_${{ github.sha }}.dll + dir + python -m awscli s3 cp xgboost4j_${{ github.sha }}.dll s3://xgboost-nightly-builds/${{ steps.extract_branch.outputs.branch }}/ --acl public-read + if: | + (github.ref == 'refs/heads/master' || contains(github.ref, 'refs/heads/release_')) && + matrix.os == 'windows-latest' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_IAM_S3_UPLOADER }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_IAM_S3_UPLOADER }} + + + - name: Test XGBoost4J-Spark + run: | + rm -rfv build/ + cd jvm-packages + mvn -B test + if: matrix.os == 'ubuntu-latest' # Distributed training doesn't work on Windows + env: + RABIT_MOCK: ON diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8f100f052..81989e2ee 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -62,45 +62,6 @@ jobs: cd build ctest --extra-verbose - python-sdist-test: - name: Test installing XGBoost Python source package - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-10.15, windows-latest] - python-version: ["3.8"] - steps: - - uses: actions/checkout@v2 - with: - submodules: 'true' - - name: Install osx system dependencies - if: matrix.os == 'macos-10.15' - run: | - brew install ninja libomp - - name: Install Ubuntu system dependencies - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt-get install -y --no-install-recommends ninja-build - - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - python-version: ${{ matrix.python-version }} - activate-environment: test - - name: Display Conda env - shell: bash -l {0} - run: | - conda info - conda list - - name: Build and install XGBoost - shell: bash -l {0} - run: | - cd python-package - python --version - python setup.py sdist - pip install -v ./dist/xgboost-*.tar.gz - cd .. - python -c 'import xgboost' - c-api-demo: name: Test installing XGBoost lib + building the C API demo runs-on: ${{ matrix.os }} @@ -144,76 +105,6 @@ jobs: cd .. ./build/api-demo - test-with-jvm: - name: Test JVM on OS ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [windows-latest, ubuntu-latest] - - steps: - - uses: actions/checkout@v2 - with: - submodules: 'true' - - - uses: actions/setup-python@v2 - with: - python-version: '3.8' - architecture: 'x64' - - - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - - name: Install Python packages - run: | - python -m pip install wheel setuptools - python -m pip install awscli - - - name: Cache Maven packages - uses: actions/cache@v2 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - - name: Test XGBoost4J - run: | - cd jvm-packages - mvn test -B -pl :xgboost4j_2.12 - - - name: Extract branch name - shell: bash - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - id: extract_branch - if: | - (github.ref == 'refs/heads/master' || contains(github.ref, 'refs/heads/release_')) && - matrix.os == 'windows-latest' - - - name: Publish artifact xgboost4j.dll to S3 - run: | - cd lib/ - Rename-Item -Path xgboost4j.dll -NewName xgboost4j_${{ github.sha }}.dll - dir - python -m awscli s3 cp xgboost4j_${{ github.sha }}.dll s3://xgboost-nightly-builds/${{ steps.extract_branch.outputs.branch }}/ --acl public-read - if: | - (github.ref == 'refs/heads/master' || contains(github.ref, 'refs/heads/release_')) && - matrix.os == 'windows-latest' - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_IAM_S3_UPLOADER }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_IAM_S3_UPLOADER }} - - - - name: Test XGBoost4J-Spark - run: | - rm -rfv build/ - cd jvm-packages - mvn -B test - if: matrix.os == 'ubuntu-latest' # Distributed training doesn't work on Windows - env: - RABIT_MOCK: ON - lint: runs-on: ubuntu-latest name: Code linting for Python and C++ diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml new file mode 100644 index 000000000..411caa0ef --- /dev/null +++ b/.github/workflows/python_tests.yml @@ -0,0 +1,91 @@ +name: XGBoost-Python-Tests + +on: [push, pull_request] + +jobs: + python-sdist-test: + runs-on: ${{ matrix.os }} + name: Test installing XGBoost Python source package on ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-10.15, windows-latest] + python-version: ["3.8"] + steps: + - uses: actions/checkout@v2 + with: + submodules: 'true' + - name: Install osx system dependencies + if: matrix.os == 'macos-10.15' + run: | + brew install ninja libomp + - name: Install Ubuntu system dependencies + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt-get install -y --no-install-recommends ninja-build + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ matrix.python-version }} + activate-environment: test + - name: Display Conda env + shell: bash -l {0} + run: | + conda info + conda list + - name: Build and install XGBoost + shell: bash -l {0} + run: | + cd python-package + python --version + python setup.py sdist + pip install -v ./dist/xgboost-*.tar.gz + cd .. + python -c 'import xgboost' + + python-tests: + name: Test XGBoost Python package on ${{ matrix.config.os }} + runs-on: ${{ matrix.config.os }} + strategy: + matrix: + config: + - {os: windows-2016, compiler: 'msvc', python-version: '3.8'} + + steps: + - uses: actions/checkout@v2 + with: + submodules: 'true' + + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ matrix.config.python-version }} + activate-environment: win64_test + environment-file: tests/ci_build/conda_env/win64_cpu_test.yml + + - name: Display Conda env + shell: bash -l {0} + run: | + conda info + conda list + + - name: Build XGBoost with msvc + shell: bash -l {0} + if: matrix.config.compiler == 'msvc' + run: | + mkdir build_msvc + cd build_msvc + cmake .. -G"Visual Studio 15 2017" -DCMAKE_CONFIGURATION_TYPES="Release" -A x64 -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON + cmake --build . --config Release --parallel $(nproc) + + - name: Install Python package + shell: bash -l {0} + run: | + cd python-package + python --version + python setup.py bdist_wheel --universal + pip install ./dist/*.whl + + - name: Test Python package + shell: bash -l {0} + run: | + pytest -s -v ./tests/python diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index e90df234e..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,71 +0,0 @@ -environment: - matrix: - - target: msvc - ver: 2015 - generator: "Visual Studio 14 2015 Win64" - configuration: Debug - - target: msvc - ver: 2015 - generator: "Visual Studio 14 2015 Win64" - configuration: Release - - target: mingw - generator: "Unix Makefiles" - -#matrix: -# fast_finish: true - -platform: - - x64 - -install: - - git submodule update --init --recursive - # MinGW - - set PATH=C:\msys64\mingw64\bin;C:\msys64\usr\bin;%PATH% - - gcc -v - - ls -l C:\ - # Miniconda3 - - call C:\Miniconda3-x64\Scripts\activate.bat - - conda info - - where python - - python --version - # do python build for mingw and one of the msvc jobs - - set DO_PYTHON=off - - if /i "%target%" == "mingw" set DO_PYTHON=on - - if /i "%target%_%ver%_%configuration%" == "msvc_2015_Release" set DO_PYTHON=on - - if /i "%DO_PYTHON%" == "on" ( - conda config --set always_yes true && - conda update -q conda && - conda install -y numpy scipy pandas matplotlib pytest scikit-learn graphviz python-graphviz hypothesis - ) - - set PATH=C:\Miniconda3-x64\Library\bin\graphviz;%PATH% - -build_script: - - cd %APPVEYOR_BUILD_FOLDER% - - if /i "%target%" == "msvc" ( - mkdir build_msvc%ver% && - cd build_msvc%ver% && - cmake .. -G"%generator%" -DCMAKE_CONFIGURATION_TYPES="Release;Debug;" && - msbuild xgboost.sln - ) - - if /i "%target%" == "mingw" ( - mkdir build_mingw && - cd build_mingw && - cmake .. -G"%generator%" && - make -j2 - ) - # Python package - - if /i "%DO_PYTHON%" == "on" ( - cd %APPVEYOR_BUILD_FOLDER%\python-package && - python setup.py install && - mkdir wheel && - python setup.py bdist_wheel --universal --plat-name win-amd64 -d wheel - ) - -test_script: - - cd %APPVEYOR_BUILD_FOLDER% - - if /i "%DO_PYTHON%" == "on" python -m pytest tests/python - -artifacts: - # binary Python wheel package - - path: '**\*.whl' - name: Bits diff --git a/tests/ci_build/conda_env/win64_cpu_test.yml b/tests/ci_build/conda_env/win64_cpu_test.yml new file mode 100644 index 000000000..dab8e2d2d --- /dev/null +++ b/tests/ci_build/conda_env/win64_cpu_test.yml @@ -0,0 +1,21 @@ +name: win64_env +channels: +- conda-forge +dependencies: +- python +- wheel +- numpy +- scipy +- scikit-learn +- pandas +- matplotlib +- dask +- distributed +- python-graphviz +- pytest +- jsonschema +- hypothesis +- jsonschema +- pip +- pip: + - graphviz