From eb281ff9b48bf88f01ec5ee0b1f41648ca27014f Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 23 Feb 2024 06:21:32 +0800 Subject: [PATCH] [CI] Fix JVM tests on GH Action (#10064) --------- Co-authored-by: Hyunsu Cho --- .github/workflows/jvm_tests.yml | 32 ++++++------ jvm-packages/create_jni.py | 75 +++++++++++++++++----------- python-package/packager/nativelib.py | 9 ++-- 3 files changed, 69 insertions(+), 47 deletions(-) diff --git a/.github/workflows/jvm_tests.yml b/.github/workflows/jvm_tests.yml index 330c037d7..13c343d32 100644 --- a/.github/workflows/jvm_tests.yml +++ b/.github/workflows/jvm_tests.yml @@ -15,31 +15,35 @@ jobs: os: [windows-latest, ubuntu-latest, macos-11] steps: - - uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: submodules: 'true' - - uses: actions/setup-python@7f80679172b057fc5e90d70d197929d454754a5a # v4.3.0 + - uses: mamba-org/setup-micromamba@422500192359a097648154e8db4e39bdb6c6eed7 # v1.8.1 with: - python-version: '3.8' - architecture: 'x64' - - - uses: actions/setup-java@d202f5dbf7256730fb690ec59f6381650114feb2 # v3.6.0 - with: - java-version: 1.8 - - - name: Install Python packages - run: | - python -m pip install wheel setuptools - python -m pip install awscli + micromamba-version: '1.5.6-0' + environment-name: jvm_tests + extra-specs: >- + python=3.10 + awscli + cache-downloads: true + cache-env: true - name: Cache Maven packages - uses: actions/cache@6998d139ddd3e68c71e9e398d8e40b71a2f39812 # v3.2.5 + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }} restore-keys: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }} + - name: Build xgboost4j.dll + run: | + mkdir build + cd build + cmake .. -G"Visual Studio 17 2022" -A x64 -DJVM_BINDINGS=ON + cmake --build . --config Release + if: matrix.os == 'windows-latest' + - name: Test XGBoost4J (Core) run: | cd jvm-packages diff --git a/jvm-packages/create_jni.py b/jvm-packages/create_jni.py index c39d354cf..865d07fe8 100755 --- a/jvm-packages/create_jni.py +++ b/jvm-packages/create_jni.py @@ -81,40 +81,55 @@ def native_build(args): with cd(".."): build_dir = "build-gpu" if cli_args.use_cuda == "ON" else "build" maybe_makedirs(build_dir) + + if sys.platform == "linux": + maybe_parallel_build = " -- -j $(nproc)" + else: + maybe_parallel_build = "" + + if cli_args.log_capi_invocation == "ON": + CONFIG["LOG_CAPI_INVOCATION"] = "ON" + + if cli_args.use_cuda == "ON": + CONFIG["USE_CUDA"] = "ON" + CONFIG["USE_NCCL"] = "ON" + CONFIG["USE_DLOPEN_NCCL"] = "OFF" + + args = ["-D{0}:BOOL={1}".format(k, v) for k, v in CONFIG.items()] + + # if enviorment set rabit_mock + if os.getenv("RABIT_MOCK", None) is not None: + args.append("-DRABIT_MOCK:BOOL=ON") + + # if enviorment set GPU_ARCH_FLAG + gpu_arch_flag = os.getenv("GPU_ARCH_FLAG", None) + if gpu_arch_flag is not None: + args.append("%s" % gpu_arch_flag) + with cd(build_dir): - if sys.platform == "win32": - # Force x64 build on Windows. - maybe_generator = " -A x64" - else: - maybe_generator = "" - if sys.platform == "linux": - maybe_parallel_build = " -- -j $(nproc)" - else: - maybe_parallel_build = "" - - if cli_args.log_capi_invocation == "ON": - CONFIG["LOG_CAPI_INVOCATION"] = "ON" - - if cli_args.use_cuda == "ON": - CONFIG["USE_CUDA"] = "ON" - CONFIG["USE_NCCL"] = "ON" - CONFIG["USE_DLOPEN_NCCL"] = "OFF" - - args = ["-D{0}:BOOL={1}".format(k, v) for k, v in CONFIG.items()] - - # if enviorment set rabit_mock - if os.getenv("RABIT_MOCK", None) is not None: - args.append("-DRABIT_MOCK:BOOL=ON") - - # if enviorment set GPU_ARCH_FLAG - gpu_arch_flag = os.getenv("GPU_ARCH_FLAG", None) - if gpu_arch_flag is not None: - args.append("%s" % gpu_arch_flag) - lib_dir = os.path.join(os.pardir, "lib") if os.path.exists(lib_dir): shutil.rmtree(lib_dir) - run("cmake .. " + " ".join(args) + maybe_generator) + + # Same trick as Python build, just test all possible generators. + if sys.platform == "win32": + supported_generators = ( + "", # empty, decided by cmake + '-G"Visual Studio 17 2022" -A x64', + '-G"Visual Studio 16 2019" -A x64', + '-G"Visual Studio 15 2017" -A x64', + ) + for generator in supported_generators: + try: + run("cmake .. " + " ".join(args + [generator])) + break + except subprocess.CalledProcessError as e: + print(f"Failed to build with generator: {generator}", e) + with cd(os.path.pardir): + shutil.rmtree(build_dir) + maybe_makedirs(build_dir) + else: + run("cmake .. " + " ".join(args)) run("cmake --build . --config Release" + maybe_parallel_build) with cd("demo/CLI/regression"): diff --git a/python-package/packager/nativelib.py b/python-package/packager/nativelib.py index 0227cff37..42b510eef 100644 --- a/python-package/packager/nativelib.py +++ b/python-package/packager/nativelib.py @@ -32,7 +32,10 @@ def build_libxgboost( build_dir: pathlib.Path, build_config: BuildConfiguration, ) -> pathlib.Path: - """Build libxgboost in a temporary directory and obtain the path to built libxgboost""" + """Build libxgboost in a temporary directory and obtain the path to built + libxgboost. + + """ logger = logging.getLogger("xgboost.packager.build_libxgboost") if not cpp_src_dir.is_dir(): @@ -51,8 +54,8 @@ def build_libxgboost( cmake_cmd.extend(build_config.get_cmake_args()) # Flag for cross-compiling for Apple Silicon - # We use environment variable because it's the only way to pass down custom flags - # through the cibuildwheel package, which calls `pip wheel` command. + # We use environment variable because it's the only way to pass down custom + # flags through the cibuildwheel package, which calls `pip wheel` command. if "CIBW_TARGET_OSX_ARM64" in os.environ: cmake_cmd.append("-DCMAKE_OSX_ARCHITECTURES=arm64")