[CI] Fix JVM tests on Windows (#10404)

.github/workflows/jvm_tests.yml

@@ -23,6 +23,11 @@ jobs:
       with:
         submodules: 'true'
 
+    - uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9  # v4.2.1
+      with:
+        distribution: 'temurin'
+        java-version: '8'
+
     - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca  # v3.0.4
       with:
         miniforge-variant: Mambaforge
@@ -38,18 +43,11 @@ jobs:
         key: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }}
         restore-keys: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }}
 
-    - name: Build xgboost4j.dll
-      run: |
-        mkdir build
-        cd build
-        cmake .. -G"Visual Studio 17 2022" -A x64 -DJVM_BINDINGS=ON
-        cmake --build . --config Release
-      if: matrix.os == 'windows-latest'
-
     - name: Test XGBoost4J (Core)
       run: |
         cd jvm-packages
         mvn test -B -pl :xgboost4j_2.12
+      if: matrix.os == 'windows-latest'
 
     - name: Extract branch name
       shell: bash
@@ -87,16 +85,12 @@ jobs:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_IAM_S3_UPLOADER }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_IAM_S3_UPLOADER }}
 
-
     - name: Test XGBoost4J (Core, Spark, Examples)
       run: |
         rm -rfv build/
         cd jvm-packages
         mvn -B test
       if: matrix.os == 'ubuntu-latest'  # Distributed training doesn't work on Windows
-      env:
-        RABIT_MOCK: ON
-
 
     - name: Build and Test XGBoost4J with scala 2.13
       run: |
@@ -104,5 +98,3 @@ jobs:
         cd jvm-packages
         mvn -B clean install test -Pdefault,scala-2.13
       if: matrix.os == 'ubuntu-latest'  # Distributed training doesn't work on Windows
-      env:
-        RABIT_MOCK: ON

CMakeLists.txt

@@ -265,11 +265,18 @@ if(USE_NCCL)
   find_package(Nccl REQUIRED)
 endif()
 
-# dmlc-core
+if(MSVC)
-msvc_use_static_runtime()
   if(FORCE_SHARED_CRT)
-  set(DMLC_FORCE_SHARED_CRT ON)
+    message(STATUS "XGBoost: Using dynamically linked MSVC runtime...")
+    set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>DLL")
+  else()
+    message(STATUS "XGBoost: Using statically linked MSVC runtime...")
+    set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
   endif()
+endif()
+
+# dmlc-core
+set(DMLC_FORCE_SHARED_CRT ${FORCE_SHARED_CRT})
 add_subdirectory(${xgboost_SOURCE_DIR}/dmlc-core)
 
 if(MSVC)
@@ -489,11 +496,6 @@ if(GOOGLE_TEST)
   endif()
 endif()
 
-# For MSVC: Call msvc_use_static_runtime() once again to completely
-# replace /MD with /MT. See https://github.com/dmlc/xgboost/issues/4462
-# for issues caused by mixing of /MD and /MT flags
-msvc_use_static_runtime()
-
 # Add xgboost.pc
 if(ADD_PKGCONFIG)
   configure_file(${xgboost_SOURCE_DIR}/cmake/xgboost.pc.in ${xgboost_BINARY_DIR}/xgboost.pc @ONLY)

cmake/Utils.cmake

@@ -13,45 +13,6 @@ function(auto_source_group SOURCES)
   endforeach()
 endfunction()
 
-# Force static runtime for MSVC
-function(msvc_use_static_runtime)
-  if(MSVC AND (NOT BUILD_SHARED_LIBS) AND (NOT FORCE_SHARED_CRT))
-      set(variables
-          CMAKE_C_FLAGS_DEBUG
-          CMAKE_C_FLAGS_MINSIZEREL
-          CMAKE_C_FLAGS_RELEASE
-          CMAKE_C_FLAGS_RELWITHDEBINFO
-          CMAKE_CXX_FLAGS_DEBUG
-          CMAKE_CXX_FLAGS_MINSIZEREL
-          CMAKE_CXX_FLAGS_RELEASE
-          CMAKE_CXX_FLAGS_RELWITHDEBINFO
-      )
-      foreach(variable ${variables})
-          if(${variable} MATCHES "/MD")
-              string(REGEX REPLACE "/MD" "/MT" ${variable} "${${variable}}")
-              set(${variable} "${${variable}}"  PARENT_SCOPE)
-          endif()
-      endforeach()
-      set(variables
-          CMAKE_CUDA_FLAGS
-          CMAKE_CUDA_FLAGS_DEBUG
-          CMAKE_CUDA_FLAGS_MINSIZEREL
-          CMAKE_CUDA_FLAGS_RELEASE
-          CMAKE_CUDA_FLAGS_RELWITHDEBINFO
-      )
-      foreach(variable ${variables})
-          if(${variable} MATCHES "-MD")
-              string(REGEX REPLACE "-MD" "-MT" ${variable} "${${variable}}")
-              set(${variable} "${${variable}}"  PARENT_SCOPE)
-          endif()
-          if(${variable} MATCHES "/MD")
-              string(REGEX REPLACE "/MD" "/MT" ${variable} "${${variable}}")
-              set(${variable} "${${variable}}"  PARENT_SCOPE)
-          endif()
-      endforeach()
-  endif()
-endfunction()
-
 # Set output directory of target, ignoring debug or release
 function(set_output_directory target dir)
   set_target_properties(${target} PROPERTIES

dev/prepare_jvm_release.py

@@ -216,8 +216,7 @@ def main():
     )
     print(
         "   # Skip native build, since we have all needed native binaries from CI\n"
-        "   export MAVEN_SKIP_NATIVE_BUILD=1\n"
+        "   GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests -Dskip.native.build=true"
-        "   GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests"
     )
     print(
         "4. Log into https://oss.sonatype.org/. On the left menu panel, click Staging "
@@ -228,9 +227,8 @@ def main():
     )
     print(
         "5. Remove the Scala 2.12 artifacts and build Scala 2.13 artifacts:\n"
-        "   export MAVEN_SKIP_NATIVE_BUILD=1\n"
         "   python dev/change_scala_version.py --scala-version 2.13 --purge-artifacts\n"
-        "   GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests"
+        "   GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests -Dskip.native.build=true"
     )
     print(
         "6. Go to https://oss.sonatype.org/ to release the Scala 2.13 artifacts. "

dmlc-core

@@ -1 +1 @@
-Subproject commit ea21135fbb141ae103fb5fc960289b5601b468f2
+Subproject commit 13341857549852a9a86b1894b5ba84c6276ab381

jvm-packages/create_jni.py

@@ -51,7 +51,7 @@ def maybe_makedirs(path):
 
 def run(command, **kwargs):
     print(command)
-    subprocess.check_call(command, shell=True, **kwargs)
+    subprocess.run(command, shell=True, check=True, env=os.environ, **kwargs)
 
 
 def cp(source, target):
@@ -85,6 +85,8 @@ def native_build(args):
 
         if sys.platform == "linux":
             maybe_parallel_build = " -- -j $(nproc)"
+        elif sys.platform == "win32":
+            maybe_parallel_build = ' -- /m /nodeReuse:false "/consoleloggerparameters:ShowCommandLine;Verbosity=minimal"'
         else:
             maybe_parallel_build = ""
 
@@ -176,15 +178,10 @@ def native_build(args):
 
 
 if __name__ == "__main__":
-    if "MAVEN_SKIP_NATIVE_BUILD" in os.environ:
-        print("MAVEN_SKIP_NATIVE_BUILD is set. Skipping native build...")
-    else:
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--log-capi-invocation", type=str, choices=["ON", "OFF"], default="OFF"
     )
-        parser.add_argument(
+    parser.add_argument("--use-cuda", type=str, choices=["ON", "OFF"], default="OFF")
-            "--use-cuda", type=str, choices=["ON", "OFF"], default="OFF"
-        )
     cli_args = parser.parse_args()
     native_build(cli_args)

jvm-packages/pom.xml

@@ -49,6 +49,7 @@
         <cudf.classifier>cuda12</cudf.classifier>
         <scalatest.version>3.2.18</scalatest.version>
         <scala-collection-compat.version>2.12.0</scala-collection-compat.version>
+        <skip.native.build>false</skip.native.build>
 
         <!-- SPARK-36796 for JDK-17 test-->
         <extraJavaTestArgs>

jvm-packages/xgboost4j/pom.xml

@@ -98,6 +98,7 @@
                             <argument>${log.capi.invocation}</argument>
                           </arguments>
                           <workingDirectory>${user.dir}</workingDirectory>
+                          <skip>${skip.native.build}</skip>
                       </configuration>
                   </execution>
               </executions>

src/CMakeLists.txt

@@ -30,11 +30,6 @@ if(LOG_CAPI_INVOCATION)
   target_compile_definitions(objxgboost PRIVATE -DLOG_CAPI_INVOCATION=1)
 endif()
 
-# For MSVC: Call msvc_use_static_runtime() once again to completely
-# replace /MD with /MT. See https://github.com/dmlc/xgboost/issues/4462
-# for issues caused by mixing of /MD and /MT flags
-msvc_use_static_runtime()
-
 # This grouping organises source files nicely in visual studio
 auto_source_group("${CUDA_SOURCES}")
 auto_source_group("${CPU_SOURCES}")

src/common/io.h

@@ -48,11 +48,12 @@ struct MemoryFixSizeBuffer : public dmlc::SeekStream {
     curr_ptr_ += nread;
     return nread;
   }
-  void Write(const void *ptr, std::size_t size) override {
+  std::size_t Write(const void *ptr, std::size_t size) override {
-    if (size == 0) return;
+    if (size == 0) return 0;
     CHECK_LE(curr_ptr_ + size, buffer_size_);
     std::memcpy(p_buffer_ + curr_ptr_, ptr, size);
     curr_ptr_ += size;
+    return size;
   }
   void Seek(std::size_t pos) override {
     if (pos == kSeekEnd) {
@@ -91,13 +92,14 @@ struct MemoryBufferStream : public dmlc::SeekStream {
     curr_ptr_ += nread;
     return nread;
   }
-  void Write(const void *ptr, size_t size) override {
+  std::size_t Write(const void *ptr, size_t size) override {
-    if (size == 0) return;
+    if (size == 0) return 0;
     if (curr_ptr_ + size > p_buffer_->length()) {
       p_buffer_->resize(curr_ptr_+size);
     }
     std::memcpy(&(*p_buffer_)[0] + curr_ptr_, ptr, size);
     curr_ptr_ += size;
+    return size;
   }
   void Seek(size_t pos) override {
     curr_ptr_ = static_cast<size_t>(pos);
@@ -127,8 +129,9 @@ class PeekableInStream : public dmlc::Stream {
   size_t Read(void* dptr, size_t size) override;
   virtual size_t PeekRead(void* dptr, size_t size);
 
-  void Write(const void*, size_t) override {
+  std::size_t Write(const void*, size_t) override {
     LOG(FATAL) << "Not implemented";
+    return 0;
   }
 
  private:
@@ -155,8 +158,9 @@ class FixedSizeStream : public PeekableInStream {
   [[nodiscard]] std::size_t Tell() const { return pointer_; }
   void Seek(size_t pos);
 
-  void Write(const void*, size_t) override {
+  std::size_t Write(const void*, size_t) override {
     LOG(FATAL) << "Not implemented";
+    return 0;
   }
 
   /*!

tests/buildkite/build-jvm-packages-gpu.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 
 source tests/buildkite/conftest.sh
 
-echo "--- Build XGBoost JVM packages with CUDA"
+echo "--- Build and test XGBoost JVM packages with CUDA"
 
 if [[ ($is_pull_request == 1) || ($is_release_branch == 0) ]]
 then

tests/buildkite/build-jvm-packages.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 
 source tests/buildkite/conftest.sh
 
-echo "--- Build XGBoost JVM packages scala 2.12"
+echo "--- Build and test XGBoost JVM packages with Scala 2.12"
 tests/ci_build/ci_build.sh jvm tests/ci_build/build_jvm_packages.sh \
   ${SPARK_VERSION}
 
@@ -14,7 +14,7 @@ buildkite-agent artifact upload "jvm-packages/xgboost4j-spark/target/*.jar"
 buildkite-agent artifact upload "jvm-packages/xgboost4j-flink/target/*.jar"
 buildkite-agent artifact upload "jvm-packages/xgboost4j-example/target/*.jar"
 
-echo "--- Build XGBoost JVM packages scala 2.13"
+echo "--- Build and test XGBoost JVM packages with Scala 2.13"
 
 tests/ci_build/ci_build.sh jvm tests/ci_build/build_jvm_packages.sh \
   ${SPARK_VERSION} "" "" "true"

tests/buildkite/build-win64-gpu.ps1

@@ -12,13 +12,11 @@ if ( $is_release_branch -eq 0 ) {
 }
 mkdir build
 cd build
-cmake .. -G"Visual Studio 17 2022" -A x64 -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON `
+cmake .. -G"Visual Studio 17 2022" -A x64 -DUSE_CUDA=ON `
   -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DBUILD_DEPRECATED_CLI=ON ${arch_flag}
-$msbuild = -join @(
+if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
-  "C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\MSBuild\\Current"
+cmake --build . --config Release -- /m /nodeReuse:false `
-  "\\Bin\\MSBuild.exe"
+  "/consoleloggerparameters:ShowCommandLine;Verbosity=minimal"
-)
-& $msbuild xgboost.sln /m /p:Configuration=Release /nodeReuse:false
 if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
 
 Write-Host "--- Build binary wheel"

tests/buildkite/infrastructure/aws-stack-creator/metadata.py

@@ -1,27 +1,27 @@
 AMI_ID = {
     # Managed by XGBoost team
     "linux-amd64-gpu": {
-        "us-west-2": "ami-070080d04e81c5e39",
+        "us-west-2": "ami-0b4079c15bbbd0faf",
     },
     "linux-amd64-mgpu": {
-        "us-west-2": "ami-070080d04e81c5e39",
+        "us-west-2": "ami-0b4079c15bbbd0faf",
     },
     "windows-gpu": {
-        "us-west-2": "ami-07c14abcf529d816a",
+        "us-west-2": "ami-0123456bcf4cdfb82",
     },
     "windows-cpu": {
-        "us-west-2": "ami-07c14abcf529d816a",
+        "us-west-2": "ami-0123456bcf4cdfb82",
     },
     # Managed by BuildKite
     # from https://s3.amazonaws.com/buildkite-aws-stack/latest/aws-stack.yml
     "linux-amd64-cpu": {
-        "us-west-2": "ami-0180f7fb0f07eb0bc",
+        "us-west-2": "ami-0083e0ae73c175ec6",
     },
     "pipeline-loader": {
-        "us-west-2": "ami-0180f7fb0f07eb0bc",
+        "us-west-2": "ami-0083e0ae73c175ec6",
     },
     "linux-arm64-cpu": {
-        "us-west-2": "ami-00686bdc2043a5505",
+        "us-west-2": "ami-0dbf1f9da54222f21",
     },
 }
 

tests/buildkite/infrastructure/worker-image-pipeline/windows-gpu-bootstrap.yml

@@ -36,7 +36,12 @@ phases:
 
               # Install Java 11
               Write-Host '>>> Installing Java 11...'
-              choco install openjdk11jre
+              choco install openjdk11
+              if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
+
+              # Install Maven
+              Write-Host '>>> Installing Maven...'
+              choco install maven
               if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
 
               # Install GraphViz

tests/buildkite/pipeline-mgpu.yml

@@ -29,7 +29,7 @@ steps:
     key: build-cuda
     agents:
       queue: linux-amd64-cpu
-  - label: ":console: Build JVM packages with CUDA"
+  - label: ":console: Build and test JVM packages with CUDA"
     command: "tests/buildkite/build-jvm-packages-gpu.sh"
     key: build-jvm-packages-gpu
     agents: