Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CI] Fix JVM tests on Windows (#10404)

Browse Source
This commit is contained in:
Philip Hyunsu Cho 2024-06-15 00:21:40 -07:00 committed by GitHub
parent dc14f98f40
commit 1ace9c66ec
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 63 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.github/workflows/jvm_tests.yml vendored
View File

@ -23,6 +23,11 @@ jobs:
with:
submodules: 'true'
- uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1
with:
distribution: 'temurin'
java-version: '8'
- uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4
with:
miniforge-variant: Mambaforge
@ -38,18 +43,11 @@ jobs:
key: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }}
restore-keys: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }}
- name: Build xgboost4j.dll
run: |
mkdir build
cd build
cmake .. -G"Visual Studio 17 2022" -A x64 -DJVM_BINDINGS=ON
cmake --build . --config Release
if: matrix.os == 'windows-latest'
- name: Test XGBoost4J (Core)
run: |
cd jvm-packages
mvn test -B -pl :xgboost4j_2.12
if: matrix.os == 'windows-latest'
- name: Extract branch name
shell: bash
@ -87,16 +85,12 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_IAM_S3_UPLOADER }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_IAM_S3_UPLOADER }}
- name: Test XGBoost4J (Core, Spark, Examples)
run: |
rm -rfv build/
cd jvm-packages
mvn -B test
if: matrix.os == 'ubuntu-latest' # Distributed training doesn't work on Windows
env:
RABIT_MOCK: ON
- name: Build and Test XGBoost4J with scala 2.13
run: |
@ -104,5 +98,3 @@ jobs:
cd jvm-packages
mvn -B clean install test -Pdefault,scala-2.13
if: matrix.os == 'ubuntu-latest' # Distributed training doesn't work on Windows
env:
RABIT_MOCK: ON

18
CMakeLists.txt
View File

@ -265,11 +265,18 @@ if(USE_NCCL)
find_package(Nccl REQUIRED)
endif()
# dmlc-core
msvc_use_static_runtime()
if(MSVC)
if(FORCE_SHARED_CRT)
set(DMLC_FORCE_SHARED_CRT ON)
message(STATUS "XGBoost: Using dynamically linked MSVC runtime...")
set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>DLL")
else()
message(STATUS "XGBoost: Using statically linked MSVC runtime...")
set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
endif()
endif()
# dmlc-core
set(DMLC_FORCE_SHARED_CRT ${FORCE_SHARED_CRT})
add_subdirectory(${xgboost_SOURCE_DIR}/dmlc-core)
if(MSVC)
@ -489,11 +496,6 @@ if(GOOGLE_TEST)
endif()
endif()
# For MSVC: Call msvc_use_static_runtime() once again to completely
# replace /MD with /MT. See https://github.com/dmlc/xgboost/issues/4462
# for issues caused by mixing of /MD and /MT flags
msvc_use_static_runtime()
# Add xgboost.pc
if(ADD_PKGCONFIG)
configure_file(${xgboost_SOURCE_DIR}/cmake/xgboost.pc.in ${xgboost_BINARY_DIR}/xgboost.pc @ONLY)

39
cmake/Utils.cmake
View File

@ -13,45 +13,6 @@ function(auto_source_group SOURCES)
endforeach()
endfunction()
# Force static runtime for MSVC
function(msvc_use_static_runtime)
if(MSVC AND (NOT BUILD_SHARED_LIBS) AND (NOT FORCE_SHARED_CRT))
set(variables
CMAKE_C_FLAGS_DEBUG
CMAKE_C_FLAGS_MINSIZEREL
CMAKE_C_FLAGS_RELEASE
CMAKE_C_FLAGS_RELWITHDEBINFO
CMAKE_CXX_FLAGS_DEBUG
CMAKE_CXX_FLAGS_MINSIZEREL
CMAKE_CXX_FLAGS_RELEASE
CMAKE_CXX_FLAGS_RELWITHDEBINFO
)
foreach(variable ${variables})
if(${variable} MATCHES "/MD")
string(REGEX REPLACE "/MD" "/MT" ${variable} "${${variable}}")
set(${variable} "${${variable}}" PARENT_SCOPE)
endif()
endforeach()
set(variables
CMAKE_CUDA_FLAGS
CMAKE_CUDA_FLAGS_DEBUG
CMAKE_CUDA_FLAGS_MINSIZEREL
CMAKE_CUDA_FLAGS_RELEASE
CMAKE_CUDA_FLAGS_RELWITHDEBINFO
)
foreach(variable ${variables})
if(${variable} MATCHES "-MD")
string(REGEX REPLACE "-MD" "-MT" ${variable} "${${variable}}")
set(${variable} "${${variable}}" PARENT_SCOPE)
endif()
if(${variable} MATCHES "/MD")
string(REGEX REPLACE "/MD" "/MT" ${variable} "${${variable}}")
set(${variable} "${${variable}}" PARENT_SCOPE)
endif()
endforeach()
endif()
endfunction()
# Set output directory of target, ignoring debug or release
function(set_output_directory target dir)
set_target_properties(${target} PROPERTIES

6
dev/prepare_jvm_release.py
View File

@ -216,8 +216,7 @@ def main():
)
print(
" # Skip native build, since we have all needed native binaries from CI\n"
" export MAVEN_SKIP_NATIVE_BUILD=1\n"
" GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests"
" GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests -Dskip.native.build=true"
)
print(
"4. Log into https://oss.sonatype.org/. On the left menu panel, click Staging "
@ -228,9 +227,8 @@ def main():
)
print(
"5. Remove the Scala 2.12 artifacts and build Scala 2.13 artifacts:\n"
" export MAVEN_SKIP_NATIVE_BUILD=1\n"
" python dev/change_scala_version.py --scala-version 2.13 --purge-artifacts\n"
" GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests"
" GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests -Dskip.native.build=true"
)
print(
"6. Go to https://oss.sonatype.org/ to release the Scala 2.13 artifacts. "

2
dmlc-core

@ -1 +1 @@
Subproject commit ea21135fbb141ae103fb5fc960289b5601b468f2
Subproject commit 13341857549852a9a86b1894b5ba84c6276ab381

11
jvm-packages/create_jni.py
View File

@ -51,7 +51,7 @@ def maybe_makedirs(path):
def run(command, **kwargs):
print(command)
subprocess.check_call(command, shell=True, **kwargs)
subprocess.run(command, shell=True, check=True, env=os.environ, **kwargs)
def cp(source, target):
@ -85,6 +85,8 @@ def native_build(args):
if sys.platform == "linux":
maybe_parallel_build = " -- -j $(nproc)"
elif sys.platform == "win32":
maybe_parallel_build = ' -- /m /nodeReuse:false "/consoleloggerparameters:ShowCommandLine;Verbosity=minimal"'
else:
maybe_parallel_build = ""
@ -176,15 +178,10 @@ def native_build(args):
if __name__ == "__main__":
if "MAVEN_SKIP_NATIVE_BUILD" in os.environ:
print("MAVEN_SKIP_NATIVE_BUILD is set. Skipping native build...")
else:
parser = argparse.ArgumentParser()
parser.add_argument(
"--log-capi-invocation", type=str, choices=["ON", "OFF"], default="OFF"
)
parser.add_argument(
"--use-cuda", type=str, choices=["ON", "OFF"], default="OFF"
)
parser.add_argument("--use-cuda", type=str, choices=["ON", "OFF"], default="OFF")
cli_args = parser.parse_args()
native_build(cli_args)

1
jvm-packages/pom.xml
View File

@ -49,6 +49,7 @@
<cudf.classifier>cuda12</cudf.classifier>
<scalatest.version>3.2.18</scalatest.version>
<scala-collection-compat.version>2.12.0</scala-collection-compat.version>
<skip.native.build>false</skip.native.build>
<!-- SPARK-36796 for JDK-17 test-->
<extraJavaTestArgs>

1
jvm-packages/xgboost4j/pom.xml
View File

@ -98,6 +98,7 @@
<argument>${log.capi.invocation}</argument>
</arguments>
<workingDirectory>${user.dir}</workingDirectory>
<skip>${skip.native.build}</skip>
</configuration>
</execution>
</executions>

5
src/CMakeLists.txt
View File

@ -30,11 +30,6 @@ if(LOG_CAPI_INVOCATION)
target_compile_definitions(objxgboost PRIVATE -DLOG_CAPI_INVOCATION=1)
endif()
# For MSVC: Call msvc_use_static_runtime() once again to completely
# replace /MD with /MT. See https://github.com/dmlc/xgboost/issues/4462
# for issues caused by mixing of /MD and /MT flags
msvc_use_static_runtime()
# This grouping organises source files nicely in visual studio
auto_source_group("${CUDA_SOURCES}")
auto_source_group("${CPU_SOURCES}")

16
src/common/io.h
View File

@ -48,11 +48,12 @@ struct MemoryFixSizeBuffer : public dmlc::SeekStream {
curr_ptr_ += nread;
return nread;
}
void Write(const void *ptr, std::size_t size) override {
if (size == 0) return;
std::size_t Write(const void *ptr, std::size_t size) override {
if (size == 0) return 0;
CHECK_LE(curr_ptr_ + size, buffer_size_);
std::memcpy(p_buffer_ + curr_ptr_, ptr, size);
curr_ptr_ += size;
return size;
}
void Seek(std::size_t pos) override {
if (pos == kSeekEnd) {
@ -91,13 +92,14 @@ struct MemoryBufferStream : public dmlc::SeekStream {
curr_ptr_ += nread;
return nread;
}
void Write(const void *ptr, size_t size) override {
if (size == 0) return;
std::size_t Write(const void *ptr, size_t size) override {
if (size == 0) return 0;
if (curr_ptr_ + size > p_buffer_->length()) {
p_buffer_->resize(curr_ptr_+size);
}
std::memcpy(&(*p_buffer_)[0] + curr_ptr_, ptr, size);
curr_ptr_ += size;
return size;
}
void Seek(size_t pos) override {
curr_ptr_ = static_cast<size_t>(pos);
@ -127,8 +129,9 @@ class PeekableInStream : public dmlc::Stream {
size_t Read(void* dptr, size_t size) override;
virtual size_t PeekRead(void* dptr, size_t size);
void Write(const void*, size_t) override {
std::size_t Write(const void*, size_t) override {
LOG(FATAL) << "Not implemented";
return 0;
}
private:
@ -155,8 +158,9 @@ class FixedSizeStream : public PeekableInStream {
[[nodiscard]] std::size_t Tell() const { return pointer_; }
void Seek(size_t pos);
void Write(const void*, size_t) override {
std::size_t Write(const void*, size_t) override {
LOG(FATAL) << "Not implemented";
return 0;
}
/*!

2
tests/buildkite/build-jvm-packages-gpu.sh
View File

@ -4,7 +4,7 @@ set -euo pipefail
source tests/buildkite/conftest.sh
echo "--- Build XGBoost JVM packages with CUDA"
echo "--- Build and test XGBoost JVM packages with CUDA"
if [[ ($is_pull_request == 1) || ($is_release_branch == 0) ]]
then

4
tests/buildkite/build-jvm-packages.sh
View File

@ -4,7 +4,7 @@ set -euo pipefail
source tests/buildkite/conftest.sh
echo "--- Build XGBoost JVM packages scala 2.12"
echo "--- Build and test XGBoost JVM packages with Scala 2.12"
tests/ci_build/ci_build.sh jvm tests/ci_build/build_jvm_packages.sh \
${SPARK_VERSION}
@ -14,7 +14,7 @@ buildkite-agent artifact upload "jvm-packages/xgboost4j-spark/target/*.jar"
buildkite-agent artifact upload "jvm-packages/xgboost4j-flink/target/*.jar"
buildkite-agent artifact upload "jvm-packages/xgboost4j-example/target/*.jar"
echo "--- Build XGBoost JVM packages scala 2.13"
echo "--- Build and test XGBoost JVM packages with Scala 2.13"
tests/ci_build/ci_build.sh jvm tests/ci_build/build_jvm_packages.sh \
${SPARK_VERSION} "" "" "true"

10
tests/buildkite/build-win64-gpu.ps1
View File

@ -12,13 +12,11 @@ if ( $is_release_branch -eq 0 ) {
}
mkdir build
cd build
cmake .. -G"Visual Studio 17 2022" -A x64 -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON `
cmake .. -G"Visual Studio 17 2022" -A x64 -DUSE_CUDA=ON `
-DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DBUILD_DEPRECATED_CLI=ON ${arch_flag}
$msbuild = -join @(
"C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\MSBuild\\Current"
"\\Bin\\MSBuild.exe"
)
& $msbuild xgboost.sln /m /p:Configuration=Release /nodeReuse:false
if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
cmake --build . --config Release -- /m /nodeReuse:false `
"/consoleloggerparameters:ShowCommandLine;Verbosity=minimal"
if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
Write-Host "--- Build binary wheel"

14
tests/buildkite/infrastructure/aws-stack-creator/metadata.py
View File

@ -1,27 +1,27 @@
AMI_ID = {
# Managed by XGBoost team
"linux-amd64-gpu": {
"us-west-2": "ami-070080d04e81c5e39",
"us-west-2": "ami-0b4079c15bbbd0faf",
},
"linux-amd64-mgpu": {
"us-west-2": "ami-070080d04e81c5e39",
"us-west-2": "ami-0b4079c15bbbd0faf",
},
"windows-gpu": {
"us-west-2": "ami-07c14abcf529d816a",
"us-west-2": "ami-0123456bcf4cdfb82",
},
"windows-cpu": {
"us-west-2": "ami-07c14abcf529d816a",
"us-west-2": "ami-0123456bcf4cdfb82",
},
# Managed by BuildKite
# from https://s3.amazonaws.com/buildkite-aws-stack/latest/aws-stack.yml
"linux-amd64-cpu": {
"us-west-2": "ami-0180f7fb0f07eb0bc",
"us-west-2": "ami-0083e0ae73c175ec6",
},
"pipeline-loader": {
"us-west-2": "ami-0180f7fb0f07eb0bc",
"us-west-2": "ami-0083e0ae73c175ec6",
},
"linux-arm64-cpu": {
"us-west-2": "ami-00686bdc2043a5505",
"us-west-2": "ami-0dbf1f9da54222f21",
},
}

7
tests/buildkite/infrastructure/worker-image-pipeline/windows-gpu-bootstrap.yml
View File

@ -36,7 +36,12 @@ phases:
# Install Java 11
Write-Host '>>> Installing Java 11...'
choco install openjdk11jre
choco install openjdk11
if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
# Install Maven
Write-Host '>>> Installing Maven...'
choco install maven
if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
# Install GraphViz

2
tests/buildkite/pipeline-mgpu.yml
View File

@ -29,7 +29,7 @@ steps:
key: build-cuda
agents:
queue: linux-amd64-cpu
- label: ":console: Build JVM packages with CUDA"
- label: ":console: Build and test JVM packages with CUDA"
command: "tests/buildkite/build-jvm-packages-gpu.sh"
key: build-jvm-packages-gpu
agents: