Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[jvm-packages] Fix POM for xgboost-jvm metapackage (#9893)

Browse Source 
* [jvm-packages] Fix POM for xgboost-jvm metapackage

* Add script for updating the Scala version
This commit is contained in:
Philip Hyunsu Cho 2023-12-14 16:50:34 -08:00 committed by GitHub
parent ad524f76ab
commit 3acbd8692b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 112 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
dev/change_scala_version.py Normal file
View File

@ -0,0 +1,45 @@
import argparse
import pathlib
import re
import shutil
try:
import sh
except ImportError as e:
raise ImportError(
"Please install sh in your Python environment.\n"
" - Pip: pip install sh\n"
" - Conda: conda install -c conda-forge sh"
) from e
def main(args):
# Clean artifacts
for target in pathlib.Path("jvm-packages/").glob("**/target"):
if target.is_dir():
print(f"Removing {target}...")
shutil.rmtree(target)
# Update pom.xml
for pom in pathlib.Path("jvm-packages/").glob("**/pom.xml"):
print(f"Updating {pom}...")
sh.sed(
[
"-i",
f"s/<artifactId>xgboost-jvm_[0-9\\.]*/<artifactId>xgboost-jvm_{args.scala_version}/g",
str(pom),
]
)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
"--scala-version",
type=str,
required=True,
help="Version of Scala to use in the JVM packages",
choices=["2.12", "2.13"],
)
parsed_args = parser.parse_args()
main(parsed_args)

28
dev/prepare_jvm_release.py
View File

@ -2,7 +2,6 @@ import argparse
import errno
import glob
import os
import platform
import re
import shutil
import subprocess
@ -88,10 +87,6 @@ def main():
help="Version of the release being prepared",
)
args = parser.parse_args()
if sys.platform != "darwin" or platform.machine() != "arm64":
raise NotImplementedError("Please run this script using an M1 Mac")
version = args.release_version
expected_git_tag = "v" + version
current_git_tag = get_current_git_tag()
@ -141,6 +136,7 @@ def main():
("linux", "x86_64"),
("windows", "x86_64"),
("macos", "x86_64"),
("macos", "aarch64"),
]:
output_dir = f"xgboost4j/src/main/resources/lib/{os_ident}/{arch}"
maybe_makedirs(output_dir)
@ -164,6 +160,10 @@ def main():
url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/libxgboost4j_{commit_hash}.dylib",
filename="xgboost4j/src/main/resources/lib/macos/x86_64/libxgboost4j.dylib",
)
retrieve(
url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/libxgboost4j_m1_{commit_hash}.dylib",
filename="xgboost4j/src/main/resources/lib/macos/aarch64/libxgboost4j.dylib",
)
with tempfile.TemporaryDirectory() as tempdir:
# libxgboost4j.so for Linux x86_64, CPU only
@ -211,9 +211,14 @@ def main():
"https://central.sonatype.org/publish/publish-maven/"
)
print(
"3. Now on a M1 Mac machine, run the following to build Scala 2.12 artifacts:"
"3. Now on a Linux machine, run the following to build Scala 2.12 artifacts. "
"Make sure to use an Internet connection with fast upload speed:"
)
print(
" # Skip native build, since we have all needed native binaries from CI\n"
" export MAVEN_SKIP_NATIVE_BUILD=1\n"
" GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests"
)
print(" GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests")
print(
"4. Log into https://oss.sonatype.org/. On the left menu panel, click Staging "
"Repositories. Visit the URL https://oss.sonatype.org/content/repositories/mldmlc-xxxx "
@ -221,9 +226,12 @@ def main():
"artifacts to the Maven Central repository. The top-level metapackage should be "
"named xgboost-jvm_2.12."
)
print("5. Remove the Scala 2.12 artifacts and build Scala 2.13 artifacts:")
print(" rm -rf targets/")
print(" GPG_TTY=$(tty) mvn deploy -Prelease-cpu-only,scala-2.13 -DskipTests")
print(
"5. Remove the Scala 2.12 artifacts and build Scala 2.13 artifacts:\n"
" export MAVEN_SKIP_NATIVE_BUILD=1\n"
" python dev/change_scala_version.py --scala-version 2.13\n"
" GPG_TTY=$(tty) mvn deploy -Prelease-cpu-only,scala-2.13 -DskipTests"
)
print(
"6. Go to https://oss.sonatype.org/ to release the Scala 2.13 artifacts. "
"The top-level metapackage should be named xgboost-jvm_2.13."

65
jvm-packages/create_jni.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python
import errno
import argparse
import errno
import glob
import os
import platform
@ -19,11 +19,10 @@ CONFIG = {
"USE_HDFS": "OFF",
"USE_AZURE": "OFF",
"USE_S3": "OFF",
"USE_CUDA": "OFF",
"USE_NCCL": "OFF",
"JVM_BINDINGS": "ON",
"LOG_CAPI_INVOCATION": "OFF"
"LOG_CAPI_INVOCATION": "OFF",
}
@ -70,26 +69,22 @@ def normpath(path):
return normalized
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--log-capi-invocation', type=str, choices=['ON', 'OFF'], default='OFF')
parser.add_argument('--use-cuda', type=str, choices=['ON', 'OFF'], default='OFF')
cli_args = parser.parse_args()
def native_build(args):
if sys.platform == "darwin":
# Enable of your compiler supports OpenMP.
CONFIG["USE_OPENMP"] = "OFF"
os.environ["JAVA_HOME"] = subprocess.check_output(
"/usr/libexec/java_home").strip().decode()
os.environ["JAVA_HOME"] = (
subprocess.check_output("/usr/libexec/java_home").strip().decode()
)
print("building Java wrapper")
with cd(".."):
build_dir = 'build-gpu' if cli_args.use_cuda == 'ON' else 'build'
build_dir = "build-gpu" if cli_args.use_cuda == "ON" else "build"
maybe_makedirs(build_dir)
with cd(build_dir):
if sys.platform == "win32":
# Force x64 build on Windows.
maybe_generator = ' -A x64'
maybe_generator = " -A x64"
else:
maybe_generator = ""
if sys.platform == "linux":
@ -97,12 +92,12 @@ if __name__ == "__main__":
else:
maybe_parallel_build = ""
if cli_args.log_capi_invocation == 'ON':
CONFIG['LOG_CAPI_INVOCATION'] = 'ON'
if cli_args.log_capi_invocation == "ON":
CONFIG["LOG_CAPI_INVOCATION"] = "ON"
if cli_args.use_cuda == 'ON':
CONFIG['USE_CUDA'] = 'ON'
CONFIG['USE_NCCL'] = 'ON'
if cli_args.use_cuda == "ON":
CONFIG["USE_CUDA"] = "ON"
CONFIG["USE_NCCL"] = "ON"
args = ["-D{0}:BOOL={1}".format(k, v) for k, v in CONFIG.items()]
@ -115,7 +110,7 @@ if __name__ == "__main__":
if gpu_arch_flag is not None:
args.append("%s" % gpu_arch_flag)
lib_dir = os.path.join(os.pardir, 'lib')
lib_dir = os.path.join(os.pardir, "lib")
if os.path.exists(lib_dir):
shutil.rmtree(lib_dir)
run("cmake .. " + " ".join(args) + maybe_generator)
@ -125,8 +120,10 @@ if __name__ == "__main__":
run(f'"{sys.executable}" mapfeat.py')
run(f'"{sys.executable}" mknfold.py machine.txt 1')
xgboost4j = 'xgboost4j-gpu' if cli_args.use_cuda == 'ON' else 'xgboost4j'
xgboost4j_spark = 'xgboost4j-spark-gpu' if cli_args.use_cuda == 'ON' else 'xgboost4j-spark'
xgboost4j = "xgboost4j-gpu" if cli_args.use_cuda == "ON" else "xgboost4j"
xgboost4j_spark = (
"xgboost4j-spark-gpu" if cli_args.use_cuda == "ON" else "xgboost4j-spark"
)
print("copying native library")
library_name, os_folder = {
@ -141,14 +138,19 @@ if __name__ == "__main__":
"i86pc": "x86_64", # on Solaris x86_64
"sun4v": "sparc", # on Solaris sparc
"arm64": "aarch64", # on macOS & Windows ARM 64-bit
"aarch64": "aarch64"
"aarch64": "aarch64",
}[platform.machine().lower()]
output_folder = "{}/src/main/resources/lib/{}/{}".format(xgboost4j, os_folder, arch_folder)
output_folder = "{}/src/main/resources/lib/{}/{}".format(
xgboost4j, os_folder, arch_folder
)
maybe_makedirs(output_folder)
cp("../lib/" + library_name, output_folder)
print("copying pure-Python tracker")
cp("../python-package/xgboost/tracker.py", "{}/src/main/resources".format(xgboost4j))
cp(
"../python-package/xgboost/tracker.py",
"{}/src/main/resources".format(xgboost4j),
)
print("copying train/test files")
maybe_makedirs("{}/src/test/resources".format(xgboost4j_spark))
@ -164,3 +166,18 @@ if __name__ == "__main__":
maybe_makedirs("{}/src/test/resources".format(xgboost4j))
for file in glob.glob("../demo/data/agaricus.*"):
cp(file, "{}/src/test/resources".format(xgboost4j))
if __name__ == "__main__":
if "MAVEN_SKIP_NATIVE_BUILD" in os.environ:
print("MAVEN_SKIP_NATIVE_BUILD is set. Skipping native build...")
else:
parser = argparse.ArgumentParser()
parser.add_argument(
"--log-capi-invocation", type=str, choices=["ON", "OFF"], default="OFF"
)
parser.add_argument(
"--use-cuda", type=str, choices=["ON", "OFF"], default="OFF"
)
cli_args = parser.parse_args()
native_build(cli_args)

2
jvm-packages/pom.xml
View File

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost-jvm_${scala.binary.version}</artifactId>
<artifactId>xgboost-jvm_2.12</artifactId>
<version>2.0.2</version>
<packaging>pom</packaging>
<name>XGBoost JVM Package</name>

2
jvm-packages/xgboost4j-example/pom.xml
View File

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost-jvm_${scala.binary.version}</artifactId>
<artifactId>xgboost-jvm_2.12</artifactId>
<version>2.0.2</version>
</parent>
<name>xgboost4j-example</name>

2
jvm-packages/xgboost4j-flink/pom.xml
View File

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost-jvm_${scala.binary.version}</artifactId>
<artifactId>xgboost-jvm_2.12</artifactId>
<version>2.0.2</version>
</parent>

2
jvm-packages/xgboost4j-gpu/pom.xml
View File

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost-jvm_${scala.binary.version}</artifactId>
<artifactId>xgboost-jvm_2.12</artifactId>
<version>2.0.2</version>
</parent>
<artifactId>xgboost4j-gpu_${scala.binary.version}</artifactId>

2
jvm-packages/xgboost4j-spark-gpu/pom.xml
View File

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost-jvm_${scala.binary.version}</artifactId>
<artifactId>xgboost-jvm_2.12</artifactId>
<version>2.0.2</version>
</parent>
<name>xgboost4j-spark-gpu</name>

2
jvm-packages/xgboost4j-spark/pom.xml
View File

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost-jvm_${scala.binary.version}</artifactId>
<artifactId>xgboost-jvm_2.12</artifactId>
<version>2.0.2</version>
</parent>
<name>xgboost4j-spark</name>

2
jvm-packages/xgboost4j/pom.xml
View File

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost-jvm_${scala.binary.version}</artifactId>
<artifactId>xgboost-jvm_2.12</artifactId>
<version>2.0.2</version>
</parent>
<name>xgboost4j</name>