import argparse import errno import glob import os import platform import re import shutil import subprocess import sys import tempfile import zipfile from contextlib import contextmanager from urllib.request import urlretrieve def normpath(path): """Normalize UNIX path to a native path.""" normalized = os.path.join(*path.split("/")) if os.path.isabs(path): return os.path.abspath("/") + normalized else: return normalized def cp(source, target): source = normpath(source) target = normpath(target) print("cp {0} {1}".format(source, target)) shutil.copy(source, target) def maybe_makedirs(path): path = normpath(path) print("mkdir -p " + path) try: os.makedirs(path) except OSError as e: if e.errno != errno.EEXIST: raise @contextmanager def cd(path): path = normpath(path) cwd = os.getcwd() os.chdir(path) print("cd " + path) try: yield path finally: os.chdir(cwd) def run(command, **kwargs): print(command) subprocess.check_call(command, shell=True, **kwargs) def get_current_git_tag(): out = subprocess.check_output(["git", "tag", "--points-at", "HEAD"]) return out.decode().split("\n")[0] def get_current_commit_hash(): out = subprocess.check_output(["git", "rev-parse", "HEAD"]) return out.decode().split("\n")[0] def get_current_git_branch(): out = subprocess.check_output(["git", "log", "-n", "1", "--pretty=%d", "HEAD"]) m = re.search(r"release_[0-9\.]+", out.decode()) if not m: raise ValueError("Expected branch name of form release_xxx") return m.group(0) def retrieve(url, filename=None): print(f"{url} -> {filename}") return urlretrieve(url, filename) def main(): parser = argparse.ArgumentParser() parser.add_argument( "--release-version", type=str, required=True, help="Version of the release being prepared", ) args = parser.parse_args() if sys.platform != "darwin" or platform.machine() != "arm64": raise NotImplementedError("Please run this script using an M1 Mac") version = args.release_version expected_git_tag = "v" + version current_git_tag = get_current_git_tag() if current_git_tag != expected_git_tag: if not current_git_tag: raise ValueError( f"Expected git tag {expected_git_tag} but current HEAD has no tag. " f"Run: git checkout {expected_git_tag}" ) raise ValueError( f"Expected git tag {expected_git_tag} but current HEAD is at tag " f"{current_git_tag}. Run: git checkout {expected_git_tag}" ) commit_hash = get_current_commit_hash() git_branch = get_current_git_branch() print( f"Using commit {commit_hash} of branch {git_branch}, git tag {current_git_tag}" ) with cd("jvm-packages/"): print("====copying pure-Python tracker====") for use_cuda in [True, False]: xgboost4j = "xgboost4j-gpu" if use_cuda else "xgboost4j" cp( "../python-package/xgboost/tracker.py", f"{xgboost4j}/src/main/resources", ) print("====copying resources for testing====") with cd("../demo/CLI/regression"): run(f"{sys.executable} mapfeat.py") run(f"{sys.executable} mknfold.py machine.txt 1") for use_cuda in [True, False]: xgboost4j = "xgboost4j-gpu" if use_cuda else "xgboost4j" xgboost4j_spark = "xgboost4j-spark-gpu" if use_cuda else "xgboost4j-spark" maybe_makedirs(f"{xgboost4j}/src/test/resources") maybe_makedirs(f"{xgboost4j_spark}/src/test/resources") for file in glob.glob("../demo/data/agaricus.*"): cp(file, f"{xgboost4j}/src/test/resources") cp(file, f"{xgboost4j_spark}/src/test/resources") for file in glob.glob("../demo/CLI/regression/machine.txt.t*"): cp(file, f"{xgboost4j_spark}/src/test/resources") print("====Creating directories to hold native binaries====") for os_ident, arch in [ ("linux", "x86_64"), ("windows", "x86_64"), ("macos", "x86_64"), ]: output_dir = f"xgboost4j/src/main/resources/lib/{os_ident}/{arch}" maybe_makedirs(output_dir) for os_ident, arch in [("linux", "x86_64")]: output_dir = f"xgboost4j-gpu/src/main/resources/lib/{os_ident}/{arch}" maybe_makedirs(output_dir) print("====Downloading native binaries from CI====") nightly_bucket_prefix = ( "https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds" ) maven_repo_prefix = ( "https://s3-us-west-2.amazonaws.com/xgboost-maven-repo/release/ml/dmlc" ) retrieve( url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/xgboost4j_{commit_hash}.dll", filename="xgboost4j/src/main/resources/lib/windows/x86_64/xgboost4j.dll", ) retrieve( url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/libxgboost4j_{commit_hash}.dylib", filename="xgboost4j/src/main/resources/lib/macos/x86_64/libxgboost4j.dylib", ) with tempfile.TemporaryDirectory() as tempdir: # libxgboost4j.so for Linux x86_64, CPU only zip_path = os.path.join(tempdir, "xgboost4j_2.12.jar") extract_dir = os.path.join(tempdir, "xgboost4j") retrieve( url=f"{maven_repo_prefix}/xgboost4j_2.12/{version}/" f"xgboost4j_2.12-{version}.jar", filename=zip_path, ) os.mkdir(extract_dir) with zipfile.ZipFile(zip_path, "r") as t: t.extractall(extract_dir) cp( os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"), "xgboost4j/src/main/resources/lib/linux/x86_64/libxgboost4j.so", ) # libxgboost4j.so for Linux x86_64, GPU support zip_path = os.path.join(tempdir, "xgboost4j-gpu_2.12.jar") extract_dir = os.path.join(tempdir, "xgboost4j-gpu") retrieve( url=f"{maven_repo_prefix}/xgboost4j-gpu_2.12/{version}/" f"xgboost4j-gpu_2.12-{version}.jar", filename=zip_path, ) os.mkdir(extract_dir) with zipfile.ZipFile(zip_path, "r") as t: t.extractall(extract_dir) cp( os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"), "xgboost4j-gpu/src/main/resources/lib/linux/x86_64/libxgboost4j.so", ) print("====Next Steps====") print("1. Gain upload right to Maven Central repo.") print("1-1. Sign up for a JIRA account at Sonatype: ") print( "1-2. File a JIRA ticket: " "https://issues.sonatype.org/secure/CreateIssue.jspa?issuetype=21&pid=10134. Example: " "https://issues.sonatype.org/browse/OSSRH-67724" ) print( "2. Store the Sonatype credentials in .m2/settings.xml. See insturctions in " "https://central.sonatype.org/publish/publish-maven/" ) print( "3. Now on a M1 Mac machine, run the following to build Scala 2.12 artifacts:" ) print(" GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests") print( "4. Log into https://oss.sonatype.org/. On the left menu panel, click Staging " "Repositories. Visit the URL https://oss.sonatype.org/content/repositories/mldmlc-xxxx " "to inspect the staged JAR files. Finally, press Release button to publish the " "artifacts to the Maven Central repository. The top-level metapackage should be " "named xgboost-jvm_2.12." ) print("5. Remove the Scala 2.12 artifacts and build Scala 2.13 artifacts:") print(" rm -rf targets/") print(" GPG_TTY=$(tty) mvn deploy -Prelease-cpu-only,scala-2.13 -DskipTests") print( "6. Go to https://oss.sonatype.org/ to release the Scala 2.13 artifacts." "The top-level metapackage should be named xgboost-jvm_2.13." ) if __name__ == "__main__": main()