diff --git a/dev/prepare_jvm_release.py b/dev/prepare_jvm_release.py index 07769b411..0cf5796a2 100644 --- a/dev/prepare_jvm_release.py +++ b/dev/prepare_jvm_release.py @@ -21,12 +21,14 @@ def normpath(path): else: return normalized + def cp(source, target): source = normpath(source) target = normpath(target) print("cp {0} {1}".format(source, target)) shutil.copy(source, target) + def maybe_makedirs(path): path = normpath(path) print("mkdir -p " + path) @@ -36,6 +38,7 @@ def maybe_makedirs(path): if e.errno != errno.EEXIST: raise + @contextmanager def cd(path): path = normpath(path) @@ -47,18 +50,22 @@ def cd(path): finally: os.chdir(cwd) + def run(command, **kwargs): print(command) subprocess.check_call(command, shell=True, **kwargs) + def get_current_git_tag(): out = subprocess.check_output(["git", "tag", "--points-at", "HEAD"]) return out.decode().split("\n")[0] + def get_current_commit_hash(): out = subprocess.check_output(["git", "rev-parse", "HEAD"]) return out.decode().split("\n")[0] + def get_current_git_branch(): out = subprocess.check_output(["git", "log", "-n", "1", "--pretty=%d", "HEAD"]) m = re.search(r"release_[0-9\.]+", out.decode()) @@ -66,38 +73,53 @@ def get_current_git_branch(): raise ValueError("Expected branch name of form release_xxx") return m.group(0) + def retrieve(url, filename=None): print(f"{url} -> {filename}") return urlretrieve(url, filename) + def main(): parser = argparse.ArgumentParser() - parser.add_argument("--release-version", type=str, required=True, - help="Version of the release being prepared") + parser.add_argument( + "--release-version", + type=str, + required=True, + help="Version of the release being prepared", + ) args = parser.parse_args() - if sys.platform != "darwin" or platform.machine() != "x86_64": - raise NotImplementedError("Please run this script using an Intel Mac") + if sys.platform != "darwin" or platform.machine() != "arm64": + raise NotImplementedError("Please run this script using an M1 Mac") version = args.release_version expected_git_tag = "v" + version current_git_tag = get_current_git_tag() if current_git_tag != expected_git_tag: if not current_git_tag: - raise ValueError(f"Expected git tag {expected_git_tag} but current HEAD has no tag. " - f"Run: git checkout {expected_git_tag}") - raise ValueError(f"Expected git tag {expected_git_tag} but current HEAD is at tag " - f"{current_git_tag}. Run: git checkout {expected_git_tag}") + raise ValueError( + f"Expected git tag {expected_git_tag} but current HEAD has no tag. " + f"Run: git checkout {expected_git_tag}" + ) + raise ValueError( + f"Expected git tag {expected_git_tag} but current HEAD is at tag " + f"{current_git_tag}. Run: git checkout {expected_git_tag}" + ) commit_hash = get_current_commit_hash() git_branch = get_current_git_branch() - print(f"Using commit {commit_hash} of branch {git_branch}, git tag {current_git_tag}") + print( + f"Using commit {commit_hash} of branch {git_branch}, git tag {current_git_tag}" + ) with cd("jvm-packages/"): print("====copying pure-Python tracker====") for use_cuda in [True, False]: xgboost4j = "xgboost4j-gpu" if use_cuda else "xgboost4j" - cp("../python-package/xgboost/tracker.py", f"{xgboost4j}/src/main/resources") + cp( + "../python-package/xgboost/tracker.py", + f"{xgboost4j}/src/main/resources", + ) print("====copying resources for testing====") with cd("../demo/CLI/regression"): @@ -115,7 +137,11 @@ def main(): cp(file, f"{xgboost4j_spark}/src/test/resources") print("====Creating directories to hold native binaries====") - for os_ident, arch in [("linux", "x86_64"), ("windows", "x86_64"), ("macos", "x86_64")]: + for os_ident, arch in [ + ("linux", "x86_64"), + ("windows", "x86_64"), + ("macos", "x86_64"), + ]: output_dir = f"xgboost4j/src/main/resources/lib/{os_ident}/{arch}" maybe_makedirs(output_dir) for os_ident, arch in [("linux", "x86_64")]: @@ -123,52 +149,76 @@ def main(): maybe_makedirs(output_dir) print("====Downloading native binaries from CI====") - nightly_bucket_prefix = "https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds" - maven_repo_prefix = "https://s3-us-west-2.amazonaws.com/xgboost-maven-repo/release/ml/dmlc" + nightly_bucket_prefix = ( + "https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds" + ) + maven_repo_prefix = ( + "https://s3-us-west-2.amazonaws.com/xgboost-maven-repo/release/ml/dmlc" + ) - retrieve(url=f"{nightly_bucket_prefix}/{git_branch}/xgboost4j_{commit_hash}.dll", - filename="xgboost4j/src/main/resources/lib/windows/x86_64/xgboost4j.dll") + retrieve( + url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/xgboost4j_{commit_hash}.dll", + filename="xgboost4j/src/main/resources/lib/windows/x86_64/xgboost4j.dll", + ) + retrieve( + url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/libxgboost4j_{commit_hash}.dylib", + filename="xgboost4j/src/main/resources/lib/macos/x86_64/libxgboost4j.dylib", + ) with tempfile.TemporaryDirectory() as tempdir: # libxgboost4j.so for Linux x86_64, CPU only zip_path = os.path.join(tempdir, "xgboost4j_2.12.jar") extract_dir = os.path.join(tempdir, "xgboost4j") - retrieve(url=f"{maven_repo_prefix}/xgboost4j_2.12/{version}/" - f"xgboost4j_2.12-{version}.jar", - filename=zip_path) + retrieve( + url=f"{maven_repo_prefix}/xgboost4j_2.12/{version}/" + f"xgboost4j_2.12-{version}.jar", + filename=zip_path, + ) os.mkdir(extract_dir) with zipfile.ZipFile(zip_path, "r") as t: t.extractall(extract_dir) - cp(os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"), - "xgboost4j/src/main/resources/lib/linux/x86_64/libxgboost4j.so") + cp( + os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"), + "xgboost4j/src/main/resources/lib/linux/x86_64/libxgboost4j.so", + ) # libxgboost4j.so for Linux x86_64, GPU support zip_path = os.path.join(tempdir, "xgboost4j-gpu_2.12.jar") extract_dir = os.path.join(tempdir, "xgboost4j-gpu") - retrieve(url=f"{maven_repo_prefix}/xgboost4j-gpu_2.12/{version}/" - f"xgboost4j-gpu_2.12-{version}.jar", - filename=zip_path) + retrieve( + url=f"{maven_repo_prefix}/xgboost4j-gpu_2.12/{version}/" + f"xgboost4j-gpu_2.12-{version}.jar", + filename=zip_path, + ) os.mkdir(extract_dir) with zipfile.ZipFile(zip_path, "r") as t: t.extractall(extract_dir) - cp(os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"), - "xgboost4j-gpu/src/main/resources/lib/linux/x86_64/libxgboost4j.so") - + cp( + os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"), + "xgboost4j-gpu/src/main/resources/lib/linux/x86_64/libxgboost4j.so", + ) print("====Next Steps====") print("1. Gain upload right to Maven Central repo.") print("1-1. Sign up for a JIRA account at Sonatype: ") - print("1-2. File a JIRA ticket: " - "https://issues.sonatype.org/secure/CreateIssue.jspa?issuetype=21&pid=10134. Example: " - "https://issues.sonatype.org/browse/OSSRH-67724") - print("2. Store the Sonatype credentials in .m2/settings.xml. See insturctions in " - "https://central.sonatype.org/publish/publish-maven/") + print( + "1-2. File a JIRA ticket: " + "https://issues.sonatype.org/secure/CreateIssue.jspa?issuetype=21&pid=10134. Example: " + "https://issues.sonatype.org/browse/OSSRH-67724" + ) + print( + "2. Store the Sonatype credentials in .m2/settings.xml. See insturctions in " + "https://central.sonatype.org/publish/publish-maven/" + ) print("3. Now on a Mac machine, run:") print(" GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests") - print("4. Log into https://oss.sonatype.org/. On the left menu panel, click Staging " - "Repositories. Visit the URL https://oss.sonatype.org/content/repositories/mldmlc-1085 " - "to inspect the staged JAR files. Finally, press Release button to publish the " - "artifacts to the Maven Central repository.") + print( + "4. Log into https://oss.sonatype.org/. On the left menu panel, click Staging " + "Repositories. Visit the URL https://oss.sonatype.org/content/repositories/mldmlc-1085 " + "to inspect the staged JAR files. Finally, press Release button to publish the " + "artifacts to the Maven Central repository." + ) + if __name__ == "__main__": main() diff --git a/python-package/packager/nativelib.py b/python-package/packager/nativelib.py index ff38fa11d..9d3fec2bc 100644 --- a/python-package/packager/nativelib.py +++ b/python-package/packager/nativelib.py @@ -132,16 +132,28 @@ def locate_or_build_libxgboost( if build_config.use_system_libxgboost: # Find libxgboost from system prefix - sys_base_prefix = pathlib.Path(sys.base_prefix).absolute().resolve() - libxgboost_sys = sys_base_prefix / "lib" / _lib_name() - if not libxgboost_sys.exists(): - raise RuntimeError( - f"use_system_libxgboost was specified but {_lib_name()} is " - f"not found in {libxgboost_sys.parent}" - ) - - logger.info("Using system XGBoost: %s", str(libxgboost_sys)) - return libxgboost_sys + sys_prefix = pathlib.Path(sys.base_prefix) + sys_prefix_candidates = [ + sys_prefix / "lib", + # Paths possibly used on Windows + sys_prefix / "bin", + sys_prefix / "Library", + sys_prefix / "Library" / "bin", + sys_prefix / "Library" / "lib", + ] + sys_prefix_candidates = [ + p.expanduser().resolve() for p in sys_prefix_candidates + ] + for candidate_dir in sys_prefix_candidates: + libtreelite_sys = candidate_dir / _lib_name() + if libtreelite_sys.exists(): + logger.info("Using system XGBoost: %s", str(libtreelite_sys)) + return libtreelite_sys + raise RuntimeError( + f"use_system_libxgboost was specified but {_lib_name()} is " + f"not found. Paths searched (in order): \n" + + "\n".join([f"* {str(p)}" for p in sys_prefix_candidates]) + ) libxgboost = locate_local_libxgboost(toplevel_dir, logger=logger) if libxgboost is not None: diff --git a/python-package/xgboost/libpath.py b/python-package/xgboost/libpath.py index 0437f3a4c..58c78df09 100644 --- a/python-package/xgboost/libpath.py +++ b/python-package/xgboost/libpath.py @@ -31,16 +31,15 @@ def find_lib_path() -> List[str]: ] if sys.platform == "win32": - if platform.architecture()[0] == "64bit": - dll_path.append(os.path.join(curr_path, "../../windows/x64/Release/")) - # hack for pip installation when copy all parent source - # directory here - dll_path.append(os.path.join(curr_path, "./windows/x64/Release/")) - else: - dll_path.append(os.path.join(curr_path, "../../windows/Release/")) - # hack for pip installation when copy all parent source - # directory here - dll_path.append(os.path.join(curr_path, "./windows/Release/")) + # On Windows, Conda may install libs in different paths + dll_path.extend( + [ + os.path.join(sys.base_prefix, "bin"), + os.path.join(sys.base_prefix, "Library"), + os.path.join(sys.base_prefix, "Library", "bin"), + os.path.join(sys.base_prefix, "Library", "lib"), + ] + ) dll_path = [os.path.join(p, "xgboost.dll") for p in dll_path] elif sys.platform.startswith(("linux", "freebsd", "emscripten")): dll_path = [os.path.join(p, "libxgboost.so") for p in dll_path]