Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into sync-condition-2023Oct11

Browse Source
This commit is contained in:
Hui Liu 2023-10-26 11:35:48 -07:00
commit e00131c465
3 changed files with 116 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
dev/prepare_jvm_release.py
View File

@ -21,12 +21,14 @@ def normpath(path):
else: else:
return normalized return normalized
def cp(source, target): def cp(source, target):
source = normpath(source) source = normpath(source)
target = normpath(target) target = normpath(target)
print("cp {0} {1}".format(source, target)) print("cp {0} {1}".format(source, target))
shutil.copy(source, target) shutil.copy(source, target)
def maybe_makedirs(path): def maybe_makedirs(path):
path = normpath(path) path = normpath(path)
print("mkdir -p " + path) print("mkdir -p " + path)
@ -36,6 +38,7 @@ def maybe_makedirs(path):
if e.errno != errno.EEXIST: if e.errno != errno.EEXIST:
raise raise
@contextmanager @contextmanager
def cd(path): def cd(path):
path = normpath(path) path = normpath(path)
@ -47,18 +50,22 @@ def cd(path):
finally: finally:
os.chdir(cwd) os.chdir(cwd)
def run(command, **kwargs): def run(command, **kwargs):
print(command) print(command)
subprocess.check_call(command, shell=True, **kwargs) subprocess.check_call(command, shell=True, **kwargs)
def get_current_git_tag(): def get_current_git_tag():
out = subprocess.check_output(["git", "tag", "--points-at", "HEAD"]) out = subprocess.check_output(["git", "tag", "--points-at", "HEAD"])
return out.decode().split("\n")[0] return out.decode().split("\n")[0]
def get_current_commit_hash(): def get_current_commit_hash():
out = subprocess.check_output(["git", "rev-parse", "HEAD"]) out = subprocess.check_output(["git", "rev-parse", "HEAD"])
return out.decode().split("\n")[0] return out.decode().split("\n")[0]
def get_current_git_branch(): def get_current_git_branch():
out = subprocess.check_output(["git", "log", "-n", "1", "--pretty=%d", "HEAD"]) out = subprocess.check_output(["git", "log", "-n", "1", "--pretty=%d", "HEAD"])
m = re.search(r"release_[0-9\.]+", out.decode()) m = re.search(r"release_[0-9\.]+", out.decode())
@ -66,38 +73,53 @@ def get_current_git_branch():
raise ValueError("Expected branch name of form release_xxx") raise ValueError("Expected branch name of form release_xxx")
return m.group(0) return m.group(0)
def retrieve(url, filename=None): def retrieve(url, filename=None):
print(f"{url} -> {filename}") print(f"{url} -> {filename}")
return urlretrieve(url, filename) return urlretrieve(url, filename)
def main(): def main():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("--release-version", type=str, required=True, parser.add_argument(
help="Version of the release being prepared") "--release-version",
type=str,
required=True,
help="Version of the release being prepared",
)
args = parser.parse_args() args = parser.parse_args()
if sys.platform != "darwin" or platform.machine() != "x86_64": if sys.platform != "darwin" or platform.machine() != "arm64":
raise NotImplementedError("Please run this script using an Intel Mac") raise NotImplementedError("Please run this script using an M1 Mac")
version = args.release_version version = args.release_version
expected_git_tag = "v" + version expected_git_tag = "v" + version
current_git_tag = get_current_git_tag() current_git_tag = get_current_git_tag()
if current_git_tag != expected_git_tag: if current_git_tag != expected_git_tag:
if not current_git_tag: if not current_git_tag:
raise ValueError(f"Expected git tag {expected_git_tag} but current HEAD has no tag. " raise ValueError(
f"Run: git checkout {expected_git_tag}") f"Expected git tag {expected_git_tag} but current HEAD has no tag. "
raise ValueError(f"Expected git tag {expected_git_tag} but current HEAD is at tag " f"Run: git checkout {expected_git_tag}"
f"{current_git_tag}. Run: git checkout {expected_git_tag}") )
raise ValueError(
f"Expected git tag {expected_git_tag} but current HEAD is at tag "
f"{current_git_tag}. Run: git checkout {expected_git_tag}"
)
commit_hash = get_current_commit_hash() commit_hash = get_current_commit_hash()
git_branch = get_current_git_branch() git_branch = get_current_git_branch()
print(f"Using commit {commit_hash} of branch {git_branch}, git tag {current_git_tag}") print(
f"Using commit {commit_hash} of branch {git_branch}, git tag {current_git_tag}"
)
with cd("jvm-packages/"): with cd("jvm-packages/"):
print("====copying pure-Python tracker====") print("====copying pure-Python tracker====")
for use_cuda in [True, False]: for use_cuda in [True, False]:
xgboost4j = "xgboost4j-gpu" if use_cuda else "xgboost4j" xgboost4j = "xgboost4j-gpu" if use_cuda else "xgboost4j"
cp("../python-package/xgboost/tracker.py", f"{xgboost4j}/src/main/resources") cp(
"../python-package/xgboost/tracker.py",
f"{xgboost4j}/src/main/resources",
)
print("====copying resources for testing====") print("====copying resources for testing====")
with cd("../demo/CLI/regression"): with cd("../demo/CLI/regression"):
@ -115,7 +137,11 @@ def main():
cp(file, f"{xgboost4j_spark}/src/test/resources") cp(file, f"{xgboost4j_spark}/src/test/resources")
print("====Creating directories to hold native binaries====") print("====Creating directories to hold native binaries====")
for os_ident, arch in [("linux", "x86_64"), ("windows", "x86_64"), ("macos", "x86_64")]: for os_ident, arch in [
("linux", "x86_64"),
("windows", "x86_64"),
("macos", "x86_64"),
]:
output_dir = f"xgboost4j/src/main/resources/lib/{os_ident}/{arch}" output_dir = f"xgboost4j/src/main/resources/lib/{os_ident}/{arch}"
maybe_makedirs(output_dir) maybe_makedirs(output_dir)
for os_ident, arch in [("linux", "x86_64")]: for os_ident, arch in [("linux", "x86_64")]:
@ -123,52 +149,76 @@ def main():
maybe_makedirs(output_dir) maybe_makedirs(output_dir)
print("====Downloading native binaries from CI====") print("====Downloading native binaries from CI====")
nightly_bucket_prefix = "https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds" nightly_bucket_prefix = (
maven_repo_prefix = "https://s3-us-west-2.amazonaws.com/xgboost-maven-repo/release/ml/dmlc" "https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds"
)
maven_repo_prefix = (
"https://s3-us-west-2.amazonaws.com/xgboost-maven-repo/release/ml/dmlc"
)
retrieve(url=f"{nightly_bucket_prefix}/{git_branch}/xgboost4j_{commit_hash}.dll", retrieve(
filename="xgboost4j/src/main/resources/lib/windows/x86_64/xgboost4j.dll") url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/xgboost4j_{commit_hash}.dll",
filename="xgboost4j/src/main/resources/lib/windows/x86_64/xgboost4j.dll",
)
retrieve(
url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/libxgboost4j_{commit_hash}.dylib",
filename="xgboost4j/src/main/resources/lib/macos/x86_64/libxgboost4j.dylib",
)
with tempfile.TemporaryDirectory() as tempdir: with tempfile.TemporaryDirectory() as tempdir:
# libxgboost4j.so for Linux x86_64, CPU only # libxgboost4j.so for Linux x86_64, CPU only
zip_path = os.path.join(tempdir, "xgboost4j_2.12.jar") zip_path = os.path.join(tempdir, "xgboost4j_2.12.jar")
extract_dir = os.path.join(tempdir, "xgboost4j") extract_dir = os.path.join(tempdir, "xgboost4j")
retrieve(url=f"{maven_repo_prefix}/xgboost4j_2.12/{version}/" retrieve(
url=f"{maven_repo_prefix}/xgboost4j_2.12/{version}/"
f"xgboost4j_2.12-{version}.jar", f"xgboost4j_2.12-{version}.jar",
filename=zip_path) filename=zip_path,
)
os.mkdir(extract_dir) os.mkdir(extract_dir)
with zipfile.ZipFile(zip_path, "r") as t: with zipfile.ZipFile(zip_path, "r") as t:
t.extractall(extract_dir) t.extractall(extract_dir)
cp(os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"), cp(
"xgboost4j/src/main/resources/lib/linux/x86_64/libxgboost4j.so") os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"),
"xgboost4j/src/main/resources/lib/linux/x86_64/libxgboost4j.so",
)
# libxgboost4j.so for Linux x86_64, GPU support # libxgboost4j.so for Linux x86_64, GPU support
zip_path = os.path.join(tempdir, "xgboost4j-gpu_2.12.jar") zip_path = os.path.join(tempdir, "xgboost4j-gpu_2.12.jar")
extract_dir = os.path.join(tempdir, "xgboost4j-gpu") extract_dir = os.path.join(tempdir, "xgboost4j-gpu")
retrieve(url=f"{maven_repo_prefix}/xgboost4j-gpu_2.12/{version}/" retrieve(
url=f"{maven_repo_prefix}/xgboost4j-gpu_2.12/{version}/"
f"xgboost4j-gpu_2.12-{version}.jar", f"xgboost4j-gpu_2.12-{version}.jar",
filename=zip_path) filename=zip_path,
)
os.mkdir(extract_dir) os.mkdir(extract_dir)
with zipfile.ZipFile(zip_path, "r") as t: with zipfile.ZipFile(zip_path, "r") as t:
t.extractall(extract_dir) t.extractall(extract_dir)
cp(os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"), cp(
"xgboost4j-gpu/src/main/resources/lib/linux/x86_64/libxgboost4j.so") os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"),
"xgboost4j-gpu/src/main/resources/lib/linux/x86_64/libxgboost4j.so",
)
print("====Next Steps====") print("====Next Steps====")
print("1. Gain upload right to Maven Central repo.") print("1. Gain upload right to Maven Central repo.")
print("1-1. Sign up for a JIRA account at Sonatype: ") print("1-1. Sign up for a JIRA account at Sonatype: ")
print("1-2. File a JIRA ticket: " print(
"1-2. File a JIRA ticket: "
"https://issues.sonatype.org/secure/CreateIssue.jspa?issuetype=21&pid=10134. Example: " "https://issues.sonatype.org/secure/CreateIssue.jspa?issuetype=21&pid=10134. Example: "
"https://issues.sonatype.org/browse/OSSRH-67724") "https://issues.sonatype.org/browse/OSSRH-67724"
print("2. Store the Sonatype credentials in .m2/settings.xml. See insturctions in " )
"https://central.sonatype.org/publish/publish-maven/") print(
"2. Store the Sonatype credentials in .m2/settings.xml. See insturctions in "
"https://central.sonatype.org/publish/publish-maven/"
)
print("3. Now on a Mac machine, run:") print("3. Now on a Mac machine, run:")
print(" GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests") print(" GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests")
print("4. Log into https://oss.sonatype.org/. On the left menu panel, click Staging " print(
"4. Log into https://oss.sonatype.org/. On the left menu panel, click Staging "
"Repositories. Visit the URL https://oss.sonatype.org/content/repositories/mldmlc-1085 " "Repositories. Visit the URL https://oss.sonatype.org/content/repositories/mldmlc-1085 "
"to inspect the staged JAR files. Finally, press Release button to publish the " "to inspect the staged JAR files. Finally, press Release button to publish the "
"artifacts to the Maven Central repository.") "artifacts to the Maven Central repository."
)
if __name__ == "__main__": if __name__ == "__main__":
main() main()

26
python-package/packager/nativelib.py
View File

@ -132,17 +132,29 @@ def locate_or_build_libxgboost(
if build_config.use_system_libxgboost: if build_config.use_system_libxgboost:
# Find libxgboost from system prefix # Find libxgboost from system prefix
sys_base_prefix = pathlib.Path(sys.base_prefix).absolute().resolve() sys_prefix = pathlib.Path(sys.base_prefix)
libxgboost_sys = sys_base_prefix / "lib" / _lib_name() sys_prefix_candidates = [
if not libxgboost_sys.exists(): sys_prefix / "lib",
# Paths possibly used on Windows
sys_prefix / "bin",
sys_prefix / "Library",
sys_prefix / "Library" / "bin",
sys_prefix / "Library" / "lib",
]
sys_prefix_candidates = [
p.expanduser().resolve() for p in sys_prefix_candidates
]
for candidate_dir in sys_prefix_candidates:
libtreelite_sys = candidate_dir / _lib_name()
if libtreelite_sys.exists():
logger.info("Using system XGBoost: %s", str(libtreelite_sys))
return libtreelite_sys
raise RuntimeError( raise RuntimeError(
f"use_system_libxgboost was specified but {_lib_name()} is " f"use_system_libxgboost was specified but {_lib_name()} is "
f"not found in {libxgboost_sys.parent}" f"not found. Paths searched (in order): \n"
+ "\n".join([f"* {str(p)}" for p in sys_prefix_candidates])
) )
logger.info("Using system XGBoost: %s", str(libxgboost_sys))
return libxgboost_sys
libxgboost = locate_local_libxgboost(toplevel_dir, logger=logger) libxgboost = locate_local_libxgboost(toplevel_dir, logger=logger)
if libxgboost is not None: if libxgboost is not None:
return libxgboost return libxgboost

19
python-package/xgboost/libpath.py
View File

@ -31,16 +31,15 @@ def find_lib_path() -> List[str]:
] ]
if sys.platform == "win32": if sys.platform == "win32":
if platform.architecture()[0] == "64bit": # On Windows, Conda may install libs in different paths
dll_path.append(os.path.join(curr_path, "../../windows/x64/Release/")) dll_path.extend(
# hack for pip installation when copy all parent source [
# directory here os.path.join(sys.base_prefix, "bin"),
dll_path.append(os.path.join(curr_path, "./windows/x64/Release/")) os.path.join(sys.base_prefix, "Library"),
else: os.path.join(sys.base_prefix, "Library", "bin"),
dll_path.append(os.path.join(curr_path, "../../windows/Release/")) os.path.join(sys.base_prefix, "Library", "lib"),
# hack for pip installation when copy all parent source ]
# directory here )
dll_path.append(os.path.join(curr_path, "./windows/Release/"))
dll_path = [os.path.join(p, "xgboost.dll") for p in dll_path] dll_path = [os.path.join(p, "xgboost.dll") for p in dll_path]
elif sys.platform.startswith(("linux", "freebsd", "emscripten")): elif sys.platform.startswith(("linux", "freebsd", "emscripten")):
dll_path = [os.path.join(p, "libxgboost.so") for p in dll_path] dll_path = [os.path.join(p, "libxgboost.so") for p in dll_path]