Merge branch 'master' into sync-condition-2023Oct11
This commit is contained in:
Hui Liu
commit
e00131c465
@ -21,12 +21,14 @@ def normpath(path):
|
||||
else:
|
||||
return normalized
|
||||
|
||||
|
||||
def cp(source, target):
|
||||
source = normpath(source)
|
||||
target = normpath(target)
|
||||
print("cp {0} {1}".format(source, target))
|
||||
shutil.copy(source, target)
|
||||
|
||||
|
||||
def maybe_makedirs(path):
|
||||
path = normpath(path)
|
||||
print("mkdir -p " + path)
|
||||
@ -36,6 +38,7 @@ def maybe_makedirs(path):
|
||||
if e.errno != errno.EEXIST:
|
||||
raise
|
||||
|
||||
|
||||
@contextmanager
|
||||
def cd(path):
|
||||
path = normpath(path)
|
||||
@ -47,18 +50,22 @@ def cd(path):
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
|
||||
|
||||
def run(command, **kwargs):
|
||||
print(command)
|
||||
subprocess.check_call(command, shell=True, **kwargs)
|
||||
|
||||
|
||||
def get_current_git_tag():
|
||||
out = subprocess.check_output(["git", "tag", "--points-at", "HEAD"])
|
||||
return out.decode().split("\n")[0]
|
||||
|
||||
|
||||
def get_current_commit_hash():
|
||||
out = subprocess.check_output(["git", "rev-parse", "HEAD"])
|
||||
return out.decode().split("\n")[0]
|
||||
|
||||
|
||||
def get_current_git_branch():
|
||||
out = subprocess.check_output(["git", "log", "-n", "1", "--pretty=%d", "HEAD"])
|
||||
m = re.search(r"release_[0-9\.]+", out.decode())
|
||||
@ -66,38 +73,53 @@ def get_current_git_branch():
|
||||
raise ValueError("Expected branch name of form release_xxx")
|
||||
return m.group(0)
|
||||
|
||||
|
||||
def retrieve(url, filename=None):
|
||||
print(f"{url} -> {filename}")
|
||||
return urlretrieve(url, filename)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--release-version", type=str, required=True,
|
||||
help="Version of the release being prepared")
|
||||
parser.add_argument(
|
||||
"--release-version",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Version of the release being prepared",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if sys.platform != "darwin" or platform.machine() != "x86_64":
|
||||
raise NotImplementedError("Please run this script using an Intel Mac")
|
||||
if sys.platform != "darwin" or platform.machine() != "arm64":
|
||||
raise NotImplementedError("Please run this script using an M1 Mac")
|
||||
|
||||
version = args.release_version
|
||||
expected_git_tag = "v" + version
|
||||
current_git_tag = get_current_git_tag()
|
||||
if current_git_tag != expected_git_tag:
|
||||
if not current_git_tag:
|
||||
raise ValueError(f"Expected git tag {expected_git_tag} but current HEAD has no tag. "
|
||||
f"Run: git checkout {expected_git_tag}")
|
||||
raise ValueError(f"Expected git tag {expected_git_tag} but current HEAD is at tag "
|
||||
f"{current_git_tag}. Run: git checkout {expected_git_tag}")
|
||||
raise ValueError(
|
||||
f"Expected git tag {expected_git_tag} but current HEAD has no tag. "
|
||||
f"Run: git checkout {expected_git_tag}"
|
||||
)
|
||||
raise ValueError(
|
||||
f"Expected git tag {expected_git_tag} but current HEAD is at tag "
|
||||
f"{current_git_tag}. Run: git checkout {expected_git_tag}"
|
||||
)
|
||||
|
||||
commit_hash = get_current_commit_hash()
|
||||
git_branch = get_current_git_branch()
|
||||
print(f"Using commit {commit_hash} of branch {git_branch}, git tag {current_git_tag}")
|
||||
print(
|
||||
f"Using commit {commit_hash} of branch {git_branch}, git tag {current_git_tag}"
|
||||
)
|
||||
|
||||
with cd("jvm-packages/"):
|
||||
print("====copying pure-Python tracker====")
|
||||
for use_cuda in [True, False]:
|
||||
xgboost4j = "xgboost4j-gpu" if use_cuda else "xgboost4j"
|
||||
cp("../python-package/xgboost/tracker.py", f"{xgboost4j}/src/main/resources")
|
||||
cp(
|
||||
"../python-package/xgboost/tracker.py",
|
||||
f"{xgboost4j}/src/main/resources",
|
||||
)
|
||||
|
||||
print("====copying resources for testing====")
|
||||
with cd("../demo/CLI/regression"):
|
||||
@ -115,7 +137,11 @@ def main():
|
||||
cp(file, f"{xgboost4j_spark}/src/test/resources")
|
||||
|
||||
print("====Creating directories to hold native binaries====")
|
||||
for os_ident, arch in [("linux", "x86_64"), ("windows", "x86_64"), ("macos", "x86_64")]:
|
||||
for os_ident, arch in [
|
||||
("linux", "x86_64"),
|
||||
("windows", "x86_64"),
|
||||
("macos", "x86_64"),
|
||||
]:
|
||||
output_dir = f"xgboost4j/src/main/resources/lib/{os_ident}/{arch}"
|
||||
maybe_makedirs(output_dir)
|
||||
for os_ident, arch in [("linux", "x86_64")]:
|
||||
@ -123,52 +149,76 @@ def main():
|
||||
maybe_makedirs(output_dir)
|
||||
|
||||
print("====Downloading native binaries from CI====")
|
||||
nightly_bucket_prefix = "https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds"
|
||||
maven_repo_prefix = "https://s3-us-west-2.amazonaws.com/xgboost-maven-repo/release/ml/dmlc"
|
||||
nightly_bucket_prefix = (
|
||||
"https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds"
|
||||
)
|
||||
maven_repo_prefix = (
|
||||
"https://s3-us-west-2.amazonaws.com/xgboost-maven-repo/release/ml/dmlc"
|
||||
)
|
||||
|
||||
retrieve(url=f"{nightly_bucket_prefix}/{git_branch}/xgboost4j_{commit_hash}.dll",
|
||||
filename="xgboost4j/src/main/resources/lib/windows/x86_64/xgboost4j.dll")
|
||||
retrieve(
|
||||
url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/xgboost4j_{commit_hash}.dll",
|
||||
filename="xgboost4j/src/main/resources/lib/windows/x86_64/xgboost4j.dll",
|
||||
)
|
||||
retrieve(
|
||||
url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/libxgboost4j_{commit_hash}.dylib",
|
||||
filename="xgboost4j/src/main/resources/lib/macos/x86_64/libxgboost4j.dylib",
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tempdir:
|
||||
# libxgboost4j.so for Linux x86_64, CPU only
|
||||
zip_path = os.path.join(tempdir, "xgboost4j_2.12.jar")
|
||||
extract_dir = os.path.join(tempdir, "xgboost4j")
|
||||
retrieve(url=f"{maven_repo_prefix}/xgboost4j_2.12/{version}/"
|
||||
f"xgboost4j_2.12-{version}.jar",
|
||||
filename=zip_path)
|
||||
retrieve(
|
||||
url=f"{maven_repo_prefix}/xgboost4j_2.12/{version}/"
|
||||
f"xgboost4j_2.12-{version}.jar",
|
||||
filename=zip_path,
|
||||
)
|
||||
os.mkdir(extract_dir)
|
||||
with zipfile.ZipFile(zip_path, "r") as t:
|
||||
t.extractall(extract_dir)
|
||||
cp(os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"),
|
||||
"xgboost4j/src/main/resources/lib/linux/x86_64/libxgboost4j.so")
|
||||
cp(
|
||||
os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"),
|
||||
"xgboost4j/src/main/resources/lib/linux/x86_64/libxgboost4j.so",
|
||||
)
|
||||
|
||||
# libxgboost4j.so for Linux x86_64, GPU support
|
||||
zip_path = os.path.join(tempdir, "xgboost4j-gpu_2.12.jar")
|
||||
extract_dir = os.path.join(tempdir, "xgboost4j-gpu")
|
||||
retrieve(url=f"{maven_repo_prefix}/xgboost4j-gpu_2.12/{version}/"
|
||||
f"xgboost4j-gpu_2.12-{version}.jar",
|
||||
filename=zip_path)
|
||||
retrieve(
|
||||
url=f"{maven_repo_prefix}/xgboost4j-gpu_2.12/{version}/"
|
||||
f"xgboost4j-gpu_2.12-{version}.jar",
|
||||
filename=zip_path,
|
||||
)
|
||||
os.mkdir(extract_dir)
|
||||
with zipfile.ZipFile(zip_path, "r") as t:
|
||||
t.extractall(extract_dir)
|
||||
cp(os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"),
|
||||
"xgboost4j-gpu/src/main/resources/lib/linux/x86_64/libxgboost4j.so")
|
||||
|
||||
cp(
|
||||
os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"),
|
||||
"xgboost4j-gpu/src/main/resources/lib/linux/x86_64/libxgboost4j.so",
|
||||
)
|
||||
|
||||
print("====Next Steps====")
|
||||
print("1. Gain upload right to Maven Central repo.")
|
||||
print("1-1. Sign up for a JIRA account at Sonatype: ")
|
||||
print("1-2. File a JIRA ticket: "
|
||||
"https://issues.sonatype.org/secure/CreateIssue.jspa?issuetype=21&pid=10134. Example: "
|
||||
"https://issues.sonatype.org/browse/OSSRH-67724")
|
||||
print("2. Store the Sonatype credentials in .m2/settings.xml. See insturctions in "
|
||||
"https://central.sonatype.org/publish/publish-maven/")
|
||||
print(
|
||||
"1-2. File a JIRA ticket: "
|
||||
"https://issues.sonatype.org/secure/CreateIssue.jspa?issuetype=21&pid=10134. Example: "
|
||||
"https://issues.sonatype.org/browse/OSSRH-67724"
|
||||
)
|
||||
print(
|
||||
"2. Store the Sonatype credentials in .m2/settings.xml. See insturctions in "
|
||||
"https://central.sonatype.org/publish/publish-maven/"
|
||||
)
|
||||
print("3. Now on a Mac machine, run:")
|
||||
print(" GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests")
|
||||
print("4. Log into https://oss.sonatype.org/. On the left menu panel, click Staging "
|
||||
"Repositories. Visit the URL https://oss.sonatype.org/content/repositories/mldmlc-1085 "
|
||||
"to inspect the staged JAR files. Finally, press Release button to publish the "
|
||||
"artifacts to the Maven Central repository.")
|
||||
print(
|
||||
"4. Log into https://oss.sonatype.org/. On the left menu panel, click Staging "
|
||||
"Repositories. Visit the URL https://oss.sonatype.org/content/repositories/mldmlc-1085 "
|
||||
"to inspect the staged JAR files. Finally, press Release button to publish the "
|
||||
"artifacts to the Maven Central repository."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@ -132,16 +132,28 @@ def locate_or_build_libxgboost(
|
||||
|
||||
if build_config.use_system_libxgboost:
|
||||
# Find libxgboost from system prefix
|
||||
sys_base_prefix = pathlib.Path(sys.base_prefix).absolute().resolve()
|
||||
libxgboost_sys = sys_base_prefix / "lib" / _lib_name()
|
||||
if not libxgboost_sys.exists():
|
||||
raise RuntimeError(
|
||||
f"use_system_libxgboost was specified but {_lib_name()} is "
|
||||
f"not found in {libxgboost_sys.parent}"
|
||||
)
|
||||
|
||||
logger.info("Using system XGBoost: %s", str(libxgboost_sys))
|
||||
return libxgboost_sys
|
||||
sys_prefix = pathlib.Path(sys.base_prefix)
|
||||
sys_prefix_candidates = [
|
||||
sys_prefix / "lib",
|
||||
# Paths possibly used on Windows
|
||||
sys_prefix / "bin",
|
||||
sys_prefix / "Library",
|
||||
sys_prefix / "Library" / "bin",
|
||||
sys_prefix / "Library" / "lib",
|
||||
]
|
||||
sys_prefix_candidates = [
|
||||
p.expanduser().resolve() for p in sys_prefix_candidates
|
||||
]
|
||||
for candidate_dir in sys_prefix_candidates:
|
||||
libtreelite_sys = candidate_dir / _lib_name()
|
||||
if libtreelite_sys.exists():
|
||||
logger.info("Using system XGBoost: %s", str(libtreelite_sys))
|
||||
return libtreelite_sys
|
||||
raise RuntimeError(
|
||||
f"use_system_libxgboost was specified but {_lib_name()} is "
|
||||
f"not found. Paths searched (in order): \n"
|
||||
+ "\n".join([f"* {str(p)}" for p in sys_prefix_candidates])
|
||||
)
|
||||
|
||||
libxgboost = locate_local_libxgboost(toplevel_dir, logger=logger)
|
||||
if libxgboost is not None:
|
||||
|
||||
@ -31,16 +31,15 @@ def find_lib_path() -> List[str]:
|
||||
]
|
||||
|
||||
if sys.platform == "win32":
|
||||
if platform.architecture()[0] == "64bit":
|
||||
dll_path.append(os.path.join(curr_path, "../../windows/x64/Release/"))
|
||||
# hack for pip installation when copy all parent source
|
||||
# directory here
|
||||
dll_path.append(os.path.join(curr_path, "./windows/x64/Release/"))
|
||||
else:
|
||||
dll_path.append(os.path.join(curr_path, "../../windows/Release/"))
|
||||
# hack for pip installation when copy all parent source
|
||||
# directory here
|
||||
dll_path.append(os.path.join(curr_path, "./windows/Release/"))
|
||||
# On Windows, Conda may install libs in different paths
|
||||
dll_path.extend(
|
||||
[
|
||||
os.path.join(sys.base_prefix, "bin"),
|
||||
os.path.join(sys.base_prefix, "Library"),
|
||||
os.path.join(sys.base_prefix, "Library", "bin"),
|
||||
os.path.join(sys.base_prefix, "Library", "lib"),
|
||||
]
|
||||
)
|
||||
dll_path = [os.path.join(p, "xgboost.dll") for p in dll_path]
|
||||
elif sys.platform.startswith(("linux", "freebsd", "emscripten")):
|
||||
dll_path = [os.path.join(p, "libxgboost.so") for p in dll_path]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user