rocm enable for v2.0.1, rm setup.py
This commit is contained in:
Hui Liu
parent
782b73f2bb
commit
a41bc0975c
@ -1,407 +0,0 @@
|
|||||||
"""Setup xgboost package."""
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
from platform import system
|
|
||||||
from typing import List, Optional
|
|
||||||
|
|
||||||
from setuptools import Extension, find_packages, setup
|
|
||||||
from setuptools.command import build_ext, install, install_lib, sdist
|
|
||||||
|
|
||||||
# You can't use `pip install .` as pip copies setup.py to a temporary
|
|
||||||
# directory, parent directory is no longer reachable (isolated build) .
|
|
||||||
CURRENT_DIR = os.path.abspath(os.path.dirname(__file__))
|
|
||||||
sys.path.insert(0, CURRENT_DIR)
|
|
||||||
|
|
||||||
# Options only effect `python setup.py install`, building `bdist_wheel`
|
|
||||||
# requires using CMake directly.
|
|
||||||
USER_OPTIONS = {
|
|
||||||
# libxgboost options.
|
|
||||||
"use-openmp": (None, "Build with OpenMP support.", 1),
|
|
||||||
"use-cuda": (None, "Build with GPU acceleration.", 0),
|
|
||||||
"use-nccl": (None, "Build with NCCL to enable distributed GPU support.", 0),
|
|
||||||
"build-with-shared-nccl": (None, "Build with shared NCCL library.", 0),
|
|
||||||
"use-hip": (None, "Build with GPU acceleration.", 0),
|
|
||||||
"use-rccl": (None, "Build with RCCL to enable distributed GPU support.", 0),
|
|
||||||
"hide-cxx-symbols": (None, "Hide all C++ symbols during build.", 1),
|
|
||||||
"use-hdfs": (None, "Build with HDFS support", 0),
|
|
||||||
"use-azure": (None, "Build with AZURE support.", 0),
|
|
||||||
"use-s3": (None, "Build with S3 support", 0),
|
|
||||||
"plugin-dense-parser": (None, "Build dense parser plugin.", 0),
|
|
||||||
# Python specific
|
|
||||||
"use-system-libxgboost": (None, "Use libxgboost.so in system path.", 0),
|
|
||||||
}
|
|
||||||
|
|
||||||
NEED_CLEAN_TREE = set()
|
|
||||||
NEED_CLEAN_FILE = set()
|
|
||||||
BUILD_TEMP_DIR = None
|
|
||||||
|
|
||||||
|
|
||||||
def lib_name() -> str:
|
|
||||||
"""Return platform dependent shared object name."""
|
|
||||||
if system() == "Linux" or system().upper().endswith("BSD"):
|
|
||||||
name = "libxgboost.so"
|
|
||||||
elif system() == "Darwin":
|
|
||||||
name = "libxgboost.dylib"
|
|
||||||
elif system() == "Windows":
|
|
||||||
name = "xgboost.dll"
|
|
||||||
elif system() == "OS400":
|
|
||||||
name = "libxgboost.so"
|
|
||||||
return name
|
|
||||||
|
|
||||||
|
|
||||||
def copy_tree(src_dir: str, target_dir: str) -> None:
|
|
||||||
"""Copy source tree into build directory."""
|
|
||||||
|
|
||||||
def clean_copy_tree(src: str, dst: str) -> None:
|
|
||||||
shutil.copytree(src, dst)
|
|
||||||
NEED_CLEAN_TREE.add(os.path.abspath(dst))
|
|
||||||
|
|
||||||
def clean_copy_file(src: str, dst: str) -> None:
|
|
||||||
shutil.copy(src, dst)
|
|
||||||
NEED_CLEAN_FILE.add(os.path.abspath(dst))
|
|
||||||
|
|
||||||
src = os.path.join(src_dir, "src")
|
|
||||||
inc = os.path.join(src_dir, "include")
|
|
||||||
dmlc_core = os.path.join(src_dir, "dmlc-core")
|
|
||||||
gputreeshap = os.path.join(src_dir, "gputreeshap")
|
|
||||||
rocgputreeshap = os.path.join(src_dir, "rocgputreeshap")
|
|
||||||
warpprim= os.path.join(src_dir, "warp-primitives")
|
|
||||||
rabit = os.path.join(src_dir, "rabit")
|
|
||||||
cmake = os.path.join(src_dir, "cmake")
|
|
||||||
plugin = os.path.join(src_dir, "plugin")
|
|
||||||
|
|
||||||
clean_copy_tree(src, os.path.join(target_dir, "src"))
|
|
||||||
clean_copy_tree(inc, os.path.join(target_dir, "include"))
|
|
||||||
clean_copy_tree(dmlc_core, os.path.join(target_dir, "dmlc-core"))
|
|
||||||
clean_copy_tree(gputreeshap, os.path.join(target_dir, "gputreeshap"))
|
|
||||||
clean_copy_tree(rocgputreeshap, os.path.join(target_dir, "rocgputreeshap"))
|
|
||||||
clean_copy_tree(warpprim, os.path.join(target_dir, "warp-primitives"))
|
|
||||||
clean_copy_tree(rabit, os.path.join(target_dir, "rabit"))
|
|
||||||
clean_copy_tree(cmake, os.path.join(target_dir, "cmake"))
|
|
||||||
clean_copy_tree(plugin, os.path.join(target_dir, "plugin"))
|
|
||||||
|
|
||||||
cmake_list = os.path.join(src_dir, "CMakeLists.txt")
|
|
||||||
clean_copy_file(cmake_list, os.path.join(target_dir, "CMakeLists.txt"))
|
|
||||||
lic = os.path.join(src_dir, "LICENSE")
|
|
||||||
clean_copy_file(lic, os.path.join(target_dir, "LICENSE"))
|
|
||||||
|
|
||||||
|
|
||||||
def clean_up() -> None:
|
|
||||||
"""Removed copied files."""
|
|
||||||
for path in NEED_CLEAN_TREE:
|
|
||||||
shutil.rmtree(path)
|
|
||||||
for path in NEED_CLEAN_FILE:
|
|
||||||
os.remove(path)
|
|
||||||
|
|
||||||
|
|
||||||
class CMakeExtension(Extension): # pylint: disable=too-few-public-methods
|
|
||||||
"""Wrapper for extension"""
|
|
||||||
|
|
||||||
def __init__(self, name: str) -> None:
|
|
||||||
super().__init__(name=name, sources=[])
|
|
||||||
|
|
||||||
|
|
||||||
class BuildExt(build_ext.build_ext): # pylint: disable=too-many-ancestors
|
|
||||||
"""Custom build_ext command using CMake."""
|
|
||||||
|
|
||||||
logger = logging.getLogger("XGBoost build_ext")
|
|
||||||
|
|
||||||
# pylint: disable=too-many-arguments
|
|
||||||
def build(
|
|
||||||
self,
|
|
||||||
src_dir: str,
|
|
||||||
build_dir: str,
|
|
||||||
generator: str,
|
|
||||||
build_tool: Optional[str] = None,
|
|
||||||
use_omp: int = 1,
|
|
||||||
) -> None:
|
|
||||||
"""Build the core library with CMake."""
|
|
||||||
cmake_cmd = ["cmake", src_dir, generator]
|
|
||||||
|
|
||||||
for k, v in USER_OPTIONS.items():
|
|
||||||
arg = k.replace("-", "_").upper()
|
|
||||||
value = str(v[2])
|
|
||||||
if arg == "USE_SYSTEM_LIBXGBOOST":
|
|
||||||
continue
|
|
||||||
if arg == "USE_OPENMP" and use_omp == 0:
|
|
||||||
cmake_cmd.append("-D" + arg + "=0")
|
|
||||||
continue
|
|
||||||
cmake_cmd.append("-D" + arg + "=" + value)
|
|
||||||
|
|
||||||
# Flag for cross-compiling for Apple Silicon
|
|
||||||
# We use environment variable because it's the only way to pass down custom flags
|
|
||||||
# through the cibuildwheel package, which otherwise calls `python setup.py bdist_wheel`
|
|
||||||
# command.
|
|
||||||
if "CIBW_TARGET_OSX_ARM64" in os.environ:
|
|
||||||
cmake_cmd.append("-DCMAKE_OSX_ARCHITECTURES=arm64")
|
|
||||||
|
|
||||||
self.logger.info("Run CMake command: %s", str(cmake_cmd))
|
|
||||||
subprocess.check_call(cmake_cmd, cwd=build_dir)
|
|
||||||
|
|
||||||
if system() != "Windows":
|
|
||||||
nproc = os.cpu_count()
|
|
||||||
assert build_tool is not None
|
|
||||||
subprocess.check_call([build_tool, "-j" + str(nproc)], cwd=build_dir)
|
|
||||||
else:
|
|
||||||
subprocess.check_call(
|
|
||||||
["cmake", "--build", ".", "--config", "Release"], cwd=build_dir
|
|
||||||
)
|
|
||||||
|
|
||||||
def build_cmake_extension(self) -> None:
|
|
||||||
"""Configure and build using CMake"""
|
|
||||||
if USER_OPTIONS["use-system-libxgboost"][2]:
|
|
||||||
self.logger.info("Using system libxgboost.")
|
|
||||||
return
|
|
||||||
|
|
||||||
build_dir = self.build_temp
|
|
||||||
global BUILD_TEMP_DIR # pylint: disable=global-statement
|
|
||||||
BUILD_TEMP_DIR = build_dir
|
|
||||||
libxgboost = os.path.abspath(
|
|
||||||
os.path.join(CURRENT_DIR, os.path.pardir, "lib", lib_name())
|
|
||||||
)
|
|
||||||
|
|
||||||
if os.path.exists(libxgboost):
|
|
||||||
self.logger.info("Found shared library, skipping build.")
|
|
||||||
return
|
|
||||||
|
|
||||||
src_dir = "xgboost"
|
|
||||||
try:
|
|
||||||
copy_tree(
|
|
||||||
os.path.join(CURRENT_DIR, os.path.pardir),
|
|
||||||
os.path.join(self.build_temp, src_dir),
|
|
||||||
)
|
|
||||||
except Exception: # pylint: disable=broad-except
|
|
||||||
copy_tree(src_dir, os.path.join(self.build_temp, src_dir))
|
|
||||||
|
|
||||||
self.logger.info("Building from source. %s", libxgboost)
|
|
||||||
if not os.path.exists(build_dir):
|
|
||||||
os.mkdir(build_dir)
|
|
||||||
if shutil.which("ninja"):
|
|
||||||
build_tool = "ninja"
|
|
||||||
else:
|
|
||||||
build_tool = "make"
|
|
||||||
if sys.platform.startswith("os400"):
|
|
||||||
build_tool = "make"
|
|
||||||
|
|
||||||
if system() == "Windows":
|
|
||||||
# Pick up from LGB, just test every possible tool chain.
|
|
||||||
for vs in (
|
|
||||||
"-GVisual Studio 17 2022",
|
|
||||||
"-GVisual Studio 16 2019",
|
|
||||||
"-GVisual Studio 15 2017",
|
|
||||||
"-GVisual Studio 14 2015",
|
|
||||||
"-GMinGW Makefiles",
|
|
||||||
):
|
|
||||||
try:
|
|
||||||
self.build(src_dir, build_dir, vs)
|
|
||||||
self.logger.info(
|
|
||||||
"%s is used for building Windows distribution.", vs
|
|
||||||
)
|
|
||||||
break
|
|
||||||
except subprocess.CalledProcessError:
|
|
||||||
shutil.rmtree(build_dir)
|
|
||||||
os.mkdir(build_dir)
|
|
||||||
continue
|
|
||||||
else:
|
|
||||||
gen = "-GNinja" if build_tool == "ninja" else "-GUnix Makefiles"
|
|
||||||
try:
|
|
||||||
self.build(src_dir, build_dir, gen, build_tool, use_omp=1)
|
|
||||||
except subprocess.CalledProcessError:
|
|
||||||
self.logger.warning("Disabling OpenMP support.")
|
|
||||||
self.build(src_dir, build_dir, gen, build_tool, use_omp=0)
|
|
||||||
|
|
||||||
def build_extension(self, ext: Extension) -> None:
|
|
||||||
"""Override the method for dispatching."""
|
|
||||||
if isinstance(ext, CMakeExtension):
|
|
||||||
self.build_cmake_extension()
|
|
||||||
else:
|
|
||||||
super().build_extension(ext)
|
|
||||||
|
|
||||||
def copy_extensions_to_source(self) -> None:
|
|
||||||
"""Dummy override. Invoked during editable installation. Our binary
|
|
||||||
should available in `lib`.
|
|
||||||
|
|
||||||
"""
|
|
||||||
if not os.path.exists(
|
|
||||||
os.path.join(CURRENT_DIR, os.path.pardir, "lib", lib_name())
|
|
||||||
):
|
|
||||||
raise ValueError(
|
|
||||||
"For using editable installation, please "
|
|
||||||
+ "build the shared object first with CMake."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class Sdist(sdist.sdist): # pylint: disable=too-many-ancestors
|
|
||||||
"""Copy c++ source into Python directory."""
|
|
||||||
|
|
||||||
logger = logging.getLogger("xgboost sdist")
|
|
||||||
|
|
||||||
def run(self) -> None:
|
|
||||||
copy_tree(
|
|
||||||
os.path.join(CURRENT_DIR, os.path.pardir),
|
|
||||||
os.path.join(CURRENT_DIR, "xgboost"),
|
|
||||||
)
|
|
||||||
libxgboost = os.path.join(CURRENT_DIR, os.path.pardir, "lib", lib_name())
|
|
||||||
if os.path.exists(libxgboost):
|
|
||||||
self.logger.warning(
|
|
||||||
"Found shared library, removing to avoid being included in source distribution."
|
|
||||||
)
|
|
||||||
os.remove(libxgboost)
|
|
||||||
super().run()
|
|
||||||
|
|
||||||
|
|
||||||
class InstallLib(install_lib.install_lib):
|
|
||||||
"""Copy shared object into installation directory."""
|
|
||||||
|
|
||||||
logger = logging.getLogger("xgboost install_lib")
|
|
||||||
|
|
||||||
def install(self) -> List[str]:
|
|
||||||
outfiles = super().install()
|
|
||||||
|
|
||||||
if USER_OPTIONS["use-system-libxgboost"][2] != 0:
|
|
||||||
self.logger.info("Using system libxgboost.")
|
|
||||||
lib_path = os.path.join(sys.prefix, "lib")
|
|
||||||
msg = (
|
|
||||||
"use-system-libxgboost is specified, but "
|
|
||||||
+ lib_name()
|
|
||||||
+ " is not found in: "
|
|
||||||
+ lib_path
|
|
||||||
)
|
|
||||||
assert os.path.exists(os.path.join(lib_path, lib_name())), msg
|
|
||||||
return []
|
|
||||||
|
|
||||||
lib_dir = os.path.join(self.install_dir, "xgboost", "lib")
|
|
||||||
if not os.path.exists(lib_dir):
|
|
||||||
os.mkdir(lib_dir)
|
|
||||||
dst = os.path.join(self.install_dir, "xgboost", "lib", lib_name())
|
|
||||||
|
|
||||||
libxgboost_path = lib_name()
|
|
||||||
|
|
||||||
assert BUILD_TEMP_DIR is not None
|
|
||||||
dft_lib_dir = os.path.join(CURRENT_DIR, os.path.pardir, "lib")
|
|
||||||
build_dir = os.path.join(BUILD_TEMP_DIR, "xgboost", "lib")
|
|
||||||
|
|
||||||
if os.path.exists(os.path.join(dft_lib_dir, libxgboost_path)):
|
|
||||||
# The library is built by CMake directly
|
|
||||||
src = os.path.join(dft_lib_dir, libxgboost_path)
|
|
||||||
else:
|
|
||||||
# The library is built by setup.py
|
|
||||||
src = os.path.join(build_dir, libxgboost_path)
|
|
||||||
self.logger.info("Installing shared library: %s", src)
|
|
||||||
dst, _ = self.copy_file(src, dst)
|
|
||||||
outfiles.append(dst)
|
|
||||||
return outfiles
|
|
||||||
|
|
||||||
|
|
||||||
class Install(install.install): # pylint: disable=too-many-instance-attributes
|
|
||||||
"""An interface to install command, accepting XGBoost specific
|
|
||||||
arguments.
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
user_options = install.install.user_options + [
|
|
||||||
(k, v[0], v[1]) for k, v in USER_OPTIONS.items()
|
|
||||||
]
|
|
||||||
|
|
||||||
def initialize_options(self) -> None:
|
|
||||||
super().initialize_options()
|
|
||||||
self.use_openmp = 1
|
|
||||||
self.use_cuda = 0
|
|
||||||
self.use_nccl = 0
|
|
||||||
self.build_with_shared_nccl = 0
|
|
||||||
self.use_hip= 0
|
|
||||||
self.use_rccl = 0
|
|
||||||
self.hide_cxx_symbols = 1
|
|
||||||
|
|
||||||
self.use_hdfs = 0
|
|
||||||
self.use_azure = 0
|
|
||||||
self.use_s3 = 0
|
|
||||||
|
|
||||||
self.plugin_dense_parser = 0
|
|
||||||
|
|
||||||
self.use_system_libxgboost = 0
|
|
||||||
|
|
||||||
def run(self) -> None:
|
|
||||||
# setuptools will configure the options according to user supplied command line
|
|
||||||
# arguments, then here we propagate them into `USER_OPTIONS` for visibility to
|
|
||||||
# other sub-commands like `build_ext`.
|
|
||||||
for k, v in USER_OPTIONS.items():
|
|
||||||
arg = k.replace("-", "_")
|
|
||||||
if hasattr(self, arg):
|
|
||||||
USER_OPTIONS[k] = (v[0], v[1], getattr(self, arg))
|
|
||||||
super().run()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
# Supported commands:
|
|
||||||
# From internet:
|
|
||||||
# - pip install xgboost
|
|
||||||
# - pip install --no-binary :all: xgboost
|
|
||||||
|
|
||||||
# From source tree `xgboost/python-package`:
|
|
||||||
# - python setup.py build
|
|
||||||
# - python setup.py build_ext
|
|
||||||
# - python setup.py install
|
|
||||||
# - python setup.py sdist && pip install <sdist-name>
|
|
||||||
# - python setup.py bdist_wheel && pip install <wheel-name>
|
|
||||||
|
|
||||||
# When XGBoost is compiled directly with CMake:
|
|
||||||
# - pip install -e .
|
|
||||||
# - python setup.py develop # same as above
|
|
||||||
logging.basicConfig(level=logging.INFO)
|
|
||||||
|
|
||||||
with open(os.path.join(CURRENT_DIR, "README.rst"), encoding="utf-8") as fd:
|
|
||||||
description = fd.read()
|
|
||||||
with open(os.path.join(CURRENT_DIR, "xgboost/VERSION"), encoding="ascii") as fd:
|
|
||||||
version = fd.read().strip()
|
|
||||||
|
|
||||||
setup(
|
|
||||||
name="xgboost",
|
|
||||||
version=version,
|
|
||||||
description="XGBoost Python Package",
|
|
||||||
long_description=description,
|
|
||||||
long_description_content_type="text/x-rst",
|
|
||||||
install_requires=[
|
|
||||||
"numpy",
|
|
||||||
"scipy",
|
|
||||||
],
|
|
||||||
ext_modules=[CMakeExtension("libxgboost")],
|
|
||||||
# error: expected "str": "Type[Command]"
|
|
||||||
cmdclass={
|
|
||||||
"build_ext": BuildExt, # type: ignore
|
|
||||||
"sdist": Sdist, # type: ignore
|
|
||||||
"install_lib": InstallLib, # type: ignore
|
|
||||||
"install": Install, # type: ignore
|
|
||||||
},
|
|
||||||
extras_require={
|
|
||||||
"pandas": ["pandas"],
|
|
||||||
"scikit-learn": ["scikit-learn"],
|
|
||||||
"dask": ["dask", "pandas", "distributed"],
|
|
||||||
"datatable": ["datatable"],
|
|
||||||
"plotting": ["graphviz", "matplotlib"],
|
|
||||||
"pyspark": ["pyspark", "scikit-learn", "cloudpickle"],
|
|
||||||
},
|
|
||||||
maintainer="Hyunsu Cho",
|
|
||||||
maintainer_email="chohyu01@cs.washington.edu",
|
|
||||||
zip_safe=False,
|
|
||||||
packages=find_packages(),
|
|
||||||
include_package_data=True,
|
|
||||||
license="Apache-2.0",
|
|
||||||
classifiers=[
|
|
||||||
"License :: OSI Approved :: Apache Software License",
|
|
||||||
"Development Status :: 5 - Production/Stable",
|
|
||||||
"Operating System :: OS Independent",
|
|
||||||
"Programming Language :: Python",
|
|
||||||
"Programming Language :: Python :: 3",
|
|
||||||
"Programming Language :: Python :: 3.8",
|
|
||||||
"Programming Language :: Python :: 3.9",
|
|
||||||
"Programming Language :: Python :: 3.10",
|
|
||||||
],
|
|
||||||
python_requires=">=3.8",
|
|
||||||
url="https://github.com/dmlc/xgboost",
|
|
||||||
)
|
|
||||||
|
|
||||||
clean_up()
|
|
||||||
Loading…
x
Reference in New Issue
Block a user