Replace setup.py with pyproject.toml (#9021)
* Create pyproject.toml * Implement a custom build backend (see below) in packager directory. Build logic from setup.py has been refactored and migrated into the new backend. * Tested: pip wheel . (build wheel), python -m build --sdist . (source distribution)
This commit is contained in:
Philip Hyunsu Cho
committed by
GitHub
GitHub
parent
a7b3dd3176
commit
a5cd2412de
0
python-package/packager/__init__.py
Normal file
0
python-package/packager/__init__.py
Normal file
56
python-package/packager/build_config.py
Normal file
56
python-package/packager/build_config.py
Normal file
@@ -0,0 +1,56 @@
|
||||
"""Build configuration"""
|
||||
import dataclasses
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class BuildConfiguration: # pylint: disable=R0902
|
||||
"""Configurations use when building libxgboost"""
|
||||
|
||||
# Whether to hide C++ symbols in libxgboost.so
|
||||
hide_cxx_symbols: bool = True
|
||||
# Whether to enable OpenMP
|
||||
use_openmp: bool = True
|
||||
# Whether to enable CUDA
|
||||
use_cuda: bool = False
|
||||
# Whether to enable NCCL
|
||||
use_nccl: bool = False
|
||||
# Whether to enable HDFS
|
||||
use_hdfs: bool = False
|
||||
# Whether to enable Azure Storage
|
||||
use_azure: bool = False
|
||||
# Whether to enable AWS S3
|
||||
use_s3: bool = False
|
||||
# Whether to enable the dense parser plugin
|
||||
plugin_dense_parser: bool = False
|
||||
# Special option: See explanation below
|
||||
use_system_libxgboost: bool = False
|
||||
|
||||
def _set_config_setting(
|
||||
self, config_settings: Dict[str, Any], field_name: str
|
||||
) -> None:
|
||||
if field_name in config_settings:
|
||||
setattr(
|
||||
self,
|
||||
field_name,
|
||||
(config_settings[field_name].lower() in ["true", "1", "on"]),
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Field {field_name} is not a valid config_settings")
|
||||
|
||||
def update(self, config_settings: Optional[Dict[str, Any]]) -> None:
|
||||
"""Parse config_settings from Pip (or other PEP 517 frontend)"""
|
||||
if config_settings is not None:
|
||||
for field_name in [x.name for x in dataclasses.fields(self)]:
|
||||
self._set_config_setting(config_settings, field_name)
|
||||
|
||||
def get_cmake_args(self) -> List[str]:
|
||||
"""Convert build configuration to CMake args"""
|
||||
cmake_args = []
|
||||
for field_name in [x.name for x in dataclasses.fields(self)]:
|
||||
if field_name in ["use_system_libxgboost"]:
|
||||
continue
|
||||
cmake_option = field_name.upper()
|
||||
cmake_value = "ON" if getattr(self, field_name) is True else "OFF"
|
||||
cmake_args.append(f"-D{cmake_option}={cmake_value}")
|
||||
return cmake_args
|
||||
157
python-package/packager/nativelib.py
Normal file
157
python-package/packager/nativelib.py
Normal file
@@ -0,0 +1,157 @@
|
||||
"""
|
||||
Functions for building libxgboost
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from platform import system
|
||||
from typing import Optional
|
||||
|
||||
from .build_config import BuildConfiguration
|
||||
|
||||
|
||||
def _lib_name() -> str:
|
||||
"""Return platform dependent shared object name."""
|
||||
if system() in ["Linux", "OS400"] or system().upper().endswith("BSD"):
|
||||
name = "libxgboost.so"
|
||||
elif system() == "Darwin":
|
||||
name = "libxgboost.dylib"
|
||||
elif system() == "Windows":
|
||||
name = "xgboost.dll"
|
||||
else:
|
||||
raise NotImplementedError(f"System {system()} not supported")
|
||||
return name
|
||||
|
||||
|
||||
def build_libxgboost(
|
||||
cpp_src_dir: pathlib.Path,
|
||||
build_dir: pathlib.Path,
|
||||
build_config: BuildConfiguration,
|
||||
) -> pathlib.Path:
|
||||
"""Build libxgboost in a temporary directory and obtain the path to built libxgboost"""
|
||||
logger = logging.getLogger("xgboost.packager.build_libxgboost")
|
||||
|
||||
if not cpp_src_dir.is_dir():
|
||||
raise RuntimeError(f"Expected {cpp_src_dir} to be a directory")
|
||||
logger.info(
|
||||
"Building %s from the C++ source files in %s...", _lib_name(), str(cpp_src_dir)
|
||||
)
|
||||
|
||||
def _build(*, generator: str) -> None:
|
||||
cmake_cmd = [
|
||||
"cmake",
|
||||
str(cpp_src_dir),
|
||||
generator,
|
||||
"-DKEEP_BUILD_ARTIFACTS_IN_BINARY_DIR=ON",
|
||||
]
|
||||
cmake_cmd.extend(build_config.get_cmake_args())
|
||||
|
||||
# Flag for cross-compiling for Apple Silicon
|
||||
# We use environment variable because it's the only way to pass down custom flags
|
||||
# through the cibuildwheel package, which calls `pip wheel` command.
|
||||
if "CIBW_TARGET_OSX_ARM64" in os.environ:
|
||||
cmake_cmd.append("-DCMAKE_OSX_ARCHITECTURES=arm64")
|
||||
|
||||
logger.info("CMake args: %s", str(cmake_cmd))
|
||||
subprocess.check_call(cmake_cmd, cwd=build_dir)
|
||||
|
||||
if system() == "Windows":
|
||||
subprocess.check_call(
|
||||
["cmake", "--build", ".", "--config", "Release"], cwd=build_dir
|
||||
)
|
||||
else:
|
||||
nproc = os.cpu_count()
|
||||
assert build_tool is not None
|
||||
subprocess.check_call([build_tool, f"-j{nproc}"], cwd=build_dir)
|
||||
|
||||
if system() == "Windows":
|
||||
supported_generators = (
|
||||
"-GVisual Studio 17 2022",
|
||||
"-GVisual Studio 16 2019",
|
||||
"-GVisual Studio 15 2017",
|
||||
"-GMinGW Makefiles",
|
||||
)
|
||||
for generator in supported_generators:
|
||||
try:
|
||||
_build(generator=generator)
|
||||
logger.info(
|
||||
"Successfully built %s using generator %s", _lib_name(), generator
|
||||
)
|
||||
break
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.info(
|
||||
"Tried building with generator %s but failed with exception %s",
|
||||
generator,
|
||||
str(e),
|
||||
)
|
||||
# Empty build directory
|
||||
shutil.rmtree(build_dir)
|
||||
build_dir.mkdir()
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"None of the supported generators produced a successful build!"
|
||||
f"Supported generators: {supported_generators}"
|
||||
)
|
||||
else:
|
||||
build_tool = "ninja" if shutil.which("ninja") else "make"
|
||||
generator = "-GNinja" if build_tool == "ninja" else "-GUnix Makefiles"
|
||||
try:
|
||||
_build(generator=generator)
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.info("Failed to build with OpenMP. Exception: %s", str(e))
|
||||
build_config.use_openmp = False
|
||||
_build(generator=generator)
|
||||
|
||||
return build_dir / "lib" / _lib_name()
|
||||
|
||||
|
||||
def locate_local_libxgboost(
|
||||
toplevel_dir: pathlib.Path,
|
||||
logger: logging.Logger,
|
||||
) -> Optional[pathlib.Path]:
|
||||
"""
|
||||
Locate libxgboost from the local project directory's lib/ subdirectory.
|
||||
"""
|
||||
libxgboost = toplevel_dir.parent / "lib" / _lib_name()
|
||||
if libxgboost.exists():
|
||||
logger.info("Found %s at %s", libxgboost.name, str(libxgboost.parent))
|
||||
return libxgboost
|
||||
return None
|
||||
|
||||
|
||||
def locate_or_build_libxgboost(
|
||||
toplevel_dir: pathlib.Path,
|
||||
build_dir: pathlib.Path,
|
||||
build_config: BuildConfiguration,
|
||||
) -> pathlib.Path:
|
||||
"""Locate libxgboost; if not exist, build it"""
|
||||
logger = logging.getLogger("xgboost.packager.locate_or_build_libxgboost")
|
||||
|
||||
libxgboost = locate_local_libxgboost(toplevel_dir, logger=logger)
|
||||
if libxgboost is not None:
|
||||
return libxgboost
|
||||
if build_config.use_system_libxgboost:
|
||||
# Find libxgboost from system prefix
|
||||
sys_prefix = pathlib.Path(sys.prefix).absolute().resolve()
|
||||
libxgboost = sys_prefix / "lib" / _lib_name()
|
||||
if not libxgboost.exists():
|
||||
raise RuntimeError(
|
||||
f"use_system_libxgboost was specified but {_lib_name()} is "
|
||||
f"not found in {libxgboost.parent}"
|
||||
)
|
||||
|
||||
logger.info("Using system XGBoost: %s", str(libxgboost))
|
||||
return libxgboost
|
||||
|
||||
if toplevel_dir.joinpath("cpp_src").exists():
|
||||
# Source distribution; all C++ source files to be found in cpp_src/
|
||||
cpp_src_dir = toplevel_dir.joinpath("cpp_src")
|
||||
else:
|
||||
# Probably running "pip install ." from python-package/
|
||||
cpp_src_dir = toplevel_dir.parent
|
||||
if not cpp_src_dir.joinpath("CMakeLists.txt").exists():
|
||||
raise RuntimeError(f"Did not find CMakeLists.txt from {cpp_src_dir}")
|
||||
return build_libxgboost(cpp_src_dir, build_dir=build_dir, build_config=build_config)
|
||||
157
python-package/packager/pep517.py
Normal file
157
python-package/packager/pep517.py
Normal file
@@ -0,0 +1,157 @@
|
||||
"""
|
||||
Custom build backend for XGBoost Python package.
|
||||
Builds source distribution and binary wheels, following PEP 517 / PEP 660.
|
||||
Reuses components of Hatchling (https://github.com/pypa/hatch/tree/master/backend) for the sake
|
||||
of brevity.
|
||||
"""
|
||||
import dataclasses
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Dict, Iterator, Optional, Union
|
||||
|
||||
import hatchling.build
|
||||
|
||||
from .build_config import BuildConfiguration
|
||||
from .nativelib import locate_local_libxgboost, locate_or_build_libxgboost
|
||||
from .sdist import copy_cpp_src_tree
|
||||
from .util import copy_with_logging, copytree_with_logging
|
||||
|
||||
|
||||
@contextmanager
|
||||
def cd(path: Union[str, pathlib.Path]) -> Iterator[str]: # pylint: disable=C0103
|
||||
"""
|
||||
Temporarily change working directory.
|
||||
TODO(hcho3): Remove this once we adopt Python 3.11, which implements contextlib.chdir.
|
||||
"""
|
||||
path = str(path)
|
||||
path = os.path.realpath(path)
|
||||
cwd = os.getcwd()
|
||||
os.chdir(path)
|
||||
try:
|
||||
yield path
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
|
||||
|
||||
TOPLEVEL_DIR = pathlib.Path(__file__).parent.parent.absolute().resolve()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
|
||||
# Aliases
|
||||
get_requires_for_build_sdist = hatchling.build.get_requires_for_build_sdist
|
||||
get_requires_for_build_wheel = hatchling.build.get_requires_for_build_wheel
|
||||
get_requires_for_build_editable = hatchling.build.get_requires_for_build_editable
|
||||
|
||||
|
||||
def build_wheel(
|
||||
wheel_directory: str,
|
||||
config_settings: Optional[Dict[str, Any]] = None,
|
||||
metadata_directory: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Build a wheel"""
|
||||
logger = logging.getLogger("xgboost.packager.build_wheel")
|
||||
|
||||
build_config = BuildConfiguration()
|
||||
build_config.update(config_settings)
|
||||
logger.info("Parsed build configuration: %s", dataclasses.asdict(build_config))
|
||||
|
||||
# Create tempdir with Python package + libxgboost
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
td_path = pathlib.Path(td)
|
||||
build_dir = td_path / "libbuild"
|
||||
build_dir.mkdir()
|
||||
|
||||
workspace = td_path / "whl_workspace"
|
||||
workspace.mkdir()
|
||||
logger.info("Copying project files to temporary directory %s", str(workspace))
|
||||
|
||||
copy_with_logging(TOPLEVEL_DIR / "pyproject.toml", workspace, logger=logger)
|
||||
copy_with_logging(TOPLEVEL_DIR / "hatch_build.py", workspace, logger=logger)
|
||||
copy_with_logging(TOPLEVEL_DIR / "README.rst", workspace, logger=logger)
|
||||
|
||||
pkg_path = workspace / "xgboost"
|
||||
copytree_with_logging(TOPLEVEL_DIR / "xgboost", pkg_path, logger=logger)
|
||||
lib_path = pkg_path / "lib"
|
||||
lib_path.mkdir()
|
||||
libxgboost = locate_or_build_libxgboost(
|
||||
TOPLEVEL_DIR, build_dir=build_dir, build_config=build_config
|
||||
)
|
||||
copy_with_logging(libxgboost, lib_path, logger=logger)
|
||||
|
||||
with cd(workspace):
|
||||
wheel_name = hatchling.build.build_wheel(
|
||||
wheel_directory, config_settings, metadata_directory
|
||||
)
|
||||
return wheel_name
|
||||
|
||||
|
||||
def build_sdist(
|
||||
sdist_directory: str,
|
||||
config_settings: Optional[Dict[str, Any]] = None,
|
||||
) -> str:
|
||||
"""Build a source distribution"""
|
||||
logger = logging.getLogger("xgboost.packager.build_sdist")
|
||||
|
||||
if config_settings:
|
||||
raise NotImplementedError(
|
||||
"XGBoost's custom build backend doesn't support config_settings option "
|
||||
f"when building sdist. {config_settings=}"
|
||||
)
|
||||
|
||||
cpp_src_dir = TOPLEVEL_DIR.parent
|
||||
if not cpp_src_dir.joinpath("CMakeLists.txt").exists():
|
||||
raise RuntimeError(f"Did not find CMakeLists.txt from {cpp_src_dir}")
|
||||
|
||||
# Create tempdir with Python package + C++ sources
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
td_path = pathlib.Path(td)
|
||||
|
||||
workspace = td_path / "sdist_workspace"
|
||||
workspace.mkdir()
|
||||
logger.info("Copying project files to temporary directory %s", str(workspace))
|
||||
|
||||
copy_with_logging(TOPLEVEL_DIR / "pyproject.toml", workspace, logger=logger)
|
||||
copy_with_logging(TOPLEVEL_DIR / "hatch_build.py", workspace, logger=logger)
|
||||
copy_with_logging(TOPLEVEL_DIR / "README.rst", workspace, logger=logger)
|
||||
|
||||
copytree_with_logging(
|
||||
TOPLEVEL_DIR / "xgboost", workspace / "xgboost", logger=logger
|
||||
)
|
||||
copytree_with_logging(
|
||||
TOPLEVEL_DIR / "packager", workspace / "packager", logger=logger
|
||||
)
|
||||
|
||||
temp_cpp_src_dir = workspace / "cpp_src"
|
||||
copy_cpp_src_tree(cpp_src_dir, target_dir=temp_cpp_src_dir, logger=logger)
|
||||
|
||||
with cd(workspace):
|
||||
sdist_name = hatchling.build.build_sdist(sdist_directory, config_settings)
|
||||
return sdist_name
|
||||
|
||||
|
||||
def build_editable(
|
||||
wheel_directory: str,
|
||||
config_settings: Optional[Dict[str, Any]] = None,
|
||||
metadata_directory: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Build an editable installation. We mostly delegate to Hatchling."""
|
||||
logger = logging.getLogger("xgboost.packager.build_editable")
|
||||
|
||||
if config_settings:
|
||||
raise NotImplementedError(
|
||||
"XGBoost's custom build backend doesn't support config_settings option "
|
||||
f"when building editable installation. {config_settings=}"
|
||||
)
|
||||
|
||||
if locate_local_libxgboost(TOPLEVEL_DIR, logger=logger) is None:
|
||||
raise RuntimeError(
|
||||
"To use the editable installation, first build libxgboost with CMake. "
|
||||
"See https://xgboost.readthedocs.io/en/latest/build.html for detailed instructions."
|
||||
)
|
||||
|
||||
return hatchling.build.build_editable(
|
||||
wheel_directory, config_settings, metadata_directory
|
||||
)
|
||||
27
python-package/packager/sdist.py
Normal file
27
python-package/packager/sdist.py
Normal file
@@ -0,0 +1,27 @@
|
||||
"""
|
||||
Functions for building sdist
|
||||
"""
|
||||
import logging
|
||||
import pathlib
|
||||
|
||||
from .util import copy_with_logging, copytree_with_logging
|
||||
|
||||
|
||||
def copy_cpp_src_tree(
|
||||
cpp_src_dir: pathlib.Path, target_dir: pathlib.Path, logger: logging.Logger
|
||||
) -> None:
|
||||
"""Copy C++ source tree into build directory"""
|
||||
|
||||
for subdir in [
|
||||
"src",
|
||||
"include",
|
||||
"dmlc-core",
|
||||
"gputreeshap",
|
||||
"rabit",
|
||||
"cmake",
|
||||
"plugin",
|
||||
]:
|
||||
copytree_with_logging(cpp_src_dir / subdir, target_dir / subdir, logger=logger)
|
||||
|
||||
for filename in ["CMakeLists.txt", "LICENSE"]:
|
||||
copy_with_logging(cpp_src_dir.joinpath(filename), target_dir, logger=logger)
|
||||
25
python-package/packager/util.py
Normal file
25
python-package/packager/util.py
Normal file
@@ -0,0 +1,25 @@
|
||||
"""
|
||||
Utility functions for implementing PEP 517 backend
|
||||
"""
|
||||
import logging
|
||||
import pathlib
|
||||
import shutil
|
||||
|
||||
|
||||
def copytree_with_logging(
|
||||
src: pathlib.Path, dest: pathlib.Path, logger: logging.Logger
|
||||
) -> None:
|
||||
"""Call shutil.copytree() with logging"""
|
||||
logger.info("Copying %s -> %s", str(src), str(dest))
|
||||
shutil.copytree(src, dest)
|
||||
|
||||
|
||||
def copy_with_logging(
|
||||
src: pathlib.Path, dest: pathlib.Path, logger: logging.Logger
|
||||
) -> None:
|
||||
"""Call shutil.copy() with logging"""
|
||||
if dest.is_dir():
|
||||
logger.info("Copying %s -> %s", str(src), str(dest / src.name))
|
||||
else:
|
||||
logger.info("Copying %s -> %s", str(src), str(dest))
|
||||
shutil.copy(src, dest)
|
||||
Reference in New Issue
Block a user