Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

temp merge, disable 1 line, SetValid

Browse Source
This commit is contained in:
Your Name
2023-10-12 16:16:44 -07:00
parent 2e7e9d3b2d 85d3017ca5
commit ea19555474
492 changed files with 15533 additions and 9376 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
tests/ci_build/Dockerfile.gpu_build_centos7
View File

@@ -2,6 +2,7 @@ ARG CUDA_VERSION_ARG
FROM nvidia/cuda:$CUDA_VERSION_ARG-devel-centos7
ARG CUDA_VERSION_ARG
ARG NCCL_VERSION_ARG
ARG RAPIDS_VERSION_ARG
# Install all basic requirements
RUN \
@@ -16,8 +17,8 @@ RUN \
bash conda.sh -b -p /opt/mambaforge && \
/opt/mambaforge/bin/python -m pip install awscli && \
# CMake
wget -nv -nc https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.sh --no-check-certificate && \
bash cmake-3.18.0-Linux-x86_64.sh --skip-license --prefix=/usr
wget -nv -nc https://cmake.org/files/v3.26/cmake-3.26.4-linux-x86_64.sh --no-check-certificate && \
bash cmake-3.26.4-linux-x86_64.sh --skip-license --prefix=/usr
# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
RUN \
@@ -33,9 +34,21 @@ ENV PATH=/opt/mambaforge/bin:/usr/local/ninja:$PATH
ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc
ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/c++
ENV CPP=/opt/rh/devtoolset-9/root/usr/bin/cpp
ENV CUDAHOSTCXX=/opt/rh/devtoolset-9/root/usr/bin/c++
ENV GOSU_VERSION 1.10
# Install RMM
RUN git clone -b v${RAPIDS_VERSION_ARG}.00 https://github.com/rapidsai/rmm.git --recurse-submodules --depth 1 && \
pushd rmm && \
mkdir build && \
pushd build && \
cmake .. -GNinja -DCMAKE_INSTALL_PREFIX=/opt/rmm -DCUDA_STATIC_RUNTIME=ON && \
cmake --build . --target install && \
popd && \
popd && \
rm -rf rmm
# Install gRPC
RUN git clone -b v1.49.1 https://github.com/grpc/grpc.git \
--recurse-submodules --depth 1 && \

49
tests/ci_build/Dockerfile.rmm
View File

@@ -1,49 +0,0 @@
ARG CUDA_VERSION_ARG
FROM nvidia/cuda:$CUDA_VERSION_ARG-devel-ubuntu20.04
ARG CUDA_VERSION_ARG
ARG RAPIDS_VERSION_ARG
ARG NCCL_VERSION_ARG
# Environment
ENV DEBIAN_FRONTEND noninteractive
SHELL ["/bin/bash", "-c"] # Use Bash as shell
# Install all basic requirements
RUN \
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub && \
apt-get update && \
apt-get install -y wget unzip bzip2 libgomp1 build-essential ninja-build git && \
# Python
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/22.11.1-2/Mambaforge-22.11.1-2-Linux-x86_64.sh && \
bash conda.sh -b -p /opt/mambaforge
# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
RUN \
export CUDA_SHORT=`echo $CUDA_VERSION_ARG | grep -o -E '[0-9]+\.[0-9]'` && \
export NCCL_VERSION=$NCCL_VERSION_ARG && \
apt-get update && \
apt-get install -y --allow-downgrades --allow-change-held-packages libnccl2=${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-dev=${NCCL_VERSION}+cuda${CUDA_SHORT}
ENV PATH=/opt/mambaforge/bin:$PATH
# Create new Conda environment with RMM
RUN \
conda install -c conda-forge mamba && \
mamba create -n gpu_test -c rapidsai-nightly -c rapidsai -c nvidia -c conda-forge -c defaults \
python=3.10 rmm=$RAPIDS_VERSION_ARG* cudatoolkit=$CUDA_VERSION_ARG cmake && \
mamba clean --all
ENV GOSU_VERSION 1.10
# Install lightweight sudo (not bound to TTY)
RUN set -ex; \
wget -nv -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
chmod +x /usr/local/bin/gosu && \
gosu nobody true
# Default entry-point to use if running locally
# It will preserve attributes of created files
COPY entrypoint.sh /scripts/
WORKDIR /workspace
ENTRYPOINT ["/scripts/entrypoint.sh"]

2
tests/ci_build/build_python_wheels.sh
View File

@@ -35,7 +35,7 @@ if [[ "$platform_id" == macosx_* ]]; then
# MacOS, Intel
wheel_tag=macosx_10_15_x86_64.macosx_11_0_x86_64.macosx_12_0_x86_64
cpython_ver=38
export MACOSX_DEPLOYMENT_TARGET=10.13
export MACOSX_DEPLOYMENT_TARGET=10.15
#OPENMP_URL="https://anaconda.org/conda-forge/llvm-openmp/11.1.0/download/osx-64/llvm-openmp-11.1.0-hda6cdc1_1.tar.bz2"
OPENMP_URL="https://xgboost-ci-jenkins-artifacts.s3.us-west-2.amazonaws.com/llvm-openmp-11.1.0-hda6cdc1_1-osx-64.tar.bz2"
else

1
tests/ci_build/build_r_pkg_with_cuda.sh
View File

@@ -19,7 +19,6 @@ cmake .. -GNinja -DUSE_CUDA=ON -DR_LIB=ON
ninja
cd ..
rm xgboost
# This super wacky hack is found in cmake/RPackageInstall.cmake.in and
# cmake/RPackageInstallTargetSetup.cmake. This hack lets us bypass the normal build process of R
# and have R use xgboost.so that we've already built.

1
tests/ci_build/build_r_pkg_with_cuda_win64.sh
View File

@@ -22,7 +22,6 @@ cmake .. -G"Visual Studio 17 2022" -A x64 -DUSE_CUDA=ON -DR_LIB=ON -DLIBR_HOME="
cmake --build . --config Release --parallel
cd ..
rm xgboost
# This super wacky hack is found in cmake/RPackageInstall.cmake.in and
# cmake/RPackageInstallTargetSetup.cmake. This hack lets us bypass the normal build process of R
# and have R use xgboost.dll that we've already built.

2
tests/ci_build/build_via_cmake.sh
View File

@@ -24,7 +24,7 @@ fi
rm -rf build
mkdir build
cd build
cmake .. ${cmake_args} -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_ALL_WARNINGS=ON -GNinja ${cmake_prefix_flag} -DHIDE_CXX_SYMBOLS=ON
cmake .. ${cmake_args} -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_ALL_WARNINGS=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=ON -GNinja ${cmake_prefix_flag} -DHIDE_CXX_SYMBOLS=ON -DBUILD_DEPRECATED_CLI=ON
ninja clean
time ninja -v
cd ..

17
tests/ci_build/change_version.py
View File

@@ -61,8 +61,11 @@ def pypkg(
@cd(R_PACKAGE)
def rpkg(major: int, minor: int, patch: int) -> None:
version = f"{major}.{minor}.{patch}.1"
def rpkg(major: int, minor: int, patch: int, is_dev: bool) -> None:
if is_dev:
version = f"{major}.{minor}.{patch}.0"
else:
version = f"{major}.{minor}.{patch}.1"
# Version: 2.0.0.1
desc_path = "DESCRIPTION"
with open(desc_path, "r") as fd:
@@ -119,8 +122,8 @@ def main(args: argparse.Namespace) -> None:
minor = args.minor
patch = args.patch
rc = args.rc
is_rc = args.is_rc == 1
is_dev = args.is_dev == 1
is_rc = args.is_rc
is_dev = args.is_dev
if is_rc and is_dev:
raise ValueError("It cannot be both a rc and a dev branch.")
if is_rc:
@@ -130,7 +133,7 @@ def main(args: argparse.Namespace) -> None:
cmake(major, minor, patch)
pypkg(major, minor, patch, rc, is_rc, is_dev)
rpkg(major, minor, patch)
rpkg(major, minor, patch, is_dev=is_dev)
jvmpkgs(major, minor, patch, rc, is_rc, is_dev)
print(
@@ -149,8 +152,8 @@ if __name__ == "__main__":
parser.add_argument("--minor", type=int)
parser.add_argument("--patch", type=int)
parser.add_argument("--rc", type=int, default=0)
parser.add_argument("--is-rc", type=int, choices=[0, 1])
parser.add_argument("--is-dev", type=int, choices=[0, 1])
parser.add_argument("--is-rc", action="store_true")
parser.add_argument("--is-dev", action="store_true")
args = parser.parse_args()
try:
main(args)

23
tests/ci_build/lint_python.py
View File

@@ -23,20 +23,33 @@ class LintersPaths:
"tests/python/test_predict.py",
"tests/python/test_quantile_dmatrix.py",
"tests/python/test_tree_regularization.py",
"tests/python/test_shap.py",
"tests/python-gpu/test_gpu_data_iterator.py",
"tests/python-gpu/test_gpu_prediction.py",
"tests/python-gpu/load_pickle.py",
"tests/python-gpu/test_gpu_pickling.py",
"tests/python-gpu/test_gpu_eval_metrics.py",
"tests/python-gpu/test_gpu_with_sklearn.py",
"tests/test_distributed/test_with_spark/",
"tests/test_distributed/test_gpu_with_spark/",
# demo
"demo/dask/",
"demo/rmm_plugin",
"demo/json-model/json_parser.py",
"demo/guide-python/cat_in_the_dat.py",
"demo/guide-python/callbacks.py",
"demo/guide-python/categorical.py",
"demo/guide-python/feature_weights.py",
"demo/guide-python/sklearn_parallel.py",
"demo/guide-python/spark_estimator_examples.py",
"demo/guide-python/external_memory.py",
"demo/guide-python/individual_trees.py",
"demo/guide-python/quantile_regression.py",
"demo/guide-python/multioutput_regression.py",
"demo/guide-python/learning_to_rank.py",
"demo/guide-python/quantile_data_iterator.py",
"demo/guide-python/update_process.py",
"demo/aft_survival/aft_survival_viz_demo.py",
# CI
"tests/ci_build/lint_python.py",
"tests/ci_build/test_r_package.py",
@@ -66,6 +79,7 @@ class LintersPaths:
"tests/python/test_dt.py",
"tests/python/test_data_iterator.py",
"tests/python-gpu/test_gpu_data_iterator.py",
"tests/python-gpu/load_pickle.py",
"tests/test_distributed/test_with_spark/test_data.py",
"tests/test_distributed/test_gpu_with_spark/test_data.py",
"tests/test_distributed/test_gpu_with_dask/test_gpu_with_dask.py",
@@ -78,6 +92,7 @@ class LintersPaths:
"demo/guide-python/quantile_regression.py",
"demo/guide-python/multioutput_regression.py",
"demo/guide-python/learning_to_rank.py",
"demo/aft_survival/aft_survival_viz_demo.py",
# CI
"tests/ci_build/lint_python.py",
"tests/ci_build/test_r_package.py",
@@ -114,7 +129,13 @@ def run_black(rel_path: str, fix: bool) -> bool:
@cd(PY_PACKAGE)
def run_isort(rel_path: str, fix: bool) -> bool:
# Isort gets confused when trying to find the config file, so specified explicitly.
cmd = ["isort", "--settings-path", PY_PACKAGE, os.path.join(ROOT, rel_path)]
cmd = [
"isort",
"--settings-path",
PY_PACKAGE,
f"--src={PY_PACKAGE}",
os.path.join(ROOT, rel_path),
]
if not fix:
cmd += ["--check"]

15
tests/ci_build/lint_r.R
View File

@@ -20,15 +20,23 @@ my_linters <- list(
any_duplicated = lintr::any_duplicated_linter(),
any_is_na = lintr::any_is_na_linter(),
assignment_linter = lintr::assignment_linter(),
boolean_arithmetic = lintr::boolean_arithmetic_linter(),
brace_linter = lintr::brace_linter(),
class_equals = lintr::class_equals_linter(),
commas_linter = lintr::commas_linter(),
empty_assignment = lintr::empty_assignment_linter(),
equals_na = lintr::equals_na_linter(),
fixed_regex = lintr::fixed_regex_linter(),
for_loop_index = lintr::for_loop_index_linter(),
function_return = lintr::function_return_linter(),
infix_spaces_linter = lintr::infix_spaces_linter(),
is_numeric = lintr::is_numeric_linter(),
line_length_linter = lintr::line_length_linter(length = 150L),
no_tab_linter = lintr::no_tab_linter(),
lengths = lintr::lengths_linter(),
matrix = lintr::matrix_apply_linter(),
object_usage_linter = lintr::object_usage_linter(),
object_length_linter = lintr::object_length_linter(),
routine_registration = lintr::routine_registration_linter(),
semicolon = lintr::semicolon_linter(),
seq = lintr::seq_linter(),
spaces_inside_linter = lintr::spaces_inside_linter(),
@@ -37,9 +45,10 @@ my_linters <- list(
trailing_blank_lines_linter = lintr::trailing_blank_lines_linter(),
trailing_whitespace_linter = lintr::trailing_whitespace_linter(),
true_false = lintr::T_and_F_symbol_linter(),
unneeded_concatenation = lintr::unneeded_concatenation_linter(),
unnecessary_concatenation = lintr::unnecessary_concatenation_linter(),
unreachable_code = lintr::unreachable_code_linter(),
vector_logic = lintr::vector_logic_linter()
vector_logic = lintr::vector_logic_linter(),
whitespace = lintr::whitespace_linter()
)
noquote(paste0(length(FILES_TO_LINT), " R files need linting"))

2
tests/ci_build/prune_libnccl.sh
View File

@@ -26,7 +26,7 @@ set_property(TARGET test PROPERTY CUDA_ARCHITECTURES \${CMAKE_CUDA_ARCHITECTURES
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
EOF
cmake . -GNinja
cmake . -GNinja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
gen_code=$(grep -o -- '--generate-code=\S*' compile_commands.json | paste -sd ' ')
nvprune ${gen_code} /usr/lib64/libnccl_static.a -o ../libnccl_static.a