Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

merge latest changes

Browse Source
This commit is contained in:
Hui Liu
2023-12-13 21:06:28 -08:00
parent c81731308c 9c56916fd7
commit 2d7ffbdf3d
194 changed files with 4859 additions and 2838 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
tests/ci_build/Dockerfile.gpu
View File

@@ -2,6 +2,7 @@ ARG CUDA_VERSION_ARG
FROM nvidia/cuda:$CUDA_VERSION_ARG-runtime-ubuntu22.04
ARG CUDA_VERSION_ARG
ARG RAPIDS_VERSION_ARG
ARG NCCL_VERSION_ARG
# Environment
ENV DEBIAN_FRONTEND noninteractive
@@ -23,7 +24,9 @@ RUN \
conda install -c conda-forge mamba && \
mamba create -n gpu_test -c rapidsai-nightly -c rapidsai -c nvidia -c conda-forge -c defaults \
python=3.10 cudf=$RAPIDS_VERSION_ARG* rmm=$RAPIDS_VERSION_ARG* cudatoolkit=$CUDA_VERSION_ARG \
dask dask-cuda=$RAPIDS_VERSION_ARG* dask-cudf=$RAPIDS_VERSION_ARG* cupy \
nccl>=$(cut -d "-" -f 1 << $NCCL_VERSION_ARG) \
dask \
dask-cuda=$RAPIDS_VERSION_ARG* dask-cudf=$RAPIDS_VERSION_ARG* cupy \
numpy pytest pytest-timeout scipy scikit-learn pandas matplotlib wheel python-kubernetes urllib3 graphviz hypothesis \
pyspark>=3.4.0 cloudpickle cuda-python && \
mamba clean --all && \

2
tests/ci_build/Dockerfile.gpu_build_centos7
View File

@@ -27,7 +27,7 @@ RUN \
wget -nv -nc https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
yum -y update && \
yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT} && \
yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} && \
rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm;
ENV PATH=/opt/mambaforge/bin:/usr/local/ninja:$PATH

21
tests/ci_build/Dockerfile.gpu_build_r_centos7
View File

@@ -1,6 +1,7 @@
ARG CUDA_VERSION_ARG
FROM nvcr.io/nvidia/cuda:$CUDA_VERSION_ARG-devel-centos7
ARG CUDA_VERSION_ARG
ARG R_VERSION_ARG
# Install all basic requirements
RUN \
@@ -11,26 +12,28 @@ RUN \
yum -y update && \
yum install -y tar unzip wget xz git which ninja-build readline-devel libX11-devel libXt-devel \
xorg-x11-server-devel openssl-devel zlib-devel bzip2-devel xz-devel \
pcre-devel libcurl-devel texlive-* \
pcre2-devel libcurl-devel texlive-* \
devtoolset-9-gcc devtoolset-9-binutils devtoolset-9-gcc-c++ \
devtoolset-9-gcc-gfortran devtoolset-9-libquadmath-devel \
devtoolset-9-runtime devtoolset-9-libstdc++-devel
ENV PATH=/opt/mambaforge/bin:/usr/local/ninja:/opt/software/packages/bin:/opt/R/3.3.0/bin:$PATH
ENV LD_LIBRARY_PATH=/opt/software/packages/lib:/opt/R/3.3.0/lib64:$LD_LIBRARY_PATH
ENV PATH=/opt/mambaforge/bin:/usr/local/ninja:/opt/software/packages/bin:/opt/R/$R_VERSION_ARG/bin:$PATH
ENV LD_LIBRARY_PATH=/opt/software/packages/lib:/opt/R/$R_VERSION_ARG/lib64:$LD_LIBRARY_PATH
ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc
ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/c++
ENV CPP=/opt/rh/devtoolset-9/root/usr/bin/cpp
ENV F77=/opt/rh/devtoolset-9/root/usr/bin/gfortran
ENV FC=/opt/rh/devtoolset-9/root/usr/bin/gfortran
# R 3.3.0
RUN \
wget -nv -nc https://cran.r-project.org/src/base/R-3/R-3.3.0.tar.gz && \
tar xf R-3.3.0.tar.gz && \
cd R-3.3.0 && \
./configure --prefix=/opt/R/3.3.0 --enable-R-shlib && \
wget -nv -nc https://cran.r-project.org/src/base/R-4/R-$R_VERSION_ARG.tar.gz && \
tar xf R-$R_VERSION_ARG.tar.gz && \
cd R-$R_VERSION_ARG && \
./configure --prefix=/opt/R/$R_VERSION_ARG --enable-R-shlib --with-pcrel && \
make -j$(nproc) && \
make install && \
make install
run \
# Python
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/22.11.1-2/Mambaforge-22.11.1-2-Linux-x86_64.sh && \
bash conda.sh -b -p /opt/mambaforge && \

20
tests/ci_build/conda_env/linux_sycl_test.yml Normal file
View File

@@ -0,0 +1,20 @@
name: linux_sycl_test
channels:
- conda-forge
- intel
dependencies:
- python=3.8
- cmake
- c-compiler
- cxx-compiler
- pip
- wheel
- numpy
- scipy
- scikit-learn
- pandas
- hypothesis>=6.46
- pytest
- pytest-timeout
- pytest-cov
- dpcpp_linux-64

9
tests/ci_build/lint_cpp.py
View File

@@ -134,7 +134,12 @@ def process(fname, allow_type):
def main():
parser = argparse.ArgumentParser(description="run cpp lint")
parser.add_argument("path", nargs="+", help="path to traverse")
parser.add_argument(
"path",
nargs="*",
help="Path to traverse",
default=["src", "include", os.path.join("R-package", "src"), "python-package", "plugin/sycl"],
)
parser.add_argument(
"--exclude_path",
nargs="+",
@@ -148,6 +153,8 @@ def main():
allow_type += CXX_SUFFIX
for path in args.path:
if not os.path.exists(path):
raise ValueError(f"Unknown path: {path}")
if os.path.isfile(path):
normpath = os.path.normpath(path)
if normpath not in excluded_paths:

1
tests/ci_build/lint_python.py
View File

@@ -33,6 +33,7 @@ class LintersPaths:
"tests/python-gpu/test_gpu_pickling.py",
"tests/python-gpu/test_gpu_eval_metrics.py",
"tests/python-gpu/test_gpu_with_sklearn.py",
"tests/python-sycl/test_sycl_prediction.py",
"tests/test_distributed/test_with_spark/",
"tests/test_distributed/test_gpu_with_spark/",
# demo

35
tests/ci_build/prune_libnccl.sh
View File

@@ -1,35 +0,0 @@
#!/usr/bin/env bash
set -e
rm -rf tmp_nccl
mkdir tmp_nccl
pushd tmp_nccl
set -x
cat << EOF > test.cu
int main(void) { return 0; }
EOF
cat << EOF > CMakeLists.txt
cmake_minimum_required(VERSION 3.18 FATAL_ERROR)
project(gencode_extractor CXX C)
cmake_policy(SET CMP0104 NEW)
set(CMAKE_CUDA_HOST_COMPILER \${CMAKE_CXX_COMPILER})
enable_language(CUDA)
include(../cmake/Utils.cmake)
compute_cmake_cuda_archs("")
add_library(test OBJECT test.cu)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
EOF
cmake . -GNinja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
gen_code=$(grep -o -- '--generate-code=\S*' compile_commands.json | paste -sd ' ')
nvprune ${gen_code} /usr/lib64/libnccl_static.a -o ../libnccl_static.a
popd
rm -rf tmp_nccl
set +x

40
tests/ci_build/rename_whl.py
View File

@@ -1,22 +1,10 @@
import os
import sys
from contextlib import contextmanager
@contextmanager
def cd(path):
path = os.path.normpath(path)
cwd = os.getcwd()
os.chdir(path)
print("cd " + path)
try:
yield path
finally:
os.chdir(cwd)
from test_utils import DirectoryExcursion
if len(sys.argv) != 4:
print('Usage: {} [wheel to rename] [commit id] [platform tag]'.format(sys.argv[0]))
print("Usage: {} [wheel to rename] [commit id] [platform tag]".format(sys.argv[0]))
sys.exit(1)
@@ -26,20 +14,26 @@ platform_tag = sys.argv[3]
dirname, basename = os.path.dirname(whl_path), os.path.basename(whl_path)
with cd(dirname):
tokens = basename.split('-')
with DirectoryExcursion(dirname):
tokens = basename.split("-")
assert len(tokens) == 5
version = tokens[1].split('+')[0]
keywords = {'pkg_name': tokens[0],
'version': version,
'commit_id': commit_id,
'platform_tag': platform_tag}
new_name = '{pkg_name}-{version}+{commit_id}-py3-none-{platform_tag}.whl'.format(**keywords)
print('Renaming {} to {}...'.format(basename, new_name))
version = tokens[1].split("+")[0]
keywords = {
"pkg_name": tokens[0],
"version": version,
"commit_id": commit_id,
"platform_tag": platform_tag,
}
new_name = "{pkg_name}-{version}+{commit_id}-py3-none-{platform_tag}.whl".format(
**keywords
)
print("Renaming {} to {}...".format(basename, new_name))
if os.path.isfile(new_name):
os.remove(new_name)
os.rename(basename, new_name)
filesize = os.path.getsize(new_name) / 1024 / 1024 # MB
print(f"Wheel size: {filesize}")
msg = f"Limit of wheel size set by PyPI is exceeded. {new_name}: {filesize}"
assert filesize <= 300, msg