merge latest, Jan 12 2024

2024-01-12 09:57:11 -08:00
parent c42c7d99f1 73b3955dd4
commit 1e1e8be3a5
251 changed files with 9023 additions and 5012 deletions
--- a/tests/ci_build/build_r_pkg_with_cuda_win64.sh
+++ b/tests/ci_build/build_r_pkg_with_cuda_win64.sh
@@ -18,7 +18,7 @@ mv xgboost/ xgboost_rpack/

 mkdir build
 cd build
-cmake .. -G"Visual Studio 17 2022" -A x64 -DUSE_CUDA=ON -DR_LIB=ON -DLIBR_HOME="c:\\Program Files\\R\\R-3.6.3"
+cmake .. -G"Visual Studio 17 2022" -A x64 -DUSE_CUDA=ON -DR_LIB=ON -DLIBR_HOME="c:\\Program Files\\R\\R-4.3.2" -DCMAKE_PREFIX_PATH="C:\\rtools43\\x86_64-w64-mingw32.static.posix\\bin"
 cmake --build . --config Release --parallel
 cd ..

@@ -32,5 +32,5 @@ cp -v lib/xgboost.dll xgboost_rpack/src/
 echo 'all:' > xgboost_rpack/src/Makefile
 echo 'all:' > xgboost_rpack/src/Makefile.win
 mv xgboost_rpack/ xgboost/
-/c/Rtools/bin/tar -cvf xgboost_r_gpu_win64_${commit_hash}.tar xgboost/
-/c/Rtools/bin/gzip -9c xgboost_r_gpu_win64_${commit_hash}.tar > xgboost_r_gpu_win64_${commit_hash}.tar.gz
+/c/Rtools43/usr/bin/tar -cvf xgboost_r_gpu_win64_${commit_hash}.tar xgboost/
+/c/Rtools43/usr/bin/gzip -9c xgboost_r_gpu_win64_${commit_hash}.tar > xgboost_r_gpu_win64_${commit_hash}.tar.gz
--- a/tests/ci_build/ci_build.sh
+++ b/tests/ci_build/ci_build.sh
@@ -2,14 +2,14 @@
 #
 # Execute command within a docker container
 #
-# Usage: ci_build.sh <CONTAINER_TYPE> <DOCKER_BINARY>
+# Usage: ci_build.sh <CONTAINER_TYPE> [--use-gpus]
 #                    [--dockerfile <DOCKERFILE_PATH>] [-it]
 #                    [--build-arg <BUILD_ARG>] <COMMAND>
 #
 # CONTAINER_TYPE: Type of the docker container used the run the build: e.g.,
 #                 (cpu | gpu)
 #
-# DOCKER_BINARY: Command to invoke docker, e.g. (docker | nvidia-docker).
+# --use-gpus: Whether to grant the container access to NVIDIA GPUs.
 #
 # DOCKERFILE_PATH: (Optional) Path to the Dockerfile used for docker build.  If
 #                  this optional value is not supplied (via the --dockerfile
@@ -29,9 +29,12 @@ shift 1
 DOCKERFILE_PATH="${SCRIPT_DIR}/Dockerfile.${CONTAINER_TYPE}"
 DOCKER_CONTEXT_PATH="${SCRIPT_DIR}"

-# Get docker binary command (should be either docker or nvidia-docker)
-DOCKER_BINARY="$1"
-shift 1
+GPU_FLAG=''
+if [[ "$1" == "--use-gpus" ]]; then
+    echo "Using NVIDIA GPUs"
+    GPU_FLAG='--gpus all'
+    shift 1
+fi

 if [[ "$1" == "--dockerfile" ]]; then
    DOCKERFILE_PATH="$2"
@@ -144,21 +147,21 @@ then
    DOCKER_CACHE_REPO="${DOCKER_CACHE_ECR_ID}.dkr.ecr.${DOCKER_CACHE_ECR_REGION}.amazonaws.com"
    echo "Using AWS ECR; repo URL = ${DOCKER_CACHE_REPO}"
    # Login for Docker registry
-    echo "\$(aws ecr get-login --no-include-email --region ${DOCKER_CACHE_ECR_REGION} --registry-ids ${DOCKER_CACHE_ECR_ID})"
-    $(aws ecr get-login --no-include-email --region ${DOCKER_CACHE_ECR_REGION} --registry-ids ${DOCKER_CACHE_ECR_ID})
+    echo "aws ecr get-login-password --region ${DOCKER_CACHE_ECR_REGION} | docker login --username AWS --password-stdin ${DOCKER_CACHE_REPO}"
+    aws ecr get-login-password --region ${DOCKER_CACHE_ECR_REGION} | docker login --username AWS --password-stdin ${DOCKER_CACHE_REPO}
    # Pull pre-build container from Docker build cache,
    # if one exists for the particular branch or pull request
    DOCKER_TAG="${BRANCH_NAME//\//-}"  # Slashes are not allow in Docker tag
    echo "docker pull --quiet ${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:${DOCKER_TAG}"
    if time docker pull --quiet "${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:${DOCKER_TAG}"
    then
-      CACHE_FROM_CMD="--cache-from ${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:${DOCKER_TAG}"
+      CACHE_FROM_CMD="--cache-from ${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:${DOCKER_TAG} --build-arg BUILDKIT_INLINE_CACHE=1"
    else
      # If the build cache is empty of the particular branch or pull request,
      # use the build cache associated with the master branch
      echo "docker pull --quiet ${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:master"
      docker pull --quiet "${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:master" || true
-      CACHE_FROM_CMD="--cache-from ${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:master"
+      CACHE_FROM_CMD="--cache-from ${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:master --build-arg BUILDKIT_INLINE_CACHE=1"
    fi
 else
    CACHE_FROM_CMD=''
@@ -166,11 +169,15 @@ fi

 echo "docker build \
    ${CI_DOCKER_BUILD_ARG} \
+    --progress=plain \
+    --ulimit nofile=1024000:1024000 \
    -t ${DOCKER_IMG_NAME} \
    -f ${DOCKERFILE_PATH} ${DOCKER_CONTEXT_PATH} \
    ${CACHE_FROM_CMD}"
 docker build \
    ${CI_DOCKER_BUILD_ARG} \
+    --progress=plain \
+    --ulimit nofile=1024000:1024000 \
    -t "${DOCKER_IMG_NAME}" \
    -f "${DOCKERFILE_PATH}" "${DOCKER_CONTEXT_PATH}" \
    ${CACHE_FROM_CMD}
@@ -231,7 +238,8 @@ echo "Running '${COMMAND[*]}' inside ${DOCKER_IMG_NAME}..."
 # and share the PID namespace (--pid=host) so the process inside does not have
 # pid 1 and SIGKILL is propagated to the process inside (jenkins can kill it).
 set -x
-${DOCKER_BINARY} run --rm --pid=host \
+docker run --rm --pid=host \
+    ${GPU_FLAG} \
    -v "${WORKSPACE}":/workspace \
    -w /workspace \
    ${USER_IDS} \
--- a/tests/ci_build/lint_python.py
+++ b/tests/ci_build/lint_python.py
@@ -22,18 +22,17 @@ class LintersPaths:
        "tests/python/test_dmatrix.py",
        "tests/python/test_dt.py",
        "tests/python/test_demos.py",
+        "tests/python/test_multi_target.py",
        "tests/python/test_predict.py",
        "tests/python/test_quantile_dmatrix.py",
        "tests/python/test_tree_regularization.py",
        "tests/python/test_shap.py",
+        "tests/python/test_model_io.py",
        "tests/python/test_with_pandas.py",
-        "tests/python-gpu/test_gpu_data_iterator.py",
-        "tests/python-gpu/test_gpu_prediction.py",
-        "tests/python-gpu/load_pickle.py",
-        "tests/python-gpu/test_gpu_pickling.py",
-        "tests/python-gpu/test_gpu_eval_metrics.py",
-        "tests/python-gpu/test_gpu_with_sklearn.py",
-        "tests/python-sycl/test_sycl_prediction.py",
+        "tests/python-gpu/",
+        "tests/python-sycl/",
+        "tests/test_distributed/test_with_dask/",
+        "tests/test_distributed/test_gpu_with_dask/",
        "tests/test_distributed/test_with_spark/",
        "tests/test_distributed/test_gpu_with_spark/",
        # demo
@@ -84,14 +83,17 @@ class LintersPaths:
        "tests/python/test_dt.py",
        "tests/python/test_demos.py",
        "tests/python/test_data_iterator.py",
+        "tests/python/test_multi_target.py",
        "tests/python-gpu/test_gpu_data_iterator.py",
        "tests/python-gpu/load_pickle.py",
+        "tests/python/test_model_io.py",
        "tests/test_distributed/test_with_spark/test_data.py",
        "tests/test_distributed/test_gpu_with_spark/test_data.py",
        "tests/test_distributed/test_gpu_with_dask/test_gpu_with_dask.py",
        # demo
        "demo/json-model/json_parser.py",
        "demo/guide-python/external_memory.py",
+        "demo/guide-python/callbacks.py",
        "demo/guide-python/cat_in_the_dat.py",
        "demo/guide-python/categorical.py",
        "demo/guide-python/cat_pipeline.py",
--- a/tests/ci_build/test_r_package.py
+++ b/tests/ci_build/test_r_package.py
@@ -261,6 +261,8 @@ def test_with_cmake(args: argparse.Namespace) -> None:
                    "-DCMAKE_CONFIGURATION_TYPES=Release",
                    "-A",
                    "x64",
+                    "-G",
+                    "Visual Studio 17 2022",
                ]
            )
            subprocess.check_call(