Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CI] Update machine images (#9932)

Browse Source
This commit is contained in:
Philip Hyunsu Cho
2023-12-29 11:15:38 -08:00
committed by GitHub
parent a7226c0222
commit ef8bdaa047
26 changed files with 82 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
tests/ci_build/ci_build.sh
View File

@@ -2,14 +2,14 @@
#
# Execute command within a docker container
#
# Usage: ci_build.sh <CONTAINER_TYPE> <DOCKER_BINARY>
# Usage: ci_build.sh <CONTAINER_TYPE> [--use-gpus]
# [--dockerfile <DOCKERFILE_PATH>] [-it]
# [--build-arg <BUILD_ARG>] <COMMAND>
#
# CONTAINER_TYPE: Type of the docker container used the run the build: e.g.,
# (cpu | gpu)
#
# DOCKER_BINARY: Command to invoke docker, e.g. (docker | nvidia-docker).
# --use-gpus: Whether to grant the container access to NVIDIA GPUs.
#
# DOCKERFILE_PATH: (Optional) Path to the Dockerfile used for docker build. If
# this optional value is not supplied (via the --dockerfile
@@ -29,9 +29,12 @@ shift 1
DOCKERFILE_PATH="${SCRIPT_DIR}/Dockerfile.${CONTAINER_TYPE}"
DOCKER_CONTEXT_PATH="${SCRIPT_DIR}"
# Get docker binary command (should be either docker or nvidia-docker)
DOCKER_BINARY="$1"
shift 1
GPU_FLAG=''
if [[ "$1" == "--use-gpus" ]]; then
echo "Using NVIDIA GPUs"
GPU_FLAG='--gpus all'
shift 1
fi
if [[ "$1" == "--dockerfile" ]]; then
DOCKERFILE_PATH="$2"
@@ -144,21 +147,21 @@ then
DOCKER_CACHE_REPO="${DOCKER_CACHE_ECR_ID}.dkr.ecr.${DOCKER_CACHE_ECR_REGION}.amazonaws.com"
echo "Using AWS ECR; repo URL = ${DOCKER_CACHE_REPO}"
# Login for Docker registry
echo "\$(aws ecr get-login --no-include-email --region ${DOCKER_CACHE_ECR_REGION} --registry-ids ${DOCKER_CACHE_ECR_ID})"
$(aws ecr get-login --no-include-email --region ${DOCKER_CACHE_ECR_REGION} --registry-ids ${DOCKER_CACHE_ECR_ID})
echo "aws ecr get-login-password --region ${DOCKER_CACHE_ECR_REGION} | docker login --username AWS --password-stdin ${DOCKER_CACHE_REPO}"
aws ecr get-login-password --region ${DOCKER_CACHE_ECR_REGION} | docker login --username AWS --password-stdin ${DOCKER_CACHE_REPO}
# Pull pre-build container from Docker build cache,
# if one exists for the particular branch or pull request
DOCKER_TAG="${BRANCH_NAME//\//-}" # Slashes are not allow in Docker tag
echo "docker pull --quiet ${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:${DOCKER_TAG}"
if time docker pull --quiet "${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:${DOCKER_TAG}"
then
CACHE_FROM_CMD="--cache-from ${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:${DOCKER_TAG}"
CACHE_FROM_CMD="--cache-from ${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:${DOCKER_TAG} --build-arg BUILDKIT_INLINE_CACHE=1"
else
# If the build cache is empty of the particular branch or pull request,
# use the build cache associated with the master branch
echo "docker pull --quiet ${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:master"
docker pull --quiet "${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:master" || true
CACHE_FROM_CMD="--cache-from ${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:master"
CACHE_FROM_CMD="--cache-from ${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:master --build-arg BUILDKIT_INLINE_CACHE=1"
fi
else
CACHE_FROM_CMD=''
@@ -166,11 +169,15 @@ fi
echo "docker build \
${CI_DOCKER_BUILD_ARG} \
--progress=plain \
--ulimit nofile=1024000:1024000 \
-t ${DOCKER_IMG_NAME} \
-f ${DOCKERFILE_PATH} ${DOCKER_CONTEXT_PATH} \
${CACHE_FROM_CMD}"
docker build \
${CI_DOCKER_BUILD_ARG} \
--progress=plain \
--ulimit nofile=1024000:1024000 \
-t "${DOCKER_IMG_NAME}" \
-f "${DOCKERFILE_PATH}" "${DOCKER_CONTEXT_PATH}" \
${CACHE_FROM_CMD}
@@ -231,7 +238,8 @@ echo "Running '${COMMAND[*]}' inside ${DOCKER_IMG_NAME}..."
# and share the PID namespace (--pid=host) so the process inside does not have
# pid 1 and SIGKILL is propagated to the process inside (jenkins can kill it).
set -x
${DOCKER_BINARY} run --rm --pid=host \
docker run --rm --pid=host \
${GPU_FLAG} \
-v "${WORKSPACE}":/workspace \
-w /workspace \
${USER_IDS} \