Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Various CI savings (#8291)

Browse Source
This commit is contained in:
Philip Hyunsu Cho 2022-09-30 05:42:56 -07:00 committed by GitHub
parent 299e5000a4
commit 9af99760d4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 87 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
tests/buildkite/enforce_daily_budget.py Normal file
View File

@ -0,0 +1,14 @@
import json
import argparse
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--response", type=str, required=True)
args = parser.parse_args()
with open(args.response, "r") as f:
payload = f.read()
response = json.loads(payload)
if response["approved"]:
print(f"Testing approved. Reason: {response['reason']}")
else:
raise RuntimeError(f"Testing rejected. Reason: {response['reason']}")

15
tests/buildkite/enforce_daily_budget.sh Executable file
View File

@ -0,0 +1,15 @@
#!/bin/bash
set -euo pipefail
echo "--- Enforce daily budget"
source tests/buildkite/conftest.sh
if [[ $is_release_branch == 1 ]]
then
echo "Automatically approving all test jobs for release branches"
else
aws lambda invoke --function-name XGBoostCICostWatcher --invocation-type RequestResponse --region us-west-2 response.json
python3 tests/buildkite/enforce_daily_budget.py --response response.json
fi

19
tests/buildkite/infrastructure/aws-stack-creator/metadata.py
View File

@ -9,6 +9,9 @@ AMI_ID = {
"windows-gpu": {
"us-west-2": "ami-0a1a2ea551a07ad5f",
},
"windows-cpu": {
"us-west-2": "ami-0a1a2ea551a07ad5f",
},
# Managed by BuildKite
"linux-amd64-cpu": {
"us-west-2": "ami-075d4c25d5f0c17c1",
@ -37,7 +40,7 @@ STACK_PARAMS = {
"InstanceType": "g4dn.12xlarge",
"AgentsPerInstance": "1",
"MinSize": "0",
"MaxSize": "4",
"MaxSize": "1",
"OnDemandPercentage": "100",
"ScaleOutFactor": "1.0",
"ScaleInIdlePeriod": "60", # in seconds
@ -50,7 +53,17 @@ STACK_PARAMS = {
"MaxSize": "2",
"OnDemandPercentage": "100",
"ScaleOutFactor": "1.0",
"ScaleInIdlePeriod": "600", # in seconds
"ScaleInIdlePeriod": "60", # in seconds
},
"windows-cpu": {
"InstanceOperatingSystem": "windows",
"InstanceType": "c5a.2xlarge",
"AgentsPerInstance": "1",
"MinSize": "0",
"MaxSize": "2",
"OnDemandPercentage": "100",
"ScaleOutFactor": "1.0",
"ScaleInIdlePeriod": "60", # in seconds
},
"linux-amd64-cpu": {
"InstanceOperatingSystem": "linux",
@ -92,7 +105,7 @@ COMMON_STACK_PARAMS = {
"EnableCostAllocationTags": "true",
"CostAllocationTagName": "CreatedBy",
"ECRAccessPolicy": "full",
"ManagedPolicyARN": "arn:aws:iam::aws:policy/AmazonS3FullAccess",
"ManagedPolicyARN": "arn:aws:iam::aws:policy/AmazonS3FullAccess,arn:aws:iam::aws:policy/service-role/AWSLambdaRole",
"EnableSecretsPlugin": "false",
"EnableECRPlugin": "false",
"EnableDockerLoginPlugin": "false",

5
tests/buildkite/pipeline-mgpu.yml
View File

@ -5,6 +5,11 @@ env:
# Skip uploading artifacts to S3 bucket
# Also, don't build all CUDA archs; just build sm_75
steps:
- label: ":moneybag: Enforce daily budget"
command: "tests/buildkite/enforce_daily_budget.sh"
key: enforce-daily-budget
agents:
queue: pipeline-loader
- block: ":rocket: Run this test job"
if: build.pull_request.repository.fork == true
#### -------- BUILD --------

9
tests/buildkite/pipeline-win64.yml
View File

@ -1,4 +1,9 @@
steps:
- label: ":moneybag: Enforce daily budget"
command: "tests/buildkite/enforce_daily_budget.sh"
key: enforce-daily-budget
agents:
queue: pipeline-loader
- block: ":rocket: Run this test job"
if: build.pull_request.repository.fork == true
#### -------- BUILD --------
@ -6,12 +11,12 @@ steps:
command: "tests/buildkite/build-win64-gpu.ps1"
key: build-win64-gpu
agents:
queue: windows-gpu
queue: windows-cpu
- label: ":windows: Build XGBoost R package for Windows with CUDA"
command: "tests/buildkite/build-rpkg-win64-gpu.ps1"
key: build-rpkg-win64-gpu
agents:
queue: windows-gpu
queue: windows-cpu
- wait

5
tests/buildkite/pipeline.yml
View File

@ -2,6 +2,11 @@ env:
DOCKER_CACHE_ECR_ID: "492475357299"
DOCKER_CACHE_ECR_REGION: "us-west-2"
steps:
- label: ":moneybag: Enforce daily budget"
command: "tests/buildkite/enforce_daily_budget.sh"
key: enforce-daily-budget
agents:
queue: pipeline-loader
- block: ":rocket: Run this test job"
if: build.pull_request.repository.fork == true
#### -------- BUILD --------

27
tests/ci_build/ci_build.sh
View File

@ -188,8 +188,31 @@ then
echo "docker tag ${DOCKER_IMG_NAME} ${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:${BRANCH_NAME}"
docker tag "${DOCKER_IMG_NAME}" "${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:${BRANCH_NAME}"
echo "aws ecr create-repository --repository-name ${DOCKER_IMG_NAME} --region ${DOCKER_CACHE_ECR_REGION} || true"
aws ecr create-repository --repository-name ${DOCKER_IMG_NAME} --region ${DOCKER_CACHE_ECR_REGION} || true
# Attempt to create Docker repository; it will fail if the repository already exists
echo "aws ecr create-repository --repository-name ${DOCKER_IMG_NAME} --region ${DOCKER_CACHE_ECR_REGION}"
if aws ecr create-repository --repository-name ${DOCKER_IMG_NAME} --region ${DOCKER_CACHE_ECR_REGION}
then
# Repository was created. Now set expiration policy
echo "aws ecr put-lifecycle-policy --repository-name ${DOCKER_IMG_NAME} --region ${DOCKER_CACHE_ECR_REGION} --lifecycle-policy-text file:///dev/stdin"
cat <<EOF | aws ecr put-lifecycle-policy --repository-name ${DOCKER_IMG_NAME} --region ${DOCKER_CACHE_ECR_REGION} --lifecycle-policy-text file:///dev/stdin
{
"rules": [
{
"rulePriority": 1,
"selection": {
"tagStatus": "any",
"countType": "sinceImagePushed",
"countUnit": "days",
"countNumber": 30
},
"action": {
"type": "expire"
}
}
]
}
EOF
fi
echo "docker push --quiet ${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:${BRANCH_NAME}"
docker push --quiet "${DOCKER_CACHE_REPO}/${DOCKER_IMG_NAME}:${BRANCH_NAME}"