diff --git a/tests/buildkite/conftest.sh b/tests/buildkite/conftest.sh
index 40409fc50..6f33799e8 100755
--- a/tests/buildkite/conftest.sh
+++ b/tests/buildkite/conftest.sh
@@ -12,9 +12,14 @@ else
   export BRANCH_NAME=$BUILDKITE_BRANCH
 fi
 
-if [[ $BUILDKITE_BRANCH == "master" || $BUILDKITE_BRANCH == "release_"* ]]
+if [[ -z ${DISABLE_RELEASE:-} ]]
 then
-  is_release_branch=1
+  if [[ $BUILDKITE_BRANCH == "master" || $BUILDKITE_BRANCH == "release_"* ]]
+  then
+    is_release_branch=1
+  else
+    is_release_branch=0
+  fi
 else
   is_release_branch=0
 fi
diff --git a/tests/buildkite/pipeline-mgpu.yml b/tests/buildkite/pipeline-mgpu.yml
new file mode 100644
index 000000000..215c1584f
--- /dev/null
+++ b/tests/buildkite/pipeline-mgpu.yml
@@ -0,0 +1,29 @@
+env:
+  DOCKER_CACHE_ECR_ID: "492475357299"
+  DOCKER_CACHE_ECR_REGION: "us-west-2"
+  DISABLE_RELEASE: "1"
+    # Skip uploading artifacts to S3 bucket
+    # Also, don't build all CUDA archs; just build sm_75
+steps:
+  - block: ":rocket: Run this test job"
+    if: build.pull_request.repository.fork == true
+  #### -------- BUILD --------
+  - label: ":console: Build CUDA"
+    command: "tests/buildkite/build-cuda.sh"
+    key: build-cuda
+    agents:
+      queue: linux-amd64-cpu
+  - label: ":console: Build JVM packages with CUDA"
+    command: "tests/buildkite/build-jvm-packages-gpu.sh"
+    key: build-jvm-packages-gpu
+    agents:
+      queue: linux-amd64-mgpu
+
+  - wait
+
+  #### -------- TEST --------
+  - label: ":console: Test Python package, 4 GPUs"
+    command: "tests/buildkite/test-python-gpu.sh mgpu"
+    key: test-python-mgpu
+    agents:
+      queue: linux-amd64-mgpu
diff --git a/tests/buildkite/pipeline.yml b/tests/buildkite/pipeline.yml
index af5d88f53..94d12e6c7 100644
--- a/tests/buildkite/pipeline.yml
+++ b/tests/buildkite/pipeline.yml
@@ -40,11 +40,6 @@ steps:
     key: build-jvm-packages
     agents:
       queue: linux-amd64-cpu
-  - label: ":console: Build JVM packages with CUDA"
-    command: "tests/buildkite/build-jvm-packages-gpu.sh"
-    key: build-jvm-packages-gpu
-    agents:
-      queue: linux-amd64-mgpu
   - label: ":console: Build JVM package doc"
     command: "tests/buildkite/build-jvm-doc.sh"
     key: build-jvm-doc
@@ -69,11 +64,6 @@ steps:
     key: test-python-gpu
     agents:
       queue: linux-amd64-gpu
-  - label: ":console: Test Python package, 4 GPUs"
-    command: "tests/buildkite/test-python-gpu.sh mgpu"
-    key: test-python-mgpu
-    agents:
-      queue: linux-amd64-mgpu
   - label: ":console: Run Google Tests, 4 GPUs"
     command: "tests/buildkite/test-cpp-gpu.sh"
     key: test-cpp-gpu