diff --git a/tests/buildkite/test-cpp-gpu.sh b/tests/buildkite/test-cpp-gpu.sh
index d1192c763..f1ddf9d5f 100755
--- a/tests/buildkite/test-cpp-gpu.sh
+++ b/tests/buildkite/test-cpp-gpu.sh
@@ -4,7 +4,7 @@ set -euo pipefail
 
 source tests/buildkite/conftest.sh
 
-echo "--- Run Google Tests with CUDA, using 4 GPUs"
+echo "--- Run Google Tests with CUDA, using a GPU"
 buildkite-agent artifact download "build/testxgboost" . --step build-cuda
 chmod +x build/testxgboost
 tests/ci_build/ci_build.sh gpu nvidia-docker \
@@ -12,11 +12,12 @@ tests/ci_build/ci_build.sh gpu nvidia-docker \
   --build-arg RAPIDS_VERSION_ARG=$RAPIDS_VERSION \
   build/testxgboost
 
-echo "--- Run Google Tests with CUDA, using 4 GPUs, RMM enabled"
-rm -rfv build/
-buildkite-agent artifact download "build/testxgboost" . --step build-cuda-with-rmm
-chmod +x build/testxgboost
-tests/ci_build/ci_build.sh rmm nvidia-docker \
-  --build-arg CUDA_VERSION_ARG=$CUDA_VERSION \
-  --build-arg RAPIDS_VERSION_ARG=$RAPIDS_VERSION bash -c \
-  "source activate gpu_test && build/testxgboost --use-rmm-pool"
+# Disabled until https://github.com/dmlc/xgboost/issues/8619 is resolved
+# echo "--- Run Google Tests with CUDA, using a GPU, RMM enabled"
+# rm -rfv build/
+# buildkite-agent artifact download "build/testxgboost" . --step build-cuda-with-rmm
+# chmod +x build/testxgboost
+# tests/ci_build/ci_build.sh rmm nvidia-docker \
+#   --build-arg CUDA_VERSION_ARG=$CUDA_VERSION \
+#   --build-arg RAPIDS_VERSION_ARG=$RAPIDS_VERSION bash -c \
+#   "source activate gpu_test && build/testxgboost --use-rmm-pool"