diff --git a/tests/buildkite/build-gpu-rpkg.sh b/tests/buildkite/build-gpu-rpkg.sh
index 72770fc41..5e0de9f8c 100755
--- a/tests/buildkite/build-gpu-rpkg.sh
+++ b/tests/buildkite/build-gpu-rpkg.sh
@@ -6,15 +6,13 @@ source tests/buildkite/conftest.sh
 
 echo "--- Build XGBoost R package with CUDA"
 
-if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
-then
-
 tests/ci_build/ci_build.sh gpu_build_r_centos7 docker \
   --build-arg CUDA_VERSION_ARG=${CUDA_VERSION} tests/ci_build/build_r_pkg_with_cuda.sh \
   ${BUILDKITE_COMMIT}
 
-echo "--- Upload R tarball"
-aws s3 cp xgboost_r_gpu_linux_*.tar.gz s3://xgboost-nightly-builds/${BRANCH_NAME}/ \
-  --acl public-read --no-progress
-
+if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
+then
+  echo "--- Upload R tarball"
+  aws s3 cp xgboost_r_gpu_linux_*.tar.gz s3://xgboost-nightly-builds/${BRANCH_NAME}/ \
+    --acl public-read --no-progress
 fi
diff --git a/tests/buildkite/build-rpkg-win64-gpu.ps1 b/tests/buildkite/build-rpkg-win64-gpu.ps1
index c5db18480..a6947c270 100644
--- a/tests/buildkite/build-rpkg-win64-gpu.ps1
+++ b/tests/buildkite/build-rpkg-win64-gpu.ps1
@@ -5,12 +5,12 @@ $ErrorActionPreference = "Stop"
 Write-Host "--- Build XGBoost R package with CUDA"
 
 nvcc --version
+$arch_flag = "-DGPU_COMPUTE_VER=75"
+
+bash tests/ci_build/build_r_pkg_with_cuda_win64.sh $Env:BUILDKITE_COMMIT
+if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
+
 if ( $is_release_branch -eq 1 ) {
-  $arch_flag = "-DGPU_COMPUTE_VER=75"
-
-  bash tests/ci_build/build_r_pkg_with_cuda_win64.sh $Env:BUILDKITE_COMMIT
-  if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
-
   Write-Host "--- Upload R tarball"
   Get-ChildItem . -Filter xgboost_r_gpu_win64_*.tar.gz |
   Foreach-Object {
diff --git a/tests/ci_build/build_r_pkg_with_cuda.sh b/tests/ci_build/build_r_pkg_with_cuda.sh
index 1aefb3343..5d36a16e1 100755
--- a/tests/ci_build/build_r_pkg_with_cuda.sh
+++ b/tests/ci_build/build_r_pkg_with_cuda.sh
@@ -10,7 +10,7 @@ fi
 
 commit_hash="$1"
 
-make Rpack
+python tests/ci_build/test_r_package.py --task=pack
 mv xgboost/ xgboost_rpack/
 
 mkdir build
diff --git a/tests/ci_build/build_r_pkg_with_cuda_win64.sh b/tests/ci_build/build_r_pkg_with_cuda_win64.sh
index f83795775..7d32bfe6a 100644
--- a/tests/ci_build/build_r_pkg_with_cuda_win64.sh
+++ b/tests/ci_build/build_r_pkg_with_cuda_win64.sh
@@ -9,8 +9,11 @@ then
 fi
 
 commit_hash="$1"
+# Clear all positional args
+set --
 
-MAKE="/c/Rtools/bin/make" /c/Rtools/bin/make Rpack
+source activate
+python tests/ci_build/test_r_package.py --task=pack
 mv xgboost/ xgboost_rpack/
 
 mkdir build
@@ -31,4 +34,4 @@ echo 'all:' > xgboost_rpack/src/Makefile
 echo 'all:' > xgboost_rpack/src/Makefile.win
 mv xgboost_rpack/ xgboost/
 /c/Rtools/bin/tar -cvf xgboost_r_gpu_win64_${commit_hash}.tar xgboost/
-/c/Rtools/bin/gzip -9c xgboost_r_gpu_win64_${commit_hash}.tar > xgboost_r_gpu_win64_${commit_hash}.tar.gz
\ No newline at end of file
+/c/Rtools/bin/gzip -9c xgboost_r_gpu_win64_${commit_hash}.tar > xgboost_r_gpu_win64_${commit_hash}.tar.gz
diff --git a/tests/ci_build/test_r_package.py b/tests/ci_build/test_r_package.py
index a51db89c7..2f9e59c8f 100644
--- a/tests/ci_build/test_r_package.py
+++ b/tests/ci_build/test_r_package.py
@@ -251,7 +251,9 @@ def test_with_cmake(args: argparse.Namespace) -> None:
 
 @record_time
 def main(args: argparse.Namespace) -> None:
-    if args.task == "build":
+    if args.task == "pack":
+        pack_rpackage()
+    elif args.task == "build":
         src_dir = pack_rpackage()
         build_rpackage(src_dir)
     elif args.task == "doc":
@@ -280,7 +282,7 @@ if __name__ == "__main__":
     parser.add_argument(
         "--task",
         type=str,
-        choices=["build", "check", "doc"],
+        choices=["pack", "build", "check", "doc"],
         default="check",
         required=False,
     )