diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0e577817a..84f71de9f 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -351,4 +351,6 @@ jobs:
 
     - name: Check R Package
       run: |
-        make Rcheck
+        # Print stacktrace upon success of failure
+        make Rcheck || tests/ci_build/print_r_stacktrace.sh fail
+        tests/ci_build/print_r_stacktrace.sh success
diff --git a/Jenkinsfile b/Jenkinsfile
index 690b22583..964bc1022 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -86,8 +86,7 @@ pipeline {
             'test-cpp-gpu-cuda11.0': { TestCppGPU(artifact_cuda_version: '11.0', host_cuda_version: '11.0') },
             'test-jvm-jdk8': { CrossTestJVMwithJDK(jdk_version: '8', spark_version: '3.0.0') },
             'test-jvm-jdk11': { CrossTestJVMwithJDK(jdk_version: '11') },
-            'test-jvm-jdk12': { CrossTestJVMwithJDK(jdk_version: '12') },
-            'test-r-3.5.3': { TestR(use_r35: true) }
+            'test-jvm-jdk12': { CrossTestJVMwithJDK(jdk_version: '12') }
           ])
         }
       }
@@ -369,21 +368,6 @@ def CrossTestJVMwithJDK(args) {
   }
 }
 
-def TestR(args) {
-  node('linux && cpu') {
-    unstash name: 'srcs'
-    echo "Test R package"
-    def container_type = "rproject"
-    def docker_binary = "docker"
-    def use_r35_flag = (args.use_r35) ? "1" : "0"
-    def docker_args = "--build-arg USE_R35=${use_r35_flag}"
-    sh """
-    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_test_rpkg.sh || tests/ci_build/print_r_stacktrace.sh
-    """
-    deleteDir()
-  }
-}
-
 def DeployJVMPackages(args) {
   node('linux && cpu') {
     unstash name: 'srcs'
diff --git a/tests/ci_build/Dockerfile.rproject b/tests/ci_build/Dockerfile.rproject
deleted file mode 100644
index dd198f117..000000000
--- a/tests/ci_build/Dockerfile.rproject
+++ /dev/null
@@ -1,44 +0,0 @@
-FROM ubuntu:18.04
-ARG USE_R35=0
-
-# Environment
-ENV DEBIAN_FRONTEND noninteractive
-
-# Install all basic requirements
-RUN \
-    apt-get update && \
-    apt-get install -y software-properties-common tar unzip wget git build-essential doxygen graphviz libcurl4-openssl-dev libssl-dev libxml2-dev && \
-    if [ $USE_R35 -eq 1 ]; then \
-      apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \
-      add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' && \
-      apt-get update; \
-    fi && \
-    apt-get install -y r-base r-base-core r-recommended && \
-    # CMake
-    wget -nv -nc https://cmake.org/files/v3.13/cmake-3.13.0-Linux-x86_64.sh --no-check-certificate && \
-    bash cmake-3.13.0-Linux-x86_64.sh --skip-license --prefix=/usr
-
-# Use 16 workers to compile R packages
-ENV MAKE 'make -j16'
-
-# Install R packages
-RUN Rscript -e "install.packages( \
-        c('devtools', 'testthat', 'lintr', 'DiagrammeR', 'Ckmeans.1d.dp', 'vcd', 'data.table', 'stringi'), \
-        repos = 'http://cloud.r-project.org', \
-        dependencies = c('Depends', 'Imports', 'LinkingTo') \
-    )"
-
-ENV GOSU_VERSION 1.10
-
-# Install lightweight sudo (not bound to TTY)
-RUN set -ex; \
-    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
-    chmod +x /usr/local/bin/gosu && \
-    gosu nobody true
-
-# Default entry-point to use if running locally
-# It will preserve attributes of created files
-COPY entrypoint.sh /scripts/
-
-WORKDIR /workspace
-ENTRYPOINT ["/scripts/entrypoint.sh"]
diff --git a/tests/ci_build/build_test_rpkg.sh b/tests/ci_build/build_test_rpkg.sh
deleted file mode 100755
index 7a28a7d47..000000000
--- a/tests/ci_build/build_test_rpkg.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-
-set -e
-set -x
-
-make Rpack
-cd xgboost/
-
-# Run tests
-echo "Building with R CMD build"
-R CMD build --no-build-vignettes --no-manual .
-
-echo "Running R tests"
-R_PACKAGE_TARBALL=$(ls -1t *.tar.gz | head -n 1)
-
-export _R_CHECK_TIMINGS_=0
-export _R_CHECK_FORCE_SUGGESTS_=false
-R CMD check \
-  ${R_PACKAGE_TARBALL} \
-  --no-vignettes \
-  --no-manual \
-  --as-cran \
-  --install-args=--build
diff --git a/tests/ci_build/print_r_stacktrace.sh b/tests/ci_build/print_r_stacktrace.sh
index 71c449122..e7d442e72 100755
--- a/tests/ci_build/print_r_stacktrace.sh
+++ b/tests/ci_build/print_r_stacktrace.sh
@@ -3,17 +3,21 @@
 # To be called when R package tests have failed
 
 set -e
+set -x
 
-if [ -f "xgboost/xgboost.Rcheck/00install.out" ]; then
-  echo "===== xgboost/xgboost.Rcheck/00install.out ===="
-  cat xgboost/xgboost.Rcheck/00install.out
+flag="$1"
+
+if [ -f "xgboost.Rcheck/00install.out" ]; then
+  echo "===== xgboost.Rcheck/00install.out ===="
+  cat xgboost.Rcheck/00install.out
 fi
 
-if [ -f "xgboost/xgboost.Rcheck/00check.log" ]; then
-  printf "\n\n===== xgboost/xgboost.Rcheck/00check.log ====\n"
-  cat xgboost/xgboost.Rcheck/00check.log
+if [ -f "xgboost.Rcheck/00check.log" ]; then
+  printf "\n\n===== xgboost.Rcheck/00check.log ====\n"
+  cat xgboost.Rcheck/00check.log
 fi
 
-
-# Produce error code to interrupt Jenkins pipeline
-exit 1
+if [[ "$flag" == "fail" ]]
+then
+  exit 1
+fi