diff --git a/.github/workflows/r_tests.yml b/.github/workflows/r_tests.yml
index ad6853281..6565b0342 100644
--- a/.github/workflows/r_tests.yml
+++ b/.github/workflows/r_tests.yml
@@ -46,7 +46,7 @@ jobs:
         MAKEFLAGS="-j$(nproc)" R CMD INSTALL R-package/
         Rscript tests/ci_build/lint_r.R $(pwd)
 
-  test-R-on-Windows:
+  test-Rpkg:
     runs-on: ${{ matrix.config.os }}
     name: Test R on OS ${{ matrix.config.os }}, R ${{ matrix.config.r }}, Compiler ${{ matrix.config.compiler }}, Build ${{ matrix.config.build }}
     strategy:
@@ -54,11 +54,17 @@ jobs:
       matrix:
         config:
           - {os: windows-latest, r: 'release', compiler: 'mingw', build: 'autotools'}
+          - {os: ubuntu-latest, r: 'release', compiler: 'none', build: 'cmake'}
     env:
       R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
       RSPM: ${{ matrix.config.rspm }}
 
     steps:
+    - name: Install system dependencies
+      run: |
+        sudo apt update
+        sudo apt install libcurl4-openssl-dev libssl-dev libssh2-1-dev libgit2-dev libglpk-dev libxml2-dev libharfbuzz-dev libfribidi-dev
+      if: matrix.config.os == 'ubuntu-latest'
     - uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0
       with:
         submodules: 'true'
@@ -89,6 +95,12 @@ jobs:
     - name: Test R
       run: |
         python tests/ci_build/test_r_package.py --compiler='${{ matrix.config.compiler }}' --build-tool="${{ matrix.config.build }}" --task=check
+      if: matrix.config.compiler != 'none'
+
+    - name: Test R
+      run: |
+        python tests/ci_build/test_r_package.py --build-tool="${{ matrix.config.build }}" --task=check
+      if: matrix.config.compiler == 'none'
 
   test-R-on-Debian:
     name: Test R package on Debian
diff --git a/R-package/CMakeLists.txt b/R-package/CMakeLists.txt
index d3a69abc2..37c5dbf4c 100644
--- a/R-package/CMakeLists.txt
+++ b/R-package/CMakeLists.txt
@@ -26,7 +26,6 @@ endif()
 target_compile_definitions(
   xgboost-r PUBLIC
   -DXGBOOST_STRICT_R_MODE=1
-  -DXGBOOST_CUSTOMIZE_GLOBAL_PRNG=1
   -DDMLC_LOG_BEFORE_THROW=0
   -DDMLC_DISABLE_STDIN=1
   -DDMLC_LOG_CUSTOMIZE=1
diff --git a/tests/ci_build/test_r_package.py b/tests/ci_build/test_r_package.py
index dd73f850b..ddcf48674 100644
--- a/tests/ci_build/test_r_package.py
+++ b/tests/ci_build/test_r_package.py
@@ -277,6 +277,19 @@ def test_with_cmake(args: argparse.Namespace) -> None:
                     "Release",
                 ]
             )
+        elif args.compiler == "none":
+            subprocess.check_call(
+                [
+                    "cmake",
+                    os.path.pardir,
+                    "-DUSE_OPENMP=ON",
+                    "-DR_LIB=ON",
+                    "-DCMAKE_CONFIGURATION_TYPES=Release",
+                    "-G",
+                    "Unix Makefiles",
+                ]
+            )
+            subprocess.check_call(["make", "-j", "install"])
         else:
             raise ValueError("Wrong compiler")
     with DirectoryExcursion(R_PACKAGE):
@@ -333,9 +346,9 @@ if __name__ == "__main__":
     parser.add_argument(
         "--compiler",
         type=str,
-        choices=["mingw", "msvc"],
+        choices=["mingw", "msvc", "none"],
         help="Compiler used for compiling CXX code. Only relevant for windows build",
-        default="mingw",
+        default="none",
         required=False,
     )
     parser.add_argument(