diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 35fa6bf21..3416bac65 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -35,6 +35,33 @@ jobs:
       run: |
         cd build
         ctest --extra-verbose
+
+  gtest-cpu-nonomp:
+    name: Test Google C++ unittest (CPU Non-OMP)
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: 'true'
+    - name: Install system packages
+      run: |
+        sudo apt-get install -y --no-install-recommends ninja-build
+    - name: Build and install XGBoost
+      shell: bash -l {0}
+      run: |
+        mkdir build
+        cd build
+        cmake .. -GNinja -DGOOGLE_TEST=ON  -DUSE_DMLC_GTEST=ON -DUSE_OPENMP=OFF
+        ninja -v
+    - name: Run gtest binary
+      run: |
+        cd build
+        ctest --extra-verbose
+
   c-api-demo:
     name: Test installing XGBoost lib + building the C API demo
     runs-on: ${{ matrix.os }}
@@ -76,6 +103,7 @@ jobs:
         ninja -v
         cd ..
         ./build/api-demo
+
   test-with-jvm:
     name: Test JVM on OS ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
diff --git a/Jenkinsfile b/Jenkinsfile
index fde997f76..2d760c1f9 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -57,7 +57,6 @@ pipeline {
             'clang-tidy': { ClangTidy() },
             'build-cpu': { BuildCPU() },
             'build-cpu-rabit-mock': { BuildCPUMock() },
-            'build-cpu-non-omp': { BuildCPUNonOmp() },
             // Build reference, distribution-ready Python wheel with CUDA 10.0
             // using CentOS 6 image
             'build-gpu-cuda10.0': { BuildCUDA(cuda_version: '10.0') },
@@ -182,23 +181,6 @@ def BuildCPUMock() {
   }
 }
 
-def BuildCPUNonOmp() {
-  node('linux && cpu') {
-    unstash name: 'srcs'
-    echo "Build CPU without OpenMP"
-    def container_type = "cpu"
-    def docker_binary = "docker"
-    sh """
-    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh -DUSE_OPENMP=OFF
-    """
-    echo "Running Non-OpenMP C++ test..."
-    sh """
-    ${dockerRun} ${container_type} ${docker_binary} bash -c "cd build && ctest --extra-verbose"
-    """
-    deleteDir()
-  }
-}
-
 def BuildCUDA(args) {
   node('linux && cpu_build') {
     unstash name: 'srcs'