diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 9d4196feb..6a7711427 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -40,12 +40,92 @@ jobs:
         cd jvm-packages
         mvn test -pl :xgboost4j_2.12
 
+  lint:
+    runs-on: ubuntu-latest
+    name: Code linting for Python and C++
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: 'true'
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.7'
+        architecture: 'x64'
+    - name: Install Python packages
+      run: |
+        python -m pip install wheel setuptools
+        python -m pip install pylint cpplint numpy scipy scikit-learn
+    - name: Run lint
+      run: |
+        make lint
+
+  doxygen:
+    runs-on: ubuntu-latest
+    name: Generate C/C++ API doc using Doxygen
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: 'true'
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.7'
+        architecture: 'x64'
+    - name: Install system packages
+      run: |
+        sudo apt-get install -y --no-install-recommends doxygen graphviz ninja-build
+        python -m pip install wheel setuptools
+        python -m pip install awscli
+    - name: Run Doxygen
+      run: |
+        mkdir build
+        cd build
+        cmake .. -DBUILD_C_DOC=ON -GNinja
+        ninja -v doc_doxygen
+    - name: Extract branch name
+      shell: bash
+      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+      id: extract_branch
+      if: github.ref == 'refs/heads/master' || contains(github.ref, 'refs/heads/release_')
+    - name: Publish
+      run: |
+        cd build/
+        tar cvjf ${{ steps.extract_branch.outputs.branch }}.tar.bz2 doc_doxygen/
+        python -m awscli s3 cp ./${{ steps.extract_branch.outputs.branch }}.tar.bz2 s3://xgboost-docs/ --acl public-read
+      if: github.ref == 'refs/heads/master' || contains(github.ref, 'refs/heads/release_')
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_IAM_S3_UPLOADER }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_IAM_S3_UPLOADER }}
+
+  sphinx:
+    runs-on: ubuntu-latest
+    name: Build docs using Sphinx
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: 'true'
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.7'
+        architecture: 'x64'
+    - name: Install system packages
+      run: |
+        sudo apt-get install -y --no-install-recommends graphviz
+        python -m pip install wheel setuptools
+        python -m pip install -r doc/requirements.txt
+    - name: Extract branch name
+      shell: bash
+      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+      id: extract_branch
+      if: github.ref == 'refs/heads/master' || contains(github.ref, 'refs/heads/release_')
+    - name: Run Sphinx
+      run: |
+        make -C doc html
+      env:
+        SPHINX_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }}
 
   lintr:
     runs-on: ${{ matrix.config.os }}
-
     name: Run R linters on OS ${{ matrix.config.os }}, R ${{ matrix.config.r }}, Compiler ${{ matrix.config.compiler }}, Build ${{ matrix.config.build }}
-
     strategy:
       matrix:
         config:
@@ -83,23 +163,16 @@ jobs:
         R.exe CMD INSTALL .
         Rscript.exe tests/run_lint.R
 
-
   test-with-R:
     runs-on: ${{ matrix.config.os }}
-
     name: Test R on OS ${{ matrix.config.os }}, R ${{ matrix.config.r }}, Compiler ${{ matrix.config.compiler }}, Build ${{ matrix.config.build }}
-
     strategy:
       fail-fast: false
       matrix:
         config:
-          - {os: windows-latest, r: 'release', compiler: 'msvc', build: 'autotools'}
           - {os: windows-2016, r: 'release', compiler: 'msvc', build: 'autotools'}
-          - {os: windows-latest, r: 'release', compiler: 'msvc', build: 'cmake'}
           - {os: windows-2016, r: 'release', compiler: 'msvc', build: 'cmake'}
-          - {os: windows-latest, r: 'release', compiler: 'mingw', build: 'autotools'}
           - {os: windows-2016, r: 'release', compiler: 'mingw', build: 'autotools'}
-          - {os: windows-latest, r: 'release', compiler: 'mingw', build: 'cmake'}
           - {os: windows-2016, r: 'release', compiler: 'mingw', build: 'cmake'}
     env:
       R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
@@ -130,8 +203,8 @@ jobs:
 
     - uses: actions/setup-python@v2
       with:
-        python-version: '3.6' # Version range or exact version of a Python version to use, using SemVer's version range syntax
-        architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
+        python-version: '3.7'
+        architecture: 'x64'
 
     - name: Test R
       run: |
diff --git a/Jenkinsfile b/Jenkinsfile
index 60e8116f3..a251ac1a3 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -49,24 +49,12 @@ pipeline {
         stash name: 'srcs'
       }
     }
-    stage('Jenkins Linux: Formatting Check') {
-      agent none
-      steps {
-        script {
-          parallel ([
-            'clang-tidy': { ClangTidy() },
-            'lint': { Lint() },
-            'sphinx-doc': { SphinxDoc() },
-            'doxygen': { Doxygen() }
-          ])
-        }
-      }
-    }
     stage('Jenkins Linux: Build') {
       agent none
       steps {
         script {
           parallel ([
+            'clang-tidy': { ClangTidy() },
             'build-cpu': { BuildCPU() },
             'build-cpu-rabit-mock': { BuildCPUMock() },
             'build-cpu-non-omp': { BuildCPUNonOmp() },
@@ -152,50 +140,6 @@ def ClangTidy() {
   }
 }
 
-def Lint() {
-  node('linux && cpu') {
-    unstash name: 'srcs'
-    echo "Running lint..."
-    def container_type = "cpu"
-    def docker_binary = "docker"
-    sh """
-    ${dockerRun} ${container_type} ${docker_binary} bash -c "source activate cpu_test && make lint"
-    """
-    deleteDir()
-  }
-}
-
-def SphinxDoc() {
-  node('linux && cpu') {
-    unstash name: 'srcs'
-    echo "Running sphinx-doc..."
-    def container_type = "cpu"
-    def docker_binary = "docker"
-    def docker_extra_params = "CI_DOCKER_EXTRA_PARAMS_INIT='-e SPHINX_GIT_BRANCH=${BRANCH_NAME}'"
-    sh """#!/bin/bash
-    ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} bash -c "source activate cpu_test && make -C doc html"
-    """
-    deleteDir()
-  }
-}
-
-def Doxygen() {
-  node('linux && cpu') {
-    unstash name: 'srcs'
-    echo "Running doxygen..."
-    def container_type = "cpu"
-    def docker_binary = "docker"
-    sh """
-    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/doxygen.sh ${BRANCH_NAME}
-    """
-    if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) {
-      echo 'Uploading doc...'
-      s3Upload file: "build/${BRANCH_NAME}.tar.bz2", bucket: 'xgboost-docs', acl: 'PublicRead', path: "doxygen/${BRANCH_NAME}.tar.bz2"
-    }
-    deleteDir()
-  }
-}
-
 def BuildCPU() {
   node('linux && cpu') {
     unstash name: 'srcs'
diff --git a/doc/conf.py b/doc/conf.py
index d17f9594a..749d400c6 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -22,7 +22,7 @@ import subprocess
 import guzzle_sphinx_theme
 
 git_branch = os.getenv('SPHINX_GIT_BRANCH', default=None)
-if git_branch is None:
+if not git_branch:
     # If SPHINX_GIT_BRANCH environment variable is not given, run git
     # to determine branch name
     git_branch = [
@@ -30,6 +30,8 @@ if git_branch is None:
             git.branch('-r', '--contains', 'HEAD')).rstrip('\n').split('\n')
     ]
     git_branch = [x for x in git_branch if 'HEAD' not in x]
+else:
+    git_branch = [git_branch]
 print('git_branch = {}'.format(git_branch[0]))
 try:
     filename, _ = urllib.request.urlretrieve(
diff --git a/tests/ci_build/doxygen.sh b/tests/ci_build/doxygen.sh
deleted file mode 100755
index 41757eb69..000000000
--- a/tests/ci_build/doxygen.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-if [ $# -ne 1 ]; then
-  echo "Usage: $0 [branch name]"
-  exit 1
-fi
-
-set -e
-set -x
-
-branch_name=$1
-
-rm -rf build
-mkdir build
-cd build
-cmake .. -DBUILD_C_DOC=ON
-make -j
-
-tar cvjf ${branch_name}.tar.bz2 doc_doxygen/