[CI] Revise R tests. (#8430)

- Use the standard package check (check on the tarball instead of the source tree). - Run commands in parallel. - Cleanup dependencies installation. - Replace makefile. - Documentation. - Test using the image from rhub.
2022-11-09 09:12:13 +08:00 · 2022-11-09 09:12:13 +08:00 · a83748eb45
commit a83748eb45
parent 4449e30184
12 changed files with 499 additions and 363 deletions
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@ -146,7 +146,7 @@ jobs:
        python -m pip install wheel setuptools cpplint pylint
    - name: Run lint
      run: |
-        LINT_LANG=cpp make lint
+        python dmlc-core/scripts/lint.py xgboost cpp R-package/src

  sphinx:
    runs-on: ubuntu-latest
--- a/.github/workflows/r_nold.yml
+++ b/.github/workflows/r_nold.yml
@ -1,4 +1,4 @@
-# Run R tests with noLD R. Only triggered by a pull request review
+# Run expensive R tests with the help of rhub. Only triggered by a pull request review
 # See discussion at https://github.com/dmlc/xgboost/pull/6378

 name: XGBoost-R-noLD
@ -7,9 +7,6 @@ on:
  pull_request_review_comment:
    types: [created]

-env:
-  R_PACKAGES: c('XML', 'igraph', 'data.table', 'ggplot2', 'DiagrammeR', 'Ckmeans.1d.dp', 'vcd', 'testthat', 'lintr', 'knitr', 'rmarkdown', 'e1071', 'cplm', 'devtools', 'float', 'titanic')
-
 permissions:
  contents: read # to fetch code (actions/checkout)

@ -18,26 +15,22 @@ jobs:
    if: github.event.comment.body == '/gha run r-nold-test' && contains('OWNER,MEMBER,COLLABORATOR', github.event.comment.author_association)
    timeout-minutes: 120
    runs-on: ubuntu-latest
-    container: rhub/debian-gcc-devel-nold
+    container:
+      image: rhub/debian-gcc-devel-nold
    steps:
    - name: Install git and system packages
      shell: bash
      run: |
-        apt-get update && apt-get install -y git libcurl4-openssl-dev libssl-dev libssh2-1-dev libgit2-dev libxml2-dev
+        apt update && apt install libcurl4-openssl-dev libssl-dev libssh2-1-dev libgit2-dev libglpk-dev libxml2-dev libharfbuzz-dev libfribidi-dev git -y

    - uses: actions/checkout@v2
      with:
        submodules: 'true'

    - name: Install dependencies
-      shell: bash
+      shell: bash -l {0}
      run: |
-        cat > install_libs.R <<EOT
-          install.packages(${{ env.R_PACKAGES }},
-                           repos = 'http://cloud.r-project.org',
-                           dependencies = c('Depends', 'Imports', 'LinkingTo'))
-        EOT
-        /tmp/R-devel/bin/Rscript install_libs.R
+        /tmp/R-devel/bin/Rscript -e "source('./R-package/tests/helper_scripts/install_deps.R')"

    - name: Run R tests
      shell: bash
--- a/.github/workflows/r_tests.yml
+++ b/.github/workflows/r_tests.yml
@ -3,9 +3,7 @@ name: XGBoost-R-Tests
 on: [push, pull_request]

 env:
-  R_PACKAGES: c('XML', 'data.table', 'ggplot2', 'DiagrammeR', 'Ckmeans.1d.dp', 'vcd', 'testthat', 'lintr', 'knitr', 'rmarkdown', 'e1071', 'cplm', 'devtools', 'float', 'titanic')
  GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
-  _R_CHECK_EXAMPLE_TIMING_CPU_TO_ELAPSED_THRESHOLD_: 2.5

 permissions:
  contents: read # to fetch code (actions/checkout)
@ -35,29 +33,22 @@ jobs:
      uses: actions/cache@v2
      with:
        path: ${{ env.R_LIBS_USER }}
-        key: ${{ runner.os }}-r-${{ matrix.config.r }}-5-${{ hashFiles('R-package/DESCRIPTION') }}
-        restore-keys: ${{ runner.os }}-r-${{ matrix.config.r }}-5-${{ hashFiles('R-package/DESCRIPTION') }}
+        key: ${{ runner.os }}-r-${{ matrix.config.r }}-6-${{ hashFiles('R-package/DESCRIPTION') }}
+        restore-keys: ${{ runner.os }}-r-${{ matrix.config.r }}-6-${{ hashFiles('R-package/DESCRIPTION') }}

    - name: Install dependencies
      shell: Rscript {0}
      run: |
-        install.packages(${{ env.R_PACKAGES }},
-                         repos = 'http://cloud.r-project.org',
-                         dependencies = c('Depends', 'Imports', 'LinkingTo'))
-    - name: Install igraph on Windows
-      shell: Rscript {0}
-      if: matrix.config.os == 'windows-latest'
-      run: |
-        install.packages('igraph', type='binary')
+        source("./R-package/tests/helper_scripts/install_deps.R")

    - name: Run lintr
      run: |
        cd R-package
-        MAKE="make -j$(nproc)" R CMD INSTALL .
+        MAKEFLAGS="-j$(nproc)" R CMD INSTALL .
        # Disable lintr errors for now: https://github.com/dmlc/xgboost/issues/8012
        Rscript tests/helper_scripts/run_lint.R || true

-  test-with-R:
+  test-R-on-Windows:
    runs-on: ${{ matrix.config.os }}
    name: Test R on OS ${{ matrix.config.os }}, R ${{ matrix.config.r }}, Compiler ${{ matrix.config.compiler }}, Build ${{ matrix.config.build }}
    strategy:
@ -66,10 +57,8 @@ jobs:
        config:
          - {os: windows-latest, r: 'release', compiler: 'mingw', build: 'autotools'}
          - {os: windows-latest, r: 'release', compiler: 'msvc', build: 'cmake'}
-          - {os: windows-latest, r: 'release', compiler: 'mingw', build: 'cmake'}
    env:
      R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
-      _R_CHECK_EXAMPLE_TIMING_CPU_TO_ELAPSED_THRESHOLD_: 2.5
      RSPM: ${{ matrix.config.rspm }}

    steps:
@ -85,80 +74,60 @@ jobs:
      uses: actions/cache@v2
      with:
        path: ${{ env.R_LIBS_USER }}
-        key: ${{ runner.os }}-r-${{ matrix.config.r }}-5-${{ hashFiles('R-package/DESCRIPTION') }}
-        restore-keys: ${{ runner.os }}-r-${{ matrix.config.r }}-5-${{ hashFiles('R-package/DESCRIPTION') }}
-
-    - name: Install dependencies
-      shell: Rscript {0}
-      if: matrix.config.os != 'windows-latest'
-      run: |
-        install.packages(${{ env.R_PACKAGES }},
-                         repos = 'http://cloud.r-project.org',
-                         dependencies = c('Depends', 'Imports', 'LinkingTo'))
-
-    - name: Install binary dependencies
-      shell: Rscript {0}
-      if: matrix.config.os == 'windows-latest'
-      run: |
-        install.packages(${{ env.R_PACKAGES }},
-                         type = 'binary',
-                         repos = 'http://cloud.r-project.org',
-                         dependencies = c('Depends', 'Imports', 'LinkingTo'))
+        key: ${{ runner.os }}-r-${{ matrix.config.r }}-6-${{ hashFiles('R-package/DESCRIPTION') }}
+        restore-keys: ${{ runner.os }}-r-${{ matrix.config.r }}-6-${{ hashFiles('R-package/DESCRIPTION') }}

    - uses: actions/setup-python@v2
      with:
        python-version: "3.8"
        architecture: 'x64'

-    - name: Test R
-      run: |
-        python tests/ci_build/test_r_package.py --compiler='${{ matrix.config.compiler }}' --build-tool='${{ matrix.config.build }}'
-
-  test-R-CRAN:
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - {r: 'release'}
-
-    env:
-      _R_CHECK_EXAMPLE_TIMING_CPU_TO_ELAPSED_THRESHOLD_: 2.5
-      MAKE: "make -j$(nproc)"
-
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        submodules: 'true'
-
-    - uses: r-lib/actions/setup-r@v2
-      with:
-        r-version: ${{ matrix.config.r }}
-
    - uses: r-lib/actions/setup-tinytex@v2

-    - name: Install system packages
-      run: |
-        sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev libssh2-1-dev libgit2-dev pandoc pandoc-citeproc libglpk-dev
-
-    - name: Cache R packages
-      uses: actions/cache@v2
-      with:
-        path: ${{ env.R_LIBS_USER }}
-        key: ${{ runner.os }}-r-${{ matrix.config.r }}-5-${{ hashFiles('R-package/DESCRIPTION') }}
-        restore-keys: ${{ runner.os }}-r-${{ matrix.config.r }}-5-${{ hashFiles('R-package/DESCRIPTION') }}
-
    - name: Install dependencies
      shell: Rscript {0}
      run: |
-        install.packages(${{ env.R_PACKAGES }},
-                         repos = 'http://cloud.r-project.org',
-                         dependencies = c('Depends', 'Imports', 'LinkingTo'))
-        install.packages('igraph', repos = 'http://cloud.r-project.org', dependencies = c('Depends', 'Imports', 'LinkingTo'))
+        source("./R-package/tests/helper_scripts/install_deps.R")

-    - name: Check R Package
+    - name: Test R
      run: |
-        # Print stacktrace upon success of failure
-        make Rcheck || tests/ci_build/print_r_stacktrace.sh fail
-        tests/ci_build/print_r_stacktrace.sh success
+        python tests/ci_build/test_r_package.py --compiler='${{ matrix.config.compiler }}' --build-tool="${{ matrix.config.build }}" --task=check
+
+  test-R-on-Debian:
+    name: Test R package on Debian
+    runs-on: ubuntu-latest
+    container:
+      image: rhub/debian-gcc-devel
+
+    steps:
+    - name: Install system dependencies
+      run: |
+        # Must run before checkout to have the latest git installed.
+        # No need to add pandoc, the container has it figured out.
+        apt update && apt install libcurl4-openssl-dev libssl-dev libssh2-1-dev libgit2-dev libglpk-dev libxml2-dev libharfbuzz-dev libfribidi-dev git -y
+
+    - uses: actions/checkout@v2
+      with:
+        submodules: 'true'
+
+    - name: Install dependencies
+      shell: bash -l {0}
+      run: |
+        /tmp/R-devel/bin/Rscript -e "source('./R-package/tests/helper_scripts/install_deps.R')"
+
+    - name: Test R
+      shell: bash -l {0}
+      run: |
+        python3 tests/ci_build/test_r_package.py --r=/tmp/R-devel/bin/R --build-tool=autotools --task=check
+
+    - uses: dorny/paths-filter@v2
+      id: changes
+      with:
+        filters: |
+          r_package:
+            - 'R-package/**'
+
+    - name: Run document check
+      if: steps.changes.outputs.r_package == 'true'
+      run: |
+        python3 tests/ci_build/test_r_package.py --r=/tmp/R-devel/bin/R --task=doc
--- a/145
+++ b/145
@ -1,145 +0,0 @@
-ifndef DMLC_CORE
-	DMLC_CORE = dmlc-core
-endif
-
-ifndef RABIT
-	RABIT = rabit
-endif
-
-ROOTDIR = $(CURDIR)
-
-# workarounds for some buggy old make & msys2 versions seen in windows
-ifeq (NA, $(shell test ! -d "$(ROOTDIR)" && echo NA ))
-        $(warning Attempting to fix non-existing ROOTDIR [$(ROOTDIR)])
-        ROOTDIR := $(shell pwd)
-        $(warning New ROOTDIR [$(ROOTDIR)] $(shell test -d "$(ROOTDIR)" && echo " is OK" ))
-endif
-MAKE_OK := $(shell "$(MAKE)" -v 2> /dev/null)
-ifndef MAKE_OK
-        $(warning Attempting to recover non-functional MAKE [$(MAKE)])
-        MAKE := $(shell which make 2> /dev/null)
-        MAKE_OK := $(shell "$(MAKE)" -v 2> /dev/null)
-endif
-$(warning MAKE [$(MAKE)] - $(if $(MAKE_OK),checked OK,PROBLEM))
-
-include $(DMLC_CORE)/make/dmlc.mk
-
-# set compiler defaults for OSX versus *nix
-# let people override either
-OS := $(shell uname)
-ifeq ($(OS), Darwin)
-ifndef CC
-export CC = $(if $(shell which clang), clang, gcc)
-endif
-ifndef CXX
-export CXX = $(if $(shell which clang++), clang++, g++)
-endif
-else
-# linux defaults
-ifndef CC
-export CC = gcc
-endif
-ifndef CXX
-export CXX = g++
-endif
-endif
-
-export CFLAGS= -DDMLC_LOG_CUSTOMIZE=1 -std=c++14 -Wall -Wno-unknown-pragmas -Iinclude $(ADD_CFLAGS)
-CFLAGS += -I$(DMLC_CORE)/include -I$(RABIT)/include -I$(GTEST_PATH)/include
-
-ifeq ($(TEST_COVER), 1)
-	CFLAGS += -g -O0 -fprofile-arcs -ftest-coverage
-else
-	CFLAGS += -O3 -funroll-loops
-endif
-
-ifndef LINT_LANG
-	LINT_LANG= "all"
-endif
-
-# specify tensor path
-.PHONY: clean all lint clean_all doxygen rcpplint pypack Rpack Rbuild Rcheck
-
-build/%.o: src/%.cc
-	@mkdir -p $(@D)
-	$(CXX) $(CFLAGS) -MM -MT build/$*.o $< >build/$*.d
-	$(CXX) -c $(CFLAGS) $< -o $@
-
-# The should be equivalent to $(ALL_OBJ)  except for build/cli_main.o
-amalgamation/xgboost-all0.o: amalgamation/xgboost-all0.cc
-	$(CXX) -c $(CFLAGS) $< -o $@
-
-rcpplint:
-	python3 dmlc-core/scripts/lint.py xgboost ${LINT_LANG} R-package/src
-
-lint: rcpplint
-	python3 dmlc-core/scripts/lint.py --exclude_path python-package/xgboost/dmlc-core \
-	  python-package/xgboost/include python-package/xgboost/lib \
-	  python-package/xgboost/make python-package/xgboost/rabit \
-	  python-package/xgboost/src --pylint-rc ${PWD}/python-package/.pylintrc xgboost \
-	  ${LINT_LANG} include src python-package
-
-ifeq ($(TEST_COVER), 1)
-cover: check
-	@- $(foreach COV_OBJ, $(COVER_OBJ), \
-		gcov -pbcul -o $(shell dirname $(COV_OBJ)) $(COV_OBJ) > gcov.log || cat gcov.log; \
-	)
-endif
-
-
-clean:
-	$(RM) -rf build lib bin *~ */*~ */*/*~ */*/*/*~ */*.o */*/*.o */*/*/*.o #xgboost
-	$(RM) -rf build_tests *.gcov tests/cpp/xgboost_test
-	if [ -d "R-package/src" ]; then \
-		cd R-package/src; \
-		$(RM) -rf rabit src include dmlc-core amalgamation *.so *.dll; \
-		cd $(ROOTDIR); \
-	fi
-
-clean_all: clean
-	cd $(DMLC_CORE); "$(MAKE)" clean; cd $(ROOTDIR)
-	cd $(RABIT); "$(MAKE)" clean; cd $(ROOTDIR)
-
-# create pip source dist (sdist) pack for PyPI
-pippack: clean_all
-	cd python-package; python setup.py sdist; mv dist/*.tar.gz ..; cd ..
-
-# Script to make a clean installable R package.
-Rpack: clean_all
-	rm -rf xgboost xgboost*.tar.gz
-	cp -r R-package xgboost
-	rm -rf xgboost/src/*.o xgboost/src/*.so xgboost/src/*.dll
-	rm -rf xgboost/src/*/*.o
-	rm -rf xgboost/demo/*.model xgboost/demo/*.buffer xgboost/demo/*.txt
-	rm -rf xgboost/demo/runall.R
-	cp -r src xgboost/src/src
-	cp -r include xgboost/src/include
-	cp -r amalgamation xgboost/src/amalgamation
-	mkdir -p xgboost/src/rabit
-	cp -r rabit/include xgboost/src/rabit/include
-	cp -r rabit/src xgboost/src/rabit/src
-	rm -rf xgboost/src/rabit/src/*.o
-	mkdir -p xgboost/src/dmlc-core
-	cp -r dmlc-core/include xgboost/src/dmlc-core/include
-	cp -r dmlc-core/src xgboost/src/dmlc-core/src
-	cp ./LICENSE xgboost
-	cat R-package/src/Makevars.in|sed '2s/.*/PKGROOT=./' > xgboost/src/Makevars.in
-	cat R-package/src/Makevars.win|sed '2s/.*/PKGROOT=./' > xgboost/src/Makevars.win
-	rm -f xgboost/src/Makevars.win-e   # OSX sed create this extra file; remove it
-	bash R-package/remove_warning_suppression_pragma.sh
-	bash xgboost/remove_warning_suppression_pragma.sh
-	rm xgboost/remove_warning_suppression_pragma.sh
-	rm xgboost/CMakeLists.txt
-	rm -rfv xgboost/tests/helper_scripts/
-
-R ?= R
-
-Rbuild: Rpack
-	$(R) CMD build xgboost
-	rm -rf xgboost
-
-Rcheck: Rbuild
-	$(R) CMD check --as-cran xgboost*.tar.gz
-
-include build/*.d
-include build/*/*.d
--- a/R-package/demo/runall.R
+++ b/R-package/demo/runall.R
@ -1,4 +1,4 @@
-# running all scripts in demo folder
+# running all scripts in demo folder, removed during packaging.
 demo(basic_walkthrough, package = 'xgboost')
 demo(custom_objective, package = 'xgboost')
 demo(boost_from_prediction, package = 'xgboost')
--- a/R-package/tests/helper_scripts/install_deps.R
+++ b/R-package/tests/helper_scripts/install_deps.R
@ -0,0 +1,50 @@
+## Install dependencies of R package for testing. The list might not be
+## up-to-date, check DESCRIPTION for the latest list and update this one if
+## inconsistent is found.
+pkgs <- c(
+  ## CI
+  "devtools",
+  "XML",
+  "cplm",
+  "e1071",
+  ## suggests
+  "knitr",
+  "rmarkdown",
+  "ggplot2",
+  "DiagrammeR",
+  "Ckmeans.1d.dp",
+  "vcd",
+  "testthat",
+  "lintr",
+  "igraph",
+  "float",
+  "crayon",
+  "titanic",
+  ## imports
+  "Matrix",
+  "methods",
+  "data.table",
+  "jsonlite"
+)
+
+ncpus <- parallel::detectCores()
+print(paste0("Using ", ncpus, " cores to install dependencies."))
+
+if (.Platform$OS.type == "unix") {
+  print("Installing source packages on unix.")
+  install.packages(
+    pkgs,
+    repo = "https://cloud.r-project.org",
+    dependencies = c("Depends", "Imports", "LinkingTo"),
+    Ncpus = parallel::detectCores()
+  )
+} else {
+  print("Installing binary packages on Windows.")
+  install.packages(
+    pkgs,
+    repo = "https://cloud.r-project.org",
+    dependencies = c("Depends", "Imports", "LinkingTo"),
+    Ncpus = parallel::detectCores(),
+    type = "binary"
+  )
+}
--- a/doc/build.rst
+++ b/doc/build.rst
@ -503,10 +503,3 @@ XGBoost uses `Sphinx <https://www.sphinx-doc.org/en/stable/>`_ for documentation
  Checkout the ``requirements.txt`` file under ``doc/``

 Under ``xgboost/doc`` directory, run ``make <format>`` with ``<format>`` replaced by the format you want.  For a list of supported formats, run ``make help`` under the same directory.
-
-*********
-Makefiles
-*********
-
-It's only used for creating shorthands for running linters, performing packaging tasks
-etc.  So the remaining makefiles are legacy.
--- a/doc/contrib/coding_guide.rst
+++ b/doc/contrib/coding_guide.rst
@ -39,12 +39,6 @@ Code Style
  - This is mainly to be consistent with the rest of the project.
  - Another reason is we will be able to check style automatically with a linter.

- You can check the style of the code by typing the following command at root folder.
-
-  .. code-block:: bash
-
-    make rcpplint
-
 - When needed, you can disable the linter warning of certain line with ``// NOLINT(*)`` comments.
 - We use `roxygen <https://cran.r-project.org/web/packages/roxygen2/vignettes/roxygen2.html>`_ for documenting the R package.

@ -79,6 +73,7 @@ The following steps are followed to add a new Rmarkdown vignettes:

 The reason we do this is to avoid exploded repo size due to generated images.

+
 R package versioning
 ====================
 See :ref:`release`.
@ -89,6 +84,11 @@ According to `R extension manual <https://cran.r-project.org/doc/manuals/r-relea
 it is good practice to register native routines and to disable symbol search. When any changes or additions are made to the
 C++ interface of the R package, please make corresponding changes in ``src/init.c`` as well.

+Generating the Package and Running Tests
+========================================
+
+The source layout of XGBoost is a bit unusual to normal R packages as XGBoost is primarily written in C++ with multiple language bindings in mind. As a result, some special cares need to be taken to generate a standard R tarball. Most of the tests are being run on CI, and as a result, the best way to see how things work is by looking at the CI configuration files (GitHub action, at the time of writing). There are helper scripts in ``tests/ci_build`` and ``R-package/tests/helper_scripts`` for running various checks including linter and making the standard tarball.
+
 *********************************
 Running Formatting Checks Locally
 *********************************
--- a/tests/ci_build/lint_python.py
+++ b/tests/ci_build/lint_python.py
@ -6,12 +6,13 @@ from multiprocessing import Pool, cpu_count
 from typing import Dict, Tuple

 from pylint import epylint
-from test_utils import DirectoryExcursion
+from test_utils import DirectoryExcursion, print_time, record_time

 CURDIR = os.path.normpath(os.path.abspath(os.path.dirname(__file__)))
 PROJECT_ROOT = os.path.normpath(os.path.join(CURDIR, os.path.pardir, os.path.pardir))


+@record_time
 def run_black(rel_path: str) -> bool:
    cmd = ["black", "-q", "--check", rel_path]
    ret = subprocess.run(cmd).returncode
@ -27,10 +28,12 @@ Please run the following command on your machine to address the formatting error
    return True


+@record_time
 def run_isort(rel_path: str) -> bool:
    cmd = ["isort", "--check", "--profile=black", rel_path]
    ret = subprocess.run(cmd).returncode
    if ret != 0:
+        subprocess.run(["isort", "--version"])
        msg = """
 Please run the following command on your machine to address the formatting error:

@ -41,6 +44,7 @@ Please run the following command on your machine to address the formatting error
    return True


+@record_time
 def run_mypy(rel_path: str) -> bool:
    with DirectoryExcursion(os.path.join(PROJECT_ROOT, "python-package")):
        path = os.path.join(PROJECT_ROOT, rel_path)
@ -117,17 +121,13 @@ class PyLint:
        return nerr == 0


-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description=(
-            "Run static checkers for XGBoost, see `python_lint.yml' "
-            "conda env file for a list of dependencies."
-        )
-    )
-    parser.add_argument("--format", type=int, choices=[0, 1], default=1)
-    parser.add_argument("--type-check", type=int, choices=[0, 1], default=1)
-    parser.add_argument("--pylint", type=int, choices=[0, 1], default=1)
-    args = parser.parse_args()
+@record_time
+def run_pylint() -> bool:
+    return PyLint()()
+
+
+@record_time
+def main(args: argparse.Namespace) -> None:
    if args.format == 1:
        black_results = [
            run_black(path)
@ -148,6 +148,8 @@ if __name__ == "__main__":
                "tests/python/test_quantile_dmatrix.py",
                "tests/python-gpu/test_gpu_data_iterator.py",
                "tests/ci_build/lint_python.py",
+                "tests/ci_build/test_r_package.py",
+                "tests/ci_build/test_utils.py",
                "tests/test_distributed/test_with_spark/",
                "tests/test_distributed/test_gpu_with_spark/",
                # demo
@ -177,7 +179,7 @@ if __name__ == "__main__":
                "doc/",
            ]
        ]
-        if not all(black_results):
+        if not all(isort_results):
            sys.exit(-1)

    if args.type_check == 1:
@ -194,6 +196,8 @@ if __name__ == "__main__":
                "tests/python/test_data_iterator.py",
                "tests/python-gpu/test_gpu_data_iterator.py",
                "tests/ci_build/lint_python.py",
+                "tests/ci_build/test_r_package.py",
+                "tests/ci_build/test_utils.py",
                "tests/test_distributed/test_with_spark/test_data.py",
                "tests/test_distributed/test_gpu_with_spark/test_data.py",
                "tests/test_distributed/test_gpu_with_dask/test_gpu_with_dask.py",
@ -202,5 +206,22 @@ if __name__ == "__main__":
            sys.exit(-1)

    if args.pylint == 1:
-        if not PyLint()():
+        if not run_pylint():
            sys.exit(-1)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description=(
+            "Run static checkers for XGBoost, see `python_lint.yml' "
+            "conda env file for a list of dependencies."
+        )
+    )
+    parser.add_argument("--format", type=int, choices=[0, 1], default=1)
+    parser.add_argument("--type-check", type=int, choices=[0, 1], default=1)
+    parser.add_argument("--pylint", type=int, choices=[0, 1], default=1)
+    args = parser.parse_args()
+    try:
+        main(args)
+    finally:
+        print_time()
--- a/tests/ci_build/print_r_stacktrace.sh
+++ b/tests/ci_build/print_r_stacktrace.sh
@ -1,23 +0,0 @@
-#!/bin/bash
-
-# To be called when R package tests have failed
-
-set -e
-set -x
-
-flag="$1"
-
-if [ -f "xgboost.Rcheck/00install.out" ]; then
-  echo "===== xgboost.Rcheck/00install.out ===="
-  cat xgboost.Rcheck/00install.out
-fi
-
-if [ -f "xgboost.Rcheck/00check.log" ]; then
-  printf "\n\n===== xgboost.Rcheck/00check.log ====\n"
-  cat xgboost.Rcheck/00check.log
-fi
-
-if [[ "$flag" == "fail" ]]
-then
-  exit 1
-fi
--- a/tests/ci_build/test_r_package.py
+++ b/tests/ci_build/test_r_package.py
@ -1,95 +1,315 @@
+"""Utilities for packaging R code and running tests."""
 import argparse
 import os
+import shutil
 import subprocess
-from time import time
+from pathlib import Path
+from platform import system

-from test_utils import DirectoryExcursion
+from test_utils import DirectoryExcursion, cd, print_time, record_time

 ROOT = os.path.normpath(
-    os.path.join(os.path.dirname(os.path.abspath(__file__)), os.path.pardir,
-                 os.path.pardir))
-r_package = os.path.join(ROOT, 'R-package')
+    os.path.join(
+        os.path.dirname(os.path.abspath(__file__)), os.path.pardir, os.path.pardir
+    )
+)
+r_package = os.path.join(ROOT, "R-package")


-def get_mingw_bin():
-    return os.path.join('c:/rtools40/mingw64/', 'bin')
+def get_mingw_bin() -> str:
+    return os.path.join("c:/rtools40/mingw64/", "bin")


-def test_with_autotools(args):
-    with DirectoryExcursion(r_package):
-        mingw_bin = get_mingw_bin()
-        CXX = os.path.join(mingw_bin, 'g++.exe')
-        CC = os.path.join(mingw_bin, 'gcc.exe')
-        cmd = ['R.exe', 'CMD', 'INSTALL', str(os.path.curdir)]
+@cd(ROOT)
+@record_time
+def pack_rpackage() -> Path:
+    """Compose the directory used for creating R package tar ball."""
+    dest = Path("xgboost")
+
+    def pkgroot(path: str) -> None:
+        """Change makefiles according to the package layout."""
+        with open(Path("R-package") / "src" / path, "r") as fd:
+            makefile = fd.read()
+            makefile = makefile.replace("PKGROOT=../../", "PKGROOT=.", 1)
+        with open(dest / "src" / path, "w") as fd:
+            fd.write(makefile)
+
+    output = subprocess.run(["git", "clean", "-xdf", "--dry-run"], capture_output=True)
+    if output.returncode != 0:
+        raise ValueError("Failed to check git repository status.", output)
+    would_remove = output.stdout.decode("utf-8").strip().split("\n")
+
+    if would_remove and not all(f.find("tests/ci_build") != -1 for f in would_remove):
+        raise ValueError(
+            "\n".join(would_remove) + "\nPlease cleanup the working git repository."
+        )
+
+    shutil.copytree("R-package", dest)
+    os.remove(dest / "demo" / "runall.R")
+    # core
+    shutil.copytree("src", dest / "src" / "src")
+    shutil.copytree("include", dest / "src" / "include")
+    shutil.copytree("amalgamation", dest / "src" / "amalgamation")
+    # rabit
+    rabit = Path("rabit")
+    os.mkdir(dest / "src" / rabit)
+    shutil.copytree(rabit / "src", dest / "src" / "rabit" / "src")
+    shutil.copytree(rabit / "include", dest / "src" / "rabit" / "include")
+    # dmlc-core
+    dmlc_core = Path("dmlc-core")
+    os.mkdir(dest / "src" / dmlc_core)
+    shutil.copytree(dmlc_core / "include", dest / "src" / "dmlc-core" / "include")
+    shutil.copytree(dmlc_core / "src", dest / "src" / "dmlc-core" / "src")
+    # makefile & license
+    shutil.copyfile("LICENSE", dest / "LICENSE")
+    osxmakef = dest / "src" / "Makevars.win-e"
+    if os.path.exists(osxmakef):
+        os.remove(osxmakef)
+    pkgroot("Makevars.in")
+    pkgroot("Makevars.win")
+    # misc
+    rwsp = Path("R-package") / "remove_warning_suppression_pragma.sh"
+    if system() != "Windows":
+        subprocess.check_call(rwsp)
+    rwsp = dest / "remove_warning_suppression_pragma.sh"
+    if system() != "Windows":
+        subprocess.check_call(rwsp)
+    os.remove(rwsp)
+    os.remove(dest / "CMakeLists.txt")
+    shutil.rmtree(dest / "tests" / "helper_scripts")
+    return dest
+
+
+@cd(ROOT)
+@record_time
+def build_rpackage(path: str) -> str:
+    def find_tarball() -> str:
+        found = []
+        for root, subdir, files in os.walk("."):
+            for f in files:
+                if f.endswith(".tar.gz") and f.startswith("xgboost"):
+                    found.append(os.path.join(root, f))
+        if not found:
+            raise ValueError("Failed to find output tar ball.")
+        if len(found) > 1:
+            raise ValueError("Found more than one packages:", found)
+        return found[0]
+
    env = os.environ.copy()
-        env.update({'CC': CC, 'CXX': CXX, "MAKE": "make -j$(nproc)"})
+    print("Ncpus:", f"{os.cpu_count()}")
+    env.update({"MAKEFLAGS": f"-j{os.cpu_count()}"})
+    subprocess.check_call([R, "CMD", "build", path], env=env)
+
+    tarball = find_tarball()
+    return tarball
+
+
+@cd(ROOT)
+@record_time
+def check_rpackage(path: str) -> None:
+    env = os.environ.copy()
+    print("Ncpus:", f"{os.cpu_count()}")
+    env.update(
+        {
+            "MAKEFLAGS": f"-j{os.cpu_count()}",
+            # cran specific environment variables
+            "_R_CHECK_EXAMPLE_TIMING_CPU_TO_ELAPSED_THRESHOLD_": str(2.5),
+        }
+    )
+
+    # Actually we don't run this check on windows due to dependency issue.
+    if system() == "Windows":
+        # make sure compiler from rtools is used.
+        mingw_bin = get_mingw_bin()
+        CXX = os.path.join(mingw_bin, "g++.exe")
+        CC = os.path.join(mingw_bin, "gcc.exe")
+        env.update({"CC": CC, "CXX": CXX})
+
+    status = subprocess.run([R, "CMD", "check", "--as-cran", path], env=env)
+    with open(Path("xgboost.Rcheck") / "00check.log", "r") as fd:
+        check_log = fd.read()
+
+    with open(Path("xgboost.Rcheck") / "00install.out", "r") as fd:
+        install_log = fd.read()
+
+    msg = f"""
+----------------------- Install ----------------------
+{install_log}
+
+-----------------------  Check -----------------------
+{check_log}
+
+    """
+
+    if status.returncode != 0:
+        print(msg)
+        raise ValueError("Failed r package check.")
+
+    if check_log.find("WARNING") != -1:
+        print(msg)
+        raise ValueError("Has unresolved warnings.")
+    if check_log.find("Examples with CPU time") != -1:
+        print(msg)
+        raise ValueError("Suspicious NOTE.")
+
+
+@cd(r_package)
+@record_time
+def check_rmarkdown() -> None:
+    assert system() != "Windows", "Document test doesn't support Windows."
+    env = os.environ.copy()
+    env.update({"MAKEFLAGS": f"-j{os.cpu_count()}"})
+    print("Checking R document with devtools.")
+    bin_dir = os.path.dirname(R)
+    rscript = os.path.join(bin_dir, "Rscript")
+    subprocess.check_call([rscript, "-e", "devtools::document()"], env=env)
+    output = subprocess.run(["git", "diff", "--name-only"], capture_output=True)
+    if len(output.stdout.decode("utf-8").strip()) != 0:
+        raise ValueError("Please run `devtools::document()`.")
+
+
+@cd(r_package)
+@record_time
+def test_with_autotools() -> None:
+    """Windows only test. No `--as-cran` check, only unittests. We don't want to manage
+    the dependencies on Windows machine.
+
+    """
+    assert system() == "Windows"
+    mingw_bin = get_mingw_bin()
+    CXX = os.path.join(mingw_bin, "g++.exe")
+    CC = os.path.join(mingw_bin, "gcc.exe")
+    cmd = [R, "CMD", "INSTALL", str(os.path.curdir)]
+    env = os.environ.copy()
+    env.update({"CC": CC, "CXX": CXX, "MAKEFLAGS": f"-j{os.cpu_count()}"})
    subprocess.check_call(cmd, env=env)
-        subprocess.check_call([
-            'R.exe', '-q', '-e',
-            "library(testthat); setwd('tests'); source('testthat.R')"
-        ])
-        subprocess.check_call([
-            'R.exe', '-q', '-e',
-            "demo(runall, package = 'xgboost')"
-        ])
+    subprocess.check_call(
+        ["R.exe", "-q", "-e", "library(testthat); setwd('tests'); source('testthat.R')"]
+    )
+    subprocess.check_call(["R.exe", "-q", "-e", "demo(runall, package = 'xgboost')"])


-def test_with_cmake(args):
-    os.mkdir('build')
-    with DirectoryExcursion('build'):
-        if args.compiler == 'mingw':
+@record_time
+def test_with_cmake(args: argparse.Namespace) -> None:
+    os.mkdir("build")
+    with DirectoryExcursion("build"):
+        if args.compiler == "mingw":
            mingw_bin = get_mingw_bin()
-            CXX = os.path.join(mingw_bin, 'g++.exe')
-            CC = os.path.join(mingw_bin, 'gcc.exe')
+            CXX = os.path.join(mingw_bin, "g++.exe")
+            CC = os.path.join(mingw_bin, "gcc.exe")
            env = os.environ.copy()
-            env.update({'CC': CC, 'CXX': CXX})
-            subprocess.check_call([
-                'cmake', os.path.pardir, '-DUSE_OPENMP=ON', '-DR_LIB=ON',
-                '-DCMAKE_CONFIGURATION_TYPES=Release', '-G', 'Unix Makefiles',
+            env.update({"CC": CC, "CXX": CXX})
+            subprocess.check_call(
+                [
+                    "cmake",
+                    os.path.pardir,
+                    "-DUSE_OPENMP=ON",
+                    "-DR_LIB=ON",
+                    "-DCMAKE_CONFIGURATION_TYPES=Release",
+                    "-G",
+                    "Unix Makefiles",
                ],
-                                  env=env)
-            subprocess.check_call(['make', '-j', 'install'])
-        elif args.compiler == 'msvc':
-            subprocess.check_call([
-                'cmake', os.path.pardir, '-DUSE_OPENMP=ON', '-DR_LIB=ON',
-                '-DCMAKE_CONFIGURATION_TYPES=Release', '-A', 'x64'
-            ])
-            subprocess.check_call([
-                'cmake', '--build', os.path.curdir, '--target', 'install',
-                '--config', 'Release'
-            ])
+                env=env,
+            )
+            subprocess.check_call(["make", "-j", "install"])
+        elif args.compiler == "msvc":
+            subprocess.check_call(
+                [
+                    "cmake",
+                    os.path.pardir,
+                    "-DUSE_OPENMP=ON",
+                    "-DR_LIB=ON",
+                    "-DCMAKE_CONFIGURATION_TYPES=Release",
+                    "-A",
+                    "x64",
+                ]
+            )
+            subprocess.check_call(
+                [
+                    "cmake",
+                    "--build",
+                    os.path.curdir,
+                    "--target",
+                    "install",
+                    "--config",
+                    "Release",
+                ]
+            )
        else:
-            raise ValueError('Wrong compiler')
+            raise ValueError("Wrong compiler")
    with DirectoryExcursion(r_package):
-        subprocess.check_call([
-            'R.exe', '-q', '-e',
-            "library(testthat); setwd('tests'); source('testthat.R')"
-        ])
-        subprocess.check_call([
-            'R.exe', '-q', '-e',
-            "demo(runall, package = 'xgboost')"
-        ])
+        subprocess.check_call(
+            [
+                R,
+                "-q",
+                "-e",
+                "library(testthat); setwd('tests'); source('testthat.R')",
+            ]
+        )
+        subprocess.check_call([R, "-q", "-e", "demo(runall, package = 'xgboost')"])


+@record_time
 def main(args: argparse.Namespace) -> None:
-    start = time()
-    if args.build_tool == 'autotools':
-        test_with_autotools(args)
+    if args.task == "build":
+        src_dir = pack_rpackage()
+        build_rpackage(src_dir)
+    elif args.task == "doc":
+        check_rmarkdown()
+    elif args.task == "check":
+        if args.build_tool == "autotools" and system() != "Windows":
+            src_dir = pack_rpackage()
+            tarball = build_rpackage(src_dir)
+            check_rpackage(tarball)
+        elif args.build_tool == "autotools":
+            test_with_autotools()
        else:
            test_with_cmake(args)
-    print("Duration:", time() - start)
+    else:
+        raise ValueError("Unexpected task.")


-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--compiler',
-                        type=str,
-                        choices=['mingw', 'msvc'],
-                        help='Compiler used for compiling CXX code.')
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description=(
+            "Helper script for making R package and running R tests on CI. There are"
+            " also other helper scripts in the R tests directory for installing"
+            " dependencies and running linter."
+        )
+    )
    parser.add_argument(
-        '--build-tool',
+        "--task",
        type=str,
-        choices=['cmake', 'autotools'],
-        help='Build tool for compiling CXX code and install R package.')
+        choices=["build", "check", "doc"],
+        default="check",
+        required=False,
+    )
+    parser.add_argument(
+        "--compiler",
+        type=str,
+        choices=["mingw", "msvc"],
+        help="Compiler used for compiling CXX code. Only relevant for windows build",
+        default="mingw",
+        required=False,
+    )
+    parser.add_argument(
+        "--build-tool",
+        type=str,
+        choices=["cmake", "autotools"],
+        help="Build tool for compiling CXX code and install R package.",
+        default="autotools",
+        required=False,
+    )
+    parser.add_argument(
+        "--r",
+        type=str,
+        default="R" if system() != "Windows" else "R.exe",
+        help="Path to the R executable.",
+    )
    args = parser.parse_args()
+    R = args.r
+
+    try:
        main(args)
+    finally:
+        print_time()
--- a/tests/ci_build/test_utils.py
+++ b/tests/ci_build/test_utils.py
@ -1,14 +1,72 @@
+"""Utilities for the CI."""
 import os
-from typing import Union
+from datetime import datetime, timedelta
+from functools import wraps
+from typing import Any, Callable, Dict, TypedDict, TypeVar, Union


 class DirectoryExcursion:
-    def __init__(self, path: Union[os.PathLike, str]):
+    def __init__(self, path: Union[os.PathLike, str]) -> None:
        self.path = path
        self.curdir = os.path.normpath(os.path.abspath(os.path.curdir))

-    def __enter__(self):
+    def __enter__(self) -> None:
        os.chdir(self.path)

-    def __exit__(self, *args):
+    def __exit__(self, *args: Any) -> None:
        os.chdir(self.curdir)
+
+
+R = TypeVar("R")
+
+
+def cd(path: Union[os.PathLike, str]) -> Callable:
+    """Decorator for changing directory temporarily."""
+
+    def chdir(func: Callable[..., R]) -> Callable[..., R]:
+        @wraps(func)
+        def inner(*args: Any, **kwargs: Any) -> R:
+            with DirectoryExcursion(path):
+                return func(*args, **kwargs)
+
+        return inner
+
+    return chdir
+
+
+Record = TypedDict("Record", {"count": int, "total": timedelta})
+timer: Dict[str, Record] = {}
+
+
+def record_time(func: Callable[..., R]) -> Callable[..., R]:
+    """Decorator for recording function runtime."""
+    global timer
+
+    @wraps(func)
+    def inner(*args: Any, **kwargs: Any) -> R:
+        if func.__name__ not in timer:
+            timer[func.__name__] = {"count": 0, "total": timedelta(0)}
+        s = datetime.now()
+        try:
+            r = func(*args, **kwargs)
+        finally:
+            e = datetime.now()
+            timer[func.__name__]["count"] += 1
+            timer[func.__name__]["total"] += e - s
+        return r
+
+    return inner
+
+
+def print_time() -> None:
+    """Print all recorded items by :py:func:`record_time`."""
+    global timer
+    for k, v in timer.items():
+        print(
+            "Name:",
+            k,
+            "Called:",
+            v["count"],
+            "Elapsed:",
+            f"{v['total'].seconds} secs",
+        )