diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 9d4196feb..788088b5f 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -81,7 +81,7 @@ jobs:
       run: |
         cd R-package
         R.exe CMD INSTALL .
-        Rscript.exe tests/run_lint.R
+        Rscript.exe tests/helper_scripts/run_lint.R
 
 
   test-with-R:
diff --git a/Makefile b/Makefile
index 20ddace00..f82513a4c 100644
--- a/Makefile
+++ b/Makefile
@@ -133,15 +133,16 @@ Rpack: clean_all
 	sed -i -e 's/@BACKTRACE_LIB@//g' xgboost/src/Makevars.win
 	sed -i -e 's/@OPENMP_LIB@//g' xgboost/src/Makevars.win
 	rm -f xgboost/src/Makevars.win-e   # OSX sed create this extra file; remove it
-	bash R-package/remove_warning_suppression_pragma.sh
+	bash xgboost/remove_warning_suppression_pragma.sh
 	rm xgboost/remove_warning_suppression_pragma.sh
+	rm -rfv xgboost/tests/helper_scripts/
 
 Rbuild: Rpack
 	R CMD build --no-build-vignettes xgboost
 	rm -rf xgboost
 
 Rcheck: Rbuild
-	R CMD check xgboost*.tar.gz
+	R CMD check --as-cran xgboost*.tar.gz
 
 -include build/*.d
 -include build/*/*.d
diff --git a/R-package/R/xgb.train.R b/R-package/R/xgb.train.R
index a6755f5fa..85021959c 100644
--- a/R-package/R/xgb.train.R
+++ b/R-package/R/xgb.train.R
@@ -132,7 +132,7 @@
 #'   \itemize{
 #'      \item \code{rmse} root mean square error. \url{http://en.wikipedia.org/wiki/Root_mean_square_error}
 #'      \item \code{logloss} negative log-likelihood. \url{http://en.wikipedia.org/wiki/Log-likelihood}
-#'      \item \code{mlogloss} multiclass logloss. \url{http://wiki.fast.ai/index.php/Log_Loss}
+#'      \item \code{mlogloss} multiclass logloss. \url{https://scikit-learn.org/stable/modules/generated/sklearn.metrics.log_loss.html}
 #'      \item \code{error} Binary classification error rate. It is calculated as \code{(# wrong cases) / (# all cases)}.
 #'            By default, it uses the 0.5 threshold for predicted values to define negative and positive instances.
 #'            Different threshold (e.g., 0.) could be specified as "error@0."
diff --git a/R-package/man/xgb.train.Rd b/R-package/man/xgb.train.Rd
index 94db595cb..1fcd6c0e3 100644
--- a/R-package/man/xgb.train.Rd
+++ b/R-package/man/xgb.train.Rd
@@ -217,7 +217,7 @@ The following is the list of built-in metrics for which Xgboost provides optimiz
   \itemize{
      \item \code{rmse} root mean square error. \url{http://en.wikipedia.org/wiki/Root_mean_square_error}
      \item \code{logloss} negative log-likelihood. \url{http://en.wikipedia.org/wiki/Log-likelihood}
-     \item \code{mlogloss} multiclass logloss. \url{http://wiki.fast.ai/index.php/Log_Loss}
+     \item \code{mlogloss} multiclass logloss. \url{https://scikit-learn.org/stable/modules/generated/sklearn.metrics.log_loss.html}
      \item \code{error} Binary classification error rate. It is calculated as \code{(# wrong cases) / (# all cases)}.
            By default, it uses the 0.5 threshold for predicted values to define negative and positive instances.
            Different threshold (e.g., 0.) could be specified as "error@0."
diff --git a/R-package/tests/generate_models_params.R b/R-package/tests/generate_models_params.R
deleted file mode 100644
index 0f71ab22c..000000000
--- a/R-package/tests/generate_models_params.R
+++ /dev/null
@@ -1,10 +0,0 @@
-model_generator_metadata <- function() {
-  return (list(
-    kRounds = 2,
-    kRows = 1000,
-    kCols = 4,
-    kForests = 2,
-    kMaxDepth = 2,
-    kClasses = 3
-  ))
-}
diff --git a/R-package/tests/generate_models.R b/R-package/tests/helper_scripts/generate_models.R
similarity index 96%
rename from R-package/tests/generate_models.R
rename to R-package/tests/helper_scripts/generate_models.R
index 854b7944e..d38b23a19 100644
--- a/R-package/tests/generate_models.R
+++ b/R-package/tests/helper_scripts/generate_models.R
@@ -5,7 +5,14 @@ library(Matrix)
 source('./generate_models_params.R')
 
 set.seed(0)
-metadata <- model_generator_metadata()
+metadata <- list(
+  kRounds = 2,
+  kRows = 1000,
+  kCols = 4,
+  kForests = 2,
+  kMaxDepth = 2,
+  kClasses = 3
+)
 X <- Matrix(data = rnorm(metadata$kRows * metadata$kCols), nrow = metadata$kRows,
             ncol = metadata$kCols, sparse = TRUE)
 w <- runif(metadata$kRows)
diff --git a/R-package/tests/run_lint.R b/R-package/tests/helper_scripts/run_lint.R
similarity index 100%
rename from R-package/tests/run_lint.R
rename to R-package/tests/helper_scripts/run_lint.R
diff --git a/R-package/tests/testthat/test_model_compatibility.R b/R-package/tests/testthat/test_model_compatibility.R
index 7204ed891..105a60d16 100644
--- a/R-package/tests/testthat/test_model_compatibility.R
+++ b/R-package/tests/testthat/test_model_compatibility.R
@@ -1,10 +1,16 @@
 require(xgboost)
 require(jsonlite)
-source('../generate_models_params.R')
 
 context("Models from previous versions of XGBoost can be loaded")
 
-metadata <- model_generator_metadata()
+metadata <- list(
+  kRounds = 2,
+  kRows = 1000,
+  kCols = 4,
+  kForests = 2,
+  kMaxDepth = 2,
+  kClasses = 3
+)
 
 run_model_param_check <- function (config) {
   testthat::expect_equal(config$learner$learner_model_param$num_feature, '4')