[CI] Add noLD R test (#6382)

* [CI] Add noLD test

* Make noLD test only trigger with a PR comment

* [CI] Don't install stringi

* Add the Titanic example as a unit test

* Document trigger

* add to index

* Clarify that it needs to be a review comment

.github/workflows/main.yml

@@ -7,7 +7,7 @@ name: XGBoost-CI
 on: [push, pull_request]
 
 env:
-  R_PACKAGES: c('XML', 'igraph', 'data.table', 'magrittr', 'stringi', 'ggplot2', 'DiagrammeR', 'Ckmeans.1d.dp', 'vcd', 'testthat', 'lintr', 'knitr', 'rmarkdown', 'e1071', 'cplm', 'devtools', 'float')
+  R_PACKAGES: c('XML', 'igraph', 'data.table', 'magrittr', 'ggplot2', 'DiagrammeR', 'Ckmeans.1d.dp', 'vcd', 'testthat', 'lintr', 'knitr', 'rmarkdown', 'e1071', 'cplm', 'devtools', 'float', 'titanic')
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
@@ -77,7 +77,7 @@ jobs:
     - name: Install system packages
       run: |
         sudo apt-get install -y --no-install-recommends ninja-build
-    - uses: conda-incubator/setup-miniconda@v1
+    - uses: conda-incubator/setup-miniconda@v2
       with:
         auto-update-conda: true
         python-version: ${{ matrix.python-version }}

.github/workflows/r_nold.yml

@@ -0,0 +1,44 @@
+# Run R tests with noLD R. Only triggered by a pull request review
+# See discussion at https://github.com/dmlc/xgboost/pull/6378
+
+name: XGBoost-R-noLD
+
+on:
+  pull_request_review_comment:
+    types: [created]
+
+env:
+  R_PACKAGES: c('XML', 'igraph', 'data.table', 'magrittr', 'ggplot2', 'DiagrammeR', 'Ckmeans.1d.dp', 'vcd', 'testthat', 'lintr', 'knitr', 'rmarkdown', 'e1071', 'cplm', 'devtools', 'float', 'titanic')
+
+jobs:
+  test-R-noLD:
+    if: github.event.comment.body == '/gha run r-nold-test' && contains('OWNER,MEMBER,COLLABORATOR', github.event.comment.author_association)
+    timeout-minutes: 120
+    runs-on: ubuntu-latest
+    container: rhub/debian-gcc-devel-nold
+    steps:
+    - name: Install git and system packages
+      shell: bash
+      run: |
+        apt-get update && apt-get install -y git libcurl4-openssl-dev libssl-dev libssh2-1-dev libgit2-dev libxml2-dev
+
+    - uses: actions/checkout@v2
+      with:
+        submodules: 'true'
+
+    - name: Install dependencies
+      shell: bash
+      run: |
+        cat > install_libs.R <<EOT
+          install.packages(${{ env.R_PACKAGES }},
+                           repos = 'http://cloud.r-project.org',
+                           dependencies = c('Depends', 'Imports', 'LinkingTo'))
+        EOT
+        /tmp/R-devel/bin/Rscript install_libs.R
+
+    - name: Run R tests
+      shell: bash
+      run: |
+        cd R-package && \
+        /tmp/R-devel/bin/R CMD INSTALL . && \
+        /tmp/R-devel/bin/R -q -e "library(testthat); setwd('tests'); source('testthat.R')"

R-package/DESCRIPTION

@@ -55,7 +55,8 @@ Suggests:
     igraph (>= 1.0.1),
     jsonlite,
     float,
-    crayon
+    crayon,
+    titanic
 Depends:
     R (>= 3.3.0)
 Imports:

R-package/tests/testthat/test_callbacks.R

@@ -2,6 +2,7 @@
 
 require(xgboost)
 require(data.table)
+require(titanic)
 
 context("callbacks")
 
@@ -254,6 +255,26 @@ test_that("early stopping using a specific metric works", {
   expect_equal(logloss_log, logloss_pred, tolerance = 1e-5)
 })
 
+test_that("early stopping works with titanic", {
+  # This test was inspired by https://github.com/dmlc/xgboost/issues/5935
+  # It catches possible issues on noLD R
+  titanic <- titanic::titanic_train
+  titanic$Pclass <-  as.factor(titanic$Pclass)
+  dtx <- model.matrix(~ 0 + ., data = titanic[, c("Pclass", "Sex")])
+  dty <- titanic$Survived
+
+  xgboost::xgboost(
+    data = dtx,
+    label = dty,
+    objective = "binary:logistic",
+    eval_metric = "auc",
+    nrounds = 100,
+    early_stopping_rounds = 3
+  )
+
+  expect_true(TRUE)  # should not crash
+})
+
 test_that("early stopping xgb.cv works", {
   set.seed(11)
   expect_output(

doc/contrib/ci.rst

@@ -0,0 +1,27 @@
+####################################
+Automated testing in XGBoost project
+####################################
+
+This document collects tips for using the Continuous Integration (CI) service of the XGBoost
+project.
+
+**Contents**
+
+.. contents::
+  :backlinks: none
+  :local:
+
+**************
+GitHub Actions
+**************
+The configuration files are located under the directory
+`.github/workflows <https://github.com/dmlc/xgboost/tree/master/.github/workflows>`_.
+
+Most of the tests listed in the configuration files run automatically for every incoming pull
+requests and every update to branches. A few tests however require manual activation:
+
+* R tests with ``noLD`` option: Run R tests using a custom-built R with compilation flag
+  ``--disable-long-double``. See `this page <https://blog.r-hub.io/2019/05/21/nold/>`_ for more
+  details about noLD. This is a requirement for keeping XGBoost on CRAN (the R package index).
+  To invoke this test suite for a particular pull request, simply add a review comment
+  ``/gha run r-nold-test``. (Ordinary comment won't work. It needs to be a review comment.)

doc/contrib/index.rst

@@ -27,3 +27,4 @@ Here are guidelines for contributing to various aspect of the XGBoost project:
   Docs and Examples <docs>
   git_guide
   release
+  ci