Remove cleanup script in R package. (#8370)

- Bump configure.ac version.
- Remove amalgamation to reduce the build time for a single object with the added benefit that we can use parallel build during development.
- Fix c function prototype warning.
- Remove Windows automake file generation step to make the build script easier to understand.

.github/workflows/jvm_tests.yml

@@ -2,6 +2,9 @@ name: XGBoost-JVM-Tests
 
 on: [push, pull_request]
 
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
 jobs:
   test-with-jvm:
     name: Test JVM on OS ${{ matrix.os }}
@@ -9,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [windows-latest, ubuntu-latest, macos-10.15]
+        os: [windows-latest, ubuntu-latest, macos-11]
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/main.yml

@@ -6,6 +6,9 @@ name: XGBoost-CI
 # events but only for the master branch
 on: [push, pull_request]
 
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
   gtest-cpu:
@@ -14,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-10.15]
+        os: [macos-11]
     steps:
     - uses: actions/checkout@v2
       with:
@@ -129,40 +132,21 @@ jobs:
 
   lint:
     runs-on: ubuntu-latest
-    name: Code linting for Python and C++
+    name: Code linting for C++
     steps:
     - uses: actions/checkout@v2
       with:
         submodules: 'true'
     - uses: actions/setup-python@v2
       with:
-        python-version: '3.7'
+        python-version: "3.8"
         architecture: 'x64'
     - name: Install Python packages
       run: |
-        python -m pip install wheel setuptools
-        python -m pip install pylint cpplint numpy scipy scikit-learn
+        python -m pip install wheel setuptools cpplint pylint
     - name: Run lint
       run: |
-        make lint
-
-  mypy:
-    runs-on: ubuntu-latest
-    name: Type checking for Python
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        submodules: 'true'
-    - uses: actions/setup-python@v2
-      with:
-        python-version: '3.7'
-        architecture: 'x64'
-    - name: Install Python packages
-      run: |
-        python -m pip install wheel setuptools mypy pandas dask[complete] distributed
-    - name: Run mypy
-      run: |
-        make mypy
+        LINT_LANG=cpp make lint
 
   doxygen:
     runs-on: ubuntu-latest
@@ -173,7 +157,7 @@ jobs:
         submodules: 'true'
     - uses: actions/setup-python@v2
       with:
-        python-version: '3.7'
+        python-version: "3.8"
         architecture: 'x64'
     - name: Install system packages
       run: |
@@ -210,7 +194,7 @@ jobs:
         submodules: 'true'
     - uses: actions/setup-python@v2
       with:
-        python-version: '3.8'
+        python-version: "3.8"
         architecture: 'x64'
     - name: Install system packages
       run: |

.github/workflows/python_tests.yml

@@ -2,20 +2,58 @@ name: XGBoost-Python-Tests
 
 on: [push, pull_request]
 
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
 jobs:
+  python-mypy-lint:
+    runs-on: ubuntu-latest
+    name: Type and format checks for the Python package
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.8"]
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: 'true'
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        auto-update-conda: true
+        python-version: ${{ matrix.python-version }}
+        activate-environment: python_lint
+        environment-file: tests/ci_build/conda_env/python_lint.yml
+    - name: Display Conda env
+      shell: bash -l {0}
+      run: |
+        conda info
+        conda list
+    - name: Run mypy
+      shell: bash -l {0}
+      run: |
+        python tests/ci_build/lint_python.py --format=0 --type-check=1 --pylint=0
+    - name: Run formatter
+      shell: bash -l {0}
+      run: |
+        python tests/ci_build/lint_python.py --format=1 --type-check=0 --pylint=0
+    - name: Run pylint
+      shell: bash -l {0}
+      run: |
+        python tests/ci_build/lint_python.py --format=0 --type-check=0 --pylint=1
+
   python-sdist-test:
     runs-on: ${{ matrix.os }}
     name: Test installing XGBoost Python source package on ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-10.15, windows-latest]
+        os: [ubuntu-latest, macos-11, windows-latest]
         python-version: ["3.8"]
     steps:
     - uses: actions/checkout@v2
       with:
         submodules: 'true'
     - name: Install osx system dependencies
-      if: matrix.os == 'macos-10.15'
+      if: matrix.os == 'macos-11'
       run: |
         brew install ninja libomp
     - name: Install Ubuntu system dependencies
@@ -92,10 +130,11 @@ jobs:
   python-tests-on-macos:
     name: Test XGBoost Python package on ${{ matrix.config.os }}
     runs-on: ${{ matrix.config.os }}
+    timeout-minutes: 90
     strategy:
       matrix:
         config:
-          - {os: macos-10.15, python-version "3.8" }
+          - {os: macos-11, python-version "3.8" }
 
     steps:
     - uses: actions/checkout@v2
@@ -133,8 +172,7 @@ jobs:
       run: |
         cd python-package
         python --version
-        python setup.py bdist_wheel --universal
-        pip install ./dist/*.whl
+        python setup.py install
 
     - name: Test Python package
       shell: bash -l {0}

.github/workflows/python_wheels.yml

@@ -2,6 +2,9 @@ name: XGBoost-Python-Wheels
 
 on: [push, pull_request]
 
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
 jobs:
   python-wheels:
     name: Build wheel for ${{ matrix.platform_id }}
@@ -20,7 +23,7 @@ jobs:
     - name: Setup Python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.9'
+        python-version: "3.8"
     - name: Build wheels
       run: bash tests/ci_build/build_python_wheels.sh ${{ matrix.platform_id }} ${{ github.sha }}
     - name: Extract branch name

.github/workflows/r_nold.yml

@@ -10,6 +10,9 @@ on:
 env:
   R_PACKAGES: c('XML', 'igraph', 'data.table', 'ggplot2', 'DiagrammeR', 'Ckmeans.1d.dp', 'vcd', 'testthat', 'lintr', 'knitr', 'rmarkdown', 'e1071', 'cplm', 'devtools', 'float', 'titanic')
 
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
 jobs:
   test-R-noLD:
     if: github.event.comment.body == '/gha run r-nold-test' && contains('OWNER,MEMBER,COLLABORATOR', github.event.comment.author_association)

.github/workflows/r_tests.yml

@@ -6,6 +6,9 @@ env:
   R_PACKAGES: c('XML', 'data.table', 'ggplot2', 'DiagrammeR', 'Ckmeans.1d.dp', 'vcd', 'testthat', 'lintr', 'knitr', 'rmarkdown', 'e1071', 'cplm', 'devtools', 'float', 'titanic')
   GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
 
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
 jobs:
   lintr:
     runs-on: ${{ matrix.config.os }}
@@ -13,7 +16,7 @@ jobs:
     strategy:
       matrix:
         config:
-          - {os: windows-latest, r: 'release', compiler: 'mingw', build: 'autotools'}
+          - {os: ubuntu-latest, r: 'release'}
     env:
       R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
       RSPM: ${{ matrix.config.rspm }}
@@ -23,7 +26,7 @@ jobs:
       with:
         submodules: 'true'
 
-    - uses: r-lib/actions/setup-r@master
+    - uses: r-lib/actions/setup-r@v2
       with:
         r-version: ${{ matrix.config.r }}
 
@@ -31,8 +34,8 @@ jobs:
       uses: actions/cache@v2
       with:
         path: ${{ env.R_LIBS_USER }}
-        key: ${{ runner.os }}-r-${{ matrix.config.r }}-3-${{ hashFiles('R-package/DESCRIPTION') }}
-        restore-keys: ${{ runner.os }}-r-${{ matrix.config.r }}-3-${{ hashFiles('R-package/DESCRIPTION') }}
+        key: ${{ runner.os }}-r-${{ matrix.config.r }}-5-${{ hashFiles('R-package/DESCRIPTION') }}
+        restore-keys: ${{ runner.os }}-r-${{ matrix.config.r }}-5-${{ hashFiles('R-package/DESCRIPTION') }}
 
     - name: Install dependencies
       shell: Rscript {0}
@@ -49,8 +52,9 @@ jobs:
     - name: Run lintr
       run: |
         cd R-package
-        R.exe CMD INSTALL .
-        Rscript.exe tests/helper_scripts/run_lint.R
+        R CMD INSTALL .
+        # Disable lintr errors for now: https://github.com/dmlc/xgboost/issues/8012
+        Rscript tests/helper_scripts/run_lint.R || true
 
   test-with-R:
     runs-on: ${{ matrix.config.os }}
@@ -71,7 +75,7 @@ jobs:
       with:
         submodules: 'true'
 
-    - uses: r-lib/actions/setup-r@master
+    - uses: r-lib/actions/setup-r@v2
       with:
         r-version: ${{ matrix.config.r }}
 
@@ -79,24 +83,29 @@ jobs:
       uses: actions/cache@v2
       with:
         path: ${{ env.R_LIBS_USER }}
-        key: ${{ runner.os }}-r-${{ matrix.config.r }}-3-${{ hashFiles('R-package/DESCRIPTION') }}
-        restore-keys: ${{ runner.os }}-r-${{ matrix.config.r }}-3-${{ hashFiles('R-package/DESCRIPTION') }}
+        key: ${{ runner.os }}-r-${{ matrix.config.r }}-5-${{ hashFiles('R-package/DESCRIPTION') }}
+        restore-keys: ${{ runner.os }}-r-${{ matrix.config.r }}-5-${{ hashFiles('R-package/DESCRIPTION') }}
 
     - name: Install dependencies
       shell: Rscript {0}
+      if: matrix.config.os != 'windows-latest'
       run: |
         install.packages(${{ env.R_PACKAGES }},
                          repos = 'http://cloud.r-project.org',
                          dependencies = c('Depends', 'Imports', 'LinkingTo'))
-    - name: Install igraph on Windows
+
+    - name: Install binary dependencies
       shell: Rscript {0}
       if: matrix.config.os == 'windows-latest'
       run: |
-        install.packages('igraph', type='binary', dependencies = c('Depends', 'Imports', 'LinkingTo'))
+        install.packages(${{ env.R_PACKAGES }},
+                         type = 'binary',
+                         repos = 'http://cloud.r-project.org',
+                         dependencies = c('Depends', 'Imports', 'LinkingTo'))
 
     - uses: actions/setup-python@v2
       with:
-        python-version: '3.7'
+        python-version: "3.8"
         architecture: 'x64'
 
     - name: Test R
@@ -117,11 +126,11 @@ jobs:
       with:
         submodules: 'true'
 
-    - uses: r-lib/actions/setup-r@master
+    - uses: r-lib/actions/setup-r@v2
       with:
         r-version: ${{ matrix.config.r }}
 
-    - uses: r-lib/actions/setup-tinytex@master
+    - uses: r-lib/actions/setup-tinytex@v2
 
     - name: Install system packages
       run: |
@@ -131,8 +140,8 @@ jobs:
       uses: actions/cache@v2
       with:
         path: ${{ env.R_LIBS_USER }}
-        key: ${{ runner.os }}-r-${{ matrix.config.r }}-3-${{ hashFiles('R-package/DESCRIPTION') }}
-        restore-keys: ${{ runner.os }}-r-${{ matrix.config.r }}-3-${{ hashFiles('R-package/DESCRIPTION') }}
+        key: ${{ runner.os }}-r-${{ matrix.config.r }}-5-${{ hashFiles('R-package/DESCRIPTION') }}
+        restore-keys: ${{ runner.os }}-r-${{ matrix.config.r }}-5-${{ hashFiles('R-package/DESCRIPTION') }}
 
     - name: Install dependencies
       shell: Rscript {0}

.github/workflows/scorecards.yml

@@ -0,0 +1,54 @@
+name: Scorecards supply-chain security
+on:
+  # Only the default branch is supported.
+  branch_protection_rule:
+  schedule:
+    - cron: '17 2 * * 6'
+  push:
+    branches: [ "master" ]
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecards analysis
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Used to receive a badge.
+      id-token: write
+
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 # tag=v3.0.0
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@865b4092859256271290c77adbd10a43f4779972 # tag=v2.0.3
+        with:
+          results_file: results.sarif
+          results_format: sarif
+
+          # Publish the results for public repositories to enable scorecard badges. For more details, see
+          # https://github.com/ossf/scorecard-action#publishing-results. 
+          # For private repositories, `publish_results` will automatically be set to `false`, regardless 
+          # of the value entered here.
+          publish_results: true
+
+      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
+      # format to the repository Actions tab.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@6673cd052c4cd6fcf4b4e6e60ea986c889389535 # tag=v3.0.0
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard.
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@5f532563584d71fdef14ee64d17bafb34f751ce5 # tag=v1.0.26
+        with:
+          sarif_file: results.sarif

.gitignore

@@ -52,6 +52,8 @@ Debug
 R-package.Rproj
 *.cache*
 .mypy_cache/
+doxygen
+
 # java
 java/xgboost4j/target
 java/xgboost4j/tmp
@@ -97,8 +99,11 @@ metastore_db
 R-package/src/Makevars
 *.lib
 
-# Visual Studio Code
-/.vscode/
+# Visual Studio
+.vs/
+CMakeSettings.json
+*.ilk
+*.pdb
 
 # IntelliJ/CLion
 .idea
@@ -130,4 +135,7 @@ credentials.csv
 # Visual Studio code + extensions
 .vscode
 .metals
-.bloop
+.bloop
+
+# hypothesis python tests
+.hypothesis

.readthedocs.yaml

@@ -0,0 +1,35 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+submodules:
+  include: all
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.8"
+  apt_packages:
+    - graphviz
+    - cmake
+    - g++
+    - doxygen
+    - ninja-build
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+   configuration: doc/conf.py
+
+# If using Sphinx, optionally build your docs in additional formats such as PDF
+formats:
+   - pdf
+
+# Optionally declare the Python requirements required to build your docs
+python:
+  install:
+   - requirements: doc/requirements.txt
+  system_packages: true

CMakeLists.txt

@@ -1,9 +1,10 @@
 cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
-project(xgboost LANGUAGES CXX C VERSION 1.6.0)
+project(xgboost LANGUAGES CXX C VERSION 1.7.0)
 include(cmake/Utils.cmake)
 list(APPEND CMAKE_MODULE_PATH "${xgboost_SOURCE_DIR}/cmake/modules")
 cmake_policy(SET CMP0022 NEW)
 cmake_policy(SET CMP0079 NEW)
+cmake_policy(SET CMP0076 NEW)
 set(CMAKE_POLICY_DEFAULT_CMP0063 NEW)
 cmake_policy(SET CMP0063 NEW)
 
@@ -66,6 +67,7 @@ address, leak, undefined and thread.")
 ## Plugins
 option(PLUGIN_DENSE_PARSER "Build dense parser plugin" OFF)
 option(PLUGIN_RMM "Build with RAPIDS Memory Manager (RMM)" OFF)
+option(PLUGIN_FEDERATED "Build with Federated Learning" OFF)
 ## TODO: 1. Add check if DPC++ compiler is used for building
 option(PLUGIN_UPDATER_ONEAPI "DPC++ updater" OFF)
 option(ADD_PKGCONFIG "Add xgboost.pc into system." ON)
@@ -116,6 +118,20 @@ endif (BUILD_STATIC_LIB AND (R_LIB OR JVM_BINDINGS))
 if (PLUGIN_RMM AND (NOT BUILD_WITH_CUDA_CUB))
   message(SEND_ERROR "Cannot build with RMM using cub submodule.")
 endif (PLUGIN_RMM AND (NOT BUILD_WITH_CUDA_CUB))
+if (PLUGIN_FEDERATED)
+  if (CMAKE_CROSSCOMPILING)
+    message(SEND_ERROR "Cannot cross compile with federated learning support")
+  endif ()
+  if (BUILD_STATIC_LIB)
+    message(SEND_ERROR "Cannot build static lib with federated learning support")
+  endif ()
+  if (R_LIB OR JVM_BINDINGS)
+    message(SEND_ERROR "Cannot enable federated learning support when R or JVM packages are enabled.")
+  endif ()
+  if (WIN32)
+    message(SEND_ERROR "Federated learning not supported for Windows platform")
+  endif ()
+endif ()
 
 #-- Sanitizer
 if (USE_SANITIZER)
@@ -130,8 +146,8 @@ if (USE_CUDA)
   message(STATUS "Configured CUDA host compiler: ${CMAKE_CUDA_HOST_COMPILER}")
 
   enable_language(CUDA)
-  if (${CMAKE_CUDA_COMPILER_VERSION} VERSION_LESS 10.1)
-    message(FATAL_ERROR "CUDA version must be at least 10.1!")
+  if (${CMAKE_CUDA_COMPILER_VERSION} VERSION_LESS 11.0)
+    message(FATAL_ERROR "CUDA version must be at least 11.0!")
   endif()
   set(GEN_CODE "")
   format_gencode_flags("${GPU_COMPUTE_VER}" GEN_CODE)
@@ -158,6 +174,11 @@ if (USE_OPENMP)
   endif (APPLE)
   find_package(OpenMP REQUIRED)
 endif (USE_OPENMP)
+#Add for IBM i
+if (${CMAKE_SYSTEM_NAME} MATCHES "OS400")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
+  set(CMAKE_CXX_ARCHIVE_CREATE "<CMAKE_AR> -X64 qc <TARGET> <OBJECTS>")
+endif()
 
 if (USE_NCCL)
   find_package(Nccl REQUIRED)
@@ -200,6 +221,10 @@ endif (JVM_BINDINGS)
 # Plugin
 add_subdirectory(${xgboost_SOURCE_DIR}/plugin)
 
+if (PLUGIN_RMM)
+  find_package(rmm REQUIRED)
+endif (PLUGIN_RMM)
+
 #-- library
 if (BUILD_STATIC_LIB)
   add_library(xgboost STATIC)

Jenkinsfile

@@ -1,453 +0,0 @@
-#!/usr/bin/groovy
-// -*- mode: groovy -*-
-// Jenkins pipeline
-// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
-
-// Command to run command inside a docker container
-dockerRun = 'tests/ci_build/ci_build.sh'
-
-// Which CUDA version to use when building reference distribution wheel
-ref_cuda_ver = '11.0'
-
-import groovy.transform.Field
-
-@Field
-def commit_id   // necessary to pass a variable from one stage to another
-
-pipeline {
-  // Each stage specify its own agent
-  agent none
-
-  environment {
-    DOCKER_CACHE_ECR_ID = '492475357299'
-    DOCKER_CACHE_ECR_REGION = 'us-west-2'
-  }
-
-  // Setup common job properties
-  options {
-    ansiColor('xterm')
-    timestamps()
-    timeout(time: 240, unit: 'MINUTES')
-    buildDiscarder(logRotator(numToKeepStr: '10'))
-    preserveStashes()
-  }
-
-  // Build stages
-  stages {
-    stage('Jenkins Linux: Initialize') {
-      agent { label 'job_initializer' }
-      steps {
-        script {
-          def buildNumber = env.BUILD_NUMBER as int
-          if (buildNumber > 1) milestone(buildNumber - 1)
-          milestone(buildNumber)
-
-          checkoutSrcs()
-          commit_id = "${GIT_COMMIT}"
-        }
-        sh 'python3 tests/jenkins_get_approval.py'
-        stash name: 'srcs'
-      }
-    }
-    stage('Jenkins Linux: Build') {
-      agent none
-      steps {
-        script {
-          parallel ([
-            'clang-tidy': { ClangTidy() },
-            'build-cpu': { BuildCPU() },
-            'build-cpu-arm64': { BuildCPUARM64() },
-            'build-cpu-rabit-mock': { BuildCPUMock() },
-            // Build reference, distribution-ready Python wheel with CUDA 11.0
-            // using CentOS 7 image
-            'build-gpu-cuda11.0': { BuildCUDA(cuda_version: '11.0', build_rmm: true) },
-            'build-gpu-rpkg': { BuildRPackageWithCUDA(cuda_version: '11.0') },
-            'build-jvm-packages-gpu-cuda11.0': { BuildJVMPackagesWithCUDA(spark_version: '3.0.1', cuda_version: '11.0') },
-            'build-jvm-packages': { BuildJVMPackages(spark_version: '3.0.1') },
-            'build-jvm-doc': { BuildJVMDoc() }
-          ])
-        }
-      }
-    }
-    stage('Jenkins Linux: Test') {
-      agent none
-      steps {
-        script {
-          parallel ([
-            'test-python-cpu': { TestPythonCPU() },
-            'test-python-cpu-arm64': { TestPythonCPUARM64() },
-            // artifact_cuda_version doesn't apply to RMM tests; RMM tests will always match CUDA version between artifact and host env
-            'test-python-gpu-cuda11.0': { TestPythonGPU(artifact_cuda_version: '11.0', host_cuda_version: '11.0', test_rmm: true) },
-            'test-python-mgpu-cuda11.0': { TestPythonGPU(artifact_cuda_version: '11.0', host_cuda_version: '11.0', multi_gpu: true, test_rmm: true) },
-            'test-cpp-gpu-cuda11.0': { TestCppGPU(artifact_cuda_version: '11.0', host_cuda_version: '11.0', test_rmm: true) },
-            'test-jvm-jdk8': { CrossTestJVMwithJDK(jdk_version: '8', spark_version: '3.0.0') }
-          ])
-        }
-      }
-    }
-    stage('Jenkins Linux: Deploy') {
-      agent none
-      steps {
-        script {
-          parallel ([
-            'deploy-jvm-packages': { DeployJVMPackages(spark_version: '3.0.0') }
-          ])
-        }
-      }
-    }
-  }
-}
-
-// check out source code from git
-def checkoutSrcs() {
-  retry(5) {
-    try {
-      timeout(time: 2, unit: 'MINUTES') {
-        checkout scm
-        sh 'git submodule update --init'
-      }
-    } catch (exc) {
-      deleteDir()
-      error "Failed to fetch source codes"
-    }
-  }
-}
-
-def GetCUDABuildContainerType(cuda_version) {
-  return (cuda_version == ref_cuda_ver) ? 'gpu_build_centos7' : 'gpu_build'
-}
-
-def ClangTidy() {
-  node('linux && cpu_build') {
-    unstash name: 'srcs'
-    echo "Running clang-tidy job..."
-    def container_type = "clang_tidy"
-    def docker_binary = "docker"
-    def dockerArgs = "--build-arg CUDA_VERSION_ARG=11.0"
-    sh """
-    ${dockerRun} ${container_type} ${docker_binary} ${dockerArgs} python3 tests/ci_build/tidy.py --cuda-archs 75
-    """
-    deleteDir()
-  }
-}
-
-def BuildCPU() {
-  node('linux && cpu') {
-    unstash name: 'srcs'
-    echo "Build CPU"
-    def container_type = "cpu"
-    def docker_binary = "docker"
-    sh """
-    ${dockerRun} ${container_type} ${docker_binary} rm -fv dmlc-core/include/dmlc/build_config_default.h
-      # This step is not necessary, but here we include it, to ensure that DMLC_CORE_USE_CMAKE flag is correctly propagated
-      # We want to make sure that we use the configured header build/dmlc/build_config.h instead of include/dmlc/build_config_default.h.
-      # See discussion at https://github.com/dmlc/xgboost/issues/5510
-    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh -DPLUGIN_DENSE_PARSER=ON
-    ${dockerRun} ${container_type} ${docker_binary} bash -c "cd build && ctest --extra-verbose"
-    """
-    // Sanitizer test
-    def docker_extra_params = "CI_DOCKER_EXTRA_PARAMS_INIT='-e ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer -e ASAN_OPTIONS=symbolize=1 -e UBSAN_OPTIONS=print_stacktrace=1:log_path=ubsan_error.log --cap-add SYS_PTRACE'"
-    sh """
-    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh -DUSE_SANITIZER=ON -DENABLED_SANITIZERS="address;leak;undefined" \
-      -DCMAKE_BUILD_TYPE=Debug -DSANITIZER_PATH=/usr/lib/x86_64-linux-gnu/
-    ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} bash -c "cd build && ctest --exclude-regex AllTestsInDMLCUnitTests --extra-verbose"
-    """
-
-    stash name: 'xgboost_cli', includes: 'xgboost'
-    deleteDir()
-  }
-}
-
-def BuildCPUARM64() {
-  node('linux && arm64') {
-    unstash name: 'srcs'
-    echo "Build CPU ARM64"
-    def container_type = "aarch64"
-    def docker_binary = "docker"
-    def wheel_tag = "manylinux2014_aarch64"
-    sh """
-    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh --conda-env=aarch64_test -DOPEN_MP:BOOL=ON -DHIDE_CXX_SYMBOL=ON
-    ${dockerRun} ${container_type} ${docker_binary} bash -c "cd build && ctest --extra-verbose"
-    ${dockerRun} ${container_type} ${docker_binary} bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal"
-    ${dockerRun} ${container_type} ${docker_binary} python tests/ci_build/rename_whl.py python-package/dist/*.whl ${commit_id} ${wheel_tag}
-    ${dockerRun} ${container_type} ${docker_binary} bash -c "auditwheel repair --plat ${wheel_tag} python-package/dist/*.whl && python tests/ci_build/rename_whl.py wheelhouse/*.whl ${commit_id} ${wheel_tag}"
-    mv -v wheelhouse/*.whl python-package/dist/
-    # Make sure that libgomp.so is vendored in the wheel
-    ${dockerRun} ${container_type} ${docker_binary} bash -c "unzip -l python-package/dist/*.whl | grep libgomp || exit -1"
-    """
-    echo 'Stashing Python wheel...'
-    stash name: "xgboost_whl_arm64_cpu", includes: 'python-package/dist/*.whl'
-    if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) {
-      echo 'Uploading Python wheel...'
-      sh """
-      ${dockerRun} ${container_type} ${docker_binary} bash -c "source activate aarch64_test && python -m awscli s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/${BRANCH_NAME}/ --acl public-read --no-progress"
-      """
-    }
-    stash name: 'xgboost_cli_arm64', includes: 'xgboost'
-    deleteDir()
-  }
-}
-
-def BuildCPUMock() {
-  node('linux && cpu') {
-    unstash name: 'srcs'
-    echo "Build CPU with rabit mock"
-    def container_type = "cpu"
-    def docker_binary = "docker"
-    sh """
-    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_mock_cmake.sh
-    """
-    echo 'Stashing rabit C++ test executable (xgboost)...'
-    stash name: 'xgboost_rabit_tests', includes: 'xgboost'
-    deleteDir()
-  }
-}
-
-def BuildCUDA(args) {
-  node('linux && cpu_build') {
-    unstash name: 'srcs'
-    echo "Build with CUDA ${args.cuda_version}"
-    def container_type = GetCUDABuildContainerType(args.cuda_version)
-    def docker_binary = "docker"
-    def docker_args = "--build-arg CUDA_VERSION_ARG=${args.cuda_version}"
-    def arch_flag = ""
-    if (env.BRANCH_NAME != 'master' && !(env.BRANCH_NAME.startsWith('release'))) {
-      arch_flag = "-DGPU_COMPUTE_VER=75"
-    }
-    def wheel_tag = "manylinux2014_x86_64"
-    sh """
-    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_CUDA=ON -DUSE_NCCL=ON -DOPEN_MP:BOOL=ON -DHIDE_CXX_SYMBOLS=ON ${arch_flag}
-    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal"
-    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} python tests/ci_build/rename_whl.py python-package/dist/*.whl ${commit_id} ${wheel_tag}
-    """
-    if (args.cuda_version == ref_cuda_ver) {
-      sh """
-      ${dockerRun} auditwheel_x86_64 ${docker_binary} auditwheel repair --plat ${wheel_tag} python-package/dist/*.whl
-      ${dockerRun} ${container_type} ${docker_binary} ${docker_args} python tests/ci_build/rename_whl.py wheelhouse/*.whl ${commit_id} ${wheel_tag}
-      mv -v wheelhouse/*.whl python-package/dist/
-      # Make sure that libgomp.so is vendored in the wheel
-      ${dockerRun} auditwheel_x86_64 ${docker_binary} bash -c "unzip -l python-package/dist/*.whl | grep libgomp  || exit -1"
-      """
-    }
-    echo 'Stashing Python wheel...'
-    stash name: "xgboost_whl_cuda${args.cuda_version}", includes: 'python-package/dist/*.whl'
-    if (args.cuda_version == ref_cuda_ver && (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release'))) {
-      echo 'Uploading Python wheel...'
-      sh """
-      ${dockerRun} ${container_type} ${docker_binary} ${docker_args} python -m awscli s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/${BRANCH_NAME}/ --acl public-read --no-progress
-      """
-    }
-    echo 'Stashing C++ test executable (testxgboost)...'
-    stash name: "xgboost_cpp_tests_cuda${args.cuda_version}", includes: 'build/testxgboost'
-    if (args.build_rmm) {
-      echo "Build with CUDA ${args.cuda_version} and RMM"
-      container_type = "rmm"
-      docker_binary = "docker"
-      docker_args = "--build-arg CUDA_VERSION_ARG=${args.cuda_version}"
-      sh """
-      rm -rf build/
-      ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh --conda-env=gpu_test -DUSE_CUDA=ON -DUSE_NCCL=ON -DPLUGIN_RMM=ON -DBUILD_WITH_CUDA_CUB=ON ${arch_flag}
-      ${dockerRun} ${container_type} ${docker_binary} ${docker_args} bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal"
-      ${dockerRun} ${container_type} ${docker_binary} ${docker_args} python tests/ci_build/rename_whl.py python-package/dist/*.whl ${commit_id} manylinux2014_x86_64
-      """
-      echo 'Stashing Python wheel...'
-      stash name: "xgboost_whl_rmm_cuda${args.cuda_version}", includes: 'python-package/dist/*.whl'
-      echo 'Stashing C++ test executable (testxgboost)...'
-      stash name: "xgboost_cpp_tests_rmm_cuda${args.cuda_version}", includes: 'build/testxgboost'
-    }
-    deleteDir()
-  }
-}
-
-def BuildRPackageWithCUDA(args) {
-  node('linux && cpu_build') {
-    unstash name: 'srcs'
-    def container_type = 'gpu_build_r_centos7'
-    def docker_binary = "docker"
-    def docker_args = "--build-arg CUDA_VERSION_ARG=${args.cuda_version}"
-    if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) {
-      sh """
-      ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_r_pkg_with_cuda.sh ${commit_id}
-      """
-      echo 'Uploading R tarball...'
-      sh """
-      ${dockerRun} ${container_type} ${docker_binary} ${docker_args} python -m awscli s3 cp xgboost_r_gpu_linux_*.tar.gz s3://xgboost-nightly-builds/${BRANCH_NAME}/ --acl public-read --no-progress
-      """
-    }
-    deleteDir()
-  }
-}
-
-def BuildJVMPackagesWithCUDA(args) {
-  node('linux && mgpu') {
-    unstash name: 'srcs'
-    echo "Build XGBoost4J-Spark with Spark ${args.spark_version}, CUDA ${args.cuda_version}"
-    def container_type = "jvm_gpu_build"
-    def docker_binary = "nvidia-docker"
-    def docker_args = "--build-arg CUDA_VERSION_ARG=${args.cuda_version}"
-    def arch_flag = ""
-    if (env.BRANCH_NAME != 'master' && !(env.BRANCH_NAME.startsWith('release'))) {
-      arch_flag = "-DGPU_COMPUTE_VER=75"
-    }
-    // Use only 4 CPU cores
-    def docker_extra_params = "CI_DOCKER_EXTRA_PARAMS_INIT='--cpuset-cpus 0-3'"
-    sh """
-    ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_jvm_packages.sh ${args.spark_version} -Duse.cuda=ON $arch_flag
-    """
-    echo "Stashing XGBoost4J JAR with CUDA ${args.cuda_version} ..."
-    stash name: 'xgboost4j_jar_gpu', includes: "jvm-packages/xgboost4j-gpu/target/*.jar,jvm-packages/xgboost4j-spark-gpu/target/*.jar"
-    deleteDir()
-  }
-}
-
-def BuildJVMPackages(args) {
-  node('linux && cpu') {
-    unstash name: 'srcs'
-    echo "Build XGBoost4J-Spark with Spark ${args.spark_version}"
-    def container_type = "jvm"
-    def docker_binary = "docker"
-    // Use only 4 CPU cores
-    def docker_extra_params = "CI_DOCKER_EXTRA_PARAMS_INIT='--cpuset-cpus 0-3'"
-    sh """
-    ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_jvm_packages.sh ${args.spark_version}
-    """
-    echo 'Stashing XGBoost4J JAR...'
-    stash name: 'xgboost4j_jar', includes: "jvm-packages/xgboost4j/target/*.jar,jvm-packages/xgboost4j-spark/target/*.jar,jvm-packages/xgboost4j-example/target/*.jar"
-    deleteDir()
-  }
-}
-
-def BuildJVMDoc() {
-  node('linux && cpu') {
-    unstash name: 'srcs'
-    echo "Building JVM doc..."
-    def container_type = "jvm"
-    def docker_binary = "docker"
-    sh """
-    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_jvm_doc.sh ${BRANCH_NAME}
-    """
-    if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) {
-      echo 'Uploading doc...'
-      sh """
-      ${dockerRun} ${container_type} ${docker_binary} python -m awscli s3 cp jvm-packages/${BRANCH_NAME}.tar.bz2 s3://xgboost-docs/${BRANCH_NAME}.tar.bz2 --acl public-read --no-progress
-      """
-    }
-    deleteDir()
-  }
-}
-
-def TestPythonCPU() {
-  node('linux && cpu') {
-    unstash name: "xgboost_whl_cuda${ref_cuda_ver}"
-    unstash name: 'srcs'
-    unstash name: 'xgboost_cli'
-    echo "Test Python CPU"
-    def container_type = "cpu"
-    def docker_binary = "docker"
-    sh """
-    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/test_python.sh cpu
-    """
-    deleteDir()
-  }
-}
-
-def TestPythonCPUARM64() {
-  node('linux && arm64') {
-    unstash name: "xgboost_whl_arm64_cpu"
-    unstash name: 'srcs'
-    unstash name: 'xgboost_cli_arm64'
-    echo "Test Python CPU ARM64"
-    def container_type = "aarch64"
-    def docker_binary = "docker"
-    sh """
-    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/test_python.sh cpu-arm64
-    """
-    deleteDir()
-  }
-}
-
-def TestPythonGPU(args) {
-  def nodeReq = (args.multi_gpu) ? 'linux && mgpu' : 'linux && gpu'
-  def artifact_cuda_version = (args.artifact_cuda_version) ?: ref_cuda_ver
-  node(nodeReq) {
-    unstash name: "xgboost_whl_cuda${artifact_cuda_version}"
-    unstash name: "xgboost_cpp_tests_cuda${artifact_cuda_version}"
-    unstash name: 'srcs'
-    echo "Test Python GPU: CUDA ${args.host_cuda_version}"
-    def container_type = "gpu"
-    def docker_binary = "nvidia-docker"
-    def docker_args = "--build-arg CUDA_VERSION_ARG=${args.host_cuda_version}"
-    def mgpu_indicator = (args.multi_gpu) ? 'mgpu' : 'gpu'
-    // Allocate extra space in /dev/shm to enable NCCL
-    def docker_extra_params = (args.multi_gpu) ? "CI_DOCKER_EXTRA_PARAMS_INIT='--shm-size=4g'" : ''
-    sh "${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/test_python.sh ${mgpu_indicator}"
-    if (args.test_rmm) {
-      sh "rm -rfv build/ python-package/dist/"
-      unstash name: "xgboost_whl_rmm_cuda${args.host_cuda_version}"
-      unstash name: "xgboost_cpp_tests_rmm_cuda${args.host_cuda_version}"
-      sh "${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/test_python.sh ${mgpu_indicator} --use-rmm-pool"
-    }
-    deleteDir()
-  }
-}
-
-def TestCppGPU(args) {
-  def nodeReq = 'linux && mgpu'
-  def artifact_cuda_version = (args.artifact_cuda_version) ?: ref_cuda_ver
-  node(nodeReq) {
-    unstash name: "xgboost_cpp_tests_cuda${artifact_cuda_version}"
-    unstash name: 'srcs'
-    echo "Test C++, CUDA ${args.host_cuda_version}"
-    def container_type = "gpu"
-    def docker_binary = "nvidia-docker"
-    def docker_args = "--build-arg CUDA_VERSION_ARG=${args.host_cuda_version}"
-    sh "${dockerRun} ${container_type} ${docker_binary} ${docker_args} build/testxgboost"
-    if (args.test_rmm) {
-      sh "rm -rfv build/"
-      unstash name: "xgboost_cpp_tests_rmm_cuda${args.host_cuda_version}"
-      echo "Test C++, CUDA ${args.host_cuda_version} with RMM"
-      container_type = "rmm"
-      docker_binary = "nvidia-docker"
-      docker_args = "--build-arg CUDA_VERSION_ARG=${args.host_cuda_version}"
-      sh """
-      ${dockerRun} ${container_type} ${docker_binary} ${docker_args} bash -c "source activate gpu_test && build/testxgboost --use-rmm-pool --gtest_filter=-*DeathTest.*"
-      """
-    }
-    deleteDir()
-  }
-}
-
-def CrossTestJVMwithJDK(args) {
-  node('linux && cpu') {
-    unstash name: 'xgboost4j_jar'
-    unstash name: 'srcs'
-    if (args.spark_version != null) {
-      echo "Test XGBoost4J on a machine with JDK ${args.jdk_version}, Spark ${args.spark_version}"
-    } else {
-      echo "Test XGBoost4J on a machine with JDK ${args.jdk_version}"
-    }
-    def container_type = "jvm_cross"
-    def docker_binary = "docker"
-    def spark_arg = (args.spark_version != null) ? "--build-arg SPARK_VERSION=${args.spark_version}" : ""
-    def docker_args = "--build-arg JDK_VERSION=${args.jdk_version} ${spark_arg}"
-    // Run integration tests only when spark_version is given
-    def docker_extra_params = (args.spark_version != null) ? "CI_DOCKER_EXTRA_PARAMS_INIT='-e RUN_INTEGRATION_TEST=1'" : ""
-    sh """
-    ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/test_jvm_cross.sh
-    """
-    deleteDir()
-  }
-}
-
-def DeployJVMPackages(args) {
-  node('linux && cpu') {
-    unstash name: 'srcs'
-    if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) {
-      echo 'Deploying to xgboost-maven-repo S3 repo...'
-      sh """
-      ${dockerRun} jvm_gpu_build docker --build-arg CUDA_VERSION_ARG=11.0 tests/ci_build/deploy_jvm_packages.sh ${args.spark_version}
-      """
-    }
-    deleteDir()
-  }
-}

Jenkinsfile-win64

@@ -1,163 +0,0 @@
-#!/usr/bin/groovy
-// -*- mode: groovy -*-
-
-/* Jenkins pipeline for Windows AMD64 target */
-
-import groovy.transform.Field
-
-@Field
-def commit_id   // necessary to pass a variable from one stage to another
-
-pipeline {
-  agent none
-
-  // Setup common job properties
-  options {
-    timestamps()
-    timeout(time: 240, unit: 'MINUTES')
-    buildDiscarder(logRotator(numToKeepStr: '10'))
-    preserveStashes()
-  }
-
-  // Build stages
-  stages {
-    stage('Jenkins Win64: Initialize') {
-      agent { label 'job_initializer' }
-      steps {
-        script {
-          def buildNumber = env.BUILD_NUMBER as int
-          if (buildNumber > 1) milestone(buildNumber - 1)
-          milestone(buildNumber)
-          checkoutSrcs()
-          commit_id = "${GIT_COMMIT}"
-        }
-        sh 'python3 tests/jenkins_get_approval.py'
-        stash name: 'srcs'
-      }
-    }
-    stage('Jenkins Win64: Build') {
-      agent none
-      steps {
-        script {
-          parallel ([
-            'build-win64-cuda11.0': { BuildWin64() },
-            'build-rpkg-win64-cuda11.0': { BuildRPackageWithCUDAWin64() }
-          ])
-        }
-      }
-    }
-    stage('Jenkins Win64: Test') {
-      agent none
-      steps {
-        script {
-          parallel ([
-            'test-win64-cuda11.0': { TestWin64() },
-          ])
-        }
-      }
-    }
-  }
-}
-
-// check out source code from git
-def checkoutSrcs() {
-  retry(5) {
-    try {
-      timeout(time: 2, unit: 'MINUTES') {
-        checkout scm
-        sh 'git submodule update --init'
-      }
-    } catch (exc) {
-      deleteDir()
-      error "Failed to fetch source codes"
-    }
-  }
-}
-
-def BuildWin64() {
-  node('win64 && cuda11_unified') {
-    deleteDir()
-    unstash name: 'srcs'
-    echo "Building XGBoost for Windows AMD64 target..."
-    bat "nvcc --version"
-    def arch_flag = ""
-    if (env.BRANCH_NAME != 'master' && !(env.BRANCH_NAME.startsWith('release'))) {
-      arch_flag = "-DGPU_COMPUTE_VER=75"
-    }
-    bat """
-    mkdir build
-    cd build
-    cmake .. -G"Visual Studio 15 2017 Win64" -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON ${arch_flag} -DCMAKE_UNITY_BUILD=ON
-    """
-    bat """
-    cd build
-    "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe" xgboost.sln /m /p:Configuration=Release /nodeReuse:false
-    """
-    bat """
-    cd python-package
-    conda activate && python setup.py bdist_wheel --universal && for /R %%i in (dist\\*.whl) DO python ../tests/ci_build/rename_whl.py "%%i" ${commit_id} win_amd64
-    """
-    echo "Insert vcomp140.dll (OpenMP runtime) into the wheel..."
-    bat """
-    cd python-package\\dist
-    COPY /B ..\\..\\tests\\ci_build\\insert_vcomp140.py
-    conda activate && python insert_vcomp140.py *.whl
-    """
-    echo 'Stashing Python wheel...'
-    stash name: 'xgboost_whl', includes: 'python-package/dist/*.whl'
-    if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) {
-      echo 'Uploading Python wheel...'
-      path = "${BRANCH_NAME}/"
-      s3Upload bucket: 'xgboost-nightly-builds', path: path, acl: 'PublicRead', workingDir: 'python-package/dist', includePathPattern:'**/*.whl'
-    }
-    echo 'Stashing C++ test executable (testxgboost)...'
-    stash name: 'xgboost_cpp_tests', includes: 'build/testxgboost.exe'
-    stash name: 'xgboost_cli', includes: 'xgboost.exe'
-    deleteDir()
-  }
-}
-
-def BuildRPackageWithCUDAWin64() {
-  node('win64 && cuda11_unified') {
-    deleteDir()
-    unstash name: 'srcs'
-    bat "nvcc --version"
-    if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) {
-      bat """
-      bash tests/ci_build/build_r_pkg_with_cuda_win64.sh ${commit_id}
-      """
-      echo 'Uploading R tarball...'
-      path = "${BRANCH_NAME}/"
-      s3Upload bucket: 'xgboost-nightly-builds', path: path, acl: 'PublicRead', includePathPattern:'xgboost_r_gpu_win64_*.tar.gz'
-    }
-    deleteDir()
-  }
-}
-
-def TestWin64() {
-  node('win64 && cuda11_unified') {
-    deleteDir()
-    unstash name: 'srcs'
-    unstash name: 'xgboost_whl'
-    unstash name: 'xgboost_cli'
-    unstash name: 'xgboost_cpp_tests'
-    echo "Test Win64"
-    bat "nvcc --version"
-    echo "Running C++ tests..."
-    bat "build\\testxgboost.exe"
-    echo "Installing Python dependencies..."
-    def env_name = 'win64_' + UUID.randomUUID().toString().replaceAll('-', '')
-    bat "conda activate && mamba env create -n ${env_name} --file=tests/ci_build/conda_env/win64_test.yml"
-    echo "Installing Python wheel..."
-    bat """
-    conda activate ${env_name} && for /R %%i in (python-package\\dist\\*.whl) DO python -m pip install "%%i"
-    """
-    echo "Running Python tests..."
-    bat "conda activate ${env_name} && python -m pytest -v -s -rxXs --fulltrace tests\\python"
-    bat """
-    conda activate ${env_name} && python -m pytest -v -s -rxXs --fulltrace -m "(not slow) and (not mgpu)" tests\\python-gpu
-    """
-    bat "conda env remove --name ${env_name}"
-    deleteDir()
-  }
-}

Makefile

@@ -87,22 +87,6 @@ cover: check
 endif
 
 
-# dask is required to pass, others are not
-# If any of the dask tests failed, contributor won't see the other error.
-mypy:
-	cd python-package; \
-	mypy ./xgboost/dask.py && \
-	mypy ./xgboost/rabit.py && \
-	mypy ./xgboost/tracker.py && \
-	mypy ./xgboost/sklearn.py && \
-	mypy ../demo/guide-python/external_memory.py && \
-	mypy ../demo/guide-python/categorical.py && \
-	mypy ../demo/guide-python/cat_in_the_dat.py && \
-	mypy ../tests/python-gpu/test_gpu_with_dask.py && \
-	mypy ../tests/python/test_data_iterator.py && \
-	mypy ../tests/python-gpu/test_gpu_data_iterator.py  || exit 1; \
-	mypy . || true ;
-
 clean:
 	$(RM) -rf build lib bin *~ */*~ */*/*~ */*/*/*~ */*.o */*/*.o */*/*/*.o #xgboost
 	$(RM) -rf build_tests *.gcov tests/cpp/xgboost_test
@@ -139,17 +123,10 @@ Rpack: clean_all
 	cp -r dmlc-core/include xgboost/src/dmlc-core/include
 	cp -r dmlc-core/src xgboost/src/dmlc-core/src
 	cp ./LICENSE xgboost
-# Modify PKGROOT in Makevars.in
 	cat R-package/src/Makevars.in|sed '2s/.*/PKGROOT=./' > xgboost/src/Makevars.in
-# Configure Makevars.win (Windows-specific Makevars, likely using MinGW)
-	cp xgboost/src/Makevars.in xgboost/src/Makevars.win
-	cat xgboost/src/Makevars.in| sed '3s/.*/ENABLE_STD_THREAD=0/' > xgboost/src/Makevars.win
-	sed -i -e 's/@OPENMP_CXXFLAGS@/$$\(SHLIB_OPENMP_CXXFLAGS\)/g' xgboost/src/Makevars.win
-	sed -i -e 's/-pthread/$$\(SHLIB_PTHREAD_FLAGS\)/g' xgboost/src/Makevars.win
-	sed -i -e 's/@ENDIAN_FLAG@/-DDMLC_CMAKE_LITTLE_ENDIAN=1/g' xgboost/src/Makevars.win
-	sed -i -e 's/@BACKTRACE_LIB@//g' xgboost/src/Makevars.win
-	sed -i -e 's/@OPENMP_LIB@//g' xgboost/src/Makevars.win
+	cat R-package/src/Makevars.win|sed '2s/.*/PKGROOT=./' > xgboost/src/Makevars.win
 	rm -f xgboost/src/Makevars.win-e   # OSX sed create this extra file; remove it
+	rm -f xgboost/cleanup
 	bash R-package/remove_warning_suppression_pragma.sh
 	bash xgboost/remove_warning_suppression_pragma.sh
 	rm xgboost/remove_warning_suppression_pragma.sh

NEWS.md

@@ -3,6 +3,264 @@ XGBoost Change Log
 
 This file records the changes in xgboost library in reverse chronological order.
 
+## v1.6.1 (2022 May 9)
+This is a patch release for bug fixes and Spark barrier mode support. The R package is unchanged.
+
+### Experimental support for categorical data
+- Fix segfault when the number of samples is smaller than the number of categories. (https://github.com/dmlc/xgboost/pull/7853)
+- Enable partition-based split for all model types. (https://github.com/dmlc/xgboost/pull/7857)
+
+### JVM packages
+We replaced the old parallelism tracker with spark barrier mode to improve the robustness of the JVM package and fix the GPU training pipeline.
+- Fix GPU training pipeline quantile synchronization. (#7823, #7834)
+- Use barrier model in spark package. (https://github.com/dmlc/xgboost/pull/7836, https://github.com/dmlc/xgboost/pull/7840, https://github.com/dmlc/xgboost/pull/7845, https://github.com/dmlc/xgboost/pull/7846)
+- Fix shared object loading on some platforms. (https://github.com/dmlc/xgboost/pull/7844)
+
+## v1.6.0 (2022 Apr 16)
+
+After a long period of development, XGBoost v1.6.0 is packed with many new features and
+improvements. We summarize them in the following sections starting with an introduction to
+some major new features, then moving on to language binding specific changes including new
+features and notable bug fixes for that binding.
+
+### Development of categorical data support
+This version of XGBoost features new improvements and full coverage of experimental
+categorical data support in Python and C package with tree model.  Both `hist`, `approx`
+and `gpu_hist` now support training with categorical data.  Also, partition-based
+categorical split is introduced in this release. This split type is first available in
+LightGBM in the context of gradient boosting. The previous XGBoost release supported one-hot split where the splitting criteria is of form `x \in {c}`, i.e. the categorical feature `x` is tested against a single candidate. The new release allows for more expressive conditions: `x \in S` where the categorical feature `x` is tested against multiple candidates. Moreover, it is now possible to use any tree algorithms (`hist`, `approx`, `gpu_hist`) when creating categorical splits. For more
+information, please see our tutorial on [categorical
+data](https://xgboost.readthedocs.io/en/latest/tutorials/categorical.html), along with
+examples linked on that page. (#7380, #7708, #7695, #7330, #7307, #7322, #7705,
+#7652, #7592, #7666, #7576, #7569, #7529, #7575, #7393, #7465, #7385, #7371, #7745, #7810)
+
+In the future, we will continue to improve categorical data support with new features and
+optimizations. Also, we are looking forward to bringing the feature beyond Python binding,
+contributions and feedback are welcomed! Lastly, as a result of experimental status, the
+behavior might be subject to change, especially the default value of related
+hyper-parameters.
+
+### Experimental support for multi-output model
+
+XGBoost 1.6 features initial support for the multi-output model, which includes
+multi-output regression and multi-label classification. Along with this, the XGBoost
+classifier has proper support for base margin without to need for the user to flatten the
+input. In this initial support, XGBoost builds one model for each target similar to the
+sklearn meta estimator, for more details, please see our [quick
+introduction](https://xgboost.readthedocs.io/en/latest/tutorials/multioutput.html).
+
+(#7365, #7736, #7607, #7574, #7521, #7514, #7456, #7453, #7455, #7434, #7429, #7405, #7381)
+
+### External memory support
+External memory support for both approx and hist tree method is considered feature
+complete in XGBoost 1.6.  Building upon the iterator-based interface introduced in the
+previous version, now both `hist` and `approx` iterates over each batch of data during
+training and prediction.  In previous versions, `hist` concatenates all the batches into
+an internal representation, which is removed in this version.  As a result, users can
+expect higher scalability in terms of data size but might experience lower performance due
+to disk IO. (#7531, #7320, #7638, #7372)
+
+### Rewritten approx
+
+The `approx` tree method is rewritten based on the existing `hist` tree method. The
+rewrite closes the feature gap between `approx` and `hist` and improves the performance.
+Now the behavior of `approx` should be more aligned with `hist` and `gpu_hist`. Here is a
+list of user-visible changes:
+
+- Supports both `max_leaves` and `max_depth`.
+- Supports `grow_policy`.
+- Supports monotonic constraint.
+- Supports feature weights.
+- Use `max_bin` to replace `sketch_eps`.
+- Supports categorical data.
+- Faster performance for many of the datasets.
+- Improved performance and robustness for distributed training.
+- Supports prediction cache.
+- Significantly better performance for external memory when `depthwise` policy is used.
+
+### New serialization format
+Based on the existing JSON serialization format, we introduce UBJSON support as a more
+efficient alternative. Both formats will be available in the future and we plan to
+gradually [phase out](https://github.com/dmlc/xgboost/issues/7547) support for the old
+binary model format.  Users can opt to use the different formats in the serialization
+function by providing the file extension `json` or `ubj`. Also, the `save_raw` function in
+all supported languages bindings gains a new parameter for exporting the model in different
+formats, available options are `json`, `ubj`, and `deprecated`, see document for the
+language binding you are using for details. Lastly, the default internal serialization
+format is set to UBJSON, which affects Python pickle and R RDS. (#7572, #7570, #7358,
+#7571, #7556, #7549, #7416)
+
+### General new features and improvements
+Aside from the major new features mentioned above, some others are summarized here:
+
+* Users can now access the build information of XGBoost binary in Python and C
+  interface. (#7399, #7553)
+* Auto-configuration of `seed_per_iteration` is removed, now distributed training should
+  generate closer results to single node training when sampling is used. (#7009)
+* A new parameter `huber_slope` is introduced for the `Pseudo-Huber` objective.
+* During source build, XGBoost can choose cub in the system path automatically. (#7579)
+* XGBoost now honors the CPU counts from CFS, which is usually set in docker
+  environments. (#7654, #7704)
+* The metric `aucpr` is rewritten for better performance and GPU support. (#7297, #7368)
+* Metric calculation is now performed in double precision. (#7364)
+* XGBoost no longer mutates the global OpenMP thread limit. (#7537, #7519, #7608, #7590,
+  #7589, #7588, #7687)
+* The default behavior of `max_leave` and `max_depth` is now unified (#7302, #7551).
+* CUDA fat binary is now compressed. (#7601)
+* Deterministic result for evaluation metric and linear model. In previous versions of
+  XGBoost, evaluation results might differ slightly for each run due to parallel reduction
+  for floating-point values, which is now addressed. (#7362, #7303, #7316, #7349)
+* XGBoost now uses double for GPU Hist node sum, which improves the accuracy of
+  `gpu_hist`. (#7507)
+
+### Performance improvements
+Most of the performance improvements are integrated into other refactors during feature
+developments. The `approx` should see significant performance gain for many datasets as
+mentioned in the previous section, while the `hist` tree method also enjoys improved
+performance with the removal of the internal `pruner` along with some other
+refactoring. Lastly, `gpu_hist` no longer synchronizes the device during training. (#7737)
+
+### General bug fixes
+This section lists bug fixes that are not specific to any language binding.
+* The `num_parallel_tree` is now a model parameter instead of a training hyper-parameter,
+  which fixes model IO with random forest. (#7751)
+* Fixes in CMake script for exporting configuration. (#7730)
+* XGBoost can now handle unsorted sparse input. This includes text file formats like
+  libsvm and scipy sparse matrix where column index might not be sorted. (#7731)
+* Fix tree param feature type, this affects inputs with the number of columns greater than
+  the maximum value of int32. (#7565)
+* Fix external memory with gpu_hist and subsampling. (#7481)
+* Check the number of trees in inplace predict, this avoids a potential segfault when an
+  incorrect value for `iteration_range` is provided. (#7409)
+* Fix non-stable result in cox regression (#7756)
+
+### Changes in the Python package
+Other than the changes in Dask, the XGBoost Python package gained some new features and
+improvements along with small bug fixes.
+
+* Python 3.7 is required as the lowest Python version. (#7682)
+* Pre-built binary wheel for Apple Silicon. (#7621, #7612, #7747) Apple Silicon users will
+  now be able to run `pip install xgboost` to install XGBoost.
+* MacOS users no longer need to install `libomp` from Homebrew, as the XGBoost wheel now
+  bundles `libomp.dylib` library.
+* There are new parameters for users to specify the custom metric with new
+  behavior. XGBoost can now output transformed prediction values when a custom objective is
+  not supplied.  See our explanation in the
+  [tutorial](https://xgboost.readthedocs.io/en/latest/tutorials/custom_metric_obj.html#reverse-link-function)
+  for details.
+* For the sklearn interface, following the estimator guideline from scikit-learn, all
+  parameters in `fit` that are not related to input data are moved into the constructor
+  and can be set by `set_params`. (#6751, #7420, #7375, #7369)
+* Apache arrow format is now supported, which can bring better performance to users'
+  pipeline (#7512)
+* Pandas nullable types are now supported (#7760)
+* A new function `get_group` is introduced for `DMatrix` to allow users to get the group
+  information in the custom objective function. (#7564)
+* More training parameters are exposed in the sklearn interface instead of relying on the
+  `**kwargs`. (#7629)
+* A new attribute `feature_names_in_` is defined for all sklearn estimators like
+  `XGBRegressor` to follow the convention of sklearn. (#7526)
+* More work on Python type hint. (#7432, #7348, #7338, #7513, #7707)
+* Support the latest pandas Index type. (#7595)
+* Fix for Feature shape mismatch error on s390x platform (#7715)
+* Fix using feature names for constraints with multiple groups (#7711)
+* We clarified the behavior of the callback function when it contains mutable
+  states. (#7685)
+* Lastly, there are some code cleanups and maintenance work. (#7585, #7426, #7634, #7665,
+  #7667, #7377, #7360, #7498, #7438, #7667, #7752, #7749, #7751)
+
+### Changes in the Dask interface
+* Dask module now supports user-supplied host IP and port address of scheduler node.
+  Please see [introduction](https://xgboost.readthedocs.io/en/latest/tutorials/dask.html#troubleshooting) and
+  [API document](https://xgboost.readthedocs.io/en/latest/python/python_api.html#optional-dask-configuration)
+  for reference. (#7645, #7581)
+* Internal `DMatrix` construction in dask now honers thread configuration. (#7337)
+* A fix for `nthread` configuration using the Dask sklearn interface. (#7633)
+* The Dask interface can now handle empty partitions.  An empty partition is different
+  from an empty worker, the latter refers to the case when a worker has no partition of an
+  input dataset, while the former refers to some partitions on a worker that has zero
+  sizes. (#7644, #7510)
+* Scipy sparse matrix is supported as Dask array partition. (#7457)
+* Dask interface is no longer considered experimental. (#7509)
+
+### Changes in the R package
+This section summarizes the new features, improvements, and bug fixes to the R package.
+
+* `load.raw` can optionally construct a booster as return. (#7686)
+* Fix parsing decision stump, which affects both transforming text representation to data
+  table and plotting. (#7689)
+* Implement feature weights. (#7660)
+* Some improvements for complying the CRAN release policy. (#7672, #7661, #7763)
+* Support CSR data for predictions (#7615)
+* Document update (#7263, #7606)
+* New maintainer for the CRAN package (#7691, #7649)
+* Handle non-standard installation of toolchain on macos (#7759)
+
+### Changes in JVM-packages
+Some new features for JVM-packages are introduced for a more integrated GPU pipeline and
+better compatibility with musl-based Linux. Aside from this, we have a few notable bug
+fixes.
+
+* User can specify the tracker IP address for training, which helps running XGBoost on
+  restricted network environments. (#7808)
+* Add support for detecting musl-based Linux (#7624)
+* Add `DeviceQuantileDMatrix` to Scala binding (#7459)
+* Add Rapids plugin support, now more of the JVM pipeline can be accelerated by RAPIDS (#7491, #7779, #7793, #7806)
+* The setters for CPU and GPU are more aligned (#7692, #7798)
+* Control logging for early stopping (#7326)
+* Do not repartition when nWorker = 1 (#7676)
+* Fix the prediction issue for `multi:softmax` (#7694)
+* Fix for serialization of custom objective and eval (#7274)
+* Update documentation about Python tracker (#7396)
+* Remove jackson from dependency, which fixes CVE-2020-36518. (#7791)
+* Some refactoring to the training pipeline for better compatibility between CPU and
+  GPU. (#7440, #7401, #7789, #7784)
+* Maintenance work. (#7550, #7335, #7641, #7523, #6792, #4676)
+
+### Deprecation
+Other than the changes in the Python package and serialization, we removed some deprecated
+features in previous releases. Also, as mentioned in the previous section, we plan to
+phase out the old binary format in future releases.
+
+* Remove old warning in 1.3 (#7279)
+* Remove label encoder deprecated in 1.3. (#7357)
+* Remove old callback deprecated in 1.3. (#7280)
+* Pre-built binary will no longer support deprecated CUDA architectures including sm35 and
+  sm50. Users can continue to use these platforms with source build. (#7767)
+
+### Documentation
+This section lists some of the general changes to XGBoost's document, for language binding
+specific change please visit related sections.
+
+* Document is overhauled to use the new RTD theme, along with integration of Python
+  examples using Sphinx gallery. Also, we replaced most of the hard-coded URLs with sphinx
+  references. (#7347, #7346, #7468, #7522, #7530)
+* Small update along with fixes for broken links, typos, etc. (#7684, #7324, #7334, #7655,
+  #7628, #7623, #7487, #7532, #7500, #7341, #7648, #7311)
+* Update document for GPU. [skip ci] (#7403)
+* Document the status of RTD hosting. (#7353)
+* Update document for building from source. (#7664)
+* Add note about CRAN release [skip ci] (#7395)
+
+### Maintenance
+This is a summary of maintenance work that is not specific to any language binding.
+
+* Add CMake option to use /MD runtime (#7277)
+* Add clang-format configuration. (#7383)
+* Code cleanups (#7539, #7536, #7466, #7499, #7533, #7735, #7722, #7668, #7304, #7293,
+  #7321, #7356, #7345, #7387, #7577, #7548, #7469, #7680, #7433, #7398)
+* Improved tests with better coverage and latest dependency (#7573, #7446, #7650, #7520,
+  #7373, #7723, #7611, #7771)
+* Improved automation of the release process. (#7278, #7332, #7470)
+* Compiler workarounds (#7673)
+* Change shebang used in CLI demo. (#7389)
+* Update affiliation (#7289)
+
+### CI
+Some fixes and update to XGBoost's CI infrastructure. (#7739, #7701, #7382, #7662, #7646,
+#7582, #7407, #7417, #7475, #7474, #7479, #7472, #7626)
+
+
 ## v1.5.0 (2021 Oct 11)
 
 This release comes with many exciting new features and optimizations, along with some bug

R-package/DESCRIPTION

@@ -1,8 +1,8 @@
 Package: xgboost
 Type: Package
 Title: Extreme Gradient Boosting
-Version: 1.6.0.1
-Date: 2022-03-29
+Version: 1.7.0.1
+Date: 2022-10-18
 Authors@R: c(
   person("Tianqi", "Chen", role = c("aut"),
          email = "tianqi.tchen@gmail.com"),

R-package/R/xgb.DMatrix.R

@@ -54,7 +54,10 @@ xgb.DMatrix <- function(data, info = list(), missing = NA, silent = FALSE, nthre
     stop("xgb.DMatrix does not support construction from ", typeof(data))
   }
   dmat <- handle
-  attributes(dmat) <- list(.Dimnames = list(NULL, cnames), class = "xgb.DMatrix")
+  attributes(dmat) <- list(class = "xgb.DMatrix")
+  if (!is.null(cnames)) {
+    setinfo(dmat, "feature_name", cnames)
+  }
 
   info <- append(info, list(...))
   for (i in seq_along(info)) {
@@ -144,7 +147,9 @@ dim.xgb.DMatrix <- function(x) {
 #' @rdname dimnames.xgb.DMatrix
 #' @export
 dimnames.xgb.DMatrix <- function(x) {
-  attr(x, '.Dimnames')
+  fn <- getinfo(x, "feature_name")
+  ## row names is null.
+  list(NULL, fn)
 }
 
 #' @rdname dimnames.xgb.DMatrix
@@ -155,13 +160,13 @@ dimnames.xgb.DMatrix <- function(x) {
   if (!is.null(value[[1L]]))
     stop("xgb.DMatrix does not have rownames")
   if (is.null(value[[2]])) {
-    attr(x, '.Dimnames') <- NULL
+    setinfo(x, "feature_name", NULL)
     return(x)
   }
-  if (ncol(x) != length(value[[2]]))
-    stop("can't assign ", length(value[[2]]), " colnames to a ",
-         ncol(x), " column xgb.DMatrix")
-  attr(x, '.Dimnames') <- value
+  if (ncol(x) != length(value[[2]])) {
+    stop("can't assign ", length(value[[2]]), " colnames to a ", ncol(x), " column xgb.DMatrix")
+  }
+  setinfo(x, "feature_name", value[[2]])
   x
 }
 
@@ -203,13 +208,17 @@ getinfo <- function(object, ...) UseMethod("getinfo")
 #' @export
 getinfo.xgb.DMatrix <- function(object, name, ...) {
   if (typeof(name) != "character" ||
-      length(name) != 1 ||
-      !name %in% c('label', 'weight', 'base_margin', 'nrow',
-                   'label_lower_bound', 'label_upper_bound')) {
-    stop("getinfo: name must be one of the following\n",
-         "    'label', 'weight', 'base_margin', 'nrow', 'label_lower_bound', 'label_upper_bound'")
+        length(name) != 1 ||
+        !name %in% c('label', 'weight', 'base_margin', 'nrow',
+                     'label_lower_bound', 'label_upper_bound', "feature_type", "feature_name")) {
+    stop(
+      "getinfo: name must be one of the following\n",
+      "    'label', 'weight', 'base_margin', 'nrow', 'label_lower_bound', 'label_upper_bound', 'feature_type', 'feature_name'"
+    )
   }
-  if (name != "nrow"){
+  if (name == "feature_name" || name == "feature_type") {
+    ret <- .Call(XGDMatrixGetStrFeatureInfo_R, object, name)
+  } else if (name != "nrow"){
     ret <- .Call(XGDMatrixGetInfo_R, object, name)
   } else {
     ret <- nrow(object)
@@ -294,6 +303,30 @@ setinfo.xgb.DMatrix <- function(object, name, info, ...) {
     .Call(XGDMatrixSetInfo_R, object, name, as.numeric(info))
     return(TRUE)
   }
+
+  set_feat_info <- function(name) {
+    msg <- sprintf(
+      "The number of %s must equal to the number of columns in the input data. %s vs. %s",
+      name,
+      length(info),
+      ncol(object)
+    )
+    if (!is.null(info)) {
+      info <- as.list(info)
+      if (length(info) != ncol(object)) {
+        stop(msg)
+      }
+    }
+    .Call(XGDMatrixSetStrFeatureInfo_R, object, name, info)
+  }
+  if (name == "feature_name") {
+    set_feat_info("feature_name")
+    return(TRUE)
+  }
+  if (name == "feature_type") {
+    set_feat_info("feature_type")
+    return(TRUE)
+  }
   stop("setinfo: unknown info name ", name)
   return(FALSE)
 }

R-package/R/xgb.plot.multi.trees.R

@@ -62,6 +62,9 @@
 #' @export
 xgb.plot.multi.trees <- function(model, feature_names = NULL, features_keep = 5, plot_width = NULL, plot_height = NULL,
                                  render = TRUE, ...){
+  if (!requireNamespace("DiagrammeR", quietly = TRUE)) {
+    stop("DiagrammeR is required for xgb.plot.multi.trees")
+  }
   check.deprecation(...)
   tree.matrix <- xgb.model.dt.tree(feature_names = feature_names, model = model)
 

R-package/configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for xgboost 1.6-0.
+# Generated by GNU Autoconf 2.69 for xgboost 1.7.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -576,8 +576,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='xgboost'
 PACKAGE_TARNAME='xgboost'
-PACKAGE_VERSION='1.6-0'
-PACKAGE_STRING='xgboost 1.6-0'
+PACKAGE_VERSION='1.7.0'
+PACKAGE_STRING='xgboost 1.7.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -1195,7 +1195,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures xgboost 1.6-0 to adapt to many kinds of systems.
+\`configure' configures xgboost 1.7.0 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1257,7 +1257,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of xgboost 1.6-0:";;
+     short | recursive ) echo "Configuration of xgboost 1.7.0:";;
    esac
   cat <<\_ACEOF
 
@@ -1336,7 +1336,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-xgboost configure 1.6-0
+xgboost configure 1.7.0
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1479,7 +1479,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by xgboost $as_me 1.6-0, which was
+It was created by xgboost $as_me 1.7.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -3287,7 +3287,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by xgboost $as_me 1.6-0, which was
+This file was extended by xgboost $as_me 1.7.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -3340,7 +3340,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-xgboost config.status 1.6-0
+xgboost config.status 1.7.0
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

R-package/configure.ac

@@ -2,7 +2,7 @@
 
 AC_PREREQ(2.69)
 
-AC_INIT([xgboost],[1.6-0],[],[xgboost],[])
+AC_INIT([xgboost],[1.7.0],[],[xgboost],[])
 
 # Use this line to set CC variable to a C compiler
 AC_PROG_CC

R-package/src/Makevars.in

@@ -19,7 +19,78 @@ $(foreach v, $(XGB_RFLAGS), $(warning $(v)))
 PKG_CPPFLAGS=  -I$(PKGROOT)/include -I$(PKGROOT)/dmlc-core/include -I$(PKGROOT)/rabit/include -I$(PKGROOT) $(XGB_RFLAGS)
 PKG_CXXFLAGS= @OPENMP_CXXFLAGS@ @ENDIAN_FLAG@ -pthread $(CXX_VISIBILITY)
 PKG_LIBS = @OPENMP_CXXFLAGS@ @OPENMP_LIB@ @ENDIAN_FLAG@ @BACKTRACE_LIB@ -pthread
-OBJECTS= ./xgboost_R.o ./xgboost_custom.o ./xgboost_assert.o ./init.o \
-         $(PKGROOT)/amalgamation/xgboost-all0.o $(PKGROOT)/amalgamation/dmlc-minimum0.o \
-         $(PKGROOT)/rabit/src/engine.o $(PKGROOT)/rabit/src/rabit_c_api.o \
-         $(PKGROOT)/rabit/src/allreduce_base.o
+
+OBJECTS= \
+    ./xgboost_R.o \
+    ./xgboost_custom.o \
+    ./xgboost_assert.o \
+    ./init.o \
+    $(PKGROOT)/src/metric/metric.o \
+    $(PKGROOT)/src/metric/elementwise_metric.o \
+    $(PKGROOT)/src/metric/multiclass_metric.o \
+    $(PKGROOT)/src/metric/rank_metric.o \
+    $(PKGROOT)/src/metric/auc.o \
+    $(PKGROOT)/src/metric/survival_metric.o \
+    $(PKGROOT)/src/objective/objective.o \
+    $(PKGROOT)/src/objective/regression_obj.o \
+    $(PKGROOT)/src/objective/multiclass_obj.o \
+    $(PKGROOT)/src/objective/rank_obj.o \
+    $(PKGROOT)/src/objective/hinge.o \
+    $(PKGROOT)/src/objective/aft_obj.o \
+    $(PKGROOT)/src/objective/adaptive.o \
+    $(PKGROOT)/src/gbm/gbm.o \
+    $(PKGROOT)/src/gbm/gbtree.o \
+    $(PKGROOT)/src/gbm/gbtree_model.o \
+    $(PKGROOT)/src/gbm/gblinear.o \
+    $(PKGROOT)/src/gbm/gblinear_model.o \
+    $(PKGROOT)/src/data/simple_dmatrix.o \
+    $(PKGROOT)/src/data/data.o \
+    $(PKGROOT)/src/data/sparse_page_raw_format.o \
+    $(PKGROOT)/src/data/ellpack_page.o \
+    $(PKGROOT)/src/data/gradient_index.o \
+    $(PKGROOT)/src/data/gradient_index_page_source.o \
+    $(PKGROOT)/src/data/gradient_index_format.o \
+    $(PKGROOT)/src/data/sparse_page_dmatrix.o \
+    $(PKGROOT)/src/data/proxy_dmatrix.o \
+    $(PKGROOT)/src/data/iterative_dmatrix.o \
+    $(PKGROOT)/src/predictor/predictor.o \
+    $(PKGROOT)/src/predictor/cpu_predictor.o \
+    $(PKGROOT)/src/tree/constraints.o \
+    $(PKGROOT)/src/tree/param.o \
+    $(PKGROOT)/src/tree/tree_model.o \
+    $(PKGROOT)/src/tree/tree_updater.o \
+    $(PKGROOT)/src/tree/updater_approx.o \
+    $(PKGROOT)/src/tree/updater_colmaker.o \
+    $(PKGROOT)/src/tree/updater_prune.o \
+    $(PKGROOT)/src/tree/updater_quantile_hist.o \
+    $(PKGROOT)/src/tree/updater_refresh.o \
+    $(PKGROOT)/src/tree/updater_sync.o \
+    $(PKGROOT)/src/linear/linear_updater.o \
+    $(PKGROOT)/src/linear/updater_coordinate.o \
+    $(PKGROOT)/src/linear/updater_shotgun.o \
+    $(PKGROOT)/src/learner.o \
+    $(PKGROOT)/src/logging.o \
+    $(PKGROOT)/src/global_config.o \
+    $(PKGROOT)/src/collective/communicator.o \
+    $(PKGROOT)/src/collective/socket.o \
+    $(PKGROOT)/src/common/charconv.o \
+    $(PKGROOT)/src/common/column_matrix.o \
+    $(PKGROOT)/src/common/common.o \
+    $(PKGROOT)/src/common/hist_util.o \
+    $(PKGROOT)/src/common/host_device_vector.o \
+    $(PKGROOT)/src/common/io.o \
+    $(PKGROOT)/src/common/json.o \
+    $(PKGROOT)/src/common/numeric.o \
+    $(PKGROOT)/src/common/pseudo_huber.o \
+    $(PKGROOT)/src/common/quantile.o \
+    $(PKGROOT)/src/common/random.o \
+    $(PKGROOT)/src/common/survival_util.o \
+    $(PKGROOT)/src/common/threading_utils.o \
+    $(PKGROOT)/src/common/timer.o \
+    $(PKGROOT)/src/common/version.o \
+    $(PKGROOT)/src/c_api/c_api.o \
+    $(PKGROOT)/src/c_api/c_api_error.o \
+    $(PKGROOT)/amalgamation/dmlc-minimum0.o \
+    $(PKGROOT)/rabit/src/engine.o \
+    $(PKGROOT)/rabit/src/rabit_c_api.o \
+    $(PKGROOT)/rabit/src/allreduce_base.o

R-package/src/Makevars.win

@@ -1,20 +1,8 @@
 # package root
-PKGROOT=./
+PKGROOT=../../
 ENABLE_STD_THREAD=0
 # _*_ mode: Makefile; _*_
 
-# This file is only used for Windows compilation from GitHub
-# It will be replaced with Makevars.in for the CRAN version
-.PHONY: all xgblib
-all: $(SHLIB)
-$(SHLIB): xgblib
-xgblib:
-	cp -r ../../src .
-	cp -r ../../rabit .
-	cp -r ../../dmlc-core .
-	cp -r ../../include .
-	cp -r ../../amalgamation .
-
 CXX_STD = CXX14
 
 XGB_RFLAGS = -DXGBOOST_STRICT_R_MODE=1 -DDMLC_LOG_BEFORE_THROW=0\
@@ -29,11 +17,80 @@ endif
 $(foreach v, $(XGB_RFLAGS), $(warning $(v)))
 
 PKG_CPPFLAGS=  -I$(PKGROOT)/include -I$(PKGROOT)/dmlc-core/include -I$(PKGROOT)/rabit/include -I$(PKGROOT) $(XGB_RFLAGS)
-PKG_CXXFLAGS= $(SHLIB_OPENMP_CXXFLAGS) $(SHLIB_PTHREAD_FLAGS)
-PKG_LIBS = $(SHLIB_OPENMP_CXXFLAGS) $(SHLIB_PTHREAD_FLAGS)
-OBJECTS= ./xgboost_R.o ./xgboost_custom.o ./xgboost_assert.o ./init.o \
-         $(PKGROOT)/amalgamation/xgboost-all0.o $(PKGROOT)/amalgamation/dmlc-minimum0.o \
-         $(PKGROOT)/rabit/src/engine.o $(PKGROOT)/rabit/src/rabit_c_api.o \
-         $(PKGROOT)/rabit/src/allreduce_base.o
+PKG_CXXFLAGS= $(SHLIB_OPENMP_CXXFLAGS) -DDMLC_CMAKE_LITTLE_ENDIAN=1 $(SHLIB_PTHREAD_FLAGS) $(CXX_VISIBILITY)
+PKG_LIBS = $(SHLIB_OPENMP_CXXFLAGS)  -DDMLC_CMAKE_LITTLE_ENDIAN=1  $(SHLIB_PTHREAD_FLAGS) -lwsock32 -lws2_32
 
-$(OBJECTS) : xgblib
+OBJECTS= \
+    ./xgboost_R.o \
+    ./xgboost_custom.o \
+    ./xgboost_assert.o \
+    ./init.o \
+    $(PKGROOT)/src/metric/metric.o \
+    $(PKGROOT)/src/metric/elementwise_metric.o \
+    $(PKGROOT)/src/metric/multiclass_metric.o \
+    $(PKGROOT)/src/metric/rank_metric.o \
+    $(PKGROOT)/src/metric/auc.o \
+    $(PKGROOT)/src/metric/survival_metric.o \
+    $(PKGROOT)/src/objective/objective.o \
+    $(PKGROOT)/src/objective/regression_obj.o \
+    $(PKGROOT)/src/objective/multiclass_obj.o \
+    $(PKGROOT)/src/objective/rank_obj.o \
+    $(PKGROOT)/src/objective/hinge.o \
+    $(PKGROOT)/src/objective/aft_obj.o \
+    $(PKGROOT)/src/objective/adaptive.o \
+    $(PKGROOT)/src/gbm/gbm.o \
+    $(PKGROOT)/src/gbm/gbtree.o \
+    $(PKGROOT)/src/gbm/gbtree_model.o \
+    $(PKGROOT)/src/gbm/gblinear.o \
+    $(PKGROOT)/src/gbm/gblinear_model.o \
+    $(PKGROOT)/src/data/simple_dmatrix.o \
+    $(PKGROOT)/src/data/data.o \
+    $(PKGROOT)/src/data/sparse_page_raw_format.o \
+    $(PKGROOT)/src/data/ellpack_page.o \
+    $(PKGROOT)/src/data/gradient_index.o \
+    $(PKGROOT)/src/data/gradient_index_page_source.o \
+    $(PKGROOT)/src/data/gradient_index_format.o \
+    $(PKGROOT)/src/data/sparse_page_dmatrix.o \
+    $(PKGROOT)/src/data/proxy_dmatrix.o \
+    $(PKGROOT)/src/data/iterative_dmatrix.o \
+    $(PKGROOT)/src/predictor/predictor.o \
+    $(PKGROOT)/src/predictor/cpu_predictor.o \
+    $(PKGROOT)/src/tree/constraints.o \
+    $(PKGROOT)/src/tree/param.o \
+    $(PKGROOT)/src/tree/tree_model.o \
+    $(PKGROOT)/src/tree/tree_updater.o \
+    $(PKGROOT)/src/tree/updater_approx.o \
+    $(PKGROOT)/src/tree/updater_colmaker.o \
+    $(PKGROOT)/src/tree/updater_prune.o \
+    $(PKGROOT)/src/tree/updater_quantile_hist.o \
+    $(PKGROOT)/src/tree/updater_refresh.o \
+    $(PKGROOT)/src/tree/updater_sync.o \
+    $(PKGROOT)/src/linear/linear_updater.o \
+    $(PKGROOT)/src/linear/updater_coordinate.o \
+    $(PKGROOT)/src/linear/updater_shotgun.o \
+    $(PKGROOT)/src/learner.o \
+    $(PKGROOT)/src/logging.o \
+    $(PKGROOT)/src/global_config.o \
+    $(PKGROOT)/src/collective/communicator.o \
+    $(PKGROOT)/src/collective/socket.o \
+    $(PKGROOT)/src/common/charconv.o \
+    $(PKGROOT)/src/common/column_matrix.o \
+    $(PKGROOT)/src/common/common.o \
+    $(PKGROOT)/src/common/hist_util.o \
+    $(PKGROOT)/src/common/host_device_vector.o \
+    $(PKGROOT)/src/common/io.o \
+    $(PKGROOT)/src/common/json.o \
+    $(PKGROOT)/src/common/numeric.o \
+    $(PKGROOT)/src/common/pseudo_huber.o \
+    $(PKGROOT)/src/common/quantile.o \
+    $(PKGROOT)/src/common/random.o \
+    $(PKGROOT)/src/common/survival_util.o \
+    $(PKGROOT)/src/common/threading_utils.o \
+    $(PKGROOT)/src/common/timer.o \
+    $(PKGROOT)/src/common/version.o \
+    $(PKGROOT)/src/c_api/c_api.o \
+    $(PKGROOT)/src/c_api/c_api_error.o \
+    $(PKGROOT)/amalgamation/dmlc-minimum0.o \
+    $(PKGROOT)/rabit/src/engine.o \
+    $(PKGROOT)/rabit/src/rabit_c_api.o \
+    $(PKGROOT)/rabit/src/allreduce_base.o

R-package/src/init.c

@@ -42,13 +42,15 @@ extern SEXP XGDMatrixCreateFromCSR_R(SEXP, SEXP, SEXP, SEXP, SEXP);
 extern SEXP XGDMatrixCreateFromFile_R(SEXP, SEXP);
 extern SEXP XGDMatrixCreateFromMat_R(SEXP, SEXP, SEXP);
 extern SEXP XGDMatrixGetInfo_R(SEXP, SEXP);
+extern SEXP XGDMatrixGetStrFeatureInfo_R(SEXP, SEXP);
 extern SEXP XGDMatrixNumCol_R(SEXP);
 extern SEXP XGDMatrixNumRow_R(SEXP);
 extern SEXP XGDMatrixSaveBinary_R(SEXP, SEXP, SEXP);
 extern SEXP XGDMatrixSetInfo_R(SEXP, SEXP, SEXP);
+extern SEXP XGDMatrixSetStrFeatureInfo_R(SEXP, SEXP, SEXP);
 extern SEXP XGDMatrixSliceDMatrix_R(SEXP, SEXP);
 extern SEXP XGBSetGlobalConfig_R(SEXP);
-extern SEXP XGBGetGlobalConfig_R();
+extern SEXP XGBGetGlobalConfig_R(void);
 extern SEXP XGBoosterFeatureScore_R(SEXP, SEXP);
 
 static const R_CallMethodDef CallEntries[] = {
@@ -78,10 +80,12 @@ static const R_CallMethodDef CallEntries[] = {
   {"XGDMatrixCreateFromFile_R",   (DL_FUNC) &XGDMatrixCreateFromFile_R,   2},
   {"XGDMatrixCreateFromMat_R",    (DL_FUNC) &XGDMatrixCreateFromMat_R,    3},
   {"XGDMatrixGetInfo_R",          (DL_FUNC) &XGDMatrixGetInfo_R,          2},
+  {"XGDMatrixGetStrFeatureInfo_R", (DL_FUNC) &XGDMatrixGetStrFeatureInfo_R, 2},
   {"XGDMatrixNumCol_R",           (DL_FUNC) &XGDMatrixNumCol_R,           1},
   {"XGDMatrixNumRow_R",           (DL_FUNC) &XGDMatrixNumRow_R,           1},
   {"XGDMatrixSaveBinary_R",       (DL_FUNC) &XGDMatrixSaveBinary_R,       3},
   {"XGDMatrixSetInfo_R",          (DL_FUNC) &XGDMatrixSetInfo_R,          3},
+  {"XGDMatrixSetStrFeatureInfo_R", (DL_FUNC) &XGDMatrixSetStrFeatureInfo_R, 3},
   {"XGDMatrixSliceDMatrix_R",     (DL_FUNC) &XGDMatrixSliceDMatrix_R,     2},
   {"XGBSetGlobalConfig_R",        (DL_FUNC) &XGBSetGlobalConfig_R,        1},
   {"XGBGetGlobalConfig_R",        (DL_FUNC) &XGBGetGlobalConfig_R,        0},

R-package/src/xgboost_R.cc

@@ -249,15 +249,53 @@ XGB_DLL SEXP XGDMatrixSetInfo_R(SEXP handle, SEXP field, SEXP array) {
   return R_NilValue;
 }
 
+XGB_DLL SEXP XGDMatrixSetStrFeatureInfo_R(SEXP handle, SEXP field, SEXP array) {
+  R_API_BEGIN();
+  size_t len{0};
+  if (!isNull(array)) {
+    len = length(array);
+  }
+
+  const char *name = CHAR(asChar(field));
+  std::vector<std::string> str_info;
+  for (size_t i = 0; i < len; ++i) {
+    str_info.emplace_back(CHAR(asChar(VECTOR_ELT(array, i))));
+  }
+  std::vector<char const*> vec(len);
+  std::transform(str_info.cbegin(), str_info.cend(), vec.begin(),
+                 [](std::string const &str) { return str.c_str(); });
+  CHECK_CALL(XGDMatrixSetStrFeatureInfo(R_ExternalPtrAddr(handle), name, vec.data(), len));
+  R_API_END();
+  return R_NilValue;
+}
+
+XGB_DLL SEXP XGDMatrixGetStrFeatureInfo_R(SEXP handle, SEXP field) {
+  SEXP ret;
+  R_API_BEGIN();
+  char const **out_features{nullptr};
+  bst_ulong len{0};
+  const char *name = CHAR(asChar(field));
+  XGDMatrixGetStrFeatureInfo(R_ExternalPtrAddr(handle), name, &len, &out_features);
+
+  if (len > 0) {
+    ret = PROTECT(allocVector(STRSXP, len));
+    for (size_t i = 0; i < len; ++i) {
+      SET_STRING_ELT(ret, i, mkChar(out_features[i]));
+    }
+  } else {
+    ret = PROTECT(R_NilValue);
+  }
+  R_API_END();
+  UNPROTECT(1);
+  return ret;
+}
+
 XGB_DLL SEXP XGDMatrixGetInfo_R(SEXP handle, SEXP field) {
   SEXP ret;
   R_API_BEGIN();
   bst_ulong olen;
   const float *res;
-  CHECK_CALL(XGDMatrixGetFloatInfo(R_ExternalPtrAddr(handle),
-                                   CHAR(asChar(field)),
-                                 &olen,
-                                 &res));
+  CHECK_CALL(XGDMatrixGetFloatInfo(R_ExternalPtrAddr(handle), CHAR(asChar(field)), &olen, &res));
   ret = PROTECT(allocVector(REALSXP, olen));
   for (size_t i = 0; i < olen; ++i) {
     REAL(ret)[i] = res[i];

R-package/tests/helper_scripts/run_lint.R

@@ -13,7 +13,7 @@ my_linters <- list(
   object_usage_linter = lintr::object_usage_linter,
   object_length_linter = lintr::object_length_linter,
   open_curly_linter = lintr::open_curly_linter,
-  semicolon = lintr::semicolon_terminator_linter,
+  semicolon = lintr::semicolon_terminator_linter(semicolon = c("compound", "trailing")),
   seq = lintr::seq_linter,
   spaces_inside_linter = lintr::spaces_inside_linter,
   spaces_left_parentheses_linter = lintr::spaces_left_parentheses_linter,

R-package/tests/testthat/test_dmatrix.R

@@ -42,6 +42,20 @@ test_that("xgb.DMatrix: saving, loading", {
   dtest4 <- xgb.DMatrix(tmp_file, silent = TRUE)
   expect_equal(dim(dtest4), c(3, 4))
   expect_equal(getinfo(dtest4, 'label'), c(0, 1, 0))
+
+  # check that feature info is saved
+  data(agaricus.train, package = 'xgboost')
+  dtrain <- xgb.DMatrix(data = agaricus.train$data, label = agaricus.train$label)
+  cnames <- colnames(dtrain)
+  expect_equal(length(cnames), 126)
+  tmp_file <- tempfile('xgb.DMatrix_')
+  xgb.DMatrix.save(dtrain, tmp_file)
+  dtrain <- xgb.DMatrix(tmp_file)
+  expect_equal(colnames(dtrain), cnames)
+
+  ft <- rep(c("c", "q"), each=length(cnames)/2)
+  setinfo(dtrain, "feature_type", ft)
+  expect_equal(ft, getinfo(dtrain, "feature_type"))
 })
 
 test_that("xgb.DMatrix: getinfo & setinfo", {

R-package/tests/testthat/test_model_compatibility.R

@@ -77,6 +77,7 @@ test_that("Models from previous versions of XGBoost can be loaded", {
     model_xgb_ver <- m[2]
     name <- m[3]
     is_rds <- endsWith(model_file, '.rds')
+    is_json <- endsWith(model_file, '.json')
 
     cpp_warning <- capture.output({
       # Expect an R warning when a model is loaded from RDS and it was generated by version < 1.1.x
@@ -95,15 +96,13 @@ test_that("Models from previous versions of XGBoost can be loaded", {
         run_booster_check(booster, name)
       }
     })
-    if (compareVersion(model_xgb_ver, '1.0.0.0') < 0) {
-      # Expect a C++ warning when a model was generated in version < 1.0.x
-      m <- grepl(paste0('.*Loading model from XGBoost < 1\\.0\\.0, consider saving it again for ',
-                        'improved compatibility.*'), cpp_warning, perl = TRUE)
-      expect_true(length(m) > 0 && all(m))
-    } else if (is_rds && model_xgb_ver == '1.1.1.1') {
-      # Expect a C++ warning when a model is loaded from RDS and it was generated by version 1.1.x
-      m <- grepl(paste0('.*Attempted to load internal configuration for a model file that was ',
-                        'generated by a previous version of XGBoost.*'), cpp_warning, perl = TRUE)
+    cpp_warning <- paste0(cpp_warning, collapse = ' ')
+    if (is_rds && compareVersion(model_xgb_ver, '1.1.1.1') >= 0) {
+      # Expect a C++ warning when a model is loaded from RDS and it was generated by old XGBoost`
+      m <- grepl(paste0('.*If you are loading a serialized model ',
+                        '\\(like pickle in Python, RDS in R\\).*',
+                        'for more details about differences between ',
+                        'saving model and serializing.*'), cpp_warning, perl = TRUE)
       expect_true(length(m) > 0 && all(m))
     }
   })

README.md

@@ -10,6 +10,7 @@
 [![Conda version](https://img.shields.io/conda/vn/conda-forge/py-xgboost.svg)](https://anaconda.org/conda-forge/py-xgboost)
 [![Optuna](https://img.shields.io/badge/Optuna-integrated-blue)](https://optuna.org)
 [![Twitter](https://img.shields.io/badge/@XGBoostProject--_.svg?style=social&logo=twitter)](https://twitter.com/XGBoostProject)
+[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/dmlc/xgboost/badge)](https://api.securityscorecards.dev/projects/github.com/dmlc/xgboost)
 
 [Community](https://xgboost.ai/community) |
 [Documentation](https://xgboost.readthedocs.org) |
@@ -46,24 +47,11 @@ Become a sponsor and get a logo here. See details at [Sponsoring the XGBoost Pro
 ### Sponsors
 [[Become a sponsor](https://opencollective.com/xgboost#sponsor)]
 
-<!--<a href="https://opencollective.com/xgboost/sponsor/0/website" target="_blank"><img src="https://opencollective.com/xgboost/sponsor/0/avatar.svg"></a>-->
 <a href="https://www.nvidia.com/en-us/" target="_blank"><img src="https://raw.githubusercontent.com/xgboost-ai/xgboost-ai.github.io/master/images/sponsors/nvidia.jpg" alt="NVIDIA" width="72" height="72"></a>
-<a href="https://opencollective.com/xgboost/sponsor/1/website" target="_blank"><img src="https://opencollective.com/xgboost/sponsor/1/avatar.svg"></a>
-<a href="https://opencollective.com/xgboost/sponsor/2/website" target="_blank"><img src="https://opencollective.com/xgboost/sponsor/2/avatar.svg"></a>
-<a href="https://opencollective.com/xgboost/sponsor/3/website" target="_blank"><img src="https://opencollective.com/xgboost/sponsor/3/avatar.svg"></a>
-<a href="https://opencollective.com/xgboost/sponsor/4/website" target="_blank"><img src="https://opencollective.com/xgboost/sponsor/4/avatar.svg"></a>
-<a href="https://opencollective.com/xgboost/sponsor/5/website" target="_blank"><img src="https://opencollective.com/xgboost/sponsor/5/avatar.svg"></a>
-<a href="https://opencollective.com/xgboost/sponsor/6/website" target="_blank"><img src="https://opencollective.com/xgboost/sponsor/6/avatar.svg"></a>
-<a href="https://opencollective.com/xgboost/sponsor/7/website" target="_blank"><img src="https://opencollective.com/xgboost/sponsor/7/avatar.svg"></a>
-<a href="https://opencollective.com/xgboost/sponsor/8/website" target="_blank"><img src="https://opencollective.com/xgboost/sponsor/8/avatar.svg"></a>
-<a href="https://opencollective.com/xgboost/sponsor/9/website" target="_blank"><img src="https://opencollective.com/xgboost/sponsor/9/avatar.svg"></a>
+<a href="https://www.intel.com/" target="_blank"><img src="https://images.opencollective.com/intel-corporation/2fa85c1/logo/256.png" width="72" height="72"></a>
+<a href="https://getkoffie.com/?utm_source=opencollective&utm_medium=github&utm_campaign=xgboost" target="_blank"><img src="https://images.opencollective.com/koffielabs/f391ab8/logo/256.png" width="72" height="72"></a>
 
 ### Backers
 [[Become a backer](https://opencollective.com/xgboost#backer)]
 
 <a href="https://opencollective.com/xgboost#backers" target="_blank"><img src="https://opencollective.com/xgboost/backers.svg?width=890"></a>
-
-## Other sponsors
-The sponsors in this list are donating cloud hours in lieu of cash donation.
-
-<a href="https://aws.amazon.com/" target="_blank"><img src="https://raw.githubusercontent.com/xgboost-ai/xgboost-ai.github.io/master/images/sponsors/aws.png" alt="Amazon Web Services" width="72" height="72"></a>

SECURITY.md

@@ -0,0 +1,22 @@
+# Security Policy
+
+## Supported Versions
+
+<!-- Use this section to tell people about which versions of your project are
+currently being supported with security updates. -->
+Security updates are applied only to the most recent release.
+
+## Reporting a Vulnerability
+
+<!-- Use this section to tell people how to report a vulnerability.
+
+Tell them where to go, how often they can expect to get an update on a
+reported vulnerability, what to expect if the vulnerability is accepted or
+declined, etc. -->
+
+To report a security issue, please email
+[security@xgboost-ci.net](mailto:security@xgboost-ci.net)
+with a description of the issue, the steps you took to create the issue,
+affected versions, and, if known, mitigations for the issue.
+
+All support will be made on the best effort base, so please indicate the "urgency level" of the vulnerability as Critical, High, Medium or Low.

amalgamation/xgboost-all0.cc

@@ -1,89 +0,0 @@
-/*!
- * Copyright 2015-2019 by Contributors.
- * \brief XGBoost Amalgamation.
- *  This offers an alternative way to compile the entire library from this single file.
- *
- *  Example usage command.
- *  - $(CXX) -std=c++0x -fopenmp -o -shared libxgboost.so xgboost-all0.cc -ldmlc -lrabit
- *
- * \author Tianqi Chen.
- */
-
-// metrics
-#include "../src/metric/metric.cc"
-#include "../src/metric/elementwise_metric.cc"
-#include "../src/metric/multiclass_metric.cc"
-#include "../src/metric/rank_metric.cc"
-#include "../src/metric/auc.cc"
-#include "../src/metric/survival_metric.cc"
-
-// objectives
-#include "../src/objective/objective.cc"
-#include "../src/objective/regression_obj.cc"
-#include "../src/objective/multiclass_obj.cc"
-#include "../src/objective/rank_obj.cc"
-#include "../src/objective/hinge.cc"
-#include "../src/objective/aft_obj.cc"
-
-// gbms
-#include "../src/gbm/gbm.cc"
-#include "../src/gbm/gbtree.cc"
-#include "../src/gbm/gbtree_model.cc"
-#include "../src/gbm/gblinear.cc"
-#include "../src/gbm/gblinear_model.cc"
-
-// data
-#include "../src/data/simple_dmatrix.cc"
-#include "../src/data/data.cc"
-#include "../src/data/sparse_page_raw_format.cc"
-#include "../src/data/ellpack_page.cc"
-#include "../src/data/gradient_index.cc"
-#include "../src/data/gradient_index_page_source.cc"
-#include "../src/data/gradient_index_format.cc"
-#include "../src/data/sparse_page_dmatrix.cc"
-#include "../src/data/proxy_dmatrix.cc"
-
-// prediction
-#include "../src/predictor/predictor.cc"
-#include "../src/predictor/cpu_predictor.cc"
-
-// trees
-#include "../src/tree/constraints.cc"
-#include "../src/tree/hist/param.cc"
-#include "../src/tree/param.cc"
-#include "../src/tree/tree_model.cc"
-#include "../src/tree/tree_updater.cc"
-#include "../src/tree/updater_approx.cc"
-#include "../src/tree/updater_colmaker.cc"
-#include "../src/tree/updater_histmaker.cc"
-#include "../src/tree/updater_prune.cc"
-#include "../src/tree/updater_quantile_hist.cc"
-#include "../src/tree/updater_refresh.cc"
-#include "../src/tree/updater_sync.cc"
-
-// linear
-#include "../src/linear/linear_updater.cc"
-#include "../src/linear/updater_coordinate.cc"
-#include "../src/linear/updater_shotgun.cc"
-
-// global
-#include "../src/learner.cc"
-#include "../src/logging.cc"
-#include "../src/global_config.cc"
-#include "../src/common/common.cc"
-#include "../src/common/random.cc"
-#include "../src/common/charconv.cc"
-#include "../src/common/timer.cc"
-#include "../src/common/quantile.cc"
-#include "../src/common/host_device_vector.cc"
-#include "../src/common/hist_util.cc"
-#include "../src/common/io.cc"
-#include "../src/common/json.cc"
-#include "../src/common/pseudo_huber.cc"
-#include "../src/common/survival_util.cc"
-#include "../src/common/threading_utils.cc"
-#include "../src/common/version.cc"
-
-// c_api
-#include "../src/c_api/c_api.cc"
-#include "../src/c_api/c_api_error.cc"

cmake/Utils.cmake

@@ -91,21 +91,21 @@ function(format_gencode_flags flags out)
   # Set up architecture flags
   if(NOT flags)
     if (CUDA_VERSION VERSION_GREATER_EQUAL "11.1")
-      set(flags "50;52;60;61;70;75;80;86")
+      set(flags "50;60;70;80")
     elseif (CUDA_VERSION VERSION_GREATER_EQUAL "11.0")
-      set(flags "35;50;52;60;61;70;75;80")
+      set(flags "50;60;70;80")
     elseif(CUDA_VERSION VERSION_GREATER_EQUAL "10.0")
-      set(flags "35;50;52;60;61;70;75")
+      set(flags "35;50;60;70")
     elseif(CUDA_VERSION VERSION_GREATER_EQUAL "9.0")
-      set(flags "35;50;52;60;61;70")
+      set(flags "35;50;60;70")
     else()
-      set(flags "35;50;52;60;61")
+      set(flags "35;50;60")
     endif()
   endif()
 
   if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.18")
     cmake_policy(SET CMP0104 NEW)
-    list(POP_BACK flags latest_arch)
+    list(GET flags -1 latest_arch)
     list(TRANSFORM flags APPEND "-real")
     list(APPEND flags ${latest_arch})
     set(CMAKE_CUDA_ARCHITECTURES ${flags})
@@ -144,6 +144,15 @@ function(xgboost_set_cuda_flags target)
     set_property(TARGET ${target} PROPERTY CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES})
   endif (CMAKE_VERSION VERSION_GREATER_EQUAL "3.18")
 
+  if (FORCE_COLORED_OUTPUT)
+    if (FORCE_COLORED_OUTPUT AND (CMAKE_GENERATOR STREQUAL "Ninja") AND
+        ((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") OR
+          (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")))
+      target_compile_options(${target} PRIVATE
+        $<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=-fdiagnostics-color=always>)
+    endif()
+  endif (FORCE_COLORED_OUTPUT)
+
   if (USE_DEVICE_DEBUG)
     target_compile_options(${target} PRIVATE
       $<$<AND:$<CONFIG:DEBUG>,$<COMPILE_LANGUAGE:CUDA>>:-G;-src-in-ptx>)
@@ -169,10 +178,17 @@ function(xgboost_set_cuda_flags target)
       $<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=/utf-8>)
   endif (MSVC)
 
-  set_target_properties(${target} PROPERTIES
-    CUDA_STANDARD 14
-    CUDA_STANDARD_REQUIRED ON
-    CUDA_SEPARABLE_COMPILATION OFF)
+  if (PLUGIN_RMM)
+    set_target_properties(${target} PROPERTIES
+      CUDA_STANDARD 17
+      CUDA_STANDARD_REQUIRED ON
+      CUDA_SEPARABLE_COMPILATION OFF)
+  else ()
+    set_target_properties(${target} PROPERTIES
+      CUDA_STANDARD 14
+      CUDA_STANDARD_REQUIRED ON
+      CUDA_SEPARABLE_COMPILATION OFF)
+  endif (PLUGIN_RMM)
 endfunction(xgboost_set_cuda_flags)
 
 macro(xgboost_link_nccl target)
@@ -189,10 +205,18 @@ endmacro(xgboost_link_nccl)
 
 # compile options
 macro(xgboost_target_properties target)
-  set_target_properties(${target} PROPERTIES
-    CXX_STANDARD 14
-    CXX_STANDARD_REQUIRED ON
-    POSITION_INDEPENDENT_CODE ON)
+  if (PLUGIN_RMM)
+    set_target_properties(${target} PROPERTIES
+      CXX_STANDARD 17
+      CXX_STANDARD_REQUIRED ON
+      POSITION_INDEPENDENT_CODE ON)
+  else ()
+    set_target_properties(${target} PROPERTIES
+      CXX_STANDARD 14
+      CXX_STANDARD_REQUIRED ON
+      POSITION_INDEPENDENT_CODE ON)
+  endif (PLUGIN_RMM)
+
   if (HIDE_CXX_SYMBOLS)
     #-- Hide all C++ symbols
     set_target_properties(${target} PROPERTIES
@@ -204,7 +228,9 @@ macro(xgboost_target_properties target)
 
   if (ENABLE_ALL_WARNINGS)
     target_compile_options(${target} PUBLIC
-      $<IF:$<COMPILE_LANGUAGE:CUDA>,-Xcompiler=-Wall -Xcompiler=-Wextra,-Wall -Wextra>
+      $<IF:$<COMPILE_LANGUAGE:CUDA>,
+      -Xcompiler=-Wall -Xcompiler=-Wextra -Xcompiler=-Wno-expansion-to-defined,
+      -Wall -Wextra -Wno-expansion-to-defined>
     )
   endif(ENABLE_ALL_WARNINGS)
 
@@ -218,7 +244,7 @@ macro(xgboost_target_properties target)
       $<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:/utf-8>
       -D_CRT_SECURE_NO_WARNINGS
       -D_CRT_SECURE_NO_DEPRECATE
-      )
+    )
   endif (MSVC)
 
   if (WIN32 AND MINGW)
@@ -247,6 +273,10 @@ macro(xgboost_target_defs target)
       PRIVATE
       -DXGBOOST_BUILTIN_PREFETCH_PRESENT=1)
   endif (XGBOOST_BUILTIN_PREFETCH_PRESENT)
+
+  if (PLUGIN_RMM)
+    target_compile_definitions(objxgboost PUBLIC -DXGBOOST_USE_RMM=1)
+  endif (PLUGIN_RMM)
 endmacro(xgboost_target_defs)
 
 # handles dependencies
@@ -269,6 +299,10 @@ macro(xgboost_target_link_libraries target)
     xgboost_set_cuda_flags(${target})
   endif (USE_CUDA)
 
+  if (PLUGIN_RMM)
+    target_link_libraries(${target} PRIVATE rmm::rmm)
+  endif (PLUGIN_RMM)
+
   if (USE_NCCL)
     xgboost_link_nccl(${target})
   endif (USE_NCCL)
@@ -280,4 +314,8 @@ macro(xgboost_target_link_libraries target)
   if (RABIT_BUILD_MPI)
     target_link_libraries(${target} PRIVATE MPI::MPI_CXX)
   endif (RABIT_BUILD_MPI)
+
+  if (MINGW)
+    target_link_libraries(${target} PRIVATE wsock32 ws2_32)
+  endif (MINGW)
 endmacro(xgboost_target_link_libraries)

demo/README.md

@@ -78,7 +78,7 @@ XGBoost is extensively used by machine learning practitioners to create state of
 this is a list of machine learning winning solutions with XGBoost.
 Please send pull requests if you find ones that are missing here.
 
-
+- Bishwarup Bhattacharjee, 1st place winner of [Allstate Claims Severity](https://www.kaggle.com/competitions/allstate-claims-severity/overview) conducted on December 2016. Link to [discussion](https://www.kaggle.com/competitions/allstate-claims-severity/discussion/26416)
 - Benedikt Schifferer, Gilberto Titericz, Chris Deotte, Christof Henkel, Kazuki Onodera, Jiwei Liu, Bojan Tunguz, Even Oldridge, Gabriel De Souza Pereira Moreira and Ahmet Erdem, 1st place winner of [Twitter RecSys Challenge 2020](https://recsys-twitter.com/) conducted from June,20-August,20. [GPU Accelerated Feature Engineering and Training for Recommender Systems](https://medium.com/rapids-ai/winning-solution-of-recsys2020-challenge-gpu-accelerated-feature-engineering-and-training-for-cd67c5a87b1f)
 - Eugene Khvedchenya,Jessica Fridrich, Jan Butora, Yassine Yousfi 1st place winner in [ALASKA2 Image Steganalysis](https://www.kaggle.com/c/alaska2-image-steganalysis/overview). Link to [discussion](https://www.kaggle.com/c/alaska2-image-steganalysis/discussion/168546)
 - Dan Ofer, Seffi Cohen, Noa Dagan, Nurit, 1st place in WiDS Datathon 2020. Link to [discussion](https://www.kaggle.com/c/widsdatathon2020/discussion/133189)

demo/aft_survival/README.rst

@@ -0,0 +1,5 @@
+Survival Analysis Walkthrough
+=============================
+
+This is a collection of examples for using the XGBoost Python package for training
+survival models. For an introduction, see :doc:`/tutorials/aft_survival_analysis`

demo/aft_survival/aft_survival_demo.py

@@ -1,6 +1,10 @@
 """
-Demo for survival analysis (regression) using Accelerated Failure Time (AFT) model
+Demo for survival analysis (regression).
+========================================
+
+Demo for survival analysis (regression). using Accelerated Failure Time (AFT) model.
 """
+
 import os
 from sklearn.model_selection import ShuffleSplit
 import pandas as pd

demo/aft_survival/aft_survival_demo_with_optuna.py

@@ -1,6 +1,10 @@
 """
-Demo for survival analysis (regression) using Accelerated Failure Time (AFT) model, using Optuna
-to tune hyperparameters
+Demo for survival analysis (regression) with Optuna.
+====================================================
+
+Demo for survival analysis (regression) using Accelerated Failure Time (AFT) model,
+using Optuna to tune hyperparameters
+
 """
 from sklearn.model_selection import ShuffleSplit
 import pandas as pd
@@ -45,7 +49,7 @@ def objective(trial):
     params.update(base_params)
     pruning_callback = optuna.integration.XGBoostPruningCallback(trial, 'valid-aft-nloglik')
     bst = xgb.train(params, dtrain, num_boost_round=10000,
-                    evals=[(dtrain, 'train'), (dvalid, 'valid')], 
+                    evals=[(dtrain, 'train'), (dvalid, 'valid')],
                     early_stopping_rounds=50, verbose_eval=False, callbacks=[pruning_callback])
     if bst.best_iteration >= 25:
         return bst.best_score
@@ -63,7 +67,7 @@ params.update(study.best_trial.params)
 # Re-run training with the best hyperparameter combination
 print('Re-running the best trial... params = {}'.format(params))
 bst = xgb.train(params, dtrain, num_boost_round=10000,
-                evals=[(dtrain, 'train'), (dvalid, 'valid')], 
+                evals=[(dtrain, 'train'), (dvalid, 'valid')],
                 early_stopping_rounds=50)
 
 # Run prediction on the validation set

demo/aft_survival/aft_survival_viz_demo.py

@@ -1,9 +1,10 @@
 """
 Visual demo for survival analysis (regression) with Accelerated Failure Time (AFT) model.
+=========================================================================================
 
-This demo uses 1D toy data and visualizes how XGBoost fits a tree ensemble. The ensemble model
-starts out as a flat line and evolves into a step function in order to account for all ranged
-labels.
+This demo uses 1D toy data and visualizes how XGBoost fits a tree ensemble. The ensemble
+model starts out as a flat line and evolves into a step function in order to account for
+all ranged labels.
 """
 import numpy as np
 import xgboost as xgb
@@ -57,7 +58,7 @@ def plot_intermediate_model_callback(env):
     #              the corresponding predicted label (y_pred)
     acc = np.sum(np.logical_and(y_pred >= y_lower, y_pred <= y_upper)/len(X) * 100)
     accuracy_history.append(acc)
-    
+
     # Plot ranged labels as well as predictions by the model
     plt.subplot(5, 3, env.iteration + 1)
     plot_censored_labels(X, y_lower, y_upper)

demo/c-api/basic/c-api-demo.c

@@ -18,7 +18,7 @@ if (err != 0) {                                                         \
 }                                                                       \
 }
 
-int main(int argc, char** argv) {
+int main() {
   int silent = 0;
   int use_gpu = 0;  // set to 1 to use the GPU for training
 
@@ -67,10 +67,21 @@ int main(int argc, char** argv) {
 
   // predict
   bst_ulong out_len = 0;
-  const float* out_result = NULL;
   int n_print = 10;
 
-  safe_xgboost(XGBoosterPredict(booster, dtest, 0, 0, 0, &out_len, &out_result));
+  /* Run prediction with DMatrix object. */
+  char const config[] =
+      "{\"training\": false, \"type\": 0, "
+      "\"iteration_begin\": 0, \"iteration_end\": 0, \"strict_shape\": false}";
+  /* Shape of output prediction */
+  uint64_t const* out_shape;
+  /* Dimension of output prediction */
+  uint64_t out_dim;
+  /* Pointer to a thread local contigious array, assigned in prediction function. */
+  float const* out_result = NULL;
+  safe_xgboost(
+      XGBoosterPredictFromDMatrix(booster, dtest, config, &out_shape, &out_dim, &out_result));
+
   printf("y_pred: ");
   for (int i = 0; i < n_print; ++i) {
     printf("%1.4f ", out_result[i]);
@@ -98,12 +109,12 @@ int main(int argc, char** argv) {
     DMatrixHandle dmat;
     safe_xgboost(XGDMatrixCreateFromMat(values, 1, 127, 0.0, &dmat));
 
-    bst_ulong out_len = 0;
     const float* out_result = NULL;
 
-    safe_xgboost(XGBoosterPredict(booster, dmat, 0, 0, 0, &out_len,
-          &out_result));
-    assert(out_len == 1);
+    safe_xgboost(
+        XGBoosterPredictFromDMatrix(booster, dmat, config, &out_shape, &out_dim, &out_result));
+    assert(out_dim == 1);
+    assert(out_shape[0] == 1);
 
     printf("%1.4f \n", out_result[0]);
     safe_xgboost(XGDMatrixFree(dmat));
@@ -122,12 +133,12 @@ int main(int argc, char** argv) {
     safe_xgboost(XGDMatrixCreateFromCSREx(indptr, indices, data, 2, 22, 127,
       &dmat));
 
-    bst_ulong out_len = 0;
     const float* out_result = NULL;
 
-    safe_xgboost(XGBoosterPredict(booster, dmat, 0, 0, 0, &out_len,
-          &out_result));
-    assert(out_len == 1);
+    safe_xgboost(
+        XGBoosterPredictFromDMatrix(booster, dmat, config, &out_shape, &out_dim, &out_result));
+    assert(out_dim == 1);
+    assert(out_shape[0] == 1);
 
     printf("%1.4f \n", out_result[0]);
     safe_xgboost(XGDMatrixFree(dmat));
@@ -154,12 +165,12 @@ int main(int argc, char** argv) {
     safe_xgboost(XGDMatrixCreateFromCSCEx(col_ptr, indices, data, 128, 22, 1,
       &dmat));
 
-    bst_ulong out_len = 0;
     const float* out_result = NULL;
 
-    safe_xgboost(XGBoosterPredict(booster, dmat, 0, 0, 0, &out_len,
-          &out_result));
-    assert(out_len == 1);
+    safe_xgboost(
+        XGBoosterPredictFromDMatrix(booster, dmat, config, &out_shape, &out_dim, &out_result));
+    assert(out_dim == 1);
+    assert(out_shape[0] == 1);
 
     printf("%1.4f \n", out_result[0]);
     safe_xgboost(XGDMatrixFree(dmat));

demo/c-api/external-memory/external_memory.c

@@ -139,8 +139,8 @@ void TrainModel(DMatrix Xy) {
   Booster booster;
   DMatrix cache[] = {Xy};
   safe_xgboost(XGBoosterCreate(cache, 1, &booster));
-  /* Use approx for external memory training. */
-  safe_xgboost(XGBoosterSetParam(booster, "tree_method", "approx"));
+  /* Use approx or hist for external memory training. */
+  safe_xgboost(XGBoosterSetParam(booster, "tree_method", "hist"));
   safe_xgboost(XGBoosterSetParam(booster, "objective", "reg:squarederror"));
 
   /* Start training. */

demo/dask/README.rst

@@ -1,3 +1,5 @@
+.. _dask-examples:
+
 XGBoost Dask Feature Walkthrough
 ================================
 

demo/dask/gpu_training.py

@@ -3,13 +3,13 @@ Example of training with Dask on GPU
 ====================================
 """
 from dask_cuda import LocalCUDACluster
+import dask_cudf
 from dask.distributed import Client
 from dask import array as da
+from dask import dataframe as dd
 import xgboost as xgb
 from xgboost import dask as dxgb
 from xgboost.dask import DaskDMatrix
-import cupy as cp
-import argparse
 
 
 def using_dask_matrix(client: Client, X, y):
@@ -45,12 +45,12 @@ def using_quantile_device_dmatrix(client: Client, X, y):
 
     '''
     # Input must be on GPU for `DaskDeviceQuantileDMatrix`.
-    X = X.map_blocks(cp.array)
-    y = y.map_blocks(cp.array)
+    X = dask_cudf.from_dask_dataframe(dd.from_dask_array(X))
+    y = dask_cudf.from_dask_dataframe(dd.from_dask_array(y))
 
     # `DaskDeviceQuantileDMatrix` is used instead of `DaskDMatrix`, be careful
-    # that it can not be used for anything else than training.
-    dtrain = dxgb.DaskDeviceQuantileDMatrix(client, X, y)
+    # that it can not be used for anything else other than training.
+    dtrain = dxgb.DaskQuantileDMatrix(client, X, y)
     output = xgb.dask.train(client,
                             {'verbosity': 2,
                              'tree_method': 'gpu_hist'},
@@ -62,12 +62,6 @@ def using_quantile_device_dmatrix(client: Client, X, y):
 
 
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        '--ddqdm', choices=[0, 1], type=int, default=1,
-        help='''Whether should we use `DaskDeviceQuantileDMatrix`''')
-    args = parser.parse_args()
-
     # `LocalCUDACluster` is used for assigning GPU to XGBoost processes.  Here
     # `n_workers` represents the number of GPUs since we use one GPU per worker
     # process.
@@ -76,12 +70,10 @@ if __name__ == '__main__':
             # generate some random data for demonstration
             m = 100000
             n = 100
-            X = da.random.random(size=(m, n), chunks=100)
-            y = da.random.random(size=(m, ), chunks=100)
+            X = da.random.random(size=(m, n), chunks=10000)
+            y = da.random.random(size=(m, ), chunks=10000)
 
-            if args.ddqdm == 1:
-                print('Using DaskDeviceQuantileDMatrix')
-                from_ddqdm = using_quantile_device_dmatrix(client, X, y)
-            else:
-                print('Using DMatrix')
-                from_dmatrix = using_dask_matrix(client, X, y)
+            print('Using DaskQuantileDMatrix')
+            from_ddqdm = using_quantile_device_dmatrix(client, X, y)
+            print('Using DMatrix')
+            from_dmatrix = using_dask_matrix(client, X, y)

demo/guide-python/basic_walkthrough.py

@@ -1,50 +1,65 @@
 """
 Getting started with XGBoost
 ============================
+
+This is a simple example of using the native XGBoost interface, there are other
+interfaces in the Python package like scikit-learn interface and Dask interface.
+
+
+See :doc:`/python/python_intro` and :doc:`/tutorials/index` for other references.
+
 """
 import numpy as np
-import scipy.sparse
 import pickle
 import xgboost as xgb
 import os
 
+from sklearn.datasets import load_svmlight_file
+
 # Make sure the demo knows where to load the data.
 CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
 XGBOOST_ROOT_DIR = os.path.dirname(os.path.dirname(CURRENT_DIR))
-DEMO_DIR = os.path.join(XGBOOST_ROOT_DIR, 'demo')
+DEMO_DIR = os.path.join(XGBOOST_ROOT_DIR, "demo")
 
-# simple example
-# load file from text file, also binary buffer generated by xgboost
-dtrain = xgb.DMatrix(os.path.join(DEMO_DIR, 'data', 'agaricus.txt.train?indexing_mode=1'))
-dtest = xgb.DMatrix(os.path.join(DEMO_DIR, 'data', 'agaricus.txt.test?indexing_mode=1'))
+# X is a scipy csr matrix, XGBoost supports many other input types,
+X, y = load_svmlight_file(os.path.join(DEMO_DIR, "data", "agaricus.txt.train"))
+dtrain = xgb.DMatrix(X, y)
+# validation set
+X_test, y_test = load_svmlight_file(os.path.join(DEMO_DIR, "data", "agaricus.txt.test"))
+dtest = xgb.DMatrix(X_test, y_test)
 
 # specify parameters via map, definition are same as c++ version
-param = {'max_depth': 2, 'eta': 1, 'objective': 'binary:logistic'}
+param = {"max_depth": 2, "eta": 1, "objective": "binary:logistic"}
 
 # specify validations set to watch performance
-watchlist = [(dtest, 'eval'), (dtrain, 'train')]
+watchlist = [(dtest, "eval"), (dtrain, "train")]
+# number of boosting rounds
 num_round = 2
-bst = xgb.train(param, dtrain, num_round, watchlist)
+bst = xgb.train(param, dtrain, num_boost_round=num_round, evals=watchlist)
 
-# this is prediction
+# run prediction
 preds = bst.predict(dtest)
 labels = dtest.get_label()
-print('error=%f' %
-      (sum(1 for i in range(len(preds)) if int(preds[i] > 0.5) != labels[i]) /
-       float(len(preds))))
-bst.save_model('0001.model')
+print(
+    "error=%f"
+    % (
+        sum(1 for i in range(len(preds)) if int(preds[i] > 0.5) != labels[i])
+        / float(len(preds))
+    )
+)
+bst.save_model("model-0.json")
 # dump model
-bst.dump_model('dump.raw.txt')
+bst.dump_model("dump.raw.txt")
 # dump model with feature map
-bst.dump_model('dump.nice.txt', os.path.join(DEMO_DIR, 'data/featmap.txt'))
+bst.dump_model("dump.nice.txt", os.path.join(DEMO_DIR, "data/featmap.txt"))
 
 # save dmatrix into binary buffer
-dtest.save_binary('dtest.buffer')
+dtest.save_binary("dtest.dmatrix")
 # save model
-bst.save_model('xgb.model')
+bst.save_model("model-1.json")
 # load model and data in
-bst2 = xgb.Booster(model_file='xgb.model')
-dtest2 = xgb.DMatrix('dtest.buffer')
+bst2 = xgb.Booster(model_file="model-1.json")
+dtest2 = xgb.DMatrix("dtest.dmatrix")
 preds2 = bst2.predict(dtest2)
 # assert they are the same
 assert np.sum(np.abs(preds2 - preds)) == 0
@@ -56,40 +71,3 @@ bst3 = pickle.loads(pks)
 preds3 = bst3.predict(dtest2)
 # assert they are the same
 assert np.sum(np.abs(preds3 - preds)) == 0
-
-###
-# build dmatrix from scipy.sparse
-print('start running example of build DMatrix from scipy.sparse CSR Matrix')
-labels = []
-row = []
-col = []
-dat = []
-i = 0
-for l in open(os.path.join(DEMO_DIR, 'data', 'agaricus.txt.train')):
-    arr = l.split()
-    labels.append(int(arr[0]))
-    for it in arr[1:]:
-        k, v = it.split(':')
-        row.append(i)
-        col.append(int(k))
-        dat.append(float(v))
-    i += 1
-csr = scipy.sparse.csr_matrix((dat, (row, col)))
-dtrain = xgb.DMatrix(csr, label=labels)
-watchlist = [(dtest, 'eval'), (dtrain, 'train')]
-bst = xgb.train(param, dtrain, num_round, watchlist)
-
-print('start running example of build DMatrix from scipy.sparse CSC Matrix')
-# we can also construct from csc matrix
-csc = scipy.sparse.csc_matrix((dat, (row, col)))
-dtrain = xgb.DMatrix(csc, label=labels)
-watchlist = [(dtest, 'eval'), (dtrain, 'train')]
-bst = xgb.train(param, dtrain, num_round, watchlist)
-
-print('start running example of build DMatrix from numpy array')
-# NOTE: npymat is numpy array, we will convert it into scipy.sparse.csr_matrix
-# in internal implementation then convert to DMatrix
-npymat = csr.todense()
-dtrain = xgb.DMatrix(npymat, label=labels)
-watchlist = [(dtest, 'eval'), (dtrain, 'train')]
-bst = xgb.train(param, dtrain, num_round, watchlist)

demo/guide-python/cat_in_the_dat.py

@@ -19,13 +19,14 @@ Also, see the tutorial for using XGBoost with categorical data:
 """
 
 from __future__ import annotations
-from time import time
+
 import os
 from tempfile import TemporaryDirectory
+from time import time
 
 import pandas as pd
-from sklearn.model_selection import train_test_split
 from sklearn.metrics import roc_auc_score
+from sklearn.model_selection import train_test_split
 
 import xgboost as xgb
 
@@ -63,7 +64,6 @@ def load_cat_in_the_dat() -> tuple[pd.DataFrame, pd.Series]:
 
 params = {
     "tree_method": "gpu_hist",
-    "use_label_encoder": False,
     "n_estimators": 32,
     "colsample_bylevel": 0.7,
 }

demo/guide-python/categorical.py

@@ -16,11 +16,13 @@ categorical data.
     .. versionadded:: 1.5.0
 
 """
-import pandas as pd
-import numpy as np
-import xgboost as xgb
 from typing import Tuple
 
+import numpy as np
+import pandas as pd
+
+import xgboost as xgb
+
 
 def make_categorical(
     n_samples: int, n_features: int, n_categories: int, onehot: bool

demo/guide-python/continuation.py

@@ -14,13 +14,13 @@ def training_continuation(tmpdir: str, use_pickle: bool) -> None:
     """Basic training continuation."""
     # Train 128 iterations in 1 session
     X, y = load_breast_cancer(return_X_y=True)
-    clf = xgboost.XGBClassifier(n_estimators=128, use_label_encoder=False)
+    clf = xgboost.XGBClassifier(n_estimators=128)
     clf.fit(X, y, eval_set=[(X, y)], eval_metric="logloss")
     print("Total boosted rounds:", clf.get_booster().num_boosted_rounds())
 
     # Train 128 iterations in 2 sessions, with the first one runs for 32 iterations and
     # the second one runs for 96 iterations
-    clf = xgboost.XGBClassifier(n_estimators=32, use_label_encoder=False)
+    clf = xgboost.XGBClassifier(n_estimators=32)
     clf.fit(X, y, eval_set=[(X, y)], eval_metric="logloss")
     assert clf.get_booster().num_boosted_rounds() == 32
 
@@ -54,14 +54,14 @@ def training_continuation_early_stop(tmpdir: str, use_pickle: bool) -> None:
     n_estimators = 512
 
     X, y = load_breast_cancer(return_X_y=True)
-    clf = xgboost.XGBClassifier(n_estimators=n_estimators, use_label_encoder=False)
+    clf = xgboost.XGBClassifier(n_estimators=n_estimators)
     clf.fit(X, y, eval_set=[(X, y)], eval_metric="logloss", callbacks=[early_stop])
     print("Total boosted rounds:", clf.get_booster().num_boosted_rounds())
     best = clf.best_iteration
 
     # Train 512 iterations in 2 sessions, with the first one runs for 128 iterations and
     # the second one runs until early stop.
-    clf = xgboost.XGBClassifier(n_estimators=128, use_label_encoder=False)
+    clf = xgboost.XGBClassifier(n_estimators=128)
     # Reinitialize the early stop callback
     early_stop = xgboost.callback.EarlyStopping(
         rounds=early_stopping_rounds, save_best=True
@@ -79,15 +79,13 @@ def training_continuation_early_stop(tmpdir: str, use_pickle: bool) -> None:
     else:
         path = os.path.join(tmpdir, "model-first-128.json")
         clf.save_model(path)
-        loaded = xgboost.XGBClassifier(use_label_encoder=False)
+        loaded = xgboost.XGBClassifier()
         loaded.load_model(path)
 
     early_stop = xgboost.callback.EarlyStopping(
         rounds=early_stopping_rounds, save_best=True
     )
-    clf = xgboost.XGBClassifier(
-        n_estimators=n_estimators - 128, use_label_encoder=False
-    )
+    clf = xgboost.XGBClassifier(n_estimators=n_estimators - 128)
     clf.fit(
         X,
         y,

demo/guide-python/predict_first_ntree.py

@@ -35,7 +35,7 @@ def native_interface():
 def sklearn_interface():
     X_train, y_train = load_svmlight_file(train)
     X_test, y_test = load_svmlight_file(test)
-    clf = xgb.XGBClassifier(n_estimators=3, max_depth=2, eta=1, use_label_encoder=False)
+    clf = xgb.XGBClassifier(n_estimators=3, max_depth=2, eta=1)
     clf.fit(X_train, y_train, eval_set=[(X_test, y_test)])
     assert clf.n_classes_ == 2
 

demo/guide-python/sklearn_examples.py

@@ -50,8 +50,8 @@ for train_index, test_index in kf.split(X):
 print("Parameter optimization")
 xgb_model = xgb.XGBRegressor(n_jobs=1)
 clf = GridSearchCV(xgb_model,
-                   {'max_depth': [2, 4, 6],
-                    'n_estimators': [50, 100, 200]}, verbose=1, n_jobs=1)
+                   {'max_depth': [2, 4],
+                    'n_estimators': [50, 100]}, verbose=1, n_jobs=1, cv=3)
 clf.fit(X, y)
 print(clf.best_score_)
 print(clf.best_params_)

demo/guide-python/spark_estimator_examples.py

@@ -0,0 +1,96 @@
+"""
+Collection of examples for using xgboost.spark estimator interface
+==================================================================
+
+@author: Weichen Xu
+"""
+import sklearn.datasets
+from pyspark.ml.evaluation import MulticlassClassificationEvaluator, RegressionEvaluator
+from pyspark.ml.linalg import Vectors
+from pyspark.sql import SparkSession
+from pyspark.sql.functions import rand
+from sklearn.model_selection import train_test_split
+from xgboost.spark import SparkXGBClassifier, SparkXGBRegressor
+
+spark = SparkSession.builder.master("local[*]").getOrCreate()
+
+
+def create_spark_df(X, y):
+    return spark.createDataFrame(
+        spark.sparkContext.parallelize(
+            [(Vectors.dense(features), float(label)) for features, label in zip(X, y)]
+        ),
+        ["features", "label"],
+    )
+
+
+# load diabetes dataset (regression dataset)
+diabetes_X, diabetes_y = sklearn.datasets.load_diabetes(return_X_y=True)
+diabetes_X_train, diabetes_X_test, diabetes_y_train, diabetes_y_test = train_test_split(
+    diabetes_X, diabetes_y, test_size=0.3, shuffle=True
+)
+
+diabetes_train_spark_df = create_spark_df(diabetes_X_train, diabetes_y_train)
+diabetes_test_spark_df = create_spark_df(diabetes_X_test, diabetes_y_test)
+
+# train xgboost regressor model
+xgb_regressor = SparkXGBRegressor(max_depth=5)
+xgb_regressor_model = xgb_regressor.fit(diabetes_train_spark_df)
+
+transformed_diabetes_test_spark_df = xgb_regressor_model.transform(
+    diabetes_test_spark_df
+)
+regressor_evaluator = RegressionEvaluator(metricName="rmse")
+print(
+    f"regressor rmse={regressor_evaluator.evaluate(transformed_diabetes_test_spark_df)}"
+)
+
+diabetes_train_spark_df2 = diabetes_train_spark_df.withColumn(
+    "validationIndicatorCol", rand(1) > 0.7
+)
+
+# train xgboost regressor model with validation dataset
+xgb_regressor2 = SparkXGBRegressor(
+    max_depth=5, validation_indicator_col="validationIndicatorCol"
+)
+xgb_regressor_model2 = xgb_regressor2.fit(diabetes_train_spark_df2)
+transformed_diabetes_test_spark_df2 = xgb_regressor_model2.transform(
+    diabetes_test_spark_df
+)
+print(
+    f"regressor2 rmse={regressor_evaluator.evaluate(transformed_diabetes_test_spark_df2)}"
+)
+
+
+# load iris dataset (classification dataset)
+iris_X, iris_y = sklearn.datasets.load_iris(return_X_y=True)
+iris_X_train, iris_X_test, iris_y_train, iris_y_test = train_test_split(
+    iris_X, iris_y, test_size=0.3, shuffle=True
+)
+
+iris_train_spark_df = create_spark_df(iris_X_train, iris_y_train)
+iris_test_spark_df = create_spark_df(iris_X_test, iris_y_test)
+
+# train xgboost classifier model
+xgb_classifier = SparkXGBClassifier(max_depth=5)
+xgb_classifier_model = xgb_classifier.fit(iris_train_spark_df)
+
+transformed_iris_test_spark_df = xgb_classifier_model.transform(iris_test_spark_df)
+classifier_evaluator = MulticlassClassificationEvaluator(metricName="f1")
+print(f"classifier f1={classifier_evaluator.evaluate(transformed_iris_test_spark_df)}")
+
+iris_train_spark_df2 = iris_train_spark_df.withColumn(
+    "validationIndicatorCol", rand(1) > 0.7
+)
+
+# train xgboost classifier model with validation dataset
+xgb_classifier2 = SparkXGBClassifier(
+    max_depth=5, validation_indicator_col="validationIndicatorCol"
+)
+xgb_classifier_model2 = xgb_classifier2.fit(iris_train_spark_df2)
+transformed_iris_test_spark_df2 = xgb_classifier_model2.transform(iris_test_spark_df)
+print(
+    f"classifier2 f1={classifier_evaluator.evaluate(transformed_iris_test_spark_df2)}"
+)
+
+spark.stop()

demo/nvflare/README.md

@@ -0,0 +1,64 @@
+# Experimental Support of Federated XGBoost using NVFlare
+
+This directory contains a demo of Federated Learning using
+[NVFlare](https://nvidia.github.io/NVFlare/).
+
+## Training with CPU only
+
+To run the demo, first build XGBoost with the federated learning plugin enabled (see the
+[README](../../plugin/federated/README.md)).
+
+Install NVFlare (note that currently NVFlare only supports Python 3.8; for NVFlare 2.1.2 we also
+need to pin the protobuf package to 3.20.x to avoid protoc errors):
+```shell
+pip install nvflare protobuf==3.20.1
+```
+
+Prepare the data:
+```shell
+./prepare_data.sh
+```
+
+Start the NVFlare federated server:
+```shell
+./poc/server/startup/start.sh
+```
+
+In another terminal, start the first worker:
+```shell
+./poc/site-1/startup/start.sh
+```
+
+And the second worker:
+```shell
+./poc/site-2/startup/start.sh
+```
+
+Then start the admin CLI, using `admin/admin` as username/password:
+```shell
+./poc/admin/startup/fl_admin.sh
+```
+
+In the admin CLI, run the following command:
+```shell
+submit_job hello-xgboost
+```
+
+Once the training finishes, the model file should be written into
+`./poc/site-1/run_1/test.model.json` and `./poc/site-2/run_1/test.model.json`
+respectively.
+
+Finally, shutdown everything from the admin CLI:
+```shell
+shutdown client
+shutdown server
+```
+
+## Training with GPUs
+
+To demo with Federated Learning using GPUs, make sure your machine has at least 2 GPUs.
+Build XGBoost with the federated learning plugin enabled along with CUDA, but with NCCL
+turned off (see the [README](../../plugin/federated/README.md)).
+
+Modify `config/config_fed_client.json` and set `use_gpus` to `true`, then repeat the steps
+above.

demo/nvflare/config/config_fed_client.json

@@ -0,0 +1,23 @@
+{
+  "format_version": 2,
+  "executors": [
+    {
+      "tasks": [
+        "train"
+      ],
+      "executor": {
+        "path": "trainer.XGBoostTrainer",
+        "args": {
+          "server_address": "localhost:9091",
+          "world_size": 2,
+          "server_cert_path": "server-cert.pem",
+          "client_key_path": "client-key.pem",
+          "client_cert_path": "client-cert.pem",
+          "use_gpus": "false"
+        }
+      }
+    }
+  ],
+  "task_result_filters": [],
+  "task_data_filters": []
+}

demo/nvflare/config/config_fed_server.json

@@ -0,0 +1,22 @@
+{
+  "format_version": 2,
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "workflows": [
+    {
+      "id": "server_workflow",
+      "path": "controller.XGBoostController",
+      "args": {
+        "port": 9091,
+        "world_size": 2,
+        "server_key_path": "server-key.pem",
+        "server_cert_path": "server-cert.pem",
+        "client_cert_path": "client-cert.pem"
+      }
+    }
+  ],
+  "components": []
+}

demo/nvflare/custom/controller.py

@@ -0,0 +1,68 @@
+"""
+Example of training controller with NVFlare
+===========================================
+"""
+import multiprocessing
+
+import xgboost.federated
+from nvflare.apis.client import Client
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.impl.controller import Controller, Task
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+
+from trainer import SupportedTasks
+
+
+class XGBoostController(Controller):
+    def __init__(self, port: int, world_size: int, server_key_path: str,
+                 server_cert_path: str, client_cert_path: str):
+        """Controller for federated XGBoost.
+
+        Args:
+            port: the port for the gRPC server to listen on.
+            world_size: the number of sites.
+            server_key_path: the path to the server key file.
+            server_cert_path: the path to the server certificate file.
+            client_cert_path: the path to the client certificate file.
+        """
+        super().__init__()
+        self._port = port
+        self._world_size = world_size
+        self._server_key_path = server_key_path
+        self._server_cert_path = server_cert_path
+        self._client_cert_path = client_cert_path
+        self._server = None
+
+    def start_controller(self, fl_ctx: FLContext):
+        self._server = multiprocessing.Process(
+            target=xgboost.federated.run_federated_server,
+            args=(self._port, self._world_size, self._server_key_path,
+                  self._server_cert_path, self._client_cert_path))
+        self._server.start()
+
+    def stop_controller(self, fl_ctx: FLContext):
+        if self._server:
+            self._server.terminate()
+
+    def process_result_of_unknown_task(self, client: Client, task_name: str,
+                                       client_task_id: str, result: Shareable,
+                                       fl_ctx: FLContext):
+        self.log_warning(fl_ctx, f"Unknown task: {task_name} from client {client.name}.")
+
+    def control_flow(self, abort_signal: Signal, fl_ctx: FLContext):
+        self.log_info(fl_ctx, "XGBoost training control flow started.")
+        if abort_signal.triggered:
+            return
+        task = Task(name=SupportedTasks.TRAIN, data=Shareable())
+        self.broadcast_and_wait(
+            task=task,
+            min_responses=self._world_size,
+            fl_ctx=fl_ctx,
+            wait_time_after_min_received=1,
+            abort_signal=abort_signal,
+        )
+        if abort_signal.triggered:
+            return
+
+        self.log_info(fl_ctx, "XGBoost training control flow finished.")

demo/nvflare/custom/trainer.py

@@ -0,0 +1,90 @@
+import os
+
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import ReturnCode, FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable, make_reply
+from nvflare.apis.signal import Signal
+
+import xgboost as xgb
+from xgboost import callback
+
+
+class SupportedTasks(object):
+    TRAIN = "train"
+
+
+class XGBoostTrainer(Executor):
+    def __init__(self, server_address: str, world_size: int, server_cert_path: str,
+                 client_key_path: str, client_cert_path: str, use_gpus: bool):
+        """Trainer for federated XGBoost.
+
+        Args:
+            server_address: address for the gRPC server to connect to.
+            world_size: the number of sites.
+            server_cert_path: the path to the server certificate file.
+            client_key_path: the path to the client key file.
+            client_cert_path: the path to the client certificate file.
+        """
+        super().__init__()
+        self._server_address = server_address
+        self._world_size = world_size
+        self._server_cert_path = server_cert_path
+        self._client_key_path = client_key_path
+        self._client_cert_path = client_cert_path
+        self._use_gpus = use_gpus
+
+    def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext,
+                abort_signal: Signal) -> Shareable:
+        self.log_info(fl_ctx, f"Executing {task_name}")
+        try:
+            if task_name == SupportedTasks.TRAIN:
+                self._do_training(fl_ctx)
+                return make_reply(ReturnCode.OK)
+            else:
+                self.log_error(fl_ctx, f"{task_name} is not a supported task.")
+                return make_reply(ReturnCode.TASK_UNKNOWN)
+        except BaseException as e:
+            self.log_exception(fl_ctx,
+                               f"Task {task_name} failed. Exception: {e.__str__()}")
+            return make_reply(ReturnCode.EXECUTION_EXCEPTION)
+
+    def _do_training(self, fl_ctx: FLContext):
+        client_name = fl_ctx.get_prop(FLContextKey.CLIENT_NAME)
+        rank = int(client_name.split('-')[1]) - 1
+        communicator_env = {
+            'xgboost_communicator': 'federated',
+            'federated_server_address': self._server_address,
+            'federated_world_size': self._world_size,
+            'federated_rank': rank,
+            'federated_server_cert': self._server_cert_path,
+            'federated_client_key': self._client_key_path,
+            'federated_client_cert': self._client_cert_path
+        }
+        with xgb.collective.CommunicatorContext(**communicator_env):
+            # Load file, file will not be sharded in federated mode.
+            dtrain = xgb.DMatrix('agaricus.txt.train')
+            dtest = xgb.DMatrix('agaricus.txt.test')
+
+            # Specify parameters via map, definition are same as c++ version
+            param = {'max_depth': 2, 'eta': 1, 'objective': 'binary:logistic'}
+            if self._use_gpus:
+                self.log_info(fl_ctx, f'Training with GPU {rank}')
+                param['tree_method'] = 'gpu_hist'
+                param['gpu_id'] = rank
+
+            # Specify validations set to watch performance
+            watchlist = [(dtest, 'eval'), (dtrain, 'train')]
+            num_round = 20
+
+            # Run training, all the features in training API is available.
+            bst = xgb.train(param, dtrain, num_round, evals=watchlist,
+                            early_stopping_rounds=2, verbose_eval=False,
+                            callbacks=[callback.EvaluationMonitor(rank=rank)])
+
+            # Save the model.
+            workspace = fl_ctx.get_prop(FLContextKey.WORKSPACE_OBJECT)
+            run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+            run_dir = workspace.get_run_dir(run_number)
+            bst.save_model(os.path.join(run_dir, "test.model.json"))
+            xgb.collective.communicator_print("Finished training\n")

demo/nvflare/prepare_data.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -e
+
+rm -fr ./agaricus* ./*.pem ./poc
+
+world_size=2
+
+# Generate server and client certificates.
+openssl req -x509 -newkey rsa:2048 -days 7 -nodes -keyout server-key.pem -out server-cert.pem -subj "/C=US/CN=localhost"
+openssl req -x509 -newkey rsa:2048 -days 7 -nodes -keyout client-key.pem -out client-cert.pem -subj "/C=US/CN=localhost"
+
+# Split train and test files manually to simulate a federated environment.
+split -n l/${world_size} --numeric-suffixes=1 -a 1 ../data/agaricus.txt.train agaricus.txt.train-site-
+split -n l/${world_size} --numeric-suffixes=1 -a 1 ../data/agaricus.txt.test agaricus.txt.test-site-
+
+poc -n 2
+mkdir -p poc/admin/transfer/hello-xgboost
+cp -fr config custom poc/admin/transfer/hello-xgboost
+cp server-*.pem client-cert.pem poc/server/
+for id in $(eval echo "{1..$world_size}"); do
+  cp server-cert.pem client-*.pem poc/site-"$id"/
+  cp agaricus.txt.train-site-"$id" poc/site-"$id"/agaricus.txt.train
+  cp agaricus.txt.test-site-"$id" poc/site-"$id"/agaricus.txt.test
+done

dev/prepare_jvm_release.py

@@ -1,9 +1,15 @@
+import re
 import os
 import sys
+import platform
 import errno
+import argparse
 import subprocess
 import glob
 import shutil
+import tempfile
+import zipfile
+from urllib.request import urlretrieve
 from contextlib import contextmanager
 
 def normpath(path):
@@ -44,7 +50,48 @@ def run(command, **kwargs):
     print(command)
     subprocess.check_call(command, shell=True, **kwargs)
 
+def get_current_git_tag():
+    out = subprocess.check_output(["git", "tag", "--points-at", "HEAD"])
+    return out.decode().split("\n")[0]
+
+def get_current_commit_hash():
+    out = subprocess.check_output(["git", "rev-parse", "HEAD"])
+    return out.decode().split("\n")[0]
+
+def get_current_git_branch():
+    out = subprocess.check_output(["git", "log", "-n", "1", "--pretty=%d", "HEAD"])
+    m = re.search(r"release_[0-9\.]+", out.decode())
+    if not m:
+        raise ValueError("Expected branch name of form release_xxx")
+    return m.group(0)
+
+def retrieve(url, filename=None):
+    print(f"{url} -> {filename}")
+    return urlretrieve(url, filename)
+
 def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--release-version", type=str, required=True,
+                        help="Version of the release being prepared")
+    args = parser.parse_args()
+
+    if sys.platform != "darwin" or platform.machine() != "x86_64":
+        raise NotImplementedError("Please run this script using an Intel Mac")
+
+    version = args.release_version
+    expected_git_tag = "v" + version
+    current_git_tag = get_current_git_tag()
+    if current_git_tag != expected_git_tag:
+        if not current_git_tag:
+            raise ValueError(f"Expected git tag {expected_git_tag} but current HEAD has no tag. "
+                             f"Run: git checkout {expected_git_tag}")
+        raise ValueError(f"Expected git tag {expected_git_tag} but current HEAD is at tag "
+                         f"{current_git_tag}. Run: git checkout {expected_git_tag}")
+
+    commit_hash = get_current_commit_hash()
+    git_branch = get_current_git_branch()
+    print(f"Using commit {commit_hash} of branch {git_branch}, git tag {current_git_tag}")
+
     with cd("jvm-packages/"):
         print("====copying pure-Python tracker====")
         for use_cuda in [True, False]:
@@ -67,12 +114,46 @@ def main():
                 cp(file, f"{xgboost4j_spark}/src/test/resources")
 
         print("====Creating directories to hold native binaries====")
-        for os, arch in [("linux", "x86_64"), ("windows", "x86_64"), ("macos", "x86_64")]:
-            output_dir = f"xgboost4j/src/main/resources/lib/{os}/{arch}"
+        for os_ident, arch in [("linux", "x86_64"), ("windows", "x86_64"), ("macos", "x86_64")]:
+            output_dir = f"xgboost4j/src/main/resources/lib/{os_ident}/{arch}"
             maybe_makedirs(output_dir)
-        for os, arch in [("linux", "x86_64")]:
-            output_dir = f"xgboost4j-gpu/src/main/resources/lib/{os}/{arch}"
+        for os_ident, arch in [("linux", "x86_64")]:
+            output_dir = f"xgboost4j-gpu/src/main/resources/lib/{os_ident}/{arch}"
             maybe_makedirs(output_dir)
+
+        print("====Downloading native binaries from CI====")
+        nightly_bucket_prefix = "https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds"
+        maven_repo_prefix = "https://s3-us-west-2.amazonaws.com/xgboost-maven-repo/release/ml/dmlc"
+
+        retrieve(url=f"{nightly_bucket_prefix}/{git_branch}/xgboost4j_{commit_hash}.dll",
+                 filename="xgboost4j/src/main/resources/lib/windows/x86_64/xgboost4j.dll")
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            # libxgboost4j.so for Linux x86_64, CPU only
+            zip_path = os.path.join(tempdir, "xgboost4j_2.12.jar")
+            extract_dir = os.path.join(tempdir, "xgboost4j")
+            retrieve(url=f"{maven_repo_prefix}/xgboost4j_2.12/{version}/"
+                         f"xgboost4j_2.12-{version}.jar",
+                     filename=zip_path)
+            os.mkdir(extract_dir)
+            with zipfile.ZipFile(zip_path, "r") as t:
+                t.extractall(extract_dir)
+            cp(os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"),
+               "xgboost4j/src/main/resources/lib/linux/x86_64/libxgboost4j.so")
+
+            # libxgboost4j.so for Linux x86_64, GPU support
+            zip_path = os.path.join(tempdir, "xgboost4j-gpu_2.12.jar")
+            extract_dir = os.path.join(tempdir, "xgboost4j-gpu")
+            retrieve(url=f"{maven_repo_prefix}/xgboost4j-gpu_2.12/{version}/"
+                         f"xgboost4j-gpu_2.12-{version}.jar",
+                     filename=zip_path)
+            os.mkdir(extract_dir)
+            with zipfile.ZipFile(zip_path, "r") as t:
+                t.extractall(extract_dir)
+            cp(os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"),
+               "xgboost4j-gpu/src/main/resources/lib/linux/x86_64/libxgboost4j.so")
+
+
     print("====Next Steps====")
     print("1. Gain upload right to Maven Central repo.")
     print("1-1. Sign up for a JIRA account at Sonatype: ")
@@ -81,19 +162,9 @@ def main():
           "https://issues.sonatype.org/browse/OSSRH-67724")
     print("2. Store the Sonatype credentials in .m2/settings.xml. See insturctions in "
           "https://central.sonatype.org/publish/publish-maven/")
-    print("3. Obtain Linux and Windows binaries from the CI server")
-    print("3-1. Get xgboost4j_[commit].dll from "
-          "https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds/list.html. Rename it to"
-          "xgboost4j.dll.")
-    print("3-2. For Linux binaries, go to "
-          "https://s3-us-west-2.amazonaws.com/xgboost-maven-repo/list.html and navigate to the "
-          "release/ directory. Find and download two JAR files: xgboost4j_2.12-[version].jar and "
-          "xgboost4j-gpu_2.12-[version].jar. Use unzip command to extract libxgboost4j.so (one "
-          "version compiled with GPU support and another compiled without).")
-    print("4. Put the binaries in xgboost4j(-gpu)/src/main/resources/lib/[os]/[arch]")
-    print("5. Now on a Mac machine, run:")
+    print("3. Now on a Mac machine, run:")
     print("   GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests")
-    print("6. Log into https://oss.sonatype.org/. On the left menu panel, click Staging "
+    print("4. Log into https://oss.sonatype.org/. On the left menu panel, click Staging "
           "Repositories. Visit the URL https://oss.sonatype.org/content/repositories/mldmlc-1085 "
           "to inspect the staged JAR files. Finally, press Release button to publish the "
           "artifacts to the Maven Central repository.")

dev/release-py-r.py

@@ -3,11 +3,10 @@
 tqdm, sh are required to run this script.
 """
 from urllib.request import urlretrieve
-from typing import cast, Tuple
 import argparse
-from typing import List
+from typing import List, Optional
 from sh.contrib import git
-from distutils import version
+from packaging import version
 import subprocess
 import tqdm
 import os
@@ -27,7 +26,8 @@ def show_progress(block_num, block_size, total_size):
 
     downloaded = block_num * block_size
     if downloaded < total_size:
-        pbar.update(block_size / 1024)
+        upper = (total_size - downloaded) / 1024
+        pbar.update(min(block_size / 1024, upper))
     else:
         pbar.close()
         pbar = None
@@ -75,7 +75,7 @@ def download_wheels(
     return filenames
 
 
-def download_py_packages(major: int, minor: int, commit_hash: str):
+def download_py_packages(branch: str, major: int, minor: int, commit_hash: str) -> None:
     platforms = [
         "win_amd64",
         "manylinux2014_x86_64",
@@ -84,7 +84,8 @@ def download_py_packages(major: int, minor: int, commit_hash: str):
         "macosx_12_0_arm64"
     ]
 
-    dir_URL = PREFIX + str(major) + "." + str(minor) + ".0" + "/"
+    branch = branch.split("_")[1]  # release_x.y.z
+    dir_URL = PREFIX + branch + "/"
     src_filename_prefix = "xgboost-" + args.release + "%2B" + commit_hash + "-py3-none-"
     target_filename_prefix = "xgboost-" + args.release + "-py3-none-"
 
@@ -105,16 +106,17 @@ Following steps should be done manually:
     )
 
 
-def download_r_packages(release: str, rc: str, commit: str) -> None:
+def download_r_packages(release: str, branch: str, rc: str, commit: str) -> None:
     platforms = ["win64", "linux"]
     dirname = "./r-packages"
     if not os.path.exists(dirname):
         os.mkdir(dirname)
 
     filenames = []
+    branch = branch.split("_")[1]  # release_x.y.z
 
     for plat in platforms:
-        url = f"{PREFIX}{release}/xgboost_r_gpu_{plat}_{commit}.tar.gz"
+        url = f"{PREFIX}{branch}/xgboost_r_gpu_{plat}_{commit}.tar.gz"
 
         if not rc:
             filename = f"xgboost_r_gpu_{plat}_{release}.tar.gz"
@@ -136,23 +138,33 @@ def check_path():
 def main(args: argparse.Namespace) -> None:
     check_path()
 
-    rel = version.LooseVersion(args.release)
+    rel = version.parse(args.release)
+    assert isinstance(rel, version.Version)
+
+    major = rel.major
+    minor = rel.minor
+    patch = rel.micro
 
     print("Release:", rel)
-    if len(rel.version) == 3:
+    if not rel.is_prerelease:
         # Major release
-        major, minor, patch = version.StrictVersion(args.release).version
-        rc = None
-        rc_ver = None
+        rc: Optional[str] = None
+        rc_ver: Optional[int] = None
     else:
         # RC release
-        major, minor, patch, rc, rc_ver = cast(
-            Tuple[int, int, int, str, int], rel.version
-        )
+        major = rel.major
+        minor = rel.minor
+        patch = rel.micro
+        assert rel.pre is not None
+        rc, rc_ver = rel.pre
         assert rc == "rc"
 
     release = str(major) + "." + str(minor) + "." + str(patch)
-    branch = "release_" + release
+    if args.branch is not None:
+        branch = args.branch
+    else:
+        branch = "release_" + str(major) + "." + str(minor) + ".0"
+
     git.clean("-xdf")
     git.checkout(branch)
     git.pull("origin", branch)
@@ -160,10 +172,10 @@ def main(args: argparse.Namespace) -> None:
     commit_hash = latest_hash()
 
     download_r_packages(
-        release, "" if rc is None else rc + str(rc_ver), commit_hash
+        release, branch, "" if rc is None else rc + str(rc_ver), commit_hash
     )
 
-    download_py_packages(major, minor, commit_hash)
+    download_py_packages(branch, major, minor, commit_hash)
 
 
 if __name__ == "__main__":
@@ -174,5 +186,14 @@ if __name__ == "__main__":
         required=True,
         help="Version tag, e.g. '1.3.2', or '1.5.0rc1'"
     )
+    parser.add_argument(
+        "--branch",
+        type=str,
+        default=None,
+        help=(
+            "Optional branch. Usually patch releases reuse the same branch of the"
+            " major release, but there can be exception."
+        )
+    )
     args = parser.parse_args()
     main(args)

doc/Doxyfile.in

@@ -753,7 +753,7 @@ WARN_LOGFILE           =
 # spaces.
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = @PROJECT_SOURCE_DIR@/include @PROJECT_SOURCE_DIR@/src/common
+INPUT                  = @PROJECT_SOURCE_DIR@/include
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -822,7 +822,7 @@ EXCLUDE_SYMBOLS        =
 # that contain example code fragments that are included (see the \include
 # command).
 
-EXAMPLE_PATH           =
+EXAMPLE_PATH           = @PROJECT_SOURCE_DIR@/demo/c-api/
 
 # If the value of the EXAMPLE_PATH tag contains directories, you can use the
 # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
@@ -836,7 +836,7 @@ EXAMPLE_PATTERNS       =
 # irrespective of the value of the RECURSIVE tag.
 # The default value is: NO.
 
-EXAMPLE_RECURSIVE      = NO
+EXAMPLE_RECURSIVE      = YES
 
 # The IMAGE_PATH tag can be used to specify one or more files or directories
 # that contain images that are to be included in the documentation (see the
@@ -1934,7 +1934,7 @@ ENABLE_PREPROCESSING   = YES
 # The default value is: NO.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-MACRO_EXPANSION        = NO
+MACRO_EXPANSION        = YES
 
 # If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
 # the macro expansion is limited to the macros specified with the PREDEFINED and
@@ -1942,7 +1942,7 @@ MACRO_EXPANSION        = NO
 # The default value is: NO.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-EXPAND_ONLY_PREDEF     = NO
+EXPAND_ONLY_PREDEF     = YES
 
 # If the SEARCH_INCLUDES tag is set to YES the includes files in the
 # INCLUDE_PATH will be searched if a #include is found.
@@ -1974,7 +1974,9 @@ INCLUDE_FILE_PATTERNS  =
 # recursively expanded use the := operator instead of the = operator.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-PREDEFINED             = DMLC_USE_CXX11
+PREDEFINED             = DMLC_USE_CXX11 \
+                         "XGB_DLL=" \
+                         "XGB_EXTERN_C="
 
 # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
 # tag can be used to specify a list of macro names that should be expanded. The

doc/build.rst

@@ -136,9 +136,9 @@ From the command line on Linux starting from the XGBoost directory:
 
   To speed up compilation, the compute version specific to your GPU could be passed to cmake as, e.g., ``-DGPU_COMPUTE_VER=50``. A quick explanation and numbers for some architectures can be found `in this page <https://arnon.dk/matching-sm-architectures-arch-and-gencode-for-various-nvidia-cards/>`_.
 
-.. note:: Enabling distributed GPU training
+.. note:: Faster distributed GPU training with NCCL
 
-  By default, distributed GPU training is disabled and only a single GPU will be used. To enable distributed GPU training, set the option ``USE_NCCL=ON``. Distributed GPU training depends on NCCL2, available at `this link <https://developer.nvidia.com/nccl>`_. Since NCCL2 is only available for Linux machines, **distributed GPU training is available only for Linux**.
+  By default, distributed GPU training is enabled and uses Rabit for communication. For faster training, set the option ``USE_NCCL=ON``. Faster distributed GPU training depends on NCCL2, available at `this link <https://developer.nvidia.com/nccl>`_. Since NCCL2 is only available for Linux machines, **faster distributed GPU training is available only for Linux**.
 
   .. code-block:: bash
 
@@ -198,7 +198,7 @@ There are several ways to build and install the package from source:
 
     python setup.py install --use-cuda --use-nccl
 
-  Please refer to ``setup.py`` for a complete list of avaiable options.  Some other
+  Please refer to ``setup.py`` for a complete list of available options.  Some other
   options used for development are only available for using CMake directly.  See next
   section on how to use CMake with setuptools manually.
 

doc/c.rst

@@ -6,7 +6,59 @@ XGBoost implements a set of C API designed for various bindings, we maintain its
 and the CMake/make build interface.  See :doc:`/tutorials/c_api_tutorial` for an
 introduction and ``demo/c-api/`` for related examples.  Also one can generate doxygen
 document by providing ``-DBUILD_C_DOC=ON`` as parameter to ``CMake`` during build, or
-simply look at function comments in ``include/xgboost/c_api.h``.
+simply look at function comments in ``include/xgboost/c_api.h``. The reference is exported
+to sphinx with the help of breathe, which doesn't contain links to examples but might be
+easier to read. For the original doxygen pages please visit:
 
 * `C API documentation (latest master branch) <https://xgboost.readthedocs.io/en/latest/dev/c__api_8h.html>`_
 * `C API documentation (last stable release) <https://xgboost.readthedocs.io/en/stable/dev/c__api_8h.html>`_
+
+***************
+C API Reference
+***************
+
+.. contents::
+  :backlinks: none
+  :local:
+
+Library
+=======
+
+.. doxygengroup:: Library
+   :project: xgboost
+
+DMatrix
+=======
+
+.. doxygengroup:: DMatrix
+   :project: xgboost
+
+Streaming
+---------
+
+.. doxygengroup:: Streaming
+   :project: xgboost
+
+Booster
+=======
+
+.. doxygengroup:: Booster
+   :project: xgboost
+
+Prediction
+----------
+
+.. doxygengroup:: Prediction
+   :project: xgboost
+
+Serialization
+-------------
+
+.. doxygengroup:: Serialization
+   :project: xgboost
+
+Collective
+==========
+
+.. doxygengroup:: Collective
+   :project: xgboost

doc/conf.py

@@ -57,22 +57,24 @@ except HTTPError:
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
-libpath = os.path.join(curr_path, '../python-package/')
+CURR_PATH = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
+PROJECT_ROOT = os.path.normpath(os.path.join(CURR_PATH, os.path.pardir))
+libpath = os.path.join(PROJECT_ROOT, "python-package/")
 sys.path.insert(0, libpath)
-sys.path.insert(0, curr_path)
+sys.path.insert(0, CURR_PATH)
 
 # -- General configuration ------------------------------------------------
 
 # General information about the project.
-project = u'xgboost'
-author = u'%s developers' % project
-copyright = u'2021, %s' % author
-github_doc_root = 'https://github.com/dmlc/xgboost/tree/master/doc/'
+project = "xgboost"
+author = "%s developers" % project
+copyright = "2022, %s" % author
+github_doc_root = "https://github.com/dmlc/xgboost/tree/master/doc/"
 
-os.environ['XGBOOST_BUILD_DOC'] = '1'
+os.environ["XGBOOST_BUILD_DOC"] = "1"
 # Version information.
-import xgboost                  # NOQA
+import xgboost  # NOQA
+
 version = xgboost.__version__
 release = xgboost.__version__
 
@@ -91,9 +93,9 @@ extensions = [
 
 sphinx_gallery_conf = {
     # path to your example scripts
-    "examples_dirs": ["../demo/guide-python", "../demo/dask"],
+    "examples_dirs": ["../demo/guide-python", "../demo/dask", "../demo/aft_survival"],
     # path to where to save gallery generated output
-    "gallery_dirs": ["python/examples", "python/dask-examples"],
+    "gallery_dirs": ["python/examples", "python/dask-examples", "python/survival-examples"],
     "matplotlib_animations": True,
 }
 
@@ -105,7 +107,10 @@ plot_html_show_source_link = False
 plot_html_show_formats = False
 
 # Breathe extension variables
-breathe_projects = {"xgboost": "doxyxml/"}
+DOX_DIR = "doxygen"
+breathe_projects = {
+    "xgboost": os.path.join(PROJECT_ROOT, DOX_DIR, "doc_doxygen/xml")
+}
 breathe_default_project = "xgboost"
 
 # Add any paths that contain templates here, relative to this directory.
@@ -126,7 +131,7 @@ master_doc = 'index'
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
 
 autoclass_content = 'both'
 
@@ -204,34 +209,41 @@ latex_documents = [
 ]
 
 intersphinx_mapping = {
-    "python": ("https://docs.python.org/3.6", None),
+    "python": ("https://docs.python.org/3.8", None),
     "numpy": ("https://docs.scipy.org/doc/numpy/", None),
     "scipy": ("https://docs.scipy.org/doc/scipy/reference/", None),
-    "pandas": ("http://pandas-docs.github.io/pandas-docs-travis/", None),
+    "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
     "sklearn": ("https://scikit-learn.org/stable", None),
     "dask": ("https://docs.dask.org/en/stable/", None),
     "distributed": ("https://distributed.dask.org/en/stable/", None),
+    "pyspark": ("https://spark.apache.org/docs/latest/api/python/", None),
 }
 
 
 # hook for doxygen
-def run_doxygen(folder):
+def run_doxygen():
     """Run the doxygen make command in the designated folder."""
+    curdir = os.path.normpath(os.path.abspath(os.path.curdir))
     try:
-        retcode = subprocess.call("cd %s; make doxygen" % folder, shell=True)
-        if retcode < 0:
-            sys.stderr.write("doxygen terminated by signal %s" % (-retcode))
+        os.chdir(PROJECT_ROOT)
+        if not os.path.exists(DOX_DIR):
+            os.mkdir(DOX_DIR)
+        os.chdir(os.path.join(PROJECT_ROOT, DOX_DIR))
+        subprocess.check_call(["cmake", "..", "-DBUILD_C_DOC=ON", "-GNinja"])
+        subprocess.check_call(["ninja", "doc_doxygen"])
     except OSError as e:
         sys.stderr.write("doxygen execution failed: %s" % e)
+    finally:
+        os.chdir(curdir)
 
 
 def generate_doxygen_xml(app):
     """Run the doxygen make commands if we're on the ReadTheDocs server"""
     read_the_docs_build = os.environ.get('READTHEDOCS', None) == 'True'
     if read_the_docs_build:
-        run_doxygen('..')
+        run_doxygen()
 
 
-# app.add_stylesheet() is deprecated. Use app.add_css_file()
 def setup(app):
     app.add_css_file('custom.css')
+    app.connect("builder-inited", generate_doxygen_xml)

doc/contrib/ci.rst

@@ -37,3 +37,128 @@ machine in GitHub Actions, cross-compilation is needed; ``cibuildwheel`` takes c
 task of cross-compiling a Python wheel. (Note that ``cibuildwheel`` will call
 ``setup.py bdist_wheel``. Since XGBoost has a native library component, ``setup.py`` contains
 a glue code to call CMake and a C++ compiler to build the native library on the fly.)
+
+*********************************************************
+Reproduce CI testing environments using Docker containers
+*********************************************************
+In our CI pipelines, we use Docker containers extensively to package many software packages together.
+You can reproduce the same testing environment as the CI pipelines by running Docker locally.
+
+=============
+Prerequisites
+=============
+1. Install Docker: https://docs.docker.com/engine/install/ubuntu/
+2. Install NVIDIA Docker runtime: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#installing-on-ubuntu-and-debian
+   The runtime lets you access NVIDIA GPUs inside a Docker container.
+
+==============================================
+Building and Running Docker containers locally
+==============================================
+For your convenience, we provide the wrapper script ``tests/ci_build/ci_build.sh``. You can use it as follows:
+
+.. code-block:: bash
+
+  tests/ci_build/ci_build.sh <CONTAINER_TYPE> <DOCKER_BINARY> --build-arg <BUILD_ARG> \
+    <COMMAND> ...
+
+where:
+
+* ``<CONTAINER_TYPE>`` is the identifier for the container. The wrapper script will use the
+  container definition (Dockerfile) located at ``tests/ci_build/Dockerfile.<CONTAINER_TYPE>``.
+  For example, setting the container type to ``gpu`` will cause the script to load the Dockerfile
+  ``tests/ci_build/Dockerfile.gpu``.
+* ``<DOCKER_BINARY>`` must be either ``docker`` or ``nvidia-docker``. Choose ``nvidia-docker``
+  as long as you need to run any GPU code.
+* ``<BUILD_ARG>`` is a build argument to be passed to Docker. Must be of form ``VAR=VALUE``.
+  Example: ``--build-arg CUDA_VERSION_ARG=11.0``. You can pass multiple ``--build-arg``.
+* ``<COMMAND>`` is the command to run inside the Docker container. This can be more than one argument.
+  Example: ``tests/ci_build/build_via_cmake.sh -DUSE_CUDA=ON -DUSE_NCCL=ON``.
+
+Optionally, you can set the environment variable ``CI_DOCKER_EXTRA_PARAMS_INIT`` to pass extra
+arguments to Docker. For example:
+
+.. code-block:: bash
+
+  # Allocate extra space in /dev/shm to enable NCCL
+  export CI_DOCKER_EXTRA_PARAMS_INIT='--shm-size=4g'
+  # Run multi-GPU test suite
+  tests/ci_build/ci_build.sh gpu nvidia-docker --build-arg CUDA_VERSION_ARG=11.0 \
+    tests/ci_build/test_python.sh mgpu
+
+To pass multiple extra arguments:
+
+.. code-block:: bash
+
+  export CI_DOCKER_EXTRA_PARAMS_INIT='-e VAR1=VAL1 -e VAR2=VAL2 -e VAR3=VAL3'
+
+********************************************
+Update pipeline definitions for BuildKite CI
+********************************************
+
+`BuildKite <https://buildkite.com/home>`_ is a SaaS (Software as a Service) platform that orchestrates
+cloud machines to host CI pipelines. The BuildKite platform allows us to define CI pipelines as a
+declarative YAML file.
+
+The pipeline definitions are found in ``tests/buildkite/``:
+
+* ``tests/buildkite/pipeline-win64.yml``: This pipeline builds and tests XGBoost for the Windows platform.
+* ``tests/buildkite/pipeline-mgpu.yml``: This pipeline builds and tests XGBoost with access to multiple
+  NVIDIA GPUs.
+* ``tests/buildkite/pipeline.yml``: This pipeline builds and tests XGBoost with access to a single
+  NVIDIA GPU. Most tests are located here.
+
+****************************************
+Managing Elastic CI Stack with BuildKite
+****************************************
+
+BuildKite allows us to define cloud resources in
+a declarative fashion. Every configuration step is now documented explicitly as code.
+
+**Prerequisite**: You should have some knowledge of `CloudFormation <https://aws.amazon.com/cloudformation/>`_.
+CloudFormation lets us define a stack of cloud resources (EC2 machines, Lambda functions, S3 etc) using
+a single YAML file.
+
+**Prerequisite**: Gain access to the XGBoost project's AWS account (``admin@xgboost-ci.net``), and then
+set up a credential pair in order to provision resources on AWS. See
+`Creating an IAM user in your AWS account <https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html>`_.
+
+* Option 1. Give full admin privileges to your IAM user. This is the simplest option.
+* Option 2. Give limited set of permissions to your IAM user, to reduce the possibility of messing up other resources.
+  For this, use the script ``tests/buildkite/infrastructure/service-user/create_service_user.py``.
+
+=====================
+Worker Image Pipeline
+=====================
+Building images for worker machines used to be a chore: you'd provision an EC2 machine, SSH into it, and
+manually install the necessary packages. This process is not only laborous but also error-prone. You may
+forget to install a package or change a system configuration.
+
+No more. Now we have an automated pipeline for building images for worker machines.
+
+* Run ``tests/buildkite/infrastructure/worker-image-pipeline/create_worker_image_pipelines.py`` in order to provision
+  CloudFormation stacks named ``buildkite-linux-amd64-gpu-worker`` and ``buildkite-windows-gpu-worker``. They are
+  pipelines that create AMIs (Amazon Machine Images) for Linux and Windows workers, respectively.
+* Navigate to the CloudFormation web console to verify that the image builder pipelines have been provisioned. It may
+  take some time.
+* Once they pipelines have been fully provisioned, run the script
+  ``tests/buildkite/infrastructure/worker-image-pipeline/run_pipelines.py`` to execute the pipelines. New AMIs will be
+  uploaded to the EC2 service. You can locate them in the EC2 console.
+* Make sure to modify ``tests/buildkite/infrastructure/aws-stack-creator/metadata.py`` to use the correct AMI IDs.
+  (For ``linux-amd64-cpu`` and ``linux-arm64-cpu``, use the AMIs provided by BuildKite. Consult the ``AWSRegion2AMI``
+  section of https://s3.amazonaws.com/buildkite-aws-stack/latest/aws-stack.yml.)
+
+======================
+EC2 Autoscaling Groups
+======================
+In EC2, you can create auto-scaling groups, where you can dynamically adjust the number of worker instances according to
+workload. When a pull request is submitted, the following steps take place:
+
+1. GitHub sends a signal to the registered webhook, which connects to the BuildKite server.
+2. BuildKite sends a signal to a `Lambda <https://aws.amazon.com/lambda/>`_ function named ``Autoscaling``.
+3. The Lambda function sends a signal to the auto-scaling group. The group scales up and adds additional worker instances.
+4. New worker instances run the test jobs. Test results are reported back to BuildKite.
+5. When the test jobs complete, BuildKite sends a signal to ``Autoscaling``, which in turn requests the autoscaling group
+   to scale down. Idle worker instances are shut down.
+
+To set up the auto-scaling group, run the script ``tests/buildkite/infrastructure/aws-stack-creator/create_stack.py``.
+Check the CloudFormation web console to verify successful provision of auto-scaling groups.

doc/contrib/docs.rst

@@ -11,7 +11,7 @@ Documentation and Examples
 *********
 Documents
 *********
-* Documentation is built using `Sphinx <http://www.sphinx-doc.org/en/master/>`_.
+* Python and C documentation is built using `Sphinx <http://www.sphinx-doc.org/en/master/>`_.
 * Each document is written in `reStructuredText <http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html>`_.
 * You can build document locally to see the effect, by running
 

doc/contrib/donate.rst

@@ -13,9 +13,9 @@ DMLC/XGBoost has grown from a research project incubated in academia to one of t
 
 A robust and efficient **continuous integration (CI)** infrastructure is one of the most critical solutions to address the above challenge. A CI service will monitor an open-source repository and run a suite of integration tests for every incoming contribution. This way, the CI ensures that every proposed change in the codebase is compatible with existing functionalities. Furthermore, XGBoost can enable more thorough tests with a powerful CI infrastructure to cover cases which are closer to the production environment.
 
-There are several CI services available free to open source projects, such as Travis CI and AppVeyor. The XGBoost project already utilizes Travis and AppVeyor. However, the XGBoost project has needs that these free services do not adequately address. In particular, the limited usage quota of resources such as CPU and memory leaves XGBoost developers unable to bring "too-intensive" tests. In addition, they do not offer test machines with GPUs for testing XGBoost-GPU code base which has been attracting more and more interest across many organizations. Consequently, the XGBoost project self-hosts a cloud server with Jenkins software installed: https://xgboost-ci.net/.
+There are several CI services available free to open source projects, such as Travis CI and AppVeyor. The XGBoost project already utilizes GitHub Actions. However, the XGBoost project has needs that these free services do not adequately address. In particular, the limited usage quota of resources such as CPU and memory leaves XGBoost developers unable to bring "too-intensive" tests. In addition, they do not offer test machines with GPUs for testing XGBoost-GPU code base which has been attracting more and more interest across many organizations. Consequently, the XGBoost project uses a cloud-hosted test farm. We use `BuildKite <https://buildkite.com/xgboost>`_ to organize CI pipelines.
 
-The self-hosted Jenkins CI server has recurring operating expenses. It utilizes a leading cloud provider (AWS) to accommodate variable workload. The master node serving the web interface is available 24/7, to accommodate contributions from people around the globe. In addition, the master node launches slave nodes on demand, to run the test suite on incoming contributions. To save cost, the slave nodes are terminated when they are no longer needed.
+The cloud-hosted test farm has recurring operating expenses. It utilizes a leading cloud provider (AWS) to accommodate variable workload. BuildKite launches worker machines on AWS on demand, to run the test suite on incoming contributions. To save cost, the worker machines are terminated when they are no longer needed.
 
 To help defray the hosting cost, the XGBoost project seeks donations from third parties.
 
@@ -29,14 +29,14 @@ The Project Management Committee (PMC) of the XGBoost project appointed `Open So
 
 All expenses incurred for hosting CI will be submitted to the fiscal host with receipts. Only the expenses in the following categories will be approved for reimbursement:
 
-* Cloud exprenses for the Jenkins CI server (https://xgboost-ci.net)
+* Cloud exprenses for the cloud test farm (https://buildkite.com/xgboost)
 * Cost of domain https://xgboost-ci.net
-* Meetup.com account for XGBoost project
+* Monthly cost of using BuildKite
 * Hosting cost of the User Forum (https://discuss.xgboost.ai)
 
-Administration of Jenkins CI server
------------------------------------
-The PMC shall appoint committer(s) to administer the Jenkins CI server on their behalf. The current administrators are as follows:
+Administration of cloud CI infrastructure
+-----------------------------------------
+The PMC shall appoint committer(s) to administer the cloud CI infrastructure on their behalf. The current administrators are as follows:
 
 * Primary administrator: `Hyunsu Cho <https://github.com/hcho3>`_
 * Secondary administrator: `Jiaming Yuan <https://github.com/trivialfis>`_

doc/get_started.rst

@@ -19,16 +19,18 @@ Python
 
 .. code-block:: python
 
-  import xgboost as xgb
-  # read in data
-  dtrain = xgb.DMatrix('demo/data/agaricus.txt.train')
-  dtest = xgb.DMatrix('demo/data/agaricus.txt.test')
-  # specify parameters via map
-  param = {'max_depth':2, 'eta':1, 'objective':'binary:logistic' }
-  num_round = 2
-  bst = xgb.train(param, dtrain, num_round)
-  # make prediction
-  preds = bst.predict(dtest)
+  from xgboost import XGBClassifier
+  # read data
+  from sklearn.datasets import load_iris
+  from sklearn.model_selection import train_test_split
+  data = load_iris()
+  X_train, X_test, y_train, y_test = train_test_split(data['data'], data['target'], test_size=.2)
+  # create model instance
+  bst = XGBClassifier(n_estimators=2, max_depth=2, learning_rate=1, objective='binary:logistic')
+  # fit model
+  bst.fit(X_train, y_train)
+  # make predictions
+  preds = bst.predict(X_test)
 
 ***
 R

doc/gpu/index.rst

@@ -34,38 +34,8 @@ Supported parameters
 .. |tick| unicode:: U+2714
 .. |cross| unicode:: U+2718
 
-+--------------------------------+--------------+
-| parameter                      | ``gpu_hist`` |
-+================================+==============+
-| ``subsample``                  | |tick|       |
-+--------------------------------+--------------+
-| ``sampling_method``            | |tick|       |
-+--------------------------------+--------------+
-| ``colsample_bytree``           | |tick|       |
-+--------------------------------+--------------+
-| ``colsample_bylevel``          | |tick|       |
-+--------------------------------+--------------+
-| ``max_bin``                    | |tick|       |
-+--------------------------------+--------------+
-| ``gamma``                      | |tick|       |
-+--------------------------------+--------------+
-| ``gpu_id``                     | |tick|       |
-+--------------------------------+--------------+
-| ``predictor``                  | |tick|       |
-+--------------------------------+--------------+
-| ``grow_policy``                | |tick|       |
-+--------------------------------+--------------+
-| ``monotone_constraints``       | |tick|       |
-+--------------------------------+--------------+
-| ``interaction_constraints``    | |tick|       |
-+--------------------------------+--------------+
-| ``single_precision_histogram`` | |tick|       |
-+--------------------------------+--------------+
-
 GPU accelerated prediction is enabled by default for the above mentioned ``tree_method`` parameters but can be switched to CPU prediction by setting ``predictor`` to ``cpu_predictor``. This could be useful if you want to conserve GPU memory. Likewise when using CPU algorithms, GPU accelerated prediction can be enabled by setting ``predictor`` to ``gpu_predictor``.
 
-The experimental parameter ``single_precision_histogram`` can be set to True to enable building histograms using single precision. This may improve speed, in particular on older architectures.
-
 The device ordinal (which GPU to use if you have many of them) can be selected using the
 ``gpu_id`` parameter, which defaults to 0 (the first device reported by CUDA runtime).
 

doc/install.rst

@@ -64,6 +64,11 @@ Conda should be able to detect the existence of a GPU on your machine and instal
 
 Visit the `Miniconda website <https://docs.conda.io/en/latest/miniconda.html>`_ to obtain Conda.
 
+.. note:: ``py-xgboost-gpu`` not available on Windows.
+
+   The ``py-xgboost-gpu`` is currently not available on Windows. If you are using Windows,
+   please use ``pip`` to install XGBoost with GPU support.
+
 R
 -
 
@@ -101,7 +106,7 @@ R
 JVM
 ---
 
-You can use XGBoost4J in your Java/Scala application by adding XGBoost4J as a dependency:
+* XGBoost4j/XGBoost4j-Spark
 
 .. code-block:: xml
   :caption: Maven
@@ -134,6 +139,39 @@ You can use XGBoost4J in your Java/Scala application by adding XGBoost4J as a de
     "ml.dmlc" %% "xgboost4j-spark" % "latest_version_num"
   )
 
+* XGBoost4j-GPU/XGBoost4j-Spark-GPU
+
+.. code-block:: xml
+  :caption: Maven
+
+  <properties>
+    ...
+    <!-- Specify Scala version in package name -->
+    <scala.binary.version>2.12</scala.binary.version>
+  </properties>
+
+  <dependencies>
+    ...
+    <dependency>
+        <groupId>ml.dmlc</groupId>
+        <artifactId>xgboost4j-gpu_${scala.binary.version}</artifactId>
+        <version>latest_version_num</version>
+    </dependency>
+    <dependency>
+        <groupId>ml.dmlc</groupId>
+        <artifactId>xgboost4j-spark-gpu_${scala.binary.version}</artifactId>
+        <version>latest_version_num</version>
+    </dependency>
+  </dependencies>
+
+.. code-block:: scala
+  :caption: sbt
+
+  libraryDependencies ++= Seq(
+    "ml.dmlc" %% "xgboost4j-gpu" % "latest_version_num",
+    "ml.dmlc" %% "xgboost4j-spark-gpu" % "latest_version_num"
+  )
+
 This will check out the latest stable version from the Maven Central.
 
 For the latest release version number, please check `release page <https://github.com/dmlc/xgboost/releases>`_.
@@ -185,7 +223,7 @@ and Windows.) Download it and run the following commands:
 JVM
 ---
 
-First add the following Maven repository hosted by the XGBoost project:
+* XGBoost4j/XGBoost4j-Spark
 
 .. code-block:: xml
   :caption: Maven
@@ -234,6 +272,40 @@ Then add XGBoost4J as a dependency:
     "ml.dmlc" %% "xgboost4j-spark" % "latest_version_num-SNAPSHOT"
   )
 
+* XGBoost4j-GPU/XGBoost4j-Spark-GPU
+
+.. code-block:: xml
+  :caption: maven
+
+  <properties>
+    ...
+    <!-- Specify Scala version in package name -->
+    <scala.binary.version>2.12</scala.binary.version>
+  </properties>
+
+  <dependencies>
+    ...
+    <dependency>
+        <groupId>ml.dmlc</groupId>
+        <artifactId>xgboost4j-gpu_${scala.binary.version}</artifactId>
+        <version>latest_version_num-SNAPSHOT</version>
+    </dependency>
+    <dependency>
+        <groupId>ml.dmlc</groupId>
+        <artifactId>xgboost4j-spark-gpu_${scala.binary.version}</artifactId>
+        <version>latest_version_num-SNAPSHOT</version>
+    </dependency>
+  </dependencies>
+
+.. code-block:: scala
+  :caption: sbt
+
+  libraryDependencies ++= Seq(
+    "ml.dmlc" %% "xgboost4j-gpu" % "latest_version_num-SNAPSHOT",
+    "ml.dmlc" %% "xgboost4j-spark-gpu" % "latest_version_num-SNAPSHOT"
+  )
+
+
 Look up the ``version`` field in `pom.xml <https://github.com/dmlc/xgboost/blob/master/jvm-packages/pom.xml>`_ to get the correct version number.
 
 The SNAPSHOT JARs are hosted by the XGBoost project. Every commit in the ``master`` branch will automatically trigger generation of a new SNAPSHOT JAR. You can control how often Maven should upgrade your SNAPSHOT installation by specifying ``updatePolicy``. See `here <http://maven.apache.org/pom.html#Repositories>`_ for details.

doc/jvm/index.rst

@@ -35,6 +35,7 @@ Contents
 
   java_intro
   XGBoost4J-Spark Tutorial <xgboost4j_spark_tutorial>
+  XGBoost4J-Spark-GPU Tutorial <xgboost4j_spark_gpu_tutorial>
   Code Examples <https://github.com/dmlc/xgboost/tree/master/jvm-packages/xgboost4j-example>
   XGBoost4J Java API <javadocs/index>
   XGBoost4J Scala API <scaladocs/xgboost4j/index>

doc/jvm/xgboost4j_spark_gpu_tutorial.rst

@@ -0,0 +1,246 @@
+#############################################
+XGBoost4J-Spark-GPU Tutorial (version 1.6.1+)
+#############################################
+
+**XGBoost4J-Spark-GPU** is an open source library aiming to accelerate distributed XGBoost training on Apache Spark cluster from
+end to end with GPUs by leveraging the `RAPIDS Accelerator for Apache Spark <https://nvidia.github.io/spark-rapids/>`_ product.
+
+This tutorial will show you how to use **XGBoost4J-Spark-GPU**.
+
+.. contents::
+  :backlinks: none
+  :local:
+
+************************************************
+Build an ML Application with XGBoost4J-Spark-GPU
+************************************************
+
+Add XGBoost to Your Project
+===========================
+
+Before we go into the tour of how to use XGBoost4J-Spark-GPU, you should first consult
+:ref:`Installation from Maven repository <install_jvm_packages>` in order to add XGBoost4J-Spark-GPU as
+a dependency for your project. We provide both stable releases and snapshots.
+
+Data Preparation
+================
+
+In this section, we use the `Iris <https://archive.ics.uci.edu/ml/datasets/iris>`_ dataset as an example to
+showcase how we use Apache Spark to transform a raw dataset and make it fit the data interface of XGBoost.
+
+The Iris dataset is shipped in CSV format. Each instance contains 4 features, "sepal length", "sepal width",
+"petal length" and "petal width". In addition, it contains the "class" column, which is essentially the
+label with three possible values: "Iris Setosa", "Iris Versicolour" and "Iris Virginica".
+
+Read Dataset with Spark's Built-In Reader
+-----------------------------------------
+
+.. code-block:: scala
+
+  import org.apache.spark.sql.SparkSession
+  import org.apache.spark.sql.types.{DoubleType, StringType, StructField, StructType}
+
+  val spark = SparkSession.builder().getOrCreate()
+
+  val labelName = "class"
+  val schema = new StructType(Array(
+      StructField("sepal length", DoubleType, true),
+      StructField("sepal width", DoubleType, true),
+      StructField("petal length", DoubleType, true),
+      StructField("petal width", DoubleType, true),
+      StructField(labelName, StringType, true)))
+
+  val xgbInput = spark.read.option("header", "false")
+      .schema(schema)
+      .csv(dataPath)
+
+In the first line, we create an instance of a `SparkSession <https://spark.apache.org/docs/latest/sql-getting-started.html#starting-point-sparksession>`_
+which is the entry point of any Spark application working with DataFrames. The ``schema`` variable
+defines the schema of the DataFrame wrapping Iris data. With this explicitly set schema, we
+can define the column names as well as their types; otherwise the column names would be
+the default ones derived by Spark, such as ``_col0``, etc. Finally, we can use Spark's
+built-in CSV reader to load the Iris CSV file as a DataFrame named ``xgbInput``.
+
+Apache Spark also contains many built-in readers for other formats such as ORC, Parquet, Avro, JSON.
+
+
+Transform Raw Iris Dataset
+--------------------------
+
+To make the Iris dataset recognizable to XGBoost, we need to encode the String-typed
+label, i.e. "class", to the Double-typed label.
+
+One way to convert the String-typed label to Double is to use Spark's built-in feature transformer
+`StringIndexer <https://spark.apache.org/docs/2.3.1/api/scala/index.html#org.apache.spark.ml.feature.StringIndexer>`_.
+But this feature is not accelerated in RAPIDS Accelerator, which means it will fall back
+to CPU. Instead, we use an alternative way to achieve the same goal with the following code:
+
+.. code-block:: scala
+
+  import org.apache.spark.sql.expressions.Window
+  import org.apache.spark.sql.functions._
+
+  val spec = Window.orderBy(labelName)
+  val Array(train, test) = xgbInput
+      .withColumn("tmpClassName", dense_rank().over(spec) - 1)
+      .drop(labelName)
+      .withColumnRenamed("tmpClassName", labelName)
+      .randomSplit(Array(0.7, 0.3), seed = 1)
+
+  train.show(5)
+
+.. code-block:: none
+
+	+------------+-----------+------------+-----------+-----+
+	|sepal length|sepal width|petal length|petal width|class|
+	+------------+-----------+------------+-----------+-----+
+	|         4.3|        3.0|         1.1|        0.1|    0|
+	|         4.4|        2.9|         1.4|        0.2|    0|
+	|         4.4|        3.0|         1.3|        0.2|    0|
+	|         4.4|        3.2|         1.3|        0.2|    0|
+	|         4.6|        3.2|         1.4|        0.2|    0|
+	+------------+-----------+------------+-----------+-----+
+
+
+With window operations, we have mapped the string column of labels to label indices.
+
+Training
+========
+
+The GPU version of XGBoost-Spark supports both regression and classification
+models. Although we use the Iris dataset in this tutorial to show how we use
+``XGBoost/XGBoost4J-Spark-GPU`` to resolve a multi-classes classification problem, the
+usage in Regression is very similar to classification.
+
+To train a XGBoost model for classification, we need to claim a XGBoostClassifier first:
+
+.. code-block:: scala
+
+  import ml.dmlc.xgboost4j.scala.spark.XGBoostClassifier
+  val xgbParam = Map(
+      "objective" -> "multi:softprob",
+      "num_class" -> 3,
+      "num_round" -> 100,
+      "tree_method" -> "gpu_hist",
+      "num_workers" -> 1)
+
+  val featuresNames = schema.fieldNames.filter(name => name != labelName)
+
+  val xgbClassifier = new XGBoostClassifier(xgbParam)
+      .setFeaturesCol(featuresNames)
+      .setLabelCol(labelName)
+
+The available parameters for training a XGBoost model can be found in :doc:`here </parameter>`.
+Similar to the XGBoost4J-Spark package, in addition to the default set of parameters,
+XGBoost4J-Spark-GPU also supports the camel-case variant of these parameters to be
+consistent with Spark's MLlib naming convention.
+
+Specifically, each parameter in :doc:`this page </parameter>` has its equivalent form in
+XGBoost4J-Spark-GPU with camel case. For example, to set ``max_depth`` for each tree, you can pass
+parameter just like what we did in the above code snippet (as ``max_depth`` wrapped in a Map), or
+you can do it through setters in XGBoostClassifer:
+
+.. code-block:: scala
+
+  val xgbClassifier = new XGBoostClassifier(xgbParam)
+      .setFeaturesCol(featuresNames)
+      .setLabelCol(labelName)
+  xgbClassifier.setMaxDepth(2)
+
+.. note::
+
+  In contrast with XGBoost4j-Spark which accepts both a feature column with VectorUDT type and
+  an array of feature column names, XGBoost4j-Spark-GPU only accepts an array of feature
+  column names by ``setFeaturesCol(value: Array[String])``.
+
+After setting XGBoostClassifier parameters and feature/label columns, we can build a
+transformer, XGBoostClassificationModel by fitting XGBoostClassifier with the input
+DataFrame. This ``fit`` operation is essentially the training process and the generated
+model can then be used in other tasks like prediction.
+
+.. code-block:: scala
+
+  val xgbClassificationModel = xgbClassifier.fit(train)
+
+Prediction
+==========
+
+When we get a model, either a XGBoostClassificationModel or a XGBoostRegressionModel, it takes a DataFrame as an input,
+reads the column containing feature vectors, predicts for each feature vector, and outputs a new DataFrame
+with the following columns by default:
+
+* XGBoostClassificationModel will output margins (``rawPredictionCol``), probabilities(``probabilityCol``) and the eventual prediction labels (``predictionCol``) for each possible label.
+* XGBoostRegressionModel will output prediction a label(``predictionCol``).
+
+.. code-block:: scala
+
+  val xgbClassificationModel = xgbClassifier.fit(train)
+  val results = xgbClassificationModel.transform(test)
+  results.show()
+
+With the above code snippet, we get a DataFrame as result, which contains the margin, probability for each class,
+and the prediction for each instance.
+
+.. code-block:: none
+
+	+------------+-----------+------------------+-------------------+-----+--------------------+--------------------+----------+
+	|sepal length|sepal width|      petal length|        petal width|class|       rawPrediction|         probability|prediction|
+	+------------+-----------+------------------+-------------------+-----+--------------------+--------------------+----------+
+	|         4.5|        2.3|               1.3|0.30000000000000004|    0|[3.16666603088378...|[0.98853939771652...|       0.0|
+	|         4.6|        3.1|               1.5|                0.2|    0|[3.25857257843017...|[0.98969423770904...|       0.0|
+	|         4.8|        3.1|               1.6|                0.2|    0|[3.25857257843017...|[0.98969423770904...|       0.0|
+	|         4.8|        3.4|               1.6|                0.2|    0|[3.25857257843017...|[0.98969423770904...|       0.0|
+	|         4.8|        3.4|1.9000000000000001|                0.2|    0|[3.25857257843017...|[0.98969423770904...|       0.0|
+	|         4.9|        2.4|               3.3|                1.0|    1|[-2.1498908996582...|[0.00596602633595...|       1.0|
+	|         4.9|        2.5|               4.5|                1.7|    2|[-2.1498908996582...|[0.00596602633595...|       1.0|
+	|         5.0|        3.5|               1.3|0.30000000000000004|    0|[3.25857257843017...|[0.98969423770904...|       0.0|
+	|         5.1|        2.5|               3.0|                1.1|    1|[3.16666603088378...|[0.98853939771652...|       0.0|
+	|         5.1|        3.3|               1.7|                0.5|    0|[3.25857257843017...|[0.98969423770904...|       0.0|
+	|         5.1|        3.5|               1.4|                0.2|    0|[3.25857257843017...|[0.98969423770904...|       0.0|
+	|         5.1|        3.8|               1.6|                0.2|    0|[3.25857257843017...|[0.98969423770904...|       0.0|
+	|         5.2|        3.4|               1.4|                0.2|    0|[3.25857257843017...|[0.98969423770904...|       0.0|
+	|         5.2|        3.5|               1.5|                0.2|    0|[3.25857257843017...|[0.98969423770904...|       0.0|
+	|         5.2|        4.1|               1.5|                0.1|    0|[3.25857257843017...|[0.98969423770904...|       0.0|
+	|         5.4|        3.9|               1.7|                0.4|    0|[3.25857257843017...|[0.98969423770904...|       0.0|
+	|         5.5|        2.4|               3.8|                1.1|    1|[-2.1498908996582...|[0.00596602633595...|       1.0|
+	|         5.5|        4.2|               1.4|                0.2|    0|[3.25857257843017...|[0.98969423770904...|       0.0|
+	|         5.7|        2.5|               5.0|                2.0|    2|[-2.1498908996582...|[0.00280966912396...|       2.0|
+	|         5.7|        3.0|               4.2|                1.2|    1|[-2.1498908996582...|[0.00643939292058...|       1.0|
+	+------------+-----------+------------------+-------------------+-----+--------------------+--------------------+----------+
+
+**********************
+Submit the application
+**********************
+
+Here’s an example to submit an end-to-end XGBoost-4j-Spark-GPU Spark application to an
+Apache Spark Standalone cluster, assuming the application main class is Iris and the
+application jar is iris-1.0.0.jar
+
+.. code-block:: bash
+
+  cudf_version=22.02.0
+  rapids_version=22.02.0
+  xgboost_version=1.6.1
+  main_class=Iris
+  app_jar=iris-1.0.0.jar
+
+  spark-submit \
+    --master $master \
+    --packages ai.rapids:cudf:${cudf_version},com.nvidia:rapids-4-spark_2.12:${rapids_version},ml.dmlc:xgboost4j-gpu_2.12:${xgboost_version},ml.dmlc:xgboost4j-spark-gpu_2.12:${xgboost_version} \
+    --conf spark.executor.cores=12 \
+    --conf spark.task.cpus=1 \
+    --conf spark.executor.resource.gpu.amount=1 \
+    --conf spark.task.resource.gpu.amount=0.08 \
+    --conf spark.rapids.sql.csv.read.double.enabled=true \
+    --conf spark.rapids.sql.hasNans=false \
+    --conf spark.plugins=com.nvidia.spark.SQLPlugin \
+    --class ${main_class} \
+     ${app_jar}
+
+* First, we need to specify the ``RAPIDS Accelerator, cudf, xgboost4j-gpu, xgboost4j-spark-gpu`` packages by ``--packages``
+* Second, ``RAPIDS Accelerator`` is a Spark plugin, so we need to configure it by specifying ``spark.plugins=com.nvidia.spark.SQLPlugin``
+
+For details about other ``RAPIDS Accelerator`` other configurations, please refer to the `configuration <https://nvidia.github.io/spark-rapids/docs/configs.html>`_.
+
+For ``RAPIDS Accelerator Frequently Asked Questions``, please refer to the
+`frequently-asked-questions <https://nvidia.github.io/spark-rapids/docs/FAQ.html#frequently-asked-questions>`_.

doc/jvm/xgboost4j_spark_tutorial.rst

@@ -16,12 +16,6 @@ This tutorial is to cover the end-to-end process to build a machine learning pip
 * Building a Machine Learning Pipeline with XGBoost4J-Spark
 * Running XGBoost4J-Spark in Production
 
-.. note::
-
-  **SparkContext will be stopped by default when XGBoost training task fails**.
-
-  XGBoost4J-Spark 1.2.0+ exposes a parameter **kill_spark_context_on_worker_failure**. Set **kill_spark_context_on_worker_failure** to **false** so that the SparkContext will not be stopping on training failure. Instead of stopping the SparkContext, XGBoost4J-Spark will throw an exception instead. Users who want to re-use the SparkContext should wrap the training code in a try-catch block.
-
 .. contents::
   :backlinks: none
   :local:
@@ -127,6 +121,11 @@ Now, we have a DataFrame containing only two columns, "features" which contains
 "sepal length", "sepal width", "petal length" and "petal width" and "classIndex" which has Double-typed
 labels. A DataFrame like this (containing vector-represented features and numeric labels) can be fed to XGBoost4J-Spark's training engine directly.
 
+.. note::
+
+  There is no need to assemble feature columns from version 1.6.1+. Instead, users can specify an array of
+  feature column names by ``setFeaturesCol(value: Array[String])`` and XGBoost4j-Spark will do it.
+
 Dealing with missing values
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -346,11 +345,37 @@ and then loading the model in another session:
   val xgbClassificationModel2 = XGBoostClassificationModel.load(xgbClassificationModelPath)
   xgbClassificationModel2.transform(xgbInput)
 
+.. note::
+
+  Besides dumping the model to raw format, users are able to dump the model to be json or ubj format from ``version 1.7.0+``.
+
+  .. code-block:: scala
+
+    val xgbClassificationModelPath = "/tmp/xgbClassificationModel"
+    xgbClassificationModel.write.overwrite().option("format", "json").save(xgbClassificationModelPath)
+
+
 With regards to ML pipeline save and load, please refer the next section.
 
 Interact with Other Bindings of XGBoost
 ---------------------------------------
-After we train a model with XGBoost4j-Spark on massive dataset, sometimes we want to do model serving in single machine or integrate it with other single node libraries for further processing. XGBoost4j-Spark supports export model to local by:
+After we train a model with XGBoost4j-Spark on massive dataset, sometimes we want to do model serving
+in single machine or integrate it with other single node libraries for further processing.
+
+After saving the model, we can load this model with single node Python XGBoost directly from ``version 1.7.0+``.
+
+.. code-block:: scala
+
+  val xgbClassificationModelPath = "/tmp/xgbClassificationModel"
+  xgbClassificationModel.write.overwrite().save(xgbClassificationModelPath)
+
+.. code-block:: python
+
+  import xgboost as xgb
+  bst = xgb.Booster({'nthread': 4})
+  bst.load_model("/tmp/xgbClassificationModel/data/XGBoostClassificationModel")
+
+Before ``version 1.7.0``, XGBoost4j-Spark needs to export model to local manually by:
 
 .. code-block:: scala
 

doc/model.schema

@@ -207,7 +207,7 @@
         }
       }
     },
-    "pseduo_huber_param": {
+    "pseudo_huber_param": {
       "type": "object",
       "properties": {
         "huber_slope": {
@@ -247,7 +247,7 @@
       "items": [
         {
           "type": "number",
-          "const": 1
+          "minimum": 1
         },
         {
           "type": "number",
@@ -400,7 +400,6 @@
                 "reg_loss_param"
               ]
             },
-
             {
               "type": "object",
               "properties": {
@@ -433,6 +432,14 @@
                 "tweedie_regression_param"
               ]
             },
+            {
+              "properties": {
+                "name": {
+                  "const": "reg:absoluteerror"
+                }
+              },
+              "type": "object"
+            },
             {
               "type": "object",
               "properties": {

doc/parameter.rst

@@ -151,15 +151,6 @@ Parameters for Tree Booster
     - ``hist``: Faster histogram optimized approximate greedy algorithm.
     - ``gpu_hist``: GPU implementation of ``hist`` algorithm.
 
-* ``sketch_eps`` [default=0.03]
-
-  - Only used for ``updater=grow_local_histmaker``.
-  - This roughly translates into ``O(1 / sketch_eps)`` number of bins.
-    Compared to directly select number of bins, this comes with theoretical guarantee with sketch accuracy.
-  - Usually user does not have to tune this.
-    But consider setting to a lower number for more accurate enumeration of split candidates.
-  - range: (0, 1)
-
 * ``scale_pos_weight`` [default=1]
 
   - Control the balance of positive and negative weights, useful for unbalanced classes. A typical value to consider: ``sum(negative instances) / sum(positive instances)``. See :doc:`Parameters Tuning </tutorials/param_tuning>` for more discussion. Also, see Higgs Kaggle competition demo for examples: `R <https://github.com/dmlc/xgboost/blob/master/demo/kaggle-higgs/higgs-train.R>`_, `py1 <https://github.com/dmlc/xgboost/blob/master/demo/kaggle-higgs/higgs-numpy.py>`_, `py2 <https://github.com/dmlc/xgboost/blob/master/demo/kaggle-higgs/higgs-cv.py>`_, `py3 <https://github.com/dmlc/xgboost/blob/master/demo/guide-python/cross_validation.py>`_.
@@ -170,7 +161,6 @@ Parameters for Tree Booster
 
     - ``grow_colmaker``: non-distributed column-based construction of trees.
     - ``grow_histmaker``: distributed tree construction with row-based data splitting based on global proposal of histogram counting.
-    - ``grow_local_histmaker``: based on local histogram counting.
     - ``grow_quantile_histmaker``: Grow tree using quantized histogram.
     - ``grow_gpu_hist``: Grow tree with GPU.
     - ``sync``: synchronizes trees in all distributed nodes.
@@ -235,18 +225,19 @@ Parameters for Tree Booster
     list is a group of indices of features that are allowed to interact with each other.
     See :doc:`/tutorials/feature_interaction_constraint` for more information.
 
-Additional parameters for ``hist``, ``gpu_hist`` and ``approx`` tree method
-===========================================================================
+.. _cat-param:
 
-* ``single_precision_histogram``, [default= ``false``]
+Parameters for Categorical Feature
+==================================
 
-  - Use single precision to build histograms instead of double precision.
+These parameters are only used for training with categorical data. See
+:doc:`/tutorials/categorical` for more information.
 
 * ``max_cat_to_onehot``
 
-  .. versionadded:: 1.6
+  .. versionadded:: 1.6.0
 
-  .. note:: The support for this parameter is experimental.
+  .. note:: This parameter is experimental. ``exact`` tree method is not yet supported.
 
   - A threshold for deciding whether XGBoost should use one-hot encoding based split for
     categorical data.  When number of categories is lesser than the threshold then one-hot
@@ -254,6 +245,15 @@ Additional parameters for ``hist``, ``gpu_hist`` and ``approx`` tree method
     Only relevant for regression and binary classification. Also, ``exact`` tree method is
     not supported
 
+* ``max_cat_threshold``
+
+  .. versionadded:: 1.7.0
+
+  .. note:: This parameter is experimental. ``exact`` tree method is not yet supported.
+
+  - Maximum number of categories considered for each split. Used only by partition-based
+    splits for preventing over-fitting.
+
 Additional parameters for Dart Booster (``booster=dart``)
 =========================================================
 
@@ -349,6 +349,7 @@ Specify the learning task and the corresponding learning objective. The objectiv
   - ``reg:squaredlogerror``: regression with squared log loss :math:`\frac{1}{2}[log(pred + 1) - log(label + 1)]^2`.  All input labels are required to be greater than -1.  Also, see metric ``rmsle`` for possible issue  with this objective.
   - ``reg:logistic``: logistic regression.
   - ``reg:pseudohubererror``: regression with Pseudo Huber loss, a twice differentiable alternative to absolute loss.
+  - ``reg:absoluteerror``: Regression with L1 error. When tree model is used, leaf value is refreshed after tree construction. If used in distributed training, the leaf value is calculated as the mean value from all workers, which is not guaranteed to be optimal.
   - ``binary:logistic``: logistic regression for binary classification, output probability
   - ``binary:logitraw``: logistic regression for binary classification, output score before logistic transformation
   - ``binary:hinge``: hinge loss for binary classification. This makes predictions of 0 or 1, rather than producing probabilities.
@@ -369,9 +370,11 @@ Specify the learning task and the corresponding learning objective. The objectiv
   - ``reg:gamma``: gamma regression with log-link. Output is a mean of gamma distribution. It might be useful, e.g., for modeling insurance claims severity, or for any outcome that might be `gamma-distributed <https://en.wikipedia.org/wiki/Gamma_distribution#Occurrence_and_applications>`_.
   - ``reg:tweedie``: Tweedie regression with log-link. It might be useful, e.g., for modeling total loss in insurance, or for any outcome that might be `Tweedie-distributed <https://en.wikipedia.org/wiki/Tweedie_distribution#Occurrence_and_applications>`_.
 
-* ``base_score`` [default=0.5]
+* ``base_score``
 
   - The initial prediction score of all instances, global bias
+  - The parameter is automatically estimated for selected objectives before training. To
+    disable the estimation, specify a real number argument.
   - For sufficient number of iterations, changing this value will not have too much effect.
 
 * ``eval_metric`` [default according to objective]

doc/prediction.rst

@@ -153,7 +153,7 @@ underlying booster is ``gbtree`` or ``dart``, which means as long as tree model
 prediction itself should thread safe.  But the safety is only guaranteed with prediction.
 If one tries to train a model in one thread and provide prediction at the other using the
 same model the behaviour is undefined.  This happens easier than one might expect, for
-instance we might accidientally call ``clf.set_params()`` inside a predict function:
+instance we might accidentally call ``clf.set_params()`` inside a predict function:
 
 .. code-block:: python
 

doc/python/.gitignore

@@ -1,2 +1,3 @@
 examples
-dask-examples
+dask-examples
+survival-examples

doc/python/index.rst

@@ -15,3 +15,4 @@ Contents
   model
   examples/index
   dask-examples/index
+  survival-examples/index

doc/python/python_api.rst

@@ -22,6 +22,9 @@ Core Data Structure
     :members:
     :show-inheritance:
 
+.. autoclass:: xgboost.QuantileDMatrix
+    :show-inheritance:
+
 .. autoclass:: xgboost.DeviceQuantileDMatrix
     :show-inheritance:
 
@@ -147,3 +150,29 @@ Dask API
     :members:
     :inherited-members:
     :show-inheritance:
+
+
+PySpark API
+-----------
+
+.. automodule:: xgboost.spark
+
+.. autoclass:: xgboost.spark.SparkXGBClassifier
+    :members:
+    :inherited-members:
+    :show-inheritance:
+
+.. autoclass:: xgboost.spark.SparkXGBClassifierModel
+    :members:
+    :inherited-members:
+    :show-inheritance:
+
+.. autoclass:: xgboost.spark.SparkXGBRegressor
+    :members:
+    :inherited-members:
+    :show-inheritance:
+
+.. autoclass:: xgboost.spark.SparkXGBRegressorModel
+    :members:
+    :inherited-members:
+    :show-inheritance:

doc/python/python_intro.rst

@@ -45,6 +45,7 @@ including:
 - XGBoost binary buffer file.
 - LIBSVM text format file
 - Comma-separated values (CSV) file
+- Arrow table.
 
 (See :doc:`/tutorials/input_format` for detailed description of text input format.)
 
@@ -146,7 +147,7 @@ XGBoost can use either a list of pairs or a dictionary to set :doc:`parameters <
 
   .. code-block:: python
 
-    evallist = [(dtest, 'eval'), (dtrain, 'train')]
+    evallist = [(dtrain, 'train'), (dtest, 'eval')]
 
 Training
 --------

doc/requirements.txt

@@ -1,4 +1,4 @@
-sphinx>=4.4.0
+sphinx>=5.2.1
 mock
 sphinx_rtd_theme>=1.0.0
 breathe
@@ -9,4 +9,6 @@ graphviz
 numpy
 recommonmark
 xgboost_ray
-sphinx-gallery
+sphinx-gallery
+pyspark
+cloudpickle

doc/treemethod.rst

@@ -5,12 +5,12 @@ Tree Methods
 For training boosted tree models, there are 2 parameters used for choosing algorithms,
 namely ``updater`` and ``tree_method``.  XGBoost has 4 builtin tree methods, namely
 ``exact``, ``approx``, ``hist`` and ``gpu_hist``.  Along with these tree methods, there
-are also some free standing updaters including ``grow_local_histmaker``, ``refresh``,
+are also some free standing updaters including ``refresh``,
 ``prune`` and ``sync``.  The parameter ``updater`` is more primitive than ``tree_method``
 as the latter is just a pre-configuration of the former.  The difference is mostly due to
 historical reasons that each updater requires some specific configurations and might has
 missing features.  As we are moving forward, the gap between them is becoming more and
-more irrevelant.  We will collectively document them under tree methods.
+more irrelevant.  We will collectively document them under tree methods.
 
 **************
 Exact Solution
@@ -37,27 +37,18 @@ approximated training algorithms.  These algorithms build a gradient histogram f
 node and iterate through the histogram instead of real dataset.  Here we introduce the
 implementations in XGBoost below.
 
-1. ``grow_local_histmaker`` updater: An approximation tree method described in `reference
-   paper <http://arxiv.org/abs/1603.02754>`_.  This updater is rarely used in practice so
-   it's still an updater rather than tree method.  During split finding, it first runs a
-   weighted GK sketching for data points belong to current node to find split candidates,
-   using hessian as weights.  The histogram is built upon this per-node sketch.  It's
-   faster than ``exact`` in some applications, but still slow in computation.
+1. ``approx`` tree method: An approximation tree method described in `reference paper
+   <http://arxiv.org/abs/1603.02754>`_.  It runs sketching before building each tree
+   using all the rows (rows belonging to the root). Hessian is used as weights during
+   sketch.  The algorithm can be accessed by setting ``tree_method`` to ``approx``.
 
-2. ``approx`` tree method: An approximation tree method described in `reference paper
-   <http://arxiv.org/abs/1603.02754>`_.  Different from ``grow_local_histmaker``, it runs
-   sketching before building each tree using all the rows (rows belonging to the root)
-   instead of per-node dataset.  Similar to ``grow_local_histmaker`` updater, hessian is
-   used as weights during sketch.  The algorithm can be accessed by setting
-   ``tree_method`` to ``approx``.
-
-3. ``hist`` tree method: An approximation tree method used in LightGBM with slight
+2. ``hist`` tree method: An approximation tree method used in LightGBM with slight
    differences in implementation.  It runs sketching before training using only user
    provided weights instead of hessian.  The subsequent per-node histogram is built upon
    this global sketch.  This is the fastest algorithm as it runs sketching only once.  The
    algorithm can be accessed by setting ``tree_method`` to ``hist``.
 
-4. ``gpu_hist`` tree method: The ``gpu_hist`` tree method is a GPU implementation of
+3. ``gpu_hist`` tree method: The ``gpu_hist`` tree method is a GPU implementation of
    ``hist``, with additional support for gradient based sampling.  The algorithm can be
    accessed by setting ``tree_method`` to ``gpu_hist``.
 
@@ -102,19 +93,32 @@ Other Updaters
 Removed Updaters
 ****************
 
-2 Updaters were removed during development due to maintainability.  We describe them here
-solely for the interest of documentation.  First one is distributed colmaker, which was a
-distributed version of exact tree method.  It required specialization for column based
-splitting strategy and a different prediction procedure.  As the exact tree method is slow
-by itself and scaling is even less efficient, we removed it entirely.  Second one is
-``skmaker``.  Per-node weighted sketching employed by ``grow_local_histmaker`` is slow,
-the ``skmaker`` was unmaintained and seems to be a workaround trying to eliminate the
-histogram creation step and uses sketching values directly during split evaluation.  It
-was never tested and contained some unknown bugs, we decided to remove it and focus our
-resources on more promising algorithms instead.  For accuracy, most of the time
-``approx``, ``hist`` and ``gpu_hist`` are enough with some parameters tuning, so removing
-them don't have any real practical impact.
+3 Updaters were removed during development due to maintainability.  We describe them here
+solely for the interest of documentation.
 
+1. Distributed colmaker, which was a distributed version of exact tree method.  It
+   required specialization for column based splitting strategy and a different prediction
+   procedure.  As the exact tree method is slow by itself and scaling is even less
+   efficient, we removed it entirely.
+
+2. ``skmaker``.  Per-node weighted sketching employed by ``grow_local_histmaker`` is slow,
+   the ``skmaker`` was unmaintained and seems to be a workaround trying to eliminate the
+   histogram creation step and uses sketching values directly during split evaluation.  It
+   was never tested and contained some unknown bugs, we decided to remove it and focus our
+   resources on more promising algorithms instead.  For accuracy, most of the time
+   ``approx``, ``hist`` and ``gpu_hist`` are enough with some parameters tuning, so
+   removing them don't have any real practical impact.
+
+3. ``grow_local_histmaker`` updater: An approximation tree method described in `reference
+   paper <http://arxiv.org/abs/1603.02754>`_.  This updater was rarely used in practice so
+   it was still an updater rather than tree method.  During split finding, it first runs a
+   weighted GK sketching for data points belong to current node to find split candidates,
+   using hessian as weights.  The histogram is built upon this per-node sketch.  It was
+   faster than ``exact`` in some applications, but still slow in computation.  It was
+   removed because it depended on Rabit's customized reduction function that handles all
+   the data structure that can be serialized/deserialized into fixed size buffer, which is
+   not directly supported by NCCL or federated learning gRPC, making it hard to refactor
+   into a common allreducer interface.
 
 **************
 Feature Matrix

doc/tutorials/aft_survival_analysis.rst

@@ -98,7 +98,7 @@ Collect the lower bound numbers in one array (let's call it ``y_lower_bound``) a
   # 4-by-2 Data matrix
   X = np.array([[1, -1], [-1, 1], [0, 1], [1, 0]])
   dtrain = xgb.DMatrix(X)
-  
+
   # Associate ranged labels with the data matrix.
   # This example shows each kind of censored labels.
   #                         uncensored    right     left  interval
@@ -109,7 +109,7 @@ Collect the lower bound numbers in one array (let's call it ``y_lower_bound``) a
 
 .. code-block:: r
   :caption: R
-  
+
   library(xgboost)
 
   # 4-by-2 Data matrix
@@ -165,4 +165,4 @@ Currently, you can choose from three probability distributions for ``aft_loss_di
 ``extreme``               :math:`e^z e^{-\exp{z}}`
 ========================= ===========================================
 
-Note that it is not yet possible to set the ranged label using the scikit-learn interface (e.g. :class:`xgboost.XGBRegressor`). For now, you should use :class:`xgboost.train` with :class:`xgboost.DMatrix`.
+Note that it is not yet possible to set the ranged label using the scikit-learn interface (e.g. :class:`xgboost.XGBRegressor`). For now, you should use :class:`xgboost.train` with :class:`xgboost.DMatrix`. For a collection of Python examples, see :doc:`/python/survival-examples/index`

doc/tutorials/c_api_tutorial.rst

@@ -2,7 +2,7 @@
 C API Tutorial
 ##############
 
-In this tutorial, we are going to install XGBoost library & configure the CMakeLists.txt file of our C/C++ application to link XGBoost library with our application. Later on, we will see some useful tips for using C API and code snippets as examples to use various functions available in C API to perform basic task like loading, training model & predicting on test dataset.
+In this tutorial, we are going to install XGBoost library & configure the CMakeLists.txt file of our C/C++ application to link XGBoost library with our application. Later on, we will see some useful tips for using C API and code snippets as examples to use various functions available in C API to perform basic task like loading, training model & predicting on test dataset. For API reference, please visit :doc:`/c`
 
 .. contents::
   :backlinks: none

doc/tutorials/categorical.rst

@@ -36,9 +36,7 @@ parameter ``enable_categorical``:
 .. code:: python
 
   # Supported tree methods are `gpu_hist`, `approx`, and `hist`.
-  clf = xgb.XGBClassifier(
-      tree_method="gpu_hist", enable_categorical=True, use_label_encoder=False
-  )
+  clf = xgb.XGBClassifier(tree_method="gpu_hist", enable_categorical=True)
   # X is the dataframe we created in previous snippet
   clf.fit(X, y)
   # Must use JSON/UBJSON for serialization, otherwise the information is lost.
@@ -74,23 +72,20 @@ Optimal Partitioning
 .. versionadded:: 1.6
 
 Optimal partitioning is a technique for partitioning the categorical predictors for each
-node split, the proof of optimality for numerical objectives like ``RMSE`` was first
-introduced by `[1] <#references>`__. The algorithm is used in decision trees for handling
-regression and binary classification tasks `[2] <#references>`__, later LightGBM `[3]
-<#references>`__ brought it to the context of gradient boosting trees and now is also
-adopted in XGBoost as an optional feature for handling categorical splits. More
-specifically, the proof by Fisher `[1] <#references>`__ states that, when trying to
-partition a set of discrete values into groups based on the distances between a measure of
-these values, one only needs to look at sorted partitions instead of enumerating all
-possible permutations. In the context of decision trees, the discrete values are
-categories, and the measure is the output leaf value.  Intuitively, we want to group the
-categories that output similar leaf values. During split finding, we first sort the
-gradient histogram to prepare the contiguous partitions then enumerate the splits
+node split, the proof of optimality for numerical output was first introduced by `[1]
+<#references>`__. The algorithm is used in decision trees `[2] <#references>`__, later
+LightGBM `[3] <#references>`__ brought it to the context of gradient boosting trees and
+now is also adopted in XGBoost as an optional feature for handling categorical
+splits. More specifically, the proof by Fisher `[1] <#references>`__ states that, when
+trying to partition a set of discrete values into groups based on the distances between a
+measure of these values, one only needs to look at sorted partitions instead of
+enumerating all possible permutations. In the context of decision trees, the discrete
+values are categories, and the measure is the output leaf value.  Intuitively, we want to
+group the categories that output similar leaf values. During split finding, we first sort
+the gradient histogram to prepare the contiguous partitions then enumerate the splits
 according to these sorted values. One of the related parameters for XGBoost is
-``max_cat_to_one_hot``, which controls whether one-hot encoding or partitioning should be
-used for each feature, see :doc:`/parameter` for details.  When objective is not
-regression or binary classification, XGBoost will fallback to using onehot encoding
-instead.
+``max_cat_to_onehot``, which controls whether one-hot encoding or partitioning should be
+used for each feature, see :ref:`cat-param` for details.
 
 
 **********************

doc/tutorials/custom_metric_obj.rst

@@ -24,7 +24,7 @@ concepts should be readily applicable to other language bindings.
    * Breaking change was made in XGBoost 1.6.
 
 In the following two sections, we will provide a step by step walk through of implementing
-``Squared Log Error(SLE)`` objective function:
+the ``Squared Log Error (SLE)`` objective function:
 
 .. math::
    \frac{1}{2}[log(pred + 1) - log(label + 1)]^2
@@ -114,10 +114,10 @@ monitor our model's performance.  As mentioned above, the default metric for ``S
         elements = np.power(np.log1p(y) - np.log1p(predt), 2)
         return 'PyRMSLE', float(np.sqrt(np.sum(elements) / len(y)))
 
-Since we are demonstrating in Python, the metric or objective needs not be a function,
-any callable object should suffice.  Similarly to the objective function, our metric also
-accepts ``predt`` and ``dtrain`` as inputs, but returns the name of metric itself and a
-floating point value as result.  After passing it into XGBoost as argument of ``feval``
+Since we are demonstrating in Python, the metric or objective need not be a function,
+any callable object should suffice.  Similar to the objective function, our metric also
+accepts ``predt`` and ``dtrain`` as inputs, but returns the name of the metric itself and a
+floating point value as the result.  After passing it into XGBoost as argument of ``feval``
 parameter:
 
 .. code-block:: python
@@ -154,7 +154,7 @@ Reverse Link Function
 *********************
 
 When using builtin objective, the raw prediction is transformed according to the objective
-function.  When custom objective is provided XGBoost doesn't know its link function so the
+function.  When a custom objective is provided XGBoost doesn't know its link function so the
 user is responsible for making the transformation for both objective and custom evaluation
 metric.  For objective with identiy link like ``squared error`` this is trivial, but for
 other link functions like log link or inverse link the difference is significant.
@@ -162,9 +162,9 @@ other link functions like log link or inverse link the difference is significant
 For the Python package, the behaviour of prediction can be controlled by the
 ``output_margin`` parameter in ``predict`` function.  When using the ``custom_metric``
 parameter without a custom objective, the metric function will receive transformed
-prediction since the objective is defined by XGBoost. However, when custom objective is
+prediction since the objective is defined by XGBoost. However, when the custom objective is
 also provided along with that metric, then both the objective and custom metric will
-recieve raw prediction.  Following example provides a comparison between two different
+recieve raw prediction.  The following example provides a comparison between two different
 behavior with a multi-class classification model. Firstly we define 2 different Python
 metric functions implementing the same underlying metric for comparison,
 `merror_with_transform` is used when custom objective is also used, otherwise the simpler

doc/tutorials/dart.rst

@@ -11,7 +11,7 @@ This is a instruction of new tree booster ``dart``.
 **************
 Original paper
 **************
-Rashmi Korlakai Vinayak, Ran Gilad-Bachrach. "DART: Dropouts meet Multiple Additive Regression Trees." `JMLR <http://www.jmlr.org/proceedings/papers/v38/korlakaivinayak15.pdf>`_.
+Rashmi Korlakai Vinayak, Ran Gilad-Bachrach. "DART: Dropouts meet Multiple Additive Regression Trees." [`PMLR <http://proceedings.mlr.press/v38/korlakaivinayak15.pdf>`_, `arXiv <https://arxiv.org/abs/1505.01866>`_].
 
 ********
 Features

doc/tutorials/dask.rst

@@ -115,7 +115,7 @@ Alternatively, XGBoost also implements the Scikit-Learn interface with
 :py:class:`~xgboost.dask.DaskXGBRanker` and 2 random forest variances.  This wrapper is
 similar to the single node Scikit-Learn interface in xgboost, with dask collection as
 inputs and has an additional ``client`` attribute.  See following sections and
-:ref:`sphx_glr_python_dask-examples` for more examples.
+:ref:`dask-examples` for more examples.
 
 
 ******************
@@ -474,7 +474,6 @@ interface, including callback functions, custom evaluation metric and objective:
         callbacks=[early_stop],
     )
 
-
 .. _tracker-ip:
 
 ***************
@@ -504,6 +503,35 @@ dask config is used:
         reg = dxgb.DaskXGBRegressor()
 
 
+
+************
+IPv6 Support
+************
+
+.. versionadded:: 1.7.0
+
+XGBoost has initial IPv6 support for the dask interface on Linux. Due to most of the
+cluster support for IPv6 is partial (dual stack instead of IPv6 only), we require
+additional user configuration similar to :ref:`tracker-ip` to help XGBoost obtain the
+correct address information:
+
+.. code-block:: python
+
+    import dask
+    from distributed import Client
+    from xgboost import dask as dxgb
+    # let xgboost know the scheduler address, use the same bracket format as dask.
+    with dask.config.set({"xgboost.scheduler_address": "[fd20:b6f:f759:9800::]"}):
+        with Client("[fd20:b6f:f759:9800::]") as client:
+            reg = dxgb.DaskXGBRegressor(tree_method="hist")
+
+
+When GPU is used, XGBoost employs `NCCL <https://developer.nvidia.com/nccl>`_ as the
+underlying communication framework, which may require some additional configuration via
+environment variable depending on the setting of the cluster. Please note that IPv6
+support is Unix only.
+
+
 *****************************************************************************
 Why is the initialization of ``DaskDMatrix``  so slow and throws weird errors
 *****************************************************************************

doc/tutorials/index.rst

@@ -14,7 +14,9 @@ See `Awesome XGBoost <https://github.com/dmlc/xgboost/tree/master/demo>`_ for mo
   Distributed XGBoost with AWS YARN <aws_yarn>
   kubernetes
   Distributed XGBoost with XGBoost4J-Spark <https://xgboost.readthedocs.io/en/latest/jvm/xgboost4j_spark_tutorial.html>
+  Distributed XGBoost with XGBoost4J-Spark-GPU <https://xgboost.readthedocs.io/en/latest/jvm/xgboost4j_spark_gpu_tutorial.html>
   dask
+  spark_estimator
   ray
   dart
   monotonic

doc/tutorials/kubernetes.rst

@@ -2,7 +2,7 @@
 Distributed XGBoost on Kubernetes
 ###################################
 
-Distributed XGBoost training and batch prediction on `Kubernetes <https://kubernetes.io/>`_ are supported via `Kubeflow XGBoost Operator <https://github.com/kubeflow/xgboost-operator>`_.
+Distributed XGBoost training and batch prediction on `Kubernetes <https://kubernetes.io/>`_ are supported via `Kubeflow XGBoost Training Operator <https://github.com/kubeflow/training-operator>`_.
 
 ************
 Instructions
@@ -11,24 +11,24 @@ In order to run a XGBoost job in a Kubernetes cluster, perform the following ste
 
 1. Install XGBoost Operator on the Kubernetes cluster.
 
-   a. XGBoost Operator is designed to manage the scheduling and monitoring of XGBoost jobs. Follow `this installation guide <https://github.com/kubeflow/xgboost-operator#install-xgboost-operator>`_ to install XGBoost Operator.
+   a. XGBoost Operator is designed to manage the scheduling and monitoring of XGBoost jobs. Follow `this installation guide <https://www.kubeflow.org/docs/components/training/xgboost/#installing-xgboost-operator>`_ to install XGBoost Operator.
 
 2. Write application code that will be executed by the XGBoost Operator.
 
-   a. To use XGBoost Operator, you'll have to write a couple of Python scripts that implement the distributed training logic for XGBoost. Please refer to the `Iris classification example <https://github.com/kubeflow/xgboost-operator/tree/master/config/samples/xgboost-dist>`_.
+   a. To use XGBoost Operator, you'll have to write a couple of Python scripts that implement the distributed training logic for XGBoost. Please refer to the `Iris classification example <https://github.com/kubeflow/training-operator/tree/master/examples/xgboost/xgboost-dist>`_.
    b. Data reader/writer: you need to implement the data reader and writer based on the specific requirements of your chosen data source. For example, if your dataset is stored in a Hive table, you have to write the code to read from or write to the Hive table based on the index of the worker.
-   c. Model persistence: in the `Iris classification example <https://github.com/kubeflow/xgboost-operator/tree/master/config/samples/xgboost-dist>`_, the model is stored in `Alibaba OSS <https://www.alibabacloud.com/product/oss>`_. If you want to store your model in other storages such as Amazon S3 or Google NFS, you'll need to implement the model persistence logic based on the requirements of the chosen storage system.
+   c. Model persistence: in the `Iris classification example <https://github.com/kubeflow/training-operator/tree/master/examples/xgboost/xgboost-dist>`_, the model is stored in `Alibaba OSS <https://www.alibabacloud.com/product/oss>`_. If you want to store your model in other storages such as Amazon S3 or Google NFS, you'll need to implement the model persistence logic based on the requirements of the chosen storage system.
 
 3. Configure the XGBoost job using a YAML file.
 
-   a. YAML file is used to configure the computational resources and environment for your XGBoost job to run, e.g. the number of workers/masters and the number of CPU/GPUs. Please refer to this `YAML template <https://github.com/kubeflow/xgboost-operator/blob/master/config/samples/xgboost-dist/xgboostjob_v1alpha1_iris_train.yaml>`_ for an example.
+   a. YAML file is used to configure the computational resources and environment for your XGBoost job to run, e.g. the number of workers/masters and the number of CPU/GPUs. Please refer to this `YAML template <https://github.com/kubeflow/training-operator/blob/master/examples/xgboost/xgboost-dist/xgboostjob_v1alpha1_iris_train.yaml>`_ for an example.
 
 4. Submit XGBoost job to a Kubernetes cluster.
 
-   a. Use `kubectl <https://kubernetes.io/docs/reference/kubectl/overview/>`_ to submit a distributed XGBoost job as illustrated `here <https://github.com/kubeflow/xgboost-operator#creating-a-xgboost-trainingprediction-job>`_.
+   a. Use `kubectl <https://kubernetes.io/docs/reference/kubectl/overview/>`_ to submit a distributed XGBoost job as illustrated `here <https://www.kubeflow.org/docs/components/training/xgboost/#creating-a-xgboost-training-job>`_.
 
 *******
 Support
 *******
 
-Please submit an issue on `XGBoost Operator repo <https://github.com/kubeflow/xgboost-operator>`_ for any feature requests or problems.
+Please submit an issue on `XGBoost Operator repo <https://github.com/kubeflow/training-operator/issues>`_ for any feature requests or problems.

doc/tutorials/model.rst

@@ -29,7 +29,7 @@ With judicious choices for :math:`y_i`, we may express a variety of tasks, such
 The task of **training** the model amounts to finding the best parameters :math:`\theta` that best fit the training data :math:`x_i` and labels :math:`y_i`. In order to train the model, we need to define the **objective function**
 to measure how well the model fit the training data.
 
-A salient characteristic of objective functions is that they consist two parts: **training loss** and **regularization term**:
+A salient characteristic of objective functions is that they consist of two parts: **training loss** and **regularization term**:
 
 .. math::
 

doc/tutorials/saving_model.rst

@@ -44,6 +44,7 @@ with normal model IO operation. Currently, memory snapshot is used in the follow
 * Python package: when the ``Booster`` object is pickled with the built-in ``pickle`` module.
 * R package: when the ``xgb.Booster`` object is persisted with the built-in functions ``saveRDS``
   or ``save``.
+* JVM packages: when the ``Booster`` object is serialized with the built-in functions ``saveModel``.
 
 Other language bindings are still working in progress.
 
@@ -68,6 +69,18 @@ a filename with ``.json`` or ``.ubj`` as file extension, the latter is the exten
 
   xgb.save(bst, 'model_file_name.json')
 
+.. code-block:: Scala
+  :caption: Scala
+
+  val format = "json"  // or val format = "ubj"
+  model.write.option("format", format).save("model_directory_path")
+
+.. note::
+
+  Only load models from JSON files that were produced by XGBoost. Attempting to load
+  JSON files that were produced by an external source may lead to undefined behaviors
+  and crashes.
+
 While for memory snapshot, UBJSON is the default starting with xgboost 1.6.
 
 ***************************************************************
@@ -171,8 +184,6 @@ Will print out something similar to (not actual output as it's too long for demo
             "grow_gpu_hist": {
               "gpu_hist_train_param": {
                 "debug_synchronize": "0",
-                "gpu_batch_nrows": "0",
-                "single_precision_histogram": "0"
               },
               "train_param": {
                 "alpha": "0",

doc/tutorials/spark_estimator.rst

@@ -0,0 +1,226 @@
+################################
+Distributed XGBoost with PySpark
+################################
+
+Starting from version 1.7.0, xgboost supports pyspark estimator APIs.
+
+.. note::
+
+   The feature is still experimental and not yet ready for production use.
+
+.. contents::
+  :backlinks: none
+  :local:
+
+*************************
+XGBoost PySpark Estimator
+*************************
+
+SparkXGBRegressor
+=================
+
+SparkXGBRegressor is a PySpark ML estimator. It implements the XGBoost classification
+algorithm based on XGBoost python library, and it can be used in PySpark Pipeline
+and PySpark ML meta algorithms like CrossValidator/TrainValidationSplit/OneVsRest.
+
+We can create a `SparkXGBRegressor` estimator like:
+
+.. code-block:: python
+
+  from xgboost.spark import SparkXGBRegressor
+  spark_reg_estimator = SparkXGBRegressor(
+    features_col="features",
+    label_col="label",
+    num_workers=2,
+  )
+
+
+The above snippet creates a spark estimator which can fit on a spark dataset,
+and return a spark model that can transform a spark dataset and generate dataset
+with prediction column. We can set almost all of xgboost sklearn estimator parameters
+as `SparkXGBRegressor` parameters, but some parameter such as `nthread` is forbidden
+in spark estimator, and some parameters are replaced with pyspark specific parameters
+such as `weight_col`, `validation_indicator_col`, `use_gpu`, for details please see
+`SparkXGBRegressor` doc.
+
+The following code snippet shows how to train a spark xgboost regressor model,
+first we need to prepare a training dataset as a spark dataframe contains
+"label" column and "features" column(s), the "features" column(s) must be `pyspark.ml.linalg.Vector`
+type or spark array type or a list of feature column names.
+
+
+.. code-block:: python
+
+  xgb_regressor_model = xgb_regressor.fit(train_spark_dataframe)
+
+
+The following code snippet shows how to predict test data using a spark xgboost regressor model,
+first we need to prepare a test dataset as a spark dataframe contains
+"features" and "label" column, the "features" column must be `pyspark.ml.linalg.Vector`
+type or spark array type.
+
+.. code-block:: python
+
+  transformed_test_spark_dataframe = xgb_regressor.predict(test_spark_dataframe)
+
+
+The above snippet code returns a `transformed_test_spark_dataframe` that contains the input
+dataset columns and an appended column "prediction" representing the prediction results.
+
+SparkXGBClassifier
+==================
+
+`SparkXGBClassifier` estimator has similar API with `SparkXGBRegressor`, but it has some
+pyspark classifier specific params, e.g. `raw_prediction_col` and `probability_col` parameters.
+Correspondingly, by default, `SparkXGBClassifierModel` transforming test dataset will
+generate result dataset with 3 new columns:
+- "prediction": represents the predicted label.
+- "raw_prediction": represents the output margin values.
+- "probability": represents the prediction probability on each label.
+
+
+***************************
+XGBoost PySpark GPU support
+***************************
+
+XGBoost PySpark supports GPU training and prediction. To enable GPU support, first you
+need to install the XGBoost and the `cuDF <https://docs.rapids.ai/api/cudf/stable/>`_
+package. Then you can set `use_gpu` parameter to `True`.
+
+Below tutorial demonstrates how to train a model with XGBoost PySpark GPU on Spark
+standalone cluster.
+
+
+Write your PySpark application
+==============================
+
+.. code-block:: python
+
+  from xgboost.spark import SparkXGBRegressor
+  spark = SparkSession.builder.getOrCreate()
+
+  # read data into spark dataframe
+  train_data_path = "xxxx/train"
+  train_df = spark.read.parquet(data_path)
+
+  test_data_path = "xxxx/test"
+  test_df = spark.read.parquet(test_data_path)
+
+  # assume the label column is named "class"
+  label_name = "class"
+
+  # get a list with feature column names
+  feature_names = [x.name for x in train_df.schema if x.name != label]
+
+  # create a xgboost pyspark regressor estimator and set use_gpu=True
+  regressor = SparkXGBRegressor(
+    features_col=feature_names,
+    label_col=label_name,
+    num_workers=2,
+    use_gpu=True,
+  )
+
+  # train and return the model
+  model = regressor.fit(train_df)
+
+  # predict on test data
+  predict_df = model.transform(test_df)
+  predict_df.show()
+
+Prepare the necessary packages
+==============================
+
+We recommend using Conda or Virtualenv to manage python dependencies
+in PySpark. Please refer to
+`How to Manage Python Dependencies in PySpark <https://www.databricks.com/blog/2020/12/22/how-to-manage-python-dependencies-in-pyspark.html>`_.
+
+.. code-block:: bash
+
+  conda create -y -n xgboost-env -c conda-forge conda-pack python=3.9
+  conda activate xgboost-env
+  pip install xgboost
+  conda install cudf -c rapids -c nvidia -c conda-forge
+  conda pack -f -o xgboost-env.tar.gz
+
+
+Submit the PySpark application
+==============================
+
+Assuming you have configured your Spark cluster with GPU support, if not yet, please
+refer to `spark standalone configuration with GPU support <https://nvidia.github.io/spark-rapids/docs/get-started/getting-started-on-prem.html#spark-standalone-cluster>`_.
+
+.. code-block:: bash
+
+  export PYSPARK_DRIVER_PYTHON=python
+  export PYSPARK_PYTHON=./environment/bin/python
+
+  spark-submit \
+    --master spark://<master-ip>:7077 \
+    --conf spark.executor.resource.gpu.amount=1 \
+    --conf spark.task.resource.gpu.amount=1 \
+    --archives xgboost-env.tar.gz#environment \
+    xgboost_app.py
+
+
+Model Persistence
+=================
+
+Similar to standard PySpark ml estimators, one can persist and reuse the model with `save`
+and `load` methods:
+
+.. code-block:: python
+
+  regressor = SparkXGBRegressor()
+  model = regressor.fit(train_df)
+  # save the model
+  model.save("/tmp/xgboost-pyspark-model")
+  # load the model
+  model2 = SparkXGBRankerModel.load("/tmp/xgboost-pyspark-model")
+
+To export the underlying booster model used by XGBoost:
+
+.. code-block:: python
+
+  regressor = SparkXGBRegressor()
+  model = regressor.fit(train_df)
+  # the same booster object returned by xgboost.train
+  booster: xgb.Booster = model.get_booster()
+  booster.predict(...)
+  booster.save_model("model.json")
+
+This booster is shared by other Python interfaces and can be used by other language
+bindings like the C and R packages. Lastly, one can extract a booster file directly from
+saved spark estimator without going through the getter:
+
+.. code-block:: python
+
+  import xgboost as xgb
+  bst = xgb.Booster()
+  bst.load_model("/tmp/xgboost-pyspark-model/model/part-00000")
+
+Accelerate the whole pipeline of xgboost pyspark
+================================================
+
+With `RAPIDS Accelerator for Apache Spark <https://nvidia.github.io/spark-rapids/>`_,
+you can accelerate the whole pipeline (ETL, Train, Transform) for xgboost pyspark
+without any code change by leveraging GPU.
+
+Below is a simple example submit command for enabling GPU acceleration:
+
+.. code-block:: bash
+
+  export PYSPARK_DRIVER_PYTHON=python
+  export PYSPARK_PYTHON=./environment/bin/python
+
+  spark-submit \
+    --master spark://<master-ip>:7077 \
+    --conf spark.executor.resource.gpu.amount=1 \
+    --conf spark.task.resource.gpu.amount=1 \
+    --packages com.nvidia:rapids-4-spark_2.12:22.08.0 \
+    --conf spark.plugins=com.nvidia.spark.SQLPlugin \
+    --conf spark.sql.execution.arrow.maxRecordsPerBatch=1000000 \
+    --archives xgboost-env.tar.gz#environment \
+    xgboost_app.py
+
+When rapids plugin is enabled, both of the JVM rapids plugin and the cuDF Python are
+required for the acceleration.

include/xgboost/base.h

@@ -121,6 +121,8 @@ using bst_float = float;  // NOLINT
 using bst_cat_t = int32_t;  // NOLINT
 /*! \brief Type for data column (feature) index. */
 using bst_feature_t = uint32_t;  // NOLINT
+/*! \brief Type for histogram bin index. */
+using bst_bin_t = int32_t;  // NOLINT
 /*! \brief Type for data row index.
  *
  * Be careful `std::size_t' is implementation-defined.  Meaning that the binary
@@ -257,10 +259,61 @@ class GradientPairInternal {
 using GradientPair = detail::GradientPairInternal<float>;
 /*! \brief High precision gradient statistics pair */
 using GradientPairPrecise = detail::GradientPairInternal<double>;
-/*! \brief Fixed point representation for gradient pair. */
-using GradientPairInt32 = detail::GradientPairInternal<int>;
-/*! \brief Fixed point representation for high precision gradient pair. */
-using GradientPairInt64 = detail::GradientPairInternal<int64_t>;
+
+/*! \brief Fixed point representation for high precision gradient pair. Has a different interface so
+ * we don't accidentally use it in gain calculations.*/
+class GradientPairInt64 {
+  using T = int64_t;
+  T grad_ = 0;
+  T hess_ = 0;
+
+ public:
+  using ValueT = T;
+
+  XGBOOST_DEVICE GradientPairInt64(T grad, T hess) : grad_(grad), hess_(hess) {}
+  GradientPairInt64() = default;
+
+  // Copy constructor if of same value type, marked as default to be trivially_copyable
+  GradientPairInt64(const GradientPairInt64 &g) = default;
+
+  XGBOOST_DEVICE T GetQuantisedGrad() const { return grad_; }
+  XGBOOST_DEVICE T GetQuantisedHess() const { return hess_; }
+
+  XGBOOST_DEVICE GradientPairInt64 &operator+=(const GradientPairInt64 &rhs) {
+    grad_ += rhs.grad_;
+    hess_ += rhs.hess_;
+    return *this;
+  }
+
+  XGBOOST_DEVICE GradientPairInt64 operator+(const GradientPairInt64 &rhs) const {
+    GradientPairInt64 g;
+    g.grad_ = grad_ + rhs.grad_;
+    g.hess_ = hess_ + rhs.hess_;
+    return g;
+  }
+
+  XGBOOST_DEVICE GradientPairInt64 &operator-=(const GradientPairInt64 &rhs) {
+    grad_ -= rhs.grad_;
+    hess_ -= rhs.hess_;
+    return *this;
+  }
+
+  XGBOOST_DEVICE GradientPairInt64 operator-(const GradientPairInt64 &rhs) const {
+    GradientPairInt64 g;
+    g.grad_ = grad_ - rhs.grad_;
+    g.hess_ = hess_ - rhs.hess_;
+    return g;
+  }
+
+  XGBOOST_DEVICE bool operator==(const GradientPairInt64 &rhs) const {
+    return grad_ == rhs.grad_ && hess_ == rhs.hess_;
+  }
+  friend std::ostream &operator<<(std::ostream &os,
+                                  const GradientPairInt64 &g) {
+    os << g.GetQuantisedGrad() << "/" << g.GetQuantisedHess();
+    return os;
+  }
+};
 
 using Args = std::vector<std::pair<std::string, std::string> >;
 

include/xgboost/collective/socket.h

@@ -0,0 +1,536 @@
+/*!
+ * Copyright (c) 2022 by XGBoost Contributors
+ */
+#pragma once
+
+#if !defined(NOMINMAX) && defined(_WIN32)
+#define NOMINMAX
+#endif  // !defined(NOMINMAX)
+
+#include <cerrno>        // errno, EINTR, EBADF
+#include <climits>       // HOST_NAME_MAX
+#include <cstddef>       // std::size_t
+#include <cstdint>       // std::int32_t, std::uint16_t
+#include <cstring>       // memset
+#include <limits>        // std::numeric_limits
+#include <string>        // std::string
+#include <system_error>  // std::error_code, std::system_category
+#include <utility>       // std::swap
+
+#if !defined(xgboost_IS_MINGW)
+#define xgboost_IS_MINGW() defined(__MINGW32__)
+#endif  // xgboost_IS_MINGW
+
+#if defined(_WIN32)
+
+#include <winsock2.h>
+#include <ws2tcpip.h>
+
+using in_port_t = std::uint16_t;
+
+#ifdef _MSC_VER
+#pragma comment(lib, "Ws2_32.lib")
+#endif  // _MSC_VER
+
+#if !xgboost_IS_MINGW()
+using ssize_t = int;
+#endif  // !xgboost_IS_MINGW()
+
+#else  // UNIX
+
+#include <arpa/inet.h>    // inet_ntop
+#include <fcntl.h>        // fcntl, F_GETFL, O_NONBLOCK
+#include <netinet/in.h>   // sockaddr_in6, sockaddr_in, in_port_t, INET6_ADDRSTRLEN, INET_ADDRSTRLEN
+#include <netinet/in.h>   // IPPROTO_TCP
+#include <netinet/tcp.h>  // TCP_NODELAY
+#include <sys/socket.h>  // socket, SOL_SOCKET, SO_ERROR, MSG_WAITALL, recv, send, AF_INET6, AF_INET
+#include <unistd.h>      // close
+
+#if defined(__sun) || defined(sun)
+#include <sys/sockio.h>
+#endif  // defined(__sun) || defined(sun)
+
+#endif  // defined(_WIN32)
+
+#include "xgboost/base.h"         // XGBOOST_EXPECT
+#include "xgboost/logging.h"      // LOG
+#include "xgboost/string_view.h"  // StringView
+
+#if !defined(HOST_NAME_MAX)
+#define HOST_NAME_MAX 256  // macos
+#endif
+
+namespace xgboost {
+
+#if xgboost_IS_MINGW()
+// see the dummy implementation of `poll` in rabit for more info.
+inline void MingWError() { LOG(FATAL) << "Distributed training on mingw is not supported."; }
+#endif  // xgboost_IS_MINGW()
+
+namespace system {
+inline std::int32_t LastError() {
+#if defined(_WIN32)
+  return WSAGetLastError();
+#else
+  int errsv = errno;
+  return errsv;
+#endif
+}
+
+#if defined(__GLIBC__)
+inline auto ThrowAtError(StringView fn_name, std::int32_t errsv = LastError(),
+                         std::int32_t line = __builtin_LINE(),
+                         char const *file = __builtin_FILE()) {
+  auto err = std::error_code{errsv, std::system_category()};
+  LOG(FATAL) << "\n"
+             << file << "(" << line << "): Failed to call `" << fn_name << "`: " << err.message()
+             << std::endl;
+}
+#else
+inline auto ThrowAtError(StringView fn_name, std::int32_t errsv = LastError()) {
+  auto err = std::error_code{errsv, std::system_category()};
+  LOG(FATAL) << "Failed to call `" << fn_name << "`: " << err.message() << std::endl;
+}
+#endif  // defined(__GLIBC__)
+
+#if defined(_WIN32)
+using SocketT = SOCKET;
+#else
+using SocketT = int;
+#endif  // defined(_WIN32)
+
+#if !defined(xgboost_CHECK_SYS_CALL)
+#define xgboost_CHECK_SYS_CALL(exp, expected)         \
+  do {                                                \
+    if (XGBOOST_EXPECT((exp) != (expected), false)) { \
+      ::xgboost::system::ThrowAtError(#exp);          \
+    }                                                 \
+  } while (false)
+#endif  // !defined(xgboost_CHECK_SYS_CALL)
+
+inline std::int32_t CloseSocket(SocketT fd) {
+#if defined(_WIN32)
+  return closesocket(fd);
+#else
+  return close(fd);
+#endif
+}
+
+inline bool LastErrorWouldBlock() {
+  int errsv = LastError();
+#ifdef _WIN32
+  return errsv == WSAEWOULDBLOCK;
+#else
+  return errsv == EAGAIN || errsv == EWOULDBLOCK;
+#endif  // _WIN32
+}
+
+inline void SocketStartup() {
+#if defined(_WIN32)
+  WSADATA wsa_data;
+  if (WSAStartup(MAKEWORD(2, 2), &wsa_data) == -1) {
+    ThrowAtError("WSAStartup");
+  }
+  if (LOBYTE(wsa_data.wVersion) != 2 || HIBYTE(wsa_data.wVersion) != 2) {
+    WSACleanup();
+    LOG(FATAL) << "Could not find a usable version of Winsock.dll";
+  }
+#endif  // defined(_WIN32)
+}
+
+inline void SocketFinalize() {
+#if defined(_WIN32)
+  WSACleanup();
+#endif  // defined(_WIN32)
+}
+
+#if defined(_WIN32) && xgboost_IS_MINGW()
+// dummy definition for old mysys32.
+inline const char *inet_ntop(int, const void *, char *, socklen_t) {  // NOLINT
+  MingWError();
+  return nullptr;
+}
+#else
+using ::inet_ntop;
+#endif
+
+}  // namespace system
+
+namespace collective {
+class SockAddress;
+
+enum class SockDomain : std::int32_t { kV4 = AF_INET, kV6 = AF_INET6 };
+
+/**
+ * \brief Parse host address and return a SockAddress instance. Supports IPv4 and IPv6
+ *        host.
+ */
+SockAddress MakeSockAddress(StringView host, in_port_t port);
+
+class SockAddrV6 {
+  sockaddr_in6 addr_;
+
+ public:
+  explicit SockAddrV6(sockaddr_in6 addr) : addr_{addr} {}
+  SockAddrV6() { std::memset(&addr_, '\0', sizeof(addr_)); }
+
+  static SockAddrV6 Loopback();
+  static SockAddrV6 InaddrAny();
+
+  in_port_t Port() const { return ntohs(addr_.sin6_port); }
+
+  std::string Addr() const {
+    char buf[INET6_ADDRSTRLEN];
+    auto const *s = system::inet_ntop(static_cast<std::int32_t>(SockDomain::kV6), &addr_.sin6_addr,
+                                      buf, INET6_ADDRSTRLEN);
+    if (s == nullptr) {
+      system::ThrowAtError("inet_ntop");
+    }
+    return {buf};
+  }
+  sockaddr_in6 const &Handle() const { return addr_; }
+};
+
+class SockAddrV4 {
+ private:
+  sockaddr_in addr_;
+
+ public:
+  explicit SockAddrV4(sockaddr_in addr) : addr_{addr} {}
+  SockAddrV4() { std::memset(&addr_, '\0', sizeof(addr_)); }
+
+  static SockAddrV4 Loopback();
+  static SockAddrV4 InaddrAny();
+
+  in_port_t Port() const { return ntohs(addr_.sin_port); }
+
+  std::string Addr() const {
+    char buf[INET_ADDRSTRLEN];
+    auto const *s = system::inet_ntop(static_cast<std::int32_t>(SockDomain::kV4), &addr_.sin_addr,
+                                      buf, INET_ADDRSTRLEN);
+    if (s == nullptr) {
+      system::ThrowAtError("inet_ntop");
+    }
+    return {buf};
+  }
+  sockaddr_in const &Handle() const { return addr_; }
+};
+
+/**
+ * \brief Address for TCP socket, can be either IPv4 or IPv6.
+ */
+class SockAddress {
+ private:
+  SockAddrV6 v6_;
+  SockAddrV4 v4_;
+  SockDomain domain_{SockDomain::kV4};
+
+ public:
+  SockAddress() = default;
+  explicit SockAddress(SockAddrV6 const &addr) : v6_{addr}, domain_{SockDomain::kV6} {}
+  explicit SockAddress(SockAddrV4 const &addr) : v4_{addr} {}
+
+  auto Domain() const { return domain_; }
+
+  bool IsV4() const { return Domain() == SockDomain::kV4; }
+  bool IsV6() const { return !IsV4(); }
+
+  auto const &V4() const { return v4_; }
+  auto const &V6() const { return v6_; }
+};
+
+/**
+ * \brief TCP socket for simple communication.
+ */
+class TCPSocket {
+ public:
+  using HandleT = system::SocketT;
+
+ private:
+  HandleT handle_{InvalidSocket()};
+  // There's reliable no way to extract domain from a socket without first binding that
+  // socket on macos.
+#if defined(__APPLE__)
+  SockDomain domain_{SockDomain::kV4};
+#endif
+
+  constexpr static HandleT InvalidSocket() { return -1; }
+
+  explicit TCPSocket(HandleT newfd) : handle_{newfd} {}
+
+ public:
+  TCPSocket() = default;
+  /**
+   * \brief Return the socket domain.
+   */
+  auto Domain() const -> SockDomain {
+    auto ret_iafamily = [](std::int32_t domain) {
+      switch (domain) {
+        case AF_INET:
+          return SockDomain::kV4;
+        case AF_INET6:
+          return SockDomain::kV6;
+        default: {
+          LOG(FATAL) << "Unknown IA family.";
+        }
+      }
+      return SockDomain::kV4;
+    };
+
+#if defined(_WIN32)
+    WSAPROTOCOL_INFOA info;
+    socklen_t len = sizeof(info);
+    xgboost_CHECK_SYS_CALL(
+        getsockopt(handle_, SOL_SOCKET, SO_PROTOCOL_INFO, reinterpret_cast<char *>(&info), &len),
+        0);
+    return ret_iafamily(info.iAddressFamily);
+#elif defined(__APPLE__)
+    return domain_;
+#elif defined(__unix__)
+    std::int32_t domain;
+    socklen_t len = sizeof(domain);
+    xgboost_CHECK_SYS_CALL(
+        getsockopt(handle_, SOL_SOCKET, SO_DOMAIN, reinterpret_cast<char *>(&domain), &len), 0);
+    return ret_iafamily(domain);
+#else
+    LOG(FATAL) << "Unknown platform.";
+    return ret_iafamily(AF_INET);
+#endif  // platforms
+  }
+
+  bool IsClosed() const { return handle_ == InvalidSocket(); }
+
+  /** \brief get last error code if any */
+  std::int32_t GetSockError() const {
+    std::int32_t error = 0;
+    socklen_t len = sizeof(error);
+    xgboost_CHECK_SYS_CALL(
+        getsockopt(handle_, SOL_SOCKET, SO_ERROR, reinterpret_cast<char *>(&error), &len), 0);
+    return error;
+  }
+  /** \brief check if anything bad happens */
+  bool BadSocket() const {
+    if (IsClosed()) return true;
+    std::int32_t err = GetSockError();
+    if (err == EBADF || err == EINTR) return true;
+    return false;
+  }
+
+  void SetNonBlock() {
+    bool non_block{true};
+#if defined(_WIN32)
+    u_long mode = non_block ? 1 : 0;
+    xgboost_CHECK_SYS_CALL(ioctlsocket(handle_, FIONBIO, &mode), NO_ERROR);
+#else
+    std::int32_t flag = fcntl(handle_, F_GETFL, 0);
+    if (flag == -1) {
+      system::ThrowAtError("fcntl");
+    }
+    if (non_block) {
+      flag |= O_NONBLOCK;
+    } else {
+      flag &= ~O_NONBLOCK;
+    }
+    if (fcntl(handle_, F_SETFL, flag) == -1) {
+      system::ThrowAtError("fcntl");
+    }
+#endif  // _WIN32
+  }
+
+  void SetKeepAlive() {
+    std::int32_t keepalive = 1;
+    xgboost_CHECK_SYS_CALL(setsockopt(handle_, SOL_SOCKET, SO_KEEPALIVE,
+                                      reinterpret_cast<char *>(&keepalive), sizeof(keepalive)),
+                           0);
+  }
+
+  void SetNoDelay() {
+    std::int32_t tcp_no_delay = 1;
+    xgboost_CHECK_SYS_CALL(
+        setsockopt(handle_, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast<char *>(&tcp_no_delay),
+                   sizeof(tcp_no_delay)),
+        0);
+  }
+
+  /**
+   * \brief Accept new connection, returns a new TCP socket for the new connection.
+   */
+  TCPSocket Accept() {
+    HandleT newfd = accept(handle_, nullptr, nullptr);
+    if (newfd == InvalidSocket()) {
+      system::ThrowAtError("accept");
+    }
+    TCPSocket newsock{newfd};
+    return newsock;
+  }
+
+  ~TCPSocket() {
+    if (!IsClosed()) {
+      Close();
+    }
+  }
+
+  TCPSocket(TCPSocket const &that) = delete;
+  TCPSocket(TCPSocket &&that) noexcept(true) { std::swap(this->handle_, that.handle_); }
+  TCPSocket &operator=(TCPSocket const &that) = delete;
+  TCPSocket &operator=(TCPSocket &&that) {
+    std::swap(this->handle_, that.handle_);
+    return *this;
+  }
+  /**
+   * \brief Return the native socket file descriptor.
+   */
+  HandleT const &Handle() const { return handle_; }
+  /**
+   * \brief Listen to incoming requests. Should be called after bind.
+   */
+  void Listen(std::int32_t backlog = 16) { xgboost_CHECK_SYS_CALL(listen(handle_, backlog), 0); }
+  /**
+   * \brief Bind socket to INADDR_ANY, return the port selected by the OS.
+   */
+  in_port_t BindHost() {
+    if (Domain() == SockDomain::kV6) {
+      auto addr = SockAddrV6::InaddrAny();
+      auto handle = reinterpret_cast<sockaddr const *>(&addr.Handle());
+      xgboost_CHECK_SYS_CALL(
+          bind(handle_, handle, sizeof(std::remove_reference_t<decltype(addr.Handle())>)), 0);
+
+      sockaddr_in6 res_addr;
+      socklen_t addrlen = sizeof(res_addr);
+      xgboost_CHECK_SYS_CALL(
+          getsockname(handle_, reinterpret_cast<sockaddr *>(&res_addr), &addrlen), 0);
+      return ntohs(res_addr.sin6_port);
+    } else {
+      auto addr = SockAddrV4::InaddrAny();
+      auto handle = reinterpret_cast<sockaddr const *>(&addr.Handle());
+      xgboost_CHECK_SYS_CALL(
+          bind(handle_, handle, sizeof(std::remove_reference_t<decltype(addr.Handle())>)), 0);
+
+      sockaddr_in res_addr;
+      socklen_t addrlen = sizeof(res_addr);
+      xgboost_CHECK_SYS_CALL(
+          getsockname(handle_, reinterpret_cast<sockaddr *>(&res_addr), &addrlen), 0);
+      return ntohs(res_addr.sin_port);
+    }
+  }
+  /**
+   * \brief Send data, without error then all data should be sent.
+   */
+  auto SendAll(void const *buf, std::size_t len) {
+    char const *_buf = reinterpret_cast<const char *>(buf);
+    std::size_t ndone = 0;
+    while (ndone < len) {
+      ssize_t ret = send(handle_, _buf, len - ndone, 0);
+      if (ret == -1) {
+        if (system::LastErrorWouldBlock()) {
+          return ndone;
+        }
+        system::ThrowAtError("send");
+      }
+      _buf += ret;
+      ndone += ret;
+    }
+    return ndone;
+  }
+  /**
+   * \brief Receive data, without error then all data should be received.
+   */
+  auto RecvAll(void *buf, std::size_t len) {
+    char *_buf = reinterpret_cast<char *>(buf);
+    std::size_t ndone = 0;
+    while (ndone < len) {
+      ssize_t ret = recv(handle_, _buf, len - ndone, MSG_WAITALL);
+      if (ret == -1) {
+        if (system::LastErrorWouldBlock()) {
+          return ndone;
+        }
+        system::ThrowAtError("recv");
+      }
+      if (ret == 0) {
+        return ndone;
+      }
+      _buf += ret;
+      ndone += ret;
+    }
+    return ndone;
+  }
+  /**
+   * \brief Send data using the socket
+   * \param buf the pointer to the buffer
+   * \param len the size of the buffer
+   * \param flags extra flags
+   * \return size of data actually sent return -1 if error occurs
+   */
+  auto Send(const void *buf_, std::size_t len, std::int32_t flags = 0) {
+    const char *buf = reinterpret_cast<const char *>(buf_);
+    return send(handle_, buf, len, flags);
+  }
+  /**
+   * \brief receive data using the socket
+   * \param buf the pointer to the buffer
+   * \param len the size of the buffer
+   * \param flags extra flags
+   * \return size of data actually received return -1 if error occurs
+   */
+  auto Recv(void *buf, std::size_t len, std::int32_t flags = 0) {
+    char *_buf = reinterpret_cast<char *>(buf);
+    return recv(handle_, _buf, len, flags);
+  }
+  /**
+   * \brief Send string, format is matched with the Python socket wrapper in RABIT.
+   */
+  std::size_t Send(StringView str);
+  /**
+   * \brief Receive string, format is matched with the Python socket wrapper in RABIT.
+   */
+  std::size_t Recv(std::string *p_str);
+  /**
+   * \brief Close the socket, called automatically in destructor if the socket is not closed.
+   */
+  void Close() {
+    if (InvalidSocket() != handle_) {
+      xgboost_CHECK_SYS_CALL(system::CloseSocket(handle_), 0);
+      handle_ = InvalidSocket();
+    }
+  }
+  /**
+   * \brief Create a TCP socket on specified domain.
+   */
+  static TCPSocket Create(SockDomain domain) {
+#if xgboost_IS_MINGW()
+    MingWError();
+    return {};
+#else
+    auto fd = socket(static_cast<std::int32_t>(domain), SOCK_STREAM, 0);
+    if (fd == InvalidSocket()) {
+      system::ThrowAtError("socket");
+    }
+
+    TCPSocket socket{fd};
+#if defined(__APPLE__)
+    socket.domain_ = domain;
+#endif  // defined(__APPLE__)
+    return socket;
+#endif  // xgboost_IS_MINGW()
+  }
+};
+
+/**
+ * \brief Connect to remote address, returns the error code if failed (no exception is
+ *        raised so that we can retry).
+ */
+std::error_code Connect(SockAddress const &addr, TCPSocket *out);
+
+/**
+ * \brief Get the local host name.
+ */
+inline std::string GetHostName() {
+  char buf[HOST_NAME_MAX];
+  xgboost_CHECK_SYS_CALL(gethostname(&buf[0], HOST_NAME_MAX), 0);
+  return buf;
+}
+}  // namespace collective
+}  // namespace xgboost
+
+#undef xgboost_CHECK_SYS_CALL
+#undef xgboost_IS_MINGW

include/xgboost/data.h

@@ -216,7 +216,7 @@ struct BatchParam {
   /*! \brief The GPU device to use. */
   int gpu_id {-1};
   /*! \brief Maximum number of bins per feature for histograms. */
-  int max_bin{0};
+  bst_bin_t max_bin{0};
   /*! \brief Hessian, used for sketching with future approx implementation. */
   common::Span<float> hess;
   /*! \brief Whether should DMatrix regenerate the batch.  Only used for GHistIndex. */
@@ -226,17 +226,17 @@ struct BatchParam {
 
   BatchParam() = default;
   // GPU Hist
-  BatchParam(int32_t device, int32_t max_bin)
+  BatchParam(int32_t device, bst_bin_t max_bin)
       : gpu_id{device}, max_bin{max_bin} {}
   // Hist
-  BatchParam(int32_t max_bin, double sparse_thresh)
+  BatchParam(bst_bin_t max_bin, double sparse_thresh)
       : max_bin{max_bin}, sparse_thresh{sparse_thresh} {}
   // Approx
   /**
    * \brief Get batch with sketch weighted by hessian.  The batch will be regenerated if
    *        the span is changed, so caller should keep the span for each iteration.
    */
-  BatchParam(int32_t max_bin, common::Span<float> hessian, bool regenerate)
+  BatchParam(bst_bin_t max_bin, common::Span<float> hessian, bool regenerate)
       : max_bin{max_bin}, hess{hessian}, regen{regenerate} {}
 
   bool operator!=(BatchParam const& other) const {
@@ -284,12 +284,17 @@ class SparsePage {
     return {offset.ConstHostSpan(), data.ConstHostSpan()};
   }
 
-
   /*! \brief constructor */
   SparsePage() {
     this->Clear();
   }
 
+  SparsePage(SparsePage const& that) = delete;
+  SparsePage(SparsePage&& that) = default;
+  SparsePage& operator=(SparsePage const& that) = delete;
+  SparsePage& operator=(SparsePage&& that) = default;
+  virtual ~SparsePage() = default;
+
   /*! \return Number of instances in the page. */
   inline size_t Size() const {
     return offset.Size() == 0 ? 0 : offset.Size() - 1;
@@ -358,6 +363,16 @@ class CSCPage: public SparsePage {
   explicit CSCPage(SparsePage page) : SparsePage(std::move(page)) {}
 };
 
+/**
+ * \brief Sparse page for exporting DMatrix. Same as SparsePage, just a different type to
+ *        prevent being used internally.
+ */
+class ExtSparsePage {
+ public:
+  std::shared_ptr<SparsePage const> page;
+  explicit ExtSparsePage(std::shared_ptr<SparsePage const> p) : page{std::move(p)} {}
+};
+
 class SortedCSCPage : public SparsePage {
  public:
   SortedCSCPage() : SparsePage() {}
@@ -559,6 +574,7 @@ class DMatrix {
    *
    * \param iter    External data iterator
    * \param proxy   A hanlde to ProxyDMatrix
+   * \param ref     Reference Quantile DMatrix.
    * \param reset   Callback for reset
    * \param next    Callback for next
    * \param missing Value that should be treated as missing.
@@ -567,13 +583,11 @@ class DMatrix {
    *
    * \return A created quantile based DMatrix.
    */
-  template <typename DataIterHandle, typename DMatrixHandle,
-            typename DataIterResetCallback, typename XGDMatrixCallbackNext>
-  static DMatrix *Create(DataIterHandle iter, DMatrixHandle proxy,
-                         DataIterResetCallback *reset,
-                         XGDMatrixCallbackNext *next, float missing,
-                         int nthread,
-                         int max_bin);
+  template <typename DataIterHandle, typename DMatrixHandle, typename DataIterResetCallback,
+            typename XGDMatrixCallbackNext>
+  static DMatrix* Create(DataIterHandle iter, DMatrixHandle proxy, std::shared_ptr<DMatrix> ref,
+                         DataIterResetCallback* reset, XGDMatrixCallbackNext* next, float missing,
+                         int nthread, bst_bin_t max_bin);
 
   /**
    * \brief Create an external memory DMatrix with callbacks.
@@ -611,8 +625,10 @@ class DMatrix {
   virtual BatchSet<SortedCSCPage> GetSortedColumnBatches() = 0;
   virtual BatchSet<EllpackPage> GetEllpackBatches(const BatchParam& param) = 0;
   virtual BatchSet<GHistIndexMatrix> GetGradientIndex(const BatchParam& param) = 0;
+  virtual BatchSet<ExtSparsePage> GetExtBatches(BatchParam const& param) = 0;
 
   virtual bool EllpackExists() const = 0;
+  virtual bool GHistIndexExists() const = 0;
   virtual bool SparsePageExists() const = 0;
 };
 
@@ -621,11 +637,16 @@ inline BatchSet<SparsePage> DMatrix::GetBatches() {
   return GetRowBatches();
 }
 
-template<>
+template <>
 inline bool DMatrix::PageExists<EllpackPage>() const {
   return this->EllpackExists();
 }
 
+template <>
+inline bool DMatrix::PageExists<GHistIndexMatrix>() const {
+  return this->GHistIndexExists();
+}
+
 template<>
 inline bool DMatrix::PageExists<SparsePage>() const {
   return this->SparsePageExists();
@@ -646,10 +667,15 @@ inline BatchSet<EllpackPage> DMatrix::GetBatches(const BatchParam& param) {
   return GetEllpackBatches(param);
 }
 
-template<>
+template <>
 inline BatchSet<GHistIndexMatrix> DMatrix::GetBatches(const BatchParam& param) {
   return GetGradientIndex(param);
 }
+
+template <>
+inline BatchSet<ExtSparsePage> DMatrix::GetBatches() {
+  return GetExtBatches(BatchParam{});
+}
 }  // namespace xgboost
 
 namespace dmlc {

include/xgboost/gbm.h

@@ -68,21 +68,18 @@ class GradientBooster : public Model, public Configurable {
    * \param layer_end   End of booster layer. 0 means do not limit trees.
    * \param out         Output gradient booster
    */
-  virtual void Slice(int32_t layer_begin, int32_t layer_end, int32_t step,
-                     GradientBooster *out, bool* out_of_bound) const {
+  virtual void Slice(int32_t /*layer_begin*/, int32_t /*layer_end*/, int32_t /*step*/,
+                     GradientBooster* /*out*/, bool* /*out_of_bound*/) const {
     LOG(FATAL) << "Slice is not supported by current booster.";
   }
-  /*!
-   * \brief whether the model allow lazy checkpoint
-   * return true if model is only updated in DoBoost
-   * after all Allreduce calls
-   */
-  virtual bool AllowLazyCheckPoint() const {
-    return false;
-  }
   /*! \brief Return number of boosted rounds.
    */
   virtual int32_t BoostedRounds() const = 0;
+  /**
+   * \brief Whether the model has already been trained. When tree booster is chosen, then
+   *        returns true when there are existing trees.
+   */
+  virtual bool ModelFitted() const = 0;
   /*!
    * \brief perform update to the model(boosting)
    * \param p_fmat feature matrix that provide access to features
@@ -90,9 +87,8 @@ class GradientBooster : public Model, public Configurable {
    * \param prediction The output prediction cache entry that needs to be updated.
    * the booster may change content of gpair
    */
-  virtual void DoBoost(DMatrix* p_fmat,
-                       HostDeviceVector<GradientPair>* in_gpair,
-                       PredictionCacheEntry*) = 0;
+  virtual void DoBoost(DMatrix* p_fmat, HostDeviceVector<GradientPair>* in_gpair,
+                       PredictionCacheEntry*, ObjFunction const* obj) = 0;
 
   /*!
    * \brief generate predictions for given feature matrix
@@ -112,15 +108,14 @@ class GradientBooster : public Model, public Configurable {
   /*!
    * \brief Inplace prediction.
    *
-   * \param           x                      A type erased data adapter.
+   * \param           p_fmat                 A proxy DMatrix that contains the data and related
+   *                                         meta info.
    * \param           missing                Missing value in the data.
    * \param [in,out]  out_preds              The output preds.
    * \param           layer_begin (Optional) Beginning of boosted tree layer used for prediction.
    * \param           layer_end   (Optional) End of booster layer. 0 means do not limit trees.
    */
-  virtual void InplacePredict(dmlc::any const &, std::shared_ptr<DMatrix>, float,
-                              PredictionCacheEntry*,
-                              uint32_t,
+  virtual void InplacePredict(std::shared_ptr<DMatrix>, float, PredictionCacheEntry*, uint32_t,
                               uint32_t) const {
     LOG(FATAL) << "Inplace predict is not supported by current booster.";
   }

include/xgboost/json.h

@@ -58,7 +58,9 @@ class Value {
   virtual Json& operator[](int ind);
 
   virtual bool operator==(Value const& rhs) const = 0;
+#if !defined(__APPLE__)
   virtual Value& operator=(Value const& rhs) = delete;
+#endif  // !defined(__APPLE__)
 
   std::string TypeStr() const;
 
@@ -185,13 +187,17 @@ using I32Array = JsonTypedArray<int32_t, Value::ValueKind::kI32Array>;
 using I64Array = JsonTypedArray<int64_t, Value::ValueKind::kI64Array>;
 
 class JsonObject : public Value {
-  std::map<std::string, Json> object_;
+ public:
+  using Map = std::map<std::string, Json, std::less<>>;
+
+ private:
+  Map object_;
 
  public:
   JsonObject() : Value(ValueKind::kObject) {}
-  JsonObject(std::map<std::string, Json>&& object) noexcept;  // NOLINT
+  JsonObject(Map&& object) noexcept;  // NOLINT
   JsonObject(JsonObject const& that) = delete;
-  JsonObject(JsonObject && that) noexcept;
+  JsonObject(JsonObject&& that) noexcept;
 
   void Save(JsonWriter* writer) const override;
 
@@ -199,15 +205,13 @@ class JsonObject : public Value {
   Json& operator[](int ind) override { return Value::operator[](ind); }
   Json& operator[](std::string const& key) override { return object_[key]; }
 
-  std::map<std::string, Json> const& GetObject() &&      { return object_; }
-  std::map<std::string, Json> const& GetObject() const & { return object_; }
-  std::map<std::string, Json> &      GetObject() &       { return object_; }
+  Map const& GetObject() && { return object_; }
+  Map const& GetObject() const& { return object_; }
+  Map& GetObject() & { return object_; }
 
   bool operator==(Value const& rhs) const override;
 
-  static bool IsClassOf(Value const* value) {
-    return value->Type() == ValueKind::kObject;
-  }
+  static bool IsClassOf(Value const* value) { return value->Type() == ValueKind::kObject; }
   ~JsonObject() override = default;
 };
 
@@ -557,16 +561,13 @@ std::vector<T> const& GetImpl(JsonTypedArray<T, kind> const& val) {
 }
 
 // Object
-template <typename T,
-          typename std::enable_if<
-            std::is_same<T, JsonObject>::value>::type* = nullptr>
-std::map<std::string, Json>& GetImpl(T& val) {  // NOLINT
+template <typename T, typename std::enable_if<std::is_same<T, JsonObject>::value>::type* = nullptr>
+JsonObject::Map& GetImpl(T& val) {  // NOLINT
   return val.GetObject();
 }
 template <typename T,
-          typename std::enable_if<
-            std::is_same<T, JsonObject const>::value>::type* = nullptr>
-std::map<std::string, Json> const& GetImpl(T& val) {  // NOLINT
+          typename std::enable_if<std::is_same<T, JsonObject const>::value>::type* = nullptr>
+JsonObject::Map const& GetImpl(T& val) {  // NOLINT
   return val.GetObject();
 }
 }  // namespace detail

include/xgboost/json_io.h

@@ -17,6 +17,22 @@
 #include <vector>
 
 namespace xgboost {
+namespace detail {
+// Whether char is signed is undefined, as a result we might or might not need
+// static_cast and std::to_string.
+template <typename Char, std::enable_if_t<std::is_signed<Char>::value>* = nullptr>
+std::string CharToStr(Char c) {
+  static_assert(std::is_same<Char, char>::value, "");
+  return std::string{c};
+}
+
+template <typename Char, std::enable_if_t<!std::is_signed<Char>::value>* = nullptr>
+std::string CharToStr(Char c) {
+  static_assert(std::is_same<Char, char>::value, "");
+  return (c <= static_cast<char>(127) ? std::string{c} : std::to_string(c));
+}
+}  // namespace detail
+
 /*
  * \brief A json reader, currently error checking and utf-8 is not fully supported.
  */
@@ -89,7 +105,7 @@ class JsonReader {
     } else if (got == 0) {
       msg += "\\0\"";
     } else {
-      msg += (got <= 127 ? std::string{got} : std::to_string(got)) + " \"";  // NOLINT
+      msg += detail::CharToStr(got) + " \"";
     }
     Error(msg);
   }

include/xgboost/learner.h

@@ -8,10 +8,9 @@
 #ifndef XGBOOST_LEARNER_H_
 #define XGBOOST_LEARNER_H_
 
-#include <dmlc/any.h>
 #include <xgboost/base.h>
 #include <xgboost/feature_map.h>
-#include <xgboost/generic_parameters.h>
+#include <xgboost/generic_parameters.h>  // Context
 #include <xgboost/host_device_vector.h>
 #include <xgboost/model.h>
 #include <xgboost/predictor.h>
@@ -139,21 +138,16 @@ class Learner : public Model, public Configurable, public dmlc::Serializable {
   /*!
    * \brief Inplace prediction.
    *
-   * \param          x           A type erased data adapter.
-   * \param          p_m         An optional Proxy DMatrix object storing meta info like
-   *                             base margin.  Can be nullptr.
+   * \param          p_fmat      A proxy DMatrix that contains the data and related meta info.
    * \param          type        Prediction type.
    * \param          missing     Missing value in the data.
    * \param [in,out] out_preds   Pointer to output prediction vector.
    * \param          layer_begin Beginning of boosted tree layer used for prediction.
    * \param          layer_end   End of booster layer. 0 means do not limit trees.
    */
-  virtual void InplacePredict(dmlc::any const &x,
-                              std::shared_ptr<DMatrix> p_m,
-                              PredictionType type,
-                              float missing,
-                              HostDeviceVector<bst_float> **out_preds,
-                              uint32_t layer_begin, uint32_t layer_end) = 0;
+  virtual void InplacePredict(std::shared_ptr<DMatrix> p_m, PredictionType type, float missing,
+                              HostDeviceVector<bst_float>** out_preds, uint32_t layer_begin,
+                              uint32_t layer_end) = 0;
 
   /*!
    * \brief Calculate feature score.  See doc in C API for outputs.
@@ -246,10 +240,6 @@ class Learner : public Model, public Configurable, public dmlc::Serializable {
    */
   virtual void GetFeatureTypes(std::vector<std::string>* ft) const = 0;
 
-  /*!
-   * \return whether the model allow lazy checkpoint in rabit.
-   */
-  bool AllowLazyCheckPoint() const;
   /*!
    * \brief Slice the model.
    *
@@ -283,7 +273,7 @@ class Learner : public Model, public Configurable, public dmlc::Serializable {
   /**
    * \brief Return the context object of this Booster.
    */
-  virtual GenericParameter const* Ctx() const = 0;
+  virtual Context const* Ctx() const = 0;
   /*!
    * \brief Get configuration arguments currently stored by the learner
    * \return Key-value pairs representing configuration arguments
@@ -298,7 +288,7 @@ class Learner : public Model, public Configurable, public dmlc::Serializable {
   /*! \brief The evaluation metrics used to evaluate the model. */
   std::vector<std::unique_ptr<Metric> > metrics_;
   /*! \brief Training parameter. */
-  GenericParameter generic_parameters_;
+  Context ctx_;
 };
 
 struct LearnerModelParamLegacy;
@@ -307,8 +297,14 @@ struct LearnerModelParamLegacy;
  * \brief Basic Model Parameters, used to describe the booster.
  */
 struct LearnerModelParam {
-  /* \brief global bias */
-  bst_float base_score { 0.5f };
+ private:
+  /**
+   * \brief Global bias, this is just a scalar value but can be extended to vector when we
+   *        support multi-class and multi-target.
+   */
+  linalg::Tensor<float, 1> base_score_;
+
+ public:
   /* \brief number of features  */
   uint32_t num_feature { 0 };
   /* \brief number of classes, if it is multi-class classification  */
@@ -319,9 +315,20 @@ struct LearnerModelParam {
   LearnerModelParam() = default;
   // As the old `LearnerModelParamLegacy` is still used by binary IO, we keep
   // this one as an immutable copy.
-  LearnerModelParam(LearnerModelParamLegacy const& user_param, float base_margin, ObjInfo t);
+  LearnerModelParam(Context const* ctx, LearnerModelParamLegacy const& user_param,
+                    linalg::Tensor<float, 1> base_margin, ObjInfo t);
+  LearnerModelParam(LearnerModelParamLegacy const& user_param, ObjInfo t);
+  LearnerModelParam(bst_feature_t n_features, linalg::Tensor<float, 1> base_margin,
+                    uint32_t n_groups)
+      : base_score_{std::move(base_margin)}, num_feature{n_features}, num_output_group{n_groups} {}
+
+  linalg::TensorView<float const, 1> BaseScore(Context const* ctx) const;
+  linalg::TensorView<float const, 1> BaseScore(int32_t device) const;
+
+  void Copy(LearnerModelParam const& that);
+
   /* \brief Whether this parameter is initialized with LearnerModelParamLegacy. */
-  bool Initialized() const { return num_feature != 0; }
+  bool Initialized() const { return num_feature != 0 && num_output_group != 0; }
 };
 
 }  // namespace xgboost

include/xgboost/linalg.h

@@ -8,6 +8,7 @@
 
 #include <dmlc/endian.h>
 #include <xgboost/base.h>
+#include <xgboost/generic_parameters.h>
 #include <xgboost/host_device_vector.h>
 #include <xgboost/json.h>
 #include <xgboost/span.h>
@@ -16,6 +17,7 @@
 #include <cassert>
 #include <limits>
 #include <string>
+#include <tuple>
 #include <type_traits>
 #include <utility>
 #include <vector>
@@ -213,6 +215,22 @@ LINALG_HD decltype(auto) constexpr Apply(Fn &&f, Tup &&t) {
   constexpr auto kSize = std::tuple_size<Tup>::value;
   return Apply(std::forward<Fn>(f), std::forward<Tup>(t), std::make_index_sequence<kSize>{});
 }
+
+/**
+ * C++ 17 conjunction
+ */
+template <class...>
+struct Conjunction : std::true_type {};
+template <class B1>
+struct Conjunction<B1> : B1 {};
+template <class B1, class... Bn>
+struct Conjunction<B1, Bn...> : std::conditional_t<bool(B1::value), Conjunction<Bn...>, B1> {};
+
+template <typename... Index>
+using IsAllIntegral = Conjunction<std::is_integral<std::remove_reference_t<Index>>...>;
+
+template <typename... Index>
+using EnableIfIntegral = std::enable_if_t<IsAllIntegral<Index...>::value>;
 }  // namespace detail
 
 /**
@@ -317,7 +335,8 @@ class TensorView {
   }
 
   template <size_t old_dim, size_t new_dim, int32_t D, typename Index>
-  LINALG_HD size_t MakeSliceDim(size_t new_shape[D], size_t new_stride[D], Index i) const {
+  LINALG_HD size_t MakeSliceDim(DMLC_ATTRIBUTE_UNUSED size_t new_shape[D],
+                                DMLC_ATTRIBUTE_UNUSED size_t new_stride[D], Index i) const {
     static_assert(old_dim < kDim, "");
     return stride_[old_dim] * i;
   }
@@ -405,7 +424,7 @@ class TensorView {
    *
    * \endcode
    */
-  template <typename... Index>
+  template <typename... Index, detail::EnableIfIntegral<Index...> * = nullptr>
   LINALG_HD T &operator()(Index &&...index) {
     static_assert(sizeof...(index) <= kDim, "Invalid index.");
     size_t offset = detail::Offset<0ul>(stride_, 0ul, std::forward<Index>(index)...);
@@ -415,7 +434,7 @@ class TensorView {
   /**
    * \brief Index the tensor to obtain a scalar value.
    */
-  template <typename... Index>
+  template <typename... Index, detail::EnableIfIntegral<Index...> * = nullptr>
   LINALG_HD T const &operator()(Index &&...index) const {
     static_assert(sizeof...(index) <= kDim, "Invalid index.");
     size_t offset = detail::Offset<0ul>(stride_, 0ul, std::forward<Index>(index)...);
@@ -655,7 +674,7 @@ class Tensor {
     }
     if (device >= 0) {
       data_.SetDevice(device);
-      data_.DevicePointer();  // Pull to device;
+      data_.ConstDevicePointer();  // Pull to device;
     }
     CHECK_EQ(data_.Size(), detail::CalcSize(shape_));
   }
@@ -670,9 +689,13 @@ class Tensor {
    * See \ref TensorView for parameters of this constructor.
    */
   template <typename I, int32_t D>
-  explicit Tensor(I const (&shape)[D], int32_t device) {
+  explicit Tensor(I const (&shape)[D], int32_t device)
+      : Tensor{common::Span<I const, D>{shape}, device} {}
+
+  template <typename I, size_t D>
+  explicit Tensor(common::Span<I const, D> shape, int32_t device) {
     // No device unroll as this is a host only function.
-    std::copy(shape, shape + D, shape_);
+    std::copy(shape.data(), shape.data() + D, shape_);
     for (auto i = D; i < kDim; ++i) {
       shape_[i] = 1;
     }
@@ -697,12 +720,29 @@ class Tensor {
   }
 
   template <typename I, int32_t D>
-  explicit Tensor(std::initializer_list<T> data, I const (&shape)[D], int32_t device) {
+  explicit Tensor(std::initializer_list<T> data, I const (&shape)[D],
+                  int32_t device = Context::kCpuId) {
     auto &h_vec = data_.HostVector();
     h_vec = data;
     // shape
     this->Initialize(shape, device);
   }
+  /**
+   * \brief Index operator. Not thread safe, should not be used in performance critical
+   *        region. For more efficient indexing, consider getting a view first.
+   */
+  template <typename... Index>
+  T &operator()(Index &&...idx) {
+    return this->HostView()(std::forward<Index>(idx)...);
+  }
+  /**
+   * \brief Index operator. Not thread safe, should not be used in performance critical
+   *        region. For more efficient indexing, consider getting a view first.
+   */
+  template <typename... Index>
+  T const &operator()(Index &&...idx) const {
+    return this->HostView()(std::forward<Index>(idx)...);
+  }
 
   /**
    * \brief Get a \ref TensorView for this tensor.
@@ -756,7 +796,7 @@ class Tensor {
    *
    *    If the total size is changed, then data in this tensor is no longer valid.
    */
-  template <typename... S>
+  template <typename... S, detail::EnableIfIntegral<S...> * = nullptr>
   void Reshape(S &&...s) {
     static_assert(sizeof...(S) <= kDim, "Invalid shape.");
     detail::ReshapeImpl<0>(shape_, std::forward<S>(s)...);
@@ -772,15 +812,20 @@ class Tensor {
    *
    *    If the total size is changed, then data in this tensor is no longer valid.
    */
-  template <int32_t D>
-  void Reshape(size_t (&shape)[D]) {
+  template <size_t D>
+  void Reshape(common::Span<size_t const, D> shape) {
     static_assert(D <= kDim, "Invalid shape.");
-    std::copy(shape, shape + D, this->shape_);
+    std::copy(shape.data(), shape.data() + D, this->shape_);
     std::fill(shape_ + D, shape_ + kDim, 1);
     auto n = detail::CalcSize(shape_);
     data_.Resize(n);
   }
 
+  template <size_t D>
+  void Reshape(size_t (&shape)[D]) {
+    this->Reshape(common::Span<size_t const, D>{shape});
+  }
+
   /**
    * \brief Set device ordinal for this tensor.
    */

include/xgboost/metric.h

@@ -57,12 +57,8 @@ class Metric : public Configurable {
    * \brief evaluate a specific metric
    * \param preds prediction
    * \param info information, including label etc.
-   * \param distributed whether a call to Allreduce is needed to gather
-   *        the average statistics across all the node,
-   *        this is only supported by some metrics
    */
-  virtual double Eval(const HostDeviceVector<bst_float> &preds,
-                      const MetaInfo &info, bool distributed) = 0;
+  virtual double Eval(const HostDeviceVector<bst_float>& preds, const MetaInfo& info) = 0;
   /*! \return name of metric */
   virtual const char* Name() const = 0;
   /*! \brief virtual destructor */

include/xgboost/objective.h

@@ -1,5 +1,5 @@
 /*!
- * Copyright 2014-2019 by Contributors
+ * Copyright 2014-2022 by Contributors
  * \file objective.h
  * \brief interface of objective function used by xgboost.
  * \author Tianqi Chen, Kailong Chen
@@ -22,10 +22,15 @@
 
 namespace xgboost {
 
+class RegTree;
+
 /*! \brief interface of objective function */
 class ObjFunction : public Configurable {
  protected:
-  GenericParameter const* ctx_;
+  Context const* ctx_;
+
+ public:
+  static constexpr float DefaultBaseScore() { return 0.5f; }
 
  public:
   /*! \brief virtual destructor */
@@ -73,6 +78,13 @@ class ObjFunction : public Configurable {
   virtual bst_float ProbToMargin(bst_float base_score) const {
     return base_score;
   }
+  /**
+   * \brief Make initialize estimation of prediction.
+   *
+   * \param info MetaInfo that contains label.
+   * \param base_score Output estimation.
+   */
+  virtual void InitEstimation(MetaInfo const& info, linalg::Tensor<float, 1>* base_score) const;
   /*!
    * \brief Return task of this objective.
    */
@@ -88,6 +100,24 @@ class ObjFunction : public Configurable {
     return 1;
   }
 
+  /**
+   * \brief Update the leaf values after a tree is built. Needed for objectives with 0
+   *        hessian.
+   *
+   *   Note that the leaf update is not well defined for distributed training as XGBoost
+   *   computes only an average of quantile between workers. This breaks when some leaf
+   *   have no sample assigned in a local worker.
+   *
+   * \param position The leaf index for each rows.
+   * \param info MetaInfo providing labels and weights.
+   * \param prediction Model prediction after transformation.
+   * \param p_tree Tree that needs to be updated.
+   */
+  virtual void UpdateTreeLeaf(HostDeviceVector<bst_node_t> const& /*position*/,
+                              MetaInfo const& /*info*/,
+                              HostDeviceVector<float> const& /*prediction*/,
+                              RegTree* /*p_tree*/) const {}
+
   /*!
    * \brief Create an objective function according to name.
    * \param tparam Generic parameters.