Revamp the rabit implementation. (#10112)

This PR replaces the original RABIT implementation with a new one, which has already been partially merged into XGBoost. The new one features:
- Federated learning for both CPU and GPU.
- NCCL.
- More data types.
- A unified interface for all the underlying implementations.
- Improved timeout handling for both tracker and workers.
- Exhausted tests with metrics (fixed a couple of bugs along the way).
- A reusable tracker for Python and JVM packages.

tests/ci_build/Dockerfile.jvm_cross

@@ -1,6 +1,6 @@
 FROM ubuntu:18.04
 ARG JDK_VERSION=8
-ARG SPARK_VERSION=3.0.0
+ARG SPARK_VERSION=3.4.0
 
 # Environment
 ENV DEBIAN_FRONTEND noninteractive

tests/ci_build/build_jvm_packages.sh

@@ -18,7 +18,6 @@ fi
 
 rm -rf build/
 cd jvm-packages
-export RABIT_MOCK=ON
 
 if [ "x$gpu_arch" != "x" ]; then
   export GPU_ARCH_FLAG=$gpu_arch

tests/ci_build/build_mock_cmake.sh

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-rm -rf build
-mkdir build
-cd build
-cmake -DRABIT_MOCK=ON -DCMAKE_VERBOSE_MAKEFILE=ON ..
-make clean
-make -j$(nproc)
-cd ..

tests/ci_build/test_r_package.py

@@ -53,7 +53,6 @@ def pack_rpackage() -> Path:
     # rabit
     rabit = Path("rabit")
     os.mkdir(dest / "src" / rabit)
-    shutil.copytree(rabit / "src", dest / "src" / "rabit" / "src")
     shutil.copytree(rabit / "include", dest / "src" / "rabit" / "include")
     # dmlc-core
     dmlc_core = Path("dmlc-core")