From 5f3c811e84e09f68c647fafb5662340a625a8901 Mon Sep 17 00:00:00 2001
From: Philip Hyunsu Cho <chohyu01@cs.washington.edu>
Date: Fri, 31 Jul 2020 10:05:04 -0700
Subject: [PATCH] [CI] Assign larger /dev/shm to NCCL (#5966)

* [CI] Assign larger /dev/shm to NCCL

* Use 10.2 artifact to run multi-GPU Python tests

* Add CUDA 10.0 -> 11.0 cross-version test; remove CUDA 10.0 target
---
 Jenkinsfile | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d800dbc89..30b683830 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -89,10 +89,10 @@ pipeline {
         script {
           parallel ([
             'test-python-cpu': { TestPythonCPU() },
-            'test-python-gpu-cuda10.0': { TestPythonGPU(host_cuda_version: '10.0') },
             'test-python-gpu-cuda10.2': { TestPythonGPU(host_cuda_version: '10.2') },
+            'test-python-gpu-cuda11.0-cross': { TestPythonGPU(artifact_cuda_version: '10.0', host_cuda_version: '11.0') },
             'test-python-gpu-cuda11.0': { TestPythonGPU(artifact_cuda_version: '11.0', host_cuda_version: '11.0') },
-            'test-python-mgpu-cuda10.2': { TestPythonGPU(host_cuda_version: '10.2', multi_gpu: true) },
+            'test-python-mgpu-cuda10.2': { TestPythonGPU(artifact_cuda_version: '10.2', host_cuda_version: '10.2', multi_gpu: true) },
             'test-cpp-gpu-cuda10.2': { TestCppGPU(artifact_cuda_version: '10.2', host_cuda_version: '10.2') },
             'test-cpp-gpu-cuda11.0': { TestCppGPU(artifact_cuda_version: '11.0', host_cuda_version: '11.0') },
             'test-jvm-jdk8-cuda10.0': { CrossTestJVMwithJDKGPU(artifact_cuda_version: '10.0', host_cuda_version: '10.0') },
@@ -368,8 +368,10 @@ def TestPythonGPU(args) {
     def docker_args = "--build-arg CUDA_VERSION=${args.host_cuda_version}"
     if (args.multi_gpu) {
       echo "Using multiple GPUs"
+      // Allocate extra space in /dev/shm to enable NCCL
+      def docker_extra_params = "CI_DOCKER_EXTRA_PARAMS_INIT='--shm-size=4g'"
       sh """
-      ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/test_python.sh mgpu
+      ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/test_python.sh mgpu
       """
     } else {
       echo "Using a single GPU"