Add multi-GPU unit test environment (#3741)

* Add multi-GPU unit test environment

* Better assertion message

* Temporarily disable failing test

* Distinguish between multi-GPU and single-GPU CPP tests

* Consolidate Python tests. Use attributes to distinguish multi-GPU Python tests from single-CPU counterparts

tests/ci_build/jenkins_tools.Groovy

@@ -26,7 +26,7 @@ def checkoutSrcs() {
  */
 def buildFactory(buildName, conf, restricted, build_func) {
     def os = conf["os"]
-    def device = conf["withGpu"] ? "gpu" : "cpu"
+    def device = conf["withGpu"] ? (conf["multiGpu"] ? "mgpu" : "gpu") : "cpu"
     def restricted_flag = restricted ? "restricted" : "unrestricted"
     def nodeReq = "${os} && ${device} && ${restricted_flag}"
     def dockerTarget = conf["withGpu"] ? "gpu" : "cpu"
@@ -43,7 +43,7 @@ def cmakeOptions(conf) {
 }
 
 def getBuildName(conf) {
-    def gpuLabel = conf['withGpu'] ? ("_cuda" + conf['cudaVersion'] + (conf['withNccl'] ? "_nccl" : "_nonccl")) : "_cpu"
+    def gpuLabel = conf['withGpu'] ? ( (conf['multiGpu'] ? "_mgpu" : "") + "_cuda" + conf['cudaVersion'] + (conf['withNccl'] ? "_nccl" : "_nonccl")) : "_cpu"
     def ompLabel = conf['withOmp'] ? "_omp" : ""
     def pyLabel = "_py${conf['pythonVersion']}"
     return "${conf['os']}${gpuLabel}${ompLabel}${pyLabel}"

tests/ci_build/test_gpu.sh

@@ -4,6 +4,6 @@ set -e
 cd python-package
 python setup.py install --user
 cd ..
-python -m nose -v --attr='!slow' tests/python-gpu/
-./testxgboost
+python -m nose -v --eval-attr='(not slow) and (not mgpu)' tests/python-gpu/
+./testxgboost --gtest_filter=-*.MGPU_*
 

tests/ci_build/test_mgpu.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -e
+
+cd python-package
+python setup.py install --user
+cd ..
+python -m nose -v --eval-attr='(not slow) and mgpu' tests/python-gpu/
+./testxgboost --gtest_filter=*.MGPU_*