Add cuda 8/9.1 centos 6 builds, test GPU wheel on CPU only container. (#3309)

* Add cuda 8/9.1 centos 6 builds, test GPU wheel on CPU only container.

* Add Google test

Jenkinsfile

@@ -7,9 +7,9 @@
 dockerRun = 'tests/ci_build/ci_build.sh'
 
 def buildMatrix = [
-    [ "enabled": true,  "os" : "linux", "withGpu": true,  "withOmp": true, "pythonVersion": "2.7" ],
-    [ "enabled": false,  "os" : "linux", "withGpu": false, "withOmp": true, "pythonVersion": "2.7" ],
-    [ "enabled": false, "os" : "osx",   "withGpu": false, "withOmp": false, "pythonVersion": "2.7" ],
+    [ "enabled": true,  "os" : "linux", "withGpu": true, "withNccl": true,  "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "9.1" ],
+    [ "enabled": true,  "os" : "linux", "withGpu": true, "withNccl": true,  "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "8.0" ],
+    [ "enabled": false,  "os" : "linux", "withGpu": false, "withNccl": false, "withOmp": true, "pythonVersion": "2.7", "cudaVersion": ""  ],
 ]
 
 pipeline {
@@ -69,7 +69,7 @@ def buildFactory(buildName, conf) {
     def os = conf["os"]
     def nodeReq = conf["withGpu"] ? "${os} && gpu" : "${os}"
     def dockerTarget = conf["withGpu"] ? "gpu" : "cpu"
-    [ ("cmake_${buildName}") : { buildPlatformCmake("cmake_${buildName}", conf, nodeReq, dockerTarget) }
+    [ ("${buildName}") : { buildPlatformCmake("${buildName}", conf, nodeReq, dockerTarget) }
     ]
 }
 
@@ -80,6 +80,10 @@ def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) {
     def opts = cmakeOptions(conf)
     // Destination dir for artifacts
     def distDir = "dist/${buildName}"
+    def dockerArgs = ""
+    if(conf["withGpu"]){
+        dockerArgs = "--build-arg CUDA_VERSION=" + conf["cudaVersion"]
+    }
     // Build node - this is returned result
     node(nodeReq) {
         unstash name: 'srcs'
@@ -91,57 +95,33 @@ def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) {
         """.stripMargin('|')
         // Invoke command inside docker
         sh """
-        ${dockerRun} ${dockerTarget} tests/ci_build/build_via_cmake.sh ${opts}
-        ${dockerRun} ${dockerTarget} tests/ci_build/test_${dockerTarget}.sh
-        ${dockerRun} ${dockerTarget} bash -c "cd python-package; python setup.py bdist_wheel"
+        ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/build_via_cmake.sh ${opts}
+        ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/test_${dockerTarget}.sh
+        ${dockerRun} ${dockerTarget} ${dockerArgs} bash -c "cd python-package; python setup.py bdist_wheel"
         rm -rf "${distDir}"; mkdir -p "${distDir}/py"
         cp xgboost "${distDir}"
         cp -r lib "${distDir}"
         cp -r python-package/dist "${distDir}/py"
+        # Test the wheel for compatibility on a barebones CPU container
+        ${dockerRun} release ${dockerArgs} bash -c " \
+            auditwheel show xgboost-*-py2-none-any.whl
+            pip install --user python-package/dist/xgboost-*-py2-none-any.whl && \
+            python -m nose tests/python"
         """
         archiveArtifacts artifacts: "${distDir}/**/*.*", allowEmptyArchive: true
     }
 }
 
-/**
- * Build platform via make
- */
-def buildPlatformMake(buildName, conf, nodeReq, dockerTarget) {
-    def opts = makeOptions(conf)
-    // Destination dir for artifacts
-    def distDir = "dist/${buildName}"
-    // Build node
-    node(nodeReq) {
-        unstash name: 'srcs'
-        echo """
-        |===== XGBoost Make build =====
-        |  dockerTarget: ${dockerTarget}
-        |  makeOpts    : ${opts}
-        |=========================
-        """.stripMargin('|')
-        // Invoke command inside docker
-        sh """
-        ${dockerRun} ${dockerTarget} tests/ci_build/build_via_make.sh ${opts}
-        """
-    }
-}
-
-def makeOptions(conf) {
-    return ([
-        conf["withOmp"] ? 'USE_OPENMP=1' : 'USE_OPENMP=0']
-        ).join(" ")
-}
-
-
 def cmakeOptions(conf) {
     return ([
         conf["withGpu"] ? '-DUSE_CUDA=ON' : '-DUSE_CUDA=OFF',
+        conf["withNccl"] ? '-DUSE_NCCL=ON' : '-DUSE_NCCL=OFF',
         conf["withOmp"] ? '-DOPEN_MP:BOOL=ON' : '']
         ).join(" ")
 }
 
 def getBuildName(conf) {
-    def gpuLabel = conf['withGpu'] ? "_gpu" : "_cpu"
+    def gpuLabel = conf['withGpu'] ? "_cuda" + conf['cudaVersion'] : "_cpu"
     def ompLabel = conf['withOmp'] ? "_omp" : ""
     def pyLabel = "_py${conf['pythonVersion']}"
     return "${conf['os']}${gpuLabel}${ompLabel}${pyLabel}"

tests/ci_build/Dockerfile.cpu

@@ -1,49 +0,0 @@
-FROM ubuntu:14.04
-
-# Environment
-ENV DEBIAN_FRONTEND noninteractive
-
-# Install all basic requirements
-RUN \
-    apt-get update -q -y && \
-    apt-get -y dist-upgrade && \
-    apt-get -y --no-install-recommends  install \
-        build-essential \
-        wget \
-        unzip \
-        gfortran \
-        # BLAS
-        libatlas-base-dev \
-        # Python 2
-        python-setuptools \
-        python-pip \
-        python-dev \
-    && \
-    # CMake
-    wget http://www.cmake.org/files/v3.5/cmake-3.5.2.tar.gz && \
-    tar -xvzf cmake-3.5.2.tar.gz && \
-    cd cmake-3.5.2/ && ./configure && make && make install && cd ../ && \
-    # Clean up
-    rm -rf cmake-3.5.2/ && rm -rf cmake-3.5.2.tar.gz && \
-    apt-get clean && \
-    rm -rf /var/cache/apt/*
-
-
-# Install Python packages
-RUN pip install numpy nose scipy scikit-learn wheel
-
-ENV GOSU_VERSION 1.10
-
-# Install lightweight sudo (not bound to TTY)
-RUN set -ex; \
-    dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" && \
-    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" && \
-    chmod +x /usr/local/bin/gosu && \
-    gosu nobody true
-
-# Default entry-point to use if running locally
-# It will preserve attributes of created files
-COPY entrypoint.sh /scripts/
-
-WORKDIR /workspace
-ENTRYPOINT ["/scripts/entrypoint.sh"]

tests/ci_build/Dockerfile.gpu

@@ -1,43 +1,38 @@
-FROM nvidia/cuda:8.0-devel-ubuntu14.04
+ARG CUDA_VERSION
+FROM nvidia/cuda:$CUDA_VERSION-devel-centos6
 
 # Environment
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install all basic requirements
 RUN \
-    apt-get update -q -y && \
-    apt-get -y dist-upgrade && \
-    apt-get -y --no-install-recommends  install \
-        build-essential \
-        wget \
-        unzip \
-        gfortran \
-        # BLAS
-        libatlas-base-dev \
-        # Python 2
-        python-setuptools \
-        python-pip \
-        python-dev \
-    && \
+    yum -y update && \
+    yum install -y wget unzip && \
+    wget http://people.centos.org/tru/devtools-2/devtools-2.repo -O /etc/yum.repos.d/devtools-2.repo && \
+    yum install -y devtoolset-2-gcc devtoolset-2-binutils devtoolset-2-gcc-c++ && \
+    # Python
+    wget https://repo.continuum.io/miniconda/Miniconda2-4.3.27-Linux-x86_64.sh && \
+    bash Miniconda2-4.3.27-Linux-x86_64.sh -b -p /opt/python && \
     # CMake
     wget http://www.cmake.org/files/v3.5/cmake-3.5.2.tar.gz && \
     tar -xvzf cmake-3.5.2.tar.gz && \
     cd cmake-3.5.2/ && ./configure && make && make install && cd ../ && \
-    # Clean up
-    rm -rf cmake-3.5.2/ && rm -rf cmake-3.5.2.tar.gz && \
-    apt-get clean && \
-    rm -rf /var/cache/apt/*
+    rm -rf cmake-3.5.2/ && rm -rf cmake-3.5.2.tar.gz
 
+ENV PATH=/opt/python/bin:$PATH
+ENV CC=/opt/rh/devtoolset-2/root/usr/bin/gcc
+ENV CXX=/opt/rh/devtoolset-2/root/usr/bin/c++
+ENV CPP=/opt/rh/devtoolset-2/root/usr/bin/cpp
 
 # Install Python packages
-RUN pip install numpy nose scipy scikit-learn wheel
+RUN \
+    pip install numpy nose scipy scikit-learn wheel
 
 ENV GOSU_VERSION 1.10
 
 # Install lightweight sudo (not bound to TTY)
 RUN set -ex; \
-    dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" && \
-    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" && \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
     chmod +x /usr/local/bin/gosu && \
     gosu nobody true
 

tests/ci_build/Dockerfile.release

@@ -0,0 +1,34 @@
+FROM centos:6
+
+# Environment
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install all basic requirements
+RUN \
+    yum -y update && \
+    yum install -y wget graphviz && \
+    # Python
+    wget https://repo.continuum.io/miniconda/Miniconda2-4.3.27-Linux-x86_64.sh && \
+    bash Miniconda2-4.3.27-Linux-x86_64.sh -b -p /opt/python
+
+ENV PATH=/opt/python/bin:$PATH
+
+# Install Python packages
+RUN \
+    conda install numpy scipy pandas matplotlib nose scikit-learn && \
+    pip install nose wheel auditwheel graphviz
+
+ENV GOSU_VERSION 1.10
+
+# Install lightweight sudo (not bound to TTY)
+RUN set -ex; \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]

tests/ci_build/build_via_cmake.sh

@@ -1,7 +1,20 @@
 #!/usr/bin/env bash
+set -e
 
-make clean
+# Build gtest via cmake
+rm -rf gtest
+wget https://github.com/google/googletest/archive/release-1.7.0.zip
+unzip release-1.7.0.zip
+mv googletest-release-1.7.0 gtest && cd gtest
+cmake . && make 
+mkdir lib && mv libgtest.a lib
+cd ..
+rm -rf release-1.7.0.zip*
+
+rm -rf build
 mkdir build
 cd build
-cmake .. "$@"
-make
+cmake .. "$@" -DGOOGLE_TEST=ON -DGTEST_ROOT=../gtest
+make clean
+make -j
+cd ..

tests/ci_build/build_via_make.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-make clean
-make "$@"

tests/ci_build/ci_build.sh

@@ -37,6 +37,12 @@ if [[ "$1" == "-it" ]]; then
     shift 1
 fi
 
+if [[ "$1" == "--build-arg" ]]; then
+    CI_DOCKER_BUILD_ARG+="$1"
+    CI_DOCKER_BUILD_ARG+=" $2"
+    shift 2
+fi
+
 if [[ ! -f "${DOCKERFILE_PATH}" ]]; then
     echo "Invalid Dockerfile path: \"${DOCKERFILE_PATH}\""
     exit 1
@@ -76,6 +82,10 @@ WORKSPACE="${WORKSPACE:-${SCRIPT_DIR}/../../}"
 # Determine the docker image name
 DOCKER_IMG_NAME="xgb-ci.${CONTAINER_TYPE}"
 
+# Append cuda version if available
+CUDA_VERSION=$(echo "${CI_DOCKER_BUILD_ARG}" | grep CUDA_VERSION | egrep -o '[0-9]*\.[0-9]*')
+DOCKER_IMG_NAME=$DOCKER_IMG_NAME$CUDA_VERSION 
+
 # Under Jenkins matrix build, the build tag may contain characters such as
 # commas (,) and equal signs (=), which are not valid inside docker image names.
 DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | sed -e 's/=/_/g' -e 's/,/-/g')
@@ -96,6 +106,7 @@ fi
 cat <<EOF
    WORKSPACE: ${WORKSPACE}
    CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[*]}
+   CI_DOCKER_BUILD_ARG: ${CI_DOCKER_BUILD_ARG}
    COMMAND: ${COMMAND[*]}
    CONTAINER_TYPE: ${CONTAINER_TYPE}
    BUILD_TAG: ${BUILD_TAG}
@@ -108,7 +119,12 @@ EOF
 # Build the docker container.
 echo "Building container (${DOCKER_IMG_NAME})..."
 # --pull should be default
+echo "docker build \
+    ${CI_DOCKER_BUILD_ARG} \
+    -t ${DOCKER_IMG_NAME} \
+    -f ${DOCKERFILE_PATH} ${DOCKER_CONTEXT_PATH}"
 docker build \
+    ${CI_DOCKER_BUILD_ARG} \
     -t "${DOCKER_IMG_NAME}" \
     -f "${DOCKERFILE_PATH}" "${DOCKER_CONTEXT_PATH}"
 

tests/ci_build/test_cpu.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-true
-

tests/ci_build/test_gpu.sh

@@ -3,5 +3,6 @@
 cd python-package
 python setup.py install --user
 cd ..
-python -m nose tests/python-gpu
+python -m nose --attr='!slow' tests/python-gpu/
+./testxgboost