[BUILD] Dockerfile and Jenkinsfile revisited (#2514)

Includes:
  - Dockerfile changes
    - Dockerfile clean up
    - Fix execution privileges of files used from Dockerfile.
    - New Dockerfile entrypoint to replace with_user script
    - Defined a placeholders for CPU testing (script and Dockerfile)
  - Jenkinsfile
    - Jenkins file milestone defined
    - Single source code checkout and propagation via stash/unstash
    - Bash needs to be explicitly used in launching make build, since we need
access to environment
    - Jenkinsfile build factory for cmake and make style of jobs
    - Archivation of artifacts (*.so, *.whl, *.egg) produced by cmake build

Missing:
  - CPU testing
  - Python3 env build and testing

Jenkinsfile

@@ -1,15 +1,54 @@
+#!/usr/bin/groovy
 // -*- mode: groovy -*-
 // Jenkins pipeline
 // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
 
-// command to start a docker container
+// Command to run command inside a docker container
-docker_run = 'tests/ci_build/ci_build.sh'
+dockerRun = 'tests/ci_build/ci_build.sh'
 
-// timeout in minutes
+def buildMatrix = [
-max_time = 60
+    [ "enabled": true,  "os" : "linux", "withGpu": true,  "withOmp": true, "pythonVersion": "2.7" ],
+    [ "enabled": true,  "os" : "linux", "withGpu": false, "withOmp": true, "pythonVersion": "2.7" ],
+    [ "enabled": false, "os" : "osx",   "withGpu": false, "withOmp": false, "pythonVersion": "2.7" ],
+]
+
+pipeline {
+    // Each stage specify its own agent
+    agent none
+
+    // Setup common job properties
+    options {
+        ansiColor('xterm')
+        timestamps()
+        timeout(time: 120, unit: 'MINUTES')
+        buildDiscarder(logRotator(numToKeepStr: '10'))
+    }
+
+    // Build stages
+    stages {
+        stage('Get sources') {
+            agent any
+            steps {
+                checkoutSrcs()
+                stash name: 'srcs', excludes: '.git/'
+                milestone label: 'Sources ready', ordinal: 1
+            }
+        }
+        stage('Build & Test') {
+            steps {
+                script {
+                    parallel (buildMatrix.findAll{it['enabled']}.collectEntries{ c ->
+                        def buildName = getBuildName(c)
+                        buildFactory(buildName, c)
+                    })
+                }
+            }
+        }
+    }
+}
 
 // initialize source codes
-def init_git() {
+def checkoutSrcs() {
   retry(5) {
     try {
       timeout(time: 2, unit: 'MINUTES') {
@@ -23,33 +62,90 @@ def init_git() {
   }
 }
 
-stage('Build') {
+/**
-    node('GPU' && 'linux') {
+ * Creates cmake and make builds
-      ws('workspace/xgboost/build-gpu-cmake') {
+ */
-        init_git()
+def buildFactory(buildName, conf) {
-  	timeout(time: max_time, unit: 'MINUTES') {
+    def os = conf["os"]
-      		sh "${docker_run} gpu tests/ci_build/build_gpu_cmake.sh"
+    def nodeReq = conf["withGpu"] ? "${os} && gpu" : "${os}"
-	}
+    def dockerTarget = conf["withGpu"] ? "gpu" : "cpu"
-      }
+    [ ("cmake_${buildName}") : { buildPlatformCmake("cmake_${buildName}", conf, nodeReq, dockerTarget) },
-    }
+      ("make_${buildName}") : { buildPlatformMake("make_${buildName}", conf, nodeReq, dockerTarget) }
-    node('GPU' && 'linux') {
+    ]
-      ws('workspace/xgboost/build-gpu-make') {
+}
-        init_git()
+
-  	timeout(time: max_time, unit: 'MINUTES') {
+/**
-      		sh "${docker_run} gpu make PLUGIN_UPDATER_GPU=ON"
+ * Build platform and test it via cmake.
-	}
+ */
-      }
+def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) {
+    def opts = cmakeOptions(conf)
+    // Destination dir for artifacts
+    def distDir = "dist/${buildName}"
+    // Build node - this is returned result
+    node(nodeReq) {
+        unstash name: 'srcs'
+        echo """
+        |===== XGBoost CMake build =====
+        |  dockerTarget: ${dockerTarget}
+        |  cmakeOpts   : ${opts}
+        |=========================
+        """.stripMargin('|')
+        // Invoke command inside docker
+        sh """
+        ${dockerRun} ${dockerTarget} tests/ci_build/build_via_cmake.sh ${opts}
+        ${dockerRun} ${dockerTarget} tests/ci_build/test_${dockerTarget}.sh
+        ${dockerRun} ${dockerTarget} bash -c "cd python-package; python setup.py bdist_wheel"
+        rm -rf "${distDir}"; mkdir -p "${distDir}/py"
+        cp xgboost "${distDir}"
+        cp -r lib "${distDir}"
+        cp -r python-package/dist "${distDir}/py"
+        """
+        archiveArtifacts artifacts: "${distDir}/**/*.*", allowEmptyArchive: true
     }
 }
 
-
+/**
-stage('Unit Test') {
+ * Build platform via make
-    node('GPU' && 'linux') {
+ */
-      ws('workspace/xgboost/unit-test') {
+def buildPlatformMake(buildName, conf, nodeReq, dockerTarget) {
-        init_git()
+    def opts = makeOptions(conf)
-  	timeout(time: max_time, unit: 'MINUTES') {
+    // Destination dir for artifacts
-      		sh "${docker_run} gpu tests/ci_build/test_gpu.ssh"
+    def distDir = "dist/${buildName}"
-	}
+    // Build node
-      }
+    node(nodeReq) {
+        unstash name: 'srcs'
+        echo """
+        |===== XGBoost Make build =====
+        |  dockerTarget: ${dockerTarget}
+        |  makeOpts    : ${opts}
+        |=========================
+        """.stripMargin('|')
+        // Invoke command inside docker
+        sh """
+        ${dockerRun} ${dockerTarget} tests/ci_build/build_via_make.sh ${opts}
+        """
     }
 }
+
+def makeOptions(conf) {
+    return ([
+        conf["withGpu"] ? 'PLUGIN_UPDATER_GPU=ON' : 'PLUGIN_UPDATER_GPU=OFF',
+        conf["withOmp"] ? 'USE_OPENMP=1' : 'USE_OPENMP=0']
+        ).join(" ")
+}
+
+
+def cmakeOptions(conf) {
+    return ([
+        conf["withGpu"] ? '-DPLUGIN_UPDATER_GPU:BOOL=ON' : '',
+        conf["withOmp"] ? '-DOPEN_MP:BOOL=ON' : '']
+        ).join(" ")
+}
+
+def getBuildName(conf) {
+    def gpuLabel = conf['withGpu'] ? "_gpu" : "_cpu"
+    def ompLabel = conf['withOmp'] ? "_omp" : ""
+    def pyLabel = "_py${conf['pythonVersion']}"
+    return "${conf['os']}${gpuLabel}${ompLabel}${pyLabel}"
+}
+

tests/ci_build/Dockerfile.cpu

@@ -0,0 +1,49 @@
+FROM ubuntu:14.04
+
+# Environment
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install all basic requirements
+RUN \
+    apt-get update -q -y && \
+    apt-get -y dist-upgrade && \
+    apt-get -y --no-install-recommends  install \
+        build-essential \
+        wget \
+        unzip \
+        gfortran \
+        # BLAS
+        libatlas-base-dev \
+        # Python 2
+        python-setuptools \
+        python-pip \
+        python-dev \
+    && \
+    # CMake
+    wget http://www.cmake.org/files/v3.5/cmake-3.5.2.tar.gz && \
+    tar -xvzf cmake-3.5.2.tar.gz && \
+    cd cmake-3.5.2/ && ./configure && make && make install && cd ../ && \
+    # Clean up
+    rm -rf cmake-3.5.2/ && rm -rf cmake-3.5.2.tar.gz && \
+    apt-get clean && \
+    rm -rf /var/cache/apt/*
+
+
+# Install Python packages
+RUN pip install numpy nose scipy scikit-learn wheel
+
+ENV GOSU_VERSION 1.10
+
+# Install lightweight sudo (not bound to TTY)
+RUN set -ex; \
+    dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" && \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]

tests/ci_build/Dockerfile.gpu

@@ -1,16 +1,49 @@
 FROM nvidia/cuda:8.0-devel-ubuntu14.04
 
-RUN apt-get update && apt-get -y upgrade
+# Environment
-# CMAKE
+ENV DEBIAN_FRONTEND noninteractive
-RUN sudo apt-get install -y build-essential
-RUN apt-get install -y wget
-RUN wget http://www.cmake.org/files/v3.5/cmake-3.5.2.tar.gz 
-RUN tar -xvzf cmake-3.5.2.tar.gz 
-RUN cd cmake-3.5.2/ && ./configure && make && sudo make install
 
-# BLAS
+# Install all basic requirements
-RUN apt-get install -y libatlas-base-dev
+RUN \
+    apt-get update -q -y && \
+    apt-get -y dist-upgrade && \
+    apt-get -y --no-install-recommends  install \
+        build-essential \
+        wget \
+        unzip \
+        gfortran \
+        # BLAS
+        libatlas-base-dev \
+        # Python 2
+        python-setuptools \
+        python-pip \
+        python-dev \
+    && \
+    # CMake
+    wget http://www.cmake.org/files/v3.5/cmake-3.5.2.tar.gz && \
+    tar -xvzf cmake-3.5.2.tar.gz && \
+    cd cmake-3.5.2/ && ./configure && make && make install && cd ../ && \
+    # Clean up
+    rm -rf cmake-3.5.2/ && rm -rf cmake-3.5.2.tar.gz && \
+    apt-get clean && \
+    rm -rf /var/cache/apt/*
 
-# PYTHON2
+
-RUN apt-get install -y python-setuptools python-pip python-dev unzip gfortran
+# Install Python packages
-RUN pip install numpy nose scipy scikit-learn
+RUN pip install numpy nose scipy scikit-learn wheel
+
+ENV GOSU_VERSION 1.10
+
+# Install lightweight sudo (not bound to TTY)
+RUN set -ex; \
+    dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" && \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]

tests/ci_build/build_via_cmake.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
+make clean
 mkdir build
 cd build
-cmake .. -DPLUGIN_UPDATER_GPU=ON
+cmake .. "$@"
 make

tests/ci_build/build_via_make.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+make clean
+make "$@"

tests/ci_build/ci_build.sh

@@ -83,29 +83,33 @@ DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | sed -e 's/=/_/g' -e 's/,/-/g')
 # Convert to all lower-case, as per requirement of Docker image names
 DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | tr '[:upper:]' '[:lower:]')
 
-# skip with_the_same_user for non-linux
+# Bash on Ubuntu on Windows
-uname=`uname`
+UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "")
-if [[ "$uname" == "Linux" ]]; then
+# MSYS, Git Bash, etc.
-    PRE_COMMAND="tests/ci_build/with_the_same_user"
+MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "")
-else
+
-    PRE_COMMAND=""
+if [[ -z "$UBUNTU_ON_WINDOWS" ]] && [[ -z "$MSYS" ]]; then
+    USER_IDS="-e CI_BUILD_UID=$( id -u ) -e CI_BUILD_GID=$( id -g ) -e CI_BUILD_USER=$( id -un ) -e CI_BUILD_GROUP=$( id -gn ) -e CI_BUILD_HOME=${WORKSPACE}"
 fi
 
 # Print arguments.
-echo "WORKSPACE: ${WORKSPACE}"
+cat <<EOF
-echo "CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[@]}"
+   WORKSPACE: ${WORKSPACE}
-echo "COMMAND: ${COMMAND[@]}"
+   CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[*]}
-echo "CONTAINER_TYPE: ${CONTAINER_TYPE}"
+   COMMAND: ${COMMAND[*]}
-echo "BUILD_TAG: ${BUILD_TAG}"
+   CONTAINER_TYPE: ${CONTAINER_TYPE}
-echo "NODE_NAME: ${NODE_NAME}"
+   BUILD_TAG: ${BUILD_TAG}
-echo "DOCKER CONTAINER NAME: ${DOCKER_IMG_NAME}"
+   NODE_NAME: ${NODE_NAME}
-echo "PRE_COMMAND: ${PRE_COMMAND}"
+   DOCKER CONTAINER NAME: ${DOCKER_IMG_NAME}
-echo ""
+   USER_IDS: ${USER_IDS}
+EOF
 
 
 # Build the docker container.
 echo "Building container (${DOCKER_IMG_NAME})..."
-docker build -t ${DOCKER_IMG_NAME} \
+# --pull should be default
+docker build \
+    -t "${DOCKER_IMG_NAME}" \
     -f "${DOCKERFILE_PATH}" "${DOCKER_CONTEXT_PATH}"
 
 # Check docker build status
@@ -116,20 +120,16 @@ fi
 
 
 # Run the command inside the container.
-echo "Running '${COMMAND[@]}' inside ${DOCKER_IMG_NAME}..."
+echo "Running '${COMMAND[*]}' inside ${DOCKER_IMG_NAME}..."
 
 # By default we cleanup - remove the container once it finish running (--rm)
 # and share the PID namespace (--pid=host) so the process inside does not have
 # pid 1 and SIGKILL is propagated to the process inside (jenkins can kill it).
 ${DOCKER_BINARY} run --rm --pid=host \
-    -v ${WORKSPACE}:/workspace \
+    -v "${WORKSPACE}":/workspace \
     -w /workspace \
-    -e "CI_BUILD_HOME=${WORKSPACE}" \
+    ${USER_IDS} \
-    -e "CI_BUILD_USER=$(id -u -n)" \
+    "${CI_DOCKER_EXTRA_PARAMS[@]}" \
-    -e "CI_BUILD_UID=$(id -u)" \
+    "${DOCKER_IMG_NAME}" \
-    -e "CI_BUILD_GROUP=$(id -g -n)" \
+    "${COMMAND[@]}"
-    -e "CI_BUILD_GID=$(id -g)" \
+
-    ${CI_DOCKER_EXTRA_PARAMS[@]} \
-    ${DOCKER_IMG_NAME} \
-    ${PRE_COMMAND} \
-    ${COMMAND[@]}

tests/ci_build/entrypoint.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# This script is a wrapper creating the same user inside container as the one
+# running the ci_build.sh outside the container. It also set the home directory
+# for the user inside container to match the same absolute path as the workspace
+# outside of container.  Do not run this manually. It does not make sense. It is
+# intended to be called by ci_build.sh only.
+
+set -e
+
+COMMAND=("$@")
+
+if ! touch /this_is_writable_file_system; then
+  echo "You can't write to your filesystem!"
+  echo "If you are in Docker you should check you do not have too many images" \
+      "with too many files in them. Docker has some issue with it."
+  exit 1
+else
+  rm /this_is_writable_file_system
+fi
+
+if [[ -n $CI_BUILD_UID ]] && [[ -n $CI_BUILD_GID ]]; then
+    groupadd -o -g "${CI_BUILD_GID}" "${CI_BUILD_GROUP}"
+    useradd -o -m -g "${CI_BUILD_GID}" -u "${CI_BUILD_UID}" \
+        "${CI_BUILD_USER}"
+    export HOME="/home/${CI_BUILD_USER}"
+    shopt -s dotglob
+    cp -r /root/* "$HOME/"
+    chown -R "${CI_BUILD_UID}:${CI_BUILD_GID}" "$HOME"
+
+    # Allows project-specific customization
+    if [[ -e "/workspace/.pre_entry.sh" ]]; then
+        gosu "${CI_BUILD_UID}:${CI_BUILD_GID}" /workspace/.pre_entry.sh
+    fi
+
+    # Enable passwordless sudo capabilities for the user
+    chown root:"${CI_BUILD_GID}" "$(which gosu)"
+    chmod +s "$(which gosu)"; sync
+
+    exec gosu "${CI_BUILD_UID}:${CI_BUILD_GID}" "${COMMAND[@]}"
+else
+    exec "${COMMAND[@]}"
+fi

tests/ci_build/test_cpu.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+true
+

tests/ci_build/test_gpu.sh

@@ -1,10 +1,5 @@
 #!/usr/bin/env bash
 
-mkdir build
-cd build
-cmake .. -DPLUGIN_UPDATER_GPU=ON
-make
-cd ..
 cd python-package
 python setup.py install --user
 cd ../plugin/updater_gpu

tests/ci_build/with_the_same_user

@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-
-# This script is a wrapper creating the same user inside container as the one
-# running the ci_build.sh outside the container. It also set the home directory
-# for the user inside container to match the same absolute path as the workspace
-# outside of container.  Do not run this manually. It does not make sense. It is
-# intended to be called by ci_build.sh only.
-
-set -e
-
-COMMAND=("$@")
-
-if ! touch /this_is_writable_file_system; then
-  echo "You can't write to your filesystem!"
-  echo "If you are in Docker you should check you do not have too many images" \
-      "with too many files in them. Docker has some issue with it."
-  exit 1
-else
-  rm /this_is_writable_file_system
-fi
-
-getent group "${CI_BUILD_GID}" || addgroup --gid "${CI_BUILD_GID}" "${CI_BUILD_GROUP}"
-getent passwd "${CI_BUILD_UID}" || adduser --gid "${CI_BUILD_GID}" --uid "${CI_BUILD_UID}" \
-    --gecos "${CI_BUILD_USER} (generated by with_the_same_user script)" \
-    --disabled-password --home "${CI_BUILD_HOME}" --quiet "${CI_BUILD_USER}"
-usermod -a -G sudo "${CI_BUILD_USER}"
-echo "${CI_BUILD_USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-nopasswd-sudo
-
-sudo -u "#${CI_BUILD_UID}" --preserve-env "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" \
-"HOME=${CI_BUILD_HOME}" ${COMMAND[@]}