From 33ee7d161547cac40c58c8b598f74c620a6527bf Mon Sep 17 00:00:00 2001 From: Michal Malohlava Date: Wed, 12 Jul 2017 22:51:47 -0700 Subject: [PATCH] [BUILD] Dockerfile and Jenkinsfile revisited (#2514) Includes: - Dockerfile changes - Dockerfile clean up - Fix execution privileges of files used from Dockerfile. - New Dockerfile entrypoint to replace with_user script - Defined a placeholders for CPU testing (script and Dockerfile) - Jenkinsfile - Jenkins file milestone defined - Single source code checkout and propagation via stash/unstash - Bash needs to be explicitly used in launching make build, since we need access to environment - Jenkinsfile build factory for cmake and make style of jobs - Archivation of artifacts (*.so, *.whl, *.egg) produced by cmake build Missing: - CPU testing - Python3 env build and testing --- Jenkinsfile | 156 ++++++++++++++---- tests/ci_build/Dockerfile.cpu | 49 ++++++ tests/ci_build/Dockerfile.gpu | 57 +++++-- ...{build_gpu_cmake.sh => build_via_cmake.sh} | 3 +- tests/ci_build/build_via_make.sh | 4 + tests/ci_build/ci_build.sh | 54 +++--- tests/ci_build/entrypoint.sh | 43 +++++ tests/ci_build/test_cpu.sh | 4 + tests/ci_build/test_gpu.sh | 5 - tests/ci_build/with_the_same_user | 30 ---- 10 files changed, 300 insertions(+), 105 deletions(-) create mode 100644 tests/ci_build/Dockerfile.cpu rename tests/ci_build/{build_gpu_cmake.sh => build_via_cmake.sh} (58%) mode change 100644 => 100755 create mode 100755 tests/ci_build/build_via_make.sh mode change 100644 => 100755 tests/ci_build/ci_build.sh create mode 100755 tests/ci_build/entrypoint.sh create mode 100755 tests/ci_build/test_cpu.sh mode change 100644 => 100755 tests/ci_build/test_gpu.sh delete mode 100644 tests/ci_build/with_the_same_user diff --git a/Jenkinsfile b/Jenkinsfile index b63ffc91c..31cbba16a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,15 +1,54 @@ +#!/usr/bin/groovy // -*- mode: groovy -*- // Jenkins pipeline // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ -// command to start a docker container -docker_run = 'tests/ci_build/ci_build.sh' +// Command to run command inside a docker container +dockerRun = 'tests/ci_build/ci_build.sh' -// timeout in minutes -max_time = 60 +def buildMatrix = [ + [ "enabled": true, "os" : "linux", "withGpu": true, "withOmp": true, "pythonVersion": "2.7" ], + [ "enabled": true, "os" : "linux", "withGpu": false, "withOmp": true, "pythonVersion": "2.7" ], + [ "enabled": false, "os" : "osx", "withGpu": false, "withOmp": false, "pythonVersion": "2.7" ], +] + +pipeline { + // Each stage specify its own agent + agent none + + // Setup common job properties + options { + ansiColor('xterm') + timestamps() + timeout(time: 120, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '10')) + } + + // Build stages + stages { + stage('Get sources') { + agent any + steps { + checkoutSrcs() + stash name: 'srcs', excludes: '.git/' + milestone label: 'Sources ready', ordinal: 1 + } + } + stage('Build & Test') { + steps { + script { + parallel (buildMatrix.findAll{it['enabled']}.collectEntries{ c -> + def buildName = getBuildName(c) + buildFactory(buildName, c) + }) + } + } + } + } +} // initialize source codes -def init_git() { +def checkoutSrcs() { retry(5) { try { timeout(time: 2, unit: 'MINUTES') { @@ -23,33 +62,90 @@ def init_git() { } } -stage('Build') { - node('GPU' && 'linux') { - ws('workspace/xgboost/build-gpu-cmake') { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - sh "${docker_run} gpu tests/ci_build/build_gpu_cmake.sh" - } - } - } - node('GPU' && 'linux') { - ws('workspace/xgboost/build-gpu-make') { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - sh "${docker_run} gpu make PLUGIN_UPDATER_GPU=ON" - } - } +/** + * Creates cmake and make builds + */ +def buildFactory(buildName, conf) { + def os = conf["os"] + def nodeReq = conf["withGpu"] ? "${os} && gpu" : "${os}" + def dockerTarget = conf["withGpu"] ? "gpu" : "cpu" + [ ("cmake_${buildName}") : { buildPlatformCmake("cmake_${buildName}", conf, nodeReq, dockerTarget) }, + ("make_${buildName}") : { buildPlatformMake("make_${buildName}", conf, nodeReq, dockerTarget) } + ] +} + +/** + * Build platform and test it via cmake. + */ +def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) { + def opts = cmakeOptions(conf) + // Destination dir for artifacts + def distDir = "dist/${buildName}" + // Build node - this is returned result + node(nodeReq) { + unstash name: 'srcs' + echo """ + |===== XGBoost CMake build ===== + | dockerTarget: ${dockerTarget} + | cmakeOpts : ${opts} + |========================= + """.stripMargin('|') + // Invoke command inside docker + sh """ + ${dockerRun} ${dockerTarget} tests/ci_build/build_via_cmake.sh ${opts} + ${dockerRun} ${dockerTarget} tests/ci_build/test_${dockerTarget}.sh + ${dockerRun} ${dockerTarget} bash -c "cd python-package; python setup.py bdist_wheel" + rm -rf "${distDir}"; mkdir -p "${distDir}/py" + cp xgboost "${distDir}" + cp -r lib "${distDir}" + cp -r python-package/dist "${distDir}/py" + """ + archiveArtifacts artifacts: "${distDir}/**/*.*", allowEmptyArchive: true } } - -stage('Unit Test') { - node('GPU' && 'linux') { - ws('workspace/xgboost/unit-test') { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - sh "${docker_run} gpu tests/ci_build/test_gpu.ssh" - } - } +/** + * Build platform via make + */ +def buildPlatformMake(buildName, conf, nodeReq, dockerTarget) { + def opts = makeOptions(conf) + // Destination dir for artifacts + def distDir = "dist/${buildName}" + // Build node + node(nodeReq) { + unstash name: 'srcs' + echo """ + |===== XGBoost Make build ===== + | dockerTarget: ${dockerTarget} + | makeOpts : ${opts} + |========================= + """.stripMargin('|') + // Invoke command inside docker + sh """ + ${dockerRun} ${dockerTarget} tests/ci_build/build_via_make.sh ${opts} + """ } } + +def makeOptions(conf) { + return ([ + conf["withGpu"] ? 'PLUGIN_UPDATER_GPU=ON' : 'PLUGIN_UPDATER_GPU=OFF', + conf["withOmp"] ? 'USE_OPENMP=1' : 'USE_OPENMP=0'] + ).join(" ") +} + + +def cmakeOptions(conf) { + return ([ + conf["withGpu"] ? '-DPLUGIN_UPDATER_GPU:BOOL=ON' : '', + conf["withOmp"] ? '-DOPEN_MP:BOOL=ON' : ''] + ).join(" ") +} + +def getBuildName(conf) { + def gpuLabel = conf['withGpu'] ? "_gpu" : "_cpu" + def ompLabel = conf['withOmp'] ? "_omp" : "" + def pyLabel = "_py${conf['pythonVersion']}" + return "${conf['os']}${gpuLabel}${ompLabel}${pyLabel}" +} + diff --git a/tests/ci_build/Dockerfile.cpu b/tests/ci_build/Dockerfile.cpu new file mode 100644 index 000000000..c3891535e --- /dev/null +++ b/tests/ci_build/Dockerfile.cpu @@ -0,0 +1,49 @@ +FROM ubuntu:14.04 + +# Environment +ENV DEBIAN_FRONTEND noninteractive + +# Install all basic requirements +RUN \ + apt-get update -q -y && \ + apt-get -y dist-upgrade && \ + apt-get -y --no-install-recommends install \ + build-essential \ + wget \ + unzip \ + gfortran \ + # BLAS + libatlas-base-dev \ + # Python 2 + python-setuptools \ + python-pip \ + python-dev \ + && \ + # CMake + wget http://www.cmake.org/files/v3.5/cmake-3.5.2.tar.gz && \ + tar -xvzf cmake-3.5.2.tar.gz && \ + cd cmake-3.5.2/ && ./configure && make && make install && cd ../ && \ + # Clean up + rm -rf cmake-3.5.2/ && rm -rf cmake-3.5.2.tar.gz && \ + apt-get clean && \ + rm -rf /var/cache/apt/* + + +# Install Python packages +RUN pip install numpy nose scipy scikit-learn wheel + +ENV GOSU_VERSION 1.10 + +# Install lightweight sudo (not bound to TTY) +RUN set -ex; \ + dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" && \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" && \ + chmod +x /usr/local/bin/gosu && \ + gosu nobody true + +# Default entry-point to use if running locally +# It will preserve attributes of created files +COPY entrypoint.sh /scripts/ + +WORKDIR /workspace +ENTRYPOINT ["/scripts/entrypoint.sh"] diff --git a/tests/ci_build/Dockerfile.gpu b/tests/ci_build/Dockerfile.gpu index 4ca3a7b82..4893a9bb7 100644 --- a/tests/ci_build/Dockerfile.gpu +++ b/tests/ci_build/Dockerfile.gpu @@ -1,16 +1,49 @@ FROM nvidia/cuda:8.0-devel-ubuntu14.04 -RUN apt-get update && apt-get -y upgrade -# CMAKE -RUN sudo apt-get install -y build-essential -RUN apt-get install -y wget -RUN wget http://www.cmake.org/files/v3.5/cmake-3.5.2.tar.gz -RUN tar -xvzf cmake-3.5.2.tar.gz -RUN cd cmake-3.5.2/ && ./configure && make && sudo make install +# Environment +ENV DEBIAN_FRONTEND noninteractive -# BLAS -RUN apt-get install -y libatlas-base-dev +# Install all basic requirements +RUN \ + apt-get update -q -y && \ + apt-get -y dist-upgrade && \ + apt-get -y --no-install-recommends install \ + build-essential \ + wget \ + unzip \ + gfortran \ + # BLAS + libatlas-base-dev \ + # Python 2 + python-setuptools \ + python-pip \ + python-dev \ + && \ + # CMake + wget http://www.cmake.org/files/v3.5/cmake-3.5.2.tar.gz && \ + tar -xvzf cmake-3.5.2.tar.gz && \ + cd cmake-3.5.2/ && ./configure && make && make install && cd ../ && \ + # Clean up + rm -rf cmake-3.5.2/ && rm -rf cmake-3.5.2.tar.gz && \ + apt-get clean && \ + rm -rf /var/cache/apt/* -# PYTHON2 -RUN apt-get install -y python-setuptools python-pip python-dev unzip gfortran -RUN pip install numpy nose scipy scikit-learn + +# Install Python packages +RUN pip install numpy nose scipy scikit-learn wheel + +ENV GOSU_VERSION 1.10 + +# Install lightweight sudo (not bound to TTY) +RUN set -ex; \ + dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" && \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" && \ + chmod +x /usr/local/bin/gosu && \ + gosu nobody true + +# Default entry-point to use if running locally +# It will preserve attributes of created files +COPY entrypoint.sh /scripts/ + +WORKDIR /workspace +ENTRYPOINT ["/scripts/entrypoint.sh"] diff --git a/tests/ci_build/build_gpu_cmake.sh b/tests/ci_build/build_via_cmake.sh old mode 100644 new mode 100755 similarity index 58% rename from tests/ci_build/build_gpu_cmake.sh rename to tests/ci_build/build_via_cmake.sh index d8d0b9726..75c35316d --- a/tests/ci_build/build_gpu_cmake.sh +++ b/tests/ci_build/build_via_cmake.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash +make clean mkdir build cd build -cmake .. -DPLUGIN_UPDATER_GPU=ON +cmake .. "$@" make diff --git a/tests/ci_build/build_via_make.sh b/tests/ci_build/build_via_make.sh new file mode 100755 index 000000000..daef57044 --- /dev/null +++ b/tests/ci_build/build_via_make.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +make clean +make "$@" diff --git a/tests/ci_build/ci_build.sh b/tests/ci_build/ci_build.sh old mode 100644 new mode 100755 index 46a9d8880..e41230788 --- a/tests/ci_build/ci_build.sh +++ b/tests/ci_build/ci_build.sh @@ -83,29 +83,33 @@ DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | sed -e 's/=/_/g' -e 's/,/-/g') # Convert to all lower-case, as per requirement of Docker image names DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | tr '[:upper:]' '[:lower:]') -# skip with_the_same_user for non-linux -uname=`uname` -if [[ "$uname" == "Linux" ]]; then - PRE_COMMAND="tests/ci_build/with_the_same_user" -else - PRE_COMMAND="" +# Bash on Ubuntu on Windows +UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "") +# MSYS, Git Bash, etc. +MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "") + +if [[ -z "$UBUNTU_ON_WINDOWS" ]] && [[ -z "$MSYS" ]]; then + USER_IDS="-e CI_BUILD_UID=$( id -u ) -e CI_BUILD_GID=$( id -g ) -e CI_BUILD_USER=$( id -un ) -e CI_BUILD_GROUP=$( id -gn ) -e CI_BUILD_HOME=${WORKSPACE}" fi # Print arguments. -echo "WORKSPACE: ${WORKSPACE}" -echo "CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[@]}" -echo "COMMAND: ${COMMAND[@]}" -echo "CONTAINER_TYPE: ${CONTAINER_TYPE}" -echo "BUILD_TAG: ${BUILD_TAG}" -echo "NODE_NAME: ${NODE_NAME}" -echo "DOCKER CONTAINER NAME: ${DOCKER_IMG_NAME}" -echo "PRE_COMMAND: ${PRE_COMMAND}" -echo "" +cat < /etc/sudoers.d/90-nopasswd-sudo - -sudo -u "#${CI_BUILD_UID}" --preserve-env "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" \ -"HOME=${CI_BUILD_HOME}" ${COMMAND[@]}