Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[BUILD] Dockerfile and Jenkinsfile revisited (#2514)

Browse Source 
Includes:
  - Dockerfile changes
    - Dockerfile clean up
    - Fix execution privileges of files used from Dockerfile.
    - New Dockerfile entrypoint to replace with_user script
    - Defined a placeholders for CPU testing (script and Dockerfile)
  - Jenkinsfile
    - Jenkins file milestone defined
    - Single source code checkout and propagation via stash/unstash
    - Bash needs to be explicitly used in launching make build, since we need
access to environment
    - Jenkinsfile build factory for cmake and make style of jobs
    - Archivation of artifacts (*.so, *.whl, *.egg) produced by cmake build

Missing:
  - CPU testing
  - Python3 env build and testing
This commit is contained in:
Michal Malohlava 2017-07-12 22:51:47 -07:00 committed by Rory Mitchell
parent 66874f5777
commit 33ee7d1615
10 changed files with 300 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

150
Jenkinsfile vendored
View File

@ -1,15 +1,54 @@
#!/usr/bin/groovy
// -*- mode: groovy -*-
// Jenkins pipeline
// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
// command to start a docker container
docker_run = 'tests/ci_build/ci_build.sh'
// Command to run command inside a docker container
dockerRun = 'tests/ci_build/ci_build.sh'
// timeout in minutes
max_time = 60
def buildMatrix = [
[ "enabled": true, "os" : "linux", "withGpu": true, "withOmp": true, "pythonVersion": "2.7" ],
[ "enabled": true, "os" : "linux", "withGpu": false, "withOmp": true, "pythonVersion": "2.7" ],
[ "enabled": false, "os" : "osx", "withGpu": false, "withOmp": false, "pythonVersion": "2.7" ],
]
pipeline {
// Each stage specify its own agent
agent none
// Setup common job properties
options {
ansiColor('xterm')
timestamps()
timeout(time: 120, unit: 'MINUTES')
buildDiscarder(logRotator(numToKeepStr: '10'))
}
// Build stages
stages {
stage('Get sources') {
agent any
steps {
checkoutSrcs()
stash name: 'srcs', excludes: '.git/'
milestone label: 'Sources ready', ordinal: 1
}
}
stage('Build & Test') {
steps {
script {
parallel (buildMatrix.findAll{it['enabled']}.collectEntries{ c ->
def buildName = getBuildName(c)
buildFactory(buildName, c)
})
}
}
}
}
}
// initialize source codes
def init_git() {
def checkoutSrcs() {
retry(5) {
try {
timeout(time: 2, unit: 'MINUTES') {
@ -23,33 +62,90 @@ def init_git() {
}
}
stage('Build') {
node('GPU' && 'linux') {
ws('workspace/xgboost/build-gpu-cmake') {
init_git()
timeout(time: max_time, unit: 'MINUTES') {
sh "${docker_run} gpu tests/ci_build/build_gpu_cmake.sh"
}
}
}
node('GPU' && 'linux') {
ws('workspace/xgboost/build-gpu-make') {
init_git()
timeout(time: max_time, unit: 'MINUTES') {
sh "${docker_run} gpu make PLUGIN_UPDATER_GPU=ON"
}
/**
* Creates cmake and make builds
*/
def buildFactory(buildName, conf) {
def os = conf["os"]
def nodeReq = conf["withGpu"] ? "${os} && gpu" : "${os}"
def dockerTarget = conf["withGpu"] ? "gpu" : "cpu"
[ ("cmake_${buildName}") : { buildPlatformCmake("cmake_${buildName}", conf, nodeReq, dockerTarget) },
("make_${buildName}") : { buildPlatformMake("make_${buildName}", conf, nodeReq, dockerTarget) }
]
}
/**
* Build platform and test it via cmake.
*/
def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) {
def opts = cmakeOptions(conf)
// Destination dir for artifacts
def distDir = "dist/${buildName}"
// Build node - this is returned result
node(nodeReq) {
unstash name: 'srcs'
echo """
|===== XGBoost CMake build =====
| dockerTarget: ${dockerTarget}
| cmakeOpts : ${opts}
|=========================
""".stripMargin('|')
// Invoke command inside docker
sh """
${dockerRun} ${dockerTarget} tests/ci_build/build_via_cmake.sh ${opts}
${dockerRun} ${dockerTarget} tests/ci_build/test_${dockerTarget}.sh
${dockerRun} ${dockerTarget} bash -c "cd python-package; python setup.py bdist_wheel"
rm -rf "${distDir}"; mkdir -p "${distDir}/py"
cp xgboost "${distDir}"
cp -r lib "${distDir}"
cp -r python-package/dist "${distDir}/py"
"""
archiveArtifacts artifacts: "${distDir}/**/*.*", allowEmptyArchive: true
}
}
/**
* Build platform via make
*/
def buildPlatformMake(buildName, conf, nodeReq, dockerTarget) {
def opts = makeOptions(conf)
// Destination dir for artifacts
def distDir = "dist/${buildName}"
// Build node
node(nodeReq) {
unstash name: 'srcs'
echo """
|===== XGBoost Make build =====
| dockerTarget: ${dockerTarget}
| makeOpts : ${opts}
|=========================
""".stripMargin('|')
// Invoke command inside docker
sh """
${dockerRun} ${dockerTarget} tests/ci_build/build_via_make.sh ${opts}
"""
}
}
stage('Unit Test') {
node('GPU' && 'linux') {
ws('workspace/xgboost/unit-test') {
init_git()
timeout(time: max_time, unit: 'MINUTES') {
sh "${docker_run} gpu tests/ci_build/test_gpu.ssh"
}
def makeOptions(conf) {
return ([
conf["withGpu"] ? 'PLUGIN_UPDATER_GPU=ON' : 'PLUGIN_UPDATER_GPU=OFF',
conf["withOmp"] ? 'USE_OPENMP=1' : 'USE_OPENMP=0']
).join(" ")
}
def cmakeOptions(conf) {
return ([
conf["withGpu"] ? '-DPLUGIN_UPDATER_GPU:BOOL=ON' : '',
conf["withOmp"] ? '-DOPEN_MP:BOOL=ON' : '']
).join(" ")
}
def getBuildName(conf) {
def gpuLabel = conf['withGpu'] ? "_gpu" : "_cpu"
def ompLabel = conf['withOmp'] ? "_omp" : ""
def pyLabel = "_py${conf['pythonVersion']}"
return "${conf['os']}${gpuLabel}${ompLabel}${pyLabel}"
}

49
tests/ci_build/Dockerfile.cpu Normal file
View File

@ -0,0 +1,49 @@
FROM ubuntu:14.04
# Environment
ENV DEBIAN_FRONTEND noninteractive
# Install all basic requirements
RUN \
apt-get update -q -y && \
apt-get -y dist-upgrade && \
apt-get -y --no-install-recommends install \
build-essential \
wget \
unzip \
gfortran \
# BLAS
libatlas-base-dev \
# Python 2
python-setuptools \
python-pip \
python-dev \
&& \
# CMake
wget http://www.cmake.org/files/v3.5/cmake-3.5.2.tar.gz && \
tar -xvzf cmake-3.5.2.tar.gz && \
cd cmake-3.5.2/ && ./configure && make && make install && cd ../ && \
# Clean up
rm -rf cmake-3.5.2/ && rm -rf cmake-3.5.2.tar.gz && \
apt-get clean && \
rm -rf /var/cache/apt/*
# Install Python packages
RUN pip install numpy nose scipy scikit-learn wheel
ENV GOSU_VERSION 1.10
# Install lightweight sudo (not bound to TTY)
RUN set -ex; \
dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" && \
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" && \
chmod +x /usr/local/bin/gosu && \
gosu nobody true
# Default entry-point to use if running locally
# It will preserve attributes of created files
COPY entrypoint.sh /scripts/
WORKDIR /workspace
ENTRYPOINT ["/scripts/entrypoint.sh"]

55
tests/ci_build/Dockerfile.gpu
View File

@ -1,16 +1,49 @@
FROM nvidia/cuda:8.0-devel-ubuntu14.04
RUN apt-get update && apt-get -y upgrade
# CMAKE
RUN sudo apt-get install -y build-essential
RUN apt-get install -y wget
RUN wget http://www.cmake.org/files/v3.5/cmake-3.5.2.tar.gz
RUN tar -xvzf cmake-3.5.2.tar.gz
RUN cd cmake-3.5.2/ && ./configure && make && sudo make install
# Environment
ENV DEBIAN_FRONTEND noninteractive
# Install all basic requirements
RUN \
apt-get update -q -y && \
apt-get -y dist-upgrade && \
apt-get -y --no-install-recommends install \
build-essential \
wget \
unzip \
gfortran \
# BLAS
RUN apt-get install -y libatlas-base-dev
libatlas-base-dev \
# Python 2
python-setuptools \
python-pip \
python-dev \
&& \
# CMake
wget http://www.cmake.org/files/v3.5/cmake-3.5.2.tar.gz && \
tar -xvzf cmake-3.5.2.tar.gz && \
cd cmake-3.5.2/ && ./configure && make && make install && cd ../ && \
# Clean up
rm -rf cmake-3.5.2/ && rm -rf cmake-3.5.2.tar.gz && \
apt-get clean && \
rm -rf /var/cache/apt/*
# PYTHON2
RUN apt-get install -y python-setuptools python-pip python-dev unzip gfortran
RUN pip install numpy nose scipy scikit-learn
# Install Python packages
RUN pip install numpy nose scipy scikit-learn wheel
ENV GOSU_VERSION 1.10
# Install lightweight sudo (not bound to TTY)
RUN set -ex; \
dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" && \
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" && \
chmod +x /usr/local/bin/gosu && \
gosu nobody true
# Default entry-point to use if running locally
# It will preserve attributes of created files
COPY entrypoint.sh /scripts/
WORKDIR /workspace
ENTRYPOINT ["/scripts/entrypoint.sh"]

3
tests/ci_build/build_gpu_cmake.sh → tests/ci_build/build_via_cmake.sh Normal file → Executable file
View File

@ -1,6 +1,7 @@
#!/usr/bin/env bash
make clean
mkdir build
cd build
cmake .. -DPLUGIN_UPDATER_GPU=ON
cmake .. "$@"
make

4
tests/ci_build/build_via_make.sh Executable file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
make clean
make "$@"

54
tests/ci_build/ci_build.sh Normal file → Executable file
View File

@ -83,29 +83,33 @@ DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | sed -e 's/=/_/g' -e 's/,/-/g')
# Convert to all lower-case, as per requirement of Docker image names
DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | tr '[:upper:]' '[:lower:]')
# skip with_the_same_user for non-linux
uname=`uname`
if [[ "$uname" == "Linux" ]]; then
PRE_COMMAND="tests/ci_build/with_the_same_user"
else
PRE_COMMAND=""
# Bash on Ubuntu on Windows
UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "")
# MSYS, Git Bash, etc.
MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "")
if [[ -z "$UBUNTU_ON_WINDOWS" ]] && [[ -z "$MSYS" ]]; then
USER_IDS="-e CI_BUILD_UID=$( id -u ) -e CI_BUILD_GID=$( id -g ) -e CI_BUILD_USER=$( id -un ) -e CI_BUILD_GROUP=$( id -gn ) -e CI_BUILD_HOME=${WORKSPACE}"
fi
# Print arguments.
echo "WORKSPACE: ${WORKSPACE}"
echo "CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[@]}"
echo "COMMAND: ${COMMAND[@]}"
echo "CONTAINER_TYPE: ${CONTAINER_TYPE}"
echo "BUILD_TAG: ${BUILD_TAG}"
echo "NODE_NAME: ${NODE_NAME}"
echo "DOCKER CONTAINER NAME: ${DOCKER_IMG_NAME}"
echo "PRE_COMMAND: ${PRE_COMMAND}"
echo ""
cat <<EOF
WORKSPACE: ${WORKSPACE}
CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[*]}
COMMAND: ${COMMAND[*]}
CONTAINER_TYPE: ${CONTAINER_TYPE}
BUILD_TAG: ${BUILD_TAG}
NODE_NAME: ${NODE_NAME}
DOCKER CONTAINER NAME: ${DOCKER_IMG_NAME}
USER_IDS: ${USER_IDS}
EOF
# Build the docker container.
echo "Building container (${DOCKER_IMG_NAME})..."
docker build -t ${DOCKER_IMG_NAME} \
# --pull should be default
docker build \
-t "${DOCKER_IMG_NAME}" \
-f "${DOCKERFILE_PATH}" "${DOCKER_CONTEXT_PATH}"
# Check docker build status
@ -116,20 +120,16 @@ fi
# Run the command inside the container.
echo "Running '${COMMAND[@]}' inside ${DOCKER_IMG_NAME}..."
echo "Running '${COMMAND[*]}' inside ${DOCKER_IMG_NAME}..."
# By default we cleanup - remove the container once it finish running (--rm)
# and share the PID namespace (--pid=host) so the process inside does not have
# pid 1 and SIGKILL is propagated to the process inside (jenkins can kill it).
${DOCKER_BINARY} run --rm --pid=host \
-v ${WORKSPACE}:/workspace \
-v "${WORKSPACE}":/workspace \
-w /workspace \
-e "CI_BUILD_HOME=${WORKSPACE}" \
-e "CI_BUILD_USER=$(id -u -n)" \
-e "CI_BUILD_UID=$(id -u)" \
-e "CI_BUILD_GROUP=$(id -g -n)" \
-e "CI_BUILD_GID=$(id -g)" \
${CI_DOCKER_EXTRA_PARAMS[@]} \
${DOCKER_IMG_NAME} \
${PRE_COMMAND} \
${COMMAND[@]}
${USER_IDS} \
"${CI_DOCKER_EXTRA_PARAMS[@]}" \
"${DOCKER_IMG_NAME}" \
"${COMMAND[@]}"

43
tests/ci_build/entrypoint.sh Executable file
View File

@ -0,0 +1,43 @@
#!/usr/bin/env bash
# This script is a wrapper creating the same user inside container as the one
# running the ci_build.sh outside the container. It also set the home directory
# for the user inside container to match the same absolute path as the workspace
# outside of container. Do not run this manually. It does not make sense. It is
# intended to be called by ci_build.sh only.
set -e
COMMAND=("$@")
if ! touch /this_is_writable_file_system; then
echo "You can't write to your filesystem!"
echo "If you are in Docker you should check you do not have too many images" \
"with too many files in them. Docker has some issue with it."
exit 1
else
rm /this_is_writable_file_system
fi
if [[ -n $CI_BUILD_UID ]] && [[ -n $CI_BUILD_GID ]]; then
groupadd -o -g "${CI_BUILD_GID}" "${CI_BUILD_GROUP}"
useradd -o -m -g "${CI_BUILD_GID}" -u "${CI_BUILD_UID}" \
"${CI_BUILD_USER}"
export HOME="/home/${CI_BUILD_USER}"
shopt -s dotglob
cp -r /root/* "$HOME/"
chown -R "${CI_BUILD_UID}:${CI_BUILD_GID}" "$HOME"
# Allows project-specific customization
if [[ -e "/workspace/.pre_entry.sh" ]]; then
gosu "${CI_BUILD_UID}:${CI_BUILD_GID}" /workspace/.pre_entry.sh
fi
# Enable passwordless sudo capabilities for the user
chown root:"${CI_BUILD_GID}" "$(which gosu)"
chmod +s "$(which gosu)"; sync
exec gosu "${CI_BUILD_UID}:${CI_BUILD_GID}" "${COMMAND[@]}"
else
exec "${COMMAND[@]}"
fi

4
tests/ci_build/test_cpu.sh Executable file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
true

5
tests/ci_build/test_gpu.sh Normal file → Executable file
View File

@ -1,10 +1,5 @@
#!/usr/bin/env bash
mkdir build
cd build
cmake .. -DPLUGIN_UPDATER_GPU=ON
make
cd ..
cd python-package
python setup.py install --user
cd ../plugin/updater_gpu

30
tests/ci_build/with_the_same_user
View File

@ -1,30 +0,0 @@
#!/usr/bin/env bash
# This script is a wrapper creating the same user inside container as the one
# running the ci_build.sh outside the container. It also set the home directory
# for the user inside container to match the same absolute path as the workspace
# outside of container. Do not run this manually. It does not make sense. It is
# intended to be called by ci_build.sh only.
set -e
COMMAND=("$@")
if ! touch /this_is_writable_file_system; then
echo "You can't write to your filesystem!"
echo "If you are in Docker you should check you do not have too many images" \
"with too many files in them. Docker has some issue with it."
exit 1
else
rm /this_is_writable_file_system
fi
getent group "${CI_BUILD_GID}" || addgroup --gid "${CI_BUILD_GID}" "${CI_BUILD_GROUP}"
getent passwd "${CI_BUILD_UID}" || adduser --gid "${CI_BUILD_GID}" --uid "${CI_BUILD_UID}" \
--gecos "${CI_BUILD_USER} (generated by with_the_same_user script)" \
--disabled-password --home "${CI_BUILD_HOME}" --quiet "${CI_BUILD_USER}"
usermod -a -G sudo "${CI_BUILD_USER}"
echo "${CI_BUILD_USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-nopasswd-sudo
sudo -u "#${CI_BUILD_UID}" --preserve-env "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" \
"HOME=${CI_BUILD_HOME}" ${COMMAND[@]}