[CI] Pack R package tarball with pre-built xgboost.so (with GPU support) (#6827)
* Add scripts for packaging R package with GPU-enabled libxgboost.so * [CI] Automatically build R package tarball * Add comments * Don't build tarball for pull requests * Update the installation doc
This commit is contained in:
Philip Hyunsu Cho
committed by
GitHub
GitHub
parent
f294c4e023
commit
ea7a6a0321
112
tests/ci_build/Dockerfile.gpu_build_r_centos6
Normal file
112
tests/ci_build/Dockerfile.gpu_build_r_centos6
Normal file
@@ -0,0 +1,112 @@
|
||||
ARG CUDA_VERSION_ARG
|
||||
FROM nvidia/cuda:$CUDA_VERSION_ARG-devel-centos6
|
||||
ARG CUDA_VERSION_ARG
|
||||
|
||||
# Environment
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
ENV DEVTOOLSET_URL_ROOT http://vault.centos.org/6.9/sclo/x86_64/rh/devtoolset-4/
|
||||
|
||||
COPY CentOS-Base.repo /etc/yum.repos.d/
|
||||
|
||||
# Install all basic requirements
|
||||
RUN \
|
||||
yum install -y epel-release && \
|
||||
yum -y update && \
|
||||
yum install -y tar unzip wget xz git patchelf readline-devel libX11-devel libXt-devel \
|
||||
xorg-x11-server-devel openssl-devel texlive-* && \
|
||||
yum install -y $DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-5.3.1-6.1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-gfortran-5.3.1-6.1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-libquadmath-devel-5.3.1-6.1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-c++-5.3.1-6.1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-binutils-2.25.1-8.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-runtime-4.1-3.sc1.el6.x86_64.rpm \
|
||||
$DEVTOOLSET_URL_ROOT/devtoolset-4-libstdc++-devel-5.3.1-6.1.el6.x86_64.rpm
|
||||
|
||||
ENV PATH=/opt/python/bin:/usr/local/ninja:/opt/software/packages/bin:/opt/R/3.3.0/bin:$PATH
|
||||
ENV LD_LIBRARY_PATH=/opt/software/packages/lib:/opt/R/3.3.0/lib64:$LD_LIBRARY_PATH
|
||||
ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
|
||||
ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
|
||||
ENV F77=/opt/rh/devtoolset-4/root/usr/bin/gfortran
|
||||
|
||||
# A few packages have to be built from the source because CentOS 6 is a very old distribution and
|
||||
# the system packages are not sufficiently up-to-date to build R 3.3.0. We'll want to update to
|
||||
# CentOS 7 after the 1.4.0 release. Tracking issue: dmlc/xgboost#6791.
|
||||
#
|
||||
# Why choose an old Linux distro? This is so that the resulting xgboost.so is compatible with a
|
||||
# wide range of Linux OSes currently in operation. See https://www.python.org/dev/peps/pep-0571/
|
||||
RUN \
|
||||
wget https://zlib.net/fossils/zlib-1.2.5.tar.gz && \
|
||||
wget https://sourceware.org/pub/bzip2/bzip2-1.0.6.tar.gz && \
|
||||
wget http://tukaani.org/xz/xz-5.2.2.tar.gz && \
|
||||
wget https://ftp.pcre.org/pub/pcre/pcre-8.40.tar.gz && \
|
||||
wget https://www.openssl.org/source/old/1.0.0/openssl-1.0.0k.tar.gz && \
|
||||
wget --no-check-certificate https://curl.se/download/curl-7.47.1.tar.gz && \
|
||||
tar xf zlib-1.2.5.tar.gz && \
|
||||
tar xf bzip2-1.0.6.tar.gz && \
|
||||
tar xf xz-5.2.2.tar.gz && \
|
||||
tar xf pcre-8.40.tar.gz && \
|
||||
tar xf openssl-1.0.0k.tar.gz && \
|
||||
tar xf curl-7.47.1.tar.gz && \
|
||||
cd zlib-1.2.5 && \
|
||||
./configure --prefix=/opt/software/packages && \
|
||||
make -j$(nproc) && \
|
||||
make install && \
|
||||
cd ../bzip2-1.0.6 && \
|
||||
sed -i 's/CFLAGS=-Wall/CFLAGS=-fPIC -Wall/g' Makefile && \
|
||||
make -f Makefile-libbz2_so && \
|
||||
make clean && \
|
||||
make -j$(nproc) && \
|
||||
make -n install PREFIX=/opt/software/packages && \
|
||||
make install PREFIX=/opt/software/packages && \
|
||||
cd ../xz-5.2.2 && \
|
||||
./configure --prefix=/opt/software/packages && \
|
||||
make -j$(nproc) && \
|
||||
make install && \
|
||||
cd ../pcre-8.40 && \
|
||||
./configure --enable-utf8 --prefix=/opt/software/packages && \
|
||||
make -j$(nproc) && \
|
||||
make install && \
|
||||
cd ../curl-7.47.1 && \
|
||||
./configure --prefix=/opt/software/packages --with-ssl && \
|
||||
make -j$(nproc) && \
|
||||
make install && \
|
||||
export CFLAGS="-I/opt/software/packages/include" && \
|
||||
export LDFLAGS="-L/opt/software/packages/lib" && \
|
||||
cd .. && \
|
||||
# R 3.3.0
|
||||
wget -nv -nc https://cran.r-project.org/src/base/R-3/R-3.3.0.tar.gz && \
|
||||
tar xf R-3.3.0.tar.gz && \
|
||||
cd R-3.3.0 && \
|
||||
./configure --prefix=/opt/R/3.3.0 --enable-R-shlib && \
|
||||
make -j$(nproc) && \
|
||||
make install && \
|
||||
# Python
|
||||
wget -nv -nc -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
|
||||
bash Miniconda3.sh -b -p /opt/python && \
|
||||
/opt/python/bin/python -m pip install auditwheel && \
|
||||
# CMake
|
||||
wget -nv -nc https://cmake.org/files/v3.13/cmake-3.13.0-Linux-x86_64.sh --no-check-certificate && \
|
||||
bash cmake-3.13.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
|
||||
# Ninja
|
||||
mkdir -p /usr/local && \
|
||||
cd /usr/local/ && \
|
||||
wget -nv -nc https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz --no-check-certificate && \
|
||||
tar xf v1.10.0.tar.gz && mv ninja-1.10.0 ninja && rm -v v1.10.0.tar.gz && \
|
||||
cd ninja && \
|
||||
/opt/python/bin/python ./configure.py --bootstrap
|
||||
|
||||
ENV GOSU_VERSION 1.10
|
||||
|
||||
# Install lightweight sudo (not bound to TTY)
|
||||
RUN set -ex; \
|
||||
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
|
||||
chmod +x /usr/local/bin/gosu && \
|
||||
gosu nobody true
|
||||
|
||||
# Default entry-point to use if running locally
|
||||
# It will preserve attributes of created files
|
||||
COPY entrypoint.sh /scripts/
|
||||
|
||||
WORKDIR /workspace
|
||||
ENTRYPOINT ["/scripts/entrypoint.sh"]
|
||||
33
tests/ci_build/build_r_pkg_with_cuda.sh
Executable file
33
tests/ci_build/build_r_pkg_with_cuda.sh
Executable file
@@ -0,0 +1,33 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
set -x
|
||||
|
||||
if [ "$#" -ne 1 ]
|
||||
then
|
||||
echo "Build the R package tarball with CUDA code. Usage: $0 [commit hash]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
commit_hash="$1"
|
||||
|
||||
make Rpack
|
||||
mv xgboost/ xgboost_rpack/
|
||||
|
||||
mkdir build
|
||||
cd build
|
||||
cmake .. -GNinja -DUSE_CUDA=ON -DR_LIB=ON
|
||||
ninja
|
||||
cd ..
|
||||
|
||||
rm xgboost
|
||||
# This super wacky hack is found in cmake/RPackageInstall.cmake.in and
|
||||
# cmake/RPackageInstallTargetSetup.cmake. This hack lets us bypass the normal build process of R
|
||||
# and have R use xgboost.so that we've already built.
|
||||
rm -v xgboost_rpack/configure
|
||||
rm -rfv xgboost_rpack/src
|
||||
mkdir -p xgboost_rpack/src
|
||||
cp -v lib/xgboost.so xgboost_rpack/src/
|
||||
echo 'all:' > xgboost_rpack/src/Makefile
|
||||
echo 'all:' > xgboost_rpack/src/Makefile.win
|
||||
mv xgboost_rpack/ xgboost/
|
||||
tar cvzf xgboost_r_gpu_linux_${commit_hash}.tar.gz xgboost/
|
||||
Reference in New Issue
Block a user