Merge rabit

rabit/scripts/mpi_build.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+if [ -f mpich/lib/libmpich.so ]; then
+  echo "libmpich.so found -- nothing to build."
+else
+  echo "Downloading mpich source."
+  wget http://www.mpich.org/static/downloads/3.2/mpich-3.2.tar.gz
+  tar xfz mpich-3.2.tar.gz
+  rm mpich-3.2.tar.gz*
+  echo "configuring and building mpich."
+  cd mpich-3.2
+  #CC=gcc CXX=g++ CFLAGS=-m64 CXXFLAGS=-m64 FFLAGS=-m64
+  ./configure \
+          --prefix=`pwd`/../mpich \
+          --enable-static=false \
+          --enable-alloca=true \
+          --disable-long-double \
+          --enable-threads=single \
+          --enable-fortran=no \
+          --enable-fast=all \
+          --enable-g=none \
+          --enable-timing=none \
+          --enable-cxx
+  make -j4
+  make install
+  cd -
+fi

rabit/scripts/travis_osx_install.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+set -x
+
+if [ ${TRAVIS_OS_NAME} != "osx" ]; then
+    exit 0
+fi

rabit/scripts/travis_runtest.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+conda activate python3
+conda --version
+python --version
+
+make -f test.mk RABIT_BUILD_DMLC=1 model_recover_10_10k || exit -1
+make -f test.mk RABIT_BUILD_DMLC=1 model_recover_10_10k_die_same  || exit -1
+make -f test.mk RABIT_BUILD_DMLC=1 model_recover_10_10k_die_hard || exit -1
+make -f test.mk RABIT_BUILD_DMLC=1 local_recover_10_10k || exit -1
+make -f test.mk RABIT_BUILD_DMLC=1 lazy_recover_10_10k_die_hard || exit -1
+make -f test.mk RABIT_BUILD_DMLC=1 lazy_recover_10_10k_die_same || exit -1
+make -f test.mk RABIT_BUILD_DMLC=1 ringallreduce_10_10k || exit -1
+make -f test.mk RABIT_BUILD_DMLC=1 pylocal_recover_10_10k || exit -1

rabit/scripts/travis_script.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# main script of travis
+if [ ${TASK} == "lint" ]; then
+    make lint RABIT_BUILD_DMLC=1 || exit -1
+fi
+
+if [ ${TASK} == "doc" ]; then
+    make doc 2>log.txt
+    (cat log.txt| grep -v ENABLE_PREPROCESSING |grep -v "unsupported tag" |grep warning) && exit -1
+fi
+
+# we should depreciate Makefile based build
+if [ ${TASK} == "build" ]; then
+    make all RABIT_BUILD_DMLC=1 || exit -1
+fi
+
+if [ ${TASK} == "mpi-build" ]; then
+    ./scripts/mpi_build.sh
+    cd test
+    make mpi RABIT_BUILD_DMLC=1 && make speed_test.mpi RABIT_BUILD_DMLC=1 || exit -1
+fi
+#
+if [ ${TASK} == "cmake-test" ]; then
+    mkdir build
+    cd build
+    cmake -DRABIT_BUILD_TESTS=ON -DRABIT_BUILD_DMLC=ON -DGTEST_ROOT=${HOME}/.local ..
+    # known osx gtest 1.8 issue
+    cp ${HOME}/.local/lib/*.dylib .
+    make -j$(nproc)
+    make test
+    make install || exit -1
+    cd ../test
+    ../scripts/travis_runtest.sh || exit -1
+    rm -rf ../build
+fi

rabit/scripts/travis_setup.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+echo "Testing on: ${TRAVIS_OS_NAME}, Home directory: ${HOME}"
+
+# Install Miniconda
+if [ ${TRAVIS_OS_NAME} == "osx" ]; then
+  wget -O conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+else
+  wget -O conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
+fi
+bash conda.sh -b -p $HOME/miniconda
+source $HOME/miniconda/bin/activate
+conda config --set always_yes yes --set changeps1 no
+conda update -q conda
+conda info -a
+conda create -n python3 python=3.7
+conda activate python3
+conda --version
+python --version
+# Install Python packages
+conda install -c conda-forge numpy scipy urllib3 websocket-client
+python -m pip install cpplint pylint kubernetes
+
+# Install googletest under home directory
+GTEST_VERSION=1.8.1
+GTEST_RELEASE=release-${GTEST_VERSION}.tar.gz
+GTEST_TAR_BALL=googletest_${GTEST_RELEASE}
+
+wget https://github.com/google/googletest/archive/${GTEST_RELEASE} -O ${GTEST_TAR_BALL}
+echo "152b849610d91a9dfa1401293f43230c2e0c33f8 ${GTEST_TAR_BALL}" | sha1sum -c
+tar -xf ${GTEST_TAR_BALL}
+pushd .
+
+cd googletest-release-${GTEST_VERSION}
+mkdir build
+cd build
+echo "Installing to ${HOME}/.local"
+cmake .. -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=${HOME}/.local
+make -j$(nproc)
+make install
+
+popd
+
+if [ ${TRAVIS_OS_NAME} == "linux" ]; then
+    sudo apt-get install tree
+fi
+
+if [ ${TRAVIS_OS_NAME} == "osx" ]; then
+    brew install python3
+fi

rabit/scripts/travis_setup_env.sh

@@ -0,0 +1,40 @@
+# script to be sourced in travis yml
+# setup all enviroment variables
+
+export CACHE_PREFIX=${HOME}/.cache/usr
+export PATH=${HOME}/.local/bin:${PATH}
+export PATH=${PATH}:${CACHE_PREFIX}/bin
+export CPLUS_INCLUDE_PATH=${CPLUS_INCLUDE_PATH}:${CACHE_PREFIX}/include
+export C_INCLUDE_PATH=${C_INCLUDE_PATH}:${CACHE_PREFIX}/include
+export LIBRARY_PATH=${LIBRARY_PATH}:${CACHE_PREFIX}/lib
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${CACHE_PREFIX}/lib
+export DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}:${CACHE_PREFIX}/lib
+
+alias make="make -j4"
+
+# setup the cache prefix folder
+if [ ! -d ${HOME}/.cache ]; then
+    mkdir ${HOME}/.cache
+fi
+
+if [ ! -d ${CACHE_PREFIX} ]; then
+    mkdir ${CACHE_PREFIX}
+fi
+if [ ! -d ${CACHE_PREFIX}/include ]; then
+    mkdir ${CACHE_PREFIX}/include
+fi
+if [ ! -d ${CACHE_PREFIX}/lib ]; then
+    mkdir ${CACHE_PREFIX}/lib
+fi
+if [ ! -d ${CACHE_PREFIX}/bin ]; then
+    mkdir ${CACHE_PREFIX}/bin
+fi
+
+# setup CUDA path if NVCC_PREFIX exists
+if [ ! -z "$NVCC_PREFIX" ]; then
+    export PATH=${PATH}:${NVCC_PREFIX}/usr/local/cuda-7.5/bin
+    export CPLUS_INCLUDE_PATH=${CPLUS_INCLUDE_PATH}:${NVCC_PREFIX}/usr/local/cuda-7.5/include
+    export C_INCLUDE_PATH=${C_INCLUDE_PATH}:${NVCC_PREFIX}/usr/local/cuda-7.5/include
+    export LIBRARY_PATH=${LIBRARY_PATH}:${NVCC_PREFIX}/usr/local/cuda-7.5/lib64:${NVCC_PREFIX}/usr/lib/x86_64-linux-gnu
+    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NVCC_PREFIX}/usr/local/cuda-7.5/lib64:${NVCC_PREFIX}/usr/lib/x86_64-linux-gnu
+fi