diff --git a/Jenkinsfile b/Jenkinsfile index 20cc07f0c..65fb3a5f6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -293,6 +293,13 @@ def TestPythonGPU(args) { ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/test_python.sh gpu """ } + // For CUDA 10.0 target, run cuDF tests too + if (args.cuda_version == '10.0') { + echo "Running tests with cuDF..." + sh """ + ${dockerRun} cudf ${docker_binary} ${docker_args} tests/ci_build/test_python.sh cudf + """ + } deleteDir() } } diff --git a/tests/ci_build/Dockerfile.cudf b/tests/ci_build/Dockerfile.cudf new file mode 100644 index 000000000..4523e6ac5 --- /dev/null +++ b/tests/ci_build/Dockerfile.cudf @@ -0,0 +1,41 @@ +ARG CUDA_VERSION +FROM nvidia/cuda:$CUDA_VERSION-runtime-ubuntu16.04 + +# Environment +ENV DEBIAN_FRONTEND noninteractive +SHELL ["/bin/bash", "-c"] # Use Bash as shell + +# Install all basic requirements +RUN \ + apt-get update && \ + apt-get install -y wget unzip bzip2 libgomp1 build-essential && \ + # Python + wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \ + bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python + +ENV PATH=/opt/python/bin:$PATH + +# Create new Conda environment with cuDF and dask +RUN \ + conda create -n cudf_test -c rapidsai -c nvidia -c numba -c conda-forge -c anaconda \ + cudf=0.9 python=3.7 anaconda::cudatoolkit=$CUDA_VERSION dask=2.0.0 + +# Install other Python packages +RUN \ + source activate cudf_test && \ + pip install numpy pytest scipy scikit-learn pandas matplotlib wheel kubernetes urllib3 graphviz + +ENV GOSU_VERSION 1.10 + +# Install lightweight sudo (not bound to TTY) +RUN set -ex; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \ + chmod +x /usr/local/bin/gosu && \ + gosu nobody true + +# Default entry-point to use if running locally +# It will preserve attributes of created files +COPY entrypoint.sh /scripts/ + +WORKDIR /workspace +ENTRYPOINT ["/scripts/entrypoint.sh"] diff --git a/tests/ci_build/test_python.sh b/tests/ci_build/test_python.sh index e71e0e03a..dd1722163 100755 --- a/tests/ci_build/test_python.sh +++ b/tests/ci_build/test_python.sh @@ -35,6 +35,11 @@ case "$suite" in ./runtests-gpu.sh ;; + cudf) + source activate cudf_test + python -m pytest -v -s --fulltrace tests/python-gpu/test_from_columnar.py tests/python-gpu/test_gpu_with_dask.py + ;; + cpu) pytest -v -s --fulltrace tests/python cd tests/distributed @@ -42,7 +47,7 @@ case "$suite" in ;; *) - echo "Usage: $0 {gpu|mgpu|cpu}" + echo "Usage: $0 {gpu|mgpu|cudf|cpu}" exit 1 ;; esac diff --git a/tests/python-gpu/test_gpu_with_dask.py b/tests/python-gpu/test_gpu_with_dask.py index abf1e49e4..ce0beb119 100644 --- a/tests/python-gpu/test_gpu_with_dask.py +++ b/tests/python-gpu/test_gpu_with_dask.py @@ -22,6 +22,7 @@ import testing as tm @pytest.mark.skipif(**tm.no_dask()) @pytest.mark.skipif(**tm.no_cudf()) +@pytest.mark.skipif(**tm.no_dask_cudf()) def test_dask_dataframe(client): X, y = generate_array() diff --git a/tests/python/testing.py b/tests/python/testing.py index b12f00952..6073d381b 100644 --- a/tests/python/testing.py +++ b/tests/python/testing.py @@ -37,3 +37,12 @@ def no_matplotlib(): def no_cudf(): return {'condition': not CUDF_INSTALLED, 'reason': 'CUDF is not installed'} + + +def no_dask_cudf(): + reason = 'dask_cudf is not installed.' + try: + import dask_cudf as _ # noqa + return {'condition': False, 'reason': reason} + except ImportError: + return {'condition': True, 'reason': reason}