[CI] Run cuDF tests in Jenkins CI server (#4927)

Jenkinsfile

@@ -293,6 +293,13 @@ def TestPythonGPU(args) {
       ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/test_python.sh gpu
       """
     }
+    // For CUDA 10.0 target, run cuDF tests too
+    if (args.cuda_version == '10.0') {
+      echo "Running tests with cuDF..."
+      sh """
+      ${dockerRun} cudf ${docker_binary} ${docker_args} tests/ci_build/test_python.sh cudf
+      """
+    }
     deleteDir()
   }
 }

tests/ci_build/Dockerfile.cudf

@@ -0,0 +1,41 @@
+ARG CUDA_VERSION
+FROM nvidia/cuda:$CUDA_VERSION-runtime-ubuntu16.04
+
+# Environment
+ENV DEBIAN_FRONTEND noninteractive
+SHELL ["/bin/bash", "-c"]   # Use Bash as shell
+
+# Install all basic requirements
+RUN \
+    apt-get update && \
+    apt-get install -y wget unzip bzip2 libgomp1 build-essential && \
+    # Python
+    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
+    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python
+
+ENV PATH=/opt/python/bin:$PATH
+
+# Create new Conda environment with cuDF and dask
+RUN \
+    conda create -n cudf_test -c rapidsai -c nvidia -c numba -c conda-forge -c anaconda \
+        cudf=0.9 python=3.7 anaconda::cudatoolkit=$CUDA_VERSION dask=2.0.0
+
+# Install other Python packages
+RUN \
+    source activate cudf_test && \
+    pip install numpy pytest scipy scikit-learn pandas matplotlib wheel kubernetes urllib3 graphviz
+
+ENV GOSU_VERSION 1.10
+
+# Install lightweight sudo (not bound to TTY)
+RUN set -ex; \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]

tests/ci_build/test_python.sh

@@ -35,6 +35,11 @@ case "$suite" in
     ./runtests-gpu.sh
     ;;
 
+  cudf)
+    source activate cudf_test
+    python -m pytest -v -s --fulltrace tests/python-gpu/test_from_columnar.py tests/python-gpu/test_gpu_with_dask.py
+    ;;
+
   cpu)
     pytest -v -s --fulltrace tests/python
     cd tests/distributed
@@ -42,7 +47,7 @@ case "$suite" in
     ;;
 
   *)
-    echo "Usage: $0 {gpu|mgpu|cpu}"
+    echo "Usage: $0 {gpu|mgpu|cudf|cpu}"
     exit 1
     ;;
 esac

tests/python-gpu/test_gpu_with_dask.py

@@ -22,6 +22,7 @@ import testing as tm
 
 @pytest.mark.skipif(**tm.no_dask())
 @pytest.mark.skipif(**tm.no_cudf())
+@pytest.mark.skipif(**tm.no_dask_cudf())
 def test_dask_dataframe(client):
     X, y = generate_array()
 

tests/python/testing.py

@@ -37,3 +37,12 @@ def no_matplotlib():
 def no_cudf():
     return {'condition': not CUDF_INSTALLED,
             'reason': 'CUDF is not installed'}
+
+
+def no_dask_cudf():
+    reason = 'dask_cudf is not installed.'
+    try:
+        import dask_cudf as _   # noqa
+        return {'condition': False, 'reason': reason}
+    except ImportError:
+        return {'condition': True, 'reason': reason}