[CI] Fix broken Docker container 'cpu' (#5956)

Jenkinsfile

@@ -159,7 +159,7 @@ def Lint() {
     def container_type = "cpu"
     def docker_binary = "docker"
     sh """
-    ${dockerRun} ${container_type} ${docker_binary} make lint
+    ${dockerRun} ${container_type} ${docker_binary} bash -c "source activate cpu_test && make lint"
     """
     deleteDir()
   }
@@ -173,7 +173,7 @@ def SphinxDoc() {
     def docker_binary = "docker"
     def docker_extra_params = "CI_DOCKER_EXTRA_PARAMS_INIT='-e SPHINX_GIT_BRANCH=${BRANCH_NAME}'"
     sh """#!/bin/bash
-    ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} make -C doc html
+    ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} bash -c "source activate cpu_test && make -C doc html"
     """
     deleteDir()
   }

Jenkinsfile-win64

@@ -127,7 +127,7 @@ def TestWin64() {
     bat "build\\testxgboost.exe"
     echo "Installing Python dependencies..."
     def env_name = 'win64_' + UUID.randomUUID().toString().replaceAll('-', '')
-    bat "conda env create -n ${env_name} --file=tests/ci_build/win64_conda_env.yml"
+    bat "conda env create -n ${env_name} --file=tests/ci_build/conda_env/win64_test.yml"
     echo "Installing Python wheel..."
     bat """
     conda activate ${env_name} && for /R %%i in (python-package\\dist\\*.whl) DO python -m pip install "%%i"

tests/ci_build/Dockerfile.cpu

@@ -19,13 +19,9 @@ ENV PATH=/opt/python/bin:$PATH
 
 ENV GOSU_VERSION 1.10
 
-# Install Python packages in default env
-RUN \
-    pip install pyyaml cpplint pylint astroid sphinx numpy scipy pandas matplotlib sh \
-    		recommonmark guzzle_sphinx_theme mock breathe graphviz hypothesis\
-		pytest scikit-learn wheel kubernetes urllib3 jsonschema boto3 && \
-    pip install https://h2o-release.s3.amazonaws.com/datatable/stable/datatable-0.7.0/datatable-0.7.0-cp37-cp37m-linux_x86_64.whl && \
-    pip install "dask[complete]"
+# Create new Conda environment
+COPY conda_env/cpu_test.yml /scripts/
+RUN conda env create -n cpu_test --file=/scripts/cpu_test.yml
 
 # Install lightweight sudo (not bound to TTY)
 RUN set -ex; \

tests/ci_build/conda_env/cpu_test.yml

@@ -0,0 +1,33 @@
+name: cpu_test
+channels:
+- conda-forge
+dependencies:
+- python=3.7
+- pip
+- wheel
+- pyyaml
+- cpplint
+- pylint
+- numpy
+- scipy
+- scikit-learn
+- pandas
+- matplotlib
+- dask
+- python-graphviz
+- hypothesis
+- astroid
+- sphinx
+- sh
+- recommonmark
+- mock
+- breathe
+- pytest
+- python-kubernetes
+- urllib3
+- jsonschema
+- boto3
+- awscli
+- pip:
+  - guzzle_sphinx_theme
+  - datatable

tests/ci_build/runxgb.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+source activate cpu_test
+
 export DMLC_SUBMIT_CLUSTER=local
 
 submit="python3 dmlc-core/tracker/dmlc-submit"

tests/ci_build/test_python.sh

@@ -45,6 +45,7 @@ case "$suite" in
     ;;
 
   cpu)
+    source activate cpu_test
     install_xgboost
     pytest -v -s --fulltrace tests/python
     cd tests/distributed