diff --git a/.travis.yml b/.travis.yml
index 65a572e1d..0f8ee9dcd 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,6 +15,11 @@ jobs:
     - os: linux
       arch: arm64
       env: TASK=python_sdist_test
+    - os: linux
+      arch: arm64
+      env: TASK=python_test
+      services:
+        - docker
     - os: osx
       arch: amd64
       osx_image: xcode10.1
diff --git a/tests/ci_build/Dockerfile.aarch64 b/tests/ci_build/Dockerfile.aarch64
new file mode 100644
index 000000000..cba3fa432
--- /dev/null
+++ b/tests/ci_build/Dockerfile.aarch64
@@ -0,0 +1,37 @@
+FROM quay.io/pypa/manylinux2014_aarch64
+
+SHELL ["/bin/bash", "-c"]   # Use Bash as shell
+
+# Install all basic requirements
+RUN \
+    yum update -y && \
+    yum install -y tar unzip wget xz git centos-release-scl-rh yum-utils && \
+    yum-config-manager --enable centos-sclo-rh-testing && \
+    yum update -y && \
+    yum install -y devtoolset-7 && \
+    # Python
+    wget --no-verbose -O conda.sh https://github.com/conda-forge/miniforge/releases/download/4.8.2-1/Miniforge3-4.8.2-1-Linux-aarch64.sh && \
+    bash conda.sh -b -p /opt/miniforge-python
+
+ENV PATH=/opt/miniforge-python/bin:$PATH
+ENV CC=/opt/rh/devtoolset-7/root/usr/bin/gcc
+ENV CXX=/opt/rh/devtoolset-7/root/usr/bin/c++
+ENV CPP=/opt/rh/devtoolset-7/root/usr/bin/cpp
+ENV GOSU_VERSION 1.10
+
+# Create new Conda environment
+COPY conda_env/aarch64_test.yml /scripts/
+RUN conda env create -n aarch64_test --file=/scripts/aarch64_test.yml
+
+# Install lightweight sudo (not bound to TTY)
+RUN set -ex; \
+    wget --no-verbose -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-arm64" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]
diff --git a/tests/ci_build/conda_env/aarch64_test.yml b/tests/ci_build/conda_env/aarch64_test.yml
new file mode 100644
index 000000000..7c89ba58a
--- /dev/null
+++ b/tests/ci_build/conda_env/aarch64_test.yml
@@ -0,0 +1,29 @@
+name: aarch64_test
+channels:
+- conda-forge
+dependencies:
+- python=3.7
+- pip
+- wheel
+- pytest
+- pytest-cov
+- numpy
+- scipy
+- scikit-learn
+- pandas
+- matplotlib
+- dask
+- distributed
+- hypothesis
+- graphviz
+- python-graphviz
+- codecov
+- cmake
+- ninja
+- boto3
+- jsonschema
+- numba
+- llvmlite
+- pip:
+  - shap
+  - awscli
diff --git a/tests/ci_build/conda_env/cpu_test.yml b/tests/ci_build/conda_env/cpu_test.yml
index 107aebb69..6158515ba 100644
--- a/tests/ci_build/conda_env/cpu_test.yml
+++ b/tests/ci_build/conda_env/cpu_test.yml
@@ -14,6 +14,7 @@ dependencies:
 - pandas
 - matplotlib
 - dask
+- distributed
 - python-graphviz
 - hypothesis
 - astroid
@@ -23,11 +24,14 @@ dependencies:
 - mock
 - breathe
 - pytest
+- pytest-cov
 - python-kubernetes
 - urllib3
 - jsonschema
 - boto3
 - awscli
+- numba
+- llvmlite
 - pip:
   - shap
   - guzzle_sphinx_theme
diff --git a/tests/ci_build/conda_env/macos_cpu_test.yml b/tests/ci_build/conda_env/macos_cpu_test.yml
new file mode 100644
index 000000000..feb102ec2
--- /dev/null
+++ b/tests/ci_build/conda_env/macos_cpu_test.yml
@@ -0,0 +1,35 @@
+name: cpu_test
+channels:
+- conda-forge
+dependencies:
+- python=3.7
+- pip
+- wheel
+- pyyaml
+- cpplint
+- pylint
+- numpy
+- scipy
+- scikit-learn
+- pandas
+- matplotlib
+- dask
+- distributed
+- python-graphviz
+- hypothesis
+- astroid
+- sphinx
+- sh
+- recommonmark
+- mock
+- breathe
+- pytest
+- pytest-cov
+- python-kubernetes
+- urllib3
+- jsonschema
+- boto3
+- awscli
+- pip:
+  - guzzle_sphinx_theme
+  - datatable
diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh
index f65eeff50..ba00c31f0 100755
--- a/tests/travis/run_test.sh
+++ b/tests/travis/run_test.sh
@@ -9,11 +9,7 @@ if [ ${TASK} == "python_sdist_test" ]; then
 
     conda activate python3
     python --version
-    conda install numpy scipy
-    if [ ${TRAVIS_CPU_ARCH} == "arm64" ]; then
-        conda install cmake
-        cmake --version
-    fi
+    cmake --version
 
     make pippack
     python -m pip install xgboost-*.tar.gz -v --user
@@ -28,34 +24,47 @@ if [ ${TASK} == "python_test" ]; then
     fi
 
     set -e
-    # Build/test
-    rm -rf build
-    mkdir build && cd build
-    cmake .. -DUSE_OPENMP=ON -DCMAKE_VERBOSE_MAKEFILE=ON
-    make -j$(nproc)
 
-    echo "-------------------------------"
-    conda activate python3
-    conda --version
-    python --version
 
     # Build binary wheel
-    cd ../python-package
-    python setup.py bdist_wheel
-    TAG=macosx_10_13_x86_64.macosx_10_14_x86_64.macosx_10_15_x86_64
-    python ../tests/ci_build/rename_whl.py dist/*.whl ${TRAVIS_COMMIT} ${TAG}
-    python -m pip install ./dist/xgboost-*-py3-none-${TAG}.whl
+    if [ ${TRAVIS_CPU_ARCH} == "arm64" ]; then
+      # Build manylinux2014 wheel on ARM64
+      tests/ci_build/ci_build.sh aarch64 docker tests/ci_build/build_via_cmake.sh --conda-env=aarch64_test
+      tests/ci_build/ci_build.sh aarch64 docker bash -c "cd build && ctest --extra-verbose"
+      tests/ci_build/ci_build.sh aarch64 docker bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal"
+      TAG=manylinux2014_aarch64
+      tests/ci_build/ci_build.sh aarch64 docker python tests/ci_build/rename_whl.py python-package/dist/*.whl ${TRAVIS_COMMIT} ${TAG}
+    else
+      rm -rf build
+      mkdir build && cd build
+      conda activate python3
+      cmake --version
+      cmake .. -DUSE_OPENMP=ON -DCMAKE_VERBOSE_MAKEFILE=ON
+      make -j$(nproc)
+      cd ../python-package
+      python setup.py bdist_wheel
+      cd ..
+      TAG=macosx_10_13_x86_64.macosx_10_14_x86_64.macosx_10_15_x86_64
+      python tests/ci_build/rename_whl.py python-package/dist/*.whl ${TRAVIS_COMMIT} ${TAG}
+    fi
 
     # Run unit tests
-    cd ..
-    python -m pip install graphviz pytest pytest-cov codecov
-    python -m pip install datatable hypothesis
-    python -m pip install numpy scipy pandas matplotlib scikit-learn dask[complete]
-    python -m pytest -v --fulltrace -s tests/python --cov=python-package/xgboost || exit -1
+    echo "------------------------------"
+    if [ ${TRAVIS_CPU_ARCH} == "arm64" ]; then
+        tests/ci_build/ci_build.sh aarch64 docker \
+          bash -c "source activate aarch64_test && python -m pip install ./python-package/dist/xgboost-*-py3-none-${TAG}.whl && python -m pytest -v -s -rxXs --durations=0 --fulltrace tests/python --cov=python-package/xgboost"
+    else
+        conda env create -n cpu_test --file=tests/ci_build/conda_env/macos_cpu_test.yml
+        conda activate cpu_test
+        python -m pip install ./python-package/dist/xgboost-*-py3-none-${TAG}.whl
+        conda --version
+        python --version
+        python -m pytest -v -s -rxXs --durations=0 --fulltrace tests/python --cov=python-package/xgboost || exit -1
+    fi
+    conda activate python3
     codecov
 
     # Deploy binary wheel to S3
-    python -m pip install awscli
     if [ "${TRAVIS_PULL_REQUEST}" != "false" ]
     then
         S3_DEST="s3://xgboost-nightly-builds/PR-${TRAVIS_PULL_REQUEST}/"
diff --git a/tests/travis/setup.sh b/tests/travis/setup.sh
index 62f1feff5..5d5929d4b 100755
--- a/tests/travis/setup.sh
+++ b/tests/travis/setup.sh
@@ -2,11 +2,11 @@
 
 if [ ${TASK} == "python_test" ] || [ ${TASK} == "python_sdist_test" ]; then
     if [ ${TRAVIS_OS_NAME} == "osx" ]; then
-        wget -O conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+        wget --no-verbose -O conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
     elif [ ${TRAVIS_CPU_ARCH} == "arm64" ]; then
-        wget -O conda.sh https://github.com/conda-forge/miniforge/releases/download/4.8.2-1/Miniforge3-4.8.2-1-Linux-aarch64.sh
+        wget --no-verbose -O conda.sh https://github.com/conda-forge/miniforge/releases/download/4.8.2-1/Miniforge3-4.8.2-1-Linux-aarch64.sh
     else
-        wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+        wget --no-verbose -O conda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
     fi
     bash conda.sh -b -p $HOME/miniconda
     source $HOME/miniconda/bin/activate
@@ -15,7 +15,9 @@ if [ ${TASK} == "python_test" ] || [ ${TASK} == "python_sdist_test" ]; then
     conda update -q conda
     # Useful for debugging any issues with conda
     conda info -a
-    conda create -n python3 python=3.7
+    conda create -n python3 python=3.7 cmake numpy scipy codecov
+    conda activate python3
+    python -m pip install awscli
 fi
 
 if [ ${TASK} == "s390x_test" ] && [ ${TRAVIS_CPU_ARCH} == "s390x" ]; then
@@ -26,10 +28,3 @@ if [ ${TASK} == "s390x_test" ] && [ ${TRAVIS_CPU_ARCH} == "s390x" ]; then
     sudo apt-get install -y --no-install-recommends tar unzip wget git build-essential ninja-build \
       time python3 python3-pip python3-numpy python3-scipy python3-sklearn r-base
 fi
-
-if [ ${TASK} == "python_sdist_test" ] && [ ${TRAVIS_OS_NAME} == "linux" ] && [ ${TRAVIS_CPU_ARCH} == "amd64" ]; then
-    wget https://github.com/Kitware/CMake/releases/download/v3.17.1/cmake-3.17.1-Linux-x86_64.sh
-    sudo bash cmake-3.17.1-Linux-x86_64.sh --prefix=/usr/local --skip-license
-    sudo rm -rf /usr/local/cmake-3.12.4  # Remove existing CMake
-    cmake --version
-fi