diff --git a/Jenkinsfile b/Jenkinsfile
index f3733d9cb..03fc5a7f7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -249,7 +249,7 @@ def BuildCUDA(args) {
     sh """
     ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_CUDA=ON -DUSE_NCCL=ON -DOPEN_MP:BOOL=ON
     ${dockerRun} ${container_type} ${docker_binary} ${docker_args} bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal"
-    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} python3 tests/ci_build/rename_whl.py python-package/dist/*.whl ${commit_id} manylinux1_x86_64
+    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} python3 tests/ci_build/rename_whl.py python-package/dist/*.whl ${commit_id} manylinux2010_x86_64
     """
     // Stash wheel for CUDA 9.0 target
     if (args.cuda_version == '9.0') {
diff --git a/tests/ci_build/Dockerfile.gpu_build b/tests/ci_build/Dockerfile.gpu_build
index b39f5a2bc..65bccce74 100644
--- a/tests/ci_build/Dockerfile.gpu_build
+++ b/tests/ci_build/Dockerfile.gpu_build
@@ -3,7 +3,7 @@ FROM nvidia/cuda:$CUDA_VERSION-devel-centos6
 
 # Environment
 ENV DEBIAN_FRONTEND noninteractive
-ENV DEVTOOLSET_URL_ROOT http://mirror.centos.org/centos/6/sclo/x86_64/rh/devtoolset-4
+ENV DEVTOOLSET_URL_ROOT http://vault.centos.org/6.9/sclo/x86_64/rh/devtoolset-4/
 
 # Install all basic requirements
 RUN \
diff --git a/tests/ci_build/Dockerfile.jvm b/tests/ci_build/Dockerfile.jvm
index 10c1339cc..b16182102 100644
--- a/tests/ci_build/Dockerfile.jvm
+++ b/tests/ci_build/Dockerfile.jvm
@@ -1,12 +1,18 @@
 FROM centos:6
 
+ENV DEVTOOLSET_URL_ROOT http://vault.centos.org/6.9/sclo/x86_64/rh/devtoolset-4/
+
 # Install all basic requirements
 RUN \
     yum -y update && \
     yum install -y tar unzip wget xz git centos-release-scl yum-utils java-1.8.0-openjdk-devel && \
     yum-config-manager --enable centos-sclo-rh-testing && \
     yum -y update && \
-    yum install -y devtoolset-6-gcc devtoolset-6-binutils devtoolset-6-gcc-c++ && \
+    yum install -y $DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-5.3.1-6.1.el6.x86_64.rpm \
+                   $DEVTOOLSET_URL_ROOT/devtoolset-4-gcc-c++-5.3.1-6.1.el6.x86_64.rpm \
+                   $DEVTOOLSET_URL_ROOT/devtoolset-4-binutils-2.25.1-8.el6.x86_64.rpm \
+                   $DEVTOOLSET_URL_ROOT/devtoolset-4-runtime-4.1-3.sc1.el6.x86_64.rpm \
+                   $DEVTOOLSET_URL_ROOT/devtoolset-4-libstdc++-devel-5.3.1-6.1.el6.x86_64.rpm && \
     # Python
     wget -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
     bash Miniconda3.sh -b -p /opt/python && \
@@ -19,9 +25,9 @@ RUN \
     ln -s /opt/apache-maven-3.6.1/ /opt/maven
 
 ENV PATH=/opt/python/bin:/opt/maven/bin:$PATH
-ENV CC=/opt/rh/devtoolset-6/root/usr/bin/gcc
-ENV CXX=/opt/rh/devtoolset-6/root/usr/bin/c++
-ENV CPP=/opt/rh/devtoolset-6/root/usr/bin/cpp
+ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
+ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
+ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
 
 # Install Python packages
 RUN \
diff --git a/tests/ci_build/insert_vcomp140.py b/tests/ci_build/insert_vcomp140.py
index 74c3d422e..9e40ae307 100644
--- a/tests/ci_build/insert_vcomp140.py
+++ b/tests/ci_build/insert_vcomp140.py
@@ -10,7 +10,7 @@ if len(sys.argv) != 2:
 vcomp140_path = 'C:\\Windows\\System32\\vcomp140.dll'
 
 for wheel_path in sorted(glob.glob(sys.argv[1])):
-    m = re.search(r'xgboost-(.*)-py2.py3', wheel_path)
+    m = re.search(r'xgboost-(.*)-py3', wheel_path)
     assert m
     version = m.group(1)
 
diff --git a/tests/ci_build/rename_whl.py b/tests/ci_build/rename_whl.py
index 09b448729..e88aa1616 100644
--- a/tests/ci_build/rename_whl.py
+++ b/tests/ci_build/rename_whl.py
@@ -21,7 +21,7 @@ whl_path = sys.argv[1]
 commit_id = sys.argv[2]
 platform_tag = sys.argv[3]
 
-assert platform_tag in ['manylinux1_x86_64', 'win_amd64']
+assert platform_tag in ['manylinux1_x86_64', 'manylinux2010_x86_64', 'win_amd64']
 
 dirname, basename = os.path.dirname(whl_path), os.path.basename(whl_path)
 
@@ -32,6 +32,6 @@ with cd(dirname):
                 'version': tokens[1],
                 'commit_id': commit_id,
                 'platform_tag': platform_tag}
-    new_name = '{pkg_name}-{version}+{commit_id}-py2.py3-none-{platform_tag}.whl'.format(**keywords)
+    new_name = '{pkg_name}-{version}+{commit_id}-py3-none-{platform_tag}.whl'.format(**keywords)
     print('Renaming {} to {}...'.format(basename, new_name))
     os.rename(basename, new_name)
diff --git a/tests/ci_build/test_python.sh b/tests/ci_build/test_python.sh
index 72c13e4cb..78dc077c2 100755
--- a/tests/ci_build/test_python.sh
+++ b/tests/ci_build/test_python.sh
@@ -7,6 +7,7 @@ suite=$1
 # Install XGBoost Python package
 function install_xgboost {
   wheel_found=0
+  pip install --upgrade pip --user
   for file in python-package/dist/*.whl
   do
     if [ -e "${file}" ]