diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 3f45be885..ac046eccd 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -75,19 +75,18 @@ jobs:
     - uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0
       with:
         submodules: 'true'
-    - name: Install system packages
-      run: |
-        sudo apt-get install -y --no-install-recommends ninja-build
-    - uses: conda-incubator/setup-miniconda@35d1405e78aa3f784fe3ce9a2eb378d5eeb62169 # v2.1.1
+    - uses: mamba-org/provision-with-micromamba@f347426e5745fe3dfc13ec5baf20496990d0281f # v14
       with:
-        auto-update-conda: true
-        python-version: ${{ matrix.python-version }}
-        activate-environment: test
+        cache-downloads: true
+        cache-env: true
+        environment-name: cpp_test
+        environment-file: tests/ci_build/conda_env/cpp_test.yml
     - name: Display Conda env
       shell: bash -l {0}
       run: |
         conda info
         conda list
+
     - name: Build and install XGBoost static library
       shell: bash -l {0}
       run: |
@@ -109,6 +108,7 @@ jobs:
         cd ..
         rm -rf ./build
         popd
+
     - name: Build and install XGBoost shared library
       shell: bash -l {0}
       run: |
diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml
index dd10f3aee..dbc0da41d 100644
--- a/.github/workflows/python_tests.yml
+++ b/.github/workflows/python_tests.yml
@@ -40,12 +40,46 @@ jobs:
       run: |
         python tests/ci_build/lint_python.py --format=0 --type-check=0 --pylint=1
 
-  python-sdist-test:
+  python-sdist-test-on-Linux:
+    # Mismatched glibcxx version between system and conda forge.
     runs-on: ${{ matrix.os }}
     name: Test installing XGBoost Python source package on ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-11, windows-latest]
+        os: [ubuntu-latest]
+    steps:
+    - uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0
+      with:
+        submodules: 'true'
+    - uses: mamba-org/provision-with-micromamba@f347426e5745fe3dfc13ec5baf20496990d0281f # v14
+      with:
+        cache-downloads: true
+        cache-env: true
+        environment-name: sdist_test
+        environment-file: tests/ci_build/conda_env/sdist_test.yml
+    - name: Display Conda env
+      shell: bash -l {0}
+      run: |
+        conda info
+        conda list
+    - name: Build and install XGBoost
+      shell: bash -l {0}
+      run: |
+        cd python-package
+        python --version
+        python setup.py sdist
+        pip install -v ./dist/xgboost-*.tar.gz
+        cd ..
+        python -c 'import xgboost'
+
+  python-sdist-test:
+    # Use system toolchain instead of conda toolchain for macos and windows.
+    # MacOS has linker error if clang++ from conda-forge is used
+    runs-on: ${{ matrix.os }}
+    name: Test installing XGBoost Python source package on ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-11, windows-latest]
         python-version: ["3.8"]
     steps:
     - uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0
@@ -55,10 +89,6 @@ jobs:
       if: matrix.os == 'macos-11'
       run: |
         brew install ninja libomp
-    - name: Install Ubuntu system dependencies
-      if: matrix.os == 'ubuntu-latest'
-      run: |
-        sudo apt-get install -y --no-install-recommends ninja-build
     - uses: conda-incubator/setup-miniconda@35d1405e78aa3f784fe3ce9a2eb378d5eeb62169 # v2.1.1
       with:
         auto-update-conda: true
@@ -79,6 +109,63 @@ jobs:
         cd ..
         python -c 'import xgboost'
 
+  python-tests-on-macos:
+    name: Test XGBoost Python package on ${{ matrix.config.os }}
+    runs-on: ${{ matrix.config.os }}
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        config:
+          - {os: macos-11}
+
+    steps:
+    - uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0
+      with:
+        submodules: 'true'
+
+    - uses: mamba-org/provision-with-micromamba@f347426e5745fe3dfc13ec5baf20496990d0281f # v14
+      with:
+        cache-downloads: true
+        cache-env: true
+        environment-name: macos_test
+        environment-file: tests/ci_build/conda_env/macos_cpu_test.yml
+
+    - name: Display Conda env
+      shell: bash -l {0}
+      run: |
+        conda info
+        conda list
+
+    - name: Build XGBoost on macos
+      shell: bash -l {0}
+      run: |
+        brew install ninja
+
+        mkdir build
+        cd build
+        # Set prefix, to use OpenMP library from Conda env
+        # See https://github.com/dmlc/xgboost/issues/7039#issuecomment-1025038228
+        # to learn why we don't use libomp from Homebrew.
+        cmake .. -GNinja -DCMAKE_PREFIX_PATH=$CONDA_PREFIX
+        ninja
+
+    - name: Install Python package
+      shell: bash -l {0}
+      run: |
+        cd python-package
+        python --version
+        python setup.py install
+
+    - name: Test Python package
+      shell: bash -l {0}
+      run: |
+        pytest -s -v -rxXs --durations=0 ./tests/python
+
+    - name: Test Dask Interface
+      shell: bash -l {0}
+      run: |
+        pytest -s -v -rxXs --durations=0 ./tests/test_distributed/test_with_dask
+
   python-tests-on-win:
     name: Test XGBoost Python package on ${{ matrix.config.os }}
     runs-on: ${{ matrix.config.os }}
@@ -126,60 +213,3 @@ jobs:
       shell: bash -l {0}
       run: |
         pytest -s -v -rxXs --durations=0 ./tests/python
-
-  python-tests-on-macos:
-    name: Test XGBoost Python package on ${{ matrix.config.os }}
-    runs-on: ${{ matrix.config.os }}
-    timeout-minutes: 60
-    strategy:
-      matrix:
-        config:
-          - {os: macos-11}
-
-    steps:
-    - uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0
-      with:
-        submodules: 'true'
-
-    - uses: mamba-org/provision-with-micromamba@f347426e5745fe3dfc13ec5baf20496990d0281f # v14
-      with:
-        cache-downloads: true
-        cache-env: true
-        environment-name: macos_test
-        environment-file: tests/ci_build/conda_env/macos_cpu_test.yml
-
-    - name: Display Conda env
-      shell: bash -l {0}
-      run: |
-        conda info
-        conda list
-
-    - name: Build XGBoost on macos
-      shell: bash -l {0}
-      run: |
-        brew install ninja
-
-        mkdir build
-        cd build
-        # Set prefix, to use OpenMP library from Conda env
-        # See https://github.com/dmlc/xgboost/issues/7039#issuecomment-1025038228
-        # to learn why we don't use libomp from Homebrew.
-        cmake .. -GNinja -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DCMAKE_PREFIX_PATH=$CONDA_PREFIX
-        ninja
-
-    - name: Install Python package
-      shell: bash -l {0}
-      run: |
-        cd python-package
-        python --version
-        python setup.py install
-
-    - name: Test Python package
-      shell: bash -l {0}
-      run: |
-        pytest -s -v -rxXs --durations=0 ./tests/python
-
-    - name: Test Dask Interface
-      shell: bash -l {0}
-      run: |
-        pytest -s -v -rxXs --durations=0 ./tests/test_distributed/test_with_dask
diff --git a/tests/ci_build/conda_env/cpp_test.yml b/tests/ci_build/conda_env/cpp_test.yml
new file mode 100644
index 000000000..80ce65cc1
--- /dev/null
+++ b/tests/ci_build/conda_env/cpp_test.yml
@@ -0,0 +1,11 @@
+# conda environment for CPP test on Linux distributions
+name: cpp_test
+channels:
+- defaults
+- conda-forge
+dependencies:
+- cmake
+- ninja
+- c-compiler
+- cxx-compiler
+- gtest
diff --git a/tests/ci_build/conda_env/sdist_test.yml b/tests/ci_build/conda_env/sdist_test.yml
new file mode 100644
index 000000000..fc1e71097
--- /dev/null
+++ b/tests/ci_build/conda_env/sdist_test.yml
@@ -0,0 +1,13 @@
+# conda environment for source distribution test.
+name: sdist_test
+channels:
+- defaults
+- conda-forge
+dependencies:
+- python=3.8
+- pip
+- wheel
+- cmake
+- ninja
+- c-compiler
+- cxx-compiler