diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 5c73ffa27..bb6f3acd9 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -35,10 +35,19 @@ jobs:
         key: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }}
         restore-keys: ${{ runner.os }}-m2
 
-    - name: Test JVM packages
+    - name: Test XGBoost4J
       run: |
         cd jvm-packages
-        mvn test -pl :xgboost4j_2.12
+        mvn test -B -pl :xgboost4j_2.12
+
+    - name: Test XGBoost4J-Spark
+      run: |
+        rm -rfv build/
+        cd jvm-packages
+        mvn -B test
+      if: matrix.os == 'ubuntu-latest'  # Distributed training doesn't work on Windows
+      env:
+        RABIT_MOCK: ON
 
   lint:
     runs-on: ubuntu-latest
diff --git a/jvm-packages/pom.xml b/jvm-packages/pom.xml
index 03de3bd1c..4bb211ae1 100644
--- a/jvm-packages/pom.xml
+++ b/jvm-packages/pom.xml
@@ -38,6 +38,7 @@
         <scala.version>2.12.8</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
         <hadoop.version>2.7.3</hadoop.version>
+        <maven.wagon.http.retryHandler.count>5</maven.wagon.http.retryHandler.count>
         <log.capi.invocation>OFF</log.capi.invocation>
         <use.cuda>OFF</use.cuda>
     </properties>