Extract dask and spark test into distributed test. (#8395)

- Move test files.
- Run spark and dask separately to prevent conflicts.
- Gather common code into the testing module.

tests/ci_build/conda_env/python_lint.yml

@@ -17,3 +17,5 @@ dependencies:
 - isort
 - pyspark
 - cloudpickle
+- pytest
+- hypothesis

tests/ci_build/lint_python.py

@@ -103,7 +103,12 @@ class PyLint:
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
+    parser = argparse.ArgumentParser(
+        description=(
+            "Run static checkers for XGBoost, see `python_lint.yml' "
+            "conda env file for a list of dependencies."
+        )
+    )
     parser.add_argument("--format", type=int, choices=[0, 1], default=1)
     parser.add_argument("--type-check", type=int, choices=[0, 1], default=1)
     parser.add_argument("--pylint", type=int, choices=[0, 1], default=1)
@@ -125,11 +130,11 @@ if __name__ == "__main__":
                 # tests
                 "tests/python/test_config.py",
                 "tests/python/test_data_iterator.py",
-                "tests/python/test_spark/",
                 "tests/python/test_quantile_dmatrix.py",
-                "tests/python-gpu/test_gpu_spark/",
                 "tests/python-gpu/test_gpu_data_iterator.py",
                 "tests/ci_build/lint_python.py",
+                "tests/test_distributed/test_with_spark/",
+                "tests/test_distributed/test_gpu_with_spark/",
                 # demo
                 "demo/guide-python/cat_in_the_dat.py",
                 "demo/guide-python/categorical.py",
@@ -146,11 +151,11 @@ if __name__ == "__main__":
                 "demo/guide-python/external_memory.py",
                 "demo/guide-python/cat_in_the_dat.py",
                 "tests/python/test_data_iterator.py",
-                "tests/python/test_spark/test_data.py",
-                "tests/python-gpu/test_gpu_with_dask/test_gpu_with_dask.py",
                 "tests/python-gpu/test_gpu_data_iterator.py",
-                "tests/python-gpu/test_gpu_spark/test_data.py",
                 "tests/ci_build/lint_python.py",
+                "tests/test_distributed/test_with_spark/test_data.py",
+                "tests/test_distributed/test_gpu_with_spark/test_data.py",
+                "tests/test_distributed/test_gpu_with_dask/test_gpu_with_dask.py",
             ]
         ):
             sys.exit(-1)

tests/ci_build/test_python.sh

@@ -68,6 +68,8 @@ case "$suite" in
     install_xgboost
     setup_pyspark_envs
     pytest -v -s -rxXs --fulltrace --durations=0 -m "mgpu" ${args} tests/python-gpu
+    pytest -v -s -rxXs --fulltrace --durations=0 -m "mgpu" ${args} tests/test_distributed/test_gpu_with_dask
+    pytest -v -s -rxXs --fulltrace --durations=0 -m "mgpu" ${args} tests/test_distributed/test_gpu_with_spark
     unset_pyspark_envs
     uninstall_xgboost
     set +x
@@ -80,6 +82,8 @@ case "$suite" in
     export RAY_OBJECT_STORE_ALLOW_SLOW_STORAGE=1
     setup_pyspark_envs
     pytest -v -s -rxXs --fulltrace --durations=0 ${args} tests/python
+    pytest -v -s -rxXs --fulltrace --durations=0 ${args} tests/test_distributed/test_with_dask
+    pytest -v -s -rxXs --fulltrace --durations=0 ${args} tests/test_distributed/test_with_spark
     unset_pyspark_envs
     uninstall_xgboost
     set +x