[Python] Require black and isort for new Python files. (#8096)

* [Python] Require black and isort for new Python files.

- Require black and isort for spark and dask module.

These files are relatively new and are more conform to the black formatter. We will
convert the rest of the library as we move forward.

Other libraries including dask/distributed and optuna use the same formatting style and
have a more strict standard. The black formatter is indeed quite nice, automating it can
help us unify the code style.

- Gather Python checks into a single script.

tests/ci_build/conda_env/python_lint.yml

@@ -0,0 +1,15 @@
+name: python_lint
+channels:
+- conda-forge
+dependencies:
+- python=3.8
+- wheel
+- setuptools
+- mypy
+- numpy
+- scipy
+- pandas
+- dask
+- distributed
+- black
+- isort

tests/ci_build/lint_python.py

@@ -0,0 +1,63 @@
+import argparse
+import os
+import subprocess
+import sys
+
+from test_utils import DirectoryExcursion
+
+CURDIR = os.path.normpath(os.path.abspath(os.path.dirname(__file__)))
+PROJECT_ROOT = os.path.normpath(os.path.join(CURDIR, os.path.pardir, os.path.pardir))
+
+
+def run_formatter(rel_path: str):
+    path = os.path.join(PROJECT_ROOT, rel_path)
+    isort_ret = subprocess.run(["isort", "--check", "--profile=black", path]).returncode
+    black_ret = subprocess.run(
+        ["black", "--check", "./python-package/xgboost/dask.py"]
+    ).returncode
+    if isort_ret != 0 or black_ret != 0:
+        msg = (
+            "Please run the following command on your machine to address the format"
+            f" errors:\n isort --check --profile=black {rel_path}\n black {rel_path}\n"
+        )
+        print(msg, file=sys.stdout)
+        return False
+    return True
+
+
+def run_mypy(rel_path: str):
+    with DirectoryExcursion(os.path.join(PROJECT_ROOT, "python-package")):
+        path = os.path.join(PROJECT_ROOT, rel_path)
+        ret = subprocess.run(["mypy", path])
+        if ret.returncode != 0:
+            return False
+        return True
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--format", type=int, choices=[0, 1], default=1)
+    parser.add_argument("--type-check", type=int, choices=[0, 1], default=1)
+    args = parser.parse_args()
+    if args.format == 1:
+        if not all(
+            [
+                run_formatter("python-package/xgboost/dask.py"),
+                run_formatter("python-package/xgboost/spark"),
+            ]
+        ):
+            sys.exit(-1)
+
+    if args.type_check == 1:
+        if not all(
+            run_mypy(path)
+            for path in [
+                "python-package/xgboost/",
+                "demo/guide-python/external_memory.py",
+                "demo/guide-python/cat_in_the_dat.py",
+                "tests/python/test_data_iterator.py",
+                "tests/python-gpu/test_gpu_with_dask.py",
+                "tests/python-gpu/test_gpu_data_iterator.py",
+            ]
+        ):
+            sys.exit(-1)

tests/ci_build/test_r_package.py

@@ -1,6 +1,7 @@
 import argparse
 import os
 import subprocess
+from test_utils import DirectoryExcursion
 
 ROOT = os.path.normpath(
     os.path.join(os.path.dirname(os.path.abspath(__file__)), os.path.pardir,
@@ -8,18 +9,6 @@ ROOT = os.path.normpath(
 r_package = os.path.join(ROOT, 'R-package')
 
 
-class DirectoryExcursion:
-    def __init__(self, path: os.PathLike):
-        self.path = path
-        self.curdir = os.path.normpath(os.path.abspath(os.path.curdir))
-
-    def __enter__(self):
-        os.chdir(self.path)
-
-    def __exit__(self, *args):
-        os.chdir(self.curdir)
-
-
 def get_mingw_bin():
     return os.path.join('c:/rtools40/mingw64/', 'bin')
 

tests/ci_build/test_utils.py

@@ -0,0 +1,14 @@
+import os
+from typing import Union
+
+
+class DirectoryExcursion:
+    def __init__(self, path: Union[os.PathLike, str]):
+        self.path = path
+        self.curdir = os.path.normpath(os.path.abspath(os.path.curdir))
+
+    def __enter__(self):
+        os.chdir(self.path)
+
+    def __exit__(self, *args):
+        os.chdir(self.curdir)