[ci] Update PySpark version. (#9214)

tests/ci_build/Dockerfile.gpu

@@ -25,7 +25,7 @@ RUN \
         python=3.10 cudf=$RAPIDS_VERSION_ARG* rmm=$RAPIDS_VERSION_ARG* cudatoolkit=$CUDA_VERSION_ARG \
         dask dask-cuda=$RAPIDS_VERSION_ARG* dask-cudf=$RAPIDS_VERSION_ARG* cupy \
         numpy pytest pytest-timeout scipy scikit-learn pandas matplotlib wheel python-kubernetes urllib3 graphviz hypothesis \
-        pyspark cloudpickle cuda-python && \
+        pyspark>=3.4.0 cloudpickle cuda-python && \
     mamba clean --all && \
     conda run --no-capture-output -n gpu_test pip install buildkite-test-collector
 

tests/ci_build/conda_env/aarch64_test.yml

@@ -28,7 +28,7 @@ dependencies:
 - llvmlite
 - cffi
 - pyarrow
-- pyspark
+- pyspark>=3.4.0
 - cloudpickle
 - pip:
   - awscli

tests/ci_build/conda_env/linux_cpu_test.yml

@@ -38,8 +38,6 @@ dependencies:
 - protobuf
 - cloudpickle
 - modin
-# TODO: Replace it with pyspark>=3.4 once 3.4 released.
-# - https://ml-team-public-read.s3.us-west-2.amazonaws.com/pyspark-3.4.0.dev0.tar.gz
-- pyspark>=3.3.1
+- pyspark>=3.4.0
 - pip:
   - datatable

tests/ci_build/conda_env/macos_cpu_test.yml

@@ -35,7 +35,7 @@ dependencies:
 - py-ubjson
 - cffi
 - pyarrow
-- pyspark
+- pyspark>=3.4.0
 - cloudpickle
 - pip:
   - sphinx_rtd_theme

tests/ci_build/conda_env/python_lint.yml

@@ -19,6 +19,4 @@ dependencies:
 - pytest
 - hypothesis
 - hatchling
-- pip:
-  # TODO: Replace it with pyspark>=3.4 once 3.4 released.
-  - https://ml-team-public-read.s3.us-west-2.amazonaws.com/pyspark-3.4.0.dev0.tar.gz
+- pyspark>=3.4.0