[BLOCKING][CI] Upgrade to Spark 2.4.3 (#4414)

* [CI] Upgrade to Spark 2.4.2

* Pass Spark version to build script

* Allow multiple --build-arg in ci_build.sh

* Fix syntax

* Fix container name

* Update pom.xml

* Fix container name

* Update Jenkinsfile

* Update pom.xml

* Update Dockerfile.jvm_cross

tests/ci_build/Dockerfile.jvm_cross

@@ -1,5 +1,6 @@
 FROM ubuntu:19.04
 ARG JDK_VERSION=8
+ARG SPARK_VERSION=2.4.3
 
 # Environment
 ENV DEBIAN_FRONTEND noninteractive
@@ -19,9 +20,9 @@ RUN \
     tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \
     ln -s /opt/apache-maven-3.6.1/ /opt/maven && \
     # Spark
-    wget https://archive.apache.org/dist/spark/spark-2.4.1/spark-2.4.1-bin-hadoop2.7.tgz && \
-    tar xvf spark-2.4.1-bin-hadoop2.7.tgz -C /opt && \
-    ln -s /opt/spark-2.4.1-bin-hadoop2.7 /opt/spark
+    wget https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop2.7.tgz && \
+    tar xvf spark-$SPARK_VERSION-bin-hadoop2.7.tgz -C /opt && \
+    ln -s /opt/spark-$SPARK_VERSION-bin-hadoop2.7 /opt/spark
 
 ENV PATH=/opt/python/bin:/opt/spark/bin:/opt/maven/bin:$PATH
 

tests/ci_build/build_jvm_packages.sh

@@ -3,13 +3,20 @@
 set -e
 set -x
 
+if [ $# -ne 1 ]; then
+  echo "Usage: $0 [spark version]"
+  exit 1
+fi
+
+spark_version=$1
+
 # Initialize local Maven repository
 ./tests/ci_build/initialize_maven.sh
 
 rm -rf build/
 cd jvm-packages
 
-mvn --no-transfer-progress package
+mvn --no-transfer-progress package -Dspark.version=${spark_version}
 
 set +x
 set +e

tests/ci_build/ci_build.sh

@@ -51,11 +51,11 @@ if [[ "$1" == "-it" ]]; then
     shift 1
 fi
 
-if [[ "$1" == "--build-arg" ]]; then
-    CI_DOCKER_BUILD_ARG+="$1"
+while [[ "$1" == "--build-arg" ]]; do
+    CI_DOCKER_BUILD_ARG+=" $1"
     CI_DOCKER_BUILD_ARG+=" $2"
     shift 2
-fi
+done
 
 if [[ ! -f "${DOCKERFILE_PATH}" ]]; then
     echo "Invalid Dockerfile path: \"${DOCKERFILE_PATH}\""