Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[jvm-packages] upgrade to Scala 2.12 (#4574)

Browse Source 
* bump scala to 2.12 which requires java 8 and also newer flink and akka

* put scala version in artifactId

* fix appveyor

* fix for scaladoc issue that looks like https://github.com/scala/bug/issues/10509

* fix ci_build

* update versions in generate_pom.py

* fix generate_pom.py

* apache does not have a download for spark 2.4.3 distro using scala 2.12 yet, so for now i use a tgz i put on s3

* Upload spark-2.4.3-bin-scala2.12-hadoop2.7.tgz to our own S3

* Update Dockerfile.jvm_cross

* Update Dockerfile.jvm_cross
This commit is contained in:
koertkuipers 2019-07-16 11:43:34 -04:00 committed by Nan Zhu
parent 5544a730f1
commit 3c506b076e
12 changed files with 67 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
appveyor.yml
View File

@ -98,7 +98,7 @@ build_script:
cmake .. -G"%generator%" -DCMAKE_CONFIGURATION_TYPES="Release" -DR_LIB=ON &&
cmake --build . --target install --config Release
)
- if /i "%target%" == "jvm" cd jvm-packages && mvn test -pl :xgboost4j
- if /i "%target%" == "jvm" cd jvm-packages && mvn test -pl :xgboost4j_2.12
test_script:
- cd %APPVEYOR_BUILD_FOLDER%

10
jvm-packages/README.md
View File

@ -27,14 +27,14 @@ XGBoost4J, XGBoost4J-Spark, etc. in maven repository is compiled with g++-4.8.5
```
<dependency>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost4j</artifactId>
<artifactId>xgboost4j_2.12</artifactId>
<version>latest_version_num</version>
</dependency>
```
<b>sbt</b>
```sbt
"ml.dmlc" % "xgboost4j" % "latest_version_num"
"ml.dmlc" %% "xgboost4j" % "latest_version_num"
```
For the latest release version number, please check [here](https://github.com/dmlc/xgboost/releases).
@ -68,17 +68,17 @@ the add dependency as following:
```
<dependency>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost4j</artifactId>
<artifactId>xgboost4j_2.12</artifactId>
<version>latest_version_num</version>
</dependency>
```
<b>sbt</b>
```sbt
"ml.dmlc" % "xgboost4j" % "latest_version_num"
"ml.dmlc" %% "xgboost4j" % "latest_version_num"
```
For the latest release version number, please check [here](https://github.com/CodingCat/xgboost/tree/maven-repo/ml/dmlc/xgboost4j).
For the latest release version number, please check [here](https://github.com/CodingCat/xgboost/tree/maven-repo/ml/dmlc/xgboost4j_2.12).
if you want to use `xgboost4j-spark`, you just need to replace xgboost4j with `xgboost4j-spark`

20
jvm-packages/pom.xml
View File

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost-jvm</artifactId>
<artifactId>xgboost-jvm_2.12</artifactId>
<version>0.90</version>
<packaging>pom</packaging>
<name>XGBoost JVM Package</name>
@ -31,12 +31,12 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<maven.compiler.source>1.7</maven.compiler.source>
<maven.compiler.target>1.7</maven.compiler.target>
<flink.version>1.5.0</flink.version>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<flink.version>1.7.2</flink.version>
<spark.version>2.4.3</spark.version>
<scala.version>2.11.12</scala.version>
<scala.binary.version>2.11</scala.binary.version>
<scala.version>2.12.8</scala.version>
<scala.binary.version>2.12</scala.binary.version>
</properties>
<repositories>
<repository>
@ -372,7 +372,13 @@
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
<version>3.0.0</version>
<version>3.0.8</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.scalactic</groupId>
<artifactId>scalactic_${scala.binary.version}</artifactId>
<version>3.0.8</version>
<scope>test</scope>
</dependency>
</dependencies>

8
jvm-packages/xgboost4j-example/pom.xml
View File

@ -5,10 +5,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost-jvm</artifactId>
<artifactId>xgboost-jvm_2.12</artifactId>
<version>0.90</version>
</parent>
<artifactId>xgboost4j-example</artifactId>
<artifactId>xgboost4j-example_2.12</artifactId>
<version>0.90</version>
<packaging>jar</packaging>
<build>
@ -25,7 +25,7 @@
<dependencies>
<dependency>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost4j-spark</artifactId>
<artifactId>xgboost4j-spark_${scala.binary.version}</artifactId>
<version>0.90</version>
</dependency>
<dependency>
@ -36,7 +36,7 @@
</dependency>
<dependency>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost4j-flink</artifactId>
<artifactId>xgboost4j-flink_${scala.binary.version}</artifactId>
<version>0.90</version>
</dependency>
<dependency>

6
jvm-packages/xgboost4j-flink/pom.xml
View File

@ -5,10 +5,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost-jvm</artifactId>
<artifactId>xgboost-jvm_2.12</artifactId>
<version>0.90</version>
</parent>
<artifactId>xgboost4j-flink</artifactId>
<artifactId>xgboost4j-flink_2.12</artifactId>
<version>0.90</version>
<build>
<plugins>
@ -25,7 +25,7 @@
<dependencies>
<dependency>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost4j</artifactId>
<artifactId>xgboost4j_${scala.binary.version}</artifactId>
<version>0.90</version>
</dependency>
<dependency>

6
jvm-packages/xgboost4j-spark/pom.xml
View File

@ -5,10 +5,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost-jvm</artifactId>
<artifactId>xgboost-jvm_2.12</artifactId>
<version>0.90</version>
</parent>
<artifactId>xgboost4j-spark</artifactId>
<artifactId>xgboost4j-spark_2.12</artifactId>
<build>
<plugins>
<plugin>
@ -23,7 +23,7 @@
<dependencies>
<dependency>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost4j</artifactId>
<artifactId>xgboost4j_${scala.binary.version}</artifactId>
<version>0.90</version>
</dependency>
<dependency>

26
jvm-packages/xgboost4j-tester/generate_pom.py
View File

@ -8,10 +8,10 @@ pom_template = """
<modelVersion>4.0.0</modelVersion>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost4j-tester</artifactId>
<artifactId>xgboost4j-tester_2.12</artifactId>
<version>1.0-SNAPSHOT</version>
<name>xgboost4j-tester</name>
<name>xgboost4j-tester_2.12</name>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@ -51,19 +51,25 @@ pom_template = """
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-actor_${{scala.binary.version}}</artifactId>
<version>2.3.11</version>
<version>2.5.23</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-testkit_${{scala.binary.version}}</artifactId>
<version>2.3.11</version>
<version>2.5.23</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${{scala.binary.version}}</artifactId>
<version>3.0.0</version>
<version>3.0.8</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.scalactic</groupId>
<artifactId>scalactic_${{scala.binary.version}}</artifactId>
<version>3.0.8</version>
<scope>test</scope>
</dependency>
<dependency>
@ -97,12 +103,12 @@ pom_template = """
</dependency>
<dependency>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost4j</artifactId>
<artifactId>xgboost4j_${{scala.binary.version}}</artifactId>
<version>{xgboost4j_version}</version>
</dependency>
<dependency>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost4j</artifactId>
<artifactId>xgboost4j_${{scala.binary.version}}</artifactId>
<version>{xgboost4j_version}</version>
<classifier>tests</classifier>
<type>test-jar</type>
@ -110,12 +116,12 @@ pom_template = """
</dependency>
<dependency>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost4j-spark</artifactId>
<artifactId>xgboost4j-spark_${{scala.binary.version}}</artifactId>
<version>{xgboost4j_version}</version>
</dependency>
<dependency>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost4j-example</artifactId>
<artifactId>xgboost4j-example_${{scala.binary.version}}</artifactId>
<version>{xgboost4j_version}</version>
</dependency>
</dependencies>
@ -186,7 +192,7 @@ pom_template = """
<version>2.22.1</version>
<configuration>
<dependenciesToScan>
<dependency>ml.dmlc:xgboost4j</dependency>
<dependency>ml.dmlc:xgboost4j_2.12</dependency>
</dependenciesToScan>
</configuration>
</plugin>

8
jvm-packages/xgboost4j/pom.xml
View File

@ -5,10 +5,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost-jvm</artifactId>
<artifactId>xgboost-jvm_2.12</artifactId>
<version>0.90</version>
</parent>
<artifactId>xgboost4j</artifactId>
<artifactId>xgboost4j_2.12</artifactId>
<version>0.90</version>
<packaging>jar</packaging>
@ -22,13 +22,13 @@
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-actor_${scala.binary.version}</artifactId>
<version>2.3.11</version>
<version>2.5.23</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-testkit_${scala.binary.version}</artifactId>
<version>2.3.11</version>
<version>2.5.23</version>
<scope>test</scope>
</dependency>
</dependencies>

5
jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/DMatrix.java
View File

@ -78,7 +78,8 @@ public class DMatrix {
* @throws XGBoostError
*/
@Deprecated
public DMatrix(long[] headers, int[] indices, float[] data, SparseType st) throws XGBoostError {
public DMatrix(long[] headers, int[] indices, float[] data, DMatrix.SparseType st)
throws XGBoostError {
long[] out = new long[1];
if (st == SparseType.CSR) {
XGBoostJNI.checkCall(XGBoostJNI.XGDMatrixCreateFromCSREx(headers, indices, data, 0, out));
@ -100,7 +101,7 @@ public class DMatrix {
* row number
* @throws XGBoostError
*/
public DMatrix(long[] headers, int[] indices, float[] data, SparseType st, int shapeParam)
public DMatrix(long[] headers, int[] indices, float[] data, DMatrix.SparseType st, int shapeParam)
throws XGBoostError {
long[] out = new long[1];
if (st == SparseType.CSR) {

10
jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTracker.scala
View File

@ -131,9 +131,7 @@ private[scala] class RabitTracker(numWorkers: Int, port: Option[Int] = None,
}
def stop(): Unit = {
if (!system.isTerminated) {
system.shutdown()
}
system.terminate()
}
/**
@ -170,12 +168,10 @@ private[scala] class RabitTracker(numWorkers: Int, port: Option[Int] = None,
case Failure(e) =>
IRabitTracker.TrackerStatus.FAILURE.getStatusCode
}
system.shutdown()
system.terminate()
statusCode
case Failure(ex: Throwable) =>
if (!system.isTerminated) {
system.shutdown()
}
system.terminate()
IRabitTracker.TrackerStatus.FAILURE.getStatusCode
}
}

8
tests/ci_build/Dockerfile.jvm_cross
View File

@ -15,14 +15,16 @@ RUN \
# Python
wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \
/opt/python/bin/pip install awscli && \
# Maven
wget http://apache.osuosl.org/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \
tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \
ln -s /opt/apache-maven-3.6.1/ /opt/maven && \
# Spark
wget https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop2.7.tgz && \
tar xvf spark-$SPARK_VERSION-bin-hadoop2.7.tgz -C /opt && \
ln -s /opt/spark-$SPARK_VERSION-bin-hadoop2.7 /opt/spark
# This should be: wget https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz
/opt/python/bin/python -m awscli s3 cp s3://xgboost-ci-jenkins-artifacts/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz . && \
tar xvf spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz -C /opt && \
ln -s /opt/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7 /opt/spark
ENV PATH=/opt/python/bin:/opt/spark/bin:/opt/maven/bin:$PATH

14
tests/ci_build/test_jvm_cross.sh
View File

@ -16,10 +16,10 @@ scala_version=$(mvn help:evaluate -Dexpression=scala.version -q -DforceStdout)
scala_binary_version=$(mvn help:evaluate -Dexpression=scala.binary.version -q -DforceStdout)
# Install XGBoost4J JAR into local Maven repository
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j -Dversion=${xgboost4j_version} -Dpackaging=jar
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j-${xgboost4j_version}-tests.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j -Dversion=${xgboost4j_version} -Dpackaging=test-jar -Dclassifier=tests
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-spark/target/xgboost4j-spark-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-spark -Dversion=${xgboost4j_version} -Dpackaging=jar
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-example/target/xgboost4j-example-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-example -Dversion=${xgboost4j_version} -Dpackaging=jar
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}-tests.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=test-jar -Dclassifier=tests
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-spark/target/xgboost4j-spark_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-spark_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar
mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-example/target/xgboost4j-example_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-example_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar
cd xgboost4j-tester
# Generate pom.xml for XGBoost4J-tester, a dummy project to run XGBoost4J tests
@ -28,14 +28,14 @@ python3 ./generate_pom.py ${xgboost4j_version} ${maven_compiler_source} ${maven_
mvn --no-transfer-progress package
# Run integration tests with XGBoost4J
java -jar ./target/xgboost4j-tester-1.0-SNAPSHOT-jar-with-dependencies.jar
java -jar ./target/xgboost4j-tester_${scala_binary_version}-1.0-SNAPSHOT-jar-with-dependencies.jar
# Run integration tests with XGBoost4J-Spark
if [ ! -z "$RUN_INTEGRATION_TEST" ]
then
python3 get_iris.py
spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkTraining --master 'local[8]' ./target/xgboost4j-tester-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv
spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkMLlibPipeline --master 'local[8]' ./target/xgboost4j-tester-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv ${PWD}/native_model ${PWD}/pipeline_model
spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkTraining --master 'local[8]' ./target/xgboost4j-tester_${scala_binary_version}-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv
spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkMLlibPipeline --master 'local[8]' ./target/xgboost4j-tester_${scala_binary_version}-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv ${PWD}/native_model ${PWD}/pipeline_model
fi
set +x