Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CI] Refactor Jenkins CI pipeline + migrate all Linux tests to Jenkins (#4401)

Browse Source 
* All Linux tests are now in Jenkins CI
* Tests are now de-coupled from builds. We can now build XGBoost with one version of CUDA/JDK and test it with another version of CUDA/JDK
* Builds (compilation) are significantly faster because 1) They use C5 instances with faster CPU cores; and 2) build environment setup is cached using Docker containers
This commit is contained in:
Philip Hyunsu Cho
2019-04-26 18:39:12 -07:00
committed by GitHub
parent 995698b0cb
commit ea850ecd20
35 changed files with 1046 additions and 585 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
jvm-packages/build_doc.sh
View File

@@ -1,34 +0,0 @@
#!/bin/bash
if [ $# -ne 1 ]; then
echo "Usage: $0 [commit id]"
exit 1
fi
set -e
set -x
commit_id=$1
# Install JVM packages in local Maven repository
mvn install -DskipTests
# Build Scaladocs
mvn scala:doc -DskipTests
# Build Javadocs
mvn javadoc:javadoc -DskipTests
# Package JVM docs in a tarball
mkdir -p tmp/scaladocs
cp -rv xgboost4j/target/site/apidocs/ ./tmp/javadocs/
cp -rv xgboost4j/target/site/scaladocs/ ./tmp/scaladocs/xgboost4j/
cp -rv xgboost4j-spark/target/site/scaladocs/ ./tmp/scaladocs/xgboost4j-spark/
cp -rv xgboost4j-flink/target/site/scaladocs/ ./tmp/scaladocs/xgboost4j-flink/
cd tmp
tar cvjf ${commit_id}.tar.bz2 javadocs/ scaladocs/
mv ${commit_id}.tar.bz2 ..
cd ..
rm -rfv tmp/
set +x
set +e

4
jvm-packages/xgboost4j-example/src/main/java/ml/dmlc/xgboost4j/java/example/ExternalMemory.java
View File

@@ -32,8 +32,8 @@ public class ExternalMemory {
//this is the only difference, add a # followed by a cache prefix name
//several cache file with the prefix will be generated
//currently only support convert from libsvm file
DMatrix trainMat = new DMatrix("../demo/data/agaricus.txt.train#dtrain.cache");
DMatrix testMat = new DMatrix("../demo/data/agaricus.txt.test#dtest.cache");
DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train#dtrain.cache");
DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test#dtest.cache");
//specify parameters
HashMap<String, Object> params = new HashMap<String, Object>();

208
jvm-packages/xgboost4j-tester/generate_pom.py Normal file
View File

@@ -0,0 +1,208 @@
import sys
pom_template = """
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost4j-tester</artifactId>
<version>1.0-SNAPSHOT</version>
<name>xgboost4j-tester</name>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>{maven_compiler_source}</maven.compiler.source>
<maven.compiler.target>{maven_compiler_target}</maven.compiler.target>
<spark.version>{spark_version}</spark.version>
<scala.version>{scala_version}</scala.version>
<scala.binary.version>{scala_binary_version}</scala.binary.version>
</properties>
<dependencies>
<dependency>
<groupId>com.esotericsoftware.kryo</groupId>
<artifactId>kryo</artifactId>
<version>2.21</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>${{scala.version}}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${{scala.version}}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${{scala.version}}</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-actor_${{scala.binary.version}}</artifactId>
<version>2.3.11</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-testkit_${{scala.binary.version}}</artifactId>
<version>2.3.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${{scala.binary.version}}</artifactId>
<version>3.0.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.4</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${{scala.binary.version}}</artifactId>
<version>${{spark.version}}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${{scala.binary.version}}</artifactId>
<version>${{spark.version}}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-mllib_${{scala.binary.version}}</artifactId>
<version>${{spark.version}}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost4j</artifactId>
<version>{xgboost4j_version}</version>
</dependency>
<dependency>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost4j</artifactId>
<version>{xgboost4j_version}</version>
<classifier>tests</classifier>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost4j-spark</artifactId>
<version>{xgboost4j_version}</version>
</dependency>
<dependency>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost4j-example</artifactId>
<version>{xgboost4j_version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle -->
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>3.1.0</version>
</plugin>
<!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
</plugin>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>ml.dmlc.xgboost4j.tester.App</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle -->
<plugin>
<artifactId>maven-site-plugin</artifactId>
<version>3.7.1</version>
</plugin>
<plugin>
<artifactId>maven-project-info-reports-plugin</artifactId>
<version>3.0.0</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.1</version>
<configuration>
<dependenciesToScan>
<dependency>ml.dmlc:xgboost4j</dependency>
</dependenciesToScan>
</configuration>
</plugin>
</plugins>
</build>
</project>
"""
if __name__ == '__main__':
if len(sys.argv) != 7:
print('Usage: {} [xgboost4j version] [maven compiler source level] [maven compiler target level] [spark version] [scala version] [scala binary version]'.format(sys.argv[0]))
sys.exit(1)
with open('pom.xml', 'w') as f:
print(pom_template.format(xgboost4j_version=sys.argv[1],
maven_compiler_source=sys.argv[2],
maven_compiler_target=sys.argv[3],
spark_version=sys.argv[4],
scala_version=sys.argv[5],
scala_binary_version=sys.argv[6]), file=f)

10
jvm-packages/xgboost4j-tester/get_iris.py Normal file
View File

@@ -0,0 +1,10 @@
from sklearn.datasets import load_iris
import numpy as np
import pandas
X, y = load_iris(return_X_y=True)
y = y.astype(np.int)
df = pandas.DataFrame(data=X, columns=['sepal length', 'sepal width', 'petal length', 'petal width'])
class_id_to_name = {0:'Iris-setosa', 1:'Iris-versicolor', 2:'Iris-virginica'}
df['class'] = np.vectorize(class_id_to_name.get)(y)
df.to_csv('./iris.csv', float_format='%.1f', header=False, index=False)

26
jvm-packages/xgboost4j-tester/src/main/java/ml/dmlc/xgboost4j/tester/App.java Normal file
View File

@@ -0,0 +1,26 @@
package ml.dmlc.xgboost4j.tester;
import ml.dmlc.xgboost4j.java.example.*;
import java.io.IOException;
import ml.dmlc.xgboost4j.java.XGBoostError;
public class App {
public static void main(String[] args) throws IOException, XGBoostError {
String[] args2 = new String[0];
System.out.println("BoostFromPrediction");
BoostFromPrediction.main(args2);
System.out.println("CrossValidation");
CrossValidation.main(args2);
System.out.println("CustomObjective");
CustomObjective.main(args2);
System.out.println("ExternalMemory");
ExternalMemory.main(args2);
System.out.println("GeneralizedLinearModel");
GeneralizedLinearModel.main(args2);
System.out.println("PredictFirstNtree");
PredictFirstNtree.main(args2);
System.out.println("PredictLeafIndices");
PredictLeafIndices.main(args2);
}
}

20
jvm-packages/xgboost4j-tester/src/test/java/ml/dmlc/xgboost4j/tester/AppTest.java Normal file
View File

@@ -0,0 +1,20 @@
package ml.dmlc.xgboost4j.tester;
import static org.junit.Assert.assertTrue;
import org.junit.Test;
/**
* Unit test for simple App.
*/
public class AppTest
{
/**
* Rigorous Test :-)
*/
@Test
public void shouldAnswerWithTrue()
{
assertTrue( true );
}
}

12
jvm-packages/xgboost4j/pom.xml
View File

@@ -71,6 +71,18 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>