[jvm-packages] [CI] Publish XGBoost4J JARs with Scala 2.11 and 2.12

Jenkinsfile

@@ -95,6 +95,17 @@ pipeline {
         milestone ordinal: 4
       }
     }
+    stage('Jenkins Linux: Deploy') {
+      agent none
+      steps {
+        script {
+          parallel ([
+            'deploy-jvm-packages': { DeployJVMPackages(spark_version: '2.4.3') }
+          ])
+        }
+        milestone ordinal: 5
+      }
+    }
   }
 }
 
@@ -380,3 +391,18 @@ def TestR(args) {
     deleteDir()
   }
 }
+
+def DeployJVMPackages(args) {
+  node('linux && cpu') {
+    unstash name: 'srcs'
+    if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) {
+      echo 'Deploying to xgboost-maven-repo S3 repo...'
+      def container_type = "jvm"
+      def docker_binary = "docker"
+      sh """
+      ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/deploy_jvm_packages.sh ${args.spark_version}
+      """
+    }
+    deleteDir()
+  }
+}

R-package/tests/testthat/test_helpers.R

@@ -7,8 +7,8 @@ require(vcd, quietly = TRUE)
 
 float_tolerance = 5e-6
 
-# disable some tests for Win32
-win32_flag = .Platform$OS.type == "windows" && .Machine$sizeof.pointer != 8
+# disable some tests for 32-bit environment
+flag_32bit = .Machine$sizeof.pointer != 8
 
 set.seed(1982)
 data(Arthritis)
@@ -44,7 +44,7 @@ mbst.GLM <- xgboost(data = as.matrix(iris[, -5]), label = mlabel, verbose = 0,
 
 
 test_that("xgb.dump works", {
-  if (!win32_flag)
+  if (!flag_32bit)
     expect_length(xgb.dump(bst.Tree), 200)
   dump_file = file.path(tempdir(), 'xgb.model.dump')
   expect_true(xgb.dump(bst.Tree, dump_file, with_stats = T))
@@ -54,7 +54,7 @@ test_that("xgb.dump works", {
   # JSON format
   dmp <- xgb.dump(bst.Tree, dump_format = "json")
   expect_length(dmp, 1)
-  if (!win32_flag)
+  if (!flag_32bit)
     expect_length(grep('nodeid', strsplit(dmp, '\n')[[1]]), 188)
 })
 
@@ -256,7 +256,7 @@ test_that("xgb.model.dt.tree works with and without feature names", {
   names.dt.trees <- c("Tree", "Node", "ID", "Feature", "Split", "Yes", "No", "Missing", "Quality", "Cover")
   dt.tree <- xgb.model.dt.tree(feature_names = feature.names, model = bst.Tree)
   expect_equal(names.dt.trees, names(dt.tree))
-  if (!win32_flag)
+  if (!flag_32bit)
     expect_equal(dim(dt.tree), c(188, 10))
   expect_output(str(dt.tree), 'Feature.*\\"Age\\"')
 
@@ -283,7 +283,7 @@ test_that("xgb.model.dt.tree throws error for gblinear", {
 
 test_that("xgb.importance works with and without feature names", {
   importance.Tree <- xgb.importance(feature_names = feature.names, model = bst.Tree)
-  if (!win32_flag)
+  if (!flag_32bit)
     expect_equal(dim(importance.Tree), c(7, 4))
   expect_equal(colnames(importance.Tree), c("Feature", "Gain", "Cover", "Frequency"))
   expect_output(str(importance.Tree), 'Feature.*\\"Age\\"')

doc/jvm/index.rst

@@ -8,15 +8,143 @@ XGBoost JVM Package
   <img alt="Build Status" src="https://travis-ci.org/dmlc/xgboost.svg?branch=master">
   </a>
   <a href="https://github.com/dmlc/xgboost/blob/master/LICENSE">
-  <img alt="GitHub license" src="http://dmlc.github.io/img/apache2.svg">
+  <img alt="GitHub license" src="https://dmlc.github.io/img/apache2.svg">
   </a>
 
 You have found the XGBoost JVM Package!
 
+.. _install_jvm_packages:
+
 ************
 Installation
 ************
 
+.. contents::
+  :local:
+  :backlinks: none
+
+Installation from Maven repository
+==================================
+
+Access release version
+----------------------
+You can use XGBoost4J in your Java/Scala application by adding XGBoost4J as a dependency:
+
+.. code-block:: xml
+  :caption: Maven
+
+  <properties>
+    ...
+    <!-- Specify Scala version in package name -->
+    <scala.binary.version>2.12</scala.binary.version>
+  </properties>
+
+  <dependencies>
+    ...
+    <dependency>
+        <groupId>ml.dmlc</groupId>
+        <artifactId>xgboost4j_${scala.binary.version}</artifactId>
+        <version>latest_version_num</version>
+    </dependency>
+    <dependency>
+        <groupId>ml.dmlc</groupId>
+        <artifactId>xgboost4j-spark_${scala.binary.version}</artifactId>
+        <version>latest_version_num</version>
+    </dependency>
+  </dependencies>
+
+.. code-block:: scala
+  :caption: sbt
+
+  libraryDependencies ++= Seq(
+    "ml.dmlc" %% "xgboost4j" % "latest_version_num",
+    "ml.dmlc" %% "xgboost4j-spark" % "latest_version_num"
+  )
+
+This will check out the latest stable version from the Maven Central.
+
+For the latest release version number, please check `here <https://github.com/dmlc/xgboost/releases>`_.
+
+.. note:: Using Maven repository hosted by the XGBoost project
+
+  There may be some delay until a new release becomes available to Maven Central. If you would like to access the latest release immediately, add the Maven repository hosted by the XGBoost project:
+
+  .. code-block:: xml
+    :caption: Maven
+
+    <repository>
+      <id>XGBoost4J Release Repo</id>
+      <name>XGBoost4J Release Repo</name>
+      <url>https://s3-us-west-2.amazonaws.com/xgboost-maven-repo/release/</url>
+    </repository>
+
+  .. code-block:: scala
+    :caption: sbt
+
+    resolvers += "XGBoost4J Release Repo" at "https://s3-us-west-2.amazonaws.com/xgboost-maven-repo/release/"
+
+Access SNAPSHOT version
+-----------------------
+
+First add the following Maven repository hosted by the XGBoost project:
+
+.. code-block:: xml
+  :caption: Maven
+
+  <repository>
+    <id>XGBoost4J Snapshot Repo</id>
+    <name>XGBoost4J Snapshot Repo</name>
+    <url>https://s3-us-west-2.amazonaws.com/xgboost-maven-repo/snapshot/</url>
+  </repository>
+
+.. code-block:: scala
+  :caption: sbt
+
+  resolvers += "XGBoost4J Snapshot Repo" at "https://s3-us-west-2.amazonaws.com/xgboost-maven-repo/snapshot/"
+
+Then add XGBoost4J as a dependency:
+
+.. code-block:: xml
+  :caption: maven
+
+  <properties>
+    ...
+    <!-- Specify Scala version in package name -->
+    <scala.binary.version>2.12</scala.binary.version>
+  </properties>
+
+  <dependencies>
+    ...
+    <dependency>
+        <groupId>ml.dmlc</groupId>
+        <artifactId>xgboost4j_${scala.binary.version}</artifactId>
+        <version>latest_version_num-SNAPSHOT</version>
+    </dependency>
+    <dependency>
+        <groupId>ml.dmlc</groupId>
+        <artifactId>xgboost4j-spark_${scala.binary.version}</artifactId>
+        <version>latest_version_num-SNAPSHOT</version>
+    </dependency>
+  </dependencies>
+
+.. code-block:: scala
+  :caption: sbt
+
+  libraryDependencies ++= Seq(
+    "ml.dmlc" %% "xgboost4j" % "latest_version_num-SNAPSHOT",
+    "ml.dmlc" %% "xgboost4j-spark" % "latest_version_num-SNAPSHOT"
+  )
+
+Look up the ``version`` field in `pom.xml <https://github.com/dmlc/xgboost/blob/master/jvm-packages/pom.xml>`_ to get the correct version number.
+
+The SNAPSHOT JARs are hosted by the XGBoost project. Every commit in the ``master`` branch will automatically trigger generation of a new SNAPSHOT JAR. You can control how often Maven should upgrade your SNAPSHOT installation by specifying ``updatePolicy``. See `here <http://maven.apache.org/pom.html#Repositories>`_ for details.
+
+You can browse the file listing of the Maven repository at https://s3-us-west-2.amazonaws.com/xgboost-maven-repo/list.html.
+
+.. note:: Windows not supported by published JARs
+
+  The published JARs from the Maven Central and GitHub currently only supports Linux and MacOS. Windows users should consider building XGBoost4J / XGBoost4J-Spark from the source. Alternatively, checkout pre-built JARs from `criteo-forks/xgboost-jars <https://github.com/criteo-forks/xgboost-jars>`_.
+
 Installation from source
 ========================
 
@@ -64,73 +192,6 @@ If you want to use XGBoost4J-Spark, replace ``xgboost4j`` with ``xgboost4j-spark
 
   Also, make sure to install Spark directly from `Apache website <https://spark.apache.org/>`_. **Upstream XGBoost is not guaranteed to work with third-party distributions of Spark, such as Cloudera Spark.** Consult appropriate third parties to obtain their distribution of XGBoost.
 
-Installation from maven repo
-============================
-
-Access release version
-----------------------
-
-.. code-block:: xml
-  :caption: maven
-
-  <dependency>
-      <groupId>ml.dmlc</groupId>
-      <artifactId>xgboost4j</artifactId>
-      <version>latest_version_num</version>
-  </dependency>
-
-.. code-block:: scala
-  :caption: sbt
-
-  "ml.dmlc" % "xgboost4j" % "latest_version_num"
-
-This will checkout the latest stable version from the Maven Central.
-
-For the latest release version number, please check `here <https://github.com/dmlc/xgboost/releases>`_.
-
-if you want to use XGBoost4J-Spark, replace ``xgboost4j`` with ``xgboost4j-spark``.
-
-Access SNAPSHOT version
------------------------
-
-You need to add GitHub as repo:
-
-.. code-block:: xml
-  :caption: maven
-
-  <repository>
-    <id>GitHub Repo</id>
-    <name>GitHub Repo</name>
-    <url>https://raw.githubusercontent.com/CodingCat/xgboost/maven-repo/</url>
-  </repository>
-
-.. code-block:: scala
-  :caption: sbt
-
-  resolvers += "GitHub Repo" at "https://raw.githubusercontent.com/CodingCat/xgboost/maven-repo/"
-
-Then add dependency as following:
-
-.. code-block:: xml
-  :caption: maven
-
-  <dependency>
-      <groupId>ml.dmlc</groupId>
-      <artifactId>xgboost4j</artifactId>
-      <version>latest_version_num</version>
-  </dependency>
-
-.. code-block:: scala
-  :caption: sbt
-
-  "ml.dmlc" % "xgboost4j" % "latest_version_num"
-
-For the latest release version number, please check `here <https://github.com/CodingCat/xgboost/tree/maven-repo/ml/dmlc/xgboost4j>`_.
-
-.. note:: Windows not supported by published JARs
-
-  The published JARs from the Maven Central and GitHub currently only supports Linux and MacOS. Windows users should consider building XGBoost4J / XGBoost4J-Spark from the source. Alternatively, checkout pre-built JARs from `criteo-forks/xgboost-jars <https://github.com/criteo-forks/xgboost-jars>`_.
-
 Enabling OpenMP for Mac OS
 --------------------------
 If you are on Mac OS and using a compiler that supports OpenMP, you need to go to the file ``xgboost/jvm-packages/create_jni.py`` and comment out the line

doc/jvm/xgboost4j_spark_tutorial.rst

@@ -27,39 +27,7 @@ Build an ML Application with XGBoost4J-Spark
 Refer to XGBoost4J-Spark Dependency
 ===================================
 
-Before we go into the tour of how to use XGBoost4J-Spark, we would bring a brief introduction about how to build a machine learning application with XGBoost4J-Spark. The first thing you need to do is to refer to the dependency in Maven Central.
-
-You can add the following dependency in your ``pom.xml``.
-
-.. code-block:: xml
-
-  <dependency>
-    <groupId>ml.dmlc</groupId>
-    <artifactId>xgboost4j-spark</artifactId>
-    <version>latest_version_num</version>
-  </dependency>
-
-For the latest release version number, please check `here <https://github.com/dmlc/xgboost/releases>`_.
-
-We also publish some functionalities which would be included in the coming release in the form of snapshot version. To access these functionalities, you can add dependency to the snapshot artifacts. We publish snapshot version in github-based repo, so you can add the following repo in ``pom.xml``:
-
-.. code-block:: xml
-
-  <repository>
-    <id>XGBoost4J-Spark Snapshot Repo</id>
-    <name>XGBoost4J-Spark Snapshot Repo</name>
-    <url>https://raw.githubusercontent.com/CodingCat/xgboost/maven-repo/</url>
-  </repository>
-
-and then refer to the snapshot dependency by adding:
-
-.. code-block:: xml
-
-  <dependency>
-      <groupId>ml.dmlc</groupId>
-      <artifactId>xgboost4j-spark</artifactId>
-      <version>next_version_num-SNAPSHOT</version>
-  </dependency>
+Before we go into the tour of how to use XGBoost4J-Spark, you should first consult :ref:`Installation from Maven repository <install_jvm_packages>` in order to add XGBoost4J-Spark as a dependency for your project. We provide both stable releases and snapshots.
 
 .. note:: XGBoost4J-Spark requires Apache Spark 2.4+
 

jvm-packages/pom.xml

@@ -205,6 +205,29 @@
                 </plugins>
             </build>
         </profile>
+        <profile>
+            <id>release-to-s3</id>
+            <distributionManagement>
+                <snapshotRepository>
+                    <id>maven-s3-snapshot-repo</id>
+                    <url>s3://xgboost-maven-repo/snapshot</url>
+                </snapshotRepository>
+                <repository>
+                    <id>maven-s3-release-repo</id>
+                    <url>s3://xgboost-maven-repo/release</url>
+                </repository>
+            </distributionManagement>
+            <repositories>
+                <repository>
+                    <id>maven-s3-snapshot-repo</id>
+                    <url>https://s3.amazonaws.com/xgboost-maven-repo/snapshot</url>
+                </repository>
+                <repository>
+                    <id>maven-s3-release-repo</id>
+                    <url>https://s3.amazonaws.com/xgboost-maven-repo/release</url>
+                </repository>
+            </repositories>
+        </profile>
     </profiles>
     <distributionManagement>
         <snapshotRepository>
@@ -324,6 +347,13 @@
                 </executions>
             </plugin>
         </plugins>
+        <extensions>
+            <extension>
+                <groupId>org.kuali.maven.wagons</groupId>
+                <artifactId>maven-s3-wagon</artifactId>
+                <version>1.2.1</version>
+            </extension>
+        </extensions>
     </build>
     <reporting>
         <plugins>

src/common/hist_util.cc

@@ -663,7 +663,11 @@ void GHistIndexBlockMatrix::Init(const GHistIndexMatrix& gmat,
  * \brief fill a histogram by zeroes
  */
 void InitilizeHistByZeroes(GHistRow hist, size_t begin, size_t end) {
+#if defined(XGBOOST_STRICT_R_MODE) && XGBOOST_STRICT_R_MODE == 1
+  std::fill(hist.begin() + begin, hist.begin() + end, tree::GradStats());
+#else  // defined(XGBOOST_STRICT_R_MODE) && XGBOOST_STRICT_R_MODE == 1
   memset(hist.data() + begin, '\0', (end-begin)*sizeof(tree::GradStats));
+#endif  // defined(XGBOOST_STRICT_R_MODE) && XGBOOST_STRICT_R_MODE == 1
 }
 
 /*!

src/common/io.cc

@@ -117,14 +117,16 @@ std::string LoadSequentialFile(std::string fname) {
   size_t f_size_bytes = fs.st_size;
   buffer.resize(f_size_bytes + 1);
   int32_t fd = open(fname.c_str(), O_RDONLY);
+#if defined(__linux__)
   posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL);
+#endif  // defined(__linux__)
   ssize_t bytes_read = read(fd, &buffer[0], f_size_bytes);
   if (bytes_read < 0) {
     close(fd);
     ReadErr();
   }
   close(fd);
-#else
+#else  // defined(__unix__)
   FILE *f = fopen(fname.c_str(), "r");
   if (f == NULL) {
     std::string msg;

src/common/observer.h

@@ -15,15 +15,15 @@
 #include "xgboost/base.h"
 #include "xgboost/tree_model.h"
 
-#if defined(XGBOOST_STRICT_R_MODE)
+#if defined(XGBOOST_STRICT_R_MODE) && XGBOOST_STRICT_R_MODE == 1
 #define OBSERVER_PRINT LOG(INFO)
 #define OBSERVER_ENDL ""
 #define OBSERVER_NEWLINE ""
-#else
+#else  // defined(XGBOOST_STRICT_R_MODE) && XGBOOST_STRICT_R_MODE == 1
 #define OBSERVER_PRINT std::cout
 #define OBSERVER_ENDL std::endl
 #define OBSERVER_NEWLINE "\n"
-#endif  // defined(XGBOOST_STRICT_R_MODE)
+#endif  // defined(XGBOOST_STRICT_R_MODE) && XGBOOST_STRICT_R_MODE == 1
 
 namespace xgboost {
 /*\brief  An observer for logging internal data structures.

tests/ci_build/deploy_jvm_packages.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e
+set -x
+
+if [ $# -ne 1 ]; then
+  echo "Usage: $0 [spark version]"
+  exit 1
+fi
+
+spark_version=$1
+
+# Initialize local Maven repository
+./tests/ci_build/initialize_maven.sh
+
+rm -rf build/
+cd jvm-packages
+
+# Re-build package without Mock Rabit
+# Deploy to S3 bucket xgboost-maven-repo
+mvn --no-transfer-progress package deploy -P release-to-s3 -Dspark.version=${spark_version} -DskipTests
+
+# Compile XGBoost4J with Scala 2.11 too
+mvn clean
+# Rename artifactId of all XGBoost4J packages with suffix _2.11
+sed -i -e 's/<artifactId>xgboost\(.*\)_[0-9\.]\+/<artifactId>xgboost\1_2.11/' $(find . -name pom.xml)
+# Modify scala.version and scala.binary.version fields
+sed -i -e 's/<scala\.version>[0-9\.]\+/<scala.version>2.11.12/' $(find . -name pom.xml)
+sed -i -e 's/<scala\.binary\.version>[0-9\.]\+/<scala.binary.version>2.11/' $(find . -name pom.xml)
+
+# Re-build and deploy
+mvn --no-transfer-progress package deploy -P release-to-s3 -Dspark.version=${spark_version} -DskipTests
+
+set +x
+set +e