Release 1.1.0

Co-authored-by: fis <jm.yuan@outlook.com>

.travis.yml

@@ -6,7 +6,7 @@ os:
   - linux
   - osx
 
-osx_image: xcode10.3
+osx_image: xcode10.1
 dist: bionic
 
 # Use Build Matrix to do lint and build seperately
@@ -21,6 +21,10 @@ env:
     # cmake test
     - TASK=cmake_test
 
+  global:
+    - secure: "PR16i9F8QtNwn99C5NDp8nptAS+97xwDtXEJJfEiEVhxPaaRkOp0MPWhogCaK0Eclxk1TqkgWbdXFknwGycX620AzZWa/A1K3gAs+GrpzqhnPMuoBJ0Z9qxXTbSJvCyvMbYwVrjaxc/zWqdMU8waWz8A7iqKGKs/SqbQ3rO6v7c="
+    - secure: "dAGAjBokqm/0nVoLMofQni/fWIBcYSmdq4XvCBX1ZAMDsWnuOfz/4XCY6h2lEI1rVHZQ+UdZkc9PioOHGPZh5BnvE49/xVVWr9c4/61lrDOlkD01ZjSAeoV0fAZq+93V/wPl4QV+MM+Sem9hNNzFSbN5VsQLAiWCSapWsLdKzqA="
+
 matrix:
   exclude:
     - os: linux

Jenkinsfile

@@ -64,7 +64,6 @@ pipeline {
             'build-cpu': { BuildCPU() },
             'build-cpu-rabit-mock': { BuildCPUMock() },
             'build-cpu-non-omp': { BuildCPUNonOmp() },
-            'build-gpu-cuda9.0': { BuildCUDA(cuda_version: '9.0') },
             'build-gpu-cuda10.0': { BuildCUDA(cuda_version: '10.0') },
             'build-gpu-cuda10.1': { BuildCUDA(cuda_version: '10.1') },
             'build-jvm-packages': { BuildJVMPackages(spark_version: '2.4.3') },
@@ -251,10 +250,10 @@ def BuildCUDA(args) {
     ${dockerRun} ${container_type} ${docker_binary} ${docker_args} bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal"
     ${dockerRun} ${container_type} ${docker_binary} ${docker_args} python3 tests/ci_build/rename_whl.py python-package/dist/*.whl ${commit_id} manylinux2010_x86_64
     """
-    // Stash wheel for CUDA 9.0 target
-    if (args.cuda_version == '9.0') {
+    // Stash wheel for CUDA 10.0 target
+    if (args.cuda_version == '10.0') {
       echo 'Stashing Python wheel...'
-      stash name: 'xgboost_whl_cuda9', includes: 'python-package/dist/*.whl'
+      stash name: 'xgboost_whl_cuda10', includes: 'python-package/dist/*.whl'
       path = ("${BRANCH_NAME}" == 'master') ? '' : "${BRANCH_NAME}/"
       s3Upload bucket: 'xgboost-nightly-builds', path: path, acl: 'PublicRead', workingDir: 'python-package/dist', includePathPattern:'**/*.whl'
       echo 'Stashing C++ test executable (testxgboost)...'
@@ -298,7 +297,7 @@ def BuildJVMDoc() {
 
 def TestPythonCPU() {
   node('linux && cpu') {
-    unstash name: 'xgboost_whl_cuda9'
+    unstash name: 'xgboost_whl_cuda10'
     unstash name: 'srcs'
     unstash name: 'xgboost_cli'
     echo "Test Python CPU"
@@ -315,7 +314,7 @@ def TestPythonCPU() {
 def TestPythonGPU(args) {
   nodeReq = (args.multi_gpu) ? 'linux && mgpu' : 'linux && gpu'
   node(nodeReq) {
-    unstash name: 'xgboost_whl_cuda9'
+    unstash name: 'xgboost_whl_cuda10'
     unstash name: 'srcs'
     echo "Test Python GPU: CUDA ${args.cuda_version}"
     def container_type = "gpu"

R-package/vignettes/xgboostPresentation.Rmd

@@ -410,7 +410,7 @@ In some very specific cases, like when you want to pilot **XGBoost** from `caret
 
 ```{r saveLoadRBinVectorModel, message=F, warning=F}
 # save model to R's raw vector
-rawVec <- xgb.save.raw(bst)
+rawVec <- xgb.serialize(bst)
 
 # print class
 print(class(rawVec))

cmake/Python_version.in

@@ -1 +1 @@
-@xgboost_VERSION_MAJOR@.@xgboost_VERSION_MINOR@.@xgboost_VERSION_PATCH@-SNAPSHOT
+@xgboost_VERSION_MAJOR@.@xgboost_VERSION_MINOR@.@xgboost_VERSION_PATCH@

doc/dump.schema

@@ -0,0 +1,55 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "definitions": {
+        "split_node": {
+            "type": "object",
+            "properties": {
+                "nodeid": {
+                    "type": "number",
+                    "minimum": 0
+                },
+                "depth": {
+                    "type": "number",
+                    "minimum": 0
+                },
+                "yes": {
+                    "type": "number",
+                    "minimum": 0
+                },
+                "no": {
+                    "type": "number",
+                    "minimum": 0
+                },
+                "split": {
+                    "type": "string"
+                },
+                "children": {
+                    "type": "array",
+                    "items": {
+                        "oneOf": [
+                            {"$ref": "#/definitions/split_node"},
+                            {"$ref": "#/definitions/leaf_node"}
+                        ]
+                    },
+                    "maxItems": 2
+                }
+            },
+            "required": ["nodeid", "depth", "yes", "no", "split", "children"]
+        },
+        "leaf_node": {
+            "type": "object",
+            "properties": {
+                "nodeid": {
+                    "type": "number",
+                    "minimum": 0
+                },
+                "leaf": {
+                    "type": "number"
+                }
+            },
+            "required": ["nodeid", "leaf"]
+        }
+    },
+    "type": "object",
+    "$ref": "#/definitions/split_node"
+}

include/xgboost/c_api.h

@@ -418,7 +418,14 @@ XGB_DLL int XGBoosterEvalOneIter(BoosterHandle handle,
  *          4:output feature contributions to individual predictions
  * \param ntree_limit limit number of trees used for prediction, this is only valid for boosted trees
  *    when the parameter is set to 0, we will use all the trees
- * \param training Whether the prediction value is used for training.
+ * \param training Whether the prediction function is used as part of a training loop.
+ *    Prediction can be run in 2 scenarios:
+ *    1. Given data matrix X, obtain prediction y_pred from the model.
+ *    2. Obtain the prediction for computing gradients. For example, DART booster performs dropout
+ *       during training, and the prediction result will be different from the one obtained by normal
+ *       inference step due to dropped trees.
+ *    Set training=false for the first scenario. Set training=true for the second scenario.
+ *    The second scenario applies when you are defining a custom objective function.
  * \param out_len used to store length of returning result
  * \param out_result used to set a pointer to array
  * \return 0 when success, -1 when failure happens

include/xgboost/data.h

@@ -9,6 +9,7 @@
 
 #include <dmlc/base.h>
 #include <dmlc/data.h>
+#include <dmlc/serializer.h>
 #include <rabit/rabit.h>
 #include <xgboost/base.h>
 #include <xgboost/span.h>
@@ -102,7 +103,7 @@ class MetaInfo {
   /*!
    * \brief Validate all metainfo.
    */
-  void Validate() const;
+  void Validate(int32_t device) const;
 
   MetaInfo Slice(common::Span<int32_t const> ridxs) const;
   /*!
@@ -554,5 +555,21 @@ inline BatchSet<EllpackPage> DMatrix::GetBatches(const BatchParam& param) {
 
 namespace dmlc {
 DMLC_DECLARE_TRAITS(is_pod, xgboost::Entry, true);
+
+namespace serializer {
+
+template <>
+struct Handler<xgboost::Entry> {
+  inline static void Write(Stream* strm, const xgboost::Entry& data) {
+    strm->Write(data.index);
+    strm->Write(data.fvalue);
   }
+
+  inline static bool Read(Stream* strm, xgboost::Entry* data) {
+    return strm->Read(&data->index) && strm->Read(&data->fvalue);
+  }
+};
+
+}  // namespace serializer
+}  // namespace dmlc
 #endif  // XGBOOST_DATA_H_

include/xgboost/span.h

@@ -30,6 +30,7 @@
 #define XGBOOST_SPAN_H_
 
 #include <xgboost/base.h>
+#include <xgboost/logging.h>
 
 #include <cinttypes>          // size_t
 #include <limits>             // numeric_limits
@@ -85,9 +86,11 @@ namespace common {
     }                                                                          \
   } while (0);
 
-#ifdef __CUDA_ARCH__
+#if defined(__CUDA_ARCH__)
 #define SPAN_CHECK KERNEL_CHECK
-#else
+#elif defined(XGBOOST_STRICT_R_MODE) && XGBOOST_STRICT_R_MODE == 1  // R package
+#define SPAN_CHECK CHECK  // check from dmlc
+#else  // not CUDA, not R
 #define SPAN_CHECK(cond)                                                       \
   do {                                                                         \
     if (XGBOOST_EXPECT(!(cond), false)) {                                      \

jvm-packages/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>ml.dmlc</groupId>
     <artifactId>xgboost-jvm_2.12</artifactId>
-    <version>1.1.0-SNAPSHOT</version>
+    <version>1.1.0</version>
     <packaging>pom</packaging>
     <name>XGBoost JVM Package</name>
     <description>JVM Package for XGBoost</description>

jvm-packages/xgboost4j-example/pom.xml

@@ -6,10 +6,10 @@
     <parent>
         <groupId>ml.dmlc</groupId>
         <artifactId>xgboost-jvm_2.12</artifactId>
-        <version>1.1.0-SNAPSHOT</version>
+        <version>1.1.0</version>
     </parent>
     <artifactId>xgboost4j-example_2.12</artifactId>
-    <version>1.1.0-SNAPSHOT</version>
+    <version>1.1.0</version>
     <packaging>jar</packaging>
     <build>
         <plugins>
@@ -26,7 +26,7 @@
         <dependency>
             <groupId>ml.dmlc</groupId>
             <artifactId>xgboost4j-spark_${scala.binary.version}</artifactId>
-            <version>1.1.0-SNAPSHOT</version>
+            <version>1.1.0</version>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
@@ -37,7 +37,7 @@
         <dependency>
             <groupId>ml.dmlc</groupId>
             <artifactId>xgboost4j-flink_${scala.binary.version}</artifactId>
-            <version>1.1.0-SNAPSHOT</version>
+            <version>1.1.0</version>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>

jvm-packages/xgboost4j-flink/pom.xml

@@ -6,10 +6,10 @@
     <parent>
         <groupId>ml.dmlc</groupId>
         <artifactId>xgboost-jvm_2.12</artifactId>
-        <version>1.1.0-SNAPSHOT</version>
+        <version>1.1.0</version>
     </parent>
     <artifactId>xgboost4j-flink_2.12</artifactId>
-    <version>1.1.0-SNAPSHOT</version>
+    <version>1.1.0</version>
     <build>
         <plugins>
             <plugin>
@@ -26,7 +26,7 @@
         <dependency>
             <groupId>ml.dmlc</groupId>
             <artifactId>xgboost4j_${scala.binary.version}</artifactId>
-            <version>1.1.0-SNAPSHOT</version>
+            <version>1.1.0</version>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>

jvm-packages/xgboost4j-spark/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>ml.dmlc</groupId>
         <artifactId>xgboost-jvm_2.12</artifactId>
-        <version>1.1.0-SNAPSHOT</version>
+        <version>1.1.0</version>
     </parent>
     <artifactId>xgboost4j-spark_2.12</artifactId>
     <build>
@@ -24,7 +24,7 @@
         <dependency>
             <groupId>ml.dmlc</groupId>
             <artifactId>xgboost4j_${scala.binary.version}</artifactId>
-            <version>1.1.0-SNAPSHOT</version>
+            <version>1.1.0</version>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>

jvm-packages/xgboost4j/pom.xml

@@ -6,10 +6,10 @@
     <parent>
         <groupId>ml.dmlc</groupId>
         <artifactId>xgboost-jvm_2.12</artifactId>
-        <version>1.1.0-SNAPSHOT</version>
+        <version>1.1.0</version>
     </parent>
     <artifactId>xgboost4j_2.12</artifactId>
-    <version>1.1.0-SNAPSHOT</version>
+    <version>1.1.0</version>
     <packaging>jar</packaging>
 
     <dependencies>

python-package/xgboost/VERSION

@@ -1 +1 @@
-1.1.0-SNAPSHOT
+1.1.0

python-package/xgboost/core.py

@@ -159,8 +159,9 @@ def _load_lib():
             'XGBoost Library ({}) could not be loaded.\n'.format(libname) +
             'Likely causes:\n' +
             '  * OpenMP runtime is not installed ' +
-            '(vcomp140.dll or libgomp-1.dll for Windows, ' +
-            'libgomp.so for UNIX-like OSes)\n' +
+            '(vcomp140.dll or libgomp-1.dll for Windows, libomp.dylib for Mac OSX, ' +
+            'libgomp.so for Linux and other UNIX-like OSes). Mac OSX users: Run ' +
+            '`brew install libomp` to install OpenMP runtime.\n' +
             '  * You are running 32-bit Python on a 64-bit OS\n' +
             'Error message(s): {}\n'.format(os_error_list))
     lib.XGBGetLastError.restype = ctypes.c_char_p

src/c_api/c_api.cc

@@ -426,7 +426,7 @@ XGB_DLL int XGBoosterPredict(BoosterHandle handle,
                              DMatrixHandle dmat,
                              int option_mask,
                              unsigned ntree_limit,
-                             int32_t training,
+                             int training,
                              xgboost::bst_ulong *len,
                              const bst_float **out_result) {
   API_BEGIN();

src/common/column_matrix.h

@@ -82,14 +82,16 @@ template <typename BinIdxType>
 class DenseColumn: public Column<BinIdxType> {
  public:
   DenseColumn(ColumnType type, common::Span<const BinIdxType> index,
-              uint32_t index_base,
-              const std::vector<bool>::const_iterator missing_flags)
+              uint32_t index_base, const std::vector<bool>& missing_flags,
+              size_t feature_offset)
       : Column<BinIdxType>(type, index, index_base),
-        missing_flags_(missing_flags) {}
-  bool IsMissing(size_t idx) const { return missing_flags_[idx]; }
+        missing_flags_(missing_flags),
+        feature_offset_(feature_offset) {}
+  bool IsMissing(size_t idx) const { return missing_flags_[feature_offset_ + idx]; }
  private:
   /* flags for missing values in dense columns */
-  std::vector<bool>::const_iterator missing_flags_;
+  const std::vector<bool>& missing_flags_;
+  size_t feature_offset_;
 };
 
 /*! \brief a collection of columns, with support for construction from
@@ -208,10 +210,8 @@ class ColumnMatrix {
                                                  column_size };
     std::unique_ptr<const Column<BinIdxType> > res;
     if (type_[fid] == ColumnType::kDenseColumn) {
-      std::vector<bool>::const_iterator column_iterator = missing_flags_.begin();
-      advance(column_iterator, feature_offset);  // increment iterator to right position
       res.reset(new DenseColumn<BinIdxType>(type_[fid], bin_index, index_base_[fid],
-                                            column_iterator));
+                                             missing_flags_, feature_offset));
     } else {
       res.reset(new SparseColumn<BinIdxType>(type_[fid], bin_index, index_base_[fid],
                                        {&row_ind_[feature_offset], column_size}));

src/data/data.cc

@@ -37,7 +37,7 @@ template <typename T>
 void SaveScalarField(dmlc::Stream *strm, const std::string &name,
                      xgboost::DataType type, const T &field) {
   strm->Write(name);
-  strm->Write(type);
+  strm->Write(static_cast<uint8_t>(type));
   strm->Write(true);  // is_scalar=True
   strm->Write(field);
 }
@@ -47,7 +47,7 @@ void SaveVectorField(dmlc::Stream *strm, const std::string &name,
                      xgboost::DataType type, std::pair<uint64_t, uint64_t> shape,
                      const std::vector<T>& field) {
   strm->Write(name);
-  strm->Write(type);
+  strm->Write(static_cast<uint8_t>(type));
   strm->Write(false);  // is_scalar=False
   strm->Write(shape.first);
   strm->Write(shape.second);
@@ -71,7 +71,9 @@ void LoadScalarField(dmlc::Stream* strm, const std::string& expected_name,
   CHECK(strm->Read(&name)) << invalid;
   CHECK_EQ(name, expected_name)
       << invalid << " Expected field: " << expected_name << ", got: " << name;
-  CHECK(strm->Read(&type)) << invalid;
+  uint8_t type_val;
+  CHECK(strm->Read(&type_val)) << invalid;
+  type = static_cast<xgboost::DataType>(type_val);
   CHECK(type == expected_type)
       << invalid << "Expected field of type: " << static_cast<int>(expected_type) << ", "
       << "got field type: " << static_cast<int>(type);
@@ -91,7 +93,9 @@ void LoadVectorField(dmlc::Stream* strm, const std::string& expected_name,
   CHECK(strm->Read(&name)) << invalid;
   CHECK_EQ(name, expected_name)
     << invalid << " Expected field: " << expected_name << ", got: " << name;
-  CHECK(strm->Read(&type)) << invalid;
+  uint8_t type_val;
+  CHECK(strm->Read(&type_val)) << invalid;
+  type = static_cast<xgboost::DataType>(type_val);
   CHECK(type == expected_type)
     << invalid << "Expected field of type: " << static_cast<int>(expected_type) << ", "
     << "got field type: " << static_cast<int>(type);
@@ -338,7 +342,7 @@ void MetaInfo::SetInfo(const char* key, const void* dptr, DataType dtype, size_t
   }
 }
 
-void MetaInfo::Validate() const {
+void MetaInfo::Validate(int32_t device) const {
   if (group_ptr_.size() != 0 && weights_.Size() != 0) {
     CHECK_EQ(group_ptr_.size(), weights_.Size() + 1)
         << "Size of weights must equal to number of groups when ranking "
@@ -350,30 +354,44 @@ void MetaInfo::Validate() const {
         << "Invalid group structure.  Number of rows obtained from groups "
            "doesn't equal to actual number of rows given by data.";
   }
+  auto check_device = [device](HostDeviceVector<float> const &v) {
+    CHECK(v.DeviceIdx() == GenericParameter::kCpuId ||
+          device  == GenericParameter::kCpuId ||
+          v.DeviceIdx() == device)
+        << "Data is resided on a different device than `gpu_id`. "
+        << "Device that data is on: " << v.DeviceIdx() << ", "
+        << "`gpu_id` for XGBoost: " << device;
+  };
+
   if (weights_.Size() != 0) {
     CHECK_EQ(weights_.Size(), num_row_)
         << "Size of weights must equal to number of rows.";
+    check_device(weights_);
     return;
   }
   if (labels_.Size() != 0) {
     CHECK_EQ(labels_.Size(), num_row_)
         << "Size of labels must equal to number of rows.";
+    check_device(labels_);
     return;
   }
   if (labels_lower_bound_.Size() != 0) {
     CHECK_EQ(labels_lower_bound_.Size(), num_row_)
         << "Size of label_lower_bound must equal to number of rows.";
+    check_device(labels_lower_bound_);
     return;
   }
   if (labels_upper_bound_.Size() != 0) {
     CHECK_EQ(labels_upper_bound_.Size(), num_row_)
         << "Size of label_upper_bound must equal to number of rows.";
+    check_device(labels_upper_bound_);
     return;
   }
   CHECK_LE(num_nonzero_, num_col_ * num_row_);
   if (base_margin_.Size() != 0) {
     CHECK_EQ(base_margin_.Size() % num_row_, 0)
         << "Size of base margin must be a multiple of number of rows.";
+    check_device(base_margin_);
   }
 }
 

src/data/device_dmatrix.cu

@@ -201,6 +201,7 @@ template <typename AdapterT>
 DeviceDMatrix::DeviceDMatrix(AdapterT* adapter, float missing, int nthread, int max_bin) {
   common::HistogramCuts cuts =
       common::AdapterDeviceSketch(adapter, max_bin, missing);
+  dh::safe_cuda(cudaSetDevice(adapter->DeviceIdx()));
   auto& batch = adapter->Value();
   // Work out how many valid entries we have in each row
   dh::caching_device_vector<size_t> row_counts(adapter->NumRows() + 1, 0);

src/data/simple_dmatrix.cu

@@ -99,6 +99,7 @@ void CopyDataRowMajor(AdapterT* adapter, common::Span<Entry> data,
 // be supported in future. Does not currently support inferring row/column size
 template <typename AdapterT>
 SimpleDMatrix::SimpleDMatrix(AdapterT* adapter, float missing, int nthread) {
+  dh::safe_cuda(cudaSetDevice(adapter->DeviceIdx()));
   CHECK(adapter->NumRows() != kAdapterUnknownSize);
   CHECK(adapter->NumColumns() != kAdapterUnknownSize);
 

src/learner.cc

@@ -1052,7 +1052,7 @@ class LearnerImpl : public LearnerIO {
 
   void ValidateDMatrix(DMatrix* p_fmat) const {
     MetaInfo const& info = p_fmat->Info();
-    info.Validate();
+    info.Validate(generic_parameters_.gpu_id);
 
     auto const row_based_split = [this]() {
       return tparam_.dsplit == DataSplitMode::kRow ||

src/metric/survival_metric.cc

@@ -70,8 +70,7 @@ struct EvalAFT : public Metric {
 
     double nloglik_sum = 0.0;
     double weight_sum = 0.0;
-    #pragma omp parallel for default(none) \
-     firstprivate(nsize, is_null_weight, aft_loss_distribution_scale) \
+    #pragma omp parallel for \
      shared(weights, y_lower, y_upper, yhat) reduction(+:nloglik_sum, weight_sum)
     for (omp_ulong i = 0; i < nsize; ++i) {
       // If weights are empty, data is unweighted so we use 1.0 everywhere

src/objective/aft_obj.cc

@@ -56,8 +56,7 @@ class AFTObj : public ObjFunction {
     const omp_ulong nsize = static_cast<omp_ulong>(yhat.size());
     const float aft_loss_distribution_scale = param_.aft_loss_distribution_scale;
 
-    #pragma omp parallel for default(none) \
-      firstprivate(nsize, is_null_weight, aft_loss_distribution_scale) \
+    #pragma omp parallel for \
       shared(weights, y_lower, y_upper, yhat, gpair)
     for (omp_ulong i = 0; i < nsize; ++i) {
       // If weights are empty, data is unweighted so we use 1.0 everywhere
@@ -74,7 +73,7 @@ class AFTObj : public ObjFunction {
     // Trees give us a prediction in log scale, so exponentiate
     std::vector<bst_float> &preds = io_preds->HostVector();
     const long ndata = static_cast<long>(preds.size()); // NOLINT(*)
-    #pragma omp parallel for default(none) firstprivate(ndata) shared(preds)
+    #pragma omp parallel for shared(preds)
     for (long j = 0; j < ndata; ++j) {  // NOLINT(*)
       preds[j] = std::exp(preds[j]);
     }

src/tree/tree_model.cc

@@ -68,20 +68,20 @@ class TreeGenerator {
     return result;
   }
 
-  virtual std::string Indicator(RegTree const& tree, int32_t nid, uint32_t depth) {
+  virtual std::string Indicator(RegTree const& tree, int32_t nid, uint32_t depth) const {
     return "";
   }
-  virtual std::string Integer(RegTree const& tree, int32_t nid, uint32_t depth) {
+  virtual std::string Integer(RegTree const& tree, int32_t nid, uint32_t depth) const {
     return "";
   }
-  virtual std::string Quantitive(RegTree const& tree, int32_t nid, uint32_t depth) {
+  virtual std::string Quantitive(RegTree const& tree, int32_t nid, uint32_t depth) const {
     return "";
   }
-  virtual std::string NodeStat(RegTree const& tree, int32_t nid) {
+  virtual std::string NodeStat(RegTree const& tree, int32_t nid) const {
     return "";
   }
 
-  virtual std::string PlainNode(RegTree const& tree, int32_t nid, uint32_t depth) = 0;
+  virtual std::string PlainNode(RegTree const& tree, int32_t nid, uint32_t depth) const = 0;
 
   virtual std::string SplitNode(RegTree const& tree, int32_t nid, uint32_t depth) {
     auto const split_index = tree[nid].SplitIndex();
@@ -110,7 +110,7 @@ class TreeGenerator {
     return result;
   }
 
-  virtual std::string LeafNode(RegTree const& tree, int32_t nid, uint32_t depth) = 0;
+  virtual std::string LeafNode(RegTree const& tree, int32_t nid, uint32_t depth) const = 0;
   virtual std::string BuildTree(RegTree const& tree, int32_t nid, uint32_t depth) = 0;
 
  public:
@@ -181,7 +181,7 @@ class TextGenerator : public TreeGenerator {
   TextGenerator(FeatureMap const& fmap, std::string const& attrs, bool with_stats) :
       TreeGenerator(fmap, with_stats) {}
 
-  std::string LeafNode(RegTree const& tree, int32_t nid, uint32_t depth) override {
+  std::string LeafNode(RegTree const& tree, int32_t nid, uint32_t depth) const override {
     static std::string kLeafTemplate = "{tabs}{nid}:leaf={leaf}{stats}";
     static std::string kStatTemplate = ",cover={cover}";
     std::string result = SuperT::Match(
@@ -195,7 +195,7 @@ class TextGenerator : public TreeGenerator {
     return result;
   }
 
-  std::string Indicator(RegTree const& tree, int32_t nid, uint32_t depth) override {
+  std::string Indicator(RegTree const& tree, int32_t nid, uint32_t depth) const override {
     static std::string const kIndicatorTemplate = "{nid}:[{fname}] yes={yes},no={no}";
     int32_t nyes = tree[nid].DefaultLeft() ?
                    tree[nid].RightChild() : tree[nid].LeftChild();
@@ -211,7 +211,7 @@ class TextGenerator : public TreeGenerator {
 
   std::string SplitNodeImpl(
       RegTree const& tree, int32_t nid, std::string const& template_str,
-      std::string cond, uint32_t depth) {
+      std::string cond, uint32_t depth) const {
     auto split_index = tree[nid].SplitIndex();
     std::string const result = SuperT::Match(
         template_str,
@@ -226,7 +226,7 @@ class TextGenerator : public TreeGenerator {
     return result;
   }
 
-  std::string Integer(RegTree const& tree, int32_t nid, uint32_t depth) override {
+  std::string Integer(RegTree const& tree, int32_t nid, uint32_t depth) const override {
     static std::string const kIntegerTemplate =
         "{tabs}{nid}:[{fname}<{cond}] yes={left},no={right},missing={missing}";
     auto cond = tree[nid].SplitCond();
@@ -238,21 +238,21 @@ class TextGenerator : public TreeGenerator {
                          std::to_string(integer_threshold), depth);
   }
 
-  std::string Quantitive(RegTree const& tree, int32_t nid, uint32_t depth) override {
+  std::string Quantitive(RegTree const& tree, int32_t nid, uint32_t depth) const override {
     static std::string const kQuantitiveTemplate =
         "{tabs}{nid}:[{fname}<{cond}] yes={left},no={right},missing={missing}";
     auto cond = tree[nid].SplitCond();
     return SplitNodeImpl(tree, nid, kQuantitiveTemplate, SuperT::ToStr(cond), depth);
   }
 
-  std::string PlainNode(RegTree const& tree, int32_t nid, uint32_t depth) override {
+  std::string PlainNode(RegTree const& tree, int32_t nid, uint32_t depth) const override {
     auto cond = tree[nid].SplitCond();
     static std::string const kNodeTemplate =
         "{tabs}{nid}:[f{fname}<{cond}] yes={left},no={right},missing={missing}";
     return SplitNodeImpl(tree, nid, kNodeTemplate, SuperT::ToStr(cond), depth);
   }
 
-  std::string NodeStat(RegTree const& tree, int32_t nid) override {
+  std::string NodeStat(RegTree const& tree, int32_t nid) const override {
     static std::string const kStatTemplate = ",gain={loss_chg},cover={sum_hess}";
     std::string const result = SuperT::Match(
         kStatTemplate,
@@ -297,7 +297,7 @@ class JsonGenerator : public TreeGenerator {
   JsonGenerator(FeatureMap const& fmap, std::string attrs, bool with_stats) :
       TreeGenerator(fmap, with_stats) {}
 
-  std::string Indent(uint32_t depth) {
+  std::string Indent(uint32_t depth) const {
     std::string result;
     for (uint32_t i = 0; i < depth + 1; ++i) {
       result += "  ";
@@ -305,7 +305,7 @@ class JsonGenerator : public TreeGenerator {
     return result;
   }
 
-  std::string LeafNode(RegTree const& tree, int32_t nid, uint32_t depth) override {
+  std::string LeafNode(RegTree const& tree, int32_t nid, uint32_t depth) const override {
     static std::string const kLeafTemplate =
         R"L({ "nodeid": {nid}, "leaf": {leaf} {stat}})L";
     static std::string const kStatTemplate =
@@ -321,11 +321,11 @@ class JsonGenerator : public TreeGenerator {
     return result;
   }
 
-  std::string Indicator(RegTree const& tree, int32_t nid, uint32_t depth) override {
+  std::string Indicator(RegTree const& tree, int32_t nid, uint32_t depth) const override {
     int32_t nyes = tree[nid].DefaultLeft() ?
                    tree[nid].RightChild() : tree[nid].LeftChild();
     static std::string const kIndicatorTemplate =
-        R"ID( "nodeid": {nid}, "depth": {depth}, "split": "{fname}", "yes": {yes}, "no": {no}})ID";
+        R"ID( "nodeid": {nid}, "depth": {depth}, "split": "{fname}", "yes": {yes}, "no": {no})ID";
     auto split_index = tree[nid].SplitIndex();
     auto result = SuperT::Match(
         kIndicatorTemplate,
@@ -338,7 +338,8 @@ class JsonGenerator : public TreeGenerator {
   }
 
   std::string SplitNodeImpl(RegTree const &tree, int32_t nid,
-                            std::string const& template_str, std::string cond, uint32_t depth) {
+                            std::string const &template_str, std::string cond,
+                            uint32_t depth) const {
     auto split_index = tree[nid].SplitIndex();
     std::string const result = SuperT::Match(
         template_str,
@@ -353,7 +354,7 @@ class JsonGenerator : public TreeGenerator {
     return result;
   }
 
-  std::string Integer(RegTree const& tree, int32_t nid, uint32_t depth) override {
+  std::string Integer(RegTree const& tree, int32_t nid, uint32_t depth) const override {
     auto cond = tree[nid].SplitCond();
     const bst_float floored = std::floor(cond);
     const int32_t integer_threshold
@@ -367,7 +368,7 @@ class JsonGenerator : public TreeGenerator {
                          std::to_string(integer_threshold), depth);
   }
 
-  std::string Quantitive(RegTree const& tree, int32_t nid, uint32_t depth) override {
+  std::string Quantitive(RegTree const& tree, int32_t nid, uint32_t depth) const override {
     static std::string const kQuantitiveTemplate =
         R"I( "nodeid": {nid}, "depth": {depth}, "split": "{fname}", )I"
         R"I("split_condition": {cond}, "yes": {left}, "no": {right}, )I"
@@ -376,7 +377,7 @@ class JsonGenerator : public TreeGenerator {
     return SplitNodeImpl(tree, nid, kQuantitiveTemplate, SuperT::ToStr(cond), depth);
   }
 
-  std::string PlainNode(RegTree const& tree, int32_t nid, uint32_t depth) override {
+  std::string PlainNode(RegTree const& tree, int32_t nid, uint32_t depth) const override {
     auto cond = tree[nid].SplitCond();
     static std::string const kNodeTemplate =
         R"I( "nodeid": {nid}, "depth": {depth}, "split": {fname}, )I"
@@ -385,7 +386,7 @@ class JsonGenerator : public TreeGenerator {
     return SplitNodeImpl(tree, nid, kNodeTemplate, SuperT::ToStr(cond), depth);
   }
 
-  std::string NodeStat(RegTree const& tree, int32_t nid) override {
+  std::string NodeStat(RegTree const& tree, int32_t nid) const override {
     static std::string kStatTemplate =
         R"S(, "gain": {loss_chg}, "cover": {sum_hess})S";
     auto result = SuperT::Match(
@@ -529,7 +530,7 @@ class GraphvizGenerator : public TreeGenerator {
  protected:
   // Only indicator is different, so we combine all different node types into this
   // function.
-  std::string PlainNode(RegTree const& tree, int32_t nid, uint32_t depth) override {
+  std::string PlainNode(RegTree const& tree, int32_t nid, uint32_t depth) const override {
     auto split = tree[nid].SplitIndex();
     auto cond = tree[nid].SplitCond();
     static std::string const kNodeTemplate =
@@ -563,7 +564,7 @@ class GraphvizGenerator : public TreeGenerator {
     return result;
   };
 
-  std::string LeafNode(RegTree const& tree, int32_t nid, uint32_t depth) override {
+  std::string LeafNode(RegTree const& tree, int32_t nid, uint32_t depth) const override {
     static std::string const kLeafTemplate =
         "    {nid} [ label=\"leaf={leaf-value}\" {params}]\n";
     auto result = SuperT::Match(kLeafTemplate, {

tests/ci_build/rename_whl.py

@@ -21,8 +21,6 @@ whl_path = sys.argv[1]
 commit_id = sys.argv[2]
 platform_tag = sys.argv[3]
 
-assert platform_tag in ['manylinux1_x86_64', 'manylinux2010_x86_64', 'win_amd64']
-
 dirname, basename = os.path.dirname(whl_path), os.path.basename(whl_path)
 
 with cd(dirname):

tests/cpp/data/test_metainfo.cc

@@ -149,9 +149,17 @@ TEST(MetaInfo, Validate) {
   info.num_col_ = 3;
   std::vector<xgboost::bst_group_t> groups (11);
   info.SetInfo("group", groups.data(), xgboost::DataType::kUInt32, 11);
-  EXPECT_THROW(info.Validate(), dmlc::Error);
+  EXPECT_THROW(info.Validate(0), dmlc::Error);
 
   std::vector<float> labels(info.num_row_ + 1);
   info.SetInfo("label", labels.data(), xgboost::DataType::kFloat32, info.num_row_ + 1);
-  EXPECT_THROW(info.Validate(), dmlc::Error);
+  EXPECT_THROW(info.Validate(0), dmlc::Error);
+
+#if defined(XGBOOST_USE_CUDA)
+  info.group_ptr_.clear();
+  labels.resize(info.num_row_);
+  info.SetInfo("label", labels.data(), xgboost::DataType::kFloat32, info.num_row_);
+  info.labels_.SetDevice(0);
+  EXPECT_THROW(info.Validate(1), dmlc::Error);
+#endif  // defined(XGBOOST_USE_CUDA)
 }

tests/cpp/tree/test_tree_model.cc

@@ -151,6 +151,10 @@ TEST(Tree, DumpJson) {
 
   str = tree.DumpModel(fmap, false, "json");
   ASSERT_EQ(str.find("cover"), std::string::npos);
+
+
+  auto j_tree = Json::Load({str.c_str(), str.size()});
+  ASSERT_EQ(get<Array>(j_tree["children"]).size(), 2);
 }
 
 TEST(Tree, DumpText) {

tests/python-gpu/test_from_cupy.py

@@ -136,3 +136,14 @@ Arrow specification.'''
         n = 100
         X = cp.random.random((n, 2))
         xgb.DeviceQuantileDMatrix(X.toDlpack())
+
+    @pytest.mark.skipif(**tm.no_cupy())
+    @pytest.mark.mgpu
+    def test_specified_device(self):
+        import cupy as cp
+        cp.cuda.runtime.setDevice(0)
+        dtrain = dmatrix_from_cupy(
+            np.float32, xgb.DeviceQuantileDMatrix, np.nan)
+        with pytest.raises(xgb.core.XGBoostError):
+            xgb.train({'tree_method': 'gpu_hist', 'gpu_id': 1},
+                      dtrain, num_boost_round=10)

tests/python-gpu/test_gpu_basic_models.py

@@ -12,25 +12,15 @@ rng = np.random.RandomState(1994)
 class TestGPUBasicModels(unittest.TestCase):
     cputest = test_bm.TestModels()
 
-    def test_eta_decay_gpu_hist(self):
-        self.cputest.run_eta_decay('gpu_hist')
-
-    def test_deterministic_gpu_hist(self):
-        kRows = 1000
-        kCols = 64
-        kClasses = 4
-        # Create large values to force rounding.
-        X = np.random.randn(kRows, kCols) * 1e4
-        y = np.random.randint(0, kClasses, size=kRows)
-
+    def run_cls(self, X, y, deterministic):
         cls = xgb.XGBClassifier(tree_method='gpu_hist',
-                                deterministic_histogram=True,
+                                deterministic_histogram=deterministic,
                                 single_precision_histogram=True)
         cls.fit(X, y)
         cls.get_booster().save_model('test_deterministic_gpu_hist-0.json')
 
         cls = xgb.XGBClassifier(tree_method='gpu_hist',
-                                deterministic_histogram=True,
+                                deterministic_histogram=deterministic,
                                 single_precision_histogram=True)
         cls.fit(X, y)
         cls.get_booster().save_model('test_deterministic_gpu_hist-1.json')
@@ -40,7 +30,24 @@ class TestGPUBasicModels(unittest.TestCase):
         with open('test_deterministic_gpu_hist-1.json', 'r') as fd:
             model_1 = fd.read()
 
-        assert hash(model_0) == hash(model_1)
-
         os.remove('test_deterministic_gpu_hist-0.json')
         os.remove('test_deterministic_gpu_hist-1.json')
+
+        return hash(model_0), hash(model_1)
+
+    def test_eta_decay_gpu_hist(self):
+        self.cputest.run_eta_decay('gpu_hist')
+
+    def test_deterministic_gpu_hist(self):
+        kRows = 1000
+        kCols = 64
+        kClasses = 4
+        # Create large values to force rounding.
+        X = np.random.randn(kRows, kCols) * 1e4
+        y = np.random.randint(0, kClasses, size=kRows) * 1e4
+
+        model_0, model_1 = self.run_cls(X, y, True)
+        assert model_0 == model_1
+
+        model_0, model_1 = self.run_cls(X, y, False)
+        assert model_0 != model_1

tests/python-gpu/test_gpu_prediction.py

@@ -121,6 +121,7 @@ class TestGPUPredict(unittest.TestCase):
     @pytest.mark.skipif(**tm.no_cupy())
     def test_inplace_predict_cupy(self):
         import cupy as cp
+        cp.cuda.runtime.setDevice(0)
         rows = 1000
         cols = 10
         cp_rng = cp.random.RandomState(1994)

tests/python/test_basic_models.py

@@ -325,7 +325,7 @@ class TestModels(unittest.TestCase):
         assert locale.getpreferredencoding(False) == loc
 
     @pytest.mark.skipif(**tm.no_json_schema())
-    def test_json_schema(self):
+    def test_json_io_schema(self):
         import jsonschema
         model_path = 'test_json_schema.json'
         path = os.path.dirname(
@@ -342,3 +342,35 @@ class TestModels(unittest.TestCase):
         jsonschema.validate(instance=json_model(model_path, parameters),
                             schema=schema)
         os.remove(model_path)
+
+    @pytest.mark.skipif(**tm.no_json_schema())
+    def test_json_dump_schema(self):
+        import jsonschema
+
+        def validate_model(parameters):
+            X = np.random.random((100, 30))
+            y = np.random.randint(0, 4, size=(100,))
+
+            parameters['num_class'] = 4
+            m = xgb.DMatrix(X, y)
+
+            booster = xgb.train(parameters, m)
+            dump = booster.get_dump(dump_format='json')
+
+            for i in range(len(dump)):
+                jsonschema.validate(instance=json.loads(dump[i]),
+                                    schema=schema)
+
+        path = os.path.dirname(
+            os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+        doc = os.path.join(path, 'doc', 'dump.schema')
+        with open(doc, 'r') as fd:
+            schema = json.load(fd)
+
+        parameters = {'tree_method': 'hist', 'booster': 'gbtree',
+                      'objective': 'multi:softmax'}
+        validate_model(parameters)
+
+        parameters = {'tree_method': 'hist', 'booster': 'dart',
+                      'objective': 'multi:softmax'}
+        validate_model(parameters)

tests/travis/run_test.sh

@@ -23,18 +23,42 @@ if [ ${TASK} == "python_test" ]; then
     mkdir build && cd build
     cmake .. -DUSE_OPENMP=ON -DCMAKE_VERBOSE_MAKEFILE=ON
     make -j$(nproc)
-    cd ..
 
     echo "-------------------------------"
     conda activate python3
     conda --version
     python --version
 
+    # Build binary wheel
+    cd ../python-package
+    python setup.py bdist_wheel
+    TAG=macosx_10_13_x86_64.macosx_10_14_x86_64.macosx_10_15_x86_64
+    python ../tests/ci_build/rename_whl.py dist/*.whl ${TRAVIS_COMMIT} ${TAG}
+    python -m pip install ./dist/xgboost-*-py3-none-${TAG}.whl
+
+    # Run unit tests
+    cd ..
     python -m pip install graphviz pytest pytest-cov codecov
     python -m pip install datatable
     python -m pip install numpy scipy pandas matplotlib scikit-learn dask[complete]
     python -m pytest -v --fulltrace -s tests/python --cov=python-package/xgboost || exit -1
     codecov
+
+    # Deploy binary wheel to S3
+    python -m pip install awscli
+    if [ "${TRAVIS_PULL_REQUEST}" != "false" ]
+    then
+        S3_DEST="s3://xgboost-nightly-builds/PR-${TRAVIS_PULL_REQUEST}/"
+    else
+        if [ "${TRAVIS_BRANCH}" == "master" ]
+        then
+            S3_DEST="s3://xgboost-nightly-builds/"
+        elif [ -z "${TRAVIS_TAG}" ]
+        then
+            S3_DEST="s3://xgboost-nightly-builds/${TRAVIS_BRANCH}/"
+        fi
+    fi
+    python -m awscli s3 cp python-package/dist/*.whl "${S3_DEST}" --acl public-read || true
 fi
 
 if [ ${TASK} == "java_test" ]; then