De-duplicate GPU parameters. (#4454)

* Only define `gpu_id` and `n_gpus` in `LearnerTrainParam`
* Pass LearnerTrainParam through XGBoost vid factory method.
* Disable all GPU usage when GPU related parameters are not specified (fixes XGBoost choosing GPU over aggressively).
* Test learner train param io.
* Fix gpu pickling.

tests/cpp/common/test_common.cu

@@ -1,5 +1,9 @@
-#include "../../../src/common/common.h"
 #include <gtest/gtest.h>
+#include <xgboost/logging.h>
+#include "../../../src/common/common.h"
+#include "../helpers.h"
+
+#include <string>
 
 namespace xgboost {
 
@@ -36,6 +40,23 @@ TEST(GPUSet, GPUBasic) {
   }
 }
 
+TEST(GPUSet, Verbose) {
+  {
+    std::map<std::string, std::string> args {};
+    args["verbosity"] = "3";  // LOG INFO
+
+    testing::internal::CaptureStderr();
+    ConsoleLogger::Configure(args.cbegin(), args.cend());
+    GPUSet::All(0, 1);
+    std::string output = testing::internal::GetCapturedStderr();
+    ASSERT_NE(output.find("GPU ID: 0"), std::string::npos);
+    ASSERT_NE(output.find("GPUs: 1"), std::string::npos);
+
+    args["verbosity"] = "1";  // restore
+    ConsoleLogger::Configure(args.cbegin(), args.cend());
+  }
+}
+
 #if defined(XGBOOST_USE_NCCL)
 TEST(GPUSet, MGPU_GPUBasic) {
   {

tests/cpp/common/test_device_helpers.cu

@@ -5,6 +5,7 @@
 #include <thrust/device_vector.h>
 #include <xgboost/base.h>
 #include "../../../src/common/device_helpers.cuh"
+#include "../helpers.h"
 #include "gtest/gtest.h"
 
 using xgboost::common::Span;
@@ -65,7 +66,9 @@ void TestLbs() {
   }
 }
 
-TEST(cub_lbs, Test) { TestLbs(); }
+TEST(cub_lbs, Test) {
+  TestLbs();
+}
 
 TEST(sumReduce, Test) {
   thrust::device_vector<float> data(100, 1.0f);
@@ -89,4 +92,6 @@ void TestAllocator() {
 }
 
 // Define the test in a function so we can use device lambda
-TEST(bulkAllocator, Test) { TestAllocator(); }
+TEST(bulkAllocator, Test) {
+  TestAllocator();
+}

tests/cpp/common/test_enum_class_param.cc

@@ -1,6 +1,6 @@
-#include "../../../src/common/enum_class_param.h"
 #include <dmlc/parameter.h>
 #include <gtest/gtest.h>
+#include <xgboost/enum_class_param.h>
 
 enum class Foo : int {
   kBar = 0, kFrog = 1, kCat = 2, kDog = 3

tests/cpp/common/test_gpu_hist_util.cu

@@ -1,5 +1,3 @@
-#include "../../../src/common/device_helpers.cuh"
-#include "../../../src/common/hist_util.h"
 #include "gtest/gtest.h"
 #include "xgboost/c_api.h"
 #include <algorithm>
@@ -7,6 +5,10 @@
 #include <thrust/device_vector.h>
 #include <thrust/iterator/counting_iterator.h>
 
+#include "../helpers.h"
+#include "../../../src/common/device_helpers.cuh"
+#include "../../../src/common/hist_util.h"
+
 namespace xgboost {
 namespace common {
 
@@ -27,8 +29,6 @@ void TestDeviceSketch(const GPUSet& devices) {
   // parameters for finding quantiles
   tree::TrainParam p;
   p.max_bin = 20;
-  p.gpu_id = 0;
-  p.n_gpus = devices.Size();
   // ensure that the exact quantiles are found
   int gpu_batch_nrows = nrows * 10;
 
@@ -39,7 +39,7 @@ void TestDeviceSketch(const GPUSet& devices) {
   // find the cuts on the GPU
   const SparsePage& batch = *(*dmat)->GetRowBatches().begin();
   HistCutMatrix hmat_gpu;
-  DeviceSketch(batch, (*dmat)->Info(), p, &hmat_gpu, gpu_batch_nrows);
+  DeviceSketch(batch, (*dmat)->Info(), p, &hmat_gpu, gpu_batch_nrows, devices);
 
   // compare the cuts
   double eps = 1e-2;

tests/cpp/helpers.h

@@ -18,6 +18,9 @@
 #include <xgboost/objective.h>
 #include <xgboost/metric.h>
 #include <xgboost/predictor.h>
+#include <xgboost/generic_parameters.h>
+
+#include "../../src/common/common.h"
 
 #if defined(__CUDACC__)
 #define DeclareUnifiedTest(name) GPU ## name
@@ -25,6 +28,12 @@
 #define DeclareUnifiedTest(name) name
 #endif
 
+#if defined(__CUDACC__)
+#define NGPUS 1
+#else
+#define NGPUS 0
+#endif
+
 bool FileExists(const std::string& filename);
 
 int64_t GetFileSize(const std::string& filename);
@@ -158,5 +167,14 @@ std::unique_ptr<DMatrix> CreateSparsePageDMatrix(size_t n_entries, size_t page_s
 
 gbm::GBTreeModel CreateTestModel();
 
+inline LearnerTrainParam CreateEmptyGenericParam(int gpu_id, int n_gpus) {
+  xgboost::LearnerTrainParam tparam;
+  std::vector<std::pair<std::string, std::string>> args {
+    {"gpu_id", std::to_string(gpu_id)},
+    {"n_gpus", std::to_string(n_gpus)}};
+  tparam.Init(args);
+  return tparam;
+}
+
 }  // namespace xgboost
 #endif

tests/cpp/linear/test_linear.cc

@@ -7,9 +7,10 @@
 
 TEST(Linear, shotgun) {
   auto mat = xgboost::CreateDMatrix(10, 10, 0);
+  auto lparam = xgboost::CreateEmptyGenericParam(0, 0);
   {
     auto updater = std::unique_ptr<xgboost::LinearUpdater>(
-        xgboost::LinearUpdater::Create("shotgun"));
+        xgboost::LinearUpdater::Create("shotgun", &lparam));
     updater->Init({{"eta", "1."}});
     xgboost::HostDeviceVector<xgboost::GradientPair> gpair(
         (*mat)->Info().num_row_, xgboost::GradientPair(-5, 1.0));
@@ -24,7 +25,7 @@ TEST(Linear, shotgun) {
   }
   {
     auto updater = std::unique_ptr<xgboost::LinearUpdater>(
-        xgboost::LinearUpdater::Create("shotgun"));
+        xgboost::LinearUpdater::Create("shotgun", &lparam));
     EXPECT_ANY_THROW(updater->Init({{"feature_selector", "random"}}));
   }
   delete mat;
@@ -32,8 +33,9 @@ TEST(Linear, shotgun) {
 
 TEST(Linear, coordinate) {
   auto mat = xgboost::CreateDMatrix(10, 10, 0);
+  auto lparam = xgboost::CreateEmptyGenericParam(0, 0);
   auto updater = std::unique_ptr<xgboost::LinearUpdater>(
-      xgboost::LinearUpdater::Create("coord_descent"));
+      xgboost::LinearUpdater::Create("coord_descent", &lparam));
   updater->Init({{"eta", "1."}});
   xgboost::HostDeviceVector<xgboost::GradientPair> gpair(
       (*mat)->Info().num_row_, xgboost::GradientPair(-5, 1.0));

tests/cpp/linear/test_linear.cu

@@ -6,11 +6,12 @@
 namespace xgboost {
 
 TEST(Linear, GPUCoordinate) {
-  dh::safe_cuda(cudaSetDevice(0));
   auto mat = xgboost::CreateDMatrix(10, 10, 0);
+  auto lparam = CreateEmptyGenericParam(0, 1);
+  lparam.n_gpus = 1;
   auto updater = std::unique_ptr<xgboost::LinearUpdater>(
-      xgboost::LinearUpdater::Create("gpu_coord_descent"));
-  updater->Init({{"eta", "1."}, {"n_gpus", "1"}});
+      xgboost::LinearUpdater::Create("gpu_coord_descent", &lparam));
+  updater->Init({{"eta", "1."}});
   xgboost::HostDeviceVector<xgboost::GradientPair> gpair(
       (*mat)->Info().num_row_, xgboost::GradientPair(-5, 1.0));
   xgboost::gbm::GBLinearModel model;
@@ -26,12 +27,13 @@ TEST(Linear, GPUCoordinate) {
 
 #if defined(XGBOOST_USE_NCCL)
 TEST(Linear, MGPU_GPUCoordinate) {
-  dh::safe_cuda(cudaSetDevice(0));
   {
     auto mat = xgboost::CreateDMatrix(10, 10, 0);
+    auto lparam = CreateEmptyGenericParam(0, -1);
+    lparam.n_gpus = -1;
     auto updater = std::unique_ptr<xgboost::LinearUpdater>(
-        xgboost::LinearUpdater::Create("gpu_coord_descent"));
-    updater->Init({{"eta", "1."}, {"n_gpus", "-1"}});
+        xgboost::LinearUpdater::Create("gpu_coord_descent", &lparam));
+    updater->Init({{"eta", "1."}});
     xgboost::HostDeviceVector<xgboost::GradientPair> gpair(
         (*mat)->Info().num_row_, xgboost::GradientPair(-5, 1.0));
     xgboost::gbm::GBLinearModel model;
@@ -44,15 +46,13 @@ TEST(Linear, MGPU_GPUCoordinate) {
     delete mat;
   }
 
-  dh::safe_cuda(cudaSetDevice(0));
   {
+    auto lparam = CreateEmptyGenericParam(1, -1);
+    lparam.n_gpus = -1;
     auto mat = xgboost::CreateDMatrix(10, 10, 0);
     auto updater = std::unique_ptr<xgboost::LinearUpdater>(
-        xgboost::LinearUpdater::Create("gpu_coord_descent"));
-    updater->Init({
-        {"eta", "1."},
-        {"n_gpus", "-1"},
-        {"gpu_id", "1"}});
+        xgboost::LinearUpdater::Create("gpu_coord_descent", &lparam));
+    updater->Init({{"eta", "1."}});
     xgboost::HostDeviceVector<xgboost::GradientPair> gpair(
         (*mat)->Info().num_row_, xgboost::GradientPair(-5, 1.0));
     xgboost::gbm::GBLinearModel model;

tests/cpp/metric/test_elementwise_metric.cc

@@ -1,21 +1,14 @@
 /*!
- * Copyright 2018 XGBoost contributors
+ * Copyright 2018-2019 XGBoost contributors
  */
 #include <xgboost/metric.h>
 #include <map>
 #include "../helpers.h"
 
-using Arg = std::pair<std::string, std::string>;
-
-#if defined(__CUDACC__)
-#define N_GPU() Arg{"n_gpus", "1"}
-#else
-#define N_GPU() Arg{"n_gpus", "0"}
-#endif
-
 TEST(Metric, DeclareUnifiedTest(RMSE)) {
-  xgboost::Metric * metric = xgboost::Metric::Create("rmse");
-  metric->Configure({N_GPU()});
+  auto lparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  xgboost::Metric * metric = xgboost::Metric::Create("rmse", &lparam);
+  metric->Configure({});
   ASSERT_STREQ(metric->Name(), "rmse");
   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10);
   EXPECT_NEAR(GetMetricEval(metric,
@@ -27,8 +20,9 @@ TEST(Metric, DeclareUnifiedTest(RMSE)) {
 }
 
 TEST(Metric, DeclareUnifiedTest(MAE)) {
-  xgboost::Metric * metric = xgboost::Metric::Create("mae");
-  metric->Configure({N_GPU()});
+  auto lparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  xgboost::Metric * metric = xgboost::Metric::Create("mae", &lparam);
+  metric->Configure({});
   ASSERT_STREQ(metric->Name(), "mae");
   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10);
   EXPECT_NEAR(GetMetricEval(metric,
@@ -39,8 +33,9 @@ TEST(Metric, DeclareUnifiedTest(MAE)) {
 }
 
 TEST(Metric, DeclareUnifiedTest(LogLoss)) {
-  xgboost::Metric * metric = xgboost::Metric::Create("logloss");
-  metric->Configure({N_GPU()});
+  auto lparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  xgboost::Metric * metric = xgboost::Metric::Create("logloss", &lparam);
+  metric->Configure({});
   ASSERT_STREQ(metric->Name(), "logloss");
   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10);
   EXPECT_NEAR(GetMetricEval(metric,
@@ -51,8 +46,9 @@ TEST(Metric, DeclareUnifiedTest(LogLoss)) {
 }
 
 TEST(Metric, DeclareUnifiedTest(Error)) {
-  xgboost::Metric * metric = xgboost::Metric::Create("error");
-  metric->Configure({N_GPU()});
+  auto lparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  xgboost::Metric * metric = xgboost::Metric::Create("error", &lparam);
+  metric->Configure({});
   ASSERT_STREQ(metric->Name(), "error");
   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10);
   EXPECT_NEAR(GetMetricEval(metric,
@@ -60,17 +56,17 @@ TEST(Metric, DeclareUnifiedTest(Error)) {
                             {  0,   0,   1,   1}),
               0.5f, 0.001f);
 
-  EXPECT_ANY_THROW(xgboost::Metric::Create("error@abc"));
+  EXPECT_ANY_THROW(xgboost::Metric::Create("error@abc", &lparam));
   delete metric;
 
-  metric = xgboost::Metric::Create("error@0.5f");
-  metric->Configure({N_GPU()});
+  metric = xgboost::Metric::Create("error@0.5f", &lparam);
+  metric->Configure({});
   EXPECT_STREQ(metric->Name(), "error");
 
   delete metric;
 
-  metric = xgboost::Metric::Create("error@0.1");
-  metric->Configure({N_GPU()});
+  metric = xgboost::Metric::Create("error@0.1", &lparam);
+  metric->Configure({});
   ASSERT_STREQ(metric->Name(), "error@0.1");
   EXPECT_STREQ(metric->Name(), "error@0.1");
   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10);
@@ -82,8 +78,9 @@ TEST(Metric, DeclareUnifiedTest(Error)) {
 }
 
 TEST(Metric, DeclareUnifiedTest(PoissionNegLogLik)) {
-  xgboost::Metric * metric = xgboost::Metric::Create("poisson-nloglik");
-  metric->Configure({N_GPU()});
+  auto lparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  xgboost::Metric * metric = xgboost::Metric::Create("poisson-nloglik", &lparam);
+  metric->Configure({});
   ASSERT_STREQ(metric->Name(), "poisson-nloglik");
   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0.5f, 1e-10);
   EXPECT_NEAR(GetMetricEval(metric,
@@ -96,8 +93,9 @@ TEST(Metric, DeclareUnifiedTest(PoissionNegLogLik)) {
 #if defined(XGBOOST_USE_NCCL) && defined(__CUDACC__)
 TEST(Metric, MGPU_RMSE) {
   {
-    xgboost::Metric * metric = xgboost::Metric::Create("rmse");
-    metric->Configure({Arg{"n_gpus", "-1"}});
+    auto lparam = xgboost::CreateEmptyGenericParam(0, -1);
+    xgboost::Metric * metric = xgboost::Metric::Create("rmse", &lparam);
+    metric->Configure({});
     ASSERT_STREQ(metric->Name(), "rmse");
     EXPECT_NEAR(GetMetricEval(metric, {0}, {0}), 0, 1e-10);
     EXPECT_NEAR(GetMetricEval(metric,
@@ -108,8 +106,8 @@ TEST(Metric, MGPU_RMSE) {
   }
 
   {
-    xgboost::Metric * metric = xgboost::Metric::Create("rmse");
-    metric->Configure({Arg{"n_gpus", "-1"}, Arg{"gpu_id", "1"}});
+    auto lparam = xgboost::CreateEmptyGenericParam(1, -1);
+    xgboost::Metric * metric = xgboost::Metric::Create("rmse", &lparam);
     ASSERT_STREQ(metric->Name(), "rmse");
     EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10);
     EXPECT_NEAR(GetMetricEval(metric,

tests/cpp/metric/test_metric.cc

@@ -4,15 +4,16 @@
 #include "../helpers.h"
 
 TEST(Metric, UnknownMetric) {
+  auto tparam = xgboost::CreateEmptyGenericParam(0, 0);
   xgboost::Metric * metric = nullptr;
-  EXPECT_ANY_THROW(metric = xgboost::Metric::Create("unknown_name"));
-  EXPECT_NO_THROW(metric = xgboost::Metric::Create("rmse"));
+  EXPECT_ANY_THROW(metric = xgboost::Metric::Create("unknown_name", &tparam));
+  EXPECT_NO_THROW(metric = xgboost::Metric::Create("rmse", &tparam));
   if (metric) {
     delete metric;
   }
   metric = nullptr;
-  EXPECT_ANY_THROW(metric = xgboost::Metric::Create("unknown_name@1"));
-  EXPECT_NO_THROW(metric = xgboost::Metric::Create("error@0.5f"));
+  EXPECT_ANY_THROW(metric = xgboost::Metric::Create("unknown_name@1", &tparam));
+  EXPECT_NO_THROW(metric = xgboost::Metric::Create("error@0.5f", &tparam));
   if (metric) {
     delete metric;
   }

tests/cpp/metric/test_multiclass_metric.cc

@@ -4,17 +4,12 @@
 
 #include "../helpers.h"
 
-using Arg = std::pair<std::string, std::string>;
-
-#if defined(__CUDACC__)
-#define N_GPU() Arg{"n_gpus", "1"}
-#else
-#define N_GPU() Arg{"n_gpus", "0"}
-#endif
-
-inline void TestMultiClassError(std::vector<Arg> args) {
-  xgboost::Metric * metric = xgboost::Metric::Create("merror");
-  metric->Configure(args);
+inline void TestMultiClassError(xgboost::GPUSet const& devices) {
+  auto lparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  lparam.gpu_id = *devices.begin();
+  lparam.n_gpus = devices.Size();
+  xgboost::Metric * metric = xgboost::Metric::Create("merror", &lparam);
+  metric->Configure({});
   ASSERT_STREQ(metric->Name(), "merror");
   EXPECT_ANY_THROW(GetMetricEval(metric, {0}, {0, 0}));
   EXPECT_NEAR(GetMetricEval(
@@ -28,12 +23,16 @@ inline void TestMultiClassError(std::vector<Arg> args) {
 }
 
 TEST(Metric, DeclareUnifiedTest(MultiClassError)) {
-  TestMultiClassError({N_GPU()});
+  auto devices = xgboost::GPUSet::Range(0, NGPUS);
+  TestMultiClassError(devices);
 }
 
-inline void TestMultiClassLogLoss(std::vector<Arg> args) {
-  xgboost::Metric * metric = xgboost::Metric::Create("mlogloss");
-  metric->Configure(args);
+inline void TestMultiClassLogLoss(xgboost::GPUSet const& devices) {
+  auto lparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  lparam.gpu_id = *devices.begin();
+  lparam.n_gpus = devices.Size();
+  xgboost::Metric * metric = xgboost::Metric::Create("mlogloss", &lparam);
+  metric->Configure({});
   ASSERT_STREQ(metric->Name(), "mlogloss");
   EXPECT_ANY_THROW(GetMetricEval(metric, {0}, {0, 0}));
   EXPECT_NEAR(GetMetricEval(
@@ -47,15 +46,27 @@ inline void TestMultiClassLogLoss(std::vector<Arg> args) {
 }
 
 TEST(Metric, DeclareUnifiedTest(MultiClassLogLoss)) {
-  TestMultiClassLogLoss({N_GPU()});
+  auto devices = xgboost::GPUSet::Range(0, NGPUS);
+  TestMultiClassLogLoss(devices);
 }
 
 #if defined(XGBOOST_USE_NCCL) && defined(__CUDACC__)
 TEST(Metric, MGPU_MultiClassError) {
-  TestMultiClassError({Arg{"n_gpus", "-1"}});
-  TestMultiClassError({Arg{"n_gpus", "-1"}, Arg{"gpu_id", "1"}});
-
-  TestMultiClassLogLoss({Arg{"n_gpus", "-1"}});
-  TestMultiClassLogLoss({Arg{"n_gpus", "-1"}, Arg{"gpu_id", "1"}});
+  {
+    auto devices = xgboost::GPUSet::All(0, -1);
+    TestMultiClassError(devices);
+  }
+  {
+    auto devices = xgboost::GPUSet::All(1, -1);
+    TestMultiClassError(devices);
+  }
+  {
+    auto devices = xgboost::GPUSet::All(0, -1);
+    TestMultiClassLogLoss(devices);
+  }
+  {
+    auto devices = xgboost::GPUSet::All(1, -1);
+    TestMultiClassLogLoss(devices);
+  }
 }
 #endif  // defined(XGBOOST_USE_NCCL)

tests/cpp/metric/test_rank_metric.cc

@@ -4,8 +4,9 @@
 #include "../helpers.h"
 
 TEST(Metric, AMS) {
-  EXPECT_ANY_THROW(xgboost::Metric::Create("ams"));
-  xgboost::Metric * metric = xgboost::Metric::Create("ams@0.5f");
+  auto tparam = xgboost::CreateEmptyGenericParam(0, 0);
+  EXPECT_ANY_THROW(xgboost::Metric::Create("ams", &tparam));
+  xgboost::Metric * metric = xgboost::Metric::Create("ams@0.5f", &tparam);
   ASSERT_STREQ(metric->Name(), "ams@0.5");
   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0.311f, 0.001f);
   EXPECT_NEAR(GetMetricEval(metric,
@@ -14,7 +15,7 @@ TEST(Metric, AMS) {
               0.29710f, 0.001f);
 
   delete metric;
-  metric = xgboost::Metric::Create("ams@0");
+  metric = xgboost::Metric::Create("ams@0", &tparam);
   ASSERT_STREQ(metric->Name(), "ams@0");
   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0.311f, 0.001f);
 
@@ -22,7 +23,8 @@ TEST(Metric, AMS) {
 }
 
 TEST(Metric, AUC) {
-  xgboost::Metric * metric = xgboost::Metric::Create("auc");
+  auto tparam = xgboost::CreateEmptyGenericParam(0, 0);
+  xgboost::Metric * metric = xgboost::Metric::Create("auc", &tparam);
   ASSERT_STREQ(metric->Name(), "auc");
   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
   EXPECT_NEAR(GetMetricEval(metric,
@@ -36,7 +38,8 @@ TEST(Metric, AUC) {
 }
 
 TEST(Metric, AUCPR) {
-  xgboost::Metric *metric = xgboost::Metric::Create("aucpr");
+  auto tparam = xgboost::CreateEmptyGenericParam(0, 0);
+  xgboost::Metric *metric = xgboost::Metric::Create("aucpr", &tparam);
   ASSERT_STREQ(metric->Name(), "aucpr");
   EXPECT_NEAR(GetMetricEval(metric, {0, 0, 1, 1}, {0, 0, 1, 1}), 1, 1e-10);
   EXPECT_NEAR(GetMetricEval(metric, {0.1f, 0.9f, 0.1f, 0.9f}, {0, 0, 1, 1}),
@@ -62,7 +65,8 @@ TEST(Metric, Precision) {
   // When the limit for precision is not given, it takes the limit at
   // std::numeric_limits<unsigned>::max(); hence all values are very small
   // NOTE(AbdealiJK): Maybe this should be fixed to be num_row by default.
-  xgboost::Metric * metric = xgboost::Metric::Create("pre");
+  auto tparam = xgboost::CreateEmptyGenericParam(0, 0);
+  xgboost::Metric * metric = xgboost::Metric::Create("pre", &tparam);
   ASSERT_STREQ(metric->Name(), "pre");
   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-7);
   EXPECT_NEAR(GetMetricEval(metric,
@@ -71,7 +75,7 @@ TEST(Metric, Precision) {
               0, 1e-7);
 
   delete metric;
-  metric = xgboost::Metric::Create("pre@2");
+  metric = xgboost::Metric::Create("pre@2", &tparam);
   ASSERT_STREQ(metric->Name(), "pre@2");
   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0.5f, 1e-7);
   EXPECT_NEAR(GetMetricEval(metric,
@@ -85,7 +89,8 @@ TEST(Metric, Precision) {
 }
 
 TEST(Metric, NDCG) {
-  xgboost::Metric * metric = xgboost::Metric::Create("ndcg");
+  auto tparam = xgboost::CreateEmptyGenericParam(0, 0);
+  xgboost::Metric * metric = xgboost::Metric::Create("ndcg", &tparam);
   ASSERT_STREQ(metric->Name(), "ndcg");
   EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {}));
   EXPECT_NEAR(GetMetricEval(metric,
@@ -98,7 +103,7 @@ TEST(Metric, NDCG) {
               0.6509f, 0.001f);
 
   delete metric;
-  metric = xgboost::Metric::Create("ndcg@2");
+  metric = xgboost::Metric::Create("ndcg@2", &tparam);
   ASSERT_STREQ(metric->Name(), "ndcg@2");
   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
   EXPECT_NEAR(GetMetricEval(metric,
@@ -107,7 +112,7 @@ TEST(Metric, NDCG) {
               0.3868f, 0.001f);
 
   delete metric;
-  metric = xgboost::Metric::Create("ndcg@-");
+  metric = xgboost::Metric::Create("ndcg@-", &tparam);
   ASSERT_STREQ(metric->Name(), "ndcg@-");
   EXPECT_NEAR(GetMetricEval(metric,
                             xgboost::HostDeviceVector<xgboost::bst_float>{},
@@ -119,7 +124,7 @@ TEST(Metric, NDCG) {
               0.6509f, 0.001f);
 
   delete metric;
-  metric = xgboost::Metric::Create("ndcg@2-");
+  metric = xgboost::Metric::Create("ndcg@2-", &tparam);
   ASSERT_STREQ(metric->Name(), "ndcg@2-");
   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
   EXPECT_NEAR(GetMetricEval(metric,
@@ -131,7 +136,8 @@ TEST(Metric, NDCG) {
 }
 
 TEST(Metric, MAP) {
-  xgboost::Metric * metric = xgboost::Metric::Create("map");
+  auto tparam = xgboost::CreateEmptyGenericParam(0, 0);
+  xgboost::Metric * metric = xgboost::Metric::Create("map", &tparam);
   ASSERT_STREQ(metric->Name(), "map");
   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
   EXPECT_NEAR(GetMetricEval(metric,
@@ -143,14 +149,14 @@ TEST(Metric, MAP) {
                             std::vector<xgboost::bst_float>{}), 1, 1e-10);
 
   delete metric;
-  metric = xgboost::Metric::Create("map@-");
+  metric = xgboost::Metric::Create("map@-", &tparam);
   ASSERT_STREQ(metric->Name(), "map@-");
   EXPECT_NEAR(GetMetricEval(metric,
                             xgboost::HostDeviceVector<xgboost::bst_float>{},
                             {}), 0, 1e-10);
 
   delete metric;
-  metric = xgboost::Metric::Create("map@2");
+  metric = xgboost::Metric::Create("map@2", &tparam);
   ASSERT_STREQ(metric->Name(), "map@2");
   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
   EXPECT_NEAR(GetMetricEval(metric,

tests/cpp/objective/test_hinge.cc

@@ -1,13 +1,14 @@
 // Copyright by Contributors
 #include <xgboost/objective.h>
+#include <xgboost/generic_parameters.h>
 #include <limits>
 
 #include "../helpers.h"
 
 TEST(Objective, DeclareUnifiedTest(HingeObj)) {
-  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("binary:hinge");
-  std::vector<std::pair<std::string, std::string> > args;
-  obj->Configure(args);
+  xgboost::LearnerTrainParam tparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("binary:hinge", &tparam);
+
   xgboost::bst_float eps = std::numeric_limits<xgboost::bst_float>::min();
   CheckObjFunction(obj,
                    {-1.0f, -0.5f, 0.5f, 1.0f, -1.0f, -0.5f,  0.5f, 1.0f},

tests/cpp/objective/test_multiclass_obj.cc

@@ -1,13 +1,16 @@
 /*!
- * Copyright 2018 XGBoost contributors
+ * Copyright 2018-2019 XGBoost contributors
  */
 #include <xgboost/objective.h>
-
+#include <xgboost/generic_parameters.h>
+#include "../../src/common/common.h"
 #include "../helpers.h"
 
 TEST(Objective, DeclareUnifiedTest(SoftmaxMultiClassObjGPair)) {
-  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("multi:softmax");
+  xgboost::LearnerTrainParam lparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
   std::vector<std::pair<std::string, std::string>> args {{"num_class", "3"}};
+  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("multi:softmax", &lparam);
+
   obj->Configure(args);
   CheckObjFunction(obj,
 		   {1.0f, 0.0f, 2.0f, 2.0f, 0.0f, 1.0f}, // preds
@@ -22,9 +25,11 @@ TEST(Objective, DeclareUnifiedTest(SoftmaxMultiClassObjGPair)) {
 }
 
 TEST(Objective, DeclareUnifiedTest(SoftmaxMultiClassBasic)) {
-  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("multi:softmax");
-  std::vector<std::pair<std::string, std::string>> args
-    {std::pair<std::string, std::string>("num_class", "3")};
+  auto lparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  std::vector<std::pair<std::string, std::string>> args{
+    std::pair<std::string, std::string>("num_class", "3")};
+
+  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("multi:softmax", &lparam);
   obj->Configure(args);
 
   xgboost::HostDeviceVector<xgboost::bst_float>  io_preds = {2.0f, 0.0f, 1.0f,
@@ -42,9 +47,11 @@ TEST(Objective, DeclareUnifiedTest(SoftmaxMultiClassBasic)) {
 }
 
 TEST(Objective, DeclareUnifiedTest(SoftprobMultiClassBasic)) {
-  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("multi:softprob");
-  std::vector<std::pair<std::string, std::string>> args
-    {std::pair<std::string, std::string>("num_class", "3")};
+  xgboost::LearnerTrainParam lparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  std::vector<std::pair<std::string, std::string>> args {
+    std::pair<std::string, std::string>("num_class", "3")};
+
+  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("multi:softprob", &lparam);
   obj->Configure(args);
 
   xgboost::HostDeviceVector<xgboost::bst_float>  io_preds = {2.0f, 0.0f, 1.0f};

tests/cpp/objective/test_objective.cc

@@ -1,13 +1,18 @@
 // Copyright by Contributors
 #include <gtest/gtest.h>
 #include <xgboost/objective.h>
+#include <xgboost/generic_parameters.h>
 
 #include "../helpers.h"
 
 TEST(Objective, UnknownFunction) {
   xgboost::ObjFunction* obj = nullptr;
-  EXPECT_ANY_THROW(obj = xgboost::ObjFunction::Create("unknown_name"));
-  EXPECT_NO_THROW(obj = xgboost::ObjFunction::Create("reg:squarederror"));
+  xgboost::LearnerTrainParam tparam;
+  std::vector<std::pair<std::string, std::string>> args;
+  tparam.InitAllowUnknown(args);
+
+  EXPECT_ANY_THROW(obj = xgboost::ObjFunction::Create("unknown_name", &tparam));
+  EXPECT_NO_THROW(obj = xgboost::ObjFunction::Create("reg:squarederror", &tparam));
   if (obj) {
     delete obj;
   }

tests/cpp/objective/test_ranking_obj.cc

@@ -1,11 +1,15 @@
 // Copyright by Contributors
 #include <xgboost/objective.h>
-
+#include <xgboost/generic_parameters.h>
 #include "../helpers.h"
 
 TEST(Objective, PairwiseRankingGPair) {
-  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("rank:pairwise");
-  std::vector<std::pair<std::string, std::string> > args;
+  xgboost::LearnerTrainParam tparam;
+  std::vector<std::pair<std::string, std::string>> args;
+  tparam.InitAllowUnknown(args);
+
+  xgboost::ObjFunction * obj =
+      xgboost::ObjFunction::Create("rank:pairwise", &tparam);
   obj->Configure(args);
   // Test with setting sample weight to second query group
   CheckRankingObjFunction(obj,

tests/cpp/objective/test_regression_obj.cc

@@ -3,12 +3,16 @@
  */
 #include <gtest/gtest.h>
 #include <xgboost/objective.h>
-
+#include <xgboost/generic_parameters.h>
 #include "../helpers.h"
 
 TEST(Objective, DeclareUnifiedTest(LinearRegressionGPair)) {
-  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("reg:squarederror");
-  std::vector<std::pair<std::string, std::string> > args;
+  xgboost::LearnerTrainParam tparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  std::vector<std::pair<std::string, std::string>> args;
+
+  xgboost::ObjFunction * obj =
+      xgboost::ObjFunction::Create("reg:squarederror", &tparam);
+
   obj->Configure(args);
   CheckObjFunction(obj,
                    {0, 0.1f, 0.9f,   1,    0,  0.1f, 0.9f,  1},
@@ -28,8 +32,10 @@ TEST(Objective, DeclareUnifiedTest(LinearRegressionGPair)) {
 }
 
 TEST(Objective, DeclareUnifiedTest(LogisticRegressionGPair)) {
-  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("reg:logistic");
-  std::vector<std::pair<std::string, std::string> > args;
+  xgboost::LearnerTrainParam tparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  std::vector<std::pair<std::string, std::string>> args;
+  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("reg:logistic", &tparam);
+
   obj->Configure(args);
   CheckObjFunction(obj,
                    {   0,  0.1f,  0.9f,    1,    0,   0.1f,  0.9f,      1}, // preds
@@ -42,8 +48,10 @@ TEST(Objective, DeclareUnifiedTest(LogisticRegressionGPair)) {
 }
 
 TEST(Objective, DeclareUnifiedTest(LogisticRegressionBasic)) {
-  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("reg:logistic");
-  std::vector<std::pair<std::string, std::string> > args;
+  xgboost::LearnerTrainParam lparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  std::vector<std::pair<std::string, std::string>> args;
+  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("reg:logistic", &lparam);
+
   obj->Configure(args);
 
   // test label validation
@@ -70,8 +78,10 @@ TEST(Objective, DeclareUnifiedTest(LogisticRegressionBasic)) {
 }
 
 TEST(Objective, DeclareUnifiedTest(LogisticRawGPair)) {
-  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("binary:logitraw");
-  std::vector<std::pair<std::string, std::string> > args;
+  xgboost::LearnerTrainParam lparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  std::vector<std::pair<std::string, std::string>> args;
+  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("binary:logitraw", &lparam);
+
   obj->Configure(args);
   CheckObjFunction(obj,
                    {   0,  0.1f,  0.9f,    1,    0,   0.1f,   0.9f,     1},
@@ -84,8 +94,10 @@ TEST(Objective, DeclareUnifiedTest(LogisticRawGPair)) {
 }
 
 TEST(Objective, DeclareUnifiedTest(PoissonRegressionGPair)) {
-  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("count:poisson");
-  std::vector<std::pair<std::string, std::string> > args;
+  xgboost::LearnerTrainParam lparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  std::vector<std::pair<std::string, std::string>> args;
+  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("count:poisson", &lparam);
+
   args.emplace_back(std::make_pair("max_delta_step", "0.1f"));
   obj->Configure(args);
   CheckObjFunction(obj,
@@ -104,8 +116,10 @@ TEST(Objective, DeclareUnifiedTest(PoissonRegressionGPair)) {
 }
 
 TEST(Objective, DeclareUnifiedTest(PoissonRegressionBasic)) {
-  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("count:poisson");
-  std::vector<std::pair<std::string, std::string> > args;
+  xgboost::LearnerTrainParam lparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  std::vector<std::pair<std::string, std::string>> args;
+  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("count:poisson", &lparam);
+
   obj->Configure(args);
 
   // test label validation
@@ -130,8 +144,10 @@ TEST(Objective, DeclareUnifiedTest(PoissonRegressionBasic)) {
 }
 
 TEST(Objective, DeclareUnifiedTest(GammaRegressionGPair)) {
-  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("reg:gamma");
-  std::vector<std::pair<std::string, std::string> > args;
+  xgboost::LearnerTrainParam lparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  std::vector<std::pair<std::string, std::string>> args;
+  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("reg:gamma", &lparam);
+
   obj->Configure(args);
   CheckObjFunction(obj,
                    {0, 0.1f, 0.9f, 1, 0,  0.1f,  0.9f,    1},
@@ -149,8 +165,10 @@ TEST(Objective, DeclareUnifiedTest(GammaRegressionGPair)) {
 }
 
 TEST(Objective, DeclareUnifiedTest(GammaRegressionBasic)) {
-  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("reg:gamma");
-  std::vector<std::pair<std::string, std::string> > args;
+  xgboost::LearnerTrainParam lparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  std::vector<std::pair<std::string, std::string>> args;
+  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("reg:gamma", &lparam);
+
   obj->Configure(args);
 
   // test label validation
@@ -175,8 +193,10 @@ TEST(Objective, DeclareUnifiedTest(GammaRegressionBasic)) {
 }
 
 TEST(Objective, DeclareUnifiedTest(TweedieRegressionGPair)) {
-  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("reg:tweedie");
-  std::vector<std::pair<std::string, std::string> > args;
+  xgboost::LearnerTrainParam lparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  std::vector<std::pair<std::string, std::string>> args;
+  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("reg:tweedie", &lparam);
+
   args.emplace_back(std::make_pair("tweedie_variance_power", "1.1f"));
   obj->Configure(args);
   CheckObjFunction(obj,
@@ -195,9 +215,65 @@ TEST(Objective, DeclareUnifiedTest(TweedieRegressionGPair)) {
   delete obj;
 }
 
+#if defined(__CUDACC__)
+TEST(Objective, CPU_vs_CUDA) {
+  xgboost::LearnerTrainParam lparam = xgboost::CreateEmptyGenericParam(0, 1);
+
+  xgboost::ObjFunction * obj =
+      xgboost::ObjFunction::Create("reg:squarederror", &lparam);
+  xgboost::HostDeviceVector<xgboost::GradientPair> cpu_out_preds;
+  xgboost::HostDeviceVector<xgboost::GradientPair> cuda_out_preds;
+
+  constexpr size_t kRows = 400;
+  constexpr size_t kCols = 100;
+  auto ppdmat = xgboost::CreateDMatrix(kRows, kCols, 0, 0);
+  xgboost::HostDeviceVector<float> preds;
+  preds.Resize(kRows);
+  auto& h_preds = preds.HostVector();
+  for (size_t i = 0; i < h_preds.size(); ++i) {
+    h_preds[i] = static_cast<float>(i);
+  }
+  auto& info = (*ppdmat)->Info();
+
+  info.labels_.Resize(kRows);
+  auto& h_labels = info.labels_.HostVector();
+  for (size_t i = 0; i < h_labels.size(); ++i) {
+    h_labels[i] = 1 / (float)(i+1);
+  }
+
+  {
+    // CPU
+    lparam.n_gpus = 0;
+    obj->GetGradient(preds, info, 0, &cpu_out_preds);
+  }
+  {
+    // CUDA
+    lparam.n_gpus = 1;
+    obj->GetGradient(preds, info, 0, &cuda_out_preds);
+  }
+
+  auto& h_cpu_out = cpu_out_preds.HostVector();
+  auto& h_cuda_out = cuda_out_preds.HostVector();
+
+  float sgrad = 0;
+  float shess = 0;
+  for (size_t i = 0; i < kRows; ++i) {
+    sgrad += std::pow(h_cpu_out[i].GetGrad() - h_cuda_out[i].GetGrad(), 2);
+    shess += std::pow(h_cpu_out[i].GetHess() - h_cuda_out[i].GetHess(), 2);
+  }
+  ASSERT_NEAR(sgrad, 0.0f, xgboost::kRtEps);
+  ASSERT_NEAR(shess, 0.0f, xgboost::kRtEps);
+
+  delete ppdmat;
+  delete obj;
+}
+#endif
+
 TEST(Objective, DeclareUnifiedTest(TweedieRegressionBasic)) {
-  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("reg:tweedie");
-  std::vector<std::pair<std::string, std::string> > args;
+  xgboost::LearnerTrainParam lparam = xgboost::CreateEmptyGenericParam(0, NGPUS);
+  std::vector<std::pair<std::string, std::string>> args;
+  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("reg:tweedie", &lparam);
+
   obj->Configure(args);
 
   // test label validation
@@ -225,8 +301,11 @@ TEST(Objective, DeclareUnifiedTest(TweedieRegressionBasic)) {
 // CoxRegression not implemented in GPU code, no need for testing.
 #if !defined(__CUDACC__)
 TEST(Objective, CoxRegressionGPair) {
-  xgboost::ObjFunction * obj = xgboost::ObjFunction::Create("survival:cox");
-  std::vector<std::pair<std::string, std::string> > args;
+  xgboost::LearnerTrainParam lparam = xgboost::CreateEmptyGenericParam(0, 0);
+  std::vector<std::pair<std::string, std::string>> args;
+  xgboost::ObjFunction * obj =
+      xgboost::ObjFunction::Create("survival:cox", &lparam);
+
   obj->Configure(args);
   CheckObjFunction(obj,
                    { 0, 0.1f, 0.9f,       1,       0,    0.1f,   0.9f,       1},

tests/cpp/predictor/test_cpu_predictor.cc

@@ -6,8 +6,9 @@
 
 namespace xgboost {
 TEST(cpu_predictor, Test) {
+  auto lparam = CreateEmptyGenericParam(0, 0);
   std::unique_ptr<Predictor> cpu_predictor =
-      std::unique_ptr<Predictor>(Predictor::Create("cpu_predictor"));
+      std::unique_ptr<Predictor>(Predictor::Create("cpu_predictor", &lparam));
 
   gbm::GBTreeModel model = CreateTestModel();
 
@@ -56,9 +57,9 @@ TEST(cpu_predictor, Test) {
 
 TEST(cpu_predictor, ExternalMemoryTest) {
   std::unique_ptr<DMatrix> dmat = CreateSparsePageDMatrix(12, 64);
-
+  auto lparam = CreateEmptyGenericParam(0, 0);
   std::unique_ptr<Predictor> cpu_predictor =
-      std::unique_ptr<Predictor>(Predictor::Create("cpu_predictor"));
+      std::unique_ptr<Predictor>(Predictor::Create("cpu_predictor", &lparam));
 
   gbm::GBTreeModel model = CreateTestModel();
 

tests/cpp/predictor/test_gpu_predictor.cu

@@ -1,6 +1,6 @@
 
 /*!
- * Copyright 2017 XGBoost contributors
+ * Copyright 2017-2019 XGBoost contributors
  */
 #include <dmlc/logging.h>
 #include <dmlc/filesystem.h>
@@ -25,10 +25,13 @@ namespace xgboost {
 namespace predictor {
 
 TEST(gpu_predictor, Test) {
+  auto cpu_lparam = CreateEmptyGenericParam(0, 0);
+  auto gpu_lparam = CreateEmptyGenericParam(0, 1);
+
   std::unique_ptr<Predictor> gpu_predictor =
-      std::unique_ptr<Predictor>(Predictor::Create("gpu_predictor"));
+      std::unique_ptr<Predictor>(Predictor::Create("gpu_predictor", &gpu_lparam));
   std::unique_ptr<Predictor> cpu_predictor =
-      std::unique_ptr<Predictor>(Predictor::Create("cpu_predictor"));
+      std::unique_ptr<Predictor>(Predictor::Create("cpu_predictor", &cpu_lparam));
 
   gpu_predictor->Init({}, {});
   cpu_predictor->Init({}, {});
@@ -85,8 +88,9 @@ TEST(gpu_predictor, Test) {
 }
 
 TEST(gpu_predictor, ExternalMemoryTest) {
+  auto lparam = CreateEmptyGenericParam(0, 1);
   std::unique_ptr<Predictor> gpu_predictor =
-      std::unique_ptr<Predictor>(Predictor::Create("gpu_predictor"));
+      std::unique_ptr<Predictor>(Predictor::Create("gpu_predictor", &lparam));
   gpu_predictor->Init({}, {});
   gbm::GBTreeModel model = CreateTestModel();
   std::unique_ptr<DMatrix> dmat = CreateSparsePageDMatrix(32, 64);
@@ -127,8 +131,7 @@ TEST(gpu_predictor, ExternalMemoryTest) {
 #if defined(XGBOOST_USE_NCCL)
 // Test whether pickling preserves predictor parameters
 TEST(gpu_predictor, MGPU_PicklingTest) {
-  int ngpu;
-  dh::safe_cuda(cudaGetDeviceCount(&ngpu));
+  int const ngpu = GPUSet::AllVisible().Size();
 
   dmlc::TemporaryDirectory tempdir;
   const std::string tmp_file = tempdir.path + "/simple.libsvm";
@@ -201,12 +204,14 @@ TEST(gpu_predictor, MGPU_PicklingTest) {
 
 // multi-GPU predictor test
 TEST(gpu_predictor, MGPU_Test) {
-  std::unique_ptr<Predictor> gpu_predictor =
-      std::unique_ptr<Predictor>(Predictor::Create("gpu_predictor"));
-  std::unique_ptr<Predictor> cpu_predictor =
-      std::unique_ptr<Predictor>(Predictor::Create("cpu_predictor"));
+  auto cpu_lparam = CreateEmptyGenericParam(0, 0);
+  auto gpu_lparam = CreateEmptyGenericParam(0, -1);
+
+  std::unique_ptr<Predictor> gpu_predictor =
+      std::unique_ptr<Predictor>(Predictor::Create("gpu_predictor", &gpu_lparam));
+  std::unique_ptr<Predictor> cpu_predictor =
+      std::unique_ptr<Predictor>(Predictor::Create("cpu_predictor", &cpu_lparam));
 
-  gpu_predictor->Init({std::pair<std::string, std::string>("n_gpus", "-1")}, {});
   cpu_predictor->Init({}, {});
 
   for (size_t i = 1; i < 33; i *= 2) {
@@ -234,9 +239,11 @@ TEST(gpu_predictor, MGPU_Test) {
 
 // multi-GPU predictor external memory test
 TEST(gpu_predictor, MGPU_ExternalMemoryTest) {
+  auto gpu_lparam = CreateEmptyGenericParam(0, -1);
+
   std::unique_ptr<Predictor> gpu_predictor =
-      std::unique_ptr<Predictor>(Predictor::Create("gpu_predictor"));
-  gpu_predictor->Init({std::pair<std::string, std::string>("n_gpus", "-1")}, {});
+      std::unique_ptr<Predictor>(Predictor::Create("gpu_predictor", &gpu_lparam));
+  gpu_predictor->Init({}, {});
 
   gbm::GBTreeModel model = CreateTestModel();
   const int n_classes = 3;

tests/cpp/test_learner.cc

@@ -2,6 +2,7 @@
 #include <gtest/gtest.h>
 #include <vector>
 #include "helpers.h"
+
 #include "xgboost/learner.h"
 #include "dmlc/filesystem.h"
 
@@ -115,4 +116,119 @@ TEST(Learner, SLOW_CheckMultiBatch) {
   learner->UpdateOneIter(0, dmat.get());
 }
 
+#if defined(XGBOOST_USE_CUDA)
+
+TEST(Learner, IO) {
+  using Arg = std::pair<std::string, std::string>;
+  size_t constexpr kRows = 10;
+  auto pp_dmat = CreateDMatrix(kRows, 10, 0);
+  auto p_dmat = *pp_dmat;
+
+  std::vector<bst_float> labels(kRows);
+  for (size_t i = 0; i < labels.size(); ++i) {
+    labels[i] = i;
+  }
+  p_dmat->Info().labels_.HostVector() = labels;
+  std::vector<std::shared_ptr<DMatrix>> mat {p_dmat};
+
+  std::unique_ptr<Learner> learner {Learner::Create(mat)};
+  learner->Configure({Arg{"tree_method", "auto"},
+                      Arg{"predictor", "gpu_predictor"},
+                      Arg{"n_gpus", "-1"}});
+  learner->InitModel();
+  learner->UpdateOneIter(0, p_dmat.get());
+  ASSERT_EQ(learner->GetLearnerTrainParameter().gpu_id, 0);
+  ASSERT_EQ(learner->GetLearnerTrainParameter().n_gpus, -1);
+
+  dmlc::TemporaryDirectory tempdir;
+  const std::string fname = tempdir.path + "/model.bst";
+
+  {
+    // Create a scope to close the stream before next read.
+    std::unique_ptr<dmlc::Stream> fo(dmlc::Stream::Create(fname.c_str(), "w"));
+    learner->Save(fo.get());
+  }
+
+  std::unique_ptr<dmlc::Stream> fi(dmlc::Stream::Create(fname.c_str(), "r"));
+  learner->Load(fi.get());
+  ASSERT_EQ(learner->GetLearnerTrainParameter().gpu_id, 0);
+  ASSERT_EQ(learner->GetLearnerTrainParameter().n_gpus, 0);
+
+  delete pp_dmat;
+}
+
+// Tests for automatic GPU configuration.
+TEST(Learner, GPUConfiguration) {
+  using Arg = std::pair<std::string, std::string>;
+  size_t constexpr kRows = 10;
+  auto pp_dmat = CreateDMatrix(kRows, 10, 0);
+  auto p_dmat = *pp_dmat;
+  std::vector<std::shared_ptr<DMatrix>> mat {p_dmat};
+  std::vector<bst_float> labels(kRows);
+  for (size_t i = 0; i < labels.size(); ++i) {
+    labels[i] = i;
+  }
+  p_dmat->Info().labels_.HostVector() = labels;
+  {
+    std::unique_ptr<Learner> learner {Learner::Create(mat)};
+    learner->Configure({Arg{"booster", "gblinear"},
+                        Arg{"updater", "gpu_coord_descent"}});
+    learner->InitModel();
+    learner->UpdateOneIter(0, p_dmat.get());
+    ASSERT_EQ(learner->GetLearnerTrainParameter().gpu_id, 0);
+    ASSERT_EQ(learner->GetLearnerTrainParameter().n_gpus, 1);
+  }
+  {
+    std::unique_ptr<Learner> learner {Learner::Create(mat)};
+    learner->Configure({Arg{"tree_method", "gpu_exact"}});
+    learner->InitModel();
+    learner->UpdateOneIter(0, p_dmat.get());
+    ASSERT_EQ(learner->GetLearnerTrainParameter().gpu_id, 0);
+    ASSERT_EQ(learner->GetLearnerTrainParameter().n_gpus, 1);
+  }
+  {
+    std::unique_ptr<Learner> learner {Learner::Create(mat)};
+    learner->Configure({Arg{"tree_method", "gpu_hist"}});
+    learner->InitModel();
+    learner->UpdateOneIter(0, p_dmat.get());
+    ASSERT_EQ(learner->GetLearnerTrainParameter().gpu_id, 0);
+    ASSERT_EQ(learner->GetLearnerTrainParameter().n_gpus, 1);
+  }
+  {
+    // with CPU algorithm
+    std::unique_ptr<Learner> learner {Learner::Create(mat)};
+    learner->Configure({Arg{"tree_method", "hist"}});
+    learner->InitModel();
+    learner->UpdateOneIter(0, p_dmat.get());
+    ASSERT_EQ(learner->GetLearnerTrainParameter().gpu_id, 0);
+    ASSERT_EQ(learner->GetLearnerTrainParameter().n_gpus, 0);
+  }
+  {
+    // with CPU algorithm, but `n_gpus` takes priority
+    std::unique_ptr<Learner> learner {Learner::Create(mat)};
+    learner->Configure({Arg{"tree_method", "hist"},
+                        Arg{"n_gpus", "1"}});
+    learner->InitModel();
+    learner->UpdateOneIter(0, p_dmat.get());
+    ASSERT_EQ(learner->GetLearnerTrainParameter().gpu_id, 0);
+    ASSERT_EQ(learner->GetLearnerTrainParameter().n_gpus, 1);
+  }
+  {
+    // With CPU algorithm but GPU Predictor, this is to simulate when
+    // XGBoost is only used for prediction, so tree method is not
+    // specified.
+    std::unique_ptr<Learner> learner {Learner::Create(mat)};
+    learner->Configure({Arg{"tree_method", "hist"},
+                        Arg{"predictor", "gpu_predictor"}});
+    learner->InitModel();
+    learner->UpdateOneIter(0, p_dmat.get());
+    ASSERT_EQ(learner->GetLearnerTrainParameter().gpu_id, 0);
+    ASSERT_EQ(learner->GetLearnerTrainParameter().n_gpus, 1);
+  }
+
+  delete pp_dmat;
+}
+
+#endif  // XGBOOST_USE_CUDA
+
 }  // namespace xgboost

tests/cpp/test_logging.cc

@@ -52,6 +52,9 @@ TEST(Logging, Basic) {
   LOG(CONSOLE) << "Test Log Console";  // ignore global setting.
   output = testing::internal::GetCapturedStderr();
   ASSERT_NE(output.find("Test Log Console"), std::string::npos);
+
+  args["verbosity"] = "1";  // restore
+  ConsoleLogger::Configure(args.cbegin(), args.cend());
 }
 
 }  // namespace xgboost

tests/cpp/test_main.cc

@@ -1,7 +1,12 @@
 // Copyright by Contributors
 #include <gtest/gtest.h>
+#include <xgboost/logging.h>
+#include <string>
+#include <vector>
 
 int main(int argc, char ** argv) {
+  std::vector<std::pair<std::string, std::string>> args {{"verbosity", "3"}};
+  xgboost::ConsoleLogger::Configure(args.begin(), args.end());
   testing::InitGoogleTest(&argc, argv);
   testing::FLAGS_gtest_death_test_style = "threadsafe";
   return RUN_ALL_TESTS();

tests/cpp/tree/test_gpu_exact.cu

@@ -12,12 +12,10 @@ namespace tree {
 
 TEST(GPUExact, Update) {
   using Arg = std::pair<std::string, std::string>;
-  std::vector<Arg> args{
-    {"n_gpus", "1"},
-    {"gpu_id", "0"},
-    {"max_depth", "1"}};
+  auto lparam = CreateEmptyGenericParam(0, 1);
+  std::vector<Arg> args{{"max_depth", "1"}};
 
-  auto* p_gpuexact_maker = TreeUpdater::Create("grow_gpu");
+  auto* p_gpuexact_maker = TreeUpdater::Create("grow_gpu", &lparam);
   p_gpuexact_maker->Init(args);
 
   size_t constexpr kNRows = 4;

tests/cpp/tree/test_gpu_hist.cu

@@ -86,7 +86,6 @@ TEST(GpuHist, BuildGidxDense) {
   int constexpr kNRows = 16, kNCols = 8;
   TrainParam param;
   param.max_depth = 1;
-  param.n_gpus = 1;
   param.max_leaves = 0;
 
   DeviceShard<GradientPairPrecise> shard(0, 0, 0, kNRows, param, kNCols);
@@ -125,7 +124,6 @@ TEST(GpuHist, BuildGidxSparse) {
   int constexpr kNRows = 16, kNCols = 8;
   TrainParam param;
   param.max_depth = 1;
-  param.n_gpus = 1;
   param.max_leaves = 0;
 
   DeviceShard<GradientPairPrecise> shard(0, 0, 0, kNRows, param, kNCols);
@@ -169,7 +167,6 @@ void TestBuildHist(GPUHistBuilderBase<GradientSumT>& builder) {
 
   TrainParam param;
   param.max_depth = 6;
-  param.n_gpus = 1;
   param.max_leaves = 0;
 
   DeviceShard<GradientSumT> shard(0, 0, 0, kNRows, param, kNCols);
@@ -264,7 +261,6 @@ TEST(GpuHist, EvaluateSplits) {
 
   TrainParam param;
   param.max_depth = 1;
-  param.n_gpus = 1;
   param.colsample_bynode = 1;
   param.colsample_bylevel = 1;
   param.colsample_bytree = 1;

tests/cpp/tree/test_prune.cc

@@ -1,5 +1,5 @@
 /*!
- * Copyright 2018 by Contributors
+ * Copyright 2018-2019 by Contributors
  */
 #include "../helpers.h"
 #include "../../../src/common/host_device_vector.h"
@@ -29,12 +29,14 @@ TEST(Updater, Prune) {
         {0.25f, 0.24f}, {0.25f, 0.24f}, {0.25f, 0.24f}, {0.25f, 0.24f} };
   auto dmat = CreateDMatrix(32, 16, 0.4, 3);
 
+  auto lparam = CreateEmptyGenericParam(0, 0);
+
   // prepare tree
   RegTree tree = RegTree();
   tree.param.InitAllowUnknown(cfg);
   std::vector<RegTree*> trees {&tree};
   // prepare pruner
-  std::unique_ptr<TreeUpdater> pruner(TreeUpdater::Create("prune"));
+  std::unique_ptr<TreeUpdater> pruner(TreeUpdater::Create("prune", &lparam));
   pruner->Init(cfg);
 
   // loss_chg < min_split_loss;

tests/cpp/tree/test_refresh.cc

@@ -1,5 +1,5 @@
 /*!
- * Copyright 2018 by Contributors
+ * Copyright 2018-2019 by Contributors
  */
 #include "../helpers.h"
 #include "../../../src/common/host_device_vector.h"
@@ -25,9 +25,10 @@ TEST(Updater, Refresh) {
     {"reg_lambda", "1"}};
 
   RegTree tree = RegTree();
+  auto lparam = CreateEmptyGenericParam(0, 0);
   tree.param.InitAllowUnknown(cfg);
   std::vector<RegTree*> trees {&tree};
-  std::unique_ptr<TreeUpdater> refresher(TreeUpdater::Create("refresh"));
+  std::unique_ptr<TreeUpdater> refresher(TreeUpdater::Create("refresh", &lparam));
 
   tree.ExpandNode(0, 2, 0.2f, false, 0.0, 0.2f, 0.8f, 0.0f, 0.0f);
   int cleft = tree[0].LeftChild();