diff --git a/src/linear/updater_coordinate.cc b/src/linear/updater_coordinate.cc
index d4c294c9a..468911d19 100644
--- a/src/linear/updater_coordinate.cc
+++ b/src/linear/updater_coordinate.cc
@@ -27,7 +27,7 @@ struct CoordinateTrainParam : public dmlc::Parameter<CoordinateTrainParam> {
   DMLC_DECLARE_PARAMETER(CoordinateTrainParam) {
     DMLC_DECLARE_FIELD(learning_rate)
         .set_lower_bound(0.0f)
-        .set_default(1.0f)
+        .set_default(0.5f)
         .describe("Learning rate of each update.");
     DMLC_DECLARE_FIELD(reg_lambda)
         .set_lower_bound(0.0f)
diff --git a/src/objective/multiclass_obj.cc b/src/objective/multiclass_obj.cc
index 9341682e1..af212d25a 100644
--- a/src/objective/multiclass_obj.cc
+++ b/src/objective/multiclass_obj.cc
@@ -65,7 +65,8 @@ class SoftmaxMultiClassObj : public ObjFunction {
         const bst_float wt = info.GetWeight(i);
         for (int k = 0; k < nclass; ++k) {
           bst_float p = rec[k];
-          const bst_float h = 2.0f * p * (1.0f - p) * wt;
+          const float eps = 1e-16f;
+          const bst_float h = fmax(2.0f * p * (1.0f - p) * wt, eps);
           if (label == k) {
             gpair[i * nclass + k] = GradientPair((p - 1.0f) * wt, h);
           } else {
diff --git a/tests/cpp/linear/test_linear.cc b/tests/cpp/linear/test_linear.cc
index ac6a01e9b..0246935cc 100644
--- a/tests/cpp/linear/test_linear.cc
+++ b/tests/cpp/linear/test_linear.cc
@@ -31,7 +31,7 @@ TEST(Linear, coordinate) {
   mat->InitColAccess(enabled, 1.0f, 1 << 16, false);
   auto updater = std::unique_ptr<xgboost::LinearUpdater>(
       xgboost::LinearUpdater::Create("coord_descent"));
-  updater->Init({});
+  updater->Init({{"eta", "1."}});
   xgboost::HostDeviceVector<xgboost::GradientPair> gpair(
       mat->Info().num_row_, xgboost::GradientPair(-5, 1.0));
   xgboost::gbm::GBLinearModel model;