Pseudo-huber loss metric added (#5647)

- Add pseudo huber loss objective.
- Add pseudo huber loss metric.

Co-authored-by: Reetz <s02reetz@iavgroup.local>

doc/parameter.rst

@@ -342,6 +342,7 @@ Specify the learning task and the corresponding learning objective. The objectiv
   - ``reg:squarederror``: regression with squared loss.
   - ``reg:squaredlogerror``: regression with squared log loss :math:`\frac{1}{2}[log(pred + 1) - log(label + 1)]^2`.  All input labels are required to be greater than -1.  Also, see metric ``rmsle`` for possible issue  with this objective.
   - ``reg:logistic``: logistic regression
+  - ``reg:pseudohubererror``: regression with Pseudo Huber loss, a twice differentiable alternative to absolute loss.
   - ``binary:logistic``: logistic regression for binary classification, output probability
   - ``binary:logitraw``: logistic regression for binary classification, output score before logistic transformation
   - ``binary:hinge``: hinge loss for binary classification. This makes predictions of 0 or 1, rather than producing probabilities.
@@ -376,6 +377,7 @@ Specify the learning task and the corresponding learning objective. The objectiv
     - ``rmse``: `root mean square error <http://en.wikipedia.org/wiki/Root_mean_square_error>`_
     - ``rmsle``: root mean square log error: :math:`\sqrt{\frac{1}{N}[log(pred + 1) - log(label + 1)]^2}`. Default metric of ``reg:squaredlogerror`` objective. This metric reduces errors generated by outliers in dataset.  But because ``log`` function is employed, ``rmsle`` might output ``nan`` when prediction value is less than -1.  See ``reg:squaredlogerror`` for other requirements.
     - ``mae``: `mean absolute error <https://en.wikipedia.org/wiki/Mean_absolute_error>`_
+    - ``mphe``: `mean Pseudo Huber error <https://en.wikipedia.org/wiki/Huber_loss>`_. Default metric of ``reg:pseudohubererror`` objective.
     - ``logloss``: `negative log-likelihood <http://en.wikipedia.org/wiki/Log-likelihood>`_
     - ``error``: Binary classification error rate. It is calculated as ``#(wrong cases)/#(all cases)``. For the predictions, the evaluation will regard the instances with prediction value larger than 0.5 as positive instances, and the others as negative instances.
     - ``error@t``: a different than 0.5 binary classification threshold value could be specified by providing a numerical value through 't'.

src/metric/elementwise_metric.cu

@@ -190,6 +190,19 @@ struct EvalRowLogLoss {
   }
 };
 
+struct EvalRowMPHE {
+  char const *Name() const {
+    return "mphe";
+  }
+  XGBOOST_DEVICE bst_float EvalRow(bst_float label, bst_float pred) const {
+    bst_float diff = label - pred;
+    return std::sqrt( 1 + diff * diff) - 1;
+  }
+  static bst_float GetFinal(bst_float esum, bst_float wsum) {
+    return wsum == 0 ? esum : esum / wsum;
+  }
+};
+
 struct EvalError {
   explicit EvalError(const char* param) {
     if (param != nullptr) {
@@ -359,6 +372,10 @@ XGBOOST_REGISTER_METRIC(MAE, "mae")
 .describe("Mean absolute error.")
 .set_body([](const char* param) { return new EvalEWiseBase<EvalRowMAE>(); });
 
+XGBOOST_REGISTER_METRIC(MPHE, "mphe")
+.describe("Mean Pseudo Huber error.")
+.set_body([](const char* param) { return new EvalEWiseBase<EvalRowMPHE>(); });
+
 XGBOOST_REGISTER_METRIC(LogLoss, "logloss")
 .describe("Negative loglikelihood for logistic regression.")
 .set_body([](const char* param) { return new EvalEWiseBase<EvalRowLogLoss>(); });

src/objective/regression_loss.h

@@ -98,6 +98,37 @@ struct LogisticRegression {
   static const char* Name() { return "reg:logistic"; }
 };
 
+struct PseudoHuberError {
+  XGBOOST_DEVICE static bst_float PredTransform(bst_float x) {
+    return x;
+  }
+  XGBOOST_DEVICE static bool CheckLabel(bst_float label) {
+    return true;
+  }
+  XGBOOST_DEVICE static bst_float FirstOrderGradient(bst_float predt, bst_float label) {
+    const float z = predt - label;
+    const float scale_sqrt = std::sqrt(1 + std::pow(z, 2));
+    return z/scale_sqrt;
+  }
+  XGBOOST_DEVICE static bst_float SecondOrderGradient(bst_float predt, bst_float label) {
+    const float scale = 1 + std::pow(predt - label, 2);
+    const float scale_sqrt = std::sqrt(scale);
+    return 1/(scale*scale_sqrt);
+  }
+  static bst_float ProbToMargin(bst_float base_score) {
+    return base_score;
+  }
+  static const char* LabelErrorMsg() {
+    return "";
+  }
+  static const char* DefaultEvalMetric() {
+    return "mphe";
+  }
+  static const char* Name() {
+    return "reg:pseudohubererror";
+  }
+};
+
 // logistic loss for binary classification task
 struct LogisticClassification : public LogisticRegression {
   static const char* DefaultEvalMetric() { return "error"; }

src/objective/regression_obj.cu

@@ -152,6 +152,10 @@ XGBOOST_REGISTER_OBJECTIVE(LogisticRegression, LogisticRegression::Name())
 .describe("Logistic regression for probability regression task.")
 .set_body([]() { return new RegLossObj<LogisticRegression>(); });
 
+XGBOOST_REGISTER_OBJECTIVE(PseudoHuberError, PseudoHuberError::Name())
+.describe("Regression Pseudo Huber error.")
+.set_body([]() { return new RegLossObj<PseudoHuberError>(); });
+
 XGBOOST_REGISTER_OBJECTIVE(LogisticClassification, LogisticClassification::Name())
 .describe("Logistic regression for binary classification task.")
 .set_body([]() { return new RegLossObj<LogisticClassification>(); });

tests/cpp/metric/test_elementwise_metric.cc

@@ -44,6 +44,18 @@ TEST(Metric, DeclareUnifiedTest(MAE)) {
   delete metric;
 }
 
+TEST(Metric, DeclareUnifiedTest(MPHE)) {
+  auto lparam = xgboost::CreateEmptyGenericParam(GPUIDX);
+  xgboost::Metric * metric = xgboost::Metric::Create("mphe", &lparam);
+  metric->Configure({});
+  ASSERT_STREQ(metric->Name(), "mphe");
+  EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-10);
+  EXPECT_NEAR(GetMetricEval(metric,
+                            {0.1f, 0.9f, 0.1f, 0.9f},
+                            {  0,   0,   1,   1}), 0.17517f, 1e-4);
+  delete metric;
+}
+
 TEST(Metric, DeclareUnifiedTest(LogLoss)) {
   auto lparam = xgboost::CreateEmptyGenericParam(GPUIDX);
   xgboost::Metric * metric = xgboost::Metric::Create("logloss", &lparam);

tests/cpp/objective/test_regression_obj.cc

@@ -55,6 +55,29 @@ TEST(Objective, DeclareUnifiedTest(SquaredLog)) {
   ASSERT_EQ(obj->DefaultEvalMetric(), std::string{"rmsle"});
 }
 
+TEST(Objective, DeclareUnifiedTest(PseudoHuber)) {
+  GenericParameter tparam = CreateEmptyGenericParam(GPUIDX);
+  std::vector<std::pair<std::string, std::string>> args;
+
+  std::unique_ptr<ObjFunction> obj { ObjFunction::Create("reg:pseudohubererror", &tparam) };
+  obj->Configure(args);
+  CheckConfigReload(obj, "reg:pseudohubererror");
+
+  CheckObjFunction(obj,
+                   {0.1f, 0.2f, 0.4f, 0.8f, 1.6f},  // pred
+                   {1.0f, 1.0f, 1.0f, 1.0f, 1.0f},  // labels
+                   {1.0f, 1.0f, 1.0f, 1.0f, 1.0f},  // weights
+                   {-0.668965f, -0.624695f, -0.514496f, -0.196116f, 0.514496f}, // out_grad
+                   { 0.410660f,  0.476140f,  0.630510f,  0.9428660f, 0.630510f}); // out_hess
+  CheckObjFunction(obj,
+                   {0.1f, 0.2f, 0.4f, 0.8f, 1.6f},  // pred
+                   {1.0f, 1.0f, 1.0f, 1.0f, 1.0f},  // labels
+                   {},                              // empty weights
+                   {-0.668965f, -0.624695f, -0.514496f, -0.196116f, 0.514496f}, // out_grad
+                   { 0.410660f,  0.476140f,  0.630510f,  0.9428660f, 0.630510f}); // out_hess
+  ASSERT_EQ(obj->DefaultEvalMetric(), std::string{"mphe"});
+}
+
 TEST(Objective, DeclareUnifiedTest(LogisticRegressionGPair)) {
   GenericParameter tparam = CreateEmptyGenericParam(GPUIDX);
   std::vector<std::pair<std::string, std::string>> args;