[Breaking] Set output margin to True for custom objective. (#5564)

* Set output margin to True for custom objective in Python and R.

* Add a demo for writing multi-class custom objective function.

* Run tests on selected demos.

R-package/R/utils.R

@@ -145,7 +145,7 @@ xgb.iter.update <- function(booster_handle, dtrain, iter, obj = NULL) {
   if (is.null(obj)) {
     .Call(XGBoosterUpdateOneIter_R, booster_handle, as.integer(iter), dtrain)
   } else {
-    pred <- predict(booster_handle, dtrain, training = TRUE)
+    pred <- predict(booster_handle, dtrain, outputmargin = TRUE, training = TRUE)
     gpair <- obj(pred, dtrain)
     .Call(XGBoosterBoostOneIter_R, booster_handle, dtrain, gpair$grad, gpair$hess)
   }

R-package/tests/testthat/test_custom_objective.R

@@ -58,3 +58,20 @@ test_that("custom objective using DMatrix attr works", {
   bst <- xgb.train(param, dtrain, num_round, watchlist)
   expect_equal(class(bst), "xgb.Booster")
 })
+
+test_that("custom objective with multi-class works", {
+  data = as.matrix(iris[, -5])
+  label =  as.numeric(iris$Species) - 1
+  dtrain <- xgb.DMatrix(data = data, label = label)
+  nclasses <- 3
+
+  fake_softprob <- function(preds, dtrain) {
+    expect_true(all(matrix(preds) == 0.5))
+    grad <- rnorm(dim(as.matrix(preds))[1])
+    expect_equal(dim(data)[1] * nclasses, dim(as.matrix(preds))[1])
+    hess <- rnorm(dim(as.matrix(preds))[1])
+    return (list(grad = grad, hess = hess))
+  }
+  param$objective = fake_softprob
+  bst <- xgb.train(param, dtrain, 1, num_class=nclasses)
+})