[Breaking] Fix custom metric for multi output. (#5954)

* Set output margin to true for custom metric.  This fixes only R and Python.

R-package/R/utils.R

@@ -173,7 +173,7 @@ xgb.iter.eval <- function(booster_handle, watchlist, iter, feval = NULL) {
   } else {
     res <- sapply(seq_along(watchlist), function(j) {
       w <- watchlist[[j]]
-      preds <- predict(booster_handle, w, ntreelimit = 0) # predict using all trees
+      preds <- predict(booster_handle, w, outputmargin = TRUE, ntreelimit = 0) # predict using all trees
       eval_res <- feval(preds, w)
       out <- eval_res$value
       names(out) <- paste0(evnames[j], "-", eval_res$metric)

R-package/tests/testthat/test_custom_objective.R

@@ -79,6 +79,10 @@ test_that("custom objective with multi-class works", {
     hess <- rnorm(dim(as.matrix(preds))[1])
     return (list(grad = grad, hess = hess))
   }
+  fake_merror <- function(preds, dtrain) {
+    expect_equal(dim(data)[1] * nclasses, dim(as.matrix(preds))[1])
+  }
   param$objective <- fake_softprob
+  param$eval_metric <- fake_merror
   bst <- xgb.train(param, dtrain, 1, num_class = nclasses)
 })