From 54fb49ee5ce5208f89217d4b689090322788263a Mon Sep 17 00:00:00 2001 From: hetong007 Date: Tue, 5 May 2015 16:31:49 -0700 Subject: [PATCH 1/4] add early stopping to R --- R-package/NAMESPACE | 2 +- R-package/R/xgb.train.R | 58 ++++++++++++++++++- R-package/R/xgboost.R | 10 +++- R-package/man/agaricus.test.Rd | 2 +- R-package/man/agaricus.train.Rd | 2 +- R-package/man/getinfo.Rd | 2 +- R-package/man/nrow-xgb.DMatrix-method.Rd | 2 +- R-package/man/predict-xgb.Booster-method.Rd | 2 +- .../man/predict-xgb.Booster.handle-method.Rd | 2 +- R-package/man/setinfo.Rd | 2 +- R-package/man/slice.Rd | 2 +- R-package/man/xgb.DMatrix.Rd | 2 +- R-package/man/xgb.DMatrix.save.Rd | 2 +- R-package/man/xgb.cv.Rd | 2 +- R-package/man/xgb.dump.Rd | 2 +- R-package/man/xgb.importance.Rd | 2 +- R-package/man/xgb.load.Rd | 2 +- R-package/man/xgb.model.dt.tree.Rd | 2 +- R-package/man/xgb.plot.importance.Rd | 2 +- R-package/man/xgb.plot.tree.Rd | 2 +- R-package/man/xgb.save.Rd | 2 +- R-package/man/xgb.save.raw.Rd | 2 +- R-package/man/xgb.train.Rd | 18 ++++-- R-package/man/xgboost.Rd | 11 +++- 24 files changed, 106 insertions(+), 31 deletions(-) diff --git a/R-package/NAMESPACE b/R-package/NAMESPACE index a4f07799a..d7f9e455c 100644 --- a/R-package/NAMESPACE +++ b/R-package/NAMESPACE @@ -1,4 +1,4 @@ -# Generated by roxygen2 (4.1.1): do not edit by hand +# Generated by roxygen2 (4.1.0): do not edit by hand export(getinfo) export(setinfo) diff --git a/R-package/R/xgb.train.R b/R-package/R/xgb.train.R index a99740f64..6f6c1a900 100644 --- a/R-package/R/xgb.train.R +++ b/R-package/R/xgb.train.R @@ -66,7 +66,11 @@ #' prediction and dtrain, #' @param verbose If 0, xgboost will stay silent. If 1, xgboost will print #' information of performance. If 2, xgboost will print information of both -#' +#' @param earlyStopRound If \code{NULL}, the early stopping function is not triggered. +#' If set to an integer \code{k}, training with a validation set will stop if the performance +#' keeps getting worse consecutively for \code{k} rounds. +#' @param maximize If \code{feval} and \code{earlyStopRound} are set, then \code{maximize} must be set as well. +#' \code{maximize=TRUE} means the larger the evaluation score the better. #' @param ... other parameters to pass to \code{params}. #' #' @details @@ -114,7 +118,8 @@ #' @export #' xgb.train <- function(params=list(), data, nrounds, watchlist = list(), - obj = NULL, feval = NULL, verbose = 1, ...) { + obj = NULL, feval = NULL, verbose = 1, + earlyStopRound = NULL, maximize = NULL, ...) { dtrain <- data if (typeof(params) != "list") { stop("xgb.train: first argument params must be list") @@ -133,6 +138,33 @@ xgb.train <- function(params=list(), data, nrounds, watchlist = list(), } params = append(params, list(...)) + # Early stopping + if (!is.null(feval) && is.null(maximize)) + stop('Please set maximize to note whether the model is maximizing the evaluation or not.') + if (length(watchlist) == 0 && !is.null(earlyStopRound)) + stop('For early stopping you need at least one set in watchlist.') + if (is.null(maximize) && is.null(params$eval_metric)) + stop('Please set maximize to note whether the model is maximizing the evaluation or not.') + if (is.null(maximize)) + { + if (params$eval_metric %in% c('rmse','logloss','error','merror','mlogloss')) { + maximize = FALSE + } else { + maximize = TRUE + } + } + + if (maximize) { + bestScore = 0 + } else { + bestScore = Inf + } + bestInd = 0 + earlyStopflag = FALSE + + if (length(watchlist)>1 && !is.null(earlyStopRound)) + warning('Only the first data set in watchlist is used for early stopping process.') + handle <- xgb.Booster(params, append(watchlist, dtrain)) bst <- xgb.handleToBooster(handle) for (i in 1:nrounds) { @@ -140,8 +172,30 @@ xgb.train <- function(params=list(), data, nrounds, watchlist = list(), if (length(watchlist) != 0) { msg <- xgb.iter.eval(bst$handle, watchlist, i - 1, feval) cat(paste(msg, "\n", sep="")) + if (!is.null(earlyStopRound)) + { + score = strsplit(msg,'\\s+')[[1]][1] + score = strsplit(score,':')[[1]][2] + score = as.numeric(score) + if ((maximize && score>bestScore) || (!maximize && scoreearlyStopRound) { + earlyStopflag = TRUE + } + } + } + } + if (earlyStopflag) { + cat('Stopping. Best iteration:',bestInd) + break } } bst <- xgb.Booster.check(bst) + if (!is.null(earlyStopRound)) { + bst$bestScore = bestScore + bst$bestInd = bestInd + } return(bst) } diff --git a/R-package/R/xgboost.R b/R-package/R/xgboost.R index ede53b116..367a149e7 100644 --- a/R-package/R/xgboost.R +++ b/R-package/R/xgboost.R @@ -30,6 +30,11 @@ #' performance and construction progress information #' @param missing Missing is only used when input is dense matrix, pick a float #' value that represents missing value. Sometimes a data use 0 or other extreme value to represents missing values. +#' @param earlyStopRound If \code{NULL}, the early stopping function is not triggered. +#' If set to an integer \code{k}, training with a validation set will stop if the performance +#' keeps getting worse consecutively for \code{k} rounds. +#' @param maximize If \code{feval} and \code{earlyStopRound} are set, then \code{maximize} must be set as well. +#' \code{maximize=TRUE} means the larger the evaluation score the better. #' @param ... other parameters to pass to \code{params}. #' #' @details @@ -51,7 +56,7 @@ #' @export #' xgboost <- function(data = NULL, label = NULL, missing = NULL, params = list(), nrounds, - verbose = 1, ...) { + verbose = 1, earlyStopRound = NULL, maximize = NULL, ...) { if (is.null(missing)) { dtrain <- xgb.get.DMatrix(data, label) } else { @@ -66,7 +71,8 @@ xgboost <- function(data = NULL, label = NULL, missing = NULL, params = list(), watchlist <- list() } - bst <- xgb.train(params, dtrain, nrounds, watchlist, verbose=verbose) + bst <- xgb.train(params, dtrain, nrounds, watchlist, verbose = verbose, + earlyStopRound = earlyStopRound) return(bst) } diff --git a/R-package/man/agaricus.test.Rd b/R-package/man/agaricus.test.Rd index c54e30ba3..556425379 100644 --- a/R-package/man/agaricus.test.Rd +++ b/R-package/man/agaricus.test.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/xgboost.R \docType{data} \name{agaricus.test} diff --git a/R-package/man/agaricus.train.Rd b/R-package/man/agaricus.train.Rd index 955257148..879b3d5df 100644 --- a/R-package/man/agaricus.train.Rd +++ b/R-package/man/agaricus.train.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/xgboost.R \docType{data} \name{agaricus.train} diff --git a/R-package/man/getinfo.Rd b/R-package/man/getinfo.Rd index 87c507566..618d0d44b 100644 --- a/R-package/man/getinfo.Rd +++ b/R-package/man/getinfo.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/getinfo.xgb.DMatrix.R \docType{methods} \name{getinfo} diff --git a/R-package/man/nrow-xgb.DMatrix-method.Rd b/R-package/man/nrow-xgb.DMatrix-method.Rd index f86709afd..953e620bf 100644 --- a/R-package/man/nrow-xgb.DMatrix-method.Rd +++ b/R-package/man/nrow-xgb.DMatrix-method.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/nrow.xgb.DMatrix.R \docType{methods} \name{nrow,xgb.DMatrix-method} diff --git a/R-package/man/predict-xgb.Booster-method.Rd b/R-package/man/predict-xgb.Booster-method.Rd index 3ce2e2025..06fdb2ca8 100644 --- a/R-package/man/predict-xgb.Booster-method.Rd +++ b/R-package/man/predict-xgb.Booster-method.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/predict.xgb.Booster.R \docType{methods} \name{predict,xgb.Booster-method} diff --git a/R-package/man/predict-xgb.Booster.handle-method.Rd b/R-package/man/predict-xgb.Booster.handle-method.Rd index 7eb237a94..cc9ba29f9 100644 --- a/R-package/man/predict-xgb.Booster.handle-method.Rd +++ b/R-package/man/predict-xgb.Booster.handle-method.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/predict.xgb.Booster.handle.R \docType{methods} \name{predict,xgb.Booster.handle-method} diff --git a/R-package/man/setinfo.Rd b/R-package/man/setinfo.Rd index edf5284bd..9512f1dfb 100644 --- a/R-package/man/setinfo.Rd +++ b/R-package/man/setinfo.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/setinfo.xgb.DMatrix.R \docType{methods} \name{setinfo} diff --git a/R-package/man/slice.Rd b/R-package/man/slice.Rd index 20a78a383..a7812e886 100644 --- a/R-package/man/slice.Rd +++ b/R-package/man/slice.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/slice.xgb.DMatrix.R \docType{methods} \name{slice} diff --git a/R-package/man/xgb.DMatrix.Rd b/R-package/man/xgb.DMatrix.Rd index 9d4d19d37..ea644a291 100644 --- a/R-package/man/xgb.DMatrix.Rd +++ b/R-package/man/xgb.DMatrix.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/xgb.DMatrix.R \name{xgb.DMatrix} \alias{xgb.DMatrix} diff --git a/R-package/man/xgb.DMatrix.save.Rd b/R-package/man/xgb.DMatrix.save.Rd index 3ba36f55a..6bbc277b3 100644 --- a/R-package/man/xgb.DMatrix.save.Rd +++ b/R-package/man/xgb.DMatrix.save.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/xgb.DMatrix.save.R \name{xgb.DMatrix.save} \alias{xgb.DMatrix.save} diff --git a/R-package/man/xgb.cv.Rd b/R-package/man/xgb.cv.Rd index 19ab788f9..feee4e18f 100644 --- a/R-package/man/xgb.cv.Rd +++ b/R-package/man/xgb.cv.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/xgb.cv.R \name{xgb.cv} \alias{xgb.cv} diff --git a/R-package/man/xgb.dump.Rd b/R-package/man/xgb.dump.Rd index eaf1ca521..124535211 100644 --- a/R-package/man/xgb.dump.Rd +++ b/R-package/man/xgb.dump.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/xgb.dump.R \name{xgb.dump} \alias{xgb.dump} diff --git a/R-package/man/xgb.importance.Rd b/R-package/man/xgb.importance.Rd index 11740e4ac..674a54622 100644 --- a/R-package/man/xgb.importance.Rd +++ b/R-package/man/xgb.importance.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/xgb.importance.R \name{xgb.importance} \alias{xgb.importance} diff --git a/R-package/man/xgb.load.Rd b/R-package/man/xgb.load.Rd index 1331ff249..4caef6239 100644 --- a/R-package/man/xgb.load.Rd +++ b/R-package/man/xgb.load.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/xgb.load.R \name{xgb.load} \alias{xgb.load} diff --git a/R-package/man/xgb.model.dt.tree.Rd b/R-package/man/xgb.model.dt.tree.Rd index c53ed057f..df308a954 100644 --- a/R-package/man/xgb.model.dt.tree.Rd +++ b/R-package/man/xgb.model.dt.tree.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/xgb.model.dt.tree.R \name{xgb.model.dt.tree} \alias{xgb.model.dt.tree} diff --git a/R-package/man/xgb.plot.importance.Rd b/R-package/man/xgb.plot.importance.Rd index 4147278b9..0797b89c2 100644 --- a/R-package/man/xgb.plot.importance.Rd +++ b/R-package/man/xgb.plot.importance.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/xgb.plot.importance.R \name{xgb.plot.importance} \alias{xgb.plot.importance} diff --git a/R-package/man/xgb.plot.tree.Rd b/R-package/man/xgb.plot.tree.Rd index 4501d87ce..476dbda11 100644 --- a/R-package/man/xgb.plot.tree.Rd +++ b/R-package/man/xgb.plot.tree.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/xgb.plot.tree.R \name{xgb.plot.tree} \alias{xgb.plot.tree} diff --git a/R-package/man/xgb.save.Rd b/R-package/man/xgb.save.Rd index eca097fac..6e6b23e54 100644 --- a/R-package/man/xgb.save.Rd +++ b/R-package/man/xgb.save.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/xgb.save.R \name{xgb.save} \alias{xgb.save} diff --git a/R-package/man/xgb.save.raw.Rd b/R-package/man/xgb.save.raw.Rd index 79c356c0f..94ae29416 100644 --- a/R-package/man/xgb.save.raw.Rd +++ b/R-package/man/xgb.save.raw.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/xgb.save.raw.R \name{xgb.save.raw} \alias{xgb.save.raw} diff --git a/R-package/man/xgb.train.Rd b/R-package/man/xgb.train.Rd index 1bd243d60..6a1aa874b 100644 --- a/R-package/man/xgb.train.Rd +++ b/R-package/man/xgb.train.Rd @@ -1,11 +1,12 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/xgb.train.R \name{xgb.train} \alias{xgb.train} \title{eXtreme Gradient Boosting Training} \usage{ xgb.train(params = list(), data, nrounds, watchlist = list(), obj = NULL, - feval = NULL, verbose = 1, ...) + feval = NULL, verbose = 1, earlyStopRound = NULL, maximize = NULL, + ...) } \arguments{ \item{params}{the list of parameters. @@ -49,7 +50,7 @@ xgb.train(params = list(), data, nrounds, watchlist = list(), obj = NULL, \item \code{binary:logistic} logistic regression for binary classification. Output probability. \item \code{binary:logitraw} logistic regression for binary classification, output score before logistic transformation. \item \code{num_class} set the number of classes. To use only with multiclass objectives. - \item \code{multi:softmax} set xgboost to do multiclass classification using the softmax objective. Class is a number and should be from 0 \code{tonum_class} + \item \code{multi:softmax} set xgboost to do multiclass classification using the softmax objective. Class is represented by a number and should be from 0 to \code{tonum_class}. \item \code{multi:softprob} same as softmax, but output a vector of ndata * nclass, which can be further reshaped to ndata, nclass matrix. The result contains predicted probabilities of each data point belonging to each class. \item \code{rank:pairwise} set xgboost to do ranking task by minimizing the pairwise loss. } @@ -75,7 +76,14 @@ gradient with given prediction and dtrain,} prediction and dtrain,} \item{verbose}{If 0, xgboost will stay silent. If 1, xgboost will print - information of performance. If 2, xgboost will print information of both} +information of performance. If 2, xgboost will print information of both} + +\item{earlyStopRound}{If \code{NULL}, the early stopping function is not triggered. +If set to an integer \code{k}, training with a validation set will stop if the performance +keeps getting worse consecutively for \code{k} rounds.} + +\item{maximize}{If \code{feval} and \code{earlyStopRound} are set, then \code{maximize} must be set as well. +\code{maximize=TRUE} means the larger the evaluation score the better.} \item{...}{other parameters to pass to \code{params}.} } @@ -98,7 +106,7 @@ Number of threads can also be manually specified via \code{nthread} parameter. \item \code{error} Binary classification error rate. It is calculated as \code{(wrong cases) / (all cases)}. For the predictions, the evaluation will regard the instances with prediction value larger than 0.5 as positive instances, and the others as negative instances. \item \code{merror} Multiclass classification error rate. It is calculated as \code{(wrong cases) / (all cases)}. \item \code{auc} Area under the curve. \url{http://en.wikipedia.org/wiki/Receiver_operating_characteristic#'Area_under_curve} for ranking evaluation. - \item \code{ndcg} Normalized Discounted Cumulative Gain. \url{http://en.wikipedia.org/wiki/NDCG} + \item \code{ndcg} Normalized Discounted Cumulative Gain (for ranking task). \url{http://en.wikipedia.org/wiki/NDCG} } Full list of parameters is available in the Wiki \url{https://github.com/dmlc/xgboost/wiki/Parameters}. diff --git a/R-package/man/xgboost.Rd b/R-package/man/xgboost.Rd index 79cff207a..bc2311a2b 100644 --- a/R-package/man/xgboost.Rd +++ b/R-package/man/xgboost.Rd @@ -1,11 +1,11 @@ -% Generated by roxygen2 (4.1.1): do not edit by hand +% Generated by roxygen2 (4.1.0): do not edit by hand % Please edit documentation in R/xgboost.R \name{xgboost} \alias{xgboost} \title{eXtreme Gradient Boosting (Tree) library} \usage{ xgboost(data = NULL, label = NULL, missing = NULL, params = list(), - nrounds, verbose = 1, ...) + nrounds, verbose = 1, earlyStopRound = NULL, maximize = NULL, ...) } \arguments{ \item{data}{takes \code{matrix}, \code{dgCMatrix}, local data file or @@ -41,6 +41,13 @@ Commonly used ones are: information of performance. If 2, xgboost will print information of both performance and construction progress information} +\item{earlyStopRound}{If \code{NULL}, the early stopping function is not triggered. +If set to an integer \code{k}, training with a validation set will stop if the performance +keeps getting worse consecutively for \code{k} rounds.} + +\item{maximize}{If \code{feval} and \code{earlyStopRound} are set, then \code{maximize} must be set as well. +\code{maximize=TRUE} means the larger the evaluation score the better.} + \item{...}{other parameters to pass to \code{params}.} } \description{ From 0f182b0b66d95306e1ab3b16c6e1aadcfdc5256b Mon Sep 17 00:00:00 2001 From: hetong007 Date: Tue, 5 May 2015 16:44:36 -0700 Subject: [PATCH 2/4] fix logic --- R-package/R/xgb.train.R | 4 +-- R-package/demo/00Index | 1 + R-package/demo/early_Stopping.R | 58 +++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 R-package/demo/early_Stopping.R diff --git a/R-package/R/xgb.train.R b/R-package/R/xgb.train.R index 6f6c1a900..636ad3cad 100644 --- a/R-package/R/xgb.train.R +++ b/R-package/R/xgb.train.R @@ -139,11 +139,11 @@ xgb.train <- function(params=list(), data, nrounds, watchlist = list(), params = append(params, list(...)) # Early stopping - if (!is.null(feval) && is.null(maximize)) + if (!is.null(feval) && is.null(maximize) && !is.null(earlyStopRound)) stop('Please set maximize to note whether the model is maximizing the evaluation or not.') if (length(watchlist) == 0 && !is.null(earlyStopRound)) stop('For early stopping you need at least one set in watchlist.') - if (is.null(maximize) && is.null(params$eval_metric)) + if (is.null(maximize) && is.null(params$eval_metric) && !is.null(earlyStopRound)) stop('Please set maximize to note whether the model is maximizing the evaluation or not.') if (is.null(maximize)) { diff --git a/R-package/demo/00Index b/R-package/demo/00Index index 969da0d91..f0b41ec2a 100644 --- a/R-package/demo/00Index +++ b/R-package/demo/00Index @@ -6,3 +6,4 @@ generalized_linear_model Generalized Linear Model cross_validation Cross validation create_sparse_matrix Create Sparse Matrix predict_leaf_indices Predicting the corresponding leaves +early_Stopping Early Stop in training diff --git a/R-package/demo/early_Stopping.R b/R-package/demo/early_Stopping.R new file mode 100644 index 000000000..3253c3828 --- /dev/null +++ b/R-package/demo/early_Stopping.R @@ -0,0 +1,58 @@ +require(xgboost) +# load in the agaricus dataset +data(agaricus.train, package='xgboost') +data(agaricus.test, package='xgboost') +dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label) +dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label) +# note: for customized objective function, we leave objective as default +# note: what we are getting is margin value in prediction +# you must know what you are doing +param <- list(max.depth=2,eta=1,nthread = 2, silent=1) +watchlist <- list(eval = dtest) +num_round <- 20 +# user define objective function, given prediction, return gradient and second order gradient +# this is loglikelihood loss +logregobj <- function(preds, dtrain) { + labels <- getinfo(dtrain, "label") + preds <- 1/(1 + exp(-preds)) + grad <- preds - labels + hess <- preds * (1 - preds) + return(list(grad = grad, hess = hess)) +} +# user defined evaluation function, return a pair metric_name, result +# NOTE: when you do customized loss function, the default prediction value is margin +# this may make buildin evalution metric not function properly +# for example, we are doing logistic loss, the prediction is score before logistic transformation +# the buildin evaluation error assumes input is after logistic transformation +# Take this in mind when you use the customization, and maybe you need write customized evaluation function +evalerror <- function(preds, dtrain) { + labels <- getinfo(dtrain, "label") + err <- as.numeric(sum(labels != (preds > 0)))/length(labels) + return(list(metric = "error", value = err)) +} +print ('start training with user customized objective') +# training with customized objective, we can also do step by step training +# simply look at xgboost.py's implementation of train +bst <- xgb.train(param, dtrain, num_round, watchlist, logregobj, evalerror, maximize = FALSE, + earlyStopRound = 3) +# +# there can be cases where you want additional information +# being considered besides the property of DMatrix you can get by getinfo +# you can set additional information as attributes if DMatrix +# set label attribute of dtrain to be label, we use label as an example, it can be anything +attr(dtrain, 'label') <- getinfo(dtrain, 'label') +# this is new customized objective, where you can access things you set +# same thing applies to customized evaluation function +logregobjattr <- function(preds, dtrain) { + # now you can access the attribute in customized function + labels <- attr(dtrain, 'label') + preds <- 1/(1 + exp(-preds)) + grad <- preds - labels + hess <- preds * (1 - preds) + return(list(grad = grad, hess = hess)) +} +print ('start training with user customized objective, with additional attributes in DMatrix') +# training with customized objective, we can also do step by step training +# simply look at xgboost.py's implementation of train +bst <- xgb.train(param, dtrain, num_round, watchlist, logregobjattr, evalerror, maximize = FALSE, + earlyStopRound = 3) \ No newline at end of file From 419e4dbda6ac69d7c905663dcecc3d18800ed31f Mon Sep 17 00:00:00 2001 From: hetong007 Date: Wed, 6 May 2015 15:14:29 -0700 Subject: [PATCH 3/4] add demo for early_stopping in R --- R-package/R/xgb.train.R | 56 ++++++++++++++++----------------- R-package/demo/early_Stopping.R | 23 +------------- 2 files changed, 29 insertions(+), 50 deletions(-) diff --git a/R-package/R/xgb.train.R b/R-package/R/xgb.train.R index 636ad3cad..01de306a0 100644 --- a/R-package/R/xgb.train.R +++ b/R-package/R/xgb.train.R @@ -139,31 +139,34 @@ xgb.train <- function(params=list(), data, nrounds, watchlist = list(), params = append(params, list(...)) # Early stopping - if (!is.null(feval) && is.null(maximize) && !is.null(earlyStopRound)) - stop('Please set maximize to note whether the model is maximizing the evaluation or not.') - if (length(watchlist) == 0 && !is.null(earlyStopRound)) - stop('For early stopping you need at least one set in watchlist.') - if (is.null(maximize) && is.null(params$eval_metric) && !is.null(earlyStopRound)) - stop('Please set maximize to note whether the model is maximizing the evaluation or not.') - if (is.null(maximize)) - { - if (params$eval_metric %in% c('rmse','logloss','error','merror','mlogloss')) { - maximize = FALSE - } else { - maximize = TRUE + if (!is.null(earlyStopRound)){ + if (!is.null(feval) && is.null(maximize)) + stop('Please set maximize to note whether the model is maximizing the evaluation or not.') + if (length(watchlist) == 0) + stop('For early stopping you need at least one set in watchlist.') + if (is.null(maximize) && is.null(params$eval_metric)) + stop('Please set maximize to note whether the model is maximizing the evaluation or not.') + if (is.null(maximize)) + { + if (params$eval_metric %in% c('rmse','logloss','error','merror','mlogloss')) { + maximize = FALSE + } else { + maximize = TRUE + } } + + if (maximize) { + bestScore = 0 + } else { + bestScore = Inf + } + bestInd = 0 + earlyStopflag = FALSE + + if (length(watchlist)>1) + warning('Only the first data set in watchlist is used for early stopping process.') } - if (maximize) { - bestScore = 0 - } else { - bestScore = Inf - } - bestInd = 0 - earlyStopflag = FALSE - - if (length(watchlist)>1 && !is.null(earlyStopRound)) - warning('Only the first data set in watchlist is used for early stopping process.') handle <- xgb.Booster(params, append(watchlist, dtrain)) bst <- xgb.handleToBooster(handle) @@ -174,8 +177,7 @@ xgb.train <- function(params=list(), data, nrounds, watchlist = list(), cat(paste(msg, "\n", sep="")) if (!is.null(earlyStopRound)) { - score = strsplit(msg,'\\s+')[[1]][1] - score = strsplit(score,':')[[1]][2] + score = strsplit(msg,':|\\s+')[[1]][3] score = as.numeric(score) if ((maximize && score>bestScore) || (!maximize && scoreearlyStopRound) { earlyStopflag = TRUE + cat('Stopping. Best iteration:',bestInd) + break } } } } - if (earlyStopflag) { - cat('Stopping. Best iteration:',bestInd) - break - } } bst <- xgb.Booster.check(bst) if (!is.null(earlyStopRound)) { diff --git a/R-package/demo/early_Stopping.R b/R-package/demo/early_Stopping.R index 3253c3828..4cab385ca 100644 --- a/R-package/demo/early_Stopping.R +++ b/R-package/demo/early_Stopping.R @@ -30,29 +30,8 @@ evalerror <- function(preds, dtrain) { err <- as.numeric(sum(labels != (preds > 0)))/length(labels) return(list(metric = "error", value = err)) } -print ('start training with user customized objective') +print ('start training with early Stopping setting') # training with customized objective, we can also do step by step training # simply look at xgboost.py's implementation of train bst <- xgb.train(param, dtrain, num_round, watchlist, logregobj, evalerror, maximize = FALSE, earlyStopRound = 3) -# -# there can be cases where you want additional information -# being considered besides the property of DMatrix you can get by getinfo -# you can set additional information as attributes if DMatrix -# set label attribute of dtrain to be label, we use label as an example, it can be anything -attr(dtrain, 'label') <- getinfo(dtrain, 'label') -# this is new customized objective, where you can access things you set -# same thing applies to customized evaluation function -logregobjattr <- function(preds, dtrain) { - # now you can access the attribute in customized function - labels <- attr(dtrain, 'label') - preds <- 1/(1 + exp(-preds)) - grad <- preds - labels - hess <- preds * (1 - preds) - return(list(grad = grad, hess = hess)) -} -print ('start training with user customized objective, with additional attributes in DMatrix') -# training with customized objective, we can also do step by step training -# simply look at xgboost.py's implementation of train -bst <- xgb.train(param, dtrain, num_round, watchlist, logregobjattr, evalerror, maximize = FALSE, - earlyStopRound = 3) \ No newline at end of file From 993d7b9da3e723eac0b1d34bb8e8d89db2f108fd Mon Sep 17 00:00:00 2001 From: hetong007 Date: Wed, 6 May 2015 15:23:37 -0700 Subject: [PATCH 4/4] update roxygen2 --- R-package/NAMESPACE | 2 +- R-package/man/agaricus.test.Rd | 2 +- R-package/man/agaricus.train.Rd | 2 +- R-package/man/getinfo.Rd | 2 +- R-package/man/nrow-xgb.DMatrix-method.Rd | 2 +- R-package/man/predict-xgb.Booster-method.Rd | 2 +- R-package/man/predict-xgb.Booster.handle-method.Rd | 2 +- R-package/man/setinfo.Rd | 2 +- R-package/man/slice.Rd | 2 +- R-package/man/xgb.DMatrix.Rd | 2 +- R-package/man/xgb.DMatrix.save.Rd | 2 +- R-package/man/xgb.cv.Rd | 2 +- R-package/man/xgb.dump.Rd | 2 +- R-package/man/xgb.importance.Rd | 2 +- R-package/man/xgb.load.Rd | 2 +- R-package/man/xgb.model.dt.tree.Rd | 2 +- R-package/man/xgb.plot.importance.Rd | 2 +- R-package/man/xgb.plot.tree.Rd | 2 +- R-package/man/xgb.save.Rd | 2 +- R-package/man/xgb.save.raw.Rd | 2 +- R-package/man/xgb.train.Rd | 2 +- R-package/man/xgboost.Rd | 2 +- 22 files changed, 22 insertions(+), 22 deletions(-) diff --git a/R-package/NAMESPACE b/R-package/NAMESPACE index d7f9e455c..a4f07799a 100644 --- a/R-package/NAMESPACE +++ b/R-package/NAMESPACE @@ -1,4 +1,4 @@ -# Generated by roxygen2 (4.1.0): do not edit by hand +# Generated by roxygen2 (4.1.1): do not edit by hand export(getinfo) export(setinfo) diff --git a/R-package/man/agaricus.test.Rd b/R-package/man/agaricus.test.Rd index 556425379..c54e30ba3 100644 --- a/R-package/man/agaricus.test.Rd +++ b/R-package/man/agaricus.test.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/xgboost.R \docType{data} \name{agaricus.test} diff --git a/R-package/man/agaricus.train.Rd b/R-package/man/agaricus.train.Rd index 879b3d5df..955257148 100644 --- a/R-package/man/agaricus.train.Rd +++ b/R-package/man/agaricus.train.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/xgboost.R \docType{data} \name{agaricus.train} diff --git a/R-package/man/getinfo.Rd b/R-package/man/getinfo.Rd index 618d0d44b..87c507566 100644 --- a/R-package/man/getinfo.Rd +++ b/R-package/man/getinfo.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/getinfo.xgb.DMatrix.R \docType{methods} \name{getinfo} diff --git a/R-package/man/nrow-xgb.DMatrix-method.Rd b/R-package/man/nrow-xgb.DMatrix-method.Rd index 953e620bf..f86709afd 100644 --- a/R-package/man/nrow-xgb.DMatrix-method.Rd +++ b/R-package/man/nrow-xgb.DMatrix-method.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/nrow.xgb.DMatrix.R \docType{methods} \name{nrow,xgb.DMatrix-method} diff --git a/R-package/man/predict-xgb.Booster-method.Rd b/R-package/man/predict-xgb.Booster-method.Rd index 06fdb2ca8..3ce2e2025 100644 --- a/R-package/man/predict-xgb.Booster-method.Rd +++ b/R-package/man/predict-xgb.Booster-method.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/predict.xgb.Booster.R \docType{methods} \name{predict,xgb.Booster-method} diff --git a/R-package/man/predict-xgb.Booster.handle-method.Rd b/R-package/man/predict-xgb.Booster.handle-method.Rd index cc9ba29f9..7eb237a94 100644 --- a/R-package/man/predict-xgb.Booster.handle-method.Rd +++ b/R-package/man/predict-xgb.Booster.handle-method.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/predict.xgb.Booster.handle.R \docType{methods} \name{predict,xgb.Booster.handle-method} diff --git a/R-package/man/setinfo.Rd b/R-package/man/setinfo.Rd index 9512f1dfb..edf5284bd 100644 --- a/R-package/man/setinfo.Rd +++ b/R-package/man/setinfo.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/setinfo.xgb.DMatrix.R \docType{methods} \name{setinfo} diff --git a/R-package/man/slice.Rd b/R-package/man/slice.Rd index a7812e886..20a78a383 100644 --- a/R-package/man/slice.Rd +++ b/R-package/man/slice.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/slice.xgb.DMatrix.R \docType{methods} \name{slice} diff --git a/R-package/man/xgb.DMatrix.Rd b/R-package/man/xgb.DMatrix.Rd index ea644a291..9d4d19d37 100644 --- a/R-package/man/xgb.DMatrix.Rd +++ b/R-package/man/xgb.DMatrix.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/xgb.DMatrix.R \name{xgb.DMatrix} \alias{xgb.DMatrix} diff --git a/R-package/man/xgb.DMatrix.save.Rd b/R-package/man/xgb.DMatrix.save.Rd index 6bbc277b3..3ba36f55a 100644 --- a/R-package/man/xgb.DMatrix.save.Rd +++ b/R-package/man/xgb.DMatrix.save.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/xgb.DMatrix.save.R \name{xgb.DMatrix.save} \alias{xgb.DMatrix.save} diff --git a/R-package/man/xgb.cv.Rd b/R-package/man/xgb.cv.Rd index feee4e18f..19ab788f9 100644 --- a/R-package/man/xgb.cv.Rd +++ b/R-package/man/xgb.cv.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/xgb.cv.R \name{xgb.cv} \alias{xgb.cv} diff --git a/R-package/man/xgb.dump.Rd b/R-package/man/xgb.dump.Rd index 124535211..eaf1ca521 100644 --- a/R-package/man/xgb.dump.Rd +++ b/R-package/man/xgb.dump.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/xgb.dump.R \name{xgb.dump} \alias{xgb.dump} diff --git a/R-package/man/xgb.importance.Rd b/R-package/man/xgb.importance.Rd index 674a54622..11740e4ac 100644 --- a/R-package/man/xgb.importance.Rd +++ b/R-package/man/xgb.importance.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/xgb.importance.R \name{xgb.importance} \alias{xgb.importance} diff --git a/R-package/man/xgb.load.Rd b/R-package/man/xgb.load.Rd index 4caef6239..1331ff249 100644 --- a/R-package/man/xgb.load.Rd +++ b/R-package/man/xgb.load.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/xgb.load.R \name{xgb.load} \alias{xgb.load} diff --git a/R-package/man/xgb.model.dt.tree.Rd b/R-package/man/xgb.model.dt.tree.Rd index df308a954..c53ed057f 100644 --- a/R-package/man/xgb.model.dt.tree.Rd +++ b/R-package/man/xgb.model.dt.tree.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/xgb.model.dt.tree.R \name{xgb.model.dt.tree} \alias{xgb.model.dt.tree} diff --git a/R-package/man/xgb.plot.importance.Rd b/R-package/man/xgb.plot.importance.Rd index 0797b89c2..4147278b9 100644 --- a/R-package/man/xgb.plot.importance.Rd +++ b/R-package/man/xgb.plot.importance.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/xgb.plot.importance.R \name{xgb.plot.importance} \alias{xgb.plot.importance} diff --git a/R-package/man/xgb.plot.tree.Rd b/R-package/man/xgb.plot.tree.Rd index 476dbda11..4501d87ce 100644 --- a/R-package/man/xgb.plot.tree.Rd +++ b/R-package/man/xgb.plot.tree.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/xgb.plot.tree.R \name{xgb.plot.tree} \alias{xgb.plot.tree} diff --git a/R-package/man/xgb.save.Rd b/R-package/man/xgb.save.Rd index 6e6b23e54..eca097fac 100644 --- a/R-package/man/xgb.save.Rd +++ b/R-package/man/xgb.save.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/xgb.save.R \name{xgb.save} \alias{xgb.save} diff --git a/R-package/man/xgb.save.raw.Rd b/R-package/man/xgb.save.raw.Rd index 94ae29416..79c356c0f 100644 --- a/R-package/man/xgb.save.raw.Rd +++ b/R-package/man/xgb.save.raw.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/xgb.save.raw.R \name{xgb.save.raw} \alias{xgb.save.raw} diff --git a/R-package/man/xgb.train.Rd b/R-package/man/xgb.train.Rd index 6a1aa874b..4444b95ae 100644 --- a/R-package/man/xgb.train.Rd +++ b/R-package/man/xgb.train.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/xgb.train.R \name{xgb.train} \alias{xgb.train} diff --git a/R-package/man/xgboost.Rd b/R-package/man/xgboost.Rd index bc2311a2b..9509dbd39 100644 --- a/R-package/man/xgboost.Rd +++ b/R-package/man/xgboost.Rd @@ -1,4 +1,4 @@ -% Generated by roxygen2 (4.1.0): do not edit by hand +% Generated by roxygen2 (4.1.1): do not edit by hand % Please edit documentation in R/xgboost.R \name{xgboost} \alias{xgboost}