From 755be4b84667d4b6a534f126280c94033ff31048 Mon Sep 17 00:00:00 2001 From: El Potaeto Date: Mon, 29 Dec 2014 10:31:17 +0100 Subject: [PATCH 1/8] Add variable type checks --- R-package/R/xgb.importance.R | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/R-package/R/xgb.importance.R b/R-package/R/xgb.importance.R index b971c58ff..3b99fbb00 100644 --- a/R-package/R/xgb.importance.R +++ b/R-package/R/xgb.importance.R @@ -29,6 +29,12 @@ #' #' @export xgb.importance <- function(feature_names, filename_dump){ + if (class(feature_names) != "character") { + stop("feature_names: Has to be a vector of character. See help to see where to get it.") + } + if (class(filename_dump) != "character" & file.exists(filename_dump)) { + stop("filename_dump: Has to be a path to the model dump file.") + } text <- readLines(filename_dump) if(text[2] == "bias:"){ result <- linearDump(feature_names, text) From 9b6a14a99d685f0c313197f1b65c945d22627d7d Mon Sep 17 00:00:00 2001 From: El Potaeto Date: Mon, 29 Dec 2014 23:56:31 +0100 Subject: [PATCH 2/8] regeneration of documentation --- R-package/man/xgb.dump.Rd | 17 +++++++++++------ R-package/man/xgb.importance.Rd | 19 ++++++++++++------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/R-package/man/xgb.dump.Rd b/R-package/man/xgb.dump.Rd index 9be2696b9..bcecc6abd 100644 --- a/R-package/man/xgb.dump.Rd +++ b/R-package/man/xgb.dump.Rd @@ -4,7 +4,7 @@ \alias{xgb.dump} \title{Save xgboost model to text file} \usage{ -xgb.dump(model, fname, fmap = "") +xgb.dump(model, fname, fmap = "", with.stats = FALSE) } \arguments{ \item{model}{the model object.} @@ -12,11 +12,16 @@ xgb.dump(model, fname, fmap = "") \item{fname}{the name of the binary file.} \item{fmap}{feature map file representing the type of feature. - Detailed description could be found at - \url{https://github.com/tqchen/xgboost/wiki/Binary-Classification#dump-model}. - See demo/ for walkthrough example in R, and - \url{https://github.com/tqchen/xgboost/blob/master/demo/data/featmap.txt} - for example Format.} +Detailed description could be found at +\url{https://github.com/tqchen/xgboost/wiki/Binary-Classification#dump-model}. +See demo/ for walkthrough example in R, and +\url{https://github.com/tqchen/xgboost/blob/master/demo/data/featmap.txt} +for example Format.} + +\item{with.stats}{whether dump statistics of splits + When this option is on, the model dump comes with two additional statistics: + gain is the approximate loss function gain we get in each split; + cover is the sum of second order gradient in each node.} } \description{ Save a xgboost model to text file. Could be parsed later. diff --git a/R-package/man/xgb.importance.Rd b/R-package/man/xgb.importance.Rd index 9609fe82f..7d02315f9 100644 --- a/R-package/man/xgb.importance.Rd +++ b/R-package/man/xgb.importance.Rd @@ -4,30 +4,35 @@ \alias{xgb.importance} \title{Show importance of features in a model} \usage{ -xgb.importance(feature_names, filename_dump) +xgb.importance(feature_names = NULL, filename_dump = NULL) } \arguments{ -\item{feature_names}{names of each feature as a character vector. Can be extracted from a sparse matrix.} +\item{feature_names}{names of each feature as a character vector. Can be extracted from a sparse matrix (see example). If model dump already contains feature names, this argument should be \code{NULL}.} -\item{filename_dump}{the name of the text file.} +\item{filename_dump}{the path to the text file storing the model.} } \description{ -Read a xgboost model in text file format. Return a data.table of the features with their weight. +Read a xgboost model in text file format. +Can be tree or linear model (text dump of linear model are only supported in dev version of Xgboost for now). +} +\details{ +Return a data.table of the features with their weight. +#' } \examples{ data(agaricus.train, package='xgboost') data(agaricus.test, package='xgboost') -#Both dataset are list with two items, a sparse matrix and labels (outcome column which will be learned). +#Both dataset are list with two items, a sparse matrix and labels (labels = outcome column which will be learned). #Each column of the sparse Matrix is a feature in one hot encoding format. train <- agaricus.train test <- agaricus.test bst <- xgboost(data = train$data, label = train$label, max.depth = 2, eta = 1, nround = 2,objective = "binary:logistic") -xgb.dump(bst, 'xgb.model.dump') +xgb.dump(bst, 'xgb.model.dump', with.stats = T) -#agaricus.test$data@Dimnames[[2]] represents the column name of the sparse matrix. +#agaricus.test$data@Dimnames[[2]] represents the column names of the sparse matrix. xgb.importance(agaricus.test$data@Dimnames[[2]], 'xgb.model.dump') } From dba1ce7050bdb3aee6c92edddcb976be997d1c1d Mon Sep 17 00:00:00 2001 From: El Potaeto Date: Mon, 29 Dec 2014 23:57:01 +0100 Subject: [PATCH 3/8] new dependency over stringr --- R-package/DESCRIPTION | 3 ++- R-package/NAMESPACE | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/R-package/DESCRIPTION b/R-package/DESCRIPTION index ce0ab9081..6f73766fa 100644 --- a/R-package/DESCRIPTION +++ b/R-package/DESCRIPTION @@ -23,4 +23,5 @@ Imports: Matrix (>= 1.1-0), methods, data.table (>= 1.9), - magrittr (>= 1.5) \ No newline at end of file + magrittr (>= 1.5), + stringr \ No newline at end of file diff --git a/R-package/NAMESPACE b/R-package/NAMESPACE index e4784bbf5..1714d2044 100644 --- a/R-package/NAMESPACE +++ b/R-package/NAMESPACE @@ -19,3 +19,4 @@ importClassesFrom(Matrix,dgeMatrix) importFrom(data.table,":=") importFrom(data.table,data.table) importFrom(magrittr,"%>%") +importFrom(stringr,str_extract) From 263f7fa69d362201cdf4eafd6ecdb4a7a8669ad0 Mon Sep 17 00:00:00 2001 From: El Potaeto Date: Mon, 29 Dec 2014 23:57:41 +0100 Subject: [PATCH 4/8] Take gain into account to discover most important variables --- R-package/R/xgb.importance.R | 39 ++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/R-package/R/xgb.importance.R b/R-package/R/xgb.importance.R index 3b99fbb00..51221b71b 100644 --- a/R-package/R/xgb.importance.R +++ b/R-package/R/xgb.importance.R @@ -8,34 +8,35 @@ #' @importFrom data.table data.table #' @importFrom magrittr %>% #' @importFrom data.table := -#' @param feature_names names of each feature as a character vector. Can be extracted from a sparse matrix. -#' @param filename_dump the name of the text file. +#' @importFrom stringr str_extract +#' @param feature_names names of each feature as a character vector. Can be extracted from a sparse matrix (see example). If model dump already contains feature names, this argument should be \code{NULL}. +#' @param filename_dump the path to the text file storing the model. #' #' @examples #' data(agaricus.train, package='xgboost') #' data(agaricus.test, package='xgboost') #' -#' #Both dataset are list with two items, a sparse matrix and labels (outcome column which will be learned). +#' #Both dataset are list with two items, a sparse matrix and labels (labels = outcome column which will be learned). #' #Each column of the sparse Matrix is a feature in one hot encoding format. #' train <- agaricus.train #' test <- agaricus.test #' #' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, #' eta = 1, nround = 2,objective = "binary:logistic") -#' xgb.dump(bst, 'xgb.model.dump') +#' xgb.dump(bst, 'xgb.model.dump', with.stats = T) #' -#' #agaricus.test$data@@Dimnames[[2]] represents the column name of the sparse matrix. +#' #agaricus.test$data@@Dimnames[[2]] represents the column names of the sparse matrix. #' xgb.importance(agaricus.test$data@@Dimnames[[2]], 'xgb.model.dump') #' #' @export -xgb.importance <- function(feature_names, filename_dump){ - if (class(feature_names) != "character") { - stop("feature_names: Has to be a vector of character. See help to see where to get it.") +xgb.importance <- function(feature_names = NULL, filename_dump = NULL){ + if (!class(feature_names) %in% c("character", "NULL")) { + stop("feature_names: Has to be a vector of character or NULL if model dump already contain feature name. See help to see where to get it.") } - if (class(filename_dump) != "character" & file.exists(filename_dump)) { + if (class(filename_dump) != "character" & file.exists(filename_dump)) { stop("filename_dump: Has to be a path to the model dump file.") } - text <- readLines(filename_dump) + text <- readLines(filename_dump) if(text[2] == "bias:"){ result <- linearDump(feature_names, text) } else { @@ -44,17 +45,21 @@ xgb.importance <- function(feature_names, filename_dump){ result } -treeDump <- function(feature_names, text){ - result <- c() +treeDump <- function(feature_names, text){ + featureVec <- c() + gainVec <- c() for(line in text){ - p <- regexec("\\[f.*\\]", line) %>% regmatches(line, .) - if (length(p[[1]]) > 0) { - splits <- sub("\\[f", "", p[[1]]) %>% sub("\\]", "", .) %>% strsplit("<") %>% .[[1]] %>% as.numeric - result <- c(result, feature_names[splits[1]+ 1]) + p <- str_extract(line, "\\[f.*<") + if (!is.na(p)) { + featureVec <- substr(p, 3, nchar(p)-1) %>% c(featureVec) + gainVec <- str_extract(line, "gain.*,") %>% substr(x = ., 6, nchar(.)-1) %>% as.numeric %>% c(gainVec) } } + if(!is.null(feature_names)) { + featureVec %<>% as.numeric %>% {c =.+1; feature_names[c]} #+1 because in R indexing start with 1 instead of 0. + } #1. Reduce, 2. %, 3. reorder - bigger top, 4. remove temp col - data.table(Feature = result)[,.N, by = Feature][, Weight:= N /sum(N)][order(-rank(Weight))][,-2,with=F] + data.table(Feature = featureVec, Weight = gainVec)[,sum(Weight), by = Feature][, Weight:= V1 /sum(V1)][order(-rank(Weight))][,-2,with=F] } linearDump <- function(feature_names, text){ From 78813d8f78d0f4b17b23e670f4cae8e3625685f7 Mon Sep 17 00:00:00 2001 From: El Potaeto Date: Tue, 30 Dec 2014 00:12:01 +0100 Subject: [PATCH 5/8] wording --- R-package/R/xgb.importance.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R-package/R/xgb.importance.R b/R-package/R/xgb.importance.R index 51221b71b..0aa13afff 100644 --- a/R-package/R/xgb.importance.R +++ b/R-package/R/xgb.importance.R @@ -31,7 +31,7 @@ #' @export xgb.importance <- function(feature_names = NULL, filename_dump = NULL){ if (!class(feature_names) %in% c("character", "NULL")) { - stop("feature_names: Has to be a vector of character or NULL if model dump already contain feature name. See help to see where to get it.") + stop("feature_names: Has to be a vector of character or NULL if the model dump already contains feature name. Look at this function documentation to see where to get feature names.") } if (class(filename_dump) != "character" & file.exists(filename_dump)) { stop("filename_dump: Has to be a path to the model dump file.") From 3694772bde99af6b7df88e849ee5e09acea83c12 Mon Sep 17 00:00:00 2001 From: El Potaeto Date: Tue, 30 Dec 2014 12:16:13 +0100 Subject: [PATCH 6/8] Add a new Weight and Gain column. Update documentation. --- R-package/R/xgb.importance.R | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/R-package/R/xgb.importance.R b/R-package/R/xgb.importance.R index 0aa13afff..8d37a9c15 100644 --- a/R-package/R/xgb.importance.R +++ b/R-package/R/xgb.importance.R @@ -3,7 +3,7 @@ #' Read a xgboost model in text file format. #' Can be tree or linear model (text dump of linear model are only supported in dev version of Xgboost for now). #' -#' Return a data.table of the features with their weight. +#' Return a data.table of the features used in the model with their average gain (and their weight for boosted tree model)in the model. #' #' #' @importFrom data.table data.table #' @importFrom magrittr %>% @@ -12,6 +12,19 @@ #' @param feature_names names of each feature as a character vector. Can be extracted from a sparse matrix (see example). If model dump already contains feature names, this argument should be \code{NULL}. #' @param filename_dump the path to the text file storing the model. #' +#' @details +#' This is the function to understand the model trained (and through your model, your data). +#' Results are returned for both linear and tree models. +#' +#' \code{data.table} is returned by the function. +#' There are 3 columns : +#' \itemize{ +#' \item \code{Features} name of the features as provided in \code{feature_names} or already present in the model dump. +#' \item \code{Gain} contribution of each feature to the model. For boosted tree model, each gain of each feature of each tree is taken into account, then average per feature to give a vision of the entire model. Highest percentage means most important feature regarding the \code{label} used for the training. +#' \item \code{Weight} percentage representing the relative number of times a feature have been taken into trees. \code{Gain} should be prefered to search the most important feature. For boosted linear model, this column has no meaning. +#' } +#' +#' #' @examples #' data(agaricus.train, package='xgboost') #' data(agaricus.test, package='xgboost') @@ -45,7 +58,7 @@ xgb.importance <- function(feature_names = NULL, filename_dump = NULL){ result } -treeDump <- function(feature_names, text){ +treeDump <- function(feature_names, text){ featureVec <- c() gainVec <- c() for(line in text){ @@ -59,7 +72,7 @@ treeDump <- function(feature_names, text){ featureVec %<>% as.numeric %>% {c =.+1; feature_names[c]} #+1 because in R indexing start with 1 instead of 0. } #1. Reduce, 2. %, 3. reorder - bigger top, 4. remove temp col - data.table(Feature = featureVec, Weight = gainVec)[,sum(Weight), by = Feature][, Weight:= V1 /sum(V1)][order(-rank(Weight))][,-2,with=F] + data.table(Feature = featureVec, Weight = gainVec)[,list(sum(Weight), .N), by = Feature][, Gain:= V1 /sum(V1)][,Weight:= N / sum(N)][order(-rank(Gain))][,-c(2,3), with = F] } linearDump <- function(feature_names, text){ From c754fd4ad0559d6bee25e4d4ee995e613afcc754 Mon Sep 17 00:00:00 2001 From: El Potaeto Date: Tue, 30 Dec 2014 12:32:21 +0100 Subject: [PATCH 7/8] documentation wording --- R-package/man/xgb.importance.Rd | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/R-package/man/xgb.importance.Rd b/R-package/man/xgb.importance.Rd index 7d02315f9..883819993 100644 --- a/R-package/man/xgb.importance.Rd +++ b/R-package/man/xgb.importance.Rd @@ -9,15 +9,25 @@ xgb.importance(feature_names = NULL, filename_dump = NULL) \arguments{ \item{feature_names}{names of each feature as a character vector. Can be extracted from a sparse matrix (see example). If model dump already contains feature names, this argument should be \code{NULL}.} -\item{filename_dump}{the path to the text file storing the model.} +\item{filename_dump}{the path to the text file storing the model. Model dump must include the gain per feature and per tree (\code{with.stats = T} in function \code{xgb.dump}).} } \description{ -Read a xgboost model in text file format. -Can be tree or linear model (text dump of linear model are only supported in dev version of Xgboost for now). +Read a xgboost model text dump. +Can be tree or linear model (text dump of linear model are only supported in dev version of \code{Xgboost} for now). +Return a data.table of the features used in the model with their average gain (and their weight for boosted tree model) in the model. } \details{ -Return a data.table of the features with their weight. -#' +This is the function to understand the model trained (and through your model, your data). + +Results are returned for both linear and tree models. + +\code{data.table} is returned by the function. +There are 3 columns : +\itemize{ + \item \code{Features} name of the features as provided in \code{feature_names} or already present in the model dump. + \item \code{Gain} contribution of each feature to the model. For boosted tree model, each gain of each feature of each tree is taken into account, then average per feature to give a vision of the entire model. Highest percentage means most important feature regarding the \code{label} used for the training. + \item \code{Weight} percentage representing the relative number of times a feature have been taken into trees. \code{Gain} should be prefered to search the most important feature. For boosted linear model, this column has no meaning. +} } \examples{ data(agaricus.train, package='xgboost') From c6f76fab561b7ada7a545b698f13b0bfd089d6c1 Mon Sep 17 00:00:00 2001 From: El Potaeto Date: Tue, 30 Dec 2014 12:32:58 +0100 Subject: [PATCH 8/8] add new Gain and Weight columns. documentation updated. --- R-package/R/xgb.importance.R | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/R-package/R/xgb.importance.R b/R-package/R/xgb.importance.R index 8d37a9c15..b2e60bed7 100644 --- a/R-package/R/xgb.importance.R +++ b/R-package/R/xgb.importance.R @@ -1,19 +1,19 @@ #' Show importance of features in a model #' -#' Read a xgboost model in text file format. -#' Can be tree or linear model (text dump of linear model are only supported in dev version of Xgboost for now). +#' Read a xgboost model text dump. +#' Can be tree or linear model (text dump of linear model are only supported in dev version of \code{Xgboost} for now). +#' Return a data.table of the features used in the model with their average gain (and their weight for boosted tree model) in the model. #' -#' Return a data.table of the features used in the model with their average gain (and their weight for boosted tree model)in the model. -#' #' #' @importFrom data.table data.table #' @importFrom magrittr %>% #' @importFrom data.table := #' @importFrom stringr str_extract #' @param feature_names names of each feature as a character vector. Can be extracted from a sparse matrix (see example). If model dump already contains feature names, this argument should be \code{NULL}. -#' @param filename_dump the path to the text file storing the model. +#' @param filename_dump the path to the text file storing the model. Model dump must include the gain per feature and per tree (\code{with.stats = T} in function \code{xgb.dump}). #' #' @details #' This is the function to understand the model trained (and through your model, your data). +#' #' Results are returned for both linear and tree models. #' #' \code{data.table} is returned by the function. @@ -72,7 +72,7 @@ treeDump <- function(feature_names, text){ featureVec %<>% as.numeric %>% {c =.+1; feature_names[c]} #+1 because in R indexing start with 1 instead of 0. } #1. Reduce, 2. %, 3. reorder - bigger top, 4. remove temp col - data.table(Feature = featureVec, Weight = gainVec)[,list(sum(Weight), .N), by = Feature][, Gain:= V1 /sum(V1)][,Weight:= N / sum(N)][order(-rank(Gain))][,-c(2,3), with = F] + data.table(Feature = featureVec, Weight = gainVec)[,list(sum(Weight), .N), by = Feature][, Gain:= V1/sum(V1)][,Weight:= N/sum(N)][order(-rank(Gain))][,-c(2,3), with = F] } linearDump <- function(feature_names, text){