Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #155 from pommedeterresautee/master

Browse Source 
fix mermaid + change in description + new plot importance feature function + fix bug in CV function + add 1 Vignette
This commit is contained in:
Tong He 2015-02-01 14:12:43 -08:00
commit 6e91846c55
27 changed files with 597 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
R-package/DESCRIPTION
View File

@ -3,7 +3,7 @@ Type: Package
Title: eXtreme Gradient Boosting Title: eXtreme Gradient Boosting
Version: 0.3-3 Version: 0.3-3
Date: 2014-12-28 Date: 2014-12-28
Author: Tianqi Chen <tianqi.tchen@gmail.com>, Tong He <hetong007@gmail.com> Author: Tianqi Chen <tianqi.tchen@gmail.com>, Tong He <hetong007@gmail.com>, Michaël Benesty <michael@benesty.fr>
Maintainer: Tong He <hetong007@gmail.com> Maintainer: Tong He <hetong007@gmail.com>
Description: This package is a R wrapper of xgboost, which is short for eXtreme Description: This package is a R wrapper of xgboost, which is short for eXtreme
Gradient Boosting. It is an efficient and scalable implementation of Gradient Boosting. It is an efficient and scalable implementation of
@ -26,5 +26,7 @@ Imports:
methods, methods,
data.table (>= 1.9.4), data.table (>= 1.9.4),
magrittr (>= 1.5), magrittr (>= 1.5),
stringr, stringr (>= 0.6.2),
DiagrammeR DiagrammeR (>= 0.4),
ggplot2 (>= 1.0.0),
Ckmeans.1d.dp (>= 3.02)

16
R-package/NAMESPACE
View File

@ -1,4 +1,4 @@
# Generated by roxygen2 (4.0.1): do not edit by hand # Generated by roxygen2 (4.1.0): do not edit by hand
export(getinfo) export(getinfo)
export(setinfo) export(setinfo)
@ -10,6 +10,7 @@ export(xgb.dump)
export(xgb.importance) export(xgb.importance)
export(xgb.load) export(xgb.load)
export(xgb.model.dt.tree) export(xgb.model.dt.tree)
export(xgb.plot.importance)
export(xgb.plot.tree) export(xgb.plot.tree)
export(xgb.save) export(xgb.save)
export(xgb.train) export(xgb.train)
@ -18,7 +19,8 @@ exportMethods(predict)
import(methods) import(methods)
importClassesFrom(Matrix,dgCMatrix) importClassesFrom(Matrix,dgCMatrix)
importClassesFrom(Matrix,dgeMatrix) importClassesFrom(Matrix,dgeMatrix)
importFrom(DiagrammeR,DiagrammeR) importFrom(Ckmeans.1d.dp,Ckmeans.1d.dp)
importFrom(DiagrammeR,mermaid)
importFrom(data.table,":=") importFrom(data.table,":=")
importFrom(data.table,as.data.table) importFrom(data.table,as.data.table)
importFrom(data.table,copy) importFrom(data.table,copy)
@ -26,6 +28,16 @@ importFrom(data.table,data.table)
importFrom(data.table,rbindlist) importFrom(data.table,rbindlist)
importFrom(data.table,set) importFrom(data.table,set)
importFrom(data.table,setnames) importFrom(data.table,setnames)
importFrom(ggplot2,aes)
importFrom(ggplot2,coord_flip)
importFrom(ggplot2,element_blank)
importFrom(ggplot2,element_text)
importFrom(ggplot2,geom_bar)
importFrom(ggplot2,ggplot)
importFrom(ggplot2,ggtitle)
importFrom(ggplot2,theme)
importFrom(ggplot2,xlab)
importFrom(ggplot2,ylab)
importFrom(magrittr,"%>%") importFrom(magrittr,"%>%")
importFrom(magrittr,add) importFrom(magrittr,add)
importFrom(magrittr,not) importFrom(magrittr,not)

5
R-package/R/xgb.cv.R
View File

@ -114,10 +114,11 @@ xgb.cv <- function(params=list(), data, nrounds, nfold, label = NULL, missing =
colnames <- str_split(string = history[1], pattern = "\t")[[1]] %>% .[2:length(.)] %>% str_extract(".*:") %>% str_replace(":","") %>% str_replace("-", ".") colnames <- str_split(string = history[1], pattern = "\t")[[1]] %>% .[2:length(.)] %>% str_extract(".*:") %>% str_replace(":","") %>% str_replace("-", ".")
colnamesMean <- paste(colnames, "mean") colnamesMean <- paste(colnames, "mean")
colnamesStd <- paste(colnames, "std") if(showsd) colnamesStd <- paste(colnames, "std")
colnames <- c() colnames <- c()
for(i in 1:length(colnamesMean)) colnames <- c(colnames, colnamesMean[i], colnamesStd[i]) if(showsd) for(i in 1:length(colnamesMean)) colnames <- c(colnames, colnamesMean[i], colnamesStd[i])
else colnames <- colnamesMean
type <- rep(x = "numeric", times = length(colnames)) type <- rep(x = "numeric", times = length(colnames))
dt <- read.table(text = "", colClasses = type, col.names = colnames) %>% as.data.table dt <- read.table(text = "", colClasses = type, col.names = colnames) %>% as.data.table

3
R-package/R/xgb.model.dt.tree.R
View File

@ -49,10 +49,9 @@
#' #'
#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, #' bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
#' eta = 1, nround = 2,objective = "binary:logistic") #' eta = 1, nround = 2,objective = "binary:logistic")
#' xgb.dump(bst, 'xgb.model.dump', with.stats = TRUE)
#' #'
#' #agaricus.test$data@@Dimnames[[2]] represents the column names of the sparse matrix. #' #agaricus.test$data@@Dimnames[[2]] represents the column names of the sparse matrix.
#' xgb.model.dt.tree(agaricus.train$data@@Dimnames[[2]], filename_dump = 'xgb.model.dump') #' xgb.model.dt.tree(agaricus.train$data@@Dimnames[[2]], model = bst)
#' #'
#' @export #' @export
xgb.model.dt.tree <- function(feature_names = NULL, filename_dump = NULL, model = NULL, text = NULL, n_first_tree = NULL){ xgb.model.dt.tree <- function(feature_names = NULL, filename_dump = NULL, model = NULL, text = NULL, n_first_tree = NULL){

59
R-package/R/xgb.plot.importance.R Normal file
View File

@ -0,0 +1,59 @@
#' Plot feature importance bar graph
#'
#' Read a data.table containing feature importance details and plot it.
#'
#' @importFrom ggplot2 ggplot
#' @importFrom ggplot2 aes
#' @importFrom ggplot2 geom_bar
#' @importFrom ggplot2 coord_flip
#' @importFrom ggplot2 xlab
#' @importFrom ggplot2 ylab
#' @importFrom ggplot2 ggtitle
#' @importFrom ggplot2 theme
#' @importFrom ggplot2 element_text
#' @importFrom ggplot2 element_blank
#' @importFrom Ckmeans.1d.dp Ckmeans.1d.dp
#' @importFrom magrittr %>%
#' @param importance_matrix a \code{data.table} returned by the \code{xgb.importance} function.
#' @param numberOfClusters a \code{numeric} vector containing the min and the max range of the possible number of clusters of bars.
#'
#' @return A \code{ggplot2} bar graph representing each feature by a horizontal bar. Longer is the bar, more important is the feature. Features are classified by importance and clustered by importance. The group is represented through the color of the bar.
#'
#' @details
#' The purpose of this function is to easily represent the importance of each feature of a model.
#' The function return a ggplot graph, therefore each of its characteristic can be overriden (to customize it).
#' In particular you may want to override the title of the graph. To do so, add \code{+ ggtitle("A GRAPH NAME")} next to the value returned by this function.
#'
#' @examples
#' data(agaricus.train, package='xgboost')
#'
#' #Both dataset are list with two items, a sparse matrix and labels
#' #(labels = outcome column which will be learned).
#' #Each column of the sparse Matrix is a feature in one hot encoding format.
#' train <- agaricus.train
#'
#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
#' eta = 1, nround = 2,objective = "binary:logistic")
#'
#' #train$data@@Dimnames[[2]] represents the column names of the sparse matrix.
#' importance_matrix <- xgb.importance(train$data@@Dimnames[[2]], model = bst)
#' xgb.plot.importance(importance_matrix)
#'
#' @export
xgb.plot.importance <- function(importance_matrix = NULL, numberOfClusters = c(1:10)){
if (!"data.table" %in% class(importance_matrix)) {
stop("importance_matrix: Should be a data.table.")
}
clusters <- suppressWarnings(Ckmeans.1d.dp(importance_matrix[,Gain], numberOfClusters))
importance_matrix[,"Cluster":=clusters$cluster %>% as.character]
plot <- ggplot(importance_matrix, aes(x=reorder(Feature, Gain), y = Gain, width= 0.05), environment = environment())+ geom_bar(aes(fill=Cluster), stat="identity", position="identity") + coord_flip() + xlab("Features") + ylab("Gain") + ggtitle("Feature importance") + theme(plot.title = element_text(lineheight=.9, face="bold"), panel.grid.major.y = element_blank() )
return(plot)
}
# Avoid error messages during CRAN check.
# The reason is that these variables are never declared
# They are mainly column names inferred by Data.table...
globalVariables(c("Feature","Gain", "Cluster"))

4
R-package/R/xgb.plot.tree.R
View File

@ -15,7 +15,7 @@
#' @importFrom stringr str_split #' @importFrom stringr str_split
#' @importFrom stringr str_extract #' @importFrom stringr str_extract
#' @importFrom stringr str_trim #' @importFrom stringr str_trim
#' @importFrom DiagrammeR DiagrammeR #' @importFrom DiagrammeR mermaid
#' @param feature_names names of each feature as a character vector. Can be extracted from a sparse matrix (see example). If model dump already contains feature names, this argument should be \code{NULL}. #' @param feature_names names of each feature as a character vector. Can be extracted from a sparse matrix (see example). If model dump already contains feature names, this argument should be \code{NULL}.
#' @param filename_dump the path to the text file storing the model. Model dump must include the gain per feature and per tree (parameter \code{with.stats = T} in function \code{xgb.dump}). Possible to provide a model directly (see \code{model} argument). #' @param filename_dump the path to the text file storing the model. Model dump must include the gain per feature and per tree (parameter \code{with.stats = T} in function \code{xgb.dump}). Possible to provide a model directly (see \code{model} argument).
#' @param model generated by the \code{xgb.train} function. Avoid the creation of a dump file. #' @param model generated by the \code{xgb.train} function. Avoid the creation of a dump file.
@ -85,7 +85,7 @@ xgb.plot.tree <- function(feature_names = NULL, filename_dump = NULL, model = NU
no <- allTrees[Feature!="Leaf", c(No)] %>% paste(collapse = ",") %>% paste("class ", ., " redNode", sep = "") no <- allTrees[Feature!="Leaf", c(No)] %>% paste(collapse = ",") %>% paste("class ", ., " redNode", sep = "")
path <- allTrees[Feature!="Leaf", c(yesPath, noPath)] %>% .[order(.)] %>% paste(sep = "", collapse = ";") %>% paste("graph LR", .,collapse = "", sep = ";") %>% paste(CSSstyle, yes, no, sep = ";") path <- allTrees[Feature!="Leaf", c(yesPath, noPath)] %>% .[order(.)] %>% paste(sep = "", collapse = ";") %>% paste("graph LR", .,collapse = "", sep = ";") %>% paste(CSSstyle, yes, no, sep = ";")
DiagrammeR(path, width, height) mermaid(path, width, height)
} }
# Avoid error messages during CRAN check. # Avoid error messages during CRAN check.

2
R-package/demo/cross_validation.R
View File

@ -19,7 +19,7 @@ cat('running cross validation, disable standard deviation display\n')
# [iteration] metric_name:mean_value+std_value # [iteration] metric_name:mean_value+std_value
# std_value is standard deviation of the metric # std_value is standard deviation of the metric
xgb.cv(param, dtrain, nround, nfold=5, xgb.cv(param, dtrain, nround, nfold=5,
metrics={'error'}, , showsd = FALSE) metrics={'error'}, showsd = FALSE)
### ###
# you can also do cross validation with cutomized loss function # you can also do cross validation with cutomized loss function

3
R-package/man/agaricus.test.Rd
View File

@ -1,4 +1,5 @@
% Generated by roxygen2 (4.0.1): do not edit by hand % Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/xgboost.R
\docType{data} \docType{data}
\name{agaricus.test} \name{agaricus.test}
\alias{agaricus.test} \alias{agaricus.test}

3
R-package/man/agaricus.train.Rd
View File

@ -1,4 +1,5 @@
% Generated by roxygen2 (4.0.1): do not edit by hand % Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/xgboost.R
\docType{data} \docType{data}
\name{agaricus.train} \name{agaricus.train}
\alias{agaricus.train} \alias{agaricus.train}

7
R-package/man/getinfo.Rd
View File

@ -1,4 +1,5 @@
% Generated by roxygen2 (4.0.1): do not edit by hand % Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/getinfo.xgb.DMatrix.R
\docType{methods} \docType{methods}
\name{getinfo} \name{getinfo}
\alias{getinfo} \alias{getinfo}
@ -12,9 +13,9 @@ getinfo(object, ...)
\arguments{ \arguments{
\item{object}{Object of class "xgb.DMatrix"} \item{object}{Object of class "xgb.DMatrix"}
\item{name}{the name of the field to get}
\item{...}{other parameters} \item{...}{other parameters}
\item{name}{the name of the field to get}
} }
\description{ \description{
Get information of an xgb.DMatrix object Get information of an xgb.DMatrix object

3
R-package/man/predict-xgb.Booster-method.Rd
View File

@ -1,4 +1,5 @@
% Generated by roxygen2 (4.0.1): do not edit by hand % Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/predict.xgb.Booster.R
\docType{methods} \docType{methods}
\name{predict,xgb.Booster-method} \name{predict,xgb.Booster-method}
\alias{predict,xgb.Booster-method} \alias{predict,xgb.Booster-method}

7
R-package/man/setinfo.Rd
View File

@ -1,4 +1,5 @@
% Generated by roxygen2 (4.0.1): do not edit by hand % Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/setinfo.xgb.DMatrix.R
\docType{methods} \docType{methods}
\name{setinfo} \name{setinfo}
\alias{setinfo} \alias{setinfo}
@ -12,11 +13,11 @@ setinfo(object, ...)
\arguments{ \arguments{
\item{object}{Object of class "xgb.DMatrix"} \item{object}{Object of class "xgb.DMatrix"}
\item{...}{other parameters}
\item{name}{the name of the field to get} \item{name}{the name of the field to get}
\item{info}{the specific field of information to set} \item{info}{the specific field of information to set}
\item{...}{other parameters}
} }
\description{ \description{
Set information of an xgb.DMatrix object Set information of an xgb.DMatrix object

7
R-package/man/slice.Rd
View File

@ -1,4 +1,5 @@
% Generated by roxygen2 (4.0.1): do not edit by hand % Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/slice.xgb.DMatrix.R
\docType{methods} \docType{methods}
\name{slice} \name{slice}
\alias{slice} \alias{slice}
@ -13,9 +14,9 @@ slice(object, ...)
\arguments{ \arguments{
\item{object}{Object of class "xgb.DMatrix"} \item{object}{Object of class "xgb.DMatrix"}
\item{idxset}{a integer vector of indices of rows needed}
\item{...}{other parameters} \item{...}{other parameters}
\item{idxset}{a integer vector of indices of rows needed}
} }
\description{ \description{
Get a new DMatrix containing the specified rows of Get a new DMatrix containing the specified rows of

3
R-package/man/xgb.DMatrix.Rd
View File

@ -1,4 +1,5 @@
% Generated by roxygen2 (4.0.1): do not edit by hand % Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/xgb.DMatrix.R
\name{xgb.DMatrix} \name{xgb.DMatrix}
\alias{xgb.DMatrix} \alias{xgb.DMatrix}
\title{Contruct xgb.DMatrix object} \title{Contruct xgb.DMatrix object}

3
R-package/man/xgb.DMatrix.save.Rd
View File

@ -1,4 +1,5 @@
% Generated by roxygen2 (4.0.1): do not edit by hand % Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/xgb.DMatrix.save.R
\name{xgb.DMatrix.save} \name{xgb.DMatrix.save}
\alias{xgb.DMatrix.save} \alias{xgb.DMatrix.save}
\title{Save xgb.DMatrix object to binary file} \title{Save xgb.DMatrix object to binary file}

3
R-package/man/xgb.cv.Rd
View File

@ -1,4 +1,5 @@
% Generated by roxygen2 (4.0.1): do not edit by hand % Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/xgb.cv.R
\name{xgb.cv} \name{xgb.cv}
\alias{xgb.cv} \alias{xgb.cv}
\title{Cross Validation} \title{Cross Validation}

3
R-package/man/xgb.dump.Rd
View File

@ -1,4 +1,5 @@
% Generated by roxygen2 (4.0.1): do not edit by hand % Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/xgb.dump.R
\name{xgb.dump} \name{xgb.dump}
\alias{xgb.dump} \alias{xgb.dump}
\title{Save xgboost model to text file} \title{Save xgboost model to text file}

3
R-package/man/xgb.importance.Rd
View File

@ -1,4 +1,5 @@
% Generated by roxygen2 (4.0.1): do not edit by hand % Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/xgb.importance.R
\name{xgb.importance} \name{xgb.importance}
\alias{xgb.importance} \alias{xgb.importance}
\title{Show importance of features in a model} \title{Show importance of features in a model}

3
R-package/man/xgb.load.Rd
View File

@ -1,4 +1,5 @@
% Generated by roxygen2 (4.0.1): do not edit by hand % Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/xgb.load.R
\name{xgb.load} \name{xgb.load}
\alias{xgb.load} \alias{xgb.load}
\title{Load xgboost model from binary file} \title{Load xgboost model from binary file}

6
R-package/man/xgb.model.dt.tree.Rd
View File

@ -1,4 +1,5 @@
% Generated by roxygen2 (4.0.1): do not edit by hand % Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/xgb.model.dt.tree.R
\name{xgb.model.dt.tree} \name{xgb.model.dt.tree}
\alias{xgb.model.dt.tree} \alias{xgb.model.dt.tree}
\title{Convert tree model dump to data.table} \title{Convert tree model dump to data.table}
@ -50,9 +51,8 @@ train <- agaricus.train
bst <- xgboost(data = train$data, label = train$label, max.depth = 2, bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
eta = 1, nround = 2,objective = "binary:logistic") eta = 1, nround = 2,objective = "binary:logistic")
xgb.dump(bst, 'xgb.model.dump', with.stats = TRUE)
#agaricus.test$data@Dimnames[[2]] represents the column names of the sparse matrix. #agaricus.test$data@Dimnames[[2]] represents the column names of the sparse matrix.
xgb.model.dt.tree(agaricus.train$data@Dimnames[[2]], filename_dump = 'xgb.model.dump') xgb.model.dt.tree(agaricus.train$data@Dimnames[[2]], model = bst)
} }

40
R-package/man/xgb.plot.importance.Rd Normal file
View File

@ -0,0 +1,40 @@
% Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/xgb.plot.importance.R
\name{xgb.plot.importance}
\alias{xgb.plot.importance}
\title{Plot feature importance bar graph}
\usage{
xgb.plot.importance(importance_matrix = NULL, numberOfClusters = c(1:10))
}
\arguments{
\item{importance_matrix}{a \code{data.table} returned by the \code{xgb.importance} function.}
\item{numberOfClusters}{a \code{numeric} vector containing the min and the max range of the possible number of clusters of bars.}
}
\value{
A \code{ggplot2} bar graph representing each feature by a horizontal bar. Longer is the bar, more important is the feature. Features are classified by importance and clustered by importance. The group is represented through the color of the bar.
}
\description{
Read a data.table containing feature importance details and plot it.
}
\details{
The purpose of this function is to easily represent the importance of each feature of a model.
The function return a ggplot graph, therefore each of its characteristic can be overriden (to customize it).
In particular you may want to override the title of the graph. To do so, add \code{+ ggtitle("A GRAPH NAME")} next to the value returned by this function.
}
\examples{
data(agaricus.train, package='xgboost')
#Both dataset are list with two items, a sparse matrix and labels
#(labels = outcome column which will be learned).
#Each column of the sparse Matrix is a feature in one hot encoding format.
train <- agaricus.train
bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
eta = 1, nround = 2,objective = "binary:logistic")
#train$data@Dimnames[[2]] represents the column names of the sparse matrix.
importance_matrix <- xgb.importance(train$data@Dimnames[[2]], model = bst)
xgb.plot.importance(importance_matrix)
}

3
R-package/man/xgb.plot.tree.Rd
View File

@ -1,4 +1,5 @@
% Generated by roxygen2 (4.0.1): do not edit by hand % Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/xgb.plot.tree.R
\name{xgb.plot.tree} \name{xgb.plot.tree}
\alias{xgb.plot.tree} \alias{xgb.plot.tree}
\title{Plot a boosted tree model} \title{Plot a boosted tree model}

3
R-package/man/xgb.save.Rd
View File

@ -1,4 +1,5 @@
% Generated by roxygen2 (4.0.1): do not edit by hand % Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/xgb.save.R
\name{xgb.save} \name{xgb.save}
\alias{xgb.save} \alias{xgb.save}
\title{Save xgboost model to binary file} \title{Save xgboost model to binary file}

3
R-package/man/xgb.train.Rd
View File

@ -1,4 +1,5 @@
% Generated by roxygen2 (4.0.1): do not edit by hand % Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/xgb.train.R
\name{xgb.train} \name{xgb.train}
\alias{xgb.train} \alias{xgb.train}
\title{eXtreme Gradient Boosting Training} \title{eXtreme Gradient Boosting Training}

9
R-package/man/xgboost.Rd
View File

@ -1,4 +1,5 @@
% Generated by roxygen2 (4.0.1): do not edit by hand % Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/xgboost.R
\name{xgboost} \name{xgboost}
\alias{xgboost} \alias{xgboost}
\title{eXtreme Gradient Boosting (Tree) library} \title{eXtreme Gradient Boosting (Tree) library}
@ -13,6 +14,9 @@ xgboost(data = NULL, label = NULL, missing = NULL, params = list(),
\item{label}{the response variable. User should not set this field, \item{label}{the response variable. User should not set this field,
if data is local data file or \code{xgb.DMatrix}.} if data is local data file or \code{xgb.DMatrix}.}
\item{missing}{Missing is only used when input is dense matrix, pick a float
value that represents missing value. Sometime a data use 0 or other extreme value to represents missing values.}
\item{params}{the list of parameters. Commonly used ones are: \item{params}{the list of parameters. Commonly used ones are:
\itemize{ \itemize{
\item \code{objective} objective function, common ones are \item \code{objective} objective function, common ones are
@ -34,9 +38,6 @@ if data is local data file or \code{xgb.DMatrix}.}
information of performance. If 2, xgboost will print information of both information of performance. If 2, xgboost will print information of both
performance and construction progress information} performance and construction progress information}
\item{missing}{Missing is only used when input is dense matrix, pick a float
value that represents missing value. Sometime a data use 0 or other extreme value to represents missing values.}
\item{...}{other parameters to pass to \code{params}.} \item{...}{other parameters to pass to \code{params}.}
} }
\description{ \description{

227
R-package/vignettes/discoverYourData.Rmd Normal file
View File

@ -0,0 +1,227 @@
---
title: "Understand your dataset with Xgboost"
output:
html_document:
css: vignette.css
number_sections: yes
toc: yes
date: "Wednesday, January 28, 2015"
---
Introduction
============
The purpose of this Vignette is to show you how to use **Xgboost** to discover and better understand your own dataset.
You may know **Xgboost** as a state of the art tool to build some kind of Machine learning models. It has been used to win several [Kaggle](http://www.kaggle.com/) competition ([more information](https://github.com/tqchen/xgboost)).
During these competition, the purpose is to make prediction. This Vignette is not about showing you how to predict anything. The purpose of this document is to explain *how to use **Xgboost** to understand the link between the features of your data and an outcome*.
For the purpose of this tutorial we will first load the required packages.
```{r libLoading, results='hold', message=F, warning=F}
require(xgboost)
require(Matrix)
require(data.table)
if (!require(vcd)) install.packages('vcd')
```
*Note that **VCD** is used for one of its embedded dataset only (and not for its own functions).*
Preparation of the dataset
==========================
According to its documentation, **Xgboost** works only on `numeric` variables.
Sometimes the dataset we have to work on have *categorical* data.
A *categorical* variable is one which have a fixed number of different values. By exemple, if for each observation a variable called *Colour* can have only *red*, *blue* or *green* as value, it is a *categorical* variable.
In **R**, *categorical* variable is called `factor`.
Type `?factor` in console for more information.
In this demo we will see how to transform a dense dataframe with *categorical* variables to a sparse matrix before analyzing it in **Xgboost**.
The method we are going to see is usually called [one hot encoding](http://en.wikipedia.org/wiki/One-hot).
The first step is to load Arthritis dataset in memory and create a copy of the dataset with `data.table` package (`data.table` is 100% compliant with **R** dataframe but its syntax is a lot more consistent and its performance are really good).
```{r, results='hide'}
data(Arthritis)
df <- data.table(Arthritis, keep.rownames = F)
```
Let's have a look to the 10 first lines of the `data.table`:
```{r}
print(df[1:10])
```
Now we will check the format of each column.
```{r}
str(df)
```
2 columns have `factor` type, one has `ordinal` type (`ordinal` variable is a categorical variable with values wich can be ordered, here: `None` > `Some` > `Marked`).
Let's add some new categorical features to see if it helps.
Of course these feature are highly correlated to the Age feature. Usually it's not a good thing in ML, but tree algorithms (including boosted trees) are able to select the best features, even in case of highly correlated features.
For the first feature we create groups of age by rounding the real age. Note that we transform it to `factor` so the algorithm treat them as independant values.
```{r}
df[,AgeDiscret:= as.factor(round(Age/10,0))][1:10]
```
Following is an even stronger simplification of the real age with an arbitrary split at 30 years old. I choose this value **based on nothing**. We will see later if simplifying the information based on arbitrary values is a good strategy (I am sure you already have an idea of how well it will work!).
```{r}
df[,AgeCat:= as.factor(ifelse(Age > 30, "Old", "Young"))][1:10]
```
We remove ID as there is nothing to learn from this feature (it will just add some noise as the dataset is small).
```{r, results='hide'}
df[,ID:=NULL]
```
Let's list the different values for the column Treatment.
```{r}
print(levels(df[,Treatment]))
```
Next step, we will transform the categorical data to dummy variables.
This is the [one hot encoding](http://en.wikipedia.org/wiki/One-hot) part.
The purpose is to transform each value of each *categorical* feature in a binary feature.
For example, the column Treatment will be replaced by two columns, Placebo, and Treated. Each of them will be *binary*. For example an observation which had the value Placebo in column Treatment before the transformation will have, after the transformation, the value 1 in the new column Placebo and the value 0 in the new column Treated.
Formulae `Improved~.-1` used below means transform all *categorical* features but column Improved to binary values.
Column Improved is excluded because it will be our output column, the one we want to predict.
```{r, warning=FALSE,message=FALSE}
sparse_matrix <- sparse.model.matrix(Improved~.-1, data = df)
print(sparse_matrix[1:10,])
```
Create the output vector (not as a sparse `Matrix`):
1. Set, for all rows, field in Y column to 0;
2. set Y to 1 when Improved == Marked;
3. Return Y column.
```{r}
output_vector = df[,Y:=0][Improved == "Marked",Y:=1][,Y]
```
Build the model
===============
The code below is very usual. For more information, you can look at the documentation of `xgboost()` function.
```{r}
bst <- xgboost(data = sparse_matrix, label = output_vector, max.depth = 4,
eta = 1, nround = 10,objective = "binary:logistic")
```
You can see plenty of `train-error: 0.XXXXX` lines followed by a number. It decreases. Each line shows how well your model explains your data. Lower is better.
A model which fits too well may [overfit](http://en.wikipedia.org/wiki/Overfitting) (meaning it copy paste too much the past, and is not that good to predict the future).
Here you can see the numbers decrease until line 7 and then increase. It probably means I am overfitting. To fix that I may reduce the number of rounds to `nround = 4`. I will let things like that because I don't really care for the purpose of this example :-)
Feature importance
==================
Measure feature importance
--------------------------
In the code below, `sparse_matrix@Dimnames[[2]]` represents the column names of the sparse matrix. These names are the values of the feature (because one binary column == one value of one *categorical* feature)
```{r}
importance <- xgb.importance(sparse_matrix@Dimnames[[2]], model = bst)
print(importance)
```
The column `Gain` provide the information we are looking for.
`Gain` is the improvement in accuracy brought by a feature to the branches it is on. The idea is that before adding a new split on a feature X to the branch there was some wrongly classified elements, after adding the split on this feature, there are two new branches, and each of these branch is more accurate (one branch saying if your observation is on this branch then it should be classified as 1, and the other branch saying the exact opposite, both new branch being more accurate than the one before the insertion of the feature).
`Cover` measure the relative quantity of observations concerned by a feature.
`Frequence` is a simpler way to measure the `Gain`. It just counts the number of times a feature is used in all generated trees. You should not use it (unless you know why you want to use it).
As you can see, features are classified by `Gain`.
Plotting the feature importance
-------------------------------
All these things are nice, but it would be even better to plot the result. Fortunately, such function already exists.
```{r}
xgb.plot.importance(importance_matrix = importance)
```
Feature have been automatically divided in 2 clusters: the interesting features... and the others.
According to the plot above, the most important feature in this dataset to predict if the treatment will work is :
* the Age;
* having received a placebo or not ;
* the sex is third but already included in the not interesting feature ;
* then we see our generated features (AgeDiscret). We can see that their contribution is very low.
*Note: Depending of the case you may have more than two clusters. Default value is to limit them to 10, but you can increase this limit. Look at the function documentation for more information.*
Does these results make sense?
------------------------------
Let's check some **Chi2** between each of these features and the outcome.
Higher **Chi2** means better correlation.
```{r, warning=FALSE, message=FALSE}
c2 <- chisq.test(df$Age, df$Y)
print(c2)
```
Pearson correlation between Age and illness disapearing is **`r round(c2$statistic, 2 )`**.
```{r, warning=FALSE, message=FALSE}
c2 <- chisq.test(df$AgeDiscret, df$Y)
print(c2)
```
Our first simplification of Age gives a Pearson correlation is **`r round(c2$statistic, 2)`**.
```{r, warning=FALSE, message=FALSE}
c2 <- chisq.test(df$AgeCat, df$Y)
print(c2)
```
The perfectly random split I did between young and old at 30 years old have a low correlation of **`r round(c2$statistic, 2)`**. It's a result we may expect as may be in my mind > 30 years is being old (I am 32 and starting feeling old, this may explain that), but for the illness we are studying, the age to be vulnerable is not the same. Don't let your *gut* lower the quality of your model. In *data science* expression, there is the word *science* :-)
Conclusion
==========
As you can see, in general *destroying information by simplying it won't improve your model*. **Chi2** just demonstrates that.
But in more complex cases, creating a new feature based on existing one which makes link with the outcome more obvious may help the algorithm and improve the model.
The case studied here is not enough complex to show that. Check Kaggle forum for some challenging datasets. However it's almost always worse when you add some arbitrary rules.
Moreover, you can notice that even if we have added some not useful new features highly correlated with other features, the boosting tree algorithm have been able to choose the best one, which in this case is the Age.
Linear model may not be that strong in these scenario.
```{r, fig.align='center', include=FALSE}
#xgb.plot.tree(sparse_matrix@Dimnames[[2]], model = bst, n_first_tree = 1, width = 1200, height = 800)
```

201
R-package/vignettes/vignette.css Normal file
View File

@ -0,0 +1,201 @@
body{
margin: 0 auto;
background-color: white;
/* --------- FONT FAMILY --------
following are some optional font families. Usually a family
is safer to choose than a specific font,
which may not be on the users computer */
/ font-family:Georgia, Palatino, serif;
font-family: "Open Sans", "Book Antiqua", Palatino, serif;
/ font-family:Arial, Helvetica, sans-serif;
/ font-family:Tahoma, Verdana, Geneva, sans-serif;
/ font-family:Courier, monospace;
/ font-family:"Times New Roman", Times, serif;
/* -------------- COLOR OPTIONS ------------
following are additional color options for base font
you could uncomment another one to easily change the base color
or add one to a specific element style below */
color: #333333; /* dark gray not black */
/ color: #000000; /* black */
/ color: #666666; /* medium gray black */
/ color: #E3E3E3; /* very light gray */
/ color: white;
line-height: 1;
max-width: 960px;
padding: 20px;
font-size: 17px;
}
p {
line-height: 150%;
/ max-width: 540px;
max-width: 960px;
font-weight: 400;
/ color: #333333
}
h1, h2, h3, h4 {
/ color: #111111;
font-weight: 400;
}
h2, h3, h4, h5, p {
margin-bottom: 20px;
padding: 0;
}
h1 {
margin-bottom: 10px;
font-size:230%;
padding: 0px;
font-variant:small-caps;
}
h2 {
font-size:130%
/ margin: 24px 0 6px;
}
h3 {
font-size:110%
text-decoration: underline;
font-style: italic;
}
h4 {
font-size:100%
font-variant:small-caps;
}
h5 {
font-size:100%
font-weight: 100;
font-style: italic;
}
h6 {
font-size:100%
font-weight: 100;
color:red;
font-variant:small-caps;
font-style: italic;
}
a {
color: #606AAA;
margin: 0;
padding: 0;
vertical-align: baseline;
}
a:hover {
text-decoration: blink;
color: green;
}
a:visited {
color: gray;
}
ul, ol {
padding: 0;
margin: 0px 0px 0px 50px;
}
ul {
list-style-type: square;
list-style-position: inside;
}
li {
line-height:150%
}
li ul, li ul {
margin-left: 24px;
}
pre {
padding: 0px 24px;
max-width: 800px;
white-space: pre-wrap;
}
code {
font-family: Consolas, Monaco, Andale Mono, monospace;
line-height: 1.5;
font-size: 15px;
}
aside {
display: block;
float: right;
width: 390px;
}
blockquote {
border-left:.5em solid #eee;
padding: 0 1em;
margin-left:0;
max-width: 476px;
}
blockquote cite {
/ font-size:14px;
line-height:20px;
color:#bfbfbf;
}
blockquote cite:before {
content: '\2014 \00A0';
}
blockquote p {
color: #666;
max-width: 460px;
}
hr {
/ width: 540px;
text-align: left;
margin: 0 auto 0 0;
color: #999;
}
/* table */
table {
width: 100%;
border-top: 1px solid #919699;
border-left: 1px solid #919699;
border-spacing: 0;
}
table th {
padding: 4px 8px 4px 8px;
text-align: center;
color: white;
background: #606AAA;
border-bottom: 1px solid #919699;
border-right: 1px solid #919699;
}
table th p {
font-weight: bold;
margin-bottom: 0px;
}
table td {
padding: 8px;
vertical-align: top;
border-bottom: 1px solid #919699;
border-right: 1px solid #919699;
}
table td:last-child {
/background: lightgray;
text-align: right;
}
table td p {
margin-bottom: 0px;
}
table td p + p {
margin-top: 5px;
}
table td p + p + p {
margin-top: 5px;
}