Update README.md

new parameter in xgboost() and xgb.train() to print every N-th progress message

.gitignore

@@ -2,18 +2,59 @@
 *.slo
 *.lo
 *.o
-
+*.page
 # Compiled Dynamic libraries
 *.so
 *.dylib
-
+*.page
 # Compiled Static libraries
 *.lai
 *.la
 *.a
 *~
+*.Rcheck
+*.rds
+*.tar.gz
 *txt*
 *conf
 *buffer
 *model
-xgboost
+*pyc
+*train
+*test
+*group
+*rar
+*vali
+*data
+*sdf
+Release
+*exe*
+*exp
+ipch
+*.filters
+*.user
+*log
+Debug
+*suo
+*test*
+.Rhistory
+*.dll
+*i386
+*x64
+*dump
+*save
+*csv
+.Rproj.user
+*.cpage.col
+*.cpage
+*.Rproj
+xgboost
+xgboost.mpi
+xgboost.mock
+train*
+rabit
+#.Rbuildignore
+R-package.Rproj
+*.cache*
+R-package/inst
+R-package/src

CHANGES.md

@@ -0,0 +1,36 @@
+Change Log
+=====
+
+xgboost-0.1
+=====
+* Initial release
+
+xgboost-0.2x
+=====
+* Python module
+* Weighted samples instances
+* Initial version of pairwise rank
+
+xgboost-0.3
+=====
+* Faster tree construction module
+  - Allows subsample columns during tree construction via ```bst:col_samplebytree=ratio```
+* Support for boosting from initial predictions
+* Experimental version of LambdaRank
+* Linear booster is now parallelized, using parallel coordinated descent.
+* Add [Code Guide](src/README.md) for customizing objective function and evaluation
+* Add R module
+
+xgboost-0.4
+=====
+* Distributed version of xgboost that runs on YARN, scales to billions of examples
+* Direct save/load data and model from/to S3 and HDFS
+* Feature importance visualization in R module, by Michael Benesty
+* Predict leaf index
+* Poisson regression for counts data
+* Early stopping option in training
+* Native save load support in R and python
+  - xgboost models now can be saved using save/load in R
+  - xgboost python model is now pickable
+* sklearn wrapper is supported in python module
+* Experimental External memory version

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2014 Tianqi Chen
+Copyright (c) 2014 by Tianqi Chen and Contributors 
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

Makefile

@@ -1,25 +1,131 @@
 export CC  = gcc
 export CXX = g++
-export CFLAGS = -Wall -O3 -msse2  -Wno-unknown-pragmas -fopenmp
+export MPICXX = mpicxx
+export LDFLAGS= -pthread -lm 
+export CFLAGS = -Wall -O3 -msse2  -Wno-unknown-pragmas -fPIC
+
+ifeq ($(OS), Windows_NT)
+	export CXX = g++ -m64
+	export CC = gcc -m64
+endif
+
+ifeq ($(no_omp),1)
+	CFLAGS += -DDISABLE_OPENMP 
+else 
+	CFLAGS += -fopenmp
+endif
+
+# by default use c++11
+ifeq ($(cxx11),1)
+	CFLAGS += -std=c++11
+else 
+endif
+
+# handling dmlc
+ifdef dmlc
+	ifndef config
+		ifneq ("$(wildcard $(dmlc)/config.mk)","")
+			config = $(dmlc)/config.mk
+		else
+			config = $(dmlc)/make/config.mk
+		endif	
+	endif
+	include $(config)
+	include $(dmlc)/make/dmlc.mk
+	LDFLAGS+= $(DMLC_LDFLAGS)
+	LIBDMLC=$(dmlc)/libdmlc.a
+else
+	LIBDMLC=dmlc_simple.o
+endif
+
+ifeq ($(OS), Windows_NT)
+	LIBRABIT = subtree/rabit/lib/librabit_empty.a
+	SLIB = wrapper/xgboost_wrapper.dll
+else
+	LIBRABIT = subtree/rabit/lib/librabit.a
+	SLIB = wrapper/libxgboostwrapper.so
+endif
 
 # specify tensor path
 BIN = xgboost
-OBJ = 
-.PHONY: clean all
+MOCKBIN = xgboost.mock
+OBJ = updater.o gbm.o io.o main.o dmlc_simple.o
+MPIBIN =
+TARGET = $(BIN) $(OBJ) $(SLIB)
 
-all: $(BIN) $(OBJ)
-export LDFLAGS= -pthread -lm 
+.PHONY: clean all mpi python Rpack
 
-xgboost: regression/xgboost_reg_main.cpp regression/*.h booster/*.h booster/*/*.hpp booster/*.hpp
+all: $(BIN) $(OBJ) $(SLIB)
+mpi: $(MPIBIN)
+
+python: wrapper/libxgboostwrapper.so
+# now the wrapper takes in two files. io and wrapper part
+updater.o: src/tree/updater.cpp  src/tree/*.hpp src/*.h src/tree/*.h src/utils/*.h
+dmlc_simple.o: src/io/dmlc_simple.cpp src/utils/*.h
+gbm.o: src/gbm/gbm.cpp src/gbm/*.hpp src/gbm/*.h 
+io.o: src/io/io.cpp src/io/*.hpp src/utils/*.h src/learner/dmatrix.h src/*.h
+main.o: src/xgboost_main.cpp src/utils/*.h src/*.h src/learner/*.hpp src/learner/*.h 
+xgboost:  updater.o gbm.o io.o main.o $(LIBRABIT) $(LIBDMLC)
+wrapper/xgboost_wrapper.dll wrapper/libxgboostwrapper.so: wrapper/xgboost_wrapper.cpp src/utils/*.h src/*.h src/learner/*.hpp src/learner/*.h  updater.o gbm.o io.o $(LIBRABIT) $(LIBDMLC)
+
+# dependency on rabit
+subtree/rabit/lib/librabit.a: subtree/rabit/src/engine.cc
+	+	cd subtree/rabit;make lib/librabit.a; cd ../..
+subtree/rabit/lib/librabit_empty.a: subtree/rabit/src/engine_empty.cc
+	+	cd subtree/rabit;make lib/librabit_empty.a; cd ../..
+subtree/rabit/lib/librabit_mock.a: subtree/rabit/src/engine_mock.cc
+	+	cd subtree/rabit;make lib/librabit_mock.a; cd ../..
+subtree/rabit/lib/librabit_mpi.a: subtree/rabit/src/engine_mpi.cc
+	+	cd subtree/rabit;make lib/librabit_mpi.a; cd ../..
 
 $(BIN) : 
-	$(CXX) $(CFLAGS) $(LDFLAGS) -o $@ $(filter %.cpp %.o %.c, $^)
+	$(CXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c %.cc %.a, $^) $(LDFLAGS) 
+
+$(MOCKBIN) : 
+	$(CXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c %.cc %.a, $^) $(LDFLAGS) 
+
+$(SLIB) :
+	$(CXX) $(CFLAGS) -fPIC -shared -o $@ $(filter %.cpp %.o %.c %.a %.cc, $^) $(LDFLAGS) $(DLLFLAGS)
 
 $(OBJ) : 
-	$(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c, $^) )
+	$(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c %.cc, $^) )
+
+$(MPIOBJ) : 
+	$(MPICXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c, $^) ) 
+
+$(MPIBIN) : 
+	$(MPICXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c %.cc %.a, $^) $(LDFLAGS) 
 
 install:
 	cp -f -r $(BIN)  $(INSTALL_PATH)
 
+Rpack:
+	make clean
+	cd subtree/rabit;make clean;cd ..
+	rm -rf xgboost xgboost*.tar.gz
+	cp -r R-package xgboost
+	rm -rf xgboost/src/*.o xgboost/src/*.so xgboost/src/*.dll
+	rm -rf xgboost/src/*/*.o
+	rm -rf subtree/rabit/src/*.o
+	rm -rf xgboost/demo/*.model xgboost/demo/*.buffer xgboost/demo/*.txt
+	rm -rf xgboost/demo/runall.R
+	cp -r src xgboost/src/src
+	mkdir xgboost/src/subtree
+	mkdir xgboost/src/subtree/rabit
+	cp -r subtree/rabit/include xgboost/src/subtree/rabit/include
+	cp -r subtree/rabit/src xgboost/src/subtree/rabit/src
+	rm -rf xgboost/src/subtree/rabit/src/*.o
+	mkdir xgboost/src/wrapper
+	cp  wrapper/xgboost_wrapper.h xgboost/src/wrapper
+	cp  wrapper/xgboost_wrapper.cpp xgboost/src/wrapper
+	cp ./LICENSE xgboost
+	cat R-package/src/Makevars|sed '2s/.*/PKGROOT=./' > xgboost/src/Makevars
+	cp xgboost/src/Makevars xgboost/src/Makevars.win
+	# R CMD build --no-build-vignettes xgboost
+	R CMD build xgboost
+	rm -rf xgboost
+	R CMD check --as-cran xgboost*.tar.gz
+
 clean:
-	$(RM) $(OBJ) $(BIN) *~
+	$(RM) -rf $(OBJ) $(BIN) $(MPIBIN) $(MPIOBJ) $(SLIB) *.o  */*.o */*/*.o *~ */*~ */*/*~
+	cd subtree/rabit; make clean; cd ..

R-package/.Rbuildignore

@@ -0,0 +1,5 @@
+\.o$
+\.so$
+\.dll$
+^.*\.Rproj$
+^\.Rproj\.user$

R-package/DESCRIPTION

@@ -0,0 +1,34 @@
+Package: xgboost
+Type: Package
+Title: eXtreme Gradient Boosting
+Version: 0.4-0
+Date: 2015-05-11
+Author: Tianqi Chen <tianqi.tchen@gmail.com>, Tong He <hetong007@gmail.com>, Michael Benesty <michael@benesty.fr>
+Maintainer: Tong He <hetong007@gmail.com>
+Description: Xgboost is short for eXtreme Gradient Boosting, which is an 
+    efficient and scalable implementation of gradient boosting framework. 
+    This package is an R wrapper of xgboost. The package includes efficient 
+    linear model solver and tree learning algorithms. The package can automatically 
+    do parallel computation with OpenMP, and it can be more than 10 times faster
+    than existing gradient boosting packages such as gbm. It supports various
+    objective functions, including regression, classification and ranking. The
+    package is made to be extensible, so that users are also allowed to define
+    their own objectives easily.
+License: Apache License (== 2.0) | file LICENSE
+URL: https://github.com/dmlc/xgboost
+BugReports: https://github.com/dmlc/xgboost/issues
+VignetteBuilder: knitr
+Suggests:
+    knitr,
+    ggplot2 (>= 1.0.0),
+    DiagrammeR (>= 0.6),
+    Ckmeans.1d.dp (>= 3.3.1),
+    vcd (>= 1.3)
+Depends:
+    R (>= 2.10)
+Imports:
+    Matrix (>= 1.1-0),
+    methods,
+    data.table (>= 1.9.4),
+    magrittr (>= 1.5),
+    stringr (>= 0.6.2)

R-package/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2014 by Tianqi Chen and Contributors 
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

R-package/NAMESPACE

@@ -0,0 +1,43 @@
+# Generated by roxygen2 (4.1.1): do not edit by hand
+
+export(getinfo)
+export(setinfo)
+export(slice)
+export(xgb.DMatrix)
+export(xgb.DMatrix.save)
+export(xgb.cv)
+export(xgb.dump)
+export(xgb.importance)
+export(xgb.load)
+export(xgb.model.dt.tree)
+export(xgb.plot.importance)
+export(xgb.plot.tree)
+export(xgb.save)
+export(xgb.save.raw)
+export(xgb.train)
+export(xgboost)
+exportMethods(nrow)
+exportMethods(predict)
+import(methods)
+importClassesFrom(Matrix,dgCMatrix)
+importClassesFrom(Matrix,dgeMatrix)
+importFrom(Matrix,cBind)
+importFrom(Matrix,colSums)
+importFrom(Matrix,sparseVector)
+importFrom(data.table,":=")
+importFrom(data.table,as.data.table)
+importFrom(data.table,copy)
+importFrom(data.table,data.table)
+importFrom(data.table,fread)
+importFrom(data.table,rbindlist)
+importFrom(data.table,set)
+importFrom(data.table,setnames)
+importFrom(magrittr,"%>%")
+importFrom(magrittr,add)
+importFrom(magrittr,not)
+importFrom(stringr,str_extract)
+importFrom(stringr,str_extract_all)
+importFrom(stringr,str_match)
+importFrom(stringr,str_replace)
+importFrom(stringr,str_split)
+importFrom(stringr,str_trim)

R-package/R/getinfo.xgb.DMatrix.R

@@ -0,0 +1,57 @@
+setClass('xgb.DMatrix')
+
+#' Get information of an xgb.DMatrix object
+#' 
+#' Get information of an xgb.DMatrix object
+#' 
+#' The information can be one of the following:
+#' 
+#' \itemize{
+#'     \item \code{label}: label Xgboost learn from ;
+#'     \item \code{weight}: to do a weight rescale ;
+#'     \item \code{base_margin}: base margin is the base prediction Xgboost will boost from ;
+#'     \item \code{nrow}: number of rows of the \code{xgb.DMatrix}.
+#' }
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' train <- agaricus.train
+#' dtrain <- xgb.DMatrix(train$data, label=train$label)
+#' labels <- getinfo(dtrain, 'label')
+#' setinfo(dtrain, 'label', 1-labels)
+#' labels2 <- getinfo(dtrain, 'label')
+#' stopifnot(all(labels2 == 1-labels))
+#' @rdname getinfo
+#' @export
+#' 
+getinfo <- function(object, ...){
+    UseMethod("getinfo")
+}
+
+
+
+#' @param object Object of class \code{xgb.DMatrix}
+#' @param name the name of the field to get
+#' @param ... other parameters
+#' @rdname getinfo
+#' @method getinfo xgb.DMatrix
+setMethod("getinfo", signature = "xgb.DMatrix", 
+          definition = function(object, name) {
+              if (typeof(name) != "character") {
+                  stop("xgb.getinfo: name must be character")
+              }
+              if (class(object) != "xgb.DMatrix") {
+                  stop("xgb.setinfo: first argument dtrain must be xgb.DMatrix")
+              }
+              if (name != "label" && name != "weight" && 
+                      name != "base_margin" && name != "nrow") {
+                  stop(paste("xgb.getinfo: unknown info name", name))
+              }
+              if (name != "nrow"){
+                  ret <- .Call("XGDMatrixGetInfo_R", object, name, PACKAGE = "xgboost")
+              } else {
+                  ret <- xgb.numrow(object)
+              }
+              return(ret)
+          })
+

R-package/R/nrow.xgb.DMatrix.R

@@ -0,0 +1,19 @@
+setGeneric("nrow")
+
+#' @title Number of xgb.DMatrix rows
+#' @description \code{nrow} return the number of rows present in the \code{xgb.DMatrix}.
+#' @param x Object of class \code{xgb.DMatrix}
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' train <- agaricus.train
+#' dtrain <- xgb.DMatrix(train$data, label=train$label)
+#' stopifnot(nrow(dtrain) == nrow(train$data))
+#' 
+#' @export
+setMethod("nrow",
+          signature = "xgb.DMatrix",
+          definition = function(x) {
+            xgb.numrow(x)
+          }
+)

R-package/R/predict.xgb.Booster.R

@@ -0,0 +1,75 @@
+setClass("xgb.Booster.handle")
+setClass("xgb.Booster",
+         slots = c(handle = "xgb.Booster.handle",
+                   raw = "raw"))
+
+#' Predict method for eXtreme Gradient Boosting model
+#' 
+#' Predicted values based on xgboost model object.
+#' 
+#' @param object Object of class "xgb.Boost"
+#' @param newdata takes \code{matrix}, \code{dgCMatrix}, local data file or 
+#'   \code{xgb.DMatrix}. 
+#' @param missing Missing is only used when input is dense matrix, pick a float 
+#'     value that represents missing value. Sometime a data use 0 or other extreme value to represents missing values.
+#' @param outputmargin whether the prediction should be shown in the original
+#'   value of sum of functions, when outputmargin=TRUE, the prediction is 
+#'   untransformed margin value. In logistic regression, outputmargin=T will
+#'   output value before logistic transformation.
+#' @param ntreelimit limit number of trees used in prediction, this parameter is
+#'  only valid for gbtree, but not for gblinear. set it to be value bigger 
+#'  than 0. It will use all trees by default.
+#' @param predleaf whether predict leaf index instead. If set to TRUE, the output will be a matrix object.
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' data(agaricus.test, package='xgboost')
+#' train <- agaricus.train
+#' test <- agaricus.test
+#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
+#'                eta = 1, nthread = 2, nround = 2,objective = "binary:logistic")
+#' pred <- predict(bst, test$data)
+#' @export
+#' 
+setMethod("predict", signature = "xgb.Booster", 
+          definition = function(object, newdata, missing = NULL, 
+                                outputmargin = FALSE, ntreelimit = NULL, predleaf = FALSE) {
+  if (class(object) != "xgb.Booster"){
+    stop("predict: model in prediction must be of class xgb.Booster")
+  } else {
+    object <- xgb.Booster.check(object, saveraw = FALSE)
+  }
+  if (class(newdata) != "xgb.DMatrix") {
+    if (is.null(missing)) {
+      newdata <- xgb.DMatrix(newdata)
+    } else {
+      newdata <- xgb.DMatrix(newdata, missing = missing)
+    }
+  }
+  if (is.null(ntreelimit)) {
+    ntreelimit <- 0
+  } else {
+    if (ntreelimit < 1){
+      stop("predict: ntreelimit must be equal to or greater than 1")
+    }
+  }
+  option = 0
+  if (outputmargin) {
+    option <- option + 1
+  }
+  if (predleaf) {
+    option <- option + 2
+  }
+  ret <- .Call("XGBoosterPredict_R", object$handle, newdata, as.integer(option), 
+               as.integer(ntreelimit), PACKAGE = "xgboost")
+  if (predleaf){
+      len <- getinfo(newdata, "nrow")
+      if (length(ret) == len){
+          ret <- matrix(ret,ncol = 1)
+      } else {
+          ret <- matrix(ret, ncol = len)
+          ret <- t(ret)
+      }
+  }
+  return(ret)
+})
+

R-package/R/predict.xgb.Booster.handle.R

@@ -0,0 +1,19 @@
+#' Predict method for eXtreme Gradient Boosting model handle
+#' 
+#' Predicted values based on xgb.Booster.handle object.
+#' 
+#' @param object Object of class "xgb.Boost.handle"
+#' @param ... Parameters pass to \code{predict.xgb.Booster}
+#' 
+setMethod("predict", signature = "xgb.Booster.handle", 
+          definition = function(object, ...) {
+  if (class(object) != "xgb.Booster.handle"){
+    stop("predict: model in prediction must be of class xgb.Booster.handle")
+  }
+  
+  bst <- xgb.handleToBooster(object)
+  
+  ret = predict(bst, ...)
+  return(ret)
+})
+

R-package/R/setinfo.xgb.DMatrix.R

@@ -0,0 +1,38 @@
+#' Set information of an xgb.DMatrix object
+#' 
+#' Set information of an xgb.DMatrix object
+#' 
+#' It can be one of the following:
+#' 
+#' \itemize{
+#'     \item \code{label}: label Xgboost learn from ;
+#'     \item \code{weight}: to do a weight rescale ;
+#'     \item \code{base_margin}: base margin is the base prediction Xgboost will boost from ;
+#'     \item \code{group}.
+#' }
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' train <- agaricus.train
+#' dtrain <- xgb.DMatrix(train$data, label=train$label)
+#' labels <- getinfo(dtrain, 'label')
+#' setinfo(dtrain, 'label', 1-labels)
+#' labels2 <- getinfo(dtrain, 'label')
+#' stopifnot(all(labels2 == 1-labels))
+#' @rdname setinfo
+#' @export
+#' 
+setinfo <- function(object, ...){
+  UseMethod("setinfo")
+}
+
+#' @param object Object of class "xgb.DMatrix"
+#' @param name the name of the field to get
+#' @param info the specific field of information to set
+#' @param ... other parameters
+#' @rdname setinfo
+#' @method setinfo xgb.DMatrix
+setMethod("setinfo", signature = "xgb.DMatrix", 
+          definition = function(object, name, info) {
+            xgb.setinfo(object, name, info)
+          })

R-package/R/slice.xgb.DMatrix.R

@@ -0,0 +1,45 @@
+setClass('xgb.DMatrix')
+
+#' Get a new DMatrix containing the specified rows of
+#' orginal xgb.DMatrix object
+#'
+#' Get a new DMatrix containing the specified rows of
+#' orginal xgb.DMatrix object
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' train <- agaricus.train
+#' dtrain <- xgb.DMatrix(train$data, label=train$label)
+#' dsub <- slice(dtrain, 1:3)
+#' @rdname slice
+#' @export
+#' 
+slice <- function(object, ...){
+    UseMethod("slice")
+}
+
+#' @param object Object of class "xgb.DMatrix"
+#' @param idxset a integer vector of indices of rows needed
+#' @param ... other parameters
+#' @rdname slice
+#' @method slice xgb.DMatrix
+setMethod("slice", signature = "xgb.DMatrix", 
+          definition = function(object, idxset, ...) {
+              if (class(object) != "xgb.DMatrix") {
+                  stop("slice: first argument dtrain must be xgb.DMatrix")
+              }
+              ret <- .Call("XGDMatrixSliceDMatrix_R", object, idxset, 
+                           PACKAGE = "xgboost")
+              
+              attr_list <- attributes(object)
+              nr <- xgb.numrow(object)
+              len <- sapply(attr_list,length)
+              ind <- which(len==nr)
+              if (length(ind)>0) {
+                  nms <- names(attr_list)[ind]
+                  for (i in 1:length(ind)) {
+                    attr(ret,nms[i]) <- attr(object,nms[i])[idxset]
+                  }
+              }
+              return(structure(ret, class = "xgb.DMatrix"))
+          })

R-package/R/utils.R

@@ -0,0 +1,344 @@
+#' @importClassesFrom Matrix dgCMatrix dgeMatrix
+#' @import methods
+
+# depends on matrix
+.onLoad <- function(libname, pkgname) {
+  library.dynam("xgboost", pkgname, libname)
+}
+.onUnload <- function(libpath) {
+  library.dynam.unload("xgboost", libpath)
+}
+
+# set information into dmatrix, this mutate dmatrix
+xgb.setinfo <- function(dmat, name, info) {
+  if (class(dmat) != "xgb.DMatrix") {
+    stop("xgb.setinfo: first argument dtrain must be xgb.DMatrix")
+  }
+  if (name == "label") {
+    if (length(info)!=xgb.numrow(dmat))
+      stop("The length of labels must equal to the number of rows in the input data")
+    .Call("XGDMatrixSetInfo_R", dmat, name, as.numeric(info), 
+          PACKAGE = "xgboost")
+    return(TRUE)
+  }
+  if (name == "weight") {
+    if (length(info)!=xgb.numrow(dmat))
+      stop("The length of weights must equal to the number of rows in the input data")
+    .Call("XGDMatrixSetInfo_R", dmat, name, as.numeric(info), 
+          PACKAGE = "xgboost")
+    return(TRUE)
+  }
+  if (name == "base_margin") {
+    # if (length(info)!=xgb.numrow(dmat))
+    #   stop("The length of base margin must equal to the number of rows in the input data")
+    .Call("XGDMatrixSetInfo_R", dmat, name, as.numeric(info), 
+          PACKAGE = "xgboost")
+    return(TRUE)
+  }
+  if (name == "group") {
+    if (sum(info)!=xgb.numrow(dmat))
+      stop("The sum of groups must equal to the number of rows in the input data")
+    .Call("XGDMatrixSetInfo_R", dmat, name, as.integer(info), 
+          PACKAGE = "xgboost")
+    return(TRUE)
+  }
+  stop(paste("xgb.setinfo: unknown info name", name))
+  return(FALSE)
+}
+
+# construct a Booster from cachelist
+xgb.Booster <- function(params = list(), cachelist = list(), modelfile = NULL) {
+  if (typeof(cachelist) != "list") {
+    stop("xgb.Booster: only accepts list of DMatrix as cachelist")
+  }
+  for (dm in cachelist) {
+    if (class(dm) != "xgb.DMatrix") {
+      stop("xgb.Booster: only accepts list of DMatrix as cachelist")
+    }
+  }
+  handle <- .Call("XGBoosterCreate_R", cachelist, PACKAGE = "xgboost")
+  if (length(params) != 0) {
+    for (i in 1:length(params)) {
+      p <- params[i]
+      .Call("XGBoosterSetParam_R", handle, gsub("\\.", "_", names(p)), as.character(p),
+            PACKAGE = "xgboost")
+    }
+  }
+  if (!is.null(modelfile)) {
+    if (typeof(modelfile) == "character") {
+      .Call("XGBoosterLoadModel_R", handle, modelfile, PACKAGE = "xgboost")
+    } else if (typeof(modelfile) == "raw") {
+      .Call("XGBoosterLoadModelFromRaw_R", handle, modelfile, PACKAGE = "xgboost")      
+    } else {
+      stop("xgb.Booster: modelfile must be character or raw vector")
+    }
+  }
+  return(structure(handle, class = "xgb.Booster.handle"))
+}
+
+# convert xgb.Booster.handle to xgb.Booster
+xgb.handleToBooster <- function(handle, raw = NULL)
+{
+  bst <- list(handle = handle, raw = raw)
+  class(bst) <- "xgb.Booster"
+  return(bst)
+}
+
+# Check whether an xgb.Booster object is complete
+xgb.Booster.check <- function(bst, saveraw = TRUE)
+{
+  isnull <- is.null(bst$handle)
+  if (!isnull) {
+    isnull <- .Call("XGCheckNullPtr_R", bst$handle, PACKAGE="xgboost")
+  }
+  if (isnull) {
+    bst$handle <- xgb.Booster(modelfile = bst$raw)
+  } else {
+    if (is.null(bst$raw) && saveraw)
+      bst$raw <- xgb.save.raw(bst$handle)
+  }
+  return(bst)
+}
+
+## ----the following are low level iteratively function, not needed if
+## you do not want to use them ---------------------------------------
+# get dmatrix from data, label
+xgb.get.DMatrix <- function(data, label = NULL, missing = NULL) {
+  inClass <- class(data)
+  if (inClass == "dgCMatrix" || inClass == "matrix") {
+    if (is.null(label)) {
+      stop("xgboost: need label when data is a matrix")
+    }
+    if (is.null(missing)){
+      dtrain <- xgb.DMatrix(data, label = label)
+    } else {
+      dtrain <- xgb.DMatrix(data, label = label, missing = missing)
+    }
+  } else {
+    if (!is.null(label)) {
+      warning("xgboost: label will be ignored.")
+    }
+    if (inClass == "character") {
+      dtrain <- xgb.DMatrix(data)
+    } else if (inClass == "xgb.DMatrix") {
+      dtrain <- data
+    } else {
+      stop("xgboost: Invalid input of data")
+    }
+  }
+  return (dtrain)
+}
+xgb.numrow <- function(dmat) {
+  nrow <- .Call("XGDMatrixNumRow_R", dmat, PACKAGE="xgboost")
+  return(nrow)
+}
+# iteratively update booster with customized statistics
+xgb.iter.boost <- function(booster, dtrain, gpair) {
+  if (class(booster) != "xgb.Booster.handle") {
+    stop("xgb.iter.update: first argument must be type xgb.Booster.handle")
+  }
+  if (class(dtrain) != "xgb.DMatrix") {
+    stop("xgb.iter.update: second argument must be type xgb.DMatrix")
+  }
+  .Call("XGBoosterBoostOneIter_R", booster, dtrain, gpair$grad, gpair$hess, 
+        PACKAGE = "xgboost")
+  return(TRUE)
+}
+
+# iteratively update booster with dtrain
+xgb.iter.update <- function(booster, dtrain, iter, obj = NULL) {
+  if (class(booster) != "xgb.Booster.handle") {
+    stop("xgb.iter.update: first argument must be type xgb.Booster.handle")
+  }
+  if (class(dtrain) != "xgb.DMatrix") {
+    stop("xgb.iter.update: second argument must be type xgb.DMatrix")
+  }
+
+  if (is.null(obj)) {
+    .Call("XGBoosterUpdateOneIter_R", booster, as.integer(iter), dtrain, 
+          PACKAGE = "xgboost")
+  } else {
+    pred <- predict(booster, dtrain)
+    gpair <- obj(pred, dtrain)
+    succ <- xgb.iter.boost(booster, dtrain, gpair)
+  }
+  return(TRUE)
+}
+
+# iteratively evaluate one iteration
+xgb.iter.eval <- function(booster, watchlist, iter, feval = NULL, prediction = FALSE) {
+  if (class(booster) != "xgb.Booster.handle") {
+    stop("xgb.eval: first argument must be type xgb.Booster")
+  }
+  if (typeof(watchlist) != "list") {
+    stop("xgb.eval: only accepts list of DMatrix as watchlist")
+  }
+  for (w in watchlist) {
+    if (class(w) != "xgb.DMatrix") {
+      stop("xgb.eval: watch list can only contain xgb.DMatrix")
+    }
+  }
+  if (length(watchlist) != 0) {
+    if (is.null(feval)) {
+      evnames <- list()
+      for (i in 1:length(watchlist)) {
+        w <- watchlist[i]
+        if (length(names(w)) == 0) {
+          stop("xgb.eval: name tag must be presented for every elements in watchlist")
+        }
+        evnames <- append(evnames, names(w))
+      }
+      msg <- .Call("XGBoosterEvalOneIter_R", booster, as.integer(iter), watchlist, 
+                   evnames, PACKAGE = "xgboost")
+    } else {
+      msg <- paste("[", iter, "]", sep="")
+      for (j in 1:length(watchlist)) {
+        w <- watchlist[j]
+        if (length(names(w)) == 0) {
+          stop("xgb.eval: name tag must be presented for every elements in watchlist")
+        }
+        preds <- predict(booster, w[[1]])
+        ret <- feval(preds, w[[1]])
+        msg <- paste(msg, "\t", names(w), "-", ret$metric, ":", ret$value, sep="")
+      }
+    }
+  } else {
+    msg <- ""
+  }
+  if (prediction){
+    preds <- predict(booster,watchlist[[2]])
+    return(list(msg,preds))
+  }
+  return(msg)
+}
+
+#------------------------------------------
+# helper functions for cross validation
+#
+xgb.cv.mknfold <- function(dall, nfold, param, stratified, folds) {
+  if (nfold <= 1) {
+    stop("nfold must be bigger than 1")
+  }
+  if(is.null(folds)) {
+    if (exists('objective', where=param) && strtrim(param[['objective']], 5) == 'rank:') {
+      stop("\tAutomatic creation of CV-folds is not implemented for ranking!\n",
+           "\tConsider providing pre-computed CV-folds through the folds parameter.")
+    }
+    y <- getinfo(dall, 'label')
+    randidx <- sample(1 : xgb.numrow(dall))
+    if (stratified & length(y) == length(randidx)) {
+      y <- y[randidx]
+      #
+      # WARNING: some heuristic logic is employed to identify classification setting!
+      #
+      # For classification, need to convert y labels to factor before making the folds,
+      # and then do stratification by factor levels.
+      # For regression, leave y numeric and do stratification by quantiles.
+      if (exists('objective', where=param)) {
+        # If 'objective' provided in params, assume that y is a classification label
+        # unless objective is reg:linear
+        if (param[['objective']] != 'reg:linear') y <- factor(y)
+      } else {
+        # If no 'objective' given in params, it means that user either wants to use
+        # the default 'reg:linear' objective or has provided a custom obj function.
+        # Here, assume classification setting when y has 5 or less unique values:
+        if (length(unique(y)) <= 5) y <- factor(y)
+      }
+      folds <- xgb.createFolds(y, nfold)
+    } else { 
+      # make simple non-stratified folds
+      kstep <- length(randidx) %/% nfold
+      folds <- list()
+      for (i in 1:(nfold-1)) {
+        folds[[i]] = randidx[1:kstep]
+        randidx = setdiff(randidx, folds[[i]])
+      }
+      folds[[nfold]] = randidx
+    }
+  }
+  ret <- list()
+  for (k in 1:nfold) {
+    dtest <- slice(dall, folds[[k]])
+    didx = c()
+    for (i in 1:nfold) {
+      if (i != k) {
+        didx <- append(didx, folds[[i]])
+      }
+    }
+    dtrain <- slice(dall, didx)
+    bst <- xgb.Booster(param, list(dtrain, dtest))
+    watchlist = list(train=dtrain, test=dtest)
+    ret[[k]] <- list(dtrain=dtrain, booster=bst, watchlist=watchlist, index=folds[[k]])
+  }
+  return (ret)
+}
+
+xgb.cv.aggcv <- function(res, showsd = TRUE) {
+  header <- res[[1]]
+  ret <- header[1]
+  for (i in 2:length(header)) {
+    kv <- strsplit(header[i], ":")[[1]]
+    ret <- paste(ret, "\t", kv[1], ":", sep="")
+    stats <- c()
+    stats[1] <- as.numeric(kv[2])    
+    for (j in 2:length(res)) {
+      tkv <- strsplit(res[[j]][i], ":")[[1]]
+      stats[j] <- as.numeric(tkv[2])
+    }
+    ret <- paste(ret, sprintf("%f", mean(stats)), sep="")
+    if (showsd) {
+      ret <- paste(ret, sprintf("+%f", sd(stats)), sep="")
+    }
+  }
+  return (ret)
+}
+
+# Shamelessly copied from caret::createFolds
+# and simplified by always returning an unnamed list of test indices
+xgb.createFolds <- function(y, k = 10)
+{
+  if(is.numeric(y)) {
+    ## Group the numeric data based on their magnitudes
+    ## and sample within those groups.
+
+    ## When the number of samples is low, we may have
+    ## issues further slicing the numeric data into
+    ## groups. The number of groups will depend on the
+    ## ratio of the number of folds to the sample size.
+    ## At most, we will use quantiles. If the sample
+    ## is too small, we just do regular unstratified
+    ## CV
+    cuts <- floor(length(y)/k)
+    if(cuts < 2) cuts <- 2
+    if(cuts > 5) cuts <- 5
+    y <- cut(y,
+             unique(quantile(y, probs = seq(0, 1, length = cuts))),
+             include.lowest = TRUE)
+  }
+
+  if(k < length(y)) {
+    ## reset levels so that the possible levels and
+    ## the levels in the vector are the same
+    y <- factor(as.character(y))
+    numInClass <- table(y)
+    foldVector <- vector(mode = "integer", length(y))
+    
+    ## For each class, balance the fold allocation as far
+    ## as possible, then resample the remainder.
+    ## The final assignment of folds is also randomized.
+    for(i in 1:length(numInClass)) {
+      ## create a vector of integers from 1:k as many times as possible without
+      ## going over the number of samples in the class. Note that if the number
+      ## of samples in a class is less than k, nothing is producd here.
+      seqVector <- rep(1:k, numInClass[i] %/% k)
+      ## add enough random integers to get  length(seqVector) == numInClass[i]
+      if(numInClass[i] %% k > 0) seqVector <- c(seqVector, sample(1:k, numInClass[i] %% k))
+      ## shuffle the integers for fold assignment and assign to this classes's data
+      foldVector[which(y == dimnames(numInClass)$y[i])] <- sample(seqVector)
+    }
+  } else foldVector <- seq(along = y)
+
+  out <- split(seq(along = y), foldVector)
+  names(out) <- NULL
+  out
+}

R-package/R/xgb.DMatrix.R

@@ -0,0 +1,45 @@
+#' Contruct xgb.DMatrix object
+#' 
+#' Contruct xgb.DMatrix object from dense matrix, sparse matrix or local file.
+#' 
+#' @param data a \code{matrix} object, a \code{dgCMatrix} object or a character 
+#'   indicating the data file.
+#' @param info a list of information of the xgb.DMatrix object
+#' @param missing Missing is only used when input is dense matrix, pick a float
+#'     value that represents missing value. Sometime a data use 0 or other extreme value to represents missing values.
+#
+#' @param ... other information to pass to \code{info}.
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' train <- agaricus.train
+#' dtrain <- xgb.DMatrix(train$data, label=train$label)
+#' xgb.DMatrix.save(dtrain, 'xgb.DMatrix.data')
+#' dtrain <- xgb.DMatrix('xgb.DMatrix.data')
+#' @export
+#' 
+xgb.DMatrix <- function(data, info = list(), missing = 0, ...) {
+  if (typeof(data) == "character") {
+    handle <- .Call("XGDMatrixCreateFromFile_R", data, as.integer(FALSE), 
+                    PACKAGE = "xgboost")
+  } else if (is.matrix(data)) {
+    handle <- .Call("XGDMatrixCreateFromMat_R", data, missing, 
+                    PACKAGE = "xgboost")
+  } else if (class(data) == "dgCMatrix") {
+    handle <- .Call("XGDMatrixCreateFromCSC_R", data@p, data@i, data@x, 
+                    PACKAGE = "xgboost")
+  } else {
+    stop(paste("xgb.DMatrix: does not support to construct from ", 
+               typeof(data)))
+  }
+  dmat <- structure(handle, class = "xgb.DMatrix")
+  
+  info <- append(info, list(...))
+  if (length(info) == 0) 
+    return(dmat)
+  for (i in 1:length(info)) {
+    p <- info[i]
+    xgb.setinfo(dmat, names(p), p[[1]])
+  }
+  return(dmat)
+} 

R-package/R/xgb.DMatrix.save.R

@@ -0,0 +1,27 @@
+#' Save xgb.DMatrix object to binary file
+#' 
+#' Save xgb.DMatrix object to binary file
+#' 
+#' @param DMatrix the DMatrix object
+#' @param fname the name of the binary file.
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' train <- agaricus.train
+#' dtrain <- xgb.DMatrix(train$data, label=train$label)
+#' xgb.DMatrix.save(dtrain, 'xgb.DMatrix.data')
+#' dtrain <- xgb.DMatrix('xgb.DMatrix.data')
+#' @export
+#' 
+xgb.DMatrix.save <- function(DMatrix, fname) {
+  if (typeof(fname) != "character") {
+    stop("xgb.save: fname must be character")
+  }
+  if (class(DMatrix) == "xgb.DMatrix") {
+    .Call("XGDMatrixSaveBinary_R", DMatrix, fname, as.integer(FALSE), 
+          PACKAGE = "xgboost")
+    return(TRUE)
+  }
+  stop("xgb.DMatrix.save: the input must be xgb.DMatrix")
+  return(FALSE)
+} 

R-package/R/xgb.cv.R

@@ -0,0 +1,232 @@
+#' Cross Validation
+#' 
+#' The cross valudation function of xgboost
+#' 
+#' @importFrom data.table data.table
+#' @importFrom data.table as.data.table
+#' @importFrom magrittr %>%
+#' @importFrom data.table :=
+#' @importFrom data.table rbindlist
+#' @importFrom stringr str_extract_all
+#' @importFrom stringr str_extract
+#' @importFrom stringr str_split
+#' @importFrom stringr str_replace
+#' @importFrom stringr str_match
+#' 
+#' @param params the list of parameters. Commonly used ones are:
+#' \itemize{
+#'   \item \code{objective} objective function, common ones are
+#'   \itemize{
+#'     \item \code{reg:linear} linear regression
+#'     \item \code{binary:logistic} logistic regression for classification
+#'   }
+#'   \item \code{eta} step size of each boosting step
+#'   \item \code{max.depth} maximum depth of the tree
+#'   \item \code{nthread} number of thread used in training, if not set, all threads are used
+#' }
+#'
+#'   See \link{xgb.train} for further details.
+#'   See also demo/ for walkthrough example in R.
+#' @param data takes an \code{xgb.DMatrix} or \code{Matrix} as the input.
+#' @param nrounds the max number of iterations
+#' @param nfold the original dataset is randomly partitioned into \code{nfold} equal size subsamples. 
+#' @param label option field, when data is \code{Matrix}
+#' @param missing Missing is only used when input is dense matrix, pick a float
+#'     value that represents missing value. Sometime a data use 0 or other extreme value to represents missing values.
+#' @param prediction A logical value indicating whether to return the prediction vector.
+#' @param showsd \code{boolean}, whether show standard deviation of cross validation
+#' @param metrics, list of evaluation metrics to be used in corss validation,
+#'   when it is not specified, the evaluation metric is chosen according to objective function.
+#'   Possible options are:
+#' \itemize{
+#'   \item \code{error} binary classification error rate
+#'   \item \code{rmse} Rooted mean square error
+#'   \item \code{logloss} negative log-likelihood function
+#'   \item \code{auc} Area under curve
+#'   \item \code{merror} Exact matching error, used to evaluate multi-class classification
+#' }
+#' @param obj customized objective function. Returns gradient and second order 
+#'   gradient with given prediction and dtrain.
+#' @param feval custimized evaluation function. Returns 
+#'   \code{list(metric='metric-name', value='metric-value')} with given 
+#'   prediction and dtrain.
+#' @param stratified \code{boolean} whether sampling of folds should be stratified by the values of labels in \code{data}
+#' @param folds \code{list} provides a possibility of using a list of pre-defined CV folds (each element must be a vector of fold's indices).
+#'   If folds are supplied, the nfold and stratified parameters would be ignored.
+#' @param verbose \code{boolean}, print the statistics during the process
+#' @param early_stop_round If \code{NULL}, the early stopping function is not triggered. 
+#'     If set to an integer \code{k}, training with a validation set will stop if the performance 
+#'     keeps getting worse consecutively for \code{k} rounds.
+#' @param early.stop.round An alternative of \code{early_stop_round}.
+#' @param maximize If \code{feval} and \code{early_stop_round} are set, then \code{maximize} must be set as well.
+#'     \code{maximize=TRUE} means the larger the evaluation score the better.
+#'     
+#' @param ... other parameters to pass to \code{params}.
+#' 
+#' @return
+#' If \code{prediction = TRUE}, a list with the following elements is returned:
+#' \itemize{
+#'   \item \code{dt} a \code{data.table} with each mean and standard deviation stat for training set and test set
+#'   \item \code{pred} an array or matrix (for multiclass classification) with predictions for each CV-fold for the model having been trained on the data in all other folds.
+#' }
+#'
+#' If \code{prediction = FALSE}, just a \code{data.table} with each mean and standard deviation stat for training set and test set is returned.
+#'
+#' @details 
+#' The original sample is randomly partitioned into \code{nfold} equal size subsamples. 
+#' 
+#' Of the \code{nfold} subsamples, a single subsample is retained as the validation data for testing the model, and the remaining \code{nfold - 1} subsamples are used as training data. 
+#' 
+#' The cross-validation process is then repeated \code{nrounds} times, with each of the \code{nfold} subsamples used exactly once as the validation data.
+#' 
+#' All observations are used for both training and validation.
+#' 
+#' Adapted from \url{http://en.wikipedia.org/wiki/Cross-validation_\%28statistics\%29#k-fold_cross-validation}
+#'
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+#' history <- xgb.cv(data = dtrain, nround=3, nthread = 2, nfold = 5, metrics=list("rmse","auc"),
+#'                   max.depth =3, eta = 1, objective = "binary:logistic")
+#' print(history)
+#' @export
+#'
+xgb.cv <- function(params=list(), data, nrounds, nfold, label = NULL, missing = NULL, 
+                   prediction = FALSE, showsd = TRUE, metrics=list(), 
+                   obj = NULL, feval = NULL, stratified = TRUE, folds = NULL, verbose = T,
+                   early_stop_round = NULL, early.stop.round = NULL, maximize = NULL, ...) {
+  if (typeof(params) != "list") {
+    stop("xgb.cv: first argument params must be list")
+  }
+  if(!is.null(folds)) {
+    if(class(folds)!="list" | length(folds) < 2) {
+      stop("folds must be a list with 2 or more elements that are vectors of indices for each CV-fold")
+    }
+    nfold <- length(folds)
+  }
+  if (nfold <= 1) {
+    stop("nfold must be bigger than 1")
+  }
+  if (is.null(missing)) {
+    dtrain <- xgb.get.DMatrix(data, label)
+  } else {
+    dtrain <- xgb.get.DMatrix(data, label, missing)
+  }
+  params <- append(params, list(...))
+  params <- append(params, list(silent=1))
+  for (mc in metrics) {
+    params <- append(params, list("eval_metric"=mc))
+  }
+  
+  # Early Stopping
+  if (is.null(early_stop_round) && !is.null(early.stop.round))
+    early_stop_round = early.stop.round
+  if (!is.null(early_stop_round)){
+    if (!is.null(feval) && is.null(maximize))
+      stop('Please set maximize to note whether the model is maximizing the evaluation or not.')
+    if (is.null(maximize) && is.null(params$eval_metric))
+      stop('Please set maximize to note whether the model is maximizing the evaluation or not.')
+    if (is.null(maximize))
+    {
+      if (params$eval_metric %in% c('rmse','logloss','error','merror','mlogloss')) {
+        maximize = FALSE
+      } else {
+        maximize = TRUE
+      }
+    }
+    
+    if (maximize) {
+      bestScore = 0
+    } else {
+      bestScore = Inf
+    }
+    bestInd = 0
+    earlyStopflag = FALSE
+    
+    if (length(metrics)>1)
+      warning('Only the first metric is used for early stopping process.')
+  }
+  
+  xgb_folds <- xgb.cv.mknfold(dtrain, nfold, params, stratified, folds)
+  obj_type = params[['objective']]
+  mat_pred = FALSE
+  if (!is.null(obj_type) && obj_type=='multi:softprob')
+  {
+    num_class = params[['num_class']]
+    if (is.null(num_class))
+      stop('must set num_class to use softmax')
+    predictValues <- matrix(0,xgb.numrow(dtrain),num_class)
+    mat_pred = TRUE
+  }
+  else
+    predictValues <- rep(0,xgb.numrow(dtrain))
+  history <- c()
+  for (i in 1:nrounds) {
+    msg <- list()
+    for (k in 1:nfold) {
+      fd <- xgb_folds[[k]]
+      succ <- xgb.iter.update(fd$booster, fd$dtrain, i - 1, obj)
+      if (i<nrounds) {
+          msg[[k]] <- xgb.iter.eval(fd$booster, fd$watchlist, i - 1, feval) %>% str_split("\t") %>% .[[1]]
+      } else {
+        if (!prediction) {
+          msg[[k]] <- xgb.iter.eval(fd$booster, fd$watchlist, i - 1, feval) %>% str_split("\t") %>% .[[1]]
+        } else {
+          res <- xgb.iter.eval(fd$booster, fd$watchlist, i - 1, feval, prediction)
+          if (mat_pred) {
+            pred_mat = matrix(res[[2]],num_class,length(fd$index))
+            predictValues[fd$index,] <- t(pred_mat)
+          } else {
+            predictValues[fd$index] <- res[[2]]
+          }
+          msg[[k]] <- res[[1]] %>% str_split("\t") %>% .[[1]]
+        }
+      }
+    }
+    ret <- xgb.cv.aggcv(msg, showsd)
+    history <- c(history, ret)
+    if(verbose) paste(ret, "\n", sep="") %>% cat
+    
+    # early_Stopping
+    if (!is.null(early_stop_round)){
+      score = strsplit(ret,'\\s+')[[1]][1+length(metrics)+1]
+      score = strsplit(score,'\\+|:')[[1]][[2]]
+      score = as.numeric(score)
+      if ((maximize && score>bestScore) || (!maximize && score<bestScore)) {
+        bestScore = score
+        bestInd = i
+      } else {
+        if (i-bestInd>=early_stop_round) {
+          earlyStopflag = TRUE
+          cat('Stopping. Best iteration:',bestInd)
+          break
+        }
+      }
+    }
+    
+  }
+  
+  colnames <- str_split(string = history[1], pattern = "\t")[[1]] %>% .[2:length(.)] %>% str_extract(".*:") %>% str_replace(":","") %>% str_replace("-", ".")
+  colnamesMean <- paste(colnames, "mean")
+  if(showsd) colnamesStd <- paste(colnames, "std")
+  
+  colnames <- c()
+  if(showsd) for(i in 1:length(colnamesMean)) colnames <- c(colnames, colnamesMean[i], colnamesStd[i])
+  else colnames <- colnamesMean
+  
+  type <- rep(x = "numeric", times = length(colnames))
+  dt <- read.table(text = "", colClasses = type, col.names = colnames) %>% as.data.table
+  split <- str_split(string = history, pattern = "\t")
+  
+  for(line in split) dt <- line[2:length(line)] %>% str_extract_all(pattern = "\\d*\\.+\\d*") %>% unlist %>% as.numeric %>% as.list %>% {rbindlist(list(dt, .), use.names = F, fill = F)}
+  
+  if (prediction) {
+    return(list(dt = dt,pred = predictValues))
+  }
+  return(dt)
+}
+
+# Avoid error messages during CRAN check.
+# The reason is that these variables are never declared
+# They are mainly column names inferred by Data.table...
+globalVariables(".")

R-package/R/xgb.dump.R

@@ -0,0 +1,71 @@
+#' Save xgboost model to text file
+#' 
+#' Save a xgboost model to text file. Could be parsed later.
+#' 
+#' @importFrom magrittr %>%
+#' @importFrom stringr str_replace
+#' @importFrom data.table fread
+#' @importFrom data.table :=
+#' @importFrom data.table setnames
+#' @param model the model object.
+#' @param fname the name of the text file where to save the model text dump. If not provided or set to \code{NULL} the function will return the model as a \code{character} vector.
+#' @param fmap feature map file representing the type of feature. 
+#'        Detailed description could be found at 
+#'        \url{https://github.com/dmlc/xgboost/wiki/Binary-Classification#dump-model}.
+#'        See demo/ for walkthrough example in R, and
+#'        \url{https://github.com/dmlc/xgboost/blob/master/demo/data/featmap.txt} 
+#'        for example Format.
+#' @param with.stats whether dump statistics of splits 
+#'        When this option is on, the model dump comes with two additional statistics:
+#'        gain is the approximate loss function gain we get in each split;
+#'        cover is the sum of second order gradient in each node.
+#'
+#' @return
+#' if fname is not provided or set to \code{NULL} the function will return the model as a \code{character} vector. Otherwise it will return \code{TRUE}.
+#'
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' data(agaricus.test, package='xgboost')
+#' train <- agaricus.train
+#' test <- agaricus.test
+#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
+#'                eta = 1, nthread = 2, nround = 2,objective = "binary:logistic")
+#' # save the model in file 'xgb.model.dump'
+#' xgb.dump(bst, 'xgb.model.dump', with.stats = TRUE)
+#' 
+#' # print the model without saving it to a file
+#' print(xgb.dump(bst))
+#' @export
+#' 
+xgb.dump <- function(model = NULL, fname = NULL, fmap = "", with.stats=FALSE) {
+  if (class(model) != "xgb.Booster") {
+    stop("model: argument must be type xgb.Booster")
+  } else {
+    model <- xgb.Booster.check(model)
+  }
+  if (!(class(fname) %in% c("character", "NULL") && length(fname) <= 1)) {
+    stop("fname: argument must be type character (when provided)")
+  }
+  if (!(class(fmap) %in% c("character", "NULL") && length(fname) <= 1)) {
+    stop("fmap: argument must be type character (when provided)")
+  }
+  
+  longString <- .Call("XGBoosterDumpModel_R", model$handle, fmap, as.integer(with.stats), PACKAGE = "xgboost")
+  
+  dt <- fread(paste(longString, collapse = ""), sep = "\n", header = F)
+
+  setnames(dt, "Lines")
+  
+  if(is.null(fname)) {
+    result <- dt[Lines != "0"][, Lines := str_replace(Lines, "^\t+", "")][Lines != ""][, paste(Lines)]
+    return(result)
+  } else {
+    result <- dt[Lines != "0"][Lines != ""][, paste(Lines)] %>% writeLines(fname)
+    return(TRUE)
+  }
+}
+
+# Avoid error messages during CRAN check.
+# The reason is that these variables are never declared
+# They are mainly column names inferred by Data.table...
+globalVariables(c("Lines", "."))

R-package/R/xgb.importance.R

@@ -0,0 +1,134 @@
+#' Show importance of features in a model
+#' 
+#' Read a xgboost model text dump. 
+#' Can be tree or linear model (text dump of linear model are only supported in dev version of \code{Xgboost} for now).
+#' 
+#' @importFrom data.table data.table
+#' @importFrom data.table setnames
+#' @importFrom data.table :=
+#' @importFrom magrittr %>%
+#' @importFrom Matrix colSums
+#' @importFrom Matrix cBind
+#' @importFrom Matrix sparseVector
+#' 
+#' @param feature_names names of each feature as a character vector. Can be extracted from a sparse matrix (see example). If model dump already contains feature names, this argument should be \code{NULL}.
+#' 
+#' @param filename_dump the path to the text file storing the model. Model dump must include the gain per feature and per tree (\code{with.stats = T} in function \code{xgb.dump}).
+#' 
+#' @param model generated by the \code{xgb.train} function. Avoid the creation of a dump file.
+#' 
+#' @param data the dataset used for the training step. Will be used with \code{label} parameter for co-occurence computation. More information in \code{Detail} part. This parameter is optional.
+#' 
+#' @param label the label vetor used for the training step. Will be used with \code{data} parameter for co-occurence computation. More information in \code{Detail} part. This parameter is optional.
+#' 
+#' @param target a function which returns \code{TRUE} or \code{1} when an observation should be count as a co-occurence and \code{FALSE} or \code{0} otherwise. Default function is provided for computing co-occurences in a binary classification. The \code{target} function should have only one parameter. This parameter will be used to provide each important feature vector after having applied the split condition, therefore these vector will be only made of 0 and 1 only, whatever was the information before. More information in \code{Detail} part. This parameter is optional.
+#'
+#' @return A \code{data.table} of the features used in the model with their average gain (and their weight for boosted tree model) in the model.
+#'
+#' @details 
+#' This is the function to understand the model trained (and through your model, your data).
+#' 
+#' Results are returned for both linear and tree models.
+#' 
+#' \code{data.table} is returned by the function. 
+#' There are 3 columns :
+#' \itemize{
+#'   \item \code{Features} name of the features as provided in \code{feature_names} or already present in the model dump.
+#'   \item \code{Gain} contribution of each feature to the model. For boosted tree model, each gain of each feature of each tree is taken into account, then average per feature to give a vision of the entire model. Highest percentage means important feature to predict the \code{label} used for the training ;
+#'   \item \code{Cover} metric of the number of observation related to this feature (only available for tree models) ;
+#'   \item \code{Weight} percentage representing the relative number of times a feature have been taken into trees. \code{Gain} should be prefered to search the most important feature. For boosted linear model, this column has no meaning.
+#' }
+#' 
+#' Co-occurence count
+#' ------------------
+#' 
+#' The gain gives you indication about the information of how a feature is important in making a branch of a decision tree more pure. However, with this information only, you can't know if this feature has to be present or not to get a specific classification. In the example code, you may wonder if odor=none should be \code{TRUE} to not eat a mushroom.
+#' 
+#' Co-occurence computation is here to help in understanding this relation between a predictor and a specific class. It will count how many observations are returned as \code{TRUE} by the \code{target} function (see parameters). When you execute the example below, there are 92 times only over the 3140 observations of the train dataset where a mushroom have no odor and can be eaten safely.
+#' 
+#' If you need to remember one thing only: until you want to leave us early, don't eat a mushroom which has no odor :-)
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' 
+#' # Both dataset are list with two items, a sparse matrix and labels 
+#' # (labels = outcome column which will be learned). 
+#' # Each column of the sparse Matrix is a feature in one hot encoding format.
+#' train <- agaricus.train
+#' 
+#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
+#'                eta = 1, nthread = 2, nround = 2,objective = "binary:logistic")
+#' 
+#' # train$data@@Dimnames[[2]] represents the column names of the sparse matrix.
+#' xgb.importance(train$data@@Dimnames[[2]], model = bst)
+#' 
+#' # Same thing with co-occurence computation this time
+#' xgb.importance(train$data@@Dimnames[[2]], model = bst, data = train$data, label = train$label)
+#' 
+#' @export
+xgb.importance <- function(feature_names = NULL, filename_dump = NULL, model = NULL, data = NULL, label = NULL, target = function(x) ((x + label) == 2)){  
+  if (!class(feature_names) %in% c("character", "NULL")) {	   
+    stop("feature_names: Has to be a vector of character or NULL if the model dump already contains feature name. Look at this function documentation to see where to get feature names.")
+  }
+  
+  if (!(class(filename_dump) %in% c("character", "NULL") && length(filename_dump) <= 1)) {
+    stop("filename_dump: Has to be a path to the model dump file.")
+  }
+  
+  if (!class(model) %in% c("xgb.Booster", "NULL")) {
+    stop("model: Has to be an object of class xgb.Booster model generaged by the xgb.train function.")
+  }
+  
+  if((is.null(data) & !is.null(label)) |(!is.null(data) & is.null(label))) {
+    stop("data/label: Provide the two arguments if you want co-occurence computation or none of them if you are not interested but not one of them only.")
+  }
+  
+  if(class(label) == "numeric"){
+    if(sum(label == 0) / length(label) > 0.5) label <- as(label, "sparseVector")
+  }
+  
+  if(is.null(model)){
+    text <- readLines(filename_dump)  
+  } else {
+    text <- xgb.dump(model = model, with.stats = T)
+  } 
+  
+  if(text[2] == "bias:"){
+    result <- readLines(filename_dump) %>% linearDump(feature_names, .)
+    if(!is.null(data) | !is.null(label)) warning("data/label: these parameters should only be provided with decision tree based models.")
+  }  else {
+    result <- treeDump(feature_names, text = text, keepDetail = !is.null(data))
+    
+    # Co-occurence computation
+    if(!is.null(data) & !is.null(label) & nrow(result) > 0) {
+      # Take care of missing column 
+      a <- data[, result[MissingNo == T,Feature], drop=FALSE] != 0
+      # Bind the two Matrix and reorder columns
+      c <- data[, result[MissingNo == F,Feature], drop=FALSE] %>% cBind(a,.) %>% .[,result[,Feature]]
+      rm(a)
+      # Apply split
+      d <- data[, result[,Feature], drop=FALSE] < as.numeric(result[,Split])
+      apply(c & d, 2, . %>% target %>% sum) -> vec
+            
+      result <- result[, "RealCover":= as.numeric(vec), with = F][, "RealCover %" := RealCover / sum(label)][,MissingNo:=NULL]
+    }    
+  }
+  result
+}
+
+treeDump <- function(feature_names, text, keepDetail){
+  if(keepDetail) groupBy <- c("Feature", "Split", "MissingNo") else groupBy <- "Feature"
+  
+  result <- xgb.model.dt.tree(feature_names = feature_names, text = text)[,"MissingNo":= Missing == No ][Feature!="Leaf",.(Gain = sum(Quality), Cover = sum(Cover), Frequence = .N), by = groupBy, with = T][,`:=`(Gain = Gain/sum(Gain), Cover = Cover/sum(Cover), Frequence = Frequence/sum(Frequence))][order(Gain, decreasing = T)]
+  
+  result  
+}
+
+linearDump <- function(feature_names, text){
+  which(text == "weight:") %>% {a=.+1;text[a:length(text)]} %>% as.numeric %>% data.table(Feature = feature_names, Weight = .)
+}
+
+# Avoid error messages during CRAN check.
+# The reason is that these variables are never declared
+# They are mainly column names inferred by Data.table...
+globalVariables(c(".", "Feature", "Split", "No", "Missing", "MissingNo", "RealCover"))

R-package/R/xgb.load.R

@@ -0,0 +1,32 @@
+#' Load xgboost model from binary file
+#' 
+#' Load xgboost model from the binary model file
+#' 
+#' @param modelfile the name of the binary file.
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' data(agaricus.test, package='xgboost')
+#' train <- agaricus.train
+#' test <- agaricus.test
+#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
+#'                eta = 1, nthread = 2, nround = 2,objective = "binary:logistic")
+#' xgb.save(bst, 'xgb.model')
+#' bst <- xgb.load('xgb.model')
+#' pred <- predict(bst, test$data)
+#' @export
+#' 
+xgb.load <- function(modelfile) {
+  if (is.null(modelfile)) 
+    stop("xgb.load: modelfile cannot be NULL")
+  
+  handle <- xgb.Booster(modelfile = modelfile)
+  # re-use modelfile if it is raw so we donot need to serialize
+  if (typeof(modelfile) == "raw") {
+    bst <- xgb.handleToBooster(handle, modelfile)
+  } else {
+    bst <- xgb.handleToBooster(handle, NULL)
+  }
+  bst <- xgb.Booster.check(bst)
+  return(bst)
+} 

R-package/R/xgb.model.dt.tree.R

@@ -0,0 +1,170 @@
+#' Convert tree model dump to data.table
+#' 
+#' Read a tree model text dump and return a data.table.
+#' 
+#' @importFrom data.table data.table
+#' @importFrom data.table set
+#' @importFrom data.table rbindlist
+#' @importFrom data.table copy
+#' @importFrom data.table :=
+#' @importFrom magrittr %>%
+#' @importFrom magrittr not
+#' @importFrom magrittr add
+#' @importFrom stringr str_extract
+#' @importFrom stringr str_split
+#' @importFrom stringr str_extract
+#' @importFrom stringr str_trim
+#' @param feature_names names of each feature as a character vector. Can be extracted from a sparse matrix (see example). If model dump already contains feature names, this argument should be \code{NULL}.
+#' @param filename_dump the path to the text file storing the model. Model dump must include the gain per feature and per tree (parameter \code{with.stats = T} in function \code{xgb.dump}).
+#' @param model dump generated by the \code{xgb.train} function. Avoid the creation of a dump file.
+#' @param text dump generated by the \code{xgb.dump} function. Avoid the creation of a dump file. Model dump must include the gain per feature and per tree (parameter \code{with.stats = T} in function \code{xgb.dump}).
+#' @param n_first_tree limit the plot to the n first trees. If \code{NULL}, all trees of the model are plotted. Performance can be low for huge models.
+#'
+#' @return A \code{data.table} of the features used in the model with their gain, cover and few other thing.
+#'
+#' @details 
+#' General function to convert a text dump of tree model to a Matrix. The purpose is to help user to explore the model and get a better understanding of it.
+#' 
+#' The content of the \code{data.table} is organised that way:
+#' 
+#' \itemize{
+#' \item \code{ID}: unique identifier of a node ;
+#'  \item \code{Feature}: feature used in the tree to operate a split. When Leaf is indicated, it is the end of a branch ;
+#'  \item \code{Split}: value of the chosen feature where is operated the split ;
+#'  \item \code{Yes}: ID of the feature for the next node in the branch when the split condition is met ;
+#'  \item \code{No}: ID of the feature for the next node in the branch when the split condition is not met ;
+#'  \item \code{Missing}: ID of the feature for the next node in the branch for observation where the feature used for the split are not provided ;
+#'  \item \code{Quality}: it's the gain related to the split in this specific node ;
+#'  \item \code{Cover}: metric to measure the number of observation affected by the split ;
+#'  \item \code{Tree}: ID of the tree. It is included in the main ID ;
+#'  \item \code{Yes.X} or \code{No.X}: data related to the pointer in \code{Yes} or \code{No} column ;
+#' } 
+#'   
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' 
+#' #Both dataset are list with two items, a sparse matrix and labels 
+#' #(labels = outcome column which will be learned). 
+#' #Each column of the sparse Matrix is a feature in one hot encoding format.
+#' train <- agaricus.train
+#' 
+#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
+#'                eta = 1, nthread = 2, nround = 2,objective = "binary:logistic")
+#' 
+#' #agaricus.test$data@@Dimnames[[2]] represents the column names of the sparse matrix.
+#' xgb.model.dt.tree(agaricus.train$data@@Dimnames[[2]], model = bst)
+#' 
+#' @export
+xgb.model.dt.tree <- function(feature_names = NULL, filename_dump = NULL, model = NULL, text = NULL, n_first_tree = NULL){
+  
+  if (!class(feature_names) %in% c("character", "NULL")) {     
+    stop("feature_names: Has to be a vector of character or NULL if the model dump already contains feature name. Look at this function documentation to see where to get feature names.")
+  }
+  if (!(class(filename_dump) %in% c("character", "NULL") && length(filename_dump) <= 1)) {
+    stop("filename_dump: Has to be a character vector of size 1 representing the path to the model dump file.")
+  } else if (!is.null(filename_dump) && !file.exists(filename_dump)) {
+    stop("filename_dump: path to the model doesn't exist.")
+  } else if(is.null(filename_dump) && is.null(model) && is.null(text)){
+    stop("filename_dump & model & text: no path to dump model, no model, no text dump, have been provided.")
+  }
+  
+  if (!class(model) %in% c("xgb.Booster", "NULL")) {
+    stop("model: Has to be an object of class xgb.Booster model generaged by the xgb.train function.")
+  }
+  
+  if (!class(text) %in% c("character", "NULL")) { 
+    stop("text: Has to be a vector of character or NULL if a path to the model dump has already been provided.")
+  }
+  
+  if (!class(n_first_tree) %in% c("numeric", "NULL") | length(n_first_tree) > 1) {
+    stop("n_first_tree: Has to be a numeric vector of size 1.")
+  }
+  
+  if(!is.null(model)){
+    text = xgb.dump(model = model, with.stats = T)
+  } else if(!is.null(filename_dump)){
+    text <- readLines(filename_dump) %>% str_trim(side = "both")  
+  }
+  
+  position <- str_match(text, "booster") %>% is.na %>% not %>% which %>% c(length(text)+1)
+  
+  extract <- function(x, pattern)  str_extract(x, pattern) %>% str_split("=") %>% lapply(function(x) x[2] %>% as.numeric) %>% unlist
+  
+  n_round <- min(length(position) - 1, n_first_tree)
+  
+  addTreeId <- function(x, i) paste(i,x,sep = "-")
+  
+  allTrees <- data.table()
+ 
+  anynumber_regex<-"[-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?" 
+  for(i in 1:n_round){
+    
+    tree <- text[(position[i]+1):(position[i+1]-1)]
+    
+    # avoid tree made of a leaf only (no split)
+    if(length(tree) <2) next
+    
+    treeID <- i-1
+    
+    notLeaf <- str_match(tree, "leaf") %>% is.na
+    leaf <- notLeaf %>% not %>% tree[.]
+    branch <- notLeaf %>% tree[.]
+    idBranch <- str_extract(branch, "\\d*:") %>% str_replace(":", "") %>% addTreeId(treeID)
+    idLeaf <- str_extract(leaf, "\\d*:") %>% str_replace(":", "") %>% addTreeId(treeID)
+    featureBranch <- str_extract(branch, "f\\d*<") %>% str_replace("<", "") %>% str_replace("f", "") %>% as.numeric 
+    if(!is.null(feature_names)){
+      featureBranch <- feature_names[featureBranch + 1]
+    }
+    featureLeaf <- rep("Leaf", length(leaf))
+    splitBranch <- str_extract(branch, paste0("<",anynumber_regex,"\\]")) %>% str_replace("<", "") %>% str_replace("\\]", "") 
+    splitLeaf <- rep(NA, length(leaf)) 
+    yesBranch <- extract(branch, "yes=\\d*") %>% addTreeId(treeID)
+    yesLeaf <- rep(NA, length(leaf)) 
+    noBranch <- extract(branch, "no=\\d*") %>% addTreeId(treeID)
+    noLeaf <- rep(NA, length(leaf))
+    missingBranch <- extract(branch, "missing=\\d+") %>% addTreeId(treeID)
+    missingLeaf <- rep(NA, length(leaf))
+    qualityBranch <- extract(branch, paste0("gain=",anynumber_regex))
+    qualityLeaf <- extract(leaf, paste0("leaf=",anynumber_regex))
+    coverBranch <- extract(branch, "cover=\\d*\\.*\\d*")
+    coverLeaf <- extract(leaf, "cover=\\d*\\.*\\d*")
+    dt <- data.table(ID = c(idBranch, idLeaf), Feature = c(featureBranch, featureLeaf), Split = c(splitBranch, splitLeaf), Yes = c(yesBranch, yesLeaf), No = c(noBranch, noLeaf), Missing = c(missingBranch, missingLeaf), Quality = c(qualityBranch, qualityLeaf), Cover = c(coverBranch, coverLeaf))[order(ID)][,Tree:=treeID]
+    
+    allTrees <- rbindlist(list(allTrees, dt), use.names = T, fill = F)
+  }
+  
+  yes <- allTrees[!is.na(Yes),Yes]
+                                                                                      
+  set(allTrees, i = which(allTrees[,Feature]!= "Leaf"), 
+      j = "Yes.Feature", 
+      value = allTrees[ID == yes,Feature])
+
+  set(allTrees, i = which(allTrees[,Feature]!= "Leaf"), 
+      j = "Yes.Cover", 
+      value = allTrees[ID == yes,Cover])
+
+  set(allTrees, i = which(allTrees[,Feature]!= "Leaf"), 
+    j = "Yes.Quality", 
+    value = allTrees[ID == yes,Quality])
+  
+  no <- allTrees[!is.na(No),No]
+  
+  set(allTrees, i = which(allTrees[,Feature]!= "Leaf"), 
+      j = "No.Feature", 
+      value = allTrees[ID == no,Feature])
+  
+  set(allTrees, i = which(allTrees[,Feature]!= "Leaf"), 
+      j = "No.Cover", 
+      value = allTrees[ID == no,Cover])
+  
+  set(allTrees, i = which(allTrees[,Feature]!= "Leaf"), 
+      j = "No.Quality", 
+      value = allTrees[ID == no,Quality])
+        
+  allTrees
+}
+
+# Avoid error messages during CRAN check.
+# The reason is that these variables are never declared
+# They are mainly column names inferred by Data.table...
+globalVariables(c("ID", "Tree", "Yes", ".", ".N", "Feature", "Cover", "Quality", "No", "Gain", "Frequence"))

R-package/R/xgb.plot.importance.R

@@ -0,0 +1,57 @@
+#' Plot feature importance bar graph
+#' 
+#' Read a data.table containing feature importance details and plot it.
+#' 
+#' @importFrom magrittr %>%
+#' @param importance_matrix a \code{data.table} returned by the \code{xgb.importance} function.
+#' @param numberOfClusters a \code{numeric} vector containing the min and the max range of the possible number of clusters of bars.
+#'
+#' @return A \code{ggplot2} bar graph representing each feature by a horizontal bar. Longer is the bar, more important is the feature. Features are classified by importance and clustered by importance. The group is represented through the color of the bar.
+#'
+#' @details 
+#' The purpose of this function is to easily represent the importance of each feature of a model.
+#' The function return a ggplot graph, therefore each of its characteristic can be overriden (to customize it).
+#' In particular you may want to override the title of the graph. To do so, add \code{+ ggtitle("A GRAPH NAME")} next to the value returned by this function. 
+#'   
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' 
+#' #Both dataset are list with two items, a sparse matrix and labels 
+#' #(labels = outcome column which will be learned). 
+#' #Each column of the sparse Matrix is a feature in one hot encoding format.
+#' train <- agaricus.train
+#' 
+#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
+#'                eta = 1, nthread = 2, nround = 2,objective = "binary:logistic")
+#' 
+#' #train$data@@Dimnames[[2]] represents the column names of the sparse matrix.
+#' importance_matrix <- xgb.importance(train$data@@Dimnames[[2]], model = bst)
+#' xgb.plot.importance(importance_matrix)
+#' 
+#' @export
+xgb.plot.importance <- function(importance_matrix = NULL, numberOfClusters = c(1:10)){
+  if (!"data.table" %in% class(importance_matrix))  {     
+    stop("importance_matrix: Should be a data.table.")
+  }
+  if (!require(ggplot2, quietly = TRUE)) {
+    stop("ggplot2 package is required for plotting the importance", call. = FALSE)
+  }
+  if (!requireNamespace("Ckmeans.1d.dp", quietly = TRUE)) {
+    stop("Ckmeans.1d.dp package is required for plotting the importance", call. = FALSE)
+  }
+
+  # To avoid issues in clustering when co-occurences are used
+  importance_matrix <- importance_matrix[, .(Gain = sum(Gain)), by = Feature]
+  
+  clusters <- suppressWarnings(Ckmeans.1d.dp::Ckmeans.1d.dp(importance_matrix[,Gain], numberOfClusters))
+  importance_matrix[,"Cluster":=clusters$cluster %>% as.character]
+    
+  plot <- ggplot(importance_matrix, aes(x=reorder(Feature, Gain), y = Gain, width= 0.05), environment = environment())+  geom_bar(aes(fill=Cluster), stat="identity", position="identity") + coord_flip() + xlab("Features") + ylab("Gain") + ggtitle("Feature importance") + theme(plot.title = element_text(lineheight=.9, face="bold"), panel.grid.major.y = element_blank() )
+  
+  return(plot)  
+}
+
+# Avoid error messages during CRAN check.
+# The reason is that these variables are never declared
+# They are mainly column names inferred by Data.table...
+globalVariables(c("Feature", "Gain", "Cluster", "ggplot", "aes", "geom_bar", "coord_flip", "xlab", "ylab", "ggtitle", "theme", "element_blank", "element_text"))

R-package/R/xgb.plot.tree.R

@@ -0,0 +1,97 @@
+#' Plot a boosted tree model
+#' 
+#' Read a tree model text dump. 
+#' Plotting only works for boosted tree model (not linear model).
+#' 
+#' @importFrom data.table data.table
+#' @importFrom data.table set
+#' @importFrom data.table rbindlist
+#' @importFrom data.table :=
+#' @importFrom data.table copy
+#' @importFrom magrittr %>%
+#' @importFrom magrittr not
+#' @importFrom magrittr add
+#' @importFrom stringr str_extract
+#' @importFrom stringr str_split
+#' @importFrom stringr str_extract
+#' @importFrom stringr str_trim
+#' @param feature_names names of each feature as a character vector. Can be extracted from a sparse matrix (see example). If model dump already contains feature names, this argument should be \code{NULL}.
+#' @param filename_dump the path to the text file storing the model. Model dump must include the gain per feature and per tree (parameter \code{with.stats = T} in function \code{xgb.dump}). Possible to provide a model directly (see \code{model} argument).
+#' @param model generated by the \code{xgb.train} function. Avoid the creation of a dump file.
+#' @param n_first_tree limit the plot to the n first trees. If \code{NULL}, all trees of the model are plotted. Performance can be low for huge models.
+#' @param CSSstyle a \code{character} vector storing a css style to customize the appearance of nodes. Look at the \href{https://github.com/knsv/mermaid/wiki}{Mermaid wiki} for more information.
+#' @param  width  the width of the diagram in pixels.
+#' @param height	the height of the diagram in pixels.
+#'
+#' @return A \code{DiagrammeR} of the model.
+#'
+#' @details 
+#' 
+#' The content of each node is organised that way:
+#' 
+#' \itemize{
+#'  \item \code{feature} value ;
+#'  \item \code{cover}: the sum of second order gradient of training data classified to the leaf, if it is square loss, this simply corresponds to the number of instances in that branch. Deeper in the tree a node is, lower this metric will be ;
+#'  \item \code{gain}: metric the importance of the node in the model.
+#' } 
+#' 
+#' Each branch finishes with a leaf. For each leaf, only the \code{cover} is indicated.
+#' It uses \href{https://github.com/knsv/mermaid/}{Mermaid} library for that purpose.
+#'  
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' 
+#' #Both dataset are list with two items, a sparse matrix and labels 
+#' #(labels = outcome column which will be learned). 
+#' #Each column of the sparse Matrix is a feature in one hot encoding format.
+#' train <- agaricus.train
+#' 
+#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
+#'                eta = 1, nthread = 2, nround = 2,objective = "binary:logistic")
+#' 
+#' #agaricus.test$data@@Dimnames[[2]] represents the column names of the sparse matrix.
+#' xgb.plot.tree(agaricus.train$data@@Dimnames[[2]], model = bst)
+#' 
+#' @export
+#' 
+xgb.plot.tree <- function(feature_names = NULL, filename_dump = NULL, model = NULL, n_first_tree = NULL, CSSstyle = NULL, width = NULL, height = NULL){  
+  
+  if (!(class(CSSstyle) %in% c("character", "NULL") && length(CSSstyle) <= 1)) {
+    stop("style: Has to be a character vector of size 1.")
+  }
+  
+  if (!class(model) %in% c("xgb.Booster", "NULL")) {
+    stop("model: Has to be an object of class xgb.Booster model generaged by the xgb.train function.")
+  }
+  
+  if (!requireNamespace("DiagrammeR", quietly = TRUE)) {
+    stop("DiagrammeR package is required for xgb.plot.tree", call. = FALSE)
+  }
+  
+  if(is.null(model)){
+    allTrees <- xgb.model.dt.tree(feature_names = feature_names, filename_dump = filename_dump, n_first_tree = n_first_tree)  
+  } else {
+    allTrees <- xgb.model.dt.tree(feature_names = feature_names, model = model, n_first_tree = n_first_tree)  
+  }
+  
+  allTrees[Feature!="Leaf" ,yesPath:= paste(ID,"(", Feature, "<br/>Cover: ", Cover, "<br/>Gain: ", Quality, ")-->|< ", Split, "|", Yes, ">", Yes.Feature, "]", sep = "")]
+  
+  allTrees[Feature!="Leaf" ,noPath:= paste(ID,"(", Feature, ")-->|>= ", Split, "|", No, ">", No.Feature, "]", sep = "")]
+  
+  
+  if(is.null(CSSstyle)){
+    CSSstyle <- "classDef greenNode fill:#A2EB86, stroke:#04C4AB, stroke-width:2px;classDef redNode fill:#FFA070, stroke:#FF5E5E, stroke-width:2px"  
+  }  
+  
+  yes <- allTrees[Feature!="Leaf", c(Yes)] %>% paste(collapse = ",") %>% paste("class ", ., " greenNode", sep = "")
+  
+  no <- allTrees[Feature!="Leaf", c(No)] %>% paste(collapse = ",") %>% paste("class ", ., " redNode", sep = "")
+  
+  path <- allTrees[Feature!="Leaf", c(yesPath, noPath)] %>% .[order(.)] %>% paste(sep = "", collapse = ";") %>% paste("graph LR", .,collapse = "", sep = ";") %>% paste(CSSstyle, yes, no, sep = ";")
+  DiagrammeR::mermaid(path, width, height)
+}
+
+# Avoid error messages during CRAN check.
+# The reason is that these variables are never declared
+# They are mainly column names inferred by Data.table...
+globalVariables(c("Feature", "yesPath", "ID", "Cover", "Quality", "Split", "Yes", "Yes.Feature", "noPath", "No", "No.Feature", "."))

R-package/R/xgb.save.R

@@ -0,0 +1,32 @@
+#' Save xgboost model to binary file
+#' 
+#' Save xgboost model from xgboost or xgb.train
+#' 
+#' @param model the model object.
+#' @param fname the name of the binary file.
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' data(agaricus.test, package='xgboost')
+#' train <- agaricus.train
+#' test <- agaricus.test
+#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
+#'                eta = 1, nthread = 2, nround = 2,objective = "binary:logistic")
+#' xgb.save(bst, 'xgb.model')
+#' bst <- xgb.load('xgb.model')
+#' pred <- predict(bst, test$data)
+#' @export
+#' 
+xgb.save <- function(model, fname) {
+  if (typeof(fname) != "character") {
+    stop("xgb.save: fname must be character")
+  }
+  if (class(model) == "xgb.Booster") {
+    model <- xgb.Booster.check(model)
+    .Call("XGBoosterSaveModel_R", model$handle, fname, PACKAGE = "xgboost")
+    return(TRUE)
+  }
+  stop("xgb.save: the input must be xgb.Booster. Use xgb.DMatrix.save to save
+       xgb.DMatrix object.")
+  return(FALSE)
+} 

R-package/R/xgb.save.raw.R

@@ -0,0 +1,30 @@
+#' Save xgboost model to R's raw vector,
+#' user can call xgb.load to load the model back from raw vector
+#' 
+#' Save xgboost model from xgboost or xgb.train
+#' 
+#' @param model the model object.
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' data(agaricus.test, package='xgboost')
+#' train <- agaricus.train
+#' test <- agaricus.test
+#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
+#'                eta = 1, nthread = 2, nround = 2,objective = "binary:logistic")
+#' raw <- xgb.save.raw(bst)
+#' bst <- xgb.load(raw)
+#' pred <- predict(bst, test$data)
+#' @export
+#' 
+xgb.save.raw <- function(model) {
+  if (class(model) == "xgb.Booster"){
+    model <- model$handle
+  }
+  if (class(model) == "xgb.Booster.handle") {
+    raw <- .Call("XGBoosterModelToRaw_R", model, PACKAGE = "xgboost")
+    return(raw)
+  }
+  stop("xgb.raw: the input must be xgb.Booster.handle. Use xgb.DMatrix.save to save
+       xgb.DMatrix object.")
+}

R-package/R/xgb.train.R

@@ -0,0 +1,208 @@
+#' eXtreme Gradient Boosting Training
+#' 
+#' An advanced interface for training xgboost model. Look at \code{\link{xgboost}} function for a simpler interface.
+#'
+#' @param params the list of parameters. 
+#' 
+#' 1. General Parameters
+#' 
+#' \itemize{
+#'   \item \code{booster} which booster to use, can be \code{gbtree} or \code{gblinear}. Default: \code{gbtree}
+#'   \item \code{silent} 0 means printing running messages, 1 means silent mode. Default: 0
+#' }
+#'  
+#' 2. Booster Parameters
+#' 
+#' 2.1. Parameter for Tree Booster
+#' 
+#' \itemize{
+#'   \item \code{eta} control the learning rate: scale the contribution of each tree by a factor of \code{0 < eta < 1} when it is added to the current approximation. Used to prevent overfitting by making the boosting process more conservative. Lower value for \code{eta} implies larger value for \code{nrounds}: low \code{eta} value means model more robust to overfitting but slower to compute. Default: 0.3
+#'   \item \code{gamma} minimum loss reduction required to make a further partition on a leaf node of the tree. the larger, the more conservative the algorithm will be. 
+#'   \item \code{max_depth} maximum depth of a tree. Default: 6
+#'   \item \code{min_child_weight} minimum sum of instance weight(hessian) needed in a child. If the tree partition step results in a leaf node with the sum of instance weight less than min_child_weight, then the building process will give up further partitioning. In linear regression mode, this simply corresponds to minimum number of instances needed to be in each node. The larger, the more conservative the algorithm will be. Default: 1
+#'   \item \code{subsample} subsample ratio of the training instance. Setting it to 0.5 means that xgboost randomly collected half of the data instances to grow trees and this will prevent overfitting. It makes computation shorter (because less data to analyse). It is advised to use this parameter with \code{eta} and increase \code{nround}. Default: 1 
+#'   \item \code{colsample_bytree} subsample ratio of columns when constructing each tree. Default: 1
+#'   \item \code{num_parallel_tree} Experimental parameter. number of trees to grow per round. Useful to test Random Forest through Xgboost (set \code{colsample_bytree < 1}, \code{subsample  < 1}  and \code{round = 1}) accordingly. Default: 1
+#' }
+#' 
+#' 2.2. Parameter for Linear Booster
+#'  
+#' \itemize{
+#'   \item \code{lambda} L2 regularization term on weights. Default: 0
+#'   \item \code{lambda_bias} L2 regularization term on bias. Default: 0
+#'   \item \code{alpha} L1 regularization term on weights. (there is no L1 reg on bias because it is not important). Default: 0
+#' }
+#' 
+#' 3. Task Parameters 
+#' 
+#' \itemize{
+#' \item \code{objective} specify the learning task and the corresponding learning objective, and the objective options are below:
+#'   \itemize{
+#'     \item \code{reg:linear} linear regression (Default).
+#'     \item \code{reg:logistic} logistic regression.
+#'     \item \code{binary:logistic} logistic regression for binary classification. Output probability.
+#'     \item \code{binary:logitraw} logistic regression for binary classification, output score before logistic transformation.
+#'     \item \code{num_class} set the number of classes. To use only with multiclass objectives.
+#'     \item \code{multi:softmax} set xgboost to do multiclass classification using the softmax objective. Class is represented by a number and should be from 0 to \code{tonum_class}.
+#'     \item \code{multi:softprob} same as softmax, but output a vector of ndata * nclass, which can be further reshaped to ndata, nclass matrix. The result contains predicted probabilities of each data point belonging to each class.
+#'     \item \code{rank:pairwise} set xgboost to do ranking task by minimizing the pairwise loss.
+#'   }
+#'   \item \code{base_score} the initial prediction score of all instances, global bias. Default: 0.5
+#'   \item \code{eval_metric} evaluation metrics for validation data. Default: metric will be assigned according to objective(rmse for regression, and error for classification, mean average precision for ranking). List is provided in detail section.
+#' }
+#' 
+#' @param data takes an \code{xgb.DMatrix} as the input.
+#' @param nrounds the max number of iterations
+#' @param watchlist what information should be printed when \code{verbose=1} or
+#'   \code{verbose=2}. Watchlist is used to specify validation set monitoring
+#'   during training. For example user can specify
+#'    watchlist=list(validation1=mat1, validation2=mat2) to watch
+#'    the performance of each round's model on mat1 and mat2
+#'
+#' @param obj customized objective function. Returns gradient and second order 
+#'   gradient with given prediction and dtrain, 
+#' @param feval custimized evaluation function. Returns 
+#'   \code{list(metric='metric-name', value='metric-value')} with given 
+#'   prediction and dtrain,
+#' @param verbose If 0, xgboost will stay silent. If 1, xgboost will print 
+#'   information of performance. If 2, xgboost will print information of both
+#' @param printEveryN Print every N progress messages when \code{verbose>0}. Default is 1 which means all messages are printed.
+#' @param early_stop_round If \code{NULL}, the early stopping function is not triggered. 
+#'     If set to an integer \code{k}, training with a validation set will stop if the performance 
+#'     keeps getting worse consecutively for \code{k} rounds.
+#' @param early.stop.round An alternative of \code{early_stop_round}.
+#' @param maximize If \code{feval} and \code{early_stop_round} are set, then \code{maximize} must be set as well.
+#'     \code{maximize=TRUE} means the larger the evaluation score the better.
+#' @param ... other parameters to pass to \code{params}.
+#' 
+#' @details 
+#' This is the training function for \code{xgboost}. 
+#' 
+#' It supports advanced features such as \code{watchlist}, customized objective function (\code{feval}),
+#' therefore it is more flexible than \code{\link{xgboost}} function.
+#'
+#' Parallelization is automatically enabled if \code{OpenMP} is present. 
+#' Number of threads can also be manually specified via \code{nthread} parameter.
+#' 
+#' \code{eval_metric} parameter (not listed above) is set automatically by Xgboost but can be overriden by parameter. Below is provided the list of different metric optimized by Xgboost to help you to understand how it works inside or to use them with the \code{watchlist} parameter.
+#'   \itemize{
+#'      \item \code{rmse} root mean square error. \url{http://en.wikipedia.org/wiki/Root_mean_square_error}
+#'      \item \code{logloss} negative log-likelihood. \url{http://en.wikipedia.org/wiki/Log-likelihood}
+#'      \item \code{error} Binary classification error rate. It is calculated as \code{(wrong cases) / (all cases)}. For the predictions, the evaluation will regard the instances with prediction value larger than 0.5 as positive instances, and the others as negative instances.
+#'      \item \code{merror} Multiclass classification error rate. It is calculated as \code{(wrong cases) / (all cases)}.
+#'      \item \code{auc} Area under the curve. \url{http://en.wikipedia.org/wiki/Receiver_operating_characteristic#'Area_under_curve} for ranking evaluation.
+#'      \item \code{ndcg} Normalized Discounted Cumulative Gain (for ranking task). \url{http://en.wikipedia.org/wiki/NDCG}
+#'   }
+#'   
+#' Full list of parameters is available in the Wiki \url{https://github.com/dmlc/xgboost/wiki/Parameters}.
+#' 
+#' This function only accepts an \code{\link{xgb.DMatrix}} object as the input.
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+#' dtest <- dtrain
+#' watchlist <- list(eval = dtest, train = dtrain)
+#' param <- list(max.depth = 2, eta = 1, silent = 1)
+#' logregobj <- function(preds, dtrain) {
+#'    labels <- getinfo(dtrain, "label")
+#'    preds <- 1/(1 + exp(-preds))
+#'    grad <- preds - labels
+#'    hess <- preds * (1 - preds)
+#'    return(list(grad = grad, hess = hess))
+#' }
+#' evalerror <- function(preds, dtrain) {
+#'   labels <- getinfo(dtrain, "label")
+#'   err <- as.numeric(sum(labels != (preds > 0)))/length(labels)
+#'   return(list(metric = "error", value = err))
+#' }
+#' bst <- xgb.train(param, dtrain, nthread = 2, nround = 2, watchlist, logregobj, evalerror)
+#' @export
+#' 
+xgb.train <- function(params=list(), data, nrounds, watchlist = list(), 
+                      obj = NULL, feval = NULL, verbose = 1, printEveryN=1L,
+                      early_stop_round = NULL, early.stop.round = NULL,
+                      maximize = NULL, ...) {
+  dtrain <- data
+  if (typeof(params) != "list") {
+    stop("xgb.train: first argument params must be list")
+  }
+  if (class(dtrain) != "xgb.DMatrix") {
+    stop("xgb.train: second argument dtrain must be xgb.DMatrix")
+  }
+  if (verbose > 1) {
+    params <- append(params, list(silent = 0))
+  } else {
+    params <- append(params, list(silent = 1))
+  }
+  if (length(watchlist) != 0 && verbose == 0) {
+    warning('watchlist is provided but verbose=0, no evaluation information will be printed')
+    watchlist <- list()
+  }
+  params = append(params, list(...))
+  
+  # Early stopping
+  if (is.null(early_stop_round) && !is.null(early.stop.round))
+    early_stop_round = early.stop.round
+  if (!is.null(early_stop_round)){
+    if (!is.null(feval) && is.null(maximize))
+      stop('Please set maximize to note whether the model is maximizing the evaluation or not.')
+    if (length(watchlist) == 0)
+      stop('For early stopping you need at least one set in watchlist.')
+    if (is.null(maximize) && is.null(params$eval_metric))
+      stop('Please set maximize to note whether the model is maximizing the evaluation or not.')
+    if (is.null(maximize))
+    {
+      if (params$eval_metric %in% c('rmse','logloss','error','merror','mlogloss')) {
+        maximize = FALSE
+      } else {
+        maximize = TRUE
+      }
+    }
+    
+    if (maximize) {
+      bestScore = 0
+    } else {
+      bestScore = Inf
+    }
+    bestInd = 0
+    earlyStopflag = FALSE
+    
+    if (length(watchlist)>1)
+      warning('Only the first data set in watchlist is used for early stopping process.')
+  }
+  
+  
+  handle <- xgb.Booster(params, append(watchlist, dtrain))
+  bst <- xgb.handleToBooster(handle)
+  printEveryN=max( as.integer(printEveryN), 1L)
+  for (i in 1:nrounds) {
+    succ <- xgb.iter.update(bst$handle, dtrain, i - 1, obj)
+    if (length(watchlist) != 0) {
+      msg <- xgb.iter.eval(bst$handle, watchlist, i - 1, feval)
+      if (0== ( (i-1) %% printEveryN))
+	    cat(paste(msg, "\n", sep=""))
+      if (!is.null(early_stop_round))
+      {
+        score = strsplit(msg,':|\\s+')[[1]][3]
+        score = as.numeric(score)
+        if ((maximize && score>bestScore) || (!maximize && score<bestScore)) {
+          bestScore = score
+          bestInd = i
+        } else {
+          if (i-bestInd>=early_stop_round) {
+            earlyStopflag = TRUE
+            cat('Stopping. Best iteration:',bestInd)
+            break
+          }
+        }
+      }
+    }
+  }
+  bst <- xgb.Booster.check(bst)
+  if (!is.null(early_stop_round)) {
+    bst$bestScore = bestScore
+    bst$bestInd = bestInd
+  }
+  return(bst)
+} 

R-package/R/xgboost.R

@@ -0,0 +1,137 @@
+#' eXtreme Gradient Boosting (Tree) library
+#' 
+#' A simple interface for training xgboost model. Look at \code{\link{xgb.train}} function for a more advanced interface.
+#' 
+#' @param data takes \code{matrix}, \code{dgCMatrix}, local data file or 
+#'   \code{xgb.DMatrix}. 
+#' @param label the response variable. User should not set this field,
+#'    if data is local data file or  \code{xgb.DMatrix}. 
+#' @param params the list of parameters.
+#' 
+#' Commonly used ones are:
+#' \itemize{
+#'   \item \code{objective} objective function, common ones are
+#'   \itemize{
+#'     \item \code{reg:linear} linear regression
+#'     \item \code{binary:logistic} logistic regression for classification
+#'   }
+#'   \item \code{eta} step size of each boosting step
+#'   \item \code{max.depth} maximum depth of the tree
+#'   \item \code{nthread} number of thread used in training, if not set, all threads are used
+#' }
+#'   
+#'   Look at \code{\link{xgb.train}} for a more complete list of parameters or \url{https://github.com/dmlc/xgboost/wiki/Parameters} for the full list.
+#'   
+#'   See also \code{demo/} for walkthrough example in R.
+#' 
+#' @param nrounds the max number of iterations
+#' @param verbose If 0, xgboost will stay silent. If 1, xgboost will print 
+#'   information of performance. If 2, xgboost will print information of both
+#'   performance and construction progress information
+#' @param printEveryN Print every N progress messages when \code{verbose>0}. Default is 1 which means all messages are printed.
+#' @param missing Missing is only used when input is dense matrix, pick a float 
+#'     value that represents missing value. Sometimes a data use 0 or other extreme value to represents missing values.
+#' @param early_stop_round If \code{NULL}, the early stopping function is not triggered. 
+#'     If set to an integer \code{k}, training with a validation set will stop if the performance 
+#'     keeps getting worse consecutively for \code{k} rounds.
+#' @param early.stop.round An alternative of \code{early_stop_round}.
+#' @param maximize If \code{feval} and \code{early_stop_round} are set, then \code{maximize} must be set as well.
+#'     \code{maximize=TRUE} means the larger the evaluation score the better.
+#' @param ... other parameters to pass to \code{params}.
+#' 
+#' @details 
+#' This is the modeling function for Xgboost.
+#' 
+#' Parallelization is automatically enabled if \code{OpenMP} is present.
+#' 
+#' Number of threads can also be manually specified via \code{nthread} parameter.
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' data(agaricus.test, package='xgboost')
+#' train <- agaricus.train
+#' test <- agaricus.test
+#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
+#'                eta = 1, nthread = 2, nround = 2, objective = "binary:logistic")
+#' pred <- predict(bst, test$data)
+#' 
+#' @export
+#' 
+xgboost <- function(data = NULL, label = NULL, missing = NULL, params = list(), nrounds, 
+                    verbose = 1, printEveryN=1L, early_stop_round = NULL, early.stop.round = NULL,
+                    maximize = NULL, ...) {
+  if (is.null(missing)) {
+    dtrain <- xgb.get.DMatrix(data, label)
+  } else {
+    dtrain <- xgb.get.DMatrix(data, label, missing)
+  }
+    
+  params <- append(params, list(...))
+  
+  if (verbose > 0) {
+    watchlist <- list(train = dtrain)
+  } else {
+    watchlist <- list()
+  }
+  
+  bst <- xgb.train(params, dtrain, nrounds, watchlist, verbose = verbose, printEveryN=printEveryN,
+                   early_stop_round = early_stop_round,
+                   early.stop.round = early.stop.round)
+  
+  return(bst)
+} 
+
+
+#' Training part from Mushroom Data Set
+#' 
+#' This data set is originally from the Mushroom data set,
+#' UCI Machine Learning Repository.
+#' 
+#' This data set includes the following fields:
+#' 
+#' \itemize{
+#'  \item \code{label} the label for each record
+#'  \item \code{data} a sparse Matrix of \code{dgCMatrix} class, with 126 columns.
+#' }
+#'
+#' @references
+#' https://archive.ics.uci.edu/ml/datasets/Mushroom
+#' 
+#' Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository 
+#' [http://archive.ics.uci.edu/ml]. Irvine, CA: University of California, 
+#' School of Information and Computer Science.
+#' 
+#' @docType data
+#' @keywords datasets
+#' @name agaricus.train
+#' @usage data(agaricus.train)
+#' @format A list containing a label vector, and a dgCMatrix object with 6513 
+#' rows and 127 variables
+NULL
+
+#' Test part from Mushroom Data Set
+#'
+#' This data set is originally from the Mushroom data set,
+#' UCI Machine Learning Repository.
+#' 
+#' This data set includes the following fields:
+#' 
+#' \itemize{
+#'  \item \code{label} the label for each record
+#'  \item \code{data} a sparse Matrix of \code{dgCMatrix} class, with 126 columns.
+#' }
+#'
+#' @references
+#' https://archive.ics.uci.edu/ml/datasets/Mushroom
+#' 
+#' Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository 
+#' [http://archive.ics.uci.edu/ml]. Irvine, CA: University of California, 
+#' School of Information and Computer Science.
+#' 
+#' @docType data
+#' @keywords datasets
+#' @name agaricus.test
+#' @usage data(agaricus.test)
+#' @format A list containing a label vector, and a dgCMatrix object with 1611 
+#' rows and 126 variables
+NULL

R-package/README.md

@@ -0,0 +1,20 @@
+# R package for xgboost.
+
+## Installation
+
+For up-to-date version (which is recommended), please install from github. Windows user will need to install [RTools](http://cran.r-project.org/bin/windows/Rtools/) first.
+
+```r
+devtools::install_github('dmlc/xgboost',subdir='R-package')
+```
+
+For stable version on CRAN, please run
+
+```r
+install.packages('xgboost')
+```
+
+## Examples
+
+* Please visit [walk through example](demo).
+* See also the [example scripts](../demo/kaggle-higgs) for Kaggle Higgs Challenge, including [speedtest script](../demo/kaggle-higgs/speedtest.R) on this dataset and the one related to [Otto challenge](../demo/kaggle-otto), including a [RMarkdown documentation](../demo/kaggle-otto/understandingXGBoostModel.Rmd).

R-package/demo/00Index

@@ -0,0 +1,10 @@
+basic_walkthrough               Basic feature walkthrough
+custom_objective                Cutomize loss function, and evaluation metric
+boost_from_prediction           Boosting from existing prediction
+predict_first_ntree             Predicting using first n trees
+generalized_linear_model        Generalized Linear Model
+cross_validation                Cross validation
+create_sparse_matrix            Create Sparse Matrix
+predict_leaf_indices            Predicting the corresponding leaves
+early_stopping                  Early Stop in training
+poisson_regression              Poisson Regression on count data

R-package/demo/README.md

@@ -0,0 +1,18 @@
+XGBoost R Feature Walkthrough
+====
+* [Basic walkthrough of wrappers](basic_walkthrough.R) 
+* [Cutomize loss function, and evaluation metric](custom_objective.R)
+* [Boosting from existing prediction](boost_from_prediction.R)
+* [Predicting using first n trees](predict_first_ntree.R)
+* [Generalized Linear Model](generalized_linear_model.R)
+* [Cross validation](cross_validation.R)
+* [Create a sparse matrix from a dense one](create_sparse_matrix.R)
+
+Benchmarks
+====
+* [Starter script for Kaggle Higgs Boson](../../demo/kaggle-higgs)
+ 
+Notes
+====
+* Contribution of examples, benchmarks is more than welcomed!
+* If you like to share how you use xgboost to solve your problem, send a pull request:)

R-package/demo/basic_walkthrough.R

@@ -0,0 +1,105 @@
+require(xgboost)
+require(methods)
+# we load in the agaricus dataset
+# In this example, we are aiming to predict whether a mushroom can be eated
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+train <- agaricus.train
+test <- agaricus.test
+# the loaded data is stored in sparseMatrix, and label is a numeric vector in {0,1}
+class(train$label)
+class(train$data)
+
+#-------------Basic Training using XGBoost-----------------
+# this is the basic usage of xgboost you can put matrix in data field
+# note: we are puting in sparse matrix here, xgboost naturally handles sparse input
+# use sparse matrix when your feature is sparse(e.g. when you using one-hot encoding vector)
+print("training xgboost with sparseMatrix")
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2, eta = 1, nround = 2,
+               nthread = 2, objective = "binary:logistic")
+# alternatively, you can put in dense matrix, i.e. basic R-matrix
+print("training xgboost with Matrix")
+bst <- xgboost(data = as.matrix(train$data), label = train$label, max.depth = 2, eta = 1, nround = 2,
+               nthread = 2, objective = "binary:logistic")
+
+# you can also put in xgb.DMatrix object, stores label, data and other meta datas needed for advanced features
+print("training xgboost with xgb.DMatrix")
+dtrain <- xgb.DMatrix(data = train$data, label = train$label)
+bst <- xgboost(data = dtrain, max.depth = 2, eta = 1, nround = 2, nthread = 2, 
+               objective = "binary:logistic")
+
+# Verbose = 0,1,2
+print ('train xgboost with verbose 0, no message')
+bst <- xgboost(data = dtrain, max.depth = 2, eta = 1, nround = 2,
+               nthread = 2, objective = "binary:logistic", verbose = 0)
+print ('train xgboost with verbose 1, print evaluation metric')
+bst <- xgboost(data = dtrain, max.depth = 2, eta = 1, nround = 2,
+               nthread = 2, objective = "binary:logistic", verbose = 1)
+print ('train xgboost with verbose 2, also print information about tree')
+bst <- xgboost(data = dtrain, max.depth = 2, eta = 1, nround = 2,
+               nthread = 2, objective = "binary:logistic", verbose = 2)
+
+# you can also specify data as file path to a LibSVM format input
+# since we do not have this file with us, the following line is just for illustration
+# bst <- xgboost(data = 'agaricus.train.svm', max.depth = 2, eta = 1, nround = 2,objective = "binary:logistic")
+
+#--------------------basic prediction using xgboost--------------
+# you can do prediction using the following line
+# you can put in Matrix, sparseMatrix, or xgb.DMatrix 
+pred <- predict(bst, test$data)
+err <- mean(as.numeric(pred > 0.5) != test$label)
+print(paste("test-error=", err))
+
+#-------------------save and load models-------------------------
+# save model to binary local file
+xgb.save(bst, "xgboost.model")
+# load binary model to R
+bst2 <- xgb.load("xgboost.model")
+pred2 <- predict(bst2, test$data)
+# pred2 should be identical to pred
+print(paste("sum(abs(pred2-pred))=", sum(abs(pred2-pred))))
+
+# save model to R's raw vector
+raw = xgb.save.raw(bst)
+# load binary model to R
+bst3 <- xgb.load(raw)
+pred3 <- predict(bst3, test$data)
+# pred2 should be identical to pred
+print(paste("sum(abs(pred3-pred))=", sum(abs(pred2-pred))))
+
+#----------------Advanced features --------------
+# to use advanced features, we need to put data in xgb.DMatrix
+dtrain <- xgb.DMatrix(data = train$data, label=train$label)
+dtest <- xgb.DMatrix(data = test$data, label=test$label)
+#---------------Using watchlist----------------
+# watchlist is a list of xgb.DMatrix, each of them tagged with name
+watchlist <- list(train=dtrain, test=dtest)
+# to train with watchlist, use xgb.train, which contains more advanced features
+# watchlist allows us to monitor the evaluation result on all data in the list 
+print ('train xgboost using xgb.train with watchlist')
+bst <- xgb.train(data=dtrain, max.depth=2, eta=1, nround=2, watchlist=watchlist,
+                 nthread = 2, objective = "binary:logistic")
+# we can change evaluation metrics, or use multiple evaluation metrics
+print ('train xgboost using xgb.train with watchlist, watch logloss and error')
+bst <- xgb.train(data=dtrain, max.depth=2, eta=1, nround=2, watchlist=watchlist,
+                 eval.metric = "error", eval.metric = "logloss",
+                 nthread = 2, objective = "binary:logistic")
+
+# xgb.DMatrix can also be saved using xgb.DMatrix.save
+xgb.DMatrix.save(dtrain, "dtrain.buffer")
+# to load it in, simply call xgb.DMatrix
+dtrain2 <- xgb.DMatrix("dtrain.buffer")
+bst <- xgb.train(data=dtrain2, max.depth=2, eta=1, nround=2, watchlist=watchlist,
+                 nthread = 2, objective = "binary:logistic")
+# information can be extracted from xgb.DMatrix using getinfo
+label = getinfo(dtest, "label")
+pred <- predict(bst, dtest)
+err <- as.numeric(sum(as.integer(pred > 0.5) != label))/length(label)
+print(paste("test-error=", err))
+
+# You can dump the tree you learned using xgb.dump into a text file
+xgb.dump(bst, "dump.raw.txt", with.stats = T)
+
+# Finally, you can check which features are the most important.
+print("Most important features (look at column Gain):")
+print(xgb.importance(feature_names = train$data@Dimnames[[2]], filename_dump = "dump.raw.txt"))

R-package/demo/boost_from_prediction.R

@@ -0,0 +1,26 @@
+require(xgboost)
+# load in the agaricus dataset
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label)
+
+watchlist <- list(eval = dtest, train = dtrain)
+###
+# advanced: start from a initial base prediction
+#
+print('start running example to start from a initial prediction')
+# train xgboost for 1 round
+param <- list(max.depth=2,eta=1,nthread = 2, silent=1,objective='binary:logistic')
+bst <- xgb.train( param, dtrain, 1, watchlist )
+# Note: we need the margin value instead of transformed prediction in set_base_margin
+# do predict with output_margin=TRUE, will always give you margin values before logistic transformation
+ptrain <- predict(bst, dtrain, outputmargin=TRUE)
+ptest  <- predict(bst, dtest, outputmargin=TRUE)
+# set the base_margin property of dtrain and dtest
+# base margin is the base prediction we will boost from
+setinfo(dtrain, "base_margin", ptrain)
+setinfo(dtest, "base_margin", ptest)
+
+print('this is result of boost from initial prediction')
+bst <- xgb.train( param, dtrain, 1, watchlist )

R-package/demo/create_sparse_matrix.R

@@ -0,0 +1,89 @@
+require(xgboost)
+require(Matrix)
+require(data.table)
+if (!require(vcd)) install.packages('vcd') #Available in Cran. Used for its dataset with categorical values.
+
+# According to its documentation, Xgboost works only on numbers.
+# Sometimes the dataset we have to work on have categorical data. 
+# A categorical variable is one which have a fixed number of values. By exemple, if for each observation a variable called "Colour" can have only "red", "blue" or "green" as value, it is a categorical variable.
+#
+# In R, categorical variable is called Factor. 
+# Type ?factor in console for more information.
+#
+# In this demo we will see how to transform a dense dataframe with categorical variables to a sparse matrix before analyzing it in Xgboost.
+# The method we are going to see is usually called "one hot encoding".
+
+#load Arthritis dataset in memory.
+data(Arthritis)
+
+# create a copy of the dataset with data.table package (data.table is 100% compliant with R dataframe but its syntax is a lot more consistent and its performance are really good).
+df <- data.table(Arthritis, keep.rownames = F)
+
+# Let's have a look to the data.table
+cat("Print the dataset\n")
+print(df)
+
+# 2 columns have factor type, one has ordinal type (ordinal variable is a categorical variable with values wich can be ordered, here: None > Some > Marked).
+cat("Structure of the dataset\n")
+str(df)
+
+# Let's add some new categorical features to see if it helps. Of course these feature are highly correlated to the Age feature. Usually it's not a good thing in ML, but Tree algorithms (including boosted trees) are able to select the best features, even in case of highly correlated features.
+
+# For the first feature we create groups of age by rounding the real age. Note that we transform it to factor (categorical data) so the algorithm treat them as independant values.
+df[,AgeDiscret:= as.factor(round(Age/10,0))]
+
+# Here is an even stronger simplification of the real age with an arbitrary split at 30 years old. I choose this value based on nothing. We will see later if simplifying the information based on arbitrary values is a good strategy (I am sure you already have an idea of how well it will work!).
+df[,AgeCat:= as.factor(ifelse(Age > 30, "Old", "Young"))]
+
+# We remove ID as there is nothing to learn from this feature (it will just add some noise as the dataset is small).
+df[,ID:=NULL]
+
+# List the different values for the column Treatment: Placebo, Treated.
+cat("Values of the categorical feature Treatment\n")
+print(levels(df[,Treatment]))
+
+# Next step, we will transform the categorical data to dummy variables.
+# This method is also called one hot encoding.
+# The purpose is to transform each value of each categorical feature in one binary feature.
+#
+# Let's take, the column Treatment will be replaced by two columns, Placebo, and Treated. Each of them will be binary. For example an observation which had the value Placebo in column Treatment before the transformation will have, after the transformation, the value 1 in the new column Placebo and the value 0 in the new column  Treated.
+#
+# Formulae Improved~.-1 used below means transform all categorical features but column Improved to binary values.
+# Column Improved is excluded because it will be our output column, the one we want to predict.
+sparse_matrix = sparse.model.matrix(Improved~.-1, data = df)
+
+cat("Encoding of the sparse Matrix\n")
+print(sparse_matrix)
+
+# Create the output vector (not sparse)
+# 1. Set, for all rows, field in Y column to 0; 
+# 2. set Y to 1 when Improved == Marked; 
+# 3. Return Y column
+output_vector = df[,Y:=0][Improved == "Marked",Y:=1][,Y]
+
+# Following is the same process as other demo
+cat("Learning...\n")
+bst <- xgboost(data = sparse_matrix, label = output_vector, max.depth = 9,
+               eta = 1, nthread = 2, nround = 10,objective = "binary:logistic")
+xgb.dump(bst, 'xgb.model.dump', with.stats = T)
+
+# sparse_matrix@Dimnames[[2]] represents the column names of the sparse matrix.
+importance <- xgb.importance(sparse_matrix@Dimnames[[2]], 'xgb.model.dump')
+print(importance)
+# According to the matrix below, the most important feature in this dataset to predict if the treatment will work is the Age. The second most important feature is having received a placebo or not. The sex is third. Then we see our generated features (AgeDiscret). We can see that their contribution is very low (Gain column).
+
+# Does these results make sense?
+# Let's check some Chi2 between each of these features and the outcome.
+
+print(chisq.test(df$Age, df$Y))
+# Pearson correlation between Age and illness disapearing is 35
+
+print(chisq.test(df$AgeDiscret, df$Y))
+# Our first simplification of Age gives a Pearson correlation of 8.
+
+print(chisq.test(df$AgeCat, df$Y))
+# The perfectly random split I did between young and old at 30 years old have a low correlation of 2. It's a result we may expect as may be in my mind > 30 years is being old (I am 32 and starting feeling old, this may explain that), but  for the illness we are studying, the age to be vulnerable is not the same. Don't let your "gut" lower the quality of your model. In "data science", there is science :-)
+
+# As you can see, in general destroying information by simplying it won't improve your model. Chi2 just demonstrates that. But in more complex cases, creating a new feature based on existing one which makes link with the outcome more obvious may help the algorithm and improve the model. The case studied here is not enough complex to show that. Check Kaggle forum for some challenging datasets.
+# However it's almost always worse when you add some arbitrary rules.
+# Moreover, you can notice that even if we have added some not useful new features highly correlated with other features, the boosting tree algorithm have been able to choose the best one, which in this case is the Age. Linear model may not be that strong in these scenario.

R-package/demo/cross_validation.R

@@ -0,0 +1,51 @@
+require(xgboost)
+# load in the agaricus dataset
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label)
+
+nround <- 2
+param <- list(max.depth=2,eta=1,silent=1,nthread = 2, objective='binary:logistic')
+
+cat('running cross validation\n')
+# do cross validation, this will print result out as
+# [iteration]  metric_name:mean_value+std_value
+# std_value is standard deviation of the metric
+xgb.cv(param, dtrain, nround, nfold=5, metrics={'error'})
+
+cat('running cross validation, disable standard deviation display\n')
+# do cross validation, this will print result out as
+# [iteration]  metric_name:mean_value+std_value
+# std_value is standard deviation of the metric
+xgb.cv(param, dtrain, nround, nfold=5,
+       metrics={'error'}, showsd = FALSE)
+
+###
+# you can also do cross validation with cutomized loss function
+# See custom_objective.R
+##
+print ('running cross validation, with cutomsized loss function')
+
+logregobj <- function(preds, dtrain) {
+  labels <- getinfo(dtrain, "label")
+  preds <- 1/(1 + exp(-preds))
+  grad <- preds - labels
+  hess <- preds * (1 - preds)
+  return(list(grad = grad, hess = hess))
+}
+evalerror <- function(preds, dtrain) {
+  labels <- getinfo(dtrain, "label")
+  err <- as.numeric(sum(labels != (preds > 0)))/length(labels)
+  return(list(metric = "error", value = err))
+}
+
+param <- list(max.depth=2,eta=1,silent=1)
+# train with customized objective
+xgb.cv(param, dtrain, nround, nfold = 5,
+       obj = logregobj, feval=evalerror)
+
+# do cross validation with prediction values for each fold
+res <- xgb.cv(param, dtrain, nround, nfold=5, prediction = TRUE)
+res$dt
+length(res$pred)

R-package/demo/custom_objective.R

@@ -0,0 +1,62 @@
+require(xgboost)
+# load in the agaricus dataset
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label)
+
+# note: for customized objective function, we leave objective as default
+# note: what we are getting is margin value in prediction
+# you must know what you are doing
+param <- list(max.depth=2,eta=1,nthread = 2, silent=1)
+watchlist <- list(eval = dtest, train = dtrain)
+num_round <- 2
+
+# user define objective function, given prediction, return gradient and second order gradient
+# this is loglikelihood loss
+logregobj <- function(preds, dtrain) {
+  labels <- getinfo(dtrain, "label")
+  preds <- 1/(1 + exp(-preds))
+  grad <- preds - labels
+  hess <- preds * (1 - preds)
+  return(list(grad = grad, hess = hess))
+}
+
+# user defined evaluation function, return a pair metric_name, result
+# NOTE: when you do customized loss function, the default prediction value is margin
+# this may make buildin evalution metric not function properly
+# for example, we are doing logistic loss, the prediction is score before logistic transformation
+# the buildin evaluation error assumes input is after logistic transformation
+# Take this in mind when you use the customization, and maybe you need write customized evaluation function
+evalerror <- function(preds, dtrain) {
+  labels <- getinfo(dtrain, "label")
+  err <- as.numeric(sum(labels != (preds > 0)))/length(labels)
+  return(list(metric = "error", value = err))
+}
+print ('start training with user customized objective')
+# training with customized objective, we can also do step by step training
+# simply look at xgboost.py's implementation of train
+bst <- xgb.train(param, dtrain, num_round, watchlist, logregobj, evalerror)
+
+#
+# there can be cases where you want additional information 
+# being considered besides the property of DMatrix you can get by getinfo
+# you can set additional information as attributes if DMatrix
+
+# set label attribute of dtrain to be label, we use label as an example, it can be anything 
+attr(dtrain, 'label') <- getinfo(dtrain, 'label')
+# this is new customized objective, where you can access things you set
+# same thing applies to customized evaluation function
+logregobjattr <- function(preds, dtrain) {
+  # now you can access the attribute in customized function
+  labels <- attr(dtrain, 'label')
+  preds <- 1/(1 + exp(-preds))
+  grad <- preds - labels
+  hess <- preds * (1 - preds)
+  return(list(grad = grad, hess = hess))
+}
+
+print ('start training with user customized objective, with additional attributes in DMatrix')
+# training with customized objective, we can also do step by step training
+# simply look at xgboost.py's implementation of train
+bst <- xgb.train(param, dtrain, num_round, watchlist, logregobjattr, evalerror)

R-package/demo/early_stopping.R

@@ -0,0 +1,39 @@
+require(xgboost)
+# load in the agaricus dataset
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label)
+# note: for customized objective function, we leave objective as default
+# note: what we are getting is margin value in prediction
+# you must know what you are doing
+param <- list(max.depth=2,eta=1,nthread = 2, silent=1)
+watchlist <- list(eval = dtest)
+num_round <- 20
+# user define objective function, given prediction, return gradient and second order gradient
+# this is loglikelihood loss
+logregobj <- function(preds, dtrain) {
+  labels <- getinfo(dtrain, "label")
+  preds <- 1/(1 + exp(-preds))
+  grad <- preds - labels
+  hess <- preds * (1 - preds)
+  return(list(grad = grad, hess = hess))
+}
+# user defined evaluation function, return a pair metric_name, result
+# NOTE: when you do customized loss function, the default prediction value is margin
+# this may make buildin evalution metric not function properly
+# for example, we are doing logistic loss, the prediction is score before logistic transformation
+# the buildin evaluation error assumes input is after logistic transformation
+# Take this in mind when you use the customization, and maybe you need write customized evaluation function
+evalerror <- function(preds, dtrain) {
+  labels <- getinfo(dtrain, "label")
+  err <- as.numeric(sum(labels != (preds > 0)))/length(labels)
+  return(list(metric = "error", value = err))
+}
+print ('start training with early Stopping setting')
+# training with customized objective, we can also do step by step training
+# simply look at xgboost.py's implementation of train
+bst <- xgb.train(param, dtrain, num_round, watchlist, logregobj, evalerror, maximize = FALSE,
+                 early.stop.round = 3)
+bst <- xgb.cv(param, dtrain, num_round, nfold=5, obj=logregobj, feval = evalerror,
+              maximize = FALSE, early.stop.round = 3)

R-package/demo/generalized_linear_model.R

@@ -0,0 +1,34 @@
+require(xgboost)
+# load in the agaricus dataset
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label)
+##
+#  this script demonstrate how to fit generalized linear model in xgboost
+#  basically, we are using linear model, instead of tree for our boosters
+#  you can fit a linear regression, or logistic regression model
+##
+
+# change booster to gblinear, so that we are fitting a linear model
+# alpha is the L1 regularizer 
+# lambda is the L2 regularizer
+# you can also set lambda_bias which is L2 regularizer on the bias term
+param <- list(objective = "binary:logistic", booster = "gblinear",
+              nthread = 2, alpha = 0.0001, lambda = 1)
+
+# normally, you do not need to set eta (step_size)
+# XGBoost uses a parallel coordinate descent algorithm (shotgun), 
+# there could be affection on convergence with parallelization on certain cases
+# setting eta to be smaller value, e.g 0.5 can make the optimization more stable
+
+##
+# the rest of settings are the same
+##
+watchlist <- list(eval = dtest, train = dtrain)
+num_round <- 2
+bst <- xgb.train(param, dtrain, num_round, watchlist)
+ypred <- predict(bst, dtest)
+labels <- getinfo(dtest, 'label')
+cat('error of preds=', mean(as.numeric(ypred>0.5)!=labels),'\n')
+

R-package/demo/poisson_regression.R

@@ -0,0 +1,7 @@
+data(mtcars)
+head(mtcars)
+bst = xgboost(data=as.matrix(mtcars[,-11]),label=mtcars[,11],
+              objective='count:poisson',nrounds=5)
+pred = predict(bst,as.matrix(mtcars[,-11]))
+sqrt(mean((pred-mtcars[,11])^2))
+

R-package/demo/predict_first_ntree.R

@@ -0,0 +1,23 @@
+require(xgboost)
+# load in the agaricus dataset
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label)
+
+param <- list(max.depth=2,eta=1,silent=1,objective='binary:logistic')
+watchlist <- list(eval = dtest, train = dtrain)
+nround = 2
+
+# training the model for two rounds
+bst = xgb.train(param, dtrain, nround, nthread = 2, watchlist)
+cat('start testing prediction from first n trees\n')
+labels <- getinfo(dtest,'label')
+
+### predict using first 1 tree
+ypred1 = predict(bst, dtest, ntreelimit=1)
+# by default, we predict using all the trees
+ypred2 = predict(bst, dtest)
+
+cat('error of ypred1=', mean(as.numeric(ypred1>0.5)!=labels),'\n')
+cat('error of ypred2=', mean(as.numeric(ypred2>0.5)!=labels),'\n')

R-package/demo/predict_leaf_indices.R

@@ -0,0 +1,21 @@
+require(xgboost)
+# load in the agaricus dataset
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label)
+
+param <- list(max.depth=2,eta=1,silent=1,objective='binary:logistic')
+watchlist <- list(eval = dtest, train = dtrain)
+nround = 5
+
+# training the model for two rounds
+bst = xgb.train(param, dtrain, nround, nthread = 2, watchlist)
+cat('start testing prediction from first n trees\n')
+
+### predict using first 2 tree
+pred_with_leaf = predict(bst, dtest, ntreelimit = 2, predleaf = TRUE)
+head(pred_with_leaf)
+# by default, we predict using all the trees
+pred_with_leaf = predict(bst, dtest, predleaf = TRUE)
+head(pred_with_leaf)

R-package/demo/runall.R

@@ -0,0 +1,11 @@
+# running all scripts in demo folder
+demo(basic_walkthrough)
+demo(custom_objective)
+demo(boost_from_prediction)
+demo(predict_first_ntree)
+demo(generalized_linear_model)
+demo(cross_validation)
+demo(create_sparse_matrix)
+demo(predict_leaf_indices)
+demo(early_stopping)
+demo(poisson_regression)

R-package/man/agaricus.test.Rd

@@ -0,0 +1,32 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/xgboost.R
+\docType{data}
+\name{agaricus.test}
+\alias{agaricus.test}
+\title{Test part from Mushroom Data Set}
+\format{A list containing a label vector, and a dgCMatrix object with 1611
+rows and 126 variables}
+\usage{
+data(agaricus.test)
+}
+\description{
+This data set is originally from the Mushroom data set,
+UCI Machine Learning Repository.
+}
+\details{
+This data set includes the following fields:
+
+\itemize{
+ \item \code{label} the label for each record
+ \item \code{data} a sparse Matrix of \code{dgCMatrix} class, with 126 columns.
+}
+}
+\references{
+https://archive.ics.uci.edu/ml/datasets/Mushroom
+
+Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository
+[http://archive.ics.uci.edu/ml]. Irvine, CA: University of California,
+School of Information and Computer Science.
+}
+\keyword{datasets}
+

R-package/man/agaricus.train.Rd

@@ -0,0 +1,32 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/xgboost.R
+\docType{data}
+\name{agaricus.train}
+\alias{agaricus.train}
+\title{Training part from Mushroom Data Set}
+\format{A list containing a label vector, and a dgCMatrix object with 6513
+rows and 127 variables}
+\usage{
+data(agaricus.train)
+}
+\description{
+This data set is originally from the Mushroom data set,
+UCI Machine Learning Repository.
+}
+\details{
+This data set includes the following fields:
+
+\itemize{
+ \item \code{label} the label for each record
+ \item \code{data} a sparse Matrix of \code{dgCMatrix} class, with 126 columns.
+}
+}
+\references{
+https://archive.ics.uci.edu/ml/datasets/Mushroom
+
+Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository
+[http://archive.ics.uci.edu/ml]. Irvine, CA: University of California,
+School of Information and Computer Science.
+}
+\keyword{datasets}
+

R-package/man/getinfo.Rd

@@ -0,0 +1,42 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/getinfo.xgb.DMatrix.R
+\docType{methods}
+\name{getinfo}
+\alias{getinfo}
+\alias{getinfo,xgb.DMatrix-method}
+\title{Get information of an xgb.DMatrix object}
+\usage{
+getinfo(object, ...)
+
+\S4method{getinfo}{xgb.DMatrix}(object, name)
+}
+\arguments{
+\item{object}{Object of class \code{xgb.DMatrix}}
+
+\item{...}{other parameters}
+
+\item{name}{the name of the field to get}
+}
+\description{
+Get information of an xgb.DMatrix object
+}
+\details{
+The information can be one of the following:
+
+\itemize{
+    \item \code{label}: label Xgboost learn from ;
+    \item \code{weight}: to do a weight rescale ;
+    \item \code{base_margin}: base margin is the base prediction Xgboost will boost from ;
+    \item \code{nrow}: number of rows of the \code{xgb.DMatrix}.
+}
+}
+\examples{
+data(agaricus.train, package='xgboost')
+train <- agaricus.train
+dtrain <- xgb.DMatrix(train$data, label=train$label)
+labels <- getinfo(dtrain, 'label')
+setinfo(dtrain, 'label', 1-labels)
+labels2 <- getinfo(dtrain, 'label')
+stopifnot(all(labels2 == 1-labels))
+}
+

R-package/man/nrow-xgb.DMatrix-method.Rd

@@ -0,0 +1,22 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/nrow.xgb.DMatrix.R
+\docType{methods}
+\name{nrow,xgb.DMatrix-method}
+\alias{nrow,xgb.DMatrix-method}
+\title{Number of xgb.DMatrix rows}
+\usage{
+\S4method{nrow}{xgb.DMatrix}(x)
+}
+\arguments{
+\item{x}{Object of class \code{xgb.DMatrix}}
+}
+\description{
+\code{nrow} return the number of rows present in the \code{xgb.DMatrix}.
+}
+\examples{
+data(agaricus.train, package='xgboost')
+train <- agaricus.train
+dtrain <- xgb.DMatrix(train$data, label=train$label)
+stopifnot(nrow(dtrain) == nrow(train$data))
+}
+

R-package/man/predict-xgb.Booster-method.Rd

@@ -0,0 +1,43 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/predict.xgb.Booster.R
+\docType{methods}
+\name{predict,xgb.Booster-method}
+\alias{predict,xgb.Booster-method}
+\title{Predict method for eXtreme Gradient Boosting model}
+\usage{
+\S4method{predict}{xgb.Booster}(object, newdata, missing = NULL,
+  outputmargin = FALSE, ntreelimit = NULL, predleaf = FALSE)
+}
+\arguments{
+\item{object}{Object of class "xgb.Boost"}
+
+\item{newdata}{takes \code{matrix}, \code{dgCMatrix}, local data file or
+\code{xgb.DMatrix}.}
+
+\item{missing}{Missing is only used when input is dense matrix, pick a float
+value that represents missing value. Sometime a data use 0 or other extreme value to represents missing values.}
+
+\item{outputmargin}{whether the prediction should be shown in the original
+value of sum of functions, when outputmargin=TRUE, the prediction is
+untransformed margin value. In logistic regression, outputmargin=T will
+output value before logistic transformation.}
+
+\item{ntreelimit}{limit number of trees used in prediction, this parameter is
+only valid for gbtree, but not for gblinear. set it to be value bigger
+than 0. It will use all trees by default.}
+
+\item{predleaf}{whether predict leaf index instead. If set to TRUE, the output will be a matrix object.}
+}
+\description{
+Predicted values based on xgboost model object.
+}
+\examples{
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+train <- agaricus.train
+test <- agaricus.test
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
+               eta = 1, nthread = 2, nround = 2,objective = "binary:logistic")
+pred <- predict(bst, test$data)
+}
+

R-package/man/predict-xgb.Booster.handle-method.Rd

@@ -0,0 +1,18 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/predict.xgb.Booster.handle.R
+\docType{methods}
+\name{predict,xgb.Booster.handle-method}
+\alias{predict,xgb.Booster.handle-method}
+\title{Predict method for eXtreme Gradient Boosting model handle}
+\usage{
+\S4method{predict}{xgb.Booster.handle}(object, ...)
+}
+\arguments{
+\item{object}{Object of class "xgb.Boost.handle"}
+
+\item{...}{Parameters pass to \code{predict.xgb.Booster}}
+}
+\description{
+Predicted values based on xgb.Booster.handle object.
+}
+

R-package/man/setinfo.Rd

@@ -0,0 +1,44 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/setinfo.xgb.DMatrix.R
+\docType{methods}
+\name{setinfo}
+\alias{setinfo}
+\alias{setinfo,xgb.DMatrix-method}
+\title{Set information of an xgb.DMatrix object}
+\usage{
+setinfo(object, ...)
+
+\S4method{setinfo}{xgb.DMatrix}(object, name, info)
+}
+\arguments{
+\item{object}{Object of class "xgb.DMatrix"}
+
+\item{...}{other parameters}
+
+\item{name}{the name of the field to get}
+
+\item{info}{the specific field of information to set}
+}
+\description{
+Set information of an xgb.DMatrix object
+}
+\details{
+It can be one of the following:
+
+\itemize{
+    \item \code{label}: label Xgboost learn from ;
+    \item \code{weight}: to do a weight rescale ;
+    \item \code{base_margin}: base margin is the base prediction Xgboost will boost from ;
+    \item \code{group}.
+}
+}
+\examples{
+data(agaricus.train, package='xgboost')
+train <- agaricus.train
+dtrain <- xgb.DMatrix(train$data, label=train$label)
+labels <- getinfo(dtrain, 'label')
+setinfo(dtrain, 'label', 1-labels)
+labels2 <- getinfo(dtrain, 'label')
+stopifnot(all(labels2 == 1-labels))
+}
+

R-package/man/slice.Rd

@@ -0,0 +1,31 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/slice.xgb.DMatrix.R
+\docType{methods}
+\name{slice}
+\alias{slice}
+\alias{slice,xgb.DMatrix-method}
+\title{Get a new DMatrix containing the specified rows of
+orginal xgb.DMatrix object}
+\usage{
+slice(object, ...)
+
+\S4method{slice}{xgb.DMatrix}(object, idxset, ...)
+}
+\arguments{
+\item{object}{Object of class "xgb.DMatrix"}
+
+\item{...}{other parameters}
+
+\item{idxset}{a integer vector of indices of rows needed}
+}
+\description{
+Get a new DMatrix containing the specified rows of
+orginal xgb.DMatrix object
+}
+\examples{
+data(agaricus.train, package='xgboost')
+train <- agaricus.train
+dtrain <- xgb.DMatrix(train$data, label=train$label)
+dsub <- slice(dtrain, 1:3)
+}
+

R-package/man/xgb.DMatrix.Rd

@@ -0,0 +1,30 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/xgb.DMatrix.R
+\name{xgb.DMatrix}
+\alias{xgb.DMatrix}
+\title{Contruct xgb.DMatrix object}
+\usage{
+xgb.DMatrix(data, info = list(), missing = 0, ...)
+}
+\arguments{
+\item{data}{a \code{matrix} object, a \code{dgCMatrix} object or a character
+indicating the data file.}
+
+\item{info}{a list of information of the xgb.DMatrix object}
+
+\item{missing}{Missing is only used when input is dense matrix, pick a float
+value that represents missing value. Sometime a data use 0 or other extreme value to represents missing values.}
+
+\item{...}{other information to pass to \code{info}.}
+}
+\description{
+Contruct xgb.DMatrix object from dense matrix, sparse matrix or local file.
+}
+\examples{
+data(agaricus.train, package='xgboost')
+train <- agaricus.train
+dtrain <- xgb.DMatrix(train$data, label=train$label)
+xgb.DMatrix.save(dtrain, 'xgb.DMatrix.data')
+dtrain <- xgb.DMatrix('xgb.DMatrix.data')
+}
+

R-package/man/xgb.DMatrix.save.Rd

@@ -0,0 +1,24 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/xgb.DMatrix.save.R
+\name{xgb.DMatrix.save}
+\alias{xgb.DMatrix.save}
+\title{Save xgb.DMatrix object to binary file}
+\usage{
+xgb.DMatrix.save(DMatrix, fname)
+}
+\arguments{
+\item{DMatrix}{the DMatrix object}
+
+\item{fname}{the name of the binary file.}
+}
+\description{
+Save xgb.DMatrix object to binary file
+}
+\examples{
+data(agaricus.train, package='xgboost')
+train <- agaricus.train
+dtrain <- xgb.DMatrix(train$data, label=train$label)
+xgb.DMatrix.save(dtrain, 'xgb.DMatrix.data')
+dtrain <- xgb.DMatrix('xgb.DMatrix.data')
+}
+

R-package/man/xgb.cv.Rd

@@ -0,0 +1,110 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/xgb.cv.R
+\name{xgb.cv}
+\alias{xgb.cv}
+\title{Cross Validation}
+\usage{
+xgb.cv(params = list(), data, nrounds, nfold, label = NULL,
+  missing = NULL, prediction = FALSE, showsd = TRUE, metrics = list(),
+  obj = NULL, feval = NULL, stratified = TRUE, folds = NULL,
+  verbose = T, early_stop_round = NULL, early.stop.round = NULL,
+  maximize = NULL, ...)
+}
+\arguments{
+\item{params}{the list of parameters. Commonly used ones are:
+\itemize{
+  \item \code{objective} objective function, common ones are
+  \itemize{
+    \item \code{reg:linear} linear regression
+    \item \code{binary:logistic} logistic regression for classification
+  }
+  \item \code{eta} step size of each boosting step
+  \item \code{max.depth} maximum depth of the tree
+  \item \code{nthread} number of thread used in training, if not set, all threads are used
+}
+
+  See \link{xgb.train} for further details.
+  See also demo/ for walkthrough example in R.}
+
+\item{data}{takes an \code{xgb.DMatrix} or \code{Matrix} as the input.}
+
+\item{nrounds}{the max number of iterations}
+
+\item{nfold}{the original dataset is randomly partitioned into \code{nfold} equal size subsamples.}
+
+\item{label}{option field, when data is \code{Matrix}}
+
+\item{missing}{Missing is only used when input is dense matrix, pick a float
+value that represents missing value. Sometime a data use 0 or other extreme value to represents missing values.}
+
+\item{prediction}{A logical value indicating whether to return the prediction vector.}
+
+\item{showsd}{\code{boolean}, whether show standard deviation of cross validation}
+
+\item{metrics,}{list of evaluation metrics to be used in corss validation,
+  when it is not specified, the evaluation metric is chosen according to objective function.
+  Possible options are:
+\itemize{
+  \item \code{error} binary classification error rate
+  \item \code{rmse} Rooted mean square error
+  \item \code{logloss} negative log-likelihood function
+  \item \code{auc} Area under curve
+  \item \code{merror} Exact matching error, used to evaluate multi-class classification
+}}
+
+\item{obj}{customized objective function. Returns gradient and second order
+gradient with given prediction and dtrain.}
+
+\item{feval}{custimized evaluation function. Returns
+\code{list(metric='metric-name', value='metric-value')} with given
+prediction and dtrain.}
+
+\item{stratified}{\code{boolean} whether sampling of folds should be stratified by the values of labels in \code{data}}
+
+\item{folds}{\code{list} provides a possibility of using a list of pre-defined CV folds (each element must be a vector of fold's indices).
+If folds are supplied, the nfold and stratified parameters would be ignored.}
+
+\item{verbose}{\code{boolean}, print the statistics during the process}
+
+\item{early_stop_round}{If \code{NULL}, the early stopping function is not triggered.
+If set to an integer \code{k}, training with a validation set will stop if the performance
+keeps getting worse consecutively for \code{k} rounds.}
+
+\item{early.stop.round}{An alternative of \code{early_stop_round}.}
+
+\item{maximize}{If \code{feval} and \code{early_stop_round} are set, then \code{maximize} must be set as well.
+    \code{maximize=TRUE} means the larger the evaluation score the better.}
+
+\item{...}{other parameters to pass to \code{params}.}
+}
+\value{
+If \code{prediction = TRUE}, a list with the following elements is returned:
+\itemize{
+  \item \code{dt} a \code{data.table} with each mean and standard deviation stat for training set and test set
+  \item \code{pred} an array or matrix (for multiclass classification) with predictions for each CV-fold for the model having been trained on the data in all other folds.
+}
+
+If \code{prediction = FALSE}, just a \code{data.table} with each mean and standard deviation stat for training set and test set is returned.
+}
+\description{
+The cross valudation function of xgboost
+}
+\details{
+The original sample is randomly partitioned into \code{nfold} equal size subsamples.
+
+Of the \code{nfold} subsamples, a single subsample is retained as the validation data for testing the model, and the remaining \code{nfold - 1} subsamples are used as training data.
+
+The cross-validation process is then repeated \code{nrounds} times, with each of the \code{nfold} subsamples used exactly once as the validation data.
+
+All observations are used for both training and validation.
+
+Adapted from \url{http://en.wikipedia.org/wiki/Cross-validation_\%28statistics\%29#k-fold_cross-validation}
+}
+\examples{
+data(agaricus.train, package='xgboost')
+dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+history <- xgb.cv(data = dtrain, nround=3, nthread = 2, nfold = 5, metrics=list("rmse","auc"),
+                  max.depth =3, eta = 1, objective = "binary:logistic")
+print(history)
+}
+

R-package/man/xgb.dump.Rd

@@ -0,0 +1,45 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/xgb.dump.R
+\name{xgb.dump}
+\alias{xgb.dump}
+\title{Save xgboost model to text file}
+\usage{
+xgb.dump(model = NULL, fname = NULL, fmap = "", with.stats = FALSE)
+}
+\arguments{
+\item{model}{the model object.}
+
+\item{fname}{the name of the text file where to save the model text dump. If not provided or set to \code{NULL} the function will return the model as a \code{character} vector.}
+
+\item{fmap}{feature map file representing the type of feature.
+Detailed description could be found at
+\url{https://github.com/dmlc/xgboost/wiki/Binary-Classification#dump-model}.
+See demo/ for walkthrough example in R, and
+\url{https://github.com/dmlc/xgboost/blob/master/demo/data/featmap.txt}
+for example Format.}
+
+\item{with.stats}{whether dump statistics of splits
+       When this option is on, the model dump comes with two additional statistics:
+       gain is the approximate loss function gain we get in each split;
+       cover is the sum of second order gradient in each node.}
+}
+\value{
+if fname is not provided or set to \code{NULL} the function will return the model as a \code{character} vector. Otherwise it will return \code{TRUE}.
+}
+\description{
+Save a xgboost model to text file. Could be parsed later.
+}
+\examples{
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+train <- agaricus.train
+test <- agaricus.test
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
+               eta = 1, nthread = 2, nround = 2,objective = "binary:logistic")
+# save the model in file 'xgb.model.dump'
+xgb.dump(bst, 'xgb.model.dump', with.stats = TRUE)
+
+# print the model without saving it to a file
+print(xgb.dump(bst))
+}
+

R-package/man/xgb.importance.Rd

@@ -0,0 +1,70 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/xgb.importance.R
+\name{xgb.importance}
+\alias{xgb.importance}
+\title{Show importance of features in a model}
+\usage{
+xgb.importance(feature_names = NULL, filename_dump = NULL, model = NULL,
+  data = NULL, label = NULL, target = function(x) ((x + label) == 2))
+}
+\arguments{
+\item{feature_names}{names of each feature as a character vector. Can be extracted from a sparse matrix (see example). If model dump already contains feature names, this argument should be \code{NULL}.}
+
+\item{filename_dump}{the path to the text file storing the model. Model dump must include the gain per feature and per tree (\code{with.stats = T} in function \code{xgb.dump}).}
+
+\item{model}{generated by the \code{xgb.train} function. Avoid the creation of a dump file.}
+
+\item{data}{the dataset used for the training step. Will be used with \code{label} parameter for co-occurence computation. More information in \code{Detail} part. This parameter is optional.}
+
+\item{label}{the label vetor used for the training step. Will be used with \code{data} parameter for co-occurence computation. More information in \code{Detail} part. This parameter is optional.}
+
+\item{target}{a function which returns \code{TRUE} or \code{1} when an observation should be count as a co-occurence and \code{FALSE} or \code{0} otherwise. Default function is provided for computing co-occurences in a binary classification. The \code{target} function should have only one parameter. This parameter will be used to provide each important feature vector after having applied the split condition, therefore these vector will be only made of 0 and 1 only, whatever was the information before. More information in \code{Detail} part. This parameter is optional.}
+}
+\value{
+A \code{data.table} of the features used in the model with their average gain (and their weight for boosted tree model) in the model.
+}
+\description{
+Read a xgboost model text dump.
+Can be tree or linear model (text dump of linear model are only supported in dev version of \code{Xgboost} for now).
+}
+\details{
+This is the function to understand the model trained (and through your model, your data).
+
+Results are returned for both linear and tree models.
+
+\code{data.table} is returned by the function.
+There are 3 columns :
+\itemize{
+  \item \code{Features} name of the features as provided in \code{feature_names} or already present in the model dump.
+  \item \code{Gain} contribution of each feature to the model. For boosted tree model, each gain of each feature of each tree is taken into account, then average per feature to give a vision of the entire model. Highest percentage means important feature to predict the \code{label} used for the training ;
+  \item \code{Cover} metric of the number of observation related to this feature (only available for tree models) ;
+  \item \code{Weight} percentage representing the relative number of times a feature have been taken into trees. \code{Gain} should be prefered to search the most important feature. For boosted linear model, this column has no meaning.
+}
+
+Co-occurence count
+------------------
+
+The gain gives you indication about the information of how a feature is important in making a branch of a decision tree more pure. However, with this information only, you can't know if this feature has to be present or not to get a specific classification. In the example code, you may wonder if odor=none should be \code{TRUE} to not eat a mushroom.
+
+Co-occurence computation is here to help in understanding this relation between a predictor and a specific class. It will count how many observations are returned as \code{TRUE} by the \code{target} function (see parameters). When you execute the example below, there are 92 times only over the 3140 observations of the train dataset where a mushroom have no odor and can be eaten safely.
+
+If you need to remember one thing only: until you want to leave us early, don't eat a mushroom which has no odor :-)
+}
+\examples{
+data(agaricus.train, package='xgboost')
+
+# Both dataset are list with two items, a sparse matrix and labels
+# (labels = outcome column which will be learned).
+# Each column of the sparse Matrix is a feature in one hot encoding format.
+train <- agaricus.train
+
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
+               eta = 1, nthread = 2, nround = 2,objective = "binary:logistic")
+
+# train$data@Dimnames[[2]] represents the column names of the sparse matrix.
+xgb.importance(train$data@Dimnames[[2]], model = bst)
+
+# Same thing with co-occurence computation this time
+xgb.importance(train$data@Dimnames[[2]], model = bst, data = train$data, label = train$label)
+}
+

R-package/man/xgb.load.Rd

@@ -0,0 +1,26 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/xgb.load.R
+\name{xgb.load}
+\alias{xgb.load}
+\title{Load xgboost model from binary file}
+\usage{
+xgb.load(modelfile)
+}
+\arguments{
+\item{modelfile}{the name of the binary file.}
+}
+\description{
+Load xgboost model from the binary model file
+}
+\examples{
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+train <- agaricus.train
+test <- agaricus.test
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
+               eta = 1, nthread = 2, nround = 2,objective = "binary:logistic")
+xgb.save(bst, 'xgb.model')
+bst <- xgb.load('xgb.model')
+pred <- predict(bst, test$data)
+}
+

R-package/man/xgb.model.dt.tree.Rd

@@ -0,0 +1,59 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/xgb.model.dt.tree.R
+\name{xgb.model.dt.tree}
+\alias{xgb.model.dt.tree}
+\title{Convert tree model dump to data.table}
+\usage{
+xgb.model.dt.tree(feature_names = NULL, filename_dump = NULL,
+  model = NULL, text = NULL, n_first_tree = NULL)
+}
+\arguments{
+\item{feature_names}{names of each feature as a character vector. Can be extracted from a sparse matrix (see example). If model dump already contains feature names, this argument should be \code{NULL}.}
+
+\item{filename_dump}{the path to the text file storing the model. Model dump must include the gain per feature and per tree (parameter \code{with.stats = T} in function \code{xgb.dump}).}
+
+\item{model}{dump generated by the \code{xgb.train} function. Avoid the creation of a dump file.}
+
+\item{text}{dump generated by the \code{xgb.dump} function. Avoid the creation of a dump file. Model dump must include the gain per feature and per tree (parameter \code{with.stats = T} in function \code{xgb.dump}).}
+
+\item{n_first_tree}{limit the plot to the n first trees. If \code{NULL}, all trees of the model are plotted. Performance can be low for huge models.}
+}
+\value{
+A \code{data.table} of the features used in the model with their gain, cover and few other thing.
+}
+\description{
+Read a tree model text dump and return a data.table.
+}
+\details{
+General function to convert a text dump of tree model to a Matrix. The purpose is to help user to explore the model and get a better understanding of it.
+
+The content of the \code{data.table} is organised that way:
+
+\itemize{
+\item \code{ID}: unique identifier of a node ;
+ \item \code{Feature}: feature used in the tree to operate a split. When Leaf is indicated, it is the end of a branch ;
+ \item \code{Split}: value of the chosen feature where is operated the split ;
+ \item \code{Yes}: ID of the feature for the next node in the branch when the split condition is met ;
+ \item \code{No}: ID of the feature for the next node in the branch when the split condition is not met ;
+ \item \code{Missing}: ID of the feature for the next node in the branch for observation where the feature used for the split are not provided ;
+ \item \code{Quality}: it's the gain related to the split in this specific node ;
+ \item \code{Cover}: metric to measure the number of observation affected by the split ;
+ \item \code{Tree}: ID of the tree. It is included in the main ID ;
+ \item \code{Yes.X} or \code{No.X}: data related to the pointer in \code{Yes} or \code{No} column ;
+}
+}
+\examples{
+data(agaricus.train, package='xgboost')
+
+#Both dataset are list with two items, a sparse matrix and labels
+#(labels = outcome column which will be learned).
+#Each column of the sparse Matrix is a feature in one hot encoding format.
+train <- agaricus.train
+
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
+               eta = 1, nthread = 2, nround = 2,objective = "binary:logistic")
+
+#agaricus.test$data@Dimnames[[2]] represents the column names of the sparse matrix.
+xgb.model.dt.tree(agaricus.train$data@Dimnames[[2]], model = bst)
+}
+

R-package/man/xgb.plot.importance.Rd

@@ -0,0 +1,40 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/xgb.plot.importance.R
+\name{xgb.plot.importance}
+\alias{xgb.plot.importance}
+\title{Plot feature importance bar graph}
+\usage{
+xgb.plot.importance(importance_matrix = NULL, numberOfClusters = c(1:10))
+}
+\arguments{
+\item{importance_matrix}{a \code{data.table} returned by the \code{xgb.importance} function.}
+
+\item{numberOfClusters}{a \code{numeric} vector containing the min and the max range of the possible number of clusters of bars.}
+}
+\value{
+A \code{ggplot2} bar graph representing each feature by a horizontal bar. Longer is the bar, more important is the feature. Features are classified by importance and clustered by importance. The group is represented through the color of the bar.
+}
+\description{
+Read a data.table containing feature importance details and plot it.
+}
+\details{
+The purpose of this function is to easily represent the importance of each feature of a model.
+The function return a ggplot graph, therefore each of its characteristic can be overriden (to customize it).
+In particular you may want to override the title of the graph. To do so, add \code{+ ggtitle("A GRAPH NAME")} next to the value returned by this function.
+}
+\examples{
+data(agaricus.train, package='xgboost')
+
+#Both dataset are list with two items, a sparse matrix and labels
+#(labels = outcome column which will be learned).
+#Each column of the sparse Matrix is a feature in one hot encoding format.
+train <- agaricus.train
+
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
+               eta = 1, nthread = 2, nround = 2,objective = "binary:logistic")
+
+#train$data@Dimnames[[2]] represents the column names of the sparse matrix.
+importance_matrix <- xgb.importance(train$data@Dimnames[[2]], model = bst)
+xgb.plot.importance(importance_matrix)
+}
+

R-package/man/xgb.plot.tree.Rd

@@ -0,0 +1,58 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/xgb.plot.tree.R
+\name{xgb.plot.tree}
+\alias{xgb.plot.tree}
+\title{Plot a boosted tree model}
+\usage{
+xgb.plot.tree(feature_names = NULL, filename_dump = NULL, model = NULL,
+  n_first_tree = NULL, CSSstyle = NULL, width = NULL, height = NULL)
+}
+\arguments{
+\item{feature_names}{names of each feature as a character vector. Can be extracted from a sparse matrix (see example). If model dump already contains feature names, this argument should be \code{NULL}.}
+
+\item{filename_dump}{the path to the text file storing the model. Model dump must include the gain per feature and per tree (parameter \code{with.stats = T} in function \code{xgb.dump}). Possible to provide a model directly (see \code{model} argument).}
+
+\item{model}{generated by the \code{xgb.train} function. Avoid the creation of a dump file.}
+
+\item{n_first_tree}{limit the plot to the n first trees. If \code{NULL}, all trees of the model are plotted. Performance can be low for huge models.}
+
+\item{CSSstyle}{a \code{character} vector storing a css style to customize the appearance of nodes. Look at the \href{https://github.com/knsv/mermaid/wiki}{Mermaid wiki} for more information.}
+
+\item{width}{the width of the diagram in pixels.}
+
+\item{height}{the height of the diagram in pixels.}
+}
+\value{
+A \code{DiagrammeR} of the model.
+}
+\description{
+Read a tree model text dump.
+Plotting only works for boosted tree model (not linear model).
+}
+\details{
+The content of each node is organised that way:
+
+\itemize{
+ \item \code{feature} value ;
+ \item \code{cover}: the sum of second order gradient of training data classified to the leaf, if it is square loss, this simply corresponds to the number of instances in that branch. Deeper in the tree a node is, lower this metric will be ;
+ \item \code{gain}: metric the importance of the node in the model.
+}
+
+Each branch finishes with a leaf. For each leaf, only the \code{cover} is indicated.
+It uses \href{https://github.com/knsv/mermaid/}{Mermaid} library for that purpose.
+}
+\examples{
+data(agaricus.train, package='xgboost')
+
+#Both dataset are list with two items, a sparse matrix and labels
+#(labels = outcome column which will be learned).
+#Each column of the sparse Matrix is a feature in one hot encoding format.
+train <- agaricus.train
+
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
+               eta = 1, nthread = 2, nround = 2,objective = "binary:logistic")
+
+#agaricus.test$data@Dimnames[[2]] represents the column names of the sparse matrix.
+xgb.plot.tree(agaricus.train$data@Dimnames[[2]], model = bst)
+}
+

R-package/man/xgb.save.Rd

@@ -0,0 +1,28 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/xgb.save.R
+\name{xgb.save}
+\alias{xgb.save}
+\title{Save xgboost model to binary file}
+\usage{
+xgb.save(model, fname)
+}
+\arguments{
+\item{model}{the model object.}
+
+\item{fname}{the name of the binary file.}
+}
+\description{
+Save xgboost model from xgboost or xgb.train
+}
+\examples{
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+train <- agaricus.train
+test <- agaricus.test
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
+               eta = 1, nthread = 2, nround = 2,objective = "binary:logistic")
+xgb.save(bst, 'xgb.model')
+bst <- xgb.load('xgb.model')
+pred <- predict(bst, test$data)
+}
+

R-package/man/xgb.save.raw.Rd

@@ -0,0 +1,27 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/xgb.save.raw.R
+\name{xgb.save.raw}
+\alias{xgb.save.raw}
+\title{Save xgboost model to R's raw vector,
+user can call xgb.load to load the model back from raw vector}
+\usage{
+xgb.save.raw(model)
+}
+\arguments{
+\item{model}{the model object.}
+}
+\description{
+Save xgboost model from xgboost or xgb.train
+}
+\examples{
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+train <- agaricus.train
+test <- agaricus.test
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
+               eta = 1, nthread = 2, nround = 2,objective = "binary:logistic")
+raw <- xgb.save.raw(bst)
+bst <- xgb.load(raw)
+pred <- predict(bst, test$data)
+}
+

R-package/man/xgb.train.Rd

@@ -0,0 +1,140 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/xgb.train.R
+\name{xgb.train}
+\alias{xgb.train}
+\title{eXtreme Gradient Boosting Training}
+\usage{
+xgb.train(params = list(), data, nrounds, watchlist = list(), obj = NULL,
+  feval = NULL, verbose = 1, printEveryN=1L, early_stop_round = NULL,
+  early.stop.round = NULL, maximize = NULL, ...)
+}
+\arguments{
+\item{params}{the list of parameters.
+
+1. General Parameters
+
+\itemize{
+  \item \code{booster} which booster to use, can be \code{gbtree} or \code{gblinear}. Default: \code{gbtree}
+  \item \code{silent} 0 means printing running messages, 1 means silent mode. Default: 0
+}
+
+2. Booster Parameters
+
+2.1. Parameter for Tree Booster
+
+\itemize{
+  \item \code{eta} control the learning rate: scale the contribution of each tree by a factor of \code{0 < eta < 1} when it is added to the current approximation. Used to prevent overfitting by making the boosting process more conservative. Lower value for \code{eta} implies larger value for \code{nrounds}: low \code{eta} value means model more robust to overfitting but slower to compute. Default: 0.3
+  \item \code{gamma} minimum loss reduction required to make a further partition on a leaf node of the tree. the larger, the more conservative the algorithm will be.
+  \item \code{max_depth} maximum depth of a tree. Default: 6
+  \item \code{min_child_weight} minimum sum of instance weight(hessian) needed in a child. If the tree partition step results in a leaf node with the sum of instance weight less than min_child_weight, then the building process will give up further partitioning. In linear regression mode, this simply corresponds to minimum number of instances needed to be in each node. The larger, the more conservative the algorithm will be. Default: 1
+  \item \code{subsample} subsample ratio of the training instance. Setting it to 0.5 means that xgboost randomly collected half of the data instances to grow trees and this will prevent overfitting. It makes computation shorter (because less data to analyse). It is advised to use this parameter with \code{eta} and increase \code{nround}. Default: 1
+  \item \code{colsample_bytree} subsample ratio of columns when constructing each tree. Default: 1
+  \item \code{num_parallel_tree} Experimental parameter. number of trees to grow per round. Useful to test Random Forest through Xgboost (set \code{colsample_bytree < 1}, \code{subsample  < 1}  and \code{round = 1}) accordingly. Default: 1
+}
+
+2.2. Parameter for Linear Booster
+
+\itemize{
+  \item \code{lambda} L2 regularization term on weights. Default: 0
+  \item \code{lambda_bias} L2 regularization term on bias. Default: 0
+  \item \code{alpha} L1 regularization term on weights. (there is no L1 reg on bias because it is not important). Default: 0
+}
+
+3. Task Parameters
+
+\itemize{
+\item \code{objective} specify the learning task and the corresponding learning objective, and the objective options are below:
+  \itemize{
+    \item \code{reg:linear} linear regression (Default).
+    \item \code{reg:logistic} logistic regression.
+    \item \code{binary:logistic} logistic regression for binary classification. Output probability.
+    \item \code{binary:logitraw} logistic regression for binary classification, output score before logistic transformation.
+    \item \code{num_class} set the number of classes. To use only with multiclass objectives.
+    \item \code{multi:softmax} set xgboost to do multiclass classification using the softmax objective. Class is represented by a number and should be from 0 to \code{tonum_class}.
+    \item \code{multi:softprob} same as softmax, but output a vector of ndata * nclass, which can be further reshaped to ndata, nclass matrix. The result contains predicted probabilities of each data point belonging to each class.
+    \item \code{rank:pairwise} set xgboost to do ranking task by minimizing the pairwise loss.
+  }
+  \item \code{base_score} the initial prediction score of all instances, global bias. Default: 0.5
+  \item \code{eval_metric} evaluation metrics for validation data. Default: metric will be assigned according to objective(rmse for regression, and error for classification, mean average precision for ranking). List is provided in detail section.
+}}
+
+\item{data}{takes an \code{xgb.DMatrix} as the input.}
+
+\item{nrounds}{the max number of iterations}
+
+\item{watchlist}{what information should be printed when \code{verbose=1} or
+  \code{verbose=2}. Watchlist is used to specify validation set monitoring
+  during training. For example user can specify
+   watchlist=list(validation1=mat1, validation2=mat2) to watch
+   the performance of each round's model on mat1 and mat2}
+
+\item{obj}{customized objective function. Returns gradient and second order
+gradient with given prediction and dtrain,}
+
+\item{feval}{custimized evaluation function. Returns
+\code{list(metric='metric-name', value='metric-value')} with given
+prediction and dtrain,}
+
+\item{verbose}{If 0, xgboost will stay silent. If 1, xgboost will print
+information of performance. If 2, xgboost will print information of both}
+
+\item{printEveryN}{Print every N progress messages when \code{verbose>0}. Default is 1 which means all messages are printed.}
+
+\item{early_stop_round}{If \code{NULL}, the early stopping function is not triggered.
+If set to an integer \code{k}, training with a validation set will stop if the performance
+keeps getting worse consecutively for \code{k} rounds.}
+
+\item{early.stop.round}{An alternative of \code{early_stop_round}.}
+
+\item{maximize}{If \code{feval} and \code{early_stop_round} are set, then \code{maximize} must be set as well.
+\code{maximize=TRUE} means the larger the evaluation score the better.}
+
+\item{...}{other parameters to pass to \code{params}.}
+}
+\description{
+An advanced interface for training xgboost model. Look at \code{\link{xgboost}} function for a simpler interface.
+}
+\details{
+This is the training function for \code{xgboost}.
+
+It supports advanced features such as \code{watchlist}, customized objective function (\code{feval}),
+therefore it is more flexible than \code{\link{xgboost}} function.
+
+Parallelization is automatically enabled if \code{OpenMP} is present.
+Number of threads can also be manually specified via \code{nthread} parameter.
+
+\code{eval_metric} parameter (not listed above) is set automatically by Xgboost but can be overriden by parameter. Below is provided the list of different metric optimized by Xgboost to help you to understand how it works inside or to use them with the \code{watchlist} parameter.
+  \itemize{
+     \item \code{rmse} root mean square error. \url{http://en.wikipedia.org/wiki/Root_mean_square_error}
+     \item \code{logloss} negative log-likelihood. \url{http://en.wikipedia.org/wiki/Log-likelihood}
+     \item \code{error} Binary classification error rate. It is calculated as \code{(wrong cases) / (all cases)}. For the predictions, the evaluation will regard the instances with prediction value larger than 0.5 as positive instances, and the others as negative instances.
+     \item \code{merror} Multiclass classification error rate. It is calculated as \code{(wrong cases) / (all cases)}.
+     \item \code{auc} Area under the curve. \url{http://en.wikipedia.org/wiki/Receiver_operating_characteristic#'Area_under_curve} for ranking evaluation.
+     \item \code{ndcg} Normalized Discounted Cumulative Gain (for ranking task). \url{http://en.wikipedia.org/wiki/NDCG}
+  }
+
+Full list of parameters is available in the Wiki \url{https://github.com/dmlc/xgboost/wiki/Parameters}.
+
+This function only accepts an \code{\link{xgb.DMatrix}} object as the input.
+}
+\examples{
+data(agaricus.train, package='xgboost')
+dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+dtest <- dtrain
+watchlist <- list(eval = dtest, train = dtrain)
+param <- list(max.depth = 2, eta = 1, silent = 1)
+logregobj <- function(preds, dtrain) {
+   labels <- getinfo(dtrain, "label")
+   preds <- 1/(1 + exp(-preds))
+   grad <- preds - labels
+   hess <- preds * (1 - preds)
+   return(list(grad = grad, hess = hess))
+}
+evalerror <- function(preds, dtrain) {
+  labels <- getinfo(dtrain, "label")
+  err <- as.numeric(sum(labels != (preds > 0)))/length(labels)
+  return(list(metric = "error", value = err))
+}
+bst <- xgb.train(param, dtrain, nthread = 2, nround = 2, watchlist, logregobj, evalerror)
+}
+

R-package/man/xgboost.Rd

@@ -0,0 +1,77 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/xgboost.R
+\name{xgboost}
+\alias{xgboost}
+\title{eXtreme Gradient Boosting (Tree) library}
+\usage{
+xgboost(data = NULL, label = NULL, missing = NULL, params = list(),
+  nrounds, verbose = 1, printEveryN=1L, early_stop_round = NULL, early.stop.round = NULL,
+  maximize = NULL, ...)
+}
+\arguments{
+\item{data}{takes \code{matrix}, \code{dgCMatrix}, local data file or
+\code{xgb.DMatrix}.}
+
+\item{label}{the response variable. User should not set this field,
+if data is local data file or  \code{xgb.DMatrix}.}
+
+\item{missing}{Missing is only used when input is dense matrix, pick a float
+value that represents missing value. Sometimes a data use 0 or other extreme value to represents missing values.}
+
+\item{params}{the list of parameters.
+
+Commonly used ones are:
+\itemize{
+  \item \code{objective} objective function, common ones are
+  \itemize{
+    \item \code{reg:linear} linear regression
+    \item \code{binary:logistic} logistic regression for classification
+  }
+  \item \code{eta} step size of each boosting step
+  \item \code{max.depth} maximum depth of the tree
+  \item \code{nthread} number of thread used in training, if not set, all threads are used
+}
+
+  Look at \code{\link{xgb.train}} for a more complete list of parameters or \url{https://github.com/dmlc/xgboost/wiki/Parameters} for the full list.
+
+  See also \code{demo/} for walkthrough example in R.}
+
+\item{nrounds}{the max number of iterations}
+
+\item{verbose}{If 0, xgboost will stay silent. If 1, xgboost will print
+information of performance. If 2, xgboost will print information of both
+performance and construction progress information}
+
+\item{printEveryN}{Print every N progress messages when \code{verbose>0}. Default is 1 which means all messages are printed.}
+
+\item{early_stop_round}{If \code{NULL}, the early stopping function is not triggered.
+If set to an integer \code{k}, training with a validation set will stop if the performance
+keeps getting worse consecutively for \code{k} rounds.}
+
+\item{early.stop.round}{An alternative of \code{early_stop_round}.}
+
+\item{maximize}{If \code{feval} and \code{early_stop_round} are set, then \code{maximize} must be set as well.
+\code{maximize=TRUE} means the larger the evaluation score the better.}
+
+\item{...}{other parameters to pass to \code{params}.}
+}
+\description{
+A simple interface for training xgboost model. Look at \code{\link{xgb.train}} function for a more advanced interface.
+}
+\details{
+This is the modeling function for Xgboost.
+
+Parallelization is automatically enabled if \code{OpenMP} is present.
+
+Number of threads can also be manually specified via \code{nthread} parameter.
+}
+\examples{
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+train <- agaricus.train
+test <- agaricus.test
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
+               eta = 1, nthread = 2, nround = 2, objective = "binary:logistic")
+pred <- predict(bst, test$data)
+}
+

R-package/src/Makevars

@@ -0,0 +1,8 @@
+# package root
+PKGROOT=../../
+# _*_ mode: Makefile; _*_
+PKG_CPPFLAGS= -DXGBOOST_CUSTOMIZE_MSG_ -DXGBOOST_CUSTOMIZE_PRNG_ -DXGBOOST_STRICT_CXX98_ -DRABIT_CUSTOMIZE_MSG_ -DRABIT_STRICT_CXX98_ -I$(PKGROOT)
+PKG_CXXFLAGS= $(SHLIB_OPENMP_CFLAGS) $(SHLIB_PTHREAD_FLAGS)
+PKG_LIBS = $(SHLIB_OPENMP_CFLAGS) $(SHLIB_PTHREAD_FLAGS)
+OBJECTS= xgboost_R.o xgboost_assert.o $(PKGROOT)/wrapper/xgboost_wrapper.o $(PKGROOT)/src/io/io.o $(PKGROOT)/src/gbm/gbm.o $(PKGROOT)/src/tree/updater.o $(PKGROOT)/subtree/rabit/src/engine_empty.o $(PKGROOT)/src/io/dmlc_simple.o
+

R-package/src/Makevars.win

@@ -0,0 +1,19 @@
+# package root
+PKGROOT=./
+# _*_ mode: Makefile; _*_
+
+# This file is only used for windows compilation from github
+# It will be replaced by Makevars in CRAN version
+.PHONY: all xgblib
+all: $(SHLIB)
+$(SHLIB): xgblib
+xgblib:
+	cp -r ../../src .
+	cp -r ../../wrapper .
+	cp -r ../../subtree .
+
+PKG_CPPFLAGS= -DXGBOOST_CUSTOMIZE_MSG_ -DXGBOOST_CUSTOMIZE_PRNG_ -DXGBOOST_STRICT_CXX98_ -DRABIT_CUSTOMIZE_MSG_ -DRABIT_STRICT_CXX98_ -I$(PKGROOT) -I../..
+PKG_CXXFLAGS= $(SHLIB_OPENMP_CFLAGS) $(SHLIB_PTHREAD_FLAGS)
+PKG_LIBS = $(SHLIB_OPENMP_CFLAGS) $(SHLIB_PTHREAD_FLAGS)
+OBJECTS= xgboost_R.o xgboost_assert.o $(PKGROOT)/wrapper/xgboost_wrapper.o $(PKGROOT)/src/io/io.o $(PKGROOT)/src/gbm/gbm.o $(PKGROOT)/src/tree/updater.o $(PKGROOT)/subtree/rabit/src/engine_empty.o $(PKGROOT)/src/io/dmlc_simple.o
+$(OBJECTS) : xgblib

R-package/src/xgboost_R.cpp

@@ -0,0 +1,322 @@
+#include <vector>
+#include <string>
+#include <utility>
+#include <cstring>
+#include <cstdio>
+#include <sstream> 
+#include "wrapper/xgboost_wrapper.h"
+#include "src/utils/utils.h"
+#include "src/utils/omp.h"
+#include "xgboost_R.h"
+
+using namespace std;
+using namespace xgboost;
+
+extern "C" {
+  void XGBoostAssert_R(int exp, const char *fmt, ...);
+  void XGBoostCheck_R(int exp, const char *fmt, ...);
+  int XGBoostSPrintf_R(char *buf, size_t size, const char *fmt, ...);
+}
+
+// implements error handling
+namespace xgboost {
+namespace utils {
+extern "C" {
+  void (*Printf)(const char *fmt, ...) = Rprintf;
+  int (*SPrintf)(char *buf, size_t size, const char *fmt, ...) = XGBoostSPrintf_R;
+  void (*Assert)(int exp, const char *fmt, ...) = XGBoostAssert_R;
+  void (*Check)(int exp, const char *fmt, ...) = XGBoostCheck_R;
+  void (*Error)(const char *fmt, ...) = error;
+}
+bool CheckNAN(double v) {
+  return ISNAN(v);
+}
+bool LogGamma(double v) {
+  return lgammafn(v);
+}
+} // namespace utils
+
+namespace random {
+void Seed(unsigned seed) {
+  warning("parameter seed is ignored, please set random seed using set.seed");
+}
+double Uniform(void) {
+  return unif_rand();
+}
+double Normal(void) {
+  return norm_rand();
+}
+}  // namespace random
+}  // namespace xgboost
+
+// call before wrapper starts
+inline void _WrapperBegin(void) {
+  GetRNGstate();
+}
+// call after wrapper starts
+inline void _WrapperEnd(void) {
+  PutRNGstate();
+}
+
+extern "C" {
+  SEXP XGCheckNullPtr_R(SEXP handle) {
+    return ScalarLogical(R_ExternalPtrAddr(handle) == NULL);
+  }
+  void _DMatrixFinalizer(SEXP ext) {    
+    if (R_ExternalPtrAddr(ext) == NULL) return;
+    XGDMatrixFree(R_ExternalPtrAddr(ext));
+    R_ClearExternalPtr(ext);
+  }
+  SEXP XGDMatrixCreateFromFile_R(SEXP fname, SEXP silent) {
+    _WrapperBegin();
+    void *handle = XGDMatrixCreateFromFile(CHAR(asChar(fname)), asInteger(silent));
+    _WrapperEnd();
+    SEXP ret = PROTECT(R_MakeExternalPtr(handle, R_NilValue, R_NilValue));
+    R_RegisterCFinalizerEx(ret, _DMatrixFinalizer, TRUE);
+    UNPROTECT(1);
+    return ret;
+  }
+  SEXP XGDMatrixCreateFromMat_R(SEXP mat, 
+                                SEXP missing) {
+    _WrapperBegin();
+    SEXP dim = getAttrib(mat, R_DimSymbol);
+    size_t nrow = static_cast<size_t>(INTEGER(dim)[0]);
+    size_t ncol = static_cast<size_t>(INTEGER(dim)[1]);
+    double *din = REAL(mat);
+    std::vector<float> data(nrow * ncol);
+    #pragma omp parallel for schedule(static)
+    for (bst_omp_uint i = 0; i < nrow; ++i) {
+      for (size_t j = 0; j < ncol; ++j) {
+        data[i * ncol +j] = din[i + nrow * j];
+      }
+    }
+    void *handle = XGDMatrixCreateFromMat(BeginPtr(data), nrow, ncol, asReal(missing));
+    _WrapperEnd();
+    SEXP ret = PROTECT(R_MakeExternalPtr(handle, R_NilValue, R_NilValue));
+    R_RegisterCFinalizerEx(ret, _DMatrixFinalizer, TRUE);
+    UNPROTECT(1);
+    return ret;    
+  }
+  SEXP XGDMatrixCreateFromCSC_R(SEXP indptr,
+                                SEXP indices,
+                                SEXP data) {
+    _WrapperBegin();
+    const int *p_indptr = INTEGER(indptr);
+    const int *p_indices = INTEGER(indices);
+    const double *p_data = REAL(data);
+    int nindptr = length(indptr);
+    int ndata = length(data);
+    std::vector<bst_ulong> col_ptr_(nindptr);
+    std::vector<unsigned> indices_(ndata);
+    std::vector<float> data_(ndata);
+
+    for (int i = 0; i < nindptr; ++i) {
+      col_ptr_[i] = static_cast<bst_ulong>(p_indptr[i]);
+    }
+    #pragma omp parallel for schedule(static)
+    for (int i = 0; i < ndata; ++i) {
+      indices_[i] = static_cast<unsigned>(p_indices[i]);
+      data_[i] = static_cast<float>(p_data[i]);
+    }
+    void *handle = XGDMatrixCreateFromCSC(BeginPtr(col_ptr_), BeginPtr(indices_),
+                                          BeginPtr(data_), nindptr, ndata);
+    _WrapperEnd();
+    SEXP ret = PROTECT(R_MakeExternalPtr(handle, R_NilValue, R_NilValue));
+    R_RegisterCFinalizerEx(ret, _DMatrixFinalizer, TRUE);
+    UNPROTECT(1);
+    return ret;
+  }
+  SEXP XGDMatrixSliceDMatrix_R(SEXP handle, SEXP idxset) {
+    _WrapperBegin();
+    int len = length(idxset);
+    std::vector<int> idxvec(len);
+    for (int i = 0; i < len; ++i) {
+      idxvec[i] = INTEGER(idxset)[i] - 1;
+    }
+    void *res = XGDMatrixSliceDMatrix(R_ExternalPtrAddr(handle), BeginPtr(idxvec), len);
+    _WrapperEnd();
+    SEXP ret = PROTECT(R_MakeExternalPtr(res, R_NilValue, R_NilValue));
+    R_RegisterCFinalizerEx(ret, _DMatrixFinalizer, TRUE);
+    UNPROTECT(1);
+    return ret;        
+  }
+  void XGDMatrixSaveBinary_R(SEXP handle, SEXP fname, SEXP silent) {
+    _WrapperBegin();
+    XGDMatrixSaveBinary(R_ExternalPtrAddr(handle),
+                        CHAR(asChar(fname)), asInteger(silent));
+    _WrapperEnd();
+  }
+  void XGDMatrixSetInfo_R(SEXP handle, SEXP field, SEXP array) {
+    _WrapperBegin();
+    int len = length(array);
+    const char *name = CHAR(asChar(field));
+    if (!strcmp("group", name)) {
+      std::vector<unsigned> vec(len);
+      #pragma omp parallel for schedule(static)      
+      for (int i = 0; i < len; ++i) {
+        vec[i] = static_cast<unsigned>(INTEGER(array)[i]);
+      }
+      XGDMatrixSetGroup(R_ExternalPtrAddr(handle), BeginPtr(vec), len);
+    } else {
+      std::vector<float> vec(len);
+      #pragma omp parallel for schedule(static)
+      for (int i = 0; i < len; ++i) {
+        vec[i] = REAL(array)[i];
+      }
+      XGDMatrixSetFloatInfo(R_ExternalPtrAddr(handle), 
+                            CHAR(asChar(field)),
+                            BeginPtr(vec), len);
+    }
+    _WrapperEnd();
+  }
+  SEXP XGDMatrixGetInfo_R(SEXP handle, SEXP field) {
+    _WrapperBegin();
+    bst_ulong olen;
+    const float *res = XGDMatrixGetFloatInfo(R_ExternalPtrAddr(handle),
+                                             CHAR(asChar(field)), &olen);
+    _WrapperEnd();
+    SEXP ret = PROTECT(allocVector(REALSXP, olen));
+    for (size_t i = 0; i < olen; ++i) {
+      REAL(ret)[i] = res[i];
+    }
+    UNPROTECT(1);
+    return ret;
+  }
+  SEXP XGDMatrixNumRow_R(SEXP handle) {
+    bst_ulong nrow = XGDMatrixNumRow(R_ExternalPtrAddr(handle));
+    return ScalarInteger(static_cast<int>(nrow));
+  }
+  // functions related to booster
+  void _BoosterFinalizer(SEXP ext) {    
+    if (R_ExternalPtrAddr(ext) == NULL) return;
+    XGBoosterFree(R_ExternalPtrAddr(ext));
+    R_ClearExternalPtr(ext);
+  }
+  SEXP XGBoosterCreate_R(SEXP dmats) {
+    _WrapperBegin();
+    int len = length(dmats);
+    std::vector<void*> dvec;
+    for (int i = 0; i < len; ++i){
+      dvec.push_back(R_ExternalPtrAddr(VECTOR_ELT(dmats, i)));
+    }
+    void *handle = XGBoosterCreate(BeginPtr(dvec), dvec.size());
+    _WrapperEnd();
+    SEXP ret = PROTECT(R_MakeExternalPtr(handle, R_NilValue, R_NilValue));
+    R_RegisterCFinalizerEx(ret, _BoosterFinalizer, TRUE);
+    UNPROTECT(1);
+    return ret;
+  }
+  void XGBoosterSetParam_R(SEXP handle, SEXP name, SEXP val) {
+    _WrapperBegin();
+    XGBoosterSetParam(R_ExternalPtrAddr(handle),
+                      CHAR(asChar(name)),
+                      CHAR(asChar(val)));
+    _WrapperEnd();
+  }
+  void XGBoosterUpdateOneIter_R(SEXP handle, SEXP iter, SEXP dtrain) {
+    _WrapperBegin();
+    XGBoosterUpdateOneIter(R_ExternalPtrAddr(handle),
+                           asInteger(iter),
+                           R_ExternalPtrAddr(dtrain));
+    _WrapperEnd();
+  }
+  void XGBoosterBoostOneIter_R(SEXP handle, SEXP dtrain, SEXP grad, SEXP hess) {
+    _WrapperBegin();
+    utils::Check(length(grad) == length(hess), "gradient and hess must have same length");
+    int len = length(grad);
+    std::vector<float> tgrad(len), thess(len);
+    #pragma omp parallel for schedule(static)
+    for (int j = 0; j < len; ++j) {
+      tgrad[j] = REAL(grad)[j];
+      thess[j] = REAL(hess)[j];
+    }
+    XGBoosterBoostOneIter(R_ExternalPtrAddr(handle),
+                          R_ExternalPtrAddr(dtrain),
+                          BeginPtr(tgrad), BeginPtr(thess), len);
+    _WrapperEnd();
+  }
+  SEXP XGBoosterEvalOneIter_R(SEXP handle, SEXP iter, SEXP dmats, SEXP evnames) {
+    _WrapperBegin();
+    utils::Check(length(dmats) == length(evnames), "dmats and evnams must have same length");
+    int len = length(dmats);
+    std::vector<void*> vec_dmats;
+    std::vector<std::string> vec_names;
+    std::vector<const char*> vec_sptr;
+    for (int i = 0; i < len; ++i) {
+      vec_dmats.push_back(R_ExternalPtrAddr(VECTOR_ELT(dmats, i)));
+      vec_names.push_back(std::string(CHAR(asChar(VECTOR_ELT(evnames, i)))));
+    }
+    for (int i = 0; i < len; ++i) {
+      vec_sptr.push_back(vec_names[i].c_str());
+    }
+    const char *ret =
+        XGBoosterEvalOneIter(R_ExternalPtrAddr(handle),
+                             asInteger(iter),
+                             BeginPtr(vec_dmats), BeginPtr(vec_sptr), len);  
+    _WrapperEnd();
+    return mkString(ret);
+  }
+  SEXP XGBoosterPredict_R(SEXP handle, SEXP dmat, SEXP option_mask, SEXP ntree_limit) {
+    _WrapperBegin();
+    bst_ulong olen;
+    const float *res = XGBoosterPredict(R_ExternalPtrAddr(handle),
+                                        R_ExternalPtrAddr(dmat),
+                                        asInteger(option_mask),
+                                        asInteger(ntree_limit),
+                                        &olen);
+    _WrapperEnd();
+    SEXP ret = PROTECT(allocVector(REALSXP, olen));
+    for (size_t i = 0; i < olen; ++i) {
+      REAL(ret)[i] = res[i];
+    }
+    UNPROTECT(1);
+    return ret;
+  }
+  void XGBoosterLoadModel_R(SEXP handle, SEXP fname) {
+    _WrapperBegin();
+    XGBoosterLoadModel(R_ExternalPtrAddr(handle), CHAR(asChar(fname)));
+    _WrapperEnd();
+  }
+  void XGBoosterSaveModel_R(SEXP handle, SEXP fname) {
+    _WrapperBegin();
+    XGBoosterSaveModel(R_ExternalPtrAddr(handle), CHAR(asChar(fname)));
+    _WrapperEnd();
+  }
+  void XGBoosterLoadModelFromRaw_R(SEXP handle, SEXP raw) {    
+    _WrapperBegin();
+    XGBoosterLoadModelFromBuffer(R_ExternalPtrAddr(handle),
+                                 RAW(raw),
+                                 length(raw));
+    _WrapperEnd();
+  }
+  SEXP XGBoosterModelToRaw_R(SEXP handle) {
+    bst_ulong olen;
+    _WrapperBegin();
+    const char *raw = XGBoosterGetModelRaw(R_ExternalPtrAddr(handle), &olen);
+    _WrapperEnd();
+    SEXP ret = PROTECT(allocVector(RAWSXP, olen));
+    if (olen != 0) {
+      memcpy(RAW(ret), raw, olen);
+    }
+    UNPROTECT(1);    
+    return ret;
+  }
+  SEXP XGBoosterDumpModel_R(SEXP handle, SEXP fmap, SEXP with_stats) {
+    _WrapperBegin();
+    bst_ulong olen;
+    const char **res =
+        XGBoosterDumpModel(R_ExternalPtrAddr(handle),
+                           CHAR(asChar(fmap)),
+                           asInteger(with_stats),
+                           &olen);
+    _WrapperEnd();
+    SEXP out = PROTECT(allocVector(STRSXP, olen));    
+    for (size_t i = 0; i < olen; ++i) {     
+      stringstream stream;
+      stream <<  "booster["<<i<<"]\n" << res[i];
+      SET_STRING_ELT(out, i, mkChar(stream.str().c_str()));
+    }
+    UNPROTECT(1);
+    return out;
+  }
+}

R-package/src/xgboost_R.h

@@ -0,0 +1,156 @@
+#ifndef XGBOOST_WRAPPER_R_H_
+#define XGBOOST_WRAPPER_R_H_
+/*!
+ * \file xgboost_wrapper_R.h
+ * \author Tianqi Chen
+ * \brief R wrapper of xgboost
+ */
+extern "C" {
+#include <Rinternals.h>
+#include <R_ext/Random.h>
+#include <Rmath.h>
+}
+
+extern "C" {
+  /*!
+   * \brief check whether a handle is NULL
+   * \param handle
+   * \return whether it is null ptr
+   */
+  SEXP XGCheckNullPtr_R(SEXP handle);
+  /*!
+   * \brief load a data matrix 
+   * \param fname name of the content
+   * \param silent whether print messages
+   * \return a loaded data matrix
+   */
+  SEXP XGDMatrixCreateFromFile_R(SEXP fname, SEXP silent);
+  /*!
+   * \brief create matrix content from dense matrix
+   * This assumes the matrix is stored in column major format
+   * \param data R Matrix object
+   * \param missing which value to represent missing value
+   * \return created dmatrix
+   */
+  SEXP XGDMatrixCreateFromMat_R(SEXP mat, 
+                                SEXP missing);
+  /*! 
+   * \brief create a matrix content from CSC format
+   * \param indptr pointer to column headers
+   * \param indices row indices
+   * \param data content of the data
+   * \return created dmatrix
+   */
+  SEXP XGDMatrixCreateFromCSC_R(SEXP indptr,
+                                SEXP indices,
+                                SEXP data);
+  /*!
+   * \brief create a new dmatrix from sliced content of existing matrix
+   * \param handle instance of data matrix to be sliced
+   * \param idxset index set
+   * \return a sliced new matrix
+   */
+  SEXP XGDMatrixSliceDMatrix_R(SEXP handle, SEXP idxset);
+  /*!
+   * \brief load a data matrix into binary file
+   * \param handle a instance of data matrix
+   * \param fname file name
+   * \param silent print statistics when saving
+   */
+  void XGDMatrixSaveBinary_R(SEXP handle, SEXP fname, SEXP silent);
+  /*!
+   * \brief set information to dmatrix
+   * \param handle a instance of data matrix
+   * \param field field name, can be label, weight
+   * \param array pointer to float vector
+   */
+  void XGDMatrixSetInfo_R(SEXP handle, SEXP field, SEXP array);
+  /*!
+   * \brief get info vector from matrix
+   * \param handle a instance of data matrix
+   * \param field field name
+   * \return info vector
+   */  
+  SEXP XGDMatrixGetInfo_R(SEXP handle, SEXP field);
+  /*!
+   * \brief return number of rows
+   * \param handle a instance of data matrix
+   */
+  SEXP XGDMatrixNumRow_R(SEXP handle);
+  /*! 
+   * \brief create xgboost learner 
+   * \param dmats a list of dmatrix handles that will be cached
+   */  
+  SEXP XGBoosterCreate_R(SEXP dmats);
+  /*! 
+   * \brief set parameters 
+   * \param handle handle
+   * \param name  parameter name
+   * \param val value of parameter
+   */
+  void XGBoosterSetParam_R(SEXP handle, SEXP name, SEXP val);
+  /*! 
+   * \brief update the model in one round using dtrain
+   * \param handle handle
+   * \param iter current iteration rounds
+   * \param dtrain training data
+   */
+  void XGBoosterUpdateOneIter_R(SEXP ext, SEXP iter, SEXP dtrain);
+  /*!
+   * \brief update the model, by directly specify gradient and second order gradient,
+   *        this can be used to replace UpdateOneIter, to support customized loss function
+   * \param handle handle
+   * \param dtrain training data
+   * \param grad gradient statistics
+   * \param hess second order gradient statistics
+   */
+  void XGBoosterBoostOneIter_R(SEXP handle, SEXP dtrain, SEXP grad, SEXP hess);
+  /*!
+   * \brief get evaluation statistics for xgboost
+   * \param handle handle
+   * \param iter current iteration rounds
+   * \param dmats list of handles to dmatrices
+   * \param evname name of evaluation
+   * \return the string containing evaluation stati
+   */
+  SEXP XGBoosterEvalOneIter_R(SEXP handle, SEXP iter, SEXP dmats, SEXP evnames);
+  /*!
+   * \brief make prediction based on dmat
+   * \param handle handle
+   * \param dmat data matrix
+   * \param option_mask output_margin:1 predict_leaf:2
+   * \param ntree_limit limit number of trees used in prediction
+   */
+  SEXP XGBoosterPredict_R(SEXP handle, SEXP dmat, SEXP option_mask, SEXP ntree_limit);
+  /*!
+   * \brief load model from existing file
+   * \param handle handle
+   * \param fname file name
+   */
+  void XGBoosterLoadModel_R(SEXP handle, SEXP fname);
+  /*!
+   * \brief save model into existing file
+   * \param handle handle
+   * \param fname file name
+   */    
+  void XGBoosterSaveModel_R(SEXP handle, SEXP fname);
+  /*!
+   * \brief load model from raw array
+   * \param handle handle
+   */    
+  void XGBoosterLoadModelFromRaw_R(SEXP handle, SEXP raw);
+  /*!
+   * \brief save model into R's raw array
+   * \param handle handle
+   * \return raw array
+   */
+  SEXP XGBoosterModelToRaw_R(SEXP handle);
+  /*!
+   * \brief dump model into a string
+   * \param handle handle
+   * \param fmap  name to fmap can be empty string
+   * \param with_stats whether dump statistics of splits
+   */
+  SEXP XGBoosterDumpModel_R(SEXP handle, SEXP fmap, SEXP with_stats);
+}
+#endif  // XGBOOST_WRAPPER_R_H_

R-package/src/xgboost_assert.c

@@ -0,0 +1,33 @@
+#include <stdio.h>
+#include <stdarg.h>
+#include <Rinternals.h>
+
+// implements error handling
+void XGBoostAssert_R(int exp, const char *fmt, ...) {
+  char buf[1024];
+  if (exp == 0) {
+    va_list args;  
+    va_start(args, fmt);
+    vsprintf(buf, fmt, args);
+    va_end(args);
+    error("AssertError:%s\n", buf);
+  }  
+}
+void XGBoostCheck_R(int exp, const char *fmt, ...) {
+  char buf[1024];
+  if (exp == 0) {
+    va_list args;  
+    va_start(args, fmt);
+    vsprintf(buf, fmt, args);
+    va_end(args);
+    error("%s\n", buf);
+  }
+}
+int XGBoostSPrintf_R(char *buf, size_t size, const char *fmt, ...) {
+  int ret;
+  va_list args;  
+  va_start(args, fmt);
+  ret = vsnprintf(buf, size, fmt, args);
+  va_end(args);
+  return ret;
+}

R-package/vignettes/discoverYourData.Rmd

@@ -0,0 +1,337 @@
+---
+title: "Understand your dataset with Xgboost"
+output: 
+  rmarkdown::html_vignette:
+    css: vignette.css
+    number_sections: yes
+    toc: yes
+author: Tianqi Chen, Tong He, Michaël Benesty
+vignette: >
+  %\VignetteIndexEntry{Discover your data}
+  %\VignetteEngine{knitr::rmarkdown}
+  \usepackage[utf8]{inputenc}
+---
+
+Introduction
+============
+
+The purpose of this Vignette is to show you how to use **Xgboost** to discover and understand your own dataset better.
+
+This Vignette is not about predicting anything (see [Xgboost presentation](https://github.com/dmlc/xgboost/blob/master/R-package/vignettes/xgboostPresentation.Rmd)). We will explain how to use **Xgboost** to highlight the *link* between the *features* of your data and the *outcome*.
+
+Pacakge loading:
+
+```{r libLoading, results='hold', message=F, warning=F}
+require(xgboost)
+require(Matrix)
+require(data.table)
+if (!require('vcd')) install.packages('vcd') 
+```
+
+> **VCD** package is used for one of its embedded dataset only.
+
+Preparation of the dataset
+==========================
+
+Numeric VS categorical variables
+--------------------------------
+
+**Xgboost** manages only `numeric` vectors.
+
+What to do when you have *categorical* data?
+
+A *categorical* variable has a fixed number of different values. For instance, if a variable called *Colour* can have only one of these three values, *red*, *blue* or *green*, then *Colour* is a *categorical* variable.
+
+> In **R**, a *categorical* variable is called `factor`.
+>
+> Type `?factor` in the console for more information.
+
+To answer the question above we will convert *categorical* variables to `numeric` one.
+
+Conversion from categorical to numeric variables
+------------------------------------------------
+
+### Looking at the raw data
+
+In this Vignette we will see how to transform a *dense* `data.frame` (*dense* = few zeroes in the matrix) with *categorical* variables to a very *sparse* matrix (*sparse* = lots of zero in the matrix) of `numeric` features.
+
+The method we are going to see is usually called [one-hot encoding](http://en.wikipedia.org/wiki/One-hot).
+
+The first step is to load `Arthritis` dataset in memory and wrap it with `data.table` package.
+
+```{r, results='hide'}
+data(Arthritis)
+df <- data.table(Arthritis, keep.rownames = F)
+```
+
+> `data.table` is 100% compliant with **R** `data.frame` but its syntax is more consistent and its performance for large dataset is [best in class](http://stackoverflow.com/questions/21435339/data-table-vs-dplyr-can-one-do-something-well-the-other-cant-or-does-poorly) (`dplyr` from **R** and `panda` from **Python** [included](https://github.com/Rdatatable/data.table/wiki/Benchmarks-%3A-Grouping)). Some parts of **Xgboost** **R** package use `data.table`.
+
+The first thing we want to do is to have a look to the first lines of the `data.table`:
+
+```{r}
+head(df)
+```
+
+Now we will check the format of each column.
+
+```{r}
+str(df)
+```
+
+2 columns have `factor` type, one has `ordinal` type.
+
+> `ordinal` variable :
+>
+> * can take a limited number of values (like `factor`) ;
+> * these values are ordered (unlike `factor`). Here these ordered values are: `Marked > Some > None`
+
+### Creation of new features based on old ones
+
+We will add some new *categorical* features to see if it helps.
+
+#### Grouping per 10 years
+
+For the first feature we create groups of age by rounding the real age.
+
+Note that we transform it to `factor` so the algorithm treat these age groups as independent values.
+
+Therefore, 20 is not closer to 30 than 60. To make it short, the distance between ages is lost in this transformation.
+
+```{r}
+head(df[,AgeDiscret := as.factor(round(Age/10,0))])
+```
+
+#### Random split in two groups
+
+Following is an even stronger simplification of the real age with an arbitrary split at 30 years old. I choose this value **based on nothing**. We will see later if simplifying the information based on arbitrary values is a good strategy (you may already have an idea of how well it will work...).
+
+```{r}
+head(df[,AgeCat:= as.factor(ifelse(Age > 30, "Old", "Young"))])
+```
+
+#### Risks in adding correlated features
+
+These new features are highly correlated to the `Age` feature because they are simple transformations of this feature. 
+
+For many machine learning algorithms, using correlated features is not a good idea. It may sometimes make prediction less accurate, and most of the time make interpretation of the model almost impossible. GLM, for instance, assumes that the features are uncorrelated.
+
+Fortunately, decision tree algorithms (including boosted trees) are very robust to these features. Therefore we have nothing to do to manage this situation.
+
+#### Cleaning data
+
+We remove ID as there is nothing to learn from this feature (it would just add some noise).
+
+```{r, results='hide'}
+df[,ID:=NULL]
+```
+
+We will list the different values for the column `Treatment`:
+
+```{r}
+levels(df[,Treatment])
+```
+
+
+### One-hot encoding
+
+Next step, we will transform the categorical data to dummy variables.
+This is the [one-hot encoding](http://en.wikipedia.org/wiki/One-hot) step.
+
+The purpose is to transform each value of each *categorical* feature in a *binary* feature `{0, 1}`.
+
+For example, the column `Treatment` will be replaced by two columns, `Placebo`, and `Treated`. Each of them will be *binary*. Therefore, an observation which has the value `Placebo` in column `Treatment` before the transformation will have after the transformation the value `1` in the new column `Placebo` and the value `0` in the new column `Treated`. The column `Treatment` will disappear during the one-hot encoding.
+
+Column `Improved` is excluded because it will be our `label` column, the one we want to predict.
+
+```{r, warning=FALSE,message=FALSE}
+sparse_matrix <- sparse.model.matrix(Improved~.-1, data = df)
+head(sparse_matrix)
+```
+
+> Formulae `Improved~.-1` used above means transform all *categorical* features but column `Improved` to binary values. The `-1` is here to remove the first column which is full of `1` (this column is generated by the conversion). For more information, you can type `?sparse.model.matrix` in the console.
+
+Create the output `numeric` vector (not as a sparse `Matrix`):
+
+```{r}
+output_vector = df[,Improved] == "Marked"
+```
+
+1. set `Y` vector to `0`; 
+2. set `Y` to `1` for rows where `Improved == Marked` is `TRUE` ; 
+3. return `Y` vector.
+
+Build the model
+===============
+
+The code below is very usual. For more information, you can look at the documentation of `xgboost` function (or at the vignette [Xgboost presentation](https://github.com/dmlc/xgboost/blob/master/R-package/vignettes/xgboostPresentation.Rmd)).
+
+```{r}
+bst <- xgboost(data = sparse_matrix, label = output_vector, max.depth = 4,
+               eta = 1, nthread = 2, nround = 10,objective = "binary:logistic")
+
+```
+
+You can see some `train-error: 0.XXXXX` lines followed by a number. It decreases. Each line shows how well the model explains your data. Lower is better.
+
+A model which fits too well may [overfit](http://en.wikipedia.org/wiki/Overfitting) (meaning it copy/paste too much the past, and won't be that good to predict the future). 
+
+> Here you can see the numbers decrease until line 7 and then increase. 
+>
+> It probably means we are overfitting. To fix that I should reduce the number of rounds to `nround = 4`. I will let things like that because I don't really care for the purpose of this example :-)
+
+Feature importance
+==================
+
+Measure feature importance
+--------------------------
+
+### Build the feature importance data.table
+
+In the code below, `sparse_matrix@Dimnames[[2]]` represents the column names of the sparse matrix. These names are the original values of the features (remember, each binary column == one value of one *categorical* feature).
+
+```{r}
+importance <- xgb.importance(sparse_matrix@Dimnames[[2]], model = bst)
+head(importance)
+```
+
+> The column `Gain` provide the information we are looking for.
+>
+> As you can see, features are classified by `Gain`.
+
+`Gain` is the improvement in accuracy brought by a feature to the branches it is on. The idea is that before adding a new split on a feature X to the branch there was some wrongly classified elements, after adding the split on this feature, there are two new branches, and each of these branch is more accurate (one branch saying if your observation is on this branch then it should be classified as `1`, and the other branch saying the exact opposite).
+
+`Cover` measures the relative quantity of observations concerned by a feature.
+
+`Frequence` is a simpler way to measure the `Gain`. It just counts the number of times a feature is used in all generated trees. You should not use it (unless you know why you want to use it).
+
+### Improvement in the interpretability of feature importance data.table
+
+We can go deeper in the analysis of the model. In the `data.table` above, we have discovered which features counts to predict if the illness will go or not. But we don't yet know the role of these features. For instance, one of the question we may want to answer would be: does receiving a placebo treatment helps to recover from the illness?
+
+One simple solution is to count the co-occurrences of a feature and a class of the classification.
+
+For that purpose we will execute the same function as above but using two more parameters, `data` and `label`.
+
+```{r}
+importanceRaw <- xgb.importance(sparse_matrix@Dimnames[[2]], model = bst, data = sparse_matrix, label = output_vector)
+
+# Cleaning for better display
+importanceClean <- importanceRaw[,`:=`(Cover=NULL, Frequence=NULL)]
+
+head(importanceClean)
+```
+
+> In the table above we have removed two not needed columns and select only the first lines.
+
+First thing you notice is the new column `Split`. It is the split applied to the feature on a branch of one of the tree. Each split is present, therefore a feature can appear several times in this table. Here we can see the feature `Age` is used several times with different splits.
+
+How the split is applied to count the co-occurrences? It is always `<`. For instance, in the second line, we measure the number of persons under 61.5 years with the illness gone after the treatment.
+
+The two other new columns are `RealCover` and `RealCover %`. In the first column it measures the number of observations in the dataset where the split is respected and the label marked as `1`. The second column is the percentage of the whole population that `RealCover` represents.
+
+Therefore, according to our findings, getting a placebo doesn't seem to help but being younger than 61 years may help (seems logic).
+
+> You may wonder how to interpret the `< 1.00001` on the first line. Basically, in a sparse `Matrix`, there is no `0`, therefore, looking for one hot-encoded categorical observations validating the rule `< 1.00001` is like just looking for `1` for this feature.
+
+Plotting the feature importance
+-------------------------------
+
+All these things are nice, but it would be even better to plot the results.
+
+```{r, fig.width=8, fig.height=5, fig.align='center'}
+xgb.plot.importance(importance_matrix = importanceRaw)
+```
+
+Feature have automatically been divided in 2 clusters: the interesting features... and the others.
+
+> Depending of the dataset and the learning parameters you may have more than two clusters. Default value is to limit them to `10`, but you can increase this limit. Look at the function documentation for more information.
+
+According to the plot above, the most important features in this dataset to predict if the treatment will work are :
+
+* the Age ;
+* having received a placebo or not ;
+* the sex is third but already included in the not interesting features group ; 
+* then we see our generated features (AgeDiscret). We can see that their contribution is very low.
+
+Do these results make sense?
+------------------------------
+
+Let's check some **Chi2** between each of these features and the label.
+
+Higher **Chi2** means better correlation.
+
+```{r, warning=FALSE, message=FALSE}
+c2 <- chisq.test(df$Age, output_vector)
+print(c2)
+```
+
+Pearson correlation between Age and illness disapearing is **`r round(c2$statistic, 2 )`**.
+
+```{r, warning=FALSE, message=FALSE}
+c2 <- chisq.test(df$AgeDiscret, output_vector)
+print(c2)
+```
+
+Our first simplification of Age gives a Pearson correlation is **`r round(c2$statistic, 2)`**.
+
+```{r, warning=FALSE, message=FALSE}
+c2 <- chisq.test(df$AgeCat, output_vector)
+print(c2)
+```
+
+The perfectly random split I did between young and old at 30 years old have a low correlation of **`r round(c2$statistic, 2)`**. It's a result we may expect as may be in my mind > 30 years is being old (I am 32 and starting feeling old, this may explain that), but for the illness we are studying, the age to be vulnerable is not the same. 
+
+Morality: don't let your *gut* lower the quality of your model. 
+
+In *data science* expression, there is the word *science* :-)
+
+Conclusion
+==========
+
+As you can see, in general *destroying information by simplifying it won't improve your model*. **Chi2** just demonstrates that. 
+
+But in more complex cases, creating a new feature based on existing one which makes link with the outcome more obvious may help the algorithm and improve the model. 
+
+The case studied here is not enough complex to show that. Check [Kaggle website](http://www.kaggle.com/) for some challenging datasets. However it's almost always worse when you add some arbitrary rules.
+
+Moreover, you can notice that even if we have added some not useful new features highly correlated with other features, the boosting tree algorithm have been able to choose the best one, which in this case is the Age.
+
+Linear model may not be that smart in this scenario.
+
+Special Note: What about Random Forests™?
+==========================================
+
+As you may know, [Random Forests™](http://en.wikipedia.org/wiki/Random_forest) algorithm is cousin with boosting and both are part of the [ensemble learning](http://en.wikipedia.org/wiki/Ensemble_learning) family.
+
+Both trains several decision trees for one dataset. The *main* difference is that in Random Forests™, trees are independent and in boosting, the tree `N+1` focus its learning on the loss (<=> what has not been well modeled by the tree `N`).
+
+This difference have an impact on a corner case in feature importance analysis: the *correlated features*.
+
+Imagine two features perfectly correlated, feature `A` and feature `B`. For one specific tree, if the algorithm needs one of them, it will choose randomly (true in both boosting and Random Forests™).
+
+However, in Random Forests™ this random choice will be done for each tree, because each tree is independent from the others. Therefore, approximatively, depending of your parameters, 50% of the trees will choose feature `A` and the other 50% will choose feature `B`. So the *importance* of the information contained in `A` and `B` (which is the same, because they are perfectly correlated) is diluted in `A` and `B`. So you won't easily know this information is important to predict what you want to predict! It is even worse when you have 10 correlated features...
+
+In boosting, when a specific link between feature and outcome have been learned by the algorithm, it will try to not refocus on it (in theory it is what happens, reality is not always that simple). Therefore, all the importance will be on feature `A` or on feature `B` (but not both). You will know that one feature have an important role in the link between the observations and the label. It is still up to you to search for the correlated features to the one detected as important if you need to know all of them.
+
+If you want to try Random Forests™ algorithm, you can tweak Xgboost parameters! 
+
+**Warning**: this is still an experimental parameter.
+
+For instance, to compute a model with 1000 trees, with a 0.5 factor on sampling rows and columns:
+
+```{r, warning=FALSE, message=FALSE}
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+train <- agaricus.train
+test <- agaricus.test
+
+#Random Forest™ - 1000 trees
+bst <- xgboost(data = train$data, label = train$label, max.depth = 4, num_parallel_tree = 1000, subsample = 0.5, colsample_bytree =0.5, nround = 1, objective = "binary:logistic")
+
+#Boosting - 3 rounds
+bst <- xgboost(data = train$data, label = train$label, max.depth = 4, nround = 3, objective = "binary:logistic")
+```
+
+> Note that the parameter `round` is set to `1`.
+
+> [**Random Forests™**](https://www.stat.berkeley.edu/~breiman/RandomForests/cc_papers.htm) is a trademark of Leo Breiman and Adele Cutler and is licensed exclusively to Salford Systems for the commercial release of the software.

R-package/vignettes/vignette.css

@@ -0,0 +1,225 @@
+body {
+    margin: 0 auto;
+    background-color: white;
+
+/*  --------- FONT FAMILY --------
+   following are some optional font families. Usually a family 
+	is safer to choose than a specific font, 
+	which may not be on the users computer		*/
+/    font-family:Georgia, Palatino, serif;
+    font-family: "Open Sans", "Book Antiqua", Palatino, serif;
+/    font-family:Arial, Helvetica, sans-serif;
+/    font-family:Tahoma, Verdana, Geneva, sans-serif;
+/    font-family:Courier, monospace;
+/    font-family:"Times New Roman", Times, serif;
+
+/*	-------------- COLOR OPTIONS ------------
+ 	following are additional color options for base font
+	you could uncomment another one to easily change the base color 
+	or add one to a specific element style below         */		
+    color: #333333; /* dark gray not black */
+/    color: #000000; /* black */
+/    color: #666666; /* medium gray  black */	
+/    color: #E3E3E3; /* very light gray */
+/    color: white; 
+
+    line-height: 100%;
+    max-width: 800px;
+    padding: 10px;
+    font-size: 17px;
+    text-align: justify;
+    text-justify: inter-word;
+}
+
+
+p {
+    line-height: 150%;
+/    max-width: 540px;
+    max-width: 960px;
+    margin-bottom: 5px;
+    font-weight: 400;    
+/    color: #333333
+}
+
+
+h1, h2, h3, h4, h5, h6 {
+    font-weight: 400;
+    margin-top: 35px;
+    margin-bottom: 15px;
+    padding-top: 10px;
+}
+
+h1 {
+    margin-top: 70px;
+    color: #606AAA;
+    font-size:230%;
+    font-variant:small-caps;
+    padding-bottom:20px;
+    width:100%;
+    border-bottom:1px solid #606AAA;
+}
+
+h2 {
+    font-size:160%;
+}
+
+h3 {
+    font-size:130%;
+}
+
+h4 {
+    font-size:120%;
+    font-variant:small-caps;
+}
+
+h5 {
+    font-size:120%;
+}
+
+h6 {
+    font-size:120%;
+    font-variant:small-caps;
+}
+
+a {
+    color: #606AAA;
+    margin: 0;
+    padding: 0;
+    vertical-align: baseline;
+}
+
+a:hover {
+    text-decoration: blink;
+    color: green;
+}
+
+a:visited {
+    color: gray;
+}
+
+ul, ol {
+    padding: 0;
+    margin: 0px 0px 0px 50px;
+}
+ul {
+    list-style-type: square;
+    list-style-position: inside;
+
+}
+
+li {
+     line-height:150%    
+}
+
+li ul, li ul {
+    margin-left: 24px;
+}
+
+pre {
+    padding: 0px 10px;
+    max-width: 800px;
+    white-space: pre-wrap;
+}
+
+code {
+    font-family: Consolas, Monaco, Andale Mono, monospace, courrier new;
+    line-height: 1.5;
+    font-size: 15px;
+    background: #F8F8F8;
+    border-radius: 4px;
+    padding: 5px;
+    display: inline-block;
+    max-width: 800px;
+    white-space: pre-wrap;
+}
+
+
+li code, p code {
+  background: #CDCDCD;
+  color: #606AAA;
+  padding: 0px 5px 0px 5px;
+}
+
+code.r, code.cpp {
+    display: block;
+    word-wrap: break-word;
+    border: 1px solid #606AAA;        
+}
+
+aside {
+    display: block;
+    float: right;
+    width: 390px;
+}
+
+blockquote {
+    border-left:.5em solid #606AAA;
+    background: #F8F8F8;
+    padding: 0em 1em 0em 1em;
+    margin-left:10px;
+    max-width: 500px;
+}
+
+blockquote cite {
+    line-height:10px;
+    color:#bfbfbf;
+}
+
+blockquote cite:before {
+    /content: '\2014 \00A0';
+}
+
+blockquote p, blockquote li {  
+    color: #666;
+}
+hr {
+/   width: 540px;
+    text-align: left;
+    margin: 0 auto 0 0;
+    color: #999;
+}
+
+
+/* table */
+
+table {
+  width: 100%;
+  border-top: 1px solid #919699;
+	border-left: 1px solid #919699;
+	border-spacing: 0;
+}
+	
+table th {
+	padding: 4px 8px 4px 8px;
+  text-align: center;
+  color: white;
+	background: #606AAA;
+	border-bottom: 1px solid #919699;
+	border-right: 1px solid #919699;
+}
+table th p {
+	font-weight: bold;
+	margin-bottom: 0px; 
+}
+	
+table td {
+	padding: 8px;	
+	vertical-align: top;
+	border-bottom: 1px solid #919699;
+	border-right: 1px solid #919699;
+}
+
+table td:last-child {
+  /background: lightgray;
+  text-align: right;
+}
+
+table td p {
+	margin-bottom: 0px; 
+}
+table td p + p  {
+	margin-top: 5px; 
+}
+table td p + p + p {
+	margin-top: 5px; 
+}

R-package/vignettes/xgboost.Rnw

@@ -0,0 +1,221 @@
+\documentclass{article}
+\RequirePackage{url}
+\usepackage{hyperref}
+\RequirePackage{amsmath}
+\RequirePackage{natbib}
+\RequirePackage[a4paper,lmargin={1.25in},rmargin={1.25in},tmargin={1in},bmargin={1in}]{geometry}
+
+\makeatletter
+% \VignetteIndexEntry{xgboost: eXtreme Gradient Boosting}
+%\VignetteKeywords{xgboost, gbm, gradient boosting machines}
+%\VignettePackage{xgboost}
+% \VignetteEngine{knitr::knitr}
+\makeatother
+
+\begin{document}
+%\SweaveOpts{concordance=TRUE}
+
+<<knitropts,echo=FALSE,message=FALSE>>=
+if (require('knitr')) opts_chunk$set(fig.width = 5, fig.height = 5, fig.align = 'center', tidy = FALSE, warning = FALSE, cache = TRUE)
+@
+
+%
+<<prelim,echo=FALSE>>=
+xgboost.version = '0.3-0'
+@
+%
+
+    \begin{center}
+    \vspace*{6\baselineskip}
+    \rule{\textwidth}{1.6pt}\vspace*{-\baselineskip}\vspace*{2pt}
+    \rule{\textwidth}{0.4pt}\\[2\baselineskip]
+    {\LARGE \textbf{xgboost: eXtreme Gradient Boosting}}\\[1.2\baselineskip]
+    \rule{\textwidth}{0.4pt}\vspace*{-\baselineskip}\vspace{3.2pt}
+    \rule{\textwidth}{1.6pt}\\[2\baselineskip]
+    {\Large Tianqi Chen, Tong He}\\[\baselineskip]
+    {\large Package Version: \Sexpr{xgboost.version}}\\[\baselineskip]
+    {\large \today}\par
+    \vfill
+    \end{center}
+
+\thispagestyle{empty}
+
+\clearpage
+
+\setcounter{page}{1}
+
+\section{Introduction}
+
+This is an introductory document of using the \verb@xgboost@ package in R. 
+
+\verb@xgboost@ is short for eXtreme Gradient Boosting package. It is an efficient
+ and scalable implementation of gradient boosting framework by \citep{friedman2001greedy} \citep{friedman2000additive}. 
+The package includes efficient linear model solver and tree learning algorithm.
+It supports various objective functions, including regression, classification
+and ranking. The package is made to be extendible, so that users are also allowed to define their own objectives easily. It has several features:
+\begin{enumerate}
+    \item{Speed: }{\verb@xgboost@ can automatically do parallel computation on 
+    Windows and Linux, with openmp. It is generally over 10 times faster than
+    \verb@gbm@.}
+    \item{Input Type: }{\verb@xgboost@ takes several types of input data:}
+    \begin{itemize}
+        \item{Dense Matrix: }{R's dense matrix, i.e. \verb@matrix@}
+        \item{Sparse Matrix: }{R's sparse matrix \verb@Matrix::dgCMatrix@}
+        \item{Data File: }{Local data files}
+        \item{xgb.DMatrix: }{\verb@xgboost@'s own class. Recommended.}
+    \end{itemize}
+    \item{Sparsity: }{\verb@xgboost@ accepts sparse input for both tree booster 
+    and linear booster, and is optimized for sparse input.}
+    \item{Customization: }{\verb@xgboost@ supports customized objective function 
+    and evaluation function}
+    \item{Performance: }{\verb@xgboost@ has better performance on several different
+    datasets.}
+\end{enumerate}
+
+
+\section{Example with Mushroom data}
+
+In this section, we will illustrate some common usage of \verb@xgboost@. The 
+Mushroom data is cited from UCI Machine Learning Repository. \citep{Bache+Lichman:2013} 
+
+<<Training and prediction with iris>>=
+library(xgboost)
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+train <- agaricus.train
+test <- agaricus.test
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2, eta = 1, 
+               nround = 2, objective = "binary:logistic")
+xgb.save(bst, 'model.save')
+bst = xgb.load('model.save')
+pred <- predict(bst, test$data)
+@
+
+\verb@xgboost@ is the main function to train a \verb@Booster@, i.e. a model.
+\verb@predict@ does prediction on the model.
+
+Here we can save the model to a binary local file, and load it when needed.
+We can't inspect the trees inside. However we have another function to save the
+model in plain text. 
+<<Dump Model>>=
+xgb.dump(bst, 'model.dump')
+@
+
+The output looks like 
+
+\begin{verbatim}
+booster[0]:
+0:[f28<1.00001] yes=1,no=2,missing=2
+  1:[f108<1.00001] yes=3,no=4,missing=4
+    3:leaf=1.85965
+    4:leaf=-1.94071
+  2:[f55<1.00001] yes=5,no=6,missing=6
+    5:leaf=-1.70044
+    6:leaf=1.71218
+booster[1]:
+0:[f59<1.00001] yes=1,no=2,missing=2
+  1:leaf=-6.23624
+  2:[f28<1.00001] yes=3,no=4,missing=4
+    3:leaf=-0.96853
+    4:leaf=0.784718
+\end{verbatim}
+
+It is important to know \verb@xgboost@'s own data type: \verb@xgb.DMatrix@.
+It speeds up \verb@xgboost@, and is needed for advanced features such as 
+training from initial prediction value, weighted training instance. 
+
+We can use \verb@xgb.DMatrix@ to construct an \verb@xgb.DMatrix@ object:
+<<xgb.DMatrix>>=
+dtrain <- xgb.DMatrix(train$data, label = train$label)
+class(dtrain)
+head(getinfo(dtrain,'label'))
+@
+
+We can also save the matrix to a binary file. Then load it simply with 
+\verb@xgb.DMatrix@
+<<save model>>=
+xgb.DMatrix.save(dtrain, 'xgb.DMatrix')
+dtrain = xgb.DMatrix('xgb.DMatrix')
+@
+
+\section{Advanced Examples}
+
+The function \verb@xgboost@ is a simple function with less parameter, in order
+to be R-friendly. The core training function is wrapped in \verb@xgb.train@. It is more flexible than \verb@xgboost@, but it requires users to read the document a bit more carefully.
+
+\verb@xgb.train@ only accept a \verb@xgb.DMatrix@ object as its input, while it supports advanced features as custom objective and evaluation functions.
+
+<<Customized loss function>>=
+logregobj <- function(preds, dtrain) {
+   labels <- getinfo(dtrain, "label")
+   preds <- 1/(1 + exp(-preds))
+   grad <- preds - labels
+   hess <- preds * (1 - preds)
+   return(list(grad = grad, hess = hess))
+}
+
+evalerror <- function(preds, dtrain) {
+  labels <- getinfo(dtrain, "label")
+  err <- sqrt(mean((preds-labels)^2))
+  return(list(metric = "MSE", value = err))
+}
+
+dtest <- xgb.DMatrix(test$data, label = test$label)
+watchlist <- list(eval = dtest, train = dtrain)
+param <- list(max.depth = 2, eta = 1, silent = 1)
+
+bst <- xgb.train(param, dtrain, nround = 2, watchlist, logregobj, evalerror)
+@
+
+The gradient and second order gradient is required for the output of customized 
+objective function. 
+
+We also have \verb@slice@ for row extraction. It is useful in 
+cross-validation.
+
+For a walkthrough demo, please see \verb@R-package/demo/@ for further 
+details.
+
+\section{The Higgs Boson competition}
+
+We have made a demo for \href{http://www.kaggle.com/c/higgs-boson}{the Higgs 
+Boson Machine Learning Challenge}. 
+
+Here are the instructions to make a submission
+\begin{enumerate}
+    \item Download the \href{http://www.kaggle.com/c/higgs-boson/data}{datasets}
+    and extract them to \verb@data/@.
+    \item Run scripts under \verb@xgboost/demo/kaggle-higgs/@: 
+    \href{https://github.com/tqchen/xgboost/blob/master/demo/kaggle-higgs/higgs-train.R}{higgs-train.R} 
+    and \href{https://github.com/tqchen/xgboost/blob/master/demo/kaggle-higgs/higgs-pred.R}{higgs-pred.R}. 
+    The computation will take less than a minute on Intel i7. 
+    \item Go to the \href{http://www.kaggle.com/c/higgs-boson/submissions/attach}{submission page} 
+    and submit your result.
+\end{enumerate}
+
+We provide \href{https://github.com/tqchen/xgboost/blob/master/demo/kaggle-higgs/speedtest.R}{a script}
+to compare the time cost on the higgs dataset with \verb@gbm@ and \verb@xgboost@. 
+The training set contains 350000 records and 30 features. 
+
+\verb@xgboost@ can automatically do parallel computation. On a machine with Intel
+i7-4700MQ and 24GB memories, we found that \verb@xgboost@ costs about 35 seconds, which is about 20 times faster
+than \verb@gbm@. When we limited \verb@xgboost@ to use only one thread, it was 
+still about two times faster than \verb@gbm@. 
+
+Meanwhile, the result from \verb@xgboost@ reaches 
+\href{http://www.kaggle.com/c/higgs-boson/details/evaluation}{3.60@AMS} with a 
+single model. This results stands in the 
+\href{http://www.kaggle.com/c/higgs-boson/leaderboard}{top 30\%} of the 
+competition. 
+
+\bibliographystyle{jss}
+\nocite{*} % list uncited references
+\bibliography{xgboost}
+
+\end{document}
+
+<<Temp file cleaning, include=FALSE>>=
+file.remove("xgb.DMatrix")
+file.remove("model.dump")
+file.remove("model.save")
+@

R-package/vignettes/xgboost.bib

@@ -0,0 +1,30 @@
+@article{friedman2001greedy,
+    title={Greedy function approximation: a gradient boosting machine},
+    author={Friedman, Jerome H},
+    journal={Annals of Statistics},
+    pages={1189--1232},
+    year={2001},
+    publisher={JSTOR}
+}
+
+@article{friedman2000additive,
+  title={Additive logistic regression: a statistical view of boosting (with discussion and a rejoinder by the authors)},
+  author={Friedman, Jerome and Hastie, Trevor and Tibshirani, Robert and others},
+  journal={The annals of statistics},
+  volume={28},
+  number={2},
+  pages={337--407},
+  year={2000},
+  publisher={Institute of Mathematical Statistics}
+}
+
+
+@misc{
+    Bache+Lichman:2013 ,
+    author = "K. Bache and M. Lichman",
+    year = "2013",
+    title = "{UCI} Machine Learning Repository",
+    url = "http://archive.ics.uci.edu/ml",
+    institution = "University of California, Irvine, School of Information and Computer Sciences" 
+}
+

R-package/vignettes/xgboostPresentation.Rmd

@@ -0,0 +1,405 @@
+---
+title: "Xgboost presentation"
+output: 
+  rmarkdown::html_vignette:
+    css: vignette.css
+    number_sections: yes
+    toc: yes
+bibliography: xgboost.bib
+author: Tianqi Chen, Tong He, Michaël Benesty
+vignette: >
+  %\VignetteIndexEntry{Xgboost presentation}
+  %\VignetteEngine{knitr::rmarkdown}
+  \usepackage[utf8]{inputenc}
+---
+
+Introduction
+============
+
+**Xgboost** is short for e**X**treme **G**radient **Boost**ing package. 
+
+The purpose of this Vignette is to show you how to use **Xgboost** to build a model and make predictions.
+
+It is an efficient and scalable implementation of gradient boosting framework by @friedman2000additive and @friedman2001greedy. Two solvers are included:
+
+- *linear* model ;
+- *tree learning* algorithm.
+
+It supports various objective functions, including *regression*, *classification* and *ranking*. The package is made to be extendible, so that users are also allowed to define their own objective functions easily. 
+
+It has been [used](https://github.com/dmlc/xgboost) to win several [Kaggle](http://www.kaggle.com) competitions. 
+
+It has several features:
+
+* Speed: it can automatically do parallel computation on *Windows* and *Linux*, with *OpenMP*. It is generally over 10 times faster than the classical `gbm`.
+* Input Type: it takes several types of input data:
+    * *Dense* Matrix: *R*'s *dense* matrix, i.e. `matrix` ;
+    * *Sparse* Matrix: *R*'s *sparse* matrix, i.e. `Matrix::dgCMatrix` ;
+    * Data File: local data files ;
+    * `xgb.DMatrix`: its own class (recommended).
+* Sparsity: it accepts *sparse* input for both *tree booster*  and *linear booster*, and is optimized for *sparse* input ;
+* Customization: it supports customized objective functions and evaluation functions.
+
+Installation
+============
+
+Github version
+--------------
+
+For up-to-date version (highly recommended), install from *Github*:
+
+```{r installGithub, eval=FALSE}
+devtools::install_github('dmlc/xgboost', subdir='R-package')
+```
+
+> *Windows* user will need to install [RTools](http://cran.r-project.org/bin/windows/Rtools/) first.
+
+Cran version
+------------
+
+For stable version on *CRAN*, run:
+
+```{r installCran, eval=FALSE}
+install.packages('xgboost')
+```
+
+Learning
+========
+
+For the purpose of this tutorial we will load **Xgboost** package.
+
+```{r libLoading, results='hold', message=F, warning=F}
+require(xgboost)
+```
+
+Dataset presentation
+--------------------
+
+In this example, we are aiming to predict whether a mushroom can be eaten or not (like in many tutorials, example data are the the same as you will use on in your every day life :-). 
+
+Mushroom data is cited from UCI Machine Learning Repository. @Bache+Lichman:2013.
+
+Dataset loading
+---------------
+
+We will load the `agaricus` datasets embedded with the package and will link them to variables.
+
+The datasets are already split in:
+
+* `train`: will be used to build the model ;
+* `test`: will be used to assess the quality of our model. 
+
+Why *split* the dataset in two parts?
+
+In the first part we will build our model. In the second part we will want to test it and assess its quality. Without dividing the dataset we would test the model on the data which the algorithm have already seen.
+
+```{r datasetLoading, results='hold', message=F, warning=F}
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+train <- agaricus.train
+test <- agaricus.test
+```
+
+> In the real world, it would be up to you to make this division between `train` and `test` data. The way to do it is out of the purpose of this article, however `caret` package may [help](http://topepo.github.io/caret/splitting.html).
+
+Each variable is a `list` containing two things, `label` and `data`:
+
+```{r dataList, message=F, warning=F}
+str(train)
+```
+
+`label` is the outcome of our dataset meaning it is the binary *classification* we will try to predict.
+
+Let's discover the dimensionality of our datasets.
+
+```{r dataSize, message=F, warning=F}
+dim(train$data)
+dim(test$data)
+```
+
+This dataset is very small to not make the **R** package too heavy, however **Xgboost** is built to manage huge dataset very efficiently.
+
+As seen below, the `data` are stored in a `dgCMatrix` which is a *sparse* matrix and `label` vector is a `numeric` vector (`{0,1}`):
+
+```{r dataClass, message=F, warning=F}
+class(train$data)[1]
+class(train$label)
+```
+
+Basic Training using Xgboost
+----------------------------
+
+This step is the most critical part of the process for the quality of our model.
+
+### Basic training
+
+We are using the `train` data. As explained above, both `data` and `label` are stored in a `list`.
+
+In a *sparse* matrix, cells containing `0` are not stored in memory. Therefore, in a dataset mainly made of `0`, memory size is reduced. It is very usual to have such dataset.
+
+We will train decision tree model using the following parameters:
+
+* `objective = "binary:logistic"`: we will train a binary classification model ;
+* `max.deph = 2`: the trees won't be deep, because our case is very simple ;
+* `nthread = 2`: the number of cpu threads we are going to use;
+* `nround = 2`: there will be two passes on the data, the second one will enhance the model by further reducing the difference between ground truth and prediction.
+
+```{r trainingSparse, message=F, warning=F}
+bstSparse <- xgboost(data = train$data, label = train$label, max.depth = 2, eta = 1, nthread = 2, nround = 2, objective = "binary:logistic")
+```
+
+> More complex the relationship between your features and your `label` is, more passes you need.
+
+### Parameter variations
+
+#### Dense matrix
+
+Alternatively, you can put your dataset in a *dense* matrix, i.e. a basic **R** matrix.
+
+```{r trainingDense, message=F, warning=F}
+bstDense <- xgboost(data = as.matrix(train$data), label = train$label, max.depth = 2, eta = 1, nthread = 2, nround = 2, objective = "binary:logistic")
+```
+
+#### xgb.DMatrix
+
+**Xgboost** offers a way to group them in a `xgb.DMatrix`. You can even add other meta data in it. It will be usefull for the most advanced features we will discover later.
+
+```{r trainingDmatrix, message=F, warning=F}
+dtrain <- xgb.DMatrix(data = train$data, label = train$label)
+bstDMatrix <- xgboost(data = dtrain, max.depth = 2, eta = 1, nthread = 2, nround = 2, objective = "binary:logistic")
+```
+
+#### Verbose option
+
+**Xgboost** has severa features to help you to view how the learning progress internally. The purpose is to help you to set the best parameters, which is the key of your model quality.
+
+One of the simplest way to see the training progress is to set the `verbose` option (see below for more advanced technics).
+
+```{r trainingVerbose0, message=T, warning=F}
+# verbose = 0, no message
+bst <- xgboost(data = dtrain, max.depth = 2, eta = 1, nthread = 2, nround = 2, objective = "binary:logistic", verbose = 0)
+```
+
+```{r trainingVerbose1, message=T, warning=F}
+# verbose = 1, print evaluation metric
+bst <- xgboost(data = dtrain, max.depth = 2, eta = 1, nthread = 2, nround = 2, objective = "binary:logistic", verbose = 1)
+```
+
+```{r trainingVerbose2, message=T, warning=F}
+# verbose = 2, also print information about tree
+bst <- xgboost(data = dtrain, max.depth = 2, eta = 1, nthread = 2, nround = 2, objective = "binary:logistic", verbose = 2)
+```
+
+Basic prediction using Xgboost
+==============================
+
+Perform the prediction
+----------------------
+
+The pupose of the model we have built is to classify new data. As explained before, we will use the `test` dataset for this step.
+
+```{r predicting, message=F, warning=F}
+pred <- predict(bst, test$data)
+
+# size of the prediction vector
+print(length(pred))
+
+# limit display of predictions to the first 10
+print(head(pred))
+```
+
+These numbers doesn't look like *binary classification* `{0,1}`. We need to perform a simple transformation before being able to use these results.
+
+Transform the regression in a binary classification
+---------------------------------------------------
+
+The only thing that **Xgboost** does is a *regression*. **Xgboost** is using `label` vector to build its *regression* model.
+
+How can we use a *regression* model to perform a binary classification?
+
+If we think about the meaning of a regression applied to our data, the numbers we get are probabilities that a datum will be classified as `1`. Therefore, we will set the rule that if this probability for a specific datum is `> 0.5` then the observation is classified as `1` (or `0` otherwise).
+
+```{r predictingTest, message=F, warning=F}
+prediction <- as.numeric(pred > 0.5)
+print(head(prediction))
+```
+
+Measuring model performance
+---------------------------
+
+To measure the model performance, we will compute a simple metric, the *average error*.
+
+```{r predictingAverageError, message=F, warning=F}
+err <- mean(as.numeric(pred > 0.5) != test$label)
+print(paste("test-error=", err))
+```
+
+> Note that the algorithm has not seen the `test` data during the model construction.
+
+Steps explanation:
+
+1. `as.numeric(pred > 0.5)` applies our rule that when the probability (<=> regression <=> prediction) is `> 0.5` the observation is classified as `1` and `0` otherwise ;
+2. `probabilityVectorPreviouslyComputed != test$label` computes the vector of error between true data and computed probabilities ;
+3. `mean(vectorOfErrors)` computes the *average error* itself.
+
+The most important thing to remember is that **to do a classification, you just do a regression to the** `label` **and then apply a threshold**. 
+
+*Multiclass* classification works in a similar way.
+
+This metric is **`r round(err, 2)`** and is pretty low: our yummly mushroom model works well!
+
+Advanced features
+=================
+
+Most of the features below have been implemented to help you to improve your model by offering a better understanding of its content.
+
+
+Dataset preparation
+-------------------
+
+For the following advanced features, we need to put data in `xgb.DMatrix` as explained above.
+
+```{r DMatrix, message=F, warning=F}
+dtrain <- xgb.DMatrix(data = train$data, label=train$label)
+dtest <- xgb.DMatrix(data = test$data, label=test$label)
+```
+
+Measure learning progress with xgb.train
+----------------------------------------
+
+Both `xgboost` (simple) and `xgb.train` (advanced) functions train models.
+
+One of the special feature of `xgb.train` is the capacity to follow the progress of the learning after each round. Because of the way boosting works, there is a time when having too many rounds lead to an overfitting. You can see this feature as a cousin of cross-validation method. The following technics will help you to avoid overfitting or optimizing the learning time in stopping it as soon as possible.
+
+One way to measure progress in learning of a model is to provide to **Xgboost** a second dataset already classified. Therefore it can learn on the first dataset and test its model on the second one. Some metrics are measured after each round during the learning.
+
+> in some way it is similar to what we have done above with the average error. The main difference is that below it was after building the model, and now it is during the construction that we measure errors.
+
+For the purpose of this example, we use `watchlist` parameter. It is a list of `xgb.DMatrix`, each of them tagged with a name.
+
+```{r watchlist, message=F, warning=F}
+watchlist <- list(train=dtrain, test=dtest)
+
+bst <- xgb.train(data=dtrain, max.depth=2, eta=1, nthread = 2, nround=2, watchlist=watchlist, objective = "binary:logistic")
+```
+
+**Xgboost** has computed at each round the same average error metric than seen above (we set `nround` to 2, that is why we have two lines). Obviously, the `train-error` number is related to the training dataset (the one the algorithm learns from) and the `test-error` number to the test dataset. 
+
+Both training and test error related metrics are very similar, and in some way, it makes sense: what we have learned from the training dataset matches the observations from the test dataset.
+
+If with your own dataset you have not such results, you should think about how you did to divide your dataset in training and test. May be there is something to fix. Again, `caret` package may [help](http://topepo.github.io/caret/splitting.html).
+
+For a better understanding of the learning progression, you may want to have some specific metric or even use multiple evaluation metrics.
+
+```{r watchlist2, message=F, warning=F}
+bst <- xgb.train(data=dtrain, max.depth=2, eta=1, nthread = 2, nround=2, watchlist=watchlist, eval.metric = "error", eval.metric = "logloss", objective = "binary:logistic")
+```
+
+> `eval.metric` allows us to monitor two new metrics for each round, `logloss` and `error`.
+
+Linear boosting
+---------------
+
+Until know, all the learnings we have performed were based on boosting trees. **Xgboost** implements a second algorithm, based on linear boosting. The only difference with previous command is `booster = "gblinear"` parameter (and removing `eta` parameter).
+
+```{r linearBoosting, message=F, warning=F}
+bst <- xgb.train(data=dtrain, booster = "gblinear", max.depth=2, nthread = 2, nround=2, watchlist=watchlist, eval.metric = "error", eval.metric = "logloss", objective = "binary:logistic")
+```
+
+In this specific case, *linear boosting* gets sligtly better performance metrics than decision trees based algorithm. 
+
+In simple cases, it will happem because there is nothing better than a linear algorithm to catch a linear link. However, decision trees are much better to catch a non linear link between predictors and outcome. Because there is no silver bullet, we advise you to check both algorithms with your own datasets to have an idea of what to use.
+
+Manipulating xgb.DMatrix
+------------------------
+
+### Save / Load
+
+Like saving models, `xgb.DMatrix` object (which groups both dataset and outcome) can also be saved using `xgb.DMatrix.save` function.
+
+```{r DMatrixSave, message=F, warning=F}
+xgb.DMatrix.save(dtrain, "dtrain.buffer")
+# to load it in, simply call xgb.DMatrix
+dtrain2 <- xgb.DMatrix("dtrain.buffer")
+bst <- xgb.train(data=dtrain2, max.depth=2, eta=1, nthread = 2, nround=2, watchlist=watchlist, objective = "binary:logistic")
+```
+
+```{r DMatrixDel, include=FALSE}
+file.remove("dtrain.buffer")
+```
+
+### Information extraction
+
+Information can be extracted from `xgb.DMatrix` using `getinfo` function. Hereafter we will extract `label` data.
+
+```{r getinfo, message=F, warning=F}
+label = getinfo(dtest, "label")
+pred <- predict(bst, dtest)
+err <- as.numeric(sum(as.integer(pred > 0.5) != label))/length(label)
+print(paste("test-error=", err))
+```
+
+View the trees from a model
+---------------------------
+
+You can dump the tree you learned using `xgb.dump` into a text file.
+
+```{r dump, message=T, warning=F}
+xgb.dump(bst, with.stats = T)
+```
+
+> if you provide a path to `fname` parameter you can save the trees to your hard drive.
+
+Save and load models
+--------------------
+
+May be your dataset is big, and it takes time to train a model on it? May be you are not a big fan of loosing time in redoing the same task again and again? In these very rare cases, you will want to save your model and load it when required.
+
+Hopefully for you, **Xgboost** implements such functions.
+
+```{r saveModel, message=F, warning=F}
+# save model to binary local file
+xgb.save(bst, "xgboost.model")
+```
+
+> `xgb.save` function should return `r TRUE` if everything goes well and crashes otherwise.
+
+An interesting test to see how identic is our saved model with the original one would be to compare the two predictions.
+
+```{r loadModel, message=F, warning=F}
+# load binary model to R
+bst2 <- xgb.load("xgboost.model")
+pred2 <- predict(bst2, test$data)
+
+# And now the test
+print(paste("sum(abs(pred2-pred))=", sum(abs(pred2-pred))))
+```
+
+```{r clean, include=FALSE}
+# delete the created model
+file.remove("./xgboost.model")
+```
+
+> result is `0`? We are good!
+
+In some very specific cases, like when you want to pilot **Xgboost** from `caret` package, you will want to save the model as a *R* binary vector. See below how to do it.
+
+```{r saveLoadRBinVectorModel, message=F, warning=F}
+# save model to R's raw vector
+rawVec <- xgb.save.raw(bst)
+
+# print class
+print(class(rawVec))
+
+# load binary model to R
+bst3 <- xgb.load(rawVec)
+pred3 <- predict(bst3, test$data)
+
+# pred2 should be identical to pred
+print(paste("sum(abs(pred3-pred))=", sum(abs(pred2-pred))))
+``` 
+
+> Again `0`? It seems that `Xgboost` works pretty well!
+
+References
+==========

README.md

@@ -1,40 +1,57 @@
-xgboost: eXtreme Gradient Boosting 
-=======
-A General purpose gradient boosting (tree) library.
+XGBoost: eXtreme Gradient Boosting 
+==================================
 
-Authors:
-* Tianqi Chen, project creater
-* Kailong Chen, contributes regression module
+An optimized general purpose gradient boosting library. The library is parallelized, and also provides an optimized distributed version.
+It implements machine learning algorithm under gradient boosting framework, including generalized linear model and gradient boosted regression tree (GBDT). XGBoost can also also distributed and scale to Terascale data
+
+Contributors: https://github.com/dmlc/xgboost/graphs/contributors
+
+Documentations: [Documentation of xgboost](doc/README.md)
+
+Issues Tracker: [https://github.com/dmlc/xgboost/issues](https://github.com/dmlc/xgboost/issues?q=is%3Aissue+label%3Aquestion)
+
+Please join [XGBoost User Group](https://groups.google.com/forum/#!forum/xgboost-user/) to ask questions and share your experience on xgboost.
+  - Use issue tracker for bug reports, feature requests etc.
+  - Use the user group to post your experience, ask questions about general usages.
+
+Gitter for developers [![Gitter chat for developers at https://gitter.im/dmlc/xgboost](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/dmlc/xgboost?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+
+Distributed Version: [Distributed XGBoost](multi-node)
+
+Highlights of Usecases: [Highlight Links](doc/README.md#highlight-links)
+
+What's New
+==========
+* XGBoost-0.4 release, see [CHANGES.md](CHANGES.md#xgboost-04)
+* XGBoost wins [WWW2015  Microsoft Malware Classification Challenge (BIG 2015)](http://www.kaggle.com/c/malware-classification/forums/t/13490/say-no-to-overfitting-approaches-sharing)
+  - Checkout the winning solution at [Highlight links](doc/README.md#highlight-links)
+* [External Memory Version](doc/external_memory.md)
 
-Turorial and Documentation: https://github.com/tqchen/xgboost/wiki
- 
 Features
-=======
-* Sparse feature format:
-  - Sparse feature format allows easy handling of missing values, and improve computation efficiency.
-* Push the limit on single machine:
-  - Efficient implementation that optimizes memory and computation.
-* Layout of gradient boosting algorithm to support generic tasks, see project wiki.
+========
+* Easily accessible in python, R, Julia, CLI
+* Fast speed and memory efficient
+  - Can be more than 10 times faster than GBM in sklearn and R
+  - Handles sparse matrices, support external memory
+* Accurate prediction, and used extensively by data scientists and kagglers
+  - See [highlight links](https://github.com/dmlc/xgboost/blob/master/doc/README.md#highlight-links)
+* Distributed and Portable
+  - The distributed version runs on Hadoop (YARN), MPI, SGE etc.
+  - Scales to billions of examples and beyond
 
-Supported key components
+Build
 =======
-* Gradient boosting models: 
-    - regression tree (GBRT)
-    - linear model/lasso
-* Objectives to support tasks: 
-    - regression
-    - classification
-* OpenMP implementation
+* Run ```bash build.sh``` (you can also type make)
+  - Normally it gives what you want
+  - See [Build Instruction](doc/build.md) for more information
 
-Planned components
+Version
 =======
-* More objective to support tasks: 
-    - ranking
-    - matrix factorization
-    - structured prediction
+* Current version xgboost-0.4, a lot improvment has been made since 0.3
+  - Change log in [CHANGES.md](CHANGES.md)
+  - This version is compatible with 0.3x versions
 
-File extension convention
-=======
-* .h are interface, utils and data structures, with detailed comment; 
-* .cpp are implementations that will be compiled, with less comment; 
-* .hpp are implementations that will be included by .cpp, with less comment
+XGBoost in Graphlab Create
+==========================
+* XGBoost is adopted as part of boosted tree toolkit in Graphlab Create (GLC). Graphlab Create is a powerful python toolkit that allows you to data manipulation, graph processing, hyper-parameter search, and visualization of TeraBytes scale data in one framework. Try the Graphlab Create in http://graphlab.com/products/create/quick-start-guide.html
+* Nice blogpost by Jay Gu using GLC boosted tree to solve kaggle bike sharing challenge: http://blog.graphlab.com/using-gradient-boosted-trees-to-predict-bike-sharing-demand

booster/linear/xgboost_linear.hpp

@@ -1,200 +0,0 @@
-#ifndef XGBOOST_LINEAR_HPP
-#define XGBOOST_LINEAR_HPP
-/*!
- * \file xgboost_linear.h
- * \brief Implementation of Linear booster, with L1/L2 regularization: Elastic Net
- *        the update rule is coordinate descent, require column major format
- * \author Tianqi Chen: tianqi.tchen@gmail.com
- */
-#include <vector>
-#include <algorithm>
-
-#include "../xgboost.h"
-#include "../../utils/xgboost_utils.h"
-
-namespace xgboost{
-    namespace booster{
-        /*! \brief linear model, with L1/L2 regularization */
-        template<typename FMatrix>
-        class LinearBooster : public InterfaceBooster<FMatrix>{
-        public:
-            LinearBooster( void ){ silent = 0;}
-            virtual ~LinearBooster( void ){}
-        public:
-            virtual void SetParam( const char *name, const char *val ){
-                if( !strcmp( name, "silent") )  silent = atoi( val );
-                if( model.weight.size() == 0 )  model.param.SetParam( name, val );
-                param.SetParam( name, val );
-            }
-            virtual void LoadModel( utils::IStream &fi ){
-                model.LoadModel( fi );
-            }
-            virtual void SaveModel( utils::IStream &fo ) const{
-                model.SaveModel( fo );
-            }
-            virtual void InitModel( void ){
-                model.InitModel();
-            }
-        public:
-            virtual void DoBoost( std::vector<float> &grad, 
-                                  std::vector<float> &hess,
-                                  const FMatrix &fmat,
-                                  const std::vector<unsigned> &root_index ){
-                utils::Assert( grad.size() < UINT_MAX, "number of instance exceed what we can handle" );
-                this->UpdateWeights( grad, hess, fmat );
-            }
-            inline float Predict( const FMatrix &fmat, bst_uint ridx, unsigned root_index ){
-                float sum = model.bias();
-                for( typename FMatrix::RowIter it = fmat.GetRow(ridx); it.Next(); ){ 
-                    sum += model.weight[ it.findex() ] * it.fvalue();
-                }
-                return sum;
-            }
-            virtual float Predict( const std::vector<float> &feat, 
-                                   const std::vector<bool>  &funknown,
-                                   unsigned rid = 0 ){
-                float sum = model.bias();
-                for( size_t i = 0; i < feat.size(); i ++ ){
-                    if( funknown[i] ) continue;
-                    sum += model.weight[ i ] * feat[ i ];
-                }
-                return sum;
-            }
-            
-        protected:
-            // training parameter
-            struct ParamTrain{
-                /*! \brief learning_rate */
-                float learning_rate;
-                /*! \brief regularization weight for L2 norm */
-                float reg_lambda;
-                /*! \brief regularization weight for L1 norm */
-                float reg_alpha;
-                 /*! \brief regularization weight for L2 norm  in bias */               
-                float reg_lambda_bias;
-                
-                ParamTrain( void ){
-                    reg_alpha = 0.0f; reg_lambda = 0.0f; reg_lambda_bias = 0.0f;
-                    learning_rate = 1.0f;
-                }            
-                inline void SetParam( const char *name, const char *val ){
-                    // sync-names
-                    if( !strcmp( "eta", name ) )    learning_rate = (float)atof( val );
-                    if( !strcmp( "lambda", name ) ) reg_lambda = (float)atof( val );
-                    if( !strcmp( "alpha", name ) )  reg_alpha  = (float)atof( val );
-                    if( !strcmp( "lambda_bias", name ) ) reg_lambda_bias = (float)atof( val );
-                    // real names
-                    if( !strcmp( "learning_rate", name ) ) learning_rate = (float)atof( val );     
-                    if( !strcmp( "reg_lambda", name ) )    reg_lambda = (float)atof( val );
-                    if( !strcmp( "reg_alpha", name ) )     reg_alpha = (float)atof( val );
-                    if( !strcmp( "reg_lambda_bias", name ) )    reg_lambda_bias = (float)atof( val );
-                }
-                // given original weight calculate delta 
-                inline double CalcDelta( double sum_grad, double sum_hess, double w ){
-                    if( sum_hess < 1e-5f ) return 0.0f;
-                    double tmp = w - ( sum_grad + reg_lambda*w )/( sum_hess + reg_lambda );
-                    if ( tmp >=0 ){
-                        return std::max(-( sum_grad + reg_lambda*w + reg_alpha)/(sum_hess+reg_lambda),-w);
-                    }else{
-                        return std::min(-( sum_grad + reg_lambda*w - reg_alpha)/(sum_hess+reg_lambda),-w);
-                    }
-                }
-                // given original weight calculate delta bias
-                inline double CalcDeltaBias( double sum_grad, double sum_hess, double w ){
-                    return - (sum_grad + reg_lambda_bias*w) / (sum_hess + reg_lambda_bias );
-                }
-            };
-            
-            // model for linear booster
-            class Model{
-            public:
-                // model parameter
-                struct Param{
-                    // number of feature dimension
-                    int num_feature;
-                    // reserved field
-                    int reserved[ 32 ];
-                    // constructor
-                    Param( void ){
-                        num_feature = 0;
-                        memset( reserved, 0, sizeof(reserved) );
-                    }
-                    inline void SetParam( const char *name, const char *val ){
-                        if( !strcmp( name, "num_feature" ) )  num_feature = atoi( val );
-                    }
-                };
-            public:
-                Param param;
-                // weight for each of feature, bias is the last one
-                std::vector<float> weight;
-            public:
-                // initialize the model parameter
-                inline void InitModel( void ){
-                    // bias is the last weight
-                    weight.resize( param.num_feature + 1 );
-                    std::fill( weight.begin(), weight.end(), 0.0f );
-                }
-                // save the model to file 
-                inline void SaveModel( utils::IStream &fo ) const{
-                    fo.Write( &param, sizeof(Param) );
-                    fo.Write( &weight[0], sizeof(float) * weight.size() );
-                }
-                // load model from file
-                inline void LoadModel( utils::IStream &fi ){
-                    utils::Assert( fi.Read( &param, sizeof(Param) ) != 0, "Load LinearBooster" );
-                    weight.resize( param.num_feature + 1 );
-                    utils::Assert( fi.Read( &weight[0], sizeof(float) * weight.size() ) != 0, "Load LinearBooster" );
-                }
-                // model bias
-                inline float &bias( void ){
-                    return weight.back();
-                }
-            };
-        private:
-            int silent;
-        protected:
-            Model model;
-            ParamTrain param;
-        protected:
-            // update weights, should work for any FMatrix
-            inline void UpdateWeights( std::vector<float> &grad,                       
-                                       const std::vector<float> &hess,
-                                       const FMatrix &smat ){
-                {// optimize bias
-                    double sum_grad = 0.0, sum_hess = 0.0;
-                    for( size_t i = 0; i < grad.size(); i ++ ){
-                        sum_grad += grad[ i ]; sum_hess += hess[ i ];
-                    }
-                    // remove bias effect
-                    double dw = param.learning_rate * param.CalcDeltaBias( sum_grad, sum_hess, model.bias() );
-                    model.bias() += dw;
-                    // update grad value 
-                    for( size_t i = 0; i < grad.size(); i ++ ){
-                        grad[ i ] += dw * hess[ i ];
-                    }
-                }
-
-                // optimize weight
-                const unsigned nfeat= (unsigned)smat.NumCol();                           
-                for( unsigned i = 0; i < nfeat; i ++ ){
-                    if( !smat.GetSortedCol( i ).Next() ) continue;
-                    double sum_grad = 0.0, sum_hess = 0.0;
-                    for( typename FMatrix::ColIter it = smat.GetSortedCol(i); it.Next(); ){
-                        const float v = it.fvalue();
-                        sum_grad += grad[ it.rindex() ] * v;
-                        sum_hess += hess[ it.rindex() ] * v * v;
-                    }
-                    float w = model.weight[ i ];
-                    double dw = param.learning_rate * param.CalcDelta( sum_grad, sum_hess, w );
-                    model.weight[ i ] += dw;
-                    // update grad value 
-                    for( typename FMatrix::ColIter it = smat.GetSortedCol(i); it.Next(); ){
-                        const float v = it.fvalue();
-                        grad[ it.rindex() ] += hess[ it.rindex() ] * v * dw;
-                    }
-                }
-            }
-        };
-    };
-};
-#endif

booster/tree/xgboost_base_treemaker.hpp

@@ -1,147 +0,0 @@
-#ifndef XGBOOST_BASE_TREEMAKER_HPP
-#define XGBOOST_BASE_TREEMAKER_HPP
-/*!
- * \file xgboost_base_treemaker.hpp
- * \brief implementation of base data structure for regression tree maker,
- *         gives common operations of tree construction steps template 
- * 
- * \author Tianqi Chen: tianqi.tchen@gmail.com 
- */
-#include <vector>
-#include "xgboost_tree_model.h"
-
-namespace xgboost{
-    namespace booster{
-        class BaseTreeMaker{
-        protected:
-            BaseTreeMaker( RegTree &tree,
-                           const TreeParamTrain &param )
-                : tree( tree ), param( param ){}
-        protected:
-            // statistics that is helpful to decide a split
-            struct SplitEntry{
-                /*! \brief loss change after split this node */
-                float  loss_chg;
-                /*! \brief split index */
-                unsigned  sindex;
-                /*! \brief split value */
-                float     split_value;
-                /*! \brief constructor */
-                SplitEntry( void ){
-                    loss_chg = 0.0f;
-                    split_value = 0.0f; sindex = 0;
-                }
-                // This function gives better priority to lower index when loss_chg equals
-                // not the best way, but helps to give consistent result during multi-thread execution
-                inline bool NeedReplace( float loss_chg, unsigned split_index ) const{
-                    if( this->split_index() <= split_index ){
-                        return loss_chg > this->loss_chg; 
-                    }else{
-                        return !(this->loss_chg > loss_chg);
-                    }
-                }
-                inline bool Update( const SplitEntry &e ){
-                    if( this->NeedReplace( e.loss_chg, e.split_index() ) ){
-                        this->loss_chg = e.loss_chg;
-                        this->sindex = e.sindex;
-                        this->split_value = e.split_value;
-                        return true;
-                    } else{
-                        return false;
-                    }
-                }
-                inline bool Update( float loss_chg, unsigned split_index, float split_value, bool default_left ){                    
-                    if( this->NeedReplace( loss_chg, split_index ) ){
-                        this->loss_chg = loss_chg;
-                        if( default_left ) split_index |= (1U << 31);
-                        this->sindex = split_index;
-                        this->split_value = split_value;
-                        return true;
-                    }else{
-                        return false;
-                    }
-                }
-                inline unsigned split_index( void ) const{
-                    return sindex & ( (1U<<31) - 1U );
-                }
-                inline bool default_left( void ) const{
-                    return (sindex >> 31) != 0;
-                }
-            };
-            struct NodeEntry{
-                /*! \brief sum gradient statistics */
-                double sum_grad;
-                /*! \brief sum hessian statistics */
-                double sum_hess;
-                /*! \brief loss of this node, without split */
-                float  root_gain;
-                /*! \brief weight calculated related to current data */
-                float  weight;
-                /*! \brief current best solution */
-                SplitEntry best;
-                NodeEntry( void ){
-                    sum_grad = sum_hess = 0.0;
-                    weight = root_gain = 0.0f;
-                }
-            };
-        private:
-            // try to prune off current leaf, return true if successful
-            inline void TryPruneLeaf( int nid, int depth ){
-                if( tree[ nid ].is_root() ) return;
-                int pid = tree[ nid ].parent();
-                RegTree::NodeStat &s = tree.stat( pid );
-                ++ s.leaf_child_cnt;
-                
-                if( s.leaf_child_cnt >= 2 && param.need_prune( s.loss_chg, depth - 1 ) ){
-                    this->stat_num_pruned += 2;
-                    // need to be pruned
-                    tree.ChangeToLeaf( pid, param.learning_rate * s.base_weight );
-                    // tail recursion
-                    this->TryPruneLeaf( pid, depth - 1 );
-                }
-            }
-        protected:
-            /*! \brief do prunning of a tree */
-            inline int DoPrune( void ){
-                this->stat_num_pruned = 0;
-                // initialize auxiliary statistics
-                for( int nid = 0; nid < tree.param.num_nodes; ++ nid ){
-                    tree.stat( nid ).leaf_child_cnt = 0;
-                    tree.stat( nid ).loss_chg = snode[ nid ].best.loss_chg;
-                    tree.stat( nid ).sum_hess = static_cast<float>( snode[ nid ].sum_hess );
-                }
-                for( int nid = 0; nid < tree.param.num_nodes; ++ nid ){
-                    if( tree[ nid ].is_leaf() ) this->TryPruneLeaf( nid, tree.GetDepth(nid) );
-                }
-                return this->stat_num_pruned;
-            }
-        protected:
-            /*! \brief update queue expand add in new leaves */
-            inline void UpdateQueueExpand( std::vector<int> &qexpand ){
-                std::vector<int> newnodes;
-                for( size_t i = 0; i < qexpand.size(); ++ i ){
-                    const int nid = qexpand[i];
-                    if( !tree[ nid ].is_leaf() ){
-                        newnodes.push_back( tree[nid].cleft() );
-                        newnodes.push_back( tree[nid].cright() );
-                    }
-                }
-                // use new nodes for qexpand
-                qexpand = newnodes;
-            }
-        protected:
-            // local helper tmp data structure
-            // statistics
-            int stat_num_pruned;
-            /*! \brief queue of nodes to be expanded */
-            std::vector<int> qexpand;
-            /*! \brief TreeNode Data: statistics for each constructed node, the derived class must maintain this */
-            std::vector<NodeEntry> snode;
-        protected:
-            // original data that supports tree construction
-            RegTree &tree;
-            const TreeParamTrain &param;
-        };
-    }; // namespace booster
-}; // namespace xgboost
-#endif // XGBOOST_BASE_TREEMAKER_HPP

booster/tree/xgboost_col_treemaker.hpp

@@ -1,335 +0,0 @@
-#ifndef XGBOOST_COL_TREEMAKER_HPP
-#define XGBOOST_COL_TREEMAKER_HPP
-/*!
- * \file xgboost_col_treemaker.hpp
- * \brief implementation of regression tree maker,
- *        use a column based approach, with OpenMP 
- * \author Tianqi Chen: tianqi.tchen@gmail.com 
- */
-// use openmp
-#include <vector>
-#include "xgboost_tree_model.h"
-#include "../../utils/xgboost_omp.h"
-#include "../../utils/xgboost_random.h"
-#include "../../utils/xgboost_fmap.h"
-#include "xgboost_base_treemaker.hpp"
-
-namespace xgboost{
-    namespace booster{
-        template<typename FMatrix>
-        class ColTreeMaker : protected BaseTreeMaker{
-        public:
-            ColTreeMaker( RegTree &tree,
-                          const TreeParamTrain &param, 
-                          const std::vector<float> &grad,
-                          const std::vector<float> &hess,
-                          const FMatrix &smat, 
-                          const std::vector<unsigned> &root_index, 
-                          const utils::FeatConstrain  &constrain )
-                : BaseTreeMaker( tree, param ), 
-                  grad(grad), hess(hess), 
-                  smat(smat), root_index(root_index), constrain(constrain) {
-                utils::Assert( grad.size() == hess.size(), "booster:invalid input" );
-                utils::Assert( smat.NumRow() == hess.size(), "booster:invalid input" );
-                utils::Assert( root_index.size() == 0 || root_index.size() == hess.size(), "booster:invalid input" );                
-                utils::Assert( smat.HaveColAccess(), "ColTreeMaker: need column access matrix" );
-            }
-            inline void Make( int& stat_max_depth, int& stat_num_pruned ){
-                this->InitData();
-                this->InitNewNode( this->qexpand );
-                stat_max_depth = 0;
-                
-                for( int depth = 0; depth < param.max_depth; ++ depth ){
-                    this->FindSplit( depth );
-                    this->UpdateQueueExpand( this->qexpand );
-                    this->InitNewNode( this->qexpand );
-                    // if nothing left to be expand, break
-                    if( qexpand.size() == 0 ) break;
-                    stat_max_depth = depth + 1;
-                }
-                // set all the rest expanding nodes to leaf
-                for( size_t i = 0; i < qexpand.size(); ++ i ){
-                    const int nid = qexpand[i];
-                    tree[ nid ].set_leaf( snode[nid].weight * param.learning_rate );                        
-                }
-                // start prunning the tree
-                stat_num_pruned = this->DoPrune();
-            }
-        private:
-            /*! \brief per thread x per node entry to store tmp data */
-            struct ThreadEntry{
-                /*! \brief sum gradient statistics */
-                double sum_grad;
-                /*! \brief sum hessian statistics */
-                double sum_hess;
-                /*! \brief last feature value scanned */
-                float  last_fvalue;
-                /*! \brief current best solution */
-                SplitEntry best;
-                /*! \brief constructor */
-                ThreadEntry( void ){                    
-                    this->ClearStats();
-                }
-                /*! \brief clear statistics */
-                inline void ClearStats( void ){
-                    sum_grad = sum_hess = 0.0;
-                }
-            };
-        private:
-            // make leaf nodes for all qexpand, update node statistics, mark leaf value
-            inline void InitNewNode( const std::vector<int> &qexpand ){
-                {// setup statistics space for each tree node
-                   for( size_t i = 0; i < stemp.size(); ++ i ){
-                        stemp[i].resize( tree.param.num_nodes, ThreadEntry() );
-                   }
-                    snode.resize( tree.param.num_nodes, NodeEntry() );
-                }
-
-                const unsigned ndata = static_cast<unsigned>( position.size() );
-                
-                #pragma omp parallel for schedule( static )
-                for( unsigned i = 0; i < ndata; ++ i ){
-                    const int tid = omp_get_thread_num();
-                    if( position[i] < 0 ) continue; 
-                    stemp[tid][ position[i] ].sum_grad += grad[i];
-                    stemp[tid][ position[i] ].sum_hess += hess[i];
-                }
-
-                for( size_t j = 0; j < qexpand.size(); ++ j ){
-                    const int nid = qexpand[ j ];
-                    double sum_grad = 0.0, sum_hess = 0.0;
-                    for( size_t tid = 0; tid < stemp.size(); tid ++ ){
-                        sum_grad += stemp[tid][nid].sum_grad;
-                        sum_hess += stemp[tid][nid].sum_hess;
-                    }
-                    // update node statistics
-                    snode[nid].sum_grad = sum_grad; 
-                    snode[nid].sum_hess = sum_hess;
-                    snode[nid].root_gain = param.CalcRootGain( sum_grad, sum_hess );
-                    if( !tree[nid].is_root() ){
-                        snode[nid].weight = param.CalcWeight( sum_grad, sum_hess, tree.stat( tree[nid].parent() ).base_weight );
-                        tree.stat(nid).base_weight = snode[nid].weight;
-                    }else{
-                        snode[nid].weight = param.CalcWeight( sum_grad, sum_hess, 0.0f );
-                        tree.stat(nid).base_weight = snode[nid].weight;
-                    }
-                }
-            }
-        private:
-            // enumerate the split values of specific feature
-            template<typename Iter>
-            inline void EnumerateSplit( Iter it, const unsigned fid, std::vector<ThreadEntry> &temp, bool is_forward_search ){
-                // clear all the temp statistics
-                for( size_t j = 0; j < qexpand.size(); ++ j ){
-                    temp[ qexpand[j] ].ClearStats();
-                }
-                
-                while( it.Next() ){
-                    const bst_uint ridx = it.rindex();
-                    const int nid = position[ ridx ];
-                    if( nid < 0 ) continue;
-
-                    const float fvalue = it.fvalue();           
-                    ThreadEntry &e = temp[ nid ];
-
-                    // test if first hit, this is fine, because we set 0 during init
-                    if( e.sum_hess == 0.0 ){
-                        e.sum_grad = grad[ ridx ];
-                        e.sum_hess = hess[ ridx ];
-                        e.last_fvalue = fvalue;
-                    }else{
-                        // try to find a split
-                        if( fabsf(fvalue - e.last_fvalue) > rt_2eps && e.sum_hess >= param.min_child_weight ){
-                            const double csum_hess = snode[ nid ].sum_hess - e.sum_hess;
-                            if( csum_hess >= param.min_child_weight ){
-                                const double csum_grad = snode[nid].sum_grad - e.sum_grad; 
-                                const double loss_chg = 
-                                    + param.CalcGain( e.sum_grad, e.sum_hess, snode[nid].weight ) 
-                                    + param.CalcGain( csum_grad , csum_hess , snode[nid].weight )
-                                    - snode[nid].root_gain;
-                                e.best.Update( loss_chg, fid, (fvalue + e.last_fvalue) * 0.5f, !is_forward_search );
-                            }
-                        }
-                        // update the statistics
-                        e.sum_grad += grad[ ridx ];
-                        e.sum_hess += hess[ ridx ];
-                        e.last_fvalue = fvalue;
-                    }
-                }
-                // finish updating all statistics, check if it is possible to include all sum statistics
-                for( size_t i = 0; i < qexpand.size(); ++ i ){
-                    const int nid = qexpand[ i ];
-                    ThreadEntry &e = temp[ nid ];
-                    const double csum_hess = snode[nid].sum_hess - e.sum_hess;
-
-                    if( e.sum_hess >= param.min_child_weight && csum_hess >= param.min_child_weight ){
-                        const double csum_grad = snode[nid].sum_grad - e.sum_grad; 
-                        const double loss_chg = 
-                            + param.CalcGain( e.sum_grad, e.sum_hess, snode[nid].weight ) 
-                            + param.CalcGain(  csum_grad,  csum_hess, snode[nid].weight )
-                            - snode[nid].root_gain;
-                        const float delta = is_forward_search ? rt_eps:-rt_eps;
-                        e.best.Update( loss_chg, fid, e.last_fvalue + delta, !is_forward_search );
-                    }
-                }
-            }
-
-            // find splits at current level
-            inline void FindSplit( int depth ){
-                const unsigned nsize = static_cast<unsigned>( feat_index.size() );
-                
-                #pragma omp parallel for schedule( dynamic, 1 )
-                for( unsigned i = 0; i < nsize; ++ i ){
-                    const unsigned fid = feat_index[i];
-                    const int tid = omp_get_thread_num();
-                    if( param.need_forward_search() ){
-                        this->EnumerateSplit( smat.GetSortedCol(fid), fid, stemp[tid], true );
-                    }
-                    if( param.need_backward_search() ){
-                        this->EnumerateSplit( smat.GetReverseSortedCol(fid), fid, stemp[tid], false );
-                    }
-                }
-
-                // after this each thread's stemp will get the best candidates, aggregate results
-                for( size_t i = 0; i < qexpand.size(); ++ i ){
-                    const int nid = qexpand[ i ];
-                    NodeEntry &e = snode[ nid ];
-                    for( int tid = 0; tid < this->nthread; ++ tid ){
-                        e.best.Update( stemp[ tid ][ nid ].best );
-                    }
-                    
-                    // now we know the solution in snode[ nid ], set split
-                    if( e.best.loss_chg > rt_eps ){
-                        tree.AddChilds( nid );
-                        tree[ nid ].set_split( e.best.split_index(), e.best.split_value, e.best.default_left() );
-                    } else{
-                        tree[ nid ].set_leaf( e.weight * param.learning_rate );
-                    }  
-                }
-
-                {// reset position 
-                    // step 1, set default direct nodes to default, and leaf nodes to -1, 
-                    const unsigned ndata = static_cast<unsigned>( position.size() );
-                    #pragma omp parallel for schedule( static )
-                    for( unsigned i = 0; i < ndata; ++ i ){
-                        const int nid = position[i];
-                        if( nid >= 0 ){
-                            if( tree[ nid ].is_leaf() ){
-                                position[i] = -1;
-                            }else{
-                                // push to default branch, correct latter
-                                position[i] = tree[nid].default_left() ? tree[nid].cleft(): tree[nid].cright();
-                            }
-                        }
-                    }
-
-                    // step 2, classify the non-default data into right places
-                    std::vector<unsigned> fsplits;
-
-                    for( size_t i = 0; i < qexpand.size(); ++ i ){
-                        const int nid = qexpand[i];
-                        if( !tree[nid].is_leaf() ) fsplits.push_back( tree[nid].split_index() );
-                    }
-                    std::sort( fsplits.begin(), fsplits.end() );
-                    fsplits.resize( std::unique( fsplits.begin(), fsplits.end() ) - fsplits.begin() );
-
-                    const unsigned nfeats = static_cast<unsigned>( fsplits.size() );
-                    #pragma omp parallel for schedule( dynamic, 1 )
-                    for( unsigned i = 0; i < nfeats; ++ i ){
-                        const unsigned fid = fsplits[i];
-                        for( typename FMatrix::ColIter it = smat.GetSortedCol( fid ); it.Next(); ){
-                            const bst_uint ridx = it.rindex();
-                            int nid = position[ ridx ];
-                            if( nid == -1 ) continue;
-                            // go back to parent, correct those who are not default
-                            nid = tree[ nid ].parent();
-                            if( tree[ nid ].split_index() == fid ){
-                                if( it.fvalue() < tree[nid].split_cond() ){
-                                    position[ ridx ] = tree[ nid ].cleft();
-                                }else{
-                                    position[ ridx ] = tree[ nid ].cright();
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        private:
-            // initialize temp data structure
-            inline void InitData( void ){
-                {
-                    position.resize( grad.size() );
-                    if( root_index.size() == 0 ){
-                        std::fill( position.begin(), position.end(), 0 );
-                    }else{
-                        for( size_t i = 0; i < root_index.size(); ++ i ){
-                            position[i] = root_index[i];
-                            utils::Assert( root_index[i] < (unsigned)tree.param.num_roots, "root index exceed setting" );
-                        }
-                    }
-                    // mark delete for the deleted datas
-                    for( size_t i = 0; i < grad.size(); ++ i ){
-                        if( hess[i] < 0.0f ) position[i] = -1;
-                    }
-                    if( param.subsample < 1.0f - 1e-6f ){
-                        for( size_t i = 0; i < grad.size(); ++ i ){
-                            if( hess[i] < 0.0f ) continue;
-                            if( random::SampleBinary( param.subsample) == 0 ){
-                                position[ i ] = -1;
-                            }
-                        }
-                    }
-                }
-                
-                {// initialize feature index
-                    int ncol = static_cast<int>( smat.NumCol() );
-                    for( int i = 0; i < ncol; i ++ ){
-                        if( smat.GetSortedCol(i).Next() && constrain.NotBanned(i) ){
-                            feat_index.push_back( i );
-                        }
-                    }
-                    random::Shuffle( feat_index );
-                }
-                {// setup temp space for each thread
-                    if( param.nthread != 0 ){
-                        omp_set_num_threads( param.nthread );
-                    }
-                    #pragma omp parallel
-                    {
-                        this->nthread = omp_get_num_threads();
-                    }
-
-                    // reserve a small space
-                    stemp.resize( this->nthread, std::vector<ThreadEntry>() );
-                    for( size_t i = 0; i < stemp.size(); ++ i ){
-                        stemp[i].reserve( 256 );
-                    }
-                    snode.reserve( 256 );
-                }
-                
-                {// expand query
-                    qexpand.reserve( 256 ); qexpand.clear();
-                    for( int i = 0; i < tree.param.num_roots; ++ i ){
-                        qexpand.push_back( i );
-                    }
-                }
-            }
-        private:
-            // number of omp thread used during training
-            int nthread;
-            // Per feature: shuffle index of each feature index
-            std::vector<int> feat_index;
-            // Instance Data: current node position in the tree of each instance
-            std::vector<int> position;
-            // PerThread x PerTreeNode: statistics for per thread construction
-            std::vector< std::vector<ThreadEntry> > stemp;
-        private:
-            const std::vector<float> &grad;
-            const std::vector<float> &hess;
-            const FMatrix            &smat;
-            const std::vector<unsigned> &root_index;
-            const utils::FeatConstrain  &constrain;
-        };
-    };
-};
-#endif

booster/tree/xgboost_row_treemaker.hpp

@@ -1,386 +0,0 @@
-#ifndef XGBOOST_ROW_TREEMAKER_HPP
-#define XGBOOST_ROW_TREEMAKER_HPP
-/*!
- * \file xgboost_row_treemaker.hpp
- * \brief implementation of regression tree maker,
- *        use a row based approach
- * \author Tianqi Chen: tianqi.tchen@gmail.com 
- */
-// use openmp
-#include <vector>
-#include "xgboost_tree_model.h"
-#include "../../utils/xgboost_omp.h"
-#include "../../utils/xgboost_random.h"
-#include "../../utils/xgboost_fmap.h"
-#include "xgboost_base_treemaker.hpp"
-
-namespace xgboost{
-    namespace booster{
-        template<typename FMatrix>
-        class RowTreeMaker : protected BaseTreeMaker{
-        public:
-            RowTreeMaker( RegTree &tree,
-                          const TreeParamTrain &param, 
-                          const std::vector<float> &grad,
-                          const std::vector<float> &hess,
-                          const FMatrix &smat, 
-                          const std::vector<unsigned> &root_index, 
-                          const utils::FeatConstrain &constrain )
-                : BaseTreeMaker( tree, param ), 
-                  grad(grad), hess(hess), 
-                  smat(smat), root_index(root_index), constrain(constrain) {
-                utils::Assert( grad.size() == hess.size(), "booster:invalid input" );
-                utils::Assert( smat.NumRow() == hess.size(), "booster:invalid input" );
-                utils::Assert( root_index.size() == 0 || root_index.size() == hess.size(), "booster:invalid input" ); 
-                {// setup temp space for each thread
-                    if( param.nthread != 0 ){
-                        omp_set_num_threads( param.nthread );
-                    }
-                    #pragma omp parallel
-                    {
-                        this->nthread = omp_get_num_threads();
-                    }
-                    tmp_rptr.resize( this->nthread, std::vector<size_t>() );
-                    snode.reserve( 256 );
-                }
-            }
-            inline void Make( int& stat_max_depth, int& stat_num_pruned ){
-                this->InitData();
-                this->InitNewNode( this->qexpand );
-                stat_max_depth = 0;
-                
-                for( int depth = 0; depth < param.max_depth; ++ depth ){                                        
-                    this->FindSplit( this->qexpand, depth );
-                    this->UpdateQueueExpand( this->qexpand );
-                    this->InitNewNode( this->qexpand );
-                    // if nothing left to be expand, break
-                    if( qexpand.size() == 0 ) break;
-                    stat_max_depth = depth + 1;
-                }
-                // set all the rest expanding nodes to leaf
-                for( size_t i = 0; i < qexpand.size(); ++ i ){
-                    const int nid = qexpand[i];
-                    tree[ nid ].set_leaf( snode[nid].weight * param.learning_rate );
-                }
-                // start prunning the tree
-                stat_num_pruned = this->DoPrune();
-            }
-            // expand a specific node
-            inline bool Expand( const std::vector<bst_uint> &valid_index, int nid ){
-                if( valid_index.size() == 0 ) return false;
-                this->InitDataExpand( valid_index, nid );
-                this->InitNewNode( this->qexpand );
-                this->FindSplit( nid, tmp_rptr[0] );
-
-                // update node statistics
-                for( size_t i = 0; i < qexpand.size(); ++ i ){
-                    const int nid = qexpand[i];
-                    tree.stat( nid ).loss_chg = snode[ nid ].best.loss_chg;
-                    tree.stat( nid ).sum_hess = static_cast<float>( snode[ nid ].sum_hess );
-                }
-                // change the leaf
-                this->UpdateQueueExpand( this->qexpand );
-                this->InitNewNode( this->qexpand );
-                
-                // set all the rest expanding nodes to leaf
-                for( size_t i = 0; i < qexpand.size(); ++ i ){
-                    const int nid = qexpand[i];
-                    
-                    tree[ nid ].set_leaf( snode[nid].weight * param.learning_rate );
-                    tree.stat( nid ).loss_chg = 0.0f;
-                    tree.stat( nid ).sum_hess = static_cast<float>( snode[ nid ].sum_hess );
-                    tree.param.max_depth = std::max( tree.param.max_depth, tree.GetDepth( nid ) );
-                }
-                if( qexpand.size() != 0 ) {
-                    return true;
-                }else{
-                    return false;
-                }
-            }
-            // collapse specific node
-            inline void Collapse( const std::vector<bst_uint> &valid_index, int nid ){
-                if( valid_index.size() == 0 ) return;
-                this->InitDataExpand( valid_index, nid );
-                this->InitNewNode( this->qexpand );
-                tree.stat( nid ).loss_chg = 0.0f;
-                tree.stat( nid ).sum_hess = static_cast<float>( snode[ nid ].sum_hess );
-                tree.CollapseToLeaf( nid, snode[nid].weight * param.learning_rate );
-            }
-        private:
-            // make leaf nodes for all qexpand, update node statistics, mark leaf value
-            inline void InitNewNode( const std::vector<int> &qexpand ){
-                snode.resize( tree.param.num_nodes, NodeEntry() );
-
-                for( size_t j = 0; j < qexpand.size(); ++j ){
-                    const int nid = qexpand[ j ];
-                    double sum_grad = 0.0, sum_hess = 0.0;
-
-                    for( bst_uint i = node_bound[nid].first; i < node_bound[nid].second; ++i ){
-                        const bst_uint ridx = row_index_set[i];
-                        sum_grad += grad[ridx]; sum_hess += hess[ridx];
-                    }
-                    // update node statistics
-                    snode[nid].sum_grad = sum_grad; 
-                    snode[nid].sum_hess = sum_hess;
-
-                    snode[nid].root_gain = param.CalcRootGain( sum_grad, sum_hess );
-                    if( !tree[nid].is_root() ){
-                        snode[nid].weight = param.CalcWeight( sum_grad, sum_hess, tree.stat( tree[nid].parent() ).base_weight );
-                        tree.stat(nid).base_weight = snode[nid].weight;
-                    }else{
-                        snode[nid].weight = param.CalcWeight( sum_grad, sum_hess, 0.0f );
-                        tree.stat(nid).base_weight = snode[nid].weight;
-                    }
-                }
-            }
-        private:
-            // enumerate the split values of specific feature
-            template<typename Iter>
-            inline void EnumerateSplit( Iter it, SplitEntry &best, const int nid, const unsigned fid, bool is_forward_search ){
-                float last_fvalue = 0.0f;
-                double sum_hess = 0.0, sum_grad = 0.0;
-                const NodeEntry enode = snode[ nid ];
-
-                while( it.Next() ){
-                    const bst_uint ridx = it.rindex();
-                    const float fvalue = it.fvalue();           
-                    
-                    if( sum_hess == 0.0 ){
-                        sum_grad = grad[ ridx ];
-                        sum_hess = hess[ ridx ];
-                        last_fvalue = fvalue;
-                    }else{
-                        // try to find a split
-                        if( fabsf(fvalue - last_fvalue) > rt_2eps && sum_hess >= param.min_child_weight ){
-                            const double csum_hess = enode.sum_hess - sum_hess;
-                            if( csum_hess >= param.min_child_weight ){
-                                const double csum_grad = enode.sum_grad - sum_grad; 
-                                const double loss_chg = 
-                                    + param.CalcGain(  sum_grad,  sum_hess, enode.weight ) 
-                                    + param.CalcGain( csum_grad, csum_hess, enode.weight )
-                                    - enode.root_gain;
-                                best.Update( loss_chg, fid, (fvalue + last_fvalue) * 0.5f, !is_forward_search );
-                            }else{
-                                // the rest part doesn't meet split condition anyway, return 
-                                return;
-                            }
-                        }
-                        // update the statistics
-                        sum_grad += grad[ ridx ];
-                        sum_hess += hess[ ridx ];
-                        last_fvalue = fvalue;
-                    }                    
-                }
-
-                const double csum_hess = enode.sum_hess - sum_hess;
-                if( sum_hess >= param.min_child_weight && csum_hess >= param.min_child_weight ){
-                    const double csum_grad = enode.sum_grad - sum_grad; 
-                    const double loss_chg = 
-                        + param.CalcGain(   sum_grad,   sum_hess, enode.weight ) 
-                        + param.CalcGain(  csum_grad,  csum_hess, enode.weight )
-                        - snode[nid].root_gain;
-                    const float delta = is_forward_search ? rt_eps:-rt_eps;
-                    best.Update( loss_chg, fid, last_fvalue + delta, !is_forward_search );
-                }
-            }
-        private:
-            inline void FindSplit( const std::vector<int> &qexpand, int depth ){
-                int nexpand = (int)qexpand.size();
-                if( depth < 3 ){ 
-                    for( int i = 0; i < nexpand; ++ i ){
-                        this->FindSplit( qexpand[i], tmp_rptr[0] );
-                    }
-                }else{
-                    // if get to enough depth, parallelize over node
-                    #pragma omp parallel for schedule(dynamic,1)
-                    for( int i = 0; i < nexpand; ++ i ){
-                        const int tid = omp_get_thread_num();
-                        utils::Assert( tid < (int)tmp_rptr.size(), "BUG: FindSplit, tid exceed tmp_rptr size" );
-                        this->FindSplit( qexpand[i], tmp_rptr[tid] );
-                    }
-                }
-            }
-        private:
-            inline void MakeSplit( int nid, unsigned gid ){
-                node_bound.resize( tree.param.num_nodes );
-                // re-organize the row_index_set after split on nid
-                const unsigned split_index = tree[nid].split_index();
-                const float    split_value = tree[nid].split_cond();
-
-                std::vector<bst_uint> right;
-                bst_uint top = node_bound[nid].first;
-                for( bst_uint i = node_bound[ nid ].first; i < node_bound[ nid ].second; ++i ){
-                    const bst_uint ridx = row_index_set[i];                    
-                    bool goleft = tree[ nid ].default_left();                    
-                    for( typename FMatrix::RowIter it = smat.GetRow(ridx,gid); it.Next(); ){
-                        if( it.findex() == split_index ){
-                            if( it.fvalue() < split_value ){
-                                goleft = true;  break;
-                            }else{
-                                goleft = false; break;
-                            }
-                        }
-                    }
-                    if( goleft ) {
-                        row_index_set[ top ++ ] = ridx;
-                    }else{
-                        right.push_back( ridx );
-                    }
-                }
-                node_bound[ tree[nid].cleft() ]  = std::make_pair( node_bound[nid].first, top );
-                node_bound[ tree[nid].cright() ] = std::make_pair( top, node_bound[nid].second );
-
-                utils::Assert( node_bound[nid].second - top == (bst_uint)right.size(), "BUG:MakeSplit" );
-                for( size_t i = 0; i < right.size(); ++ i ){
-                    row_index_set[ top ++ ] = right[ i ];
-                }
-            }
-                        
-            // find splits at current level
-            inline void FindSplit( int nid, std::vector<size_t> &tmp_rptr ){
-                if( tmp_rptr.size() == 0 ){
-                    tmp_rptr.resize( tree.param.num_feature + 1, 0 );
-                }
-                const bst_uint begin = node_bound[ nid ].first;
-                const bst_uint end   = node_bound[ nid ].second;
-                const unsigned ncgroup = smat.NumColGroup();
-                unsigned best_group = 0;
-
-                for( unsigned gid = 0; gid < ncgroup; ++gid ){
-                    // records the columns
-                    std::vector<FMatrixS::REntry> centry;
-                    // records the active features
-                    std::vector<size_t>  aclist;
-                    utils::SparseCSRMBuilder<FMatrixS::REntry,true> builder( tmp_rptr, centry, aclist );
-                    builder.InitBudget( tree.param.num_feature );
-                    for( bst_uint i = begin; i < end; ++i ){
-                        const bst_uint ridx = row_index_set[i];
-                        for( typename FMatrix::RowIter it = smat.GetRow(ridx,gid); it.Next(); ){
-                            const bst_uint findex = it.findex();
-                            if( constrain.NotBanned( findex ) ) builder.AddBudget( findex );
-                        }
-                    }
-                    builder.InitStorage();
-                    for( bst_uint i = begin; i < end; ++i ){
-                        const bst_uint ridx = row_index_set[i];
-                        for( typename FMatrix::RowIter it = smat.GetRow(ridx,gid); it.Next(); ){
-                            const bst_uint findex = it.findex();
-                            if( constrain.NotBanned( findex ) ) {
-                                builder.PushElem( findex, FMatrixS::REntry( ridx, it.fvalue() ) );
-                            }
-                        }
-                    }
-                    // --- end of building column major matrix ---                    
-                    // after this point, tmp_rptr and entry is ready to use                    
-                    int naclist = (int)aclist.size();
-                    // best entry for each thread
-                    SplitEntry nbest, tbest;
-                    #pragma omp parallel private(tbest)
-                    { 
-                        #pragma omp for schedule(dynamic,1)
-                        for( int j = 0; j < naclist; ++j ){
-                            bst_uint findex = static_cast<bst_uint>( aclist[j] );
-                            // local sort can be faster when the features are sparse
-                            std::sort( centry.begin() + tmp_rptr[findex], centry.begin() + tmp_rptr[findex+1], FMatrixS::REntry::cmp_fvalue );
-                            if( param.need_forward_search() ){
-                                this->EnumerateSplit( FMatrixS::ColIter( &centry[tmp_rptr[findex]]-1, &centry[tmp_rptr[findex+1]] - 1 ),
-                                                      tbest, nid, findex, true );
-                            }
-                            if( param.need_backward_search() ){
-                                this->EnumerateSplit( FMatrixS::ColBackIter( &centry[tmp_rptr[findex+1]], &centry[tmp_rptr[findex]] ),
-                                                      tbest, nid, findex, false );
-                            }
-                        }
-                        #pragma omp critical 
-                        {
-                            nbest.Update( tbest );
-                        }
-                    }
-                    // if current solution gives the best 
-                    if( snode[nid].best.Update( nbest ) ){
-                        best_group = gid;
-                    }
-                    // cleanup tmp_rptr for next usage
-                    builder.Cleanup();    
-                }
-                
-                // at this point, we already know the best split
-                if( snode[nid].best.loss_chg > rt_eps ){
-                    const SplitEntry &e = snode[nid].best;
-                    tree.AddChilds( nid );
-                    tree[ nid ].set_split( e.split_index(), e.split_value, e.default_left() );
-                    this->MakeSplit( nid, best_group );
-                }else{
-                    tree[ nid ].set_leaf( snode[nid].weight * param.learning_rate );                    
-                }
-            }
-        private:
-            // initialize temp data structure
-            inline void InitData( void ){
-                std::vector<bst_uint> valid_index;
-                for( size_t i = 0; i < grad.size(); ++i ){
-                    if( hess[ i ] < 0.0f ) continue;
-                    if( param.subsample > 1.0f-1e-6f || random::SampleBinary( param.subsample ) != 0 ){
-                        valid_index.push_back( static_cast<bst_uint>(i) );
-                    }
-                }
-                node_bound.resize( tree.param.num_roots );
-
-                if( root_index.size() == 0 ){
-                    row_index_set = valid_index;
-                    // set bound of root node
-                    node_bound[0] = std::make_pair( 0, (bst_uint)row_index_set.size() );
-                }else{                    
-                    std::vector<size_t>   rptr;
-                    utils::SparseCSRMBuilder<bst_uint> builder( rptr, row_index_set );
-                    builder.InitBudget( tree.param.num_roots );
-                    for( size_t i = 0; i < valid_index.size(); ++i ){
-                        const bst_uint rid = valid_index[ i ];
-                        utils::Assert( root_index[ rid ] < (unsigned)tree.param.num_roots, "root id exceed number of roots" );
-                        builder.AddBudget( root_index[ rid ] );
-                    }
-                    builder.InitStorage();
-                    for( size_t i = 0; i < valid_index.size(); ++i ){
-                        const bst_uint rid = valid_index[ i ];
-                        builder.PushElem( root_index[ rid ], rid );
-                    }
-                    for( size_t i = 1; i < rptr.size(); ++ i ){
-                        node_bound[i-1] = std::make_pair( rptr[ i - 1 ], rptr[ i ] );
-                    }
-                }
-
-                {// expand query
-                    qexpand.reserve( 256 ); qexpand.clear();
-                    for( int i = 0; i < tree.param.num_roots; ++ i ){
-                        qexpand.push_back( i );
-                    }
-                }
-            }
-
-            // initialize temp data structure
-            inline void InitDataExpand( const std::vector<bst_uint> &valid_index, int nid ){
-                row_index_set = valid_index;                
-                node_bound.resize( tree.param.num_nodes );
-                node_bound[ nid ] = std::make_pair( 0, (bst_uint)row_index_set.size() );
-             
-                qexpand.clear(); qexpand.push_back( nid );
-            }
-        private:
-            // number of omp thread used during training
-            int nthread;
-            // tmp row pointer, per thread, used for tmp data construction
-            std::vector< std::vector<size_t> > tmp_rptr;
-            // Instance row indexes corresponding to each node
-            std::vector<bst_uint> row_index_set;
-            // lower and upper bound of each nodes' row_index
-            std::vector< std::pair<bst_uint, bst_uint> > node_bound;
-        private:
-            const std::vector<float> &grad;
-            const std::vector<float> &hess;
-            const FMatrix            &smat;
-            const std::vector<unsigned> &root_index;
-            const utils::FeatConstrain  &constrain;
-        };
-    };
-};
-#endif

booster/tree/xgboost_svdf_tree.hpp

@@ -1,430 +0,0 @@
-#ifndef XGBOOST_APEX_TREE_HPP
-#define XGBOOST_APEX_TREE_HPP
-/*!
- * \file xgboost_svdf_tree.hpp
- * \brief implementation of regression tree constructor, with layerwise support
- *        this file is adapted from GBRT implementation in SVDFeature project
- * \author Tianqi Chen: tqchen@apex.sjtu.edu.cn, tianqi.tchen@gmail.com 
- */
-#include <algorithm>
-#include "xgboost_tree_model.h"
-#include "../../utils/xgboost_random.h"
-#include "../../utils/xgboost_matrix_csr.h"
-
-namespace xgboost{
-    namespace booster{        
-        inline void assert_sorted( unsigned *idset, int len ){
-            if( !rt_debug || !check_bug ) return;
-            for( int i = 1; i < len; i ++ ){
-                utils::Assert( idset[i-1] < idset[i], "idset not sorted" );
-            }
-        }
-    };
-    
-    namespace booster{                
-        // selecter of rtree to find the suitable candidate
-        class RTSelecter{
-        public:
-            struct Entry{
-                float  loss_chg;
-                size_t start;
-                int    len;
-                unsigned sindex;
-                float    split_value;
-                Entry(){}
-                Entry( float loss_chg, size_t start, int len, unsigned split_index, float split_value, bool default_left ){
-                    this->loss_chg = loss_chg;
-                    this->start    = start;
-                    this->len      = len;
-                    if( default_left ) split_index |= (1U << 31);
-                    this->sindex = split_index;
-                    this->split_value = split_value;
-                }
-                inline unsigned split_index( void ) const{
-                    return sindex & ( (1U<<31) - 1U );
-                }
-                inline bool default_left( void ) const{
-                    return (sindex >> 31) != 0;
-                }
-            };
-        private:
-            Entry best_entry;
-            const TreeParamTrain &param;
-        public:
-            RTSelecter( const TreeParamTrain &p ):param( p ){
-                memset( &best_entry, 0, sizeof(best_entry) );
-                best_entry.loss_chg = 0.0f;
-            }
-            inline void push_back( const Entry &e ){
-                if( e.loss_chg > best_entry.loss_chg ) best_entry = e;
-            }
-            inline const Entry & select( void ){            
-                return best_entry;                
-            }
-        };
-        
-        
-        // updater of rtree, allows the parameters to be stored inside, key solver
-        template<typename FMatrix>
-        class RTreeUpdater{
-        protected:
-            // training task, element of single task
-            struct Task{
-                // node id in tree
-                int nid;
-                // idset pointer, instance id in [idset,idset+len)
-                unsigned *idset;
-                // length of idset
-                unsigned len;            
-                // base_weight of parent
-                float parent_base_weight;
-                Task(){}
-                Task( int nid, unsigned *idset, unsigned len, float pweight = 0.0f ){
-                    this->nid = nid;
-                    this->idset = idset;
-                    this->len = len;
-                    this->parent_base_weight = pweight;
-                }
-            };
-            
-            // sparse column entry
-            struct SCEntry{
-                // feature value 
-                float    fvalue;
-                // row index in grad
-                unsigned rindex;
-                SCEntry(){}
-                SCEntry( float fvalue, unsigned rindex ){
-                    this->fvalue = fvalue; this->rindex = rindex;
-                }
-                inline bool operator<( const SCEntry &p ) const{
-                    return fvalue < p.fvalue;
-                }
-            };
-        private:
-            // training parameter
-            const TreeParamTrain &param;
-            // parameters, reference
-            RegTree &tree;
-            std::vector<float> &grad;
-            std::vector<float> &hess;
-            const FMatrix &smat;
-            const std::vector<unsigned> &group_id;
-        private:
-            // maximum depth up to now
-            int max_depth;
-            // number of nodes being pruned
-            int num_pruned;
-            // stack to store current task
-            std::vector<Task> task_stack;
-            // temporal space for index set
-            std::vector<unsigned> idset;
-        private:
-            // task management: NOTE DFS here
-            inline void add_task( Task tsk ){
-                task_stack.push_back( tsk );
-            }
-            inline bool next_task( Task &tsk ){
-                if( task_stack.size() == 0 ) return false;
-                tsk = task_stack.back(); 
-                task_stack.pop_back();
-                return true;
-            } 
-        private:
-            // try to prune off current leaf, return true if successful
-            inline void try_prune_leaf( int nid, int depth ){
-                if( tree[ nid ].is_root() ) return;
-                int pid = tree[ nid ].parent();
-                RegTree::NodeStat &s = tree.stat( pid );
-                s.leaf_child_cnt ++;
-                
-                if( s.leaf_child_cnt >= 2 && param.need_prune( s.loss_chg, depth - 1 ) ){
-                    // need to be pruned
-                    tree.ChangeToLeaf( pid, param.learning_rate * s.base_weight );
-                    // add statistics to number of nodes pruned
-                    num_pruned += 2;
-                    // tail recursion
-                    this->try_prune_leaf( pid, depth - 1 );
-                }
-            }
-            // make leaf for current node :)
-            inline void make_leaf( Task tsk, double sum_grad, double sum_hess, bool compute ){
-                for( unsigned i = 0; i < tsk.len; i ++ ){
-                    const unsigned ridx = tsk.idset[i];
-                    if( compute ){
-                        sum_grad += grad[ ridx ];
-                        sum_hess += hess[ ridx ];
-                    }
-                }
-                tree.stat( tsk.nid ).sum_hess = static_cast<float>( sum_hess );
-                tree[ tsk.nid ].set_leaf( param.learning_rate * param.CalcWeight( sum_grad, sum_hess, tsk.parent_base_weight ) );
-                this->try_prune_leaf( tsk.nid, tree.GetDepth( tsk.nid ) );
-            }
-        private:
-            // make split for current task, re-arrange positions in idset
-            inline void make_split( Task tsk, const SCEntry *entry, int num, float loss_chg, double sum_hess, double base_weight ){
-                // before split, first prepare statistics
-                RegTree::NodeStat &s = tree.stat( tsk.nid );
-                s.loss_chg = loss_chg; 
-                s.leaf_child_cnt = 0;
-                s.sum_hess = static_cast<float>( sum_hess );
-                s.base_weight = static_cast<float>( base_weight );
-                
-                // add childs to current node
-                tree.AddChilds( tsk.nid );
-                // assert that idset is sorted
-                assert_sorted( tsk.idset, tsk.len );
-                // use merge sort style to get the solution
-                std::vector<unsigned> qset;
-                for( int i = 0; i < num; i ++ ){
-                    qset.push_back( entry[i].rindex );
-                }
-                std::sort( qset.begin(), qset.end() );
-                // do merge sort style, make the other set, remove elements in qset
-                for( unsigned i = 0, top = 0; i < tsk.len; i ++ ){
-                    if( top < qset.size() ){
-                        if( tsk.idset[ i ] != qset[ top ] ){
-                            tsk.idset[ i - top ] = tsk.idset[ i ];
-                        }else{
-                            top ++;
-                        }
-                    }else{
-                        tsk.idset[ i - qset.size() ] = tsk.idset[ i ];
-                    }
-                }
-                // get two parts 
-                RegTree::Node &n = tree[ tsk.nid ];
-                Task def_part( n.default_left() ? n.cleft() : n.cright(), tsk.idset, tsk.len - qset.size(), s.base_weight );
-                Task spl_part( n.default_left() ? n.cright(): n.cleft() , tsk.idset + def_part.len, qset.size(), s.base_weight );  
-                // fill back split part
-                for( unsigned i = 0; i < spl_part.len; i ++ ){
-                    spl_part.idset[ i ] = qset[ i ];
-                }
-                // add tasks to the queue
-                this->add_task( def_part ); 
-                this->add_task( spl_part );
-            }
-            
-            // enumerate split point of the tree
-            inline void enumerate_split( RTSelecter &sglobal, int tlen,
-                                         double rsum_grad, double rsum_hess, double root_gain,
-                                         const SCEntry *entry, size_t start, size_t end, 
-                                         int findex, float parent_base_weight ){
-                // local selecter
-                RTSelecter slocal( param );
-                
-                if( param.need_forward_search() ){
-                    // forward process, default right
-                    double csum_grad = 0.0, csum_hess = 0.0;
-                    for( size_t j = start; j < end; j ++ ){
-                        const unsigned ridx = entry[ j ].rindex;
-                        csum_grad += grad[ ridx ];
-                        csum_hess += hess[ ridx ];
-                        // check for split
-                        if( j == end - 1 || entry[j].fvalue + rt_2eps < entry[ j + 1 ].fvalue ){
-                            if( csum_hess < param.min_child_weight ) continue;
-                            const double dsum_hess = rsum_hess - csum_hess;
-                            if( dsum_hess < param.min_child_weight ) break;                        
-                            // change of loss 
-                            double loss_chg = 
-                            param.CalcGain( csum_grad, csum_hess, parent_base_weight ) + 
-                                param.CalcGain( rsum_grad - csum_grad, dsum_hess, parent_base_weight ) - root_gain;
-                            
-                            const int clen = static_cast<int>( j + 1 - start );
-                            // add candidate to selecter
-                            slocal.push_back( RTSelecter::Entry( loss_chg, start, clen, findex, 
-                                                                 j == end - 1 ? entry[j].fvalue + rt_eps : 0.5 * (entry[j].fvalue+entry[j+1].fvalue),
-                                                                 false ) );
-                        }
-                    }
-                }
-                
-                if( param.need_backward_search() ){
-                    // backward process, default left
-                    double csum_grad = 0.0, csum_hess = 0.0;
-                    for( size_t j = end; j > start; j -- ){
-                        const unsigned ridx = entry[ j - 1 ].rindex;
-                        csum_grad += grad[ ridx ];
-                        csum_hess += hess[ ridx ];
-                        // check for split
-                        if( j == start + 1 || entry[ j - 2 ].fvalue + rt_2eps < entry[ j - 1 ].fvalue ){
-                            if( csum_hess < param.min_child_weight ) continue;
-                            const double dsum_hess = rsum_hess - csum_hess;
-                            if( dsum_hess < param.min_child_weight ) break;
-                            double loss_chg = param.CalcGain( csum_grad, csum_hess, parent_base_weight ) + 
-                                param.CalcGain( rsum_grad - csum_grad, dsum_hess, parent_base_weight ) - root_gain;
-                            const int clen = static_cast<int>( end - j + 1 );
-                            // add candidate to selecter
-                            slocal.push_back( RTSelecter::Entry( loss_chg, j - 1, clen, findex,
-                                                                 j == start + 1 ? entry[j-1].fvalue - rt_eps : 0.5 * (entry[j-2].fvalue + entry[j-1].fvalue), 
-                                                                 true ) );
-                        }
-                    }
-                }
-                sglobal.push_back( slocal.select() );
-            }
-            
-        private:
-            // temporal storage for expand column major
-            std::vector<size_t>  tmp_rptr;        
-            // find split for current task, another implementation of expand in column major manner
-            // should be more memory frugal, avoid global sorting across feature       
-            inline void expand( Task tsk ){
-                // assert that idset is sorted
-                // if reach maximum depth, make leaf from current node
-                int depth = tree.GetDepth( tsk.nid );
-                // update statistiss
-                if( depth > max_depth ) max_depth = depth; 
-                // if bigger than max depth
-                if( depth >= param.max_depth ){
-                    this->make_leaf( tsk, 0.0, 0.0, true ); return;
-                }
-                // convert to column major CSR format
-                const int nrows = tree.param.num_feature;
-                if( tmp_rptr.size() == 0 ){
-                    // initialize tmp storage in first usage
-                    tmp_rptr.resize( nrows + 1 ); 
-                    std::fill( tmp_rptr.begin(), tmp_rptr.end(), 0 );
-                }
-                // records the columns
-                std::vector<SCEntry> entry;
-                // records the active features
-                std::vector<size_t>  aclist;
-                utils::SparseCSRMBuilder<SCEntry,true> builder( tmp_rptr, entry, aclist );
-                builder.InitBudget( nrows );
-                // statistics of root
-                double rsum_grad = 0.0, rsum_hess = 0.0;            
-                for( unsigned i = 0; i < tsk.len; i ++ ){
-                    const unsigned ridx = tsk.idset[i];
-                    rsum_grad  += grad[ ridx ];
-                    rsum_hess  += hess[ ridx ];
-                    
-                    for( typename FMatrix::RowIter it = smat.GetRow(ridx); it.Next(); ){
-                        builder.AddBudget( it.findex() );
-                    }
-                }
-                
-                // if minimum split weight is not meet
-                if( param.cannot_split( rsum_hess, depth )  ){
-                    this->make_leaf( tsk, rsum_grad, rsum_hess, false ); builder.Cleanup(); return; 
-                }
-                
-                builder.InitStorage();
-                for( unsigned i = 0; i < tsk.len; i ++ ){
-                    const unsigned ridx = tsk.idset[i];
-                    for( typename FMatrix::RowIter it = smat.GetRow(ridx); it.Next(); ){
-                        builder.PushElem( it.findex(), SCEntry( it.fvalue(), ridx ) );
-                    }
-                }
-                // --- end of building column major matrix ---
-                // after this point, tmp_rptr and entry is ready to use
-                
-                // global selecter
-                RTSelecter sglobal( param );
-                // gain root 
-                const double root_gain = param.CalcRootGain( rsum_grad, rsum_hess );
-                // KEY: layerwise, weight of current node if it is leaf
-                const double base_weight = param.CalcWeight( rsum_grad, rsum_hess, tsk.parent_base_weight );
-                // enumerate feature index
-                for( size_t i = 0; i < aclist.size(); i ++ ){
-                    int findex = static_cast<int>( aclist[i] );                
-                    size_t start = tmp_rptr[ findex ];
-                    size_t end   = tmp_rptr[ findex + 1 ];
-                    utils::Assert( start < end, "bug" );
-                    // local sort can be faster when the features are sparse
-                    std::sort( entry.begin() + start, entry.begin() + end );
-                    // local selecter
-                    this->enumerate_split( sglobal, tsk.len,
-                                           rsum_grad, rsum_hess, root_gain,
-                                           &entry[0], start, end, findex, base_weight );
-                }
-                // Cleanup tmp_rptr for next use
-                builder.Cleanup();
-                // get the best solution
-                const RTSelecter::Entry &e = sglobal.select();
-                // allowed to split
-                if( e.loss_chg > rt_eps ){
-                    // add splits
-                    tree[ tsk.nid ].set_split( e.split_index(), e.split_value, e.default_left() );
-                    // re-arrange idset, push tasks
-                    this->make_split( tsk, &entry[ e.start ], e.len, e.loss_chg, rsum_hess, base_weight ); 
-                }else{
-                    // make leaf if we didn't meet requirement
-                    this->make_leaf( tsk, rsum_grad, rsum_hess, false );
-                }
-            }
-        private:
-            // initialize the tasks
-            inline void init_tasks( size_t ngrads ){
-                // add group partition if necessary
-                if( group_id.size() == 0 ){       
-                    if( param.subsample > 1.0f - 1e-6f ){ 
-                        idset.resize( 0 );
-                        for( size_t i = 0; i < ngrads; i ++ ){
-                            if( hess[i] < 0.0f ) continue;
-                            idset.push_back( (unsigned)i );
-                        } 
-                    }else{
-                        idset.resize( 0 );
-                        for( size_t i = 0; i < ngrads; i ++ ){
-                            if( random::SampleBinary( param.subsample ) != 0 ){
-                                idset.push_back( (unsigned)i );
-                            }
-                        } 
-                    }
-                    this->add_task( Task( 0, &idset[0], idset.size() ) ); return;
-                }
-                
-                utils::Assert( group_id.size() == ngrads, "number of groups must be exact" );            
-                {// new method for grouping, use CSR builder
-                    std::vector<size_t>   rptr;
-                    utils::SparseCSRMBuilder<unsigned> builder( rptr, idset );
-                    builder.InitBudget( tree.param.num_roots );
-                    for( size_t i = 0; i < group_id.size(); i ++ ){
-                        // drop invalid elements
-                        if( hess[ i ] < 0.0f ) continue;
-                        utils::Assert( group_id[ i ] < (unsigned)tree.param.num_roots, 
-                                       "group id exceed number of roots" );
-                        builder.AddBudget( group_id[ i ] );
-                    }
-                    builder.InitStorage();
-                    for( size_t i = 0; i < group_id.size(); i ++ ){
-                        // drop invalid elements
-                        if( hess[ i ] < 0.0f ) continue;
-                        builder.PushElem( group_id[ i ], static_cast<unsigned>(i) );
-                    }
-                    for( size_t i = 1; i < rptr.size(); i ++ ){
-                        const size_t start = rptr[ i - 1 ], end = rptr[ i ];
-                        if( start < end ){
-                            this->add_task( Task( i - 1, &idset[ start ], end - start ) );
-                        }
-                    }
-                }
-            }
-        public:
-            RTreeUpdater( const TreeParamTrain &pparam, 
-                          RegTree &ptree,
-                          std::vector<float> &pgrad,
-                          std::vector<float> &phess,
-                          const FMatrix &psmat, 
-                          const std::vector<unsigned> &pgroup_id ):
-                param( pparam ), tree( ptree ), grad( pgrad ), hess( phess ),
-                smat( psmat ), group_id( pgroup_id ){
-            }
-            inline int do_boost( int &num_pruned ){
-                this->init_tasks( grad.size() );
-                this->max_depth = 0;
-                this->num_pruned = 0;
-                Task tsk;
-                while( this->next_task( tsk ) ){
-                    this->expand( tsk );
-                }
-                num_pruned = this->num_pruned;
-                return max_depth;
-            }
-        };
-    };
-};
-#endif
-
-

booster/tree/xgboost_tree.hpp

@@ -1,268 +0,0 @@
-#ifndef XGBOOST_TREE_HPP
-#define XGBOOST_TREE_HPP
-/*!
- * \file xgboost_tree.hpp
- * \brief implementation of regression tree
- * \author Tianqi Chen: tianqi.tchen@gmail.com 
- */
-#include "xgboost_tree_model.h"
-
-namespace xgboost{
-    namespace booster{
-        const bool rt_debug = false;
-        // whether to check bugs
-        const bool check_bug = false;
-    
-        const float rt_eps = 1e-5f;
-        const float rt_2eps = rt_eps * 2.0f;
-        
-        inline double sqr( double a ){
-            return a * a;
-        }
-    };
-};
-#include "../../utils/xgboost_fmap.h"
-#include "xgboost_svdf_tree.hpp"
-#include "xgboost_col_treemaker.hpp"
-#include "xgboost_row_treemaker.hpp"
-
-namespace xgboost{
-    namespace booster{
-        // regression tree, construction algorithm is seperated from this class
-        // see RegTreeUpdater
-        template<typename FMatrix>
-        class RegTreeTrainer : public InterfaceBooster<FMatrix>{
-        public:
-            RegTreeTrainer( void ){ 
-                silent = 0; tree_maker = 1; 
-                // interact mode
-                interact_type = 0;
-                interact_node = 0;
-                // normally we won't have more than 64 OpenMP threads
-                threadtemp.resize( 64, ThreadEntry() );
-            }
-            virtual ~RegTreeTrainer( void ){}
-        public:
-            virtual void SetParam( const char *name, const char *val ){
-                if( !strcmp( name, "silent") )      silent = atoi( val );
-                if( !strcmp( name, "tree_maker") )  tree_maker = atoi( val );
-                if( !strncmp( name, "interact:", 9) ){
-                    const char *ename = name + 9;
-                    interact_node = atoi( val );
-                    if( !strcmp( ename, "expand") ) {
-                        interact_type = 1;
-                    }
-                    if( !strcmp( ename, "remove") ) {
-                        interact_type = 2;
-                    }                    
-                }
-                param.SetParam( name, val );
-                constrain.SetParam( name, val );
-                tree.param.SetParam( name, val );
-            }
-            virtual void LoadModel( utils::IStream &fi ){
-                tree.LoadModel( fi );
-            }
-            virtual void SaveModel( utils::IStream &fo ) const{
-                tree.SaveModel( fo );
-            }
-            virtual void InitModel( void ){
-                tree.InitModel();
-            }
-        public:
-            virtual void DoBoost( std::vector<float> &grad, 
-                                  std::vector<float> &hess,
-                                  const FMatrix &smat,
-                                  const std::vector<unsigned> &root_index ){
-                utils::Assert( grad.size() < UINT_MAX, "number of instance exceed what we can handle" );
-
-                // interactive update 
-                if( interact_type != 0 ){
-                    switch( interact_type ){
-                    case 1: this->ExpandNode( grad, hess, smat, root_index, interact_node ); return;
-                    case 2: this->CollapseNode( grad, hess, smat, root_index, interact_node ); return;
-                    default: utils::Error("unknown interact type");
-                    }
-                }
-
-                if( !silent ){
-                    printf( "\nbuild GBRT with %u instances\n", (unsigned)grad.size() );
-                }
-                int num_pruned;
-                switch( tree_maker ){
-                case 0: {
-                    utils::Assert( !constrain.HasConstrain(), "tree maker 0 does not support constrain" );
-                    RTreeUpdater<FMatrix> updater( param, tree, grad, hess, smat, root_index );
-                    tree.param.max_depth = updater.do_boost( num_pruned );
-                    break;
-                }
-                case 1:{
-                    ColTreeMaker<FMatrix> maker( tree, param, grad, hess, smat, root_index, constrain );
-                    maker.Make( tree.param.max_depth, num_pruned );
-                    break;
-                }
-                case 2:{
-                    RowTreeMaker<FMatrix> maker( tree, param, grad, hess, smat, root_index, constrain );
-                    maker.Make( tree.param.max_depth, num_pruned );
-                    break;
-                }                    
-                default: utils::Error("unknown tree maker");
-                }
-                if( !silent ){
-                    printf( "tree train end, %d roots, %d extra nodes, %d pruned nodes ,max_depth=%d\n", 
-                            tree.param.num_roots, tree.num_extra_nodes(), num_pruned, tree.MaxDepth() );
-                }
-            }            
-            virtual float Predict( const FMatrix &fmat, bst_uint ridx, unsigned gid = 0 ){
-                ThreadEntry &e = this->InitTmp();
-                this->PrepareTmp( fmat.GetRow(ridx), e );
-                int pid = this->GetLeafIndex( e.feat, e.funknown, gid );
-                this->DropTmp( fmat.GetRow(ridx), e );
-                return tree[ pid ].leaf_value();          
-            }
-            virtual int GetLeafIndex( const std::vector<float> &feat,
-                                      const std::vector<bool>  &funknown,
-                                      unsigned gid = 0 ){
-                // start from groups that belongs to current data
-                int pid = (int)gid;
-                // tranverse tree
-                while( !tree[ pid ].is_leaf() ){
-                    unsigned split_index = tree[ pid ].split_index();
-                    pid = this->GetNext( pid, feat[ split_index ], funknown[ split_index ] );
-                }
-                return pid;
-            }
-
-            virtual void PredPath( std::vector<int> &path, const FMatrix &fmat, bst_uint ridx, unsigned gid = 0 ){
-                path.clear();
-                ThreadEntry &e = this->InitTmp();
-                this->PrepareTmp( fmat.GetRow(ridx), e );
-                
-                int pid = (int)gid;
-                path.push_back( pid );
-                // tranverse tree
-                while( !tree[ pid ].is_leaf() ){                    
-                    unsigned split_index = tree[ pid ].split_index();
-                    pid = this->GetNext( pid, e.feat[ split_index ], e.funknown[ split_index ] );
-                    path.push_back( pid );
-                }
-                this->DropTmp( fmat.GetRow(ridx), e );
-            }
-            virtual float Predict( const std::vector<float> &feat, 
-                                   const std::vector<bool>  &funknown,
-                                   unsigned gid = 0 ){
-                utils::Assert( feat.size() >= (size_t)tree.param.num_feature,
-                               "input data smaller than num feature" );
-                int pid = this->GetLeafIndex( feat, funknown, gid );
-                return tree[ pid ].leaf_value();
-            }            
-            virtual void DumpModel( FILE *fo, const utils::FeatMap &fmap, bool with_stats ){
-                tree.DumpModel( fo, fmap, with_stats );
-            }
-        private:
-            inline void CollapseNode( std::vector<float> &grad, 
-                                      std::vector<float> &hess,
-                                      const FMatrix &fmat,
-                                      const std::vector<unsigned> &root_index, 
-                                      int nid ){
-                std::vector<bst_uint> valid_index;
-                for( size_t i = 0; i < grad.size(); i ++ ){
-                    ThreadEntry &e = this->InitTmp();
-                    this->PrepareTmp( fmat.GetRow(i), e );
-                    int pid = root_index.size() == 0 ? 0 : (int)root_index[i];
-                    // tranverse tree
-                    while( !tree[ pid ].is_leaf() ){                    
-                        unsigned split_index = tree[ pid ].split_index();
-                        pid = this->GetNext( pid, e.feat[ split_index ], e.funknown[ split_index ] );
-                        if( pid == nid ){
-                            valid_index.push_back( static_cast<bst_uint>(i) );  break;
-                        }
-                    }
-                    this->DropTmp( fmat.GetRow(i), e );
-                }
-                RowTreeMaker<FMatrix> maker( tree, param, grad, hess, fmat, root_index, constrain ); 
-                maker.Collapse( valid_index, nid );
-                if( !silent ){
-                    printf( "tree collapse end, max_depth=%d\n", tree.param.max_depth );
-                }                
-            }
-            inline void ExpandNode( std::vector<float> &grad, 
-                                    std::vector<float> &hess,
-                                    const FMatrix &fmat,
-                                    const std::vector<unsigned> &root_index, 
-                                    int nid ){
-                std::vector<bst_uint> valid_index;
-                for( size_t i = 0; i < grad.size(); i ++ ){
-                    ThreadEntry &e = this->InitTmp();
-                    this->PrepareTmp( fmat.GetRow(i), e );
-                    unsigned rtidx = root_index.size() == 0 ? 0 : root_index[i]; 
-                    int pid = this->GetLeafIndex( e.feat, e.funknown, rtidx );
-                    this->DropTmp( fmat.GetRow(i), e );
-                    if( pid == nid ) valid_index.push_back( static_cast<bst_uint>(i) ); 
-                }
-                RowTreeMaker<FMatrix> maker( tree, param, grad, hess, fmat, root_index, constrain ); 
-                bool success =  maker.Expand( valid_index, nid );
-                if( !silent ){
-                    printf( "tree expand end, success=%d, max_depth=%d\n", (int)success, tree.MaxDepth() );
-                }                
-            }
-        private:
-            // silent 
-            int silent;
-            RegTree tree;
-            TreeParamTrain param;
-        private:
-            // some training parameters
-            // tree maker
-            int tree_maker;
-            // interaction
-            int interact_type;
-            int interact_node;         
-            // feature constrain
-            utils::FeatConstrain  constrain;   
-        private:
-            struct ThreadEntry{
-                std::vector<float> feat;
-                std::vector<bool>  funknown;
-            };
-            std::vector<ThreadEntry> threadtemp;
-        private:
-            inline ThreadEntry& InitTmp( void ){
-                const int tid = omp_get_thread_num();
-                utils::Assert( tid < (int)threadtemp.size(), "RTreeUpdater: threadtemp pool is too small" );
-                ThreadEntry &e = threadtemp[ tid ];
-                if( e.feat.size() != (size_t)tree.param.num_feature ){
-                    e.feat.resize( tree.param.num_feature );
-                    e.funknown.resize( tree.param.num_feature );
-                    std::fill( e.funknown.begin(), e.funknown.end(), true );
-                }
-                return e;
-            }
-            inline void PrepareTmp( typename FMatrix::RowIter it, ThreadEntry &e ){
-                while( it.Next() ){
-                    const bst_uint findex = it.findex();
-                    utils::Assert( findex < (unsigned)tree.param.num_feature , "input feature execeed bound" );
-                    e.funknown[ findex ] = false;
-                    e.feat[ findex ] = it.fvalue();
-                } 
-            }
-            inline void DropTmp( typename FMatrix::RowIter it, ThreadEntry &e ){
-                while( it.Next() ){
-                    e.funknown[ it.findex() ] = true;
-                }
-            }
-
-            inline int GetNext( int pid, float fvalue, bool is_unknown ){
-                float split_value = tree[ pid ].split_cond();
-                if( is_unknown ){ 
-                    return tree[ pid ].cdefault();
-                }else{
-                    if( fvalue < split_value ) return tree[ pid ].cleft();
-                    else return tree[ pid ].cright();
-                }
-            }
-        };
-    };
-};
-
-#endif

booster/tree/xgboost_tree_model.h

@@ -1,554 +0,0 @@
-#ifndef XGBOOST_TREE_MODEL_H
-#define XGBOOST_TREE_MODEL_H
-/*!
- * \file xgboost_tree_model.h
- * \brief generic definition of model structure used in tree models
- *        used to support learning of boosting tree
- * \author Tianqi Chen: tianqi.tchen@gmail.com
- */
-#include <cstring>
-#include "../../utils/xgboost_utils.h"
-#include "../../utils/xgboost_stream.h"
-
-namespace xgboost{
-    namespace booster{
-        /*!
-         * \brief template class of TreeModel 
-         * \tparam TSplitCond data type to indicate split condition
-         * \tparam TNodeStat auxiliary statistics of node to help tree building
-         */
-        template<typename TSplitCond,typename TNodeStat>
-        class TreeModel{
-        public:
-            /*! \brief data type to indicate split condition */
-            typedef TNodeStat  NodeStat;
-            /*! \brief auxiliary statistics of node to help tree building */
-            typedef TSplitCond SplitCond;
-        public:
-            /*! \brief parameters of the tree */
-            struct Param{
-                /*! \brief number of start root */
-                int num_roots;
-                /*! \brief total number of nodes */
-                int num_nodes;
-                /*!\brief number of deleted nodes */
-                int num_deleted;
-                /*! \brief maximum depth, this is a statistics of the tree */
-                int max_depth;
-                /*! \brief  number of features used for tree construction */
-                int num_feature;
-                /*! \brief reserved part */
-                int reserved[ 32 ];
-                /*! \brief constructor */
-                Param( void ){
-                    max_depth = 0;
-                    memset( reserved, 0, sizeof( reserved ) );
-                }
-                /*! 
-                 * \brief set parameters from outside 
-                 * \param name name of the parameter
-                 * \param val  value of the parameter
-                 */
-                inline void SetParam( const char *name, const char *val ){
-                    if( !strcmp("num_roots", name ) )    num_roots = atoi( val );
-                    if( !strcmp("num_feature", name ) )  num_feature = atoi( val );
-                }
-            };
-            /*! \brief tree node */
-            class Node{
-            private:
-                friend class TreeModel<TSplitCond,TNodeStat>;
-                /*! 
-                 * \brief in leaf node, we have weights, in non-leaf nodes, 
-                 *        we have split condition 
-                 */
-                union Info{
-                    float leaf_value;
-                    TSplitCond split_cond;
-                };
-            private:
-                // pointer to parent, highest bit is used to indicate whether it's a left child or not 
-                int parent_;
-                // pointer to left, right
-                int cleft_, cright_;
-                // split feature index, left split or right split depends on the highest bit
-                unsigned sindex_;            
-                // extra info
-                Info info_;
-            private:
-                inline void set_parent( int pidx, bool is_left_child = true ){
-                    if( is_left_child ) pidx |= (1U << 31);
-                    this->parent_ = pidx;
-                }
-            public:
-                /*! \brief index of left child */
-                inline int cleft( void ) const{
-                    return this->cleft_;
-                }
-                /*! \brief index of right child */
-                inline int cright( void ) const{
-                    return this->cright_;
-                }
-                /*! \brief index of default child when feature is missing */
-                inline int cdefault( void ) const{
-                    return this->default_left() ? this->cleft() : this->cright();
-                }
-                /*! \brief feature index of split condition */
-                inline unsigned split_index( void ) const{
-                    return sindex_ & ( (1U<<31) - 1U );
-                }
-                /*! \brief when feature is unknown, whether goes to left child */
-                inline bool default_left( void ) const{
-                    return (sindex_ >> 31) != 0;
-                } 
-                /*! \brief whether current node is leaf node */
-                inline bool is_leaf( void ) const{
-                    return cleft_ == -1;
-                }
-                /*! \brief get leaf value of leaf node */
-                inline float leaf_value( void ) const{
-                    return (this->info_).leaf_value;
-                }
-                /*! \brief get split condition of the node */
-                inline TSplitCond split_cond( void ) const{
-                    return (this->info_).split_cond;
-                }
-                /*! \brief get parent of the node */
-                inline int parent( void ) const{
-                    return parent_ & ( (1U << 31) - 1 );
-                } 
-                /*! \brief whether current node is left child */
-                inline bool is_left_child( void ) const{
-                    return ( parent_ & (1U << 31)) != 0;
-                }
-                /*! \brief whether current node is root */
-                inline bool is_root( void ) const{
-                    return parent_ == -1;
-                }
-                /*! 
-                 * \brief set the right child 
-                 * \param nide node id to right child
-                 */
-                inline void set_right_child( int nid ){
-                    this->cright_ = nid;
-                }
-                /*! 
-                 * \brief set split condition of current node 
-                 * \param split_index feature index to split
-                 * \param split_cond  split condition
-                 * \param default_left the default direction when feature is unknown
-                 */
-                inline void set_split( unsigned split_index, TSplitCond split_cond, bool default_left = false ){
-                    if( default_left ) split_index |= (1U << 31);
-                    this->sindex_ = split_index;
-                    (this->info_).split_cond = split_cond;
-                }
-                /*! 
-                 * \brief set the leaf value of the node
-                 * \param value leaf value
-                 * \param right right index, could be used to store 
-                 *        additional information
-                 */
-                inline void set_leaf( float value, int right = -1 ){
-                    (this->info_).leaf_value = value;
-                    this->cleft_ = -1;
-                    this->cright_ = right;
-                }
-            };
-        protected:
-            // vector of nodes
-            std::vector<Node> nodes;
-            // stats of nodes
-            std::vector<TNodeStat> stats;
-        protected:
-            // free node space, used during training process
-            std::vector<int>  deleted_nodes;
-            // allocate a new node, 
-            // !!!!!! NOTE: may cause BUG here, nodes.resize
-            inline int AllocNode( void ){
-                if( param.num_deleted != 0 ){
-                    int nd = deleted_nodes.back();
-                    deleted_nodes.pop_back();
-                    param.num_deleted --;
-                    return nd;
-                }
-                int nd = param.num_nodes ++;
-                nodes.resize( param.num_nodes );
-                stats.resize( param.num_nodes );
-                return nd;
-            }
-            // delete a tree node
-            inline void DeleteNode( int nid ){
-                utils::Assert( nid >= param.num_roots, "can not delete root");
-                deleted_nodes.push_back( nid );
-                nodes[ nid ].set_parent( -1 );
-                param.num_deleted ++;
-            }
-        public:
-            /*! 
-             * \brief change a non leaf node to a leaf node, delete its children
-             * \param rid node id of the node
-             * \param new leaf value
-             */
-            inline void ChangeToLeaf( int rid, float value ){
-                utils::Assert( nodes[ nodes[rid].cleft()  ].is_leaf(), "can not delete a non termial child");
-                utils::Assert( nodes[ nodes[rid].cright() ].is_leaf(), "can not delete a non termial child");
-                this->DeleteNode( nodes[ rid ].cleft() ); 
-                this->DeleteNode( nodes[ rid ].cright() );
-                nodes[ rid ].set_leaf( value );
-            }
-            /*! 
-             * \brief collapse a non leaf node to a leaf node, delete its children
-             * \param rid node id of the node
-             * \param new leaf value
-             */
-            inline void CollapseToLeaf( int rid, float value ){
-                if( nodes[rid].is_leaf() ) return;
-                if( !nodes[ nodes[rid].cleft()  ].is_leaf() ){
-                    CollapseToLeaf( nodes[rid].cleft(), 0.0f );
-                }
-                if( !nodes[ nodes[rid].cright()  ].is_leaf() ){
-                    CollapseToLeaf( nodes[rid].cright(), 0.0f );
-                }
-                this->ChangeToLeaf( rid, value );
-            }
-        public:
-            /*! \brief model parameter */
-            Param param;
-        public:
-            /*! \brief constructor */
-            TreeModel( void ){
-                param.num_nodes = 1;
-                param.num_roots = 1;
-                param.num_deleted = 0;
-                nodes.resize( 1 );
-            }
-            /*! \brief get node given nid */
-            inline Node &operator[]( int nid ){
-                return nodes[ nid ];
-            }
-            /*! \brief get node statistics given nid */
-            inline NodeStat &stat( int nid ){
-                return stats[ nid ];
-            }
-            /*! \brief initialize the model */
-            inline void InitModel( void ){
-                param.num_nodes = param.num_roots;
-                nodes.resize( param.num_nodes );
-                stats.resize( param.num_nodes );
-                for( int i = 0; i < param.num_nodes; i ++ ){
-                    nodes[i].set_leaf( 0.0f );
-                    nodes[i].set_parent( -1 );
-                }
-            }
-            /*! 
-             * \brief load model from stream
-             * \param fi input stream
-             */
-            inline void LoadModel( utils::IStream &fi ){
-                utils::Assert( fi.Read( &param, sizeof(Param) ) > 0, "TreeModel" );
-                nodes.resize( param.num_nodes ); stats.resize( param.num_nodes );
-                utils::Assert( fi.Read( &nodes[0], sizeof(Node) * nodes.size() ) > 0, "TreeModel::Node" );
-                utils::Assert( fi.Read( &stats[0], sizeof(NodeStat) * stats.size() ) > 0, "TreeModel::Node" );
-
-                deleted_nodes.resize( 0 );
-                for( int i = param.num_roots; i < param.num_nodes; i ++ ){
-                    if( nodes[i].is_root() ) deleted_nodes.push_back( i );
-                }
-                utils::Assert( (int)deleted_nodes.size() == param.num_deleted, "number of deleted nodes do not match" );
-            }
-            /*! 
-             * \brief save model to stream
-             * \param fo output stream
-             */
-            inline void SaveModel( utils::IStream &fo ) const{
-                utils::Assert( param.num_nodes == (int)nodes.size() );
-                utils::Assert( param.num_nodes == (int)stats.size() );
-                fo.Write( &param, sizeof(Param) );
-                fo.Write( &nodes[0], sizeof(Node) * nodes.size() );
-                fo.Write( &stats[0], sizeof(NodeStat) * nodes.size() );
-            }
-            /*! 
-             * \brief add child nodes to node
-             * \param nid node id to add childs
-             */
-            inline void AddChilds( int nid ){
-                int pleft  = this->AllocNode();
-                int pright = this->AllocNode();
-                nodes[ nid ].cleft_  = pleft;
-                nodes[ nid ].cright_ = pright;
-                nodes[ nodes[ nid ].cleft()  ].set_parent( nid, true );
-                nodes[ nodes[ nid ].cright() ].set_parent( nid, false );
-            }
-            /*! 
-             * \brief only add a right child to a leaf node 
-             * \param node id to add right child
-             */
-            inline void AddRightChild( int nid ){
-                int pright = this->AllocNode();
-                nodes[ nid ].right  = pright;
-                nodes[ nodes[ nid ].right  ].set_parent( nid, false );
-            }
-            /*!
-             * \brief get current depth
-             * \param nid node id
-             * \param pass_rchild whether right child is not counted in depth
-             */
-            inline int GetDepth( int nid, bool pass_rchild = false ) const{
-                int depth = 0;
-                while( !nodes[ nid ].is_root() ){
-                    if( !pass_rchild || nodes[ nid ].is_left_child() ) depth ++;
-                    nid = nodes[ nid ].parent();
-                }
-                return depth;
-            }
-            /*!
-             * \brief get maximum depth
-             * \param nid node id
-             */
-            inline int MaxDepth( int nid ) const{
-                if( nodes[nid].is_leaf() ) return 0;
-                return std::max( MaxDepth( nodes[nid].cleft() )+1, 
-                                 MaxDepth( nodes[nid].cright() )+1 );
-            }
-            /*!
-             * \brief get maximum depth
-             */
-            inline int MaxDepth( void ){
-                int maxd = 0;
-                for( int i = 0; i < param.num_roots; ++ i ){
-                    maxd = std::max( maxd, MaxDepth( i ) );
-                }
-                return maxd;
-            }
-            /*! \brief number of extra nodes besides the root */
-            inline int num_extra_nodes( void ) const {
-                return param.num_nodes - param.num_roots - param.num_deleted;
-            }
-            /*! \brief dump model to text file  */
-            inline void DumpModel( FILE *fo, const utils::FeatMap& fmap, bool with_stats ){
-                this->Dump( 0, fo, fmap, 0, with_stats );
-            }
-        private:
-            void Dump( int nid, FILE *fo, const utils::FeatMap& fmap, int depth, bool with_stats ){
-                for( int  i = 0;  i < depth; ++ i ){
-                    fprintf( fo, "\t" );
-                }
-                if( nodes[ nid ].is_leaf() ){
-                    fprintf( fo, "%d:leaf=%f ", nid, nodes[ nid ].leaf_value() );
-                    if( with_stats ){
-                        stat( nid ).Print( fo, true );
-                    }
-                    fprintf( fo, "\n" );
-                }else{
-                    // right then left,
-                    TSplitCond cond = nodes[ nid ].split_cond();
-                    const unsigned split_index = nodes[ nid ].split_index();
-
-                    if( split_index < fmap.size() ){
-                        switch( fmap.type(split_index) ){
-                        case utils::FeatMap::kIndicator:{
-                            int nyes = nodes[ nid ].default_left()?nodes[nid].cright():nodes[nid].cleft();
-                            fprintf( fo, "%d:[%s] yes=%d,no=%d", 
-                                     nid, fmap.name( split_index ),
-                                     nyes, nodes[nid].cdefault() );
-                            break;                            
-                        }
-                        case utils::FeatMap::kInteger:{
-                            fprintf( fo, "%d:[%s<%d] yes=%d,no=%d,missing=%d", 
-                                     nid, fmap.name(split_index), int( float(cond)+1.0f), 
-                                     nodes[ nid ].cleft(), nodes[ nid ].cright(),
-                                     nodes[ nid ].cdefault() );
-                            break;
-                        }
-                        case utils::FeatMap::kFloat:
-                        case utils::FeatMap::kQuantitive:{
-                            fprintf( fo, "%d:[%s<%f] yes=%d,no=%d,missing=%d", 
-                                     nid, fmap.name(split_index), float(cond), 
-                                     nodes[ nid ].cleft(), nodes[ nid ].cright(),
-                                     nodes[ nid ].cdefault() );
-                            break;
-                        }
-                        default: utils::Error("unknown fmap type");
-                        }
-                    }else{
-                        fprintf( fo, "%d:[f%u<%f] yes=%d,no=%d,missing=%d", 
-                                 nid, split_index, float(cond), 
-                                 nodes[ nid ].cleft(), nodes[ nid ].cright(),
-                                 nodes[ nid ].cdefault() );
-                    }
-                    if( with_stats ){
-                        fprintf( fo, " ");
-                        stat( nid ).Print( fo, false );
-                    }
-                    fprintf( fo, "\n" );
-                    this->Dump( nodes[ nid ].cleft(), fo, fmap, depth+1, with_stats );
-                    this->Dump( nodes[ nid ].cright(), fo, fmap, depth+1, with_stats );
-                }                
-            } 
-        };
-    };
-    
-    namespace booster{
-        /*! \brief training parameters for regression tree */
-        struct TreeParamTrain{
-            // learning step size for a time
-            float learning_rate;
-            // minimum loss change required for a split
-            float min_split_loss;
-            // maximum depth of a tree
-            int   max_depth;
-            //----- the rest parameters are less important ----
-            // minimum amount of hessian(weight) allowed in a child
-            float min_child_weight;
-            // weight decay parameter used to control leaf fitting
-            float reg_lambda;
-            // reg method
-            int   reg_method;
-            // default direction choice
-            int   default_direction;
-            // whether we want to do subsample
-            float subsample;
-            // whether to use layerwise aware regularization
-            int   use_layerwise;
-            // number of threads to be used for tree construction, if OpenMP is enabled, if equals 0, use system default
-            int nthread;
-            /*! \brief constructor */
-            TreeParamTrain( void ){
-                learning_rate = 0.3f;
-                min_child_weight = 1.0f;
-                max_depth = 6;
-                reg_lambda = 1.0f;
-                reg_method = 2;
-                default_direction = 0;
-                subsample = 1.0f;
-                use_layerwise = 0;
-                nthread = 0;
-            }
-            /*! 
-             * \brief set parameters from outside 
-             * \param name name of the parameter
-             * \param val  value of the parameter
-             */            
-            inline void SetParam( const char *name, const char *val ){
-                // sync-names 
-                if( !strcmp( name, "gamma") )  min_split_loss = (float)atof( val );
-                if( !strcmp( name, "eta") )    learning_rate  = (float)atof( val );
-                if( !strcmp( name, "lambda") ) reg_lambda = (float)atof( val );
-                // normal tree prameters
-                if( !strcmp( name, "learning_rate") )     learning_rate = (float)atof( val );
-                if( !strcmp( name, "min_child_weight") )  min_child_weight = (float)atof( val );
-                if( !strcmp( name, "min_split_loss") )    min_split_loss = (float)atof( val );
-                if( !strcmp( name, "max_depth") )         max_depth = atoi( val );           
-                if( !strcmp( name, "reg_lambda") )        reg_lambda = (float)atof( val );
-                if( !strcmp( name, "reg_method") )        reg_method = (float)atof( val );
-                if( !strcmp( name, "subsample") )         subsample  = (float)atof( val );
-                if( !strcmp( name, "use_layerwise") )     use_layerwise = atoi( val );
-                if( !strcmp( name, "nthread") )           nthread = atoi( val );
-                if( !strcmp( name, "default_direction") ) {
-                    if( !strcmp( val, "learn") )  default_direction = 0;
-                    if( !strcmp( val, "left") )   default_direction = 1;
-                    if( !strcmp( val, "right") )  default_direction = 2;
-                }
-            }
-        protected:
-            // functions for L1 cost
-            static inline double ThresholdL1( double w, double lambda ){
-                if( w > +lambda ) return w - lambda;
-                if( w < -lambda ) return w + lambda;
-                return 0.0;
-            }
-            inline double CalcWeight( double sum_grad, double sum_hess )const{
-                if( sum_hess < min_child_weight ){
-                    return 0.0;
-                }else{
-                    switch( reg_method ){
-                    case 1: return - ThresholdL1( sum_grad, reg_lambda ) / sum_hess;
-                    case 2: return - sum_grad / ( sum_hess + reg_lambda );
-                        // elstic net
-                    case 3: return - ThresholdL1( sum_grad, 0.5 * reg_lambda ) / ( sum_hess + 0.5 * reg_lambda );
-                    default: return - sum_grad / sum_hess;
-                    }
-                }
-            }
-        private:
-            inline static double Sqr( double a ){
-                return a * a;
-            }
-        public:
-            // calculate the cost of loss function
-            inline double CalcGain( double sum_grad, double sum_hess ) const{
-                if( sum_hess < min_child_weight ){
-                    return 0.0;
-                }
-                switch( reg_method ){
-                case 1 : return Sqr( ThresholdL1( sum_grad, reg_lambda ) ) / sum_hess;
-                case 2 : return Sqr( sum_grad ) / ( sum_hess + reg_lambda );
-                    // elstic net
-                case 3 : return Sqr( ThresholdL1( sum_grad, 0.5 * reg_lambda ) ) / ( sum_hess + 0.5 * reg_lambda );
-                default: return Sqr( sum_grad ) / sum_hess;
-                }        
-            }
-            // KEY:layerwise
-            // calculate cost of root
-            inline double CalcRootGain( double sum_grad, double sum_hess ) const{
-                if( use_layerwise == 0 ) return this->CalcGain( sum_grad, sum_hess );
-                else return 0.0;
-            }
-            // KEY:layerwise
-            // calculate the cost after split
-            // base_weight: the base_weight of parent           
-            inline double CalcGain( double sum_grad, double sum_hess, double base_weight ) const{
-                if( use_layerwise == 0 ) return this->CalcGain( sum_grad, sum_hess );
-                else return this->CalcGain( sum_grad + sum_hess * base_weight, sum_hess );
-            }
-            // calculate the weight of leaf
-            inline double CalcWeight( double sum_grad, double sum_hess, double parent_base_weight )const{
-                if( use_layerwise == 0 ) return CalcWeight( sum_grad, sum_hess );
-                else return parent_base_weight + CalcWeight( sum_grad + parent_base_weight * sum_hess, sum_hess );
-            }           
-            /*! \brief whether need forward small to big search: default right */
-            inline bool need_forward_search( void ) const{
-                return this->default_direction != 1;
-            }
-            /*! \brief whether need forward big to small search: default left */
-            inline bool need_backward_search( void ) const{
-                return this->default_direction != 2;
-            }
-            /*! \brief given the loss change, whether we need to invode prunning */
-            inline bool need_prune( double loss_chg, int depth ) const{
-                return loss_chg < this->min_split_loss;
-            }
-            /*! \brief whether we can split with current hessian */
-            inline bool cannot_split( double sum_hess, int depth ) const{
-                return sum_hess < this->min_child_weight * 2.0; 
-            }
-        };
-    };
-    
-    namespace booster{
-        /*! \brief node statistics used in regression tree */
-        struct RTreeNodeStat{
-            /*! \brief loss chg caused by current split */
-            float loss_chg;
-            /*! \brief sum of hessian values, used to measure coverage of data */
-            float sum_hess;
-            /*! \brief weight of current node */
-            float base_weight;
-            /*! \brief number of child that is leaf node known up to now */
-            int   leaf_child_cnt;
-            /*! \brief print information of current stats to fo */
-            inline void Print( FILE *fo, bool is_leaf ) const{
-                if( !is_leaf ){
-                    fprintf( fo, "gain=%f,cover=%f", loss_chg, sum_hess );
-                }else{
-                    fprintf( fo, "cover=%f", sum_hess );
-                }
-            }
-        };
-        /*! \brief most comment structure of regression tree */
-        class RegTree: public TreeModel<bst_float,RTreeNodeStat>{
-        };
-    };
-};
-#endif

booster/xgboost-inl.hpp

@@ -1,39 +0,0 @@
-#ifndef XGBOOST_INL_HPP
-#define XGBOOST_INL_HPP
-/*!
- * \file xgboost-inl.hpp
- * \brief bootser implementations 
- * \author Tianqi Chen: tianqi.tchen@gmail.com
- */
-// implementation of boosters go to here 
-
-// A good design should have minimum functions defined interface, user should only operate on interface
-// I break it a bit, by using template and let user 'see' the implementation
-// The user should pretend that they only can use the interface, and we are all cool
-// I find this is the only way so far I can think of to make boosters invariant of data structure, 
-// while keep everything fast
-#include "xgboost.h"
-#include "../utils/xgboost_utils.h"
-#include "tree/xgboost_tree.hpp"
-#include "linear/xgboost_linear.hpp"
-
-namespace xgboost{
-	namespace booster{
-        /*!
-         * \brief create a gradient booster, given type of booster
-         * \param booster_type type of gradient booster, can be used to specify implements
-         * \tparam FMatrix input data type for booster
-         * \return the pointer to the gradient booster created
-         */
-        template<typename FMatrix>
-        inline InterfaceBooster<FMatrix> *CreateBooster( int booster_type ){
-            switch( booster_type ){
-            case 0: return new RegTreeTrainer<FMatrix>();
-            case 1: return new LinearBooster<FMatrix>();
-            default: utils::Error("unknown booster_type"); return NULL;
-            }
-		}
-	}; // namespace booster
-}; // namespace xgboost
-
-#endif // XGBOOST_INL_HPP

booster/xgboost.h

@@ -1,157 +0,0 @@
-#ifndef XGBOOST_H
-#define XGBOOST_H
-/*!
- * \file xgboost.h
- * \brief the general gradient boosting interface
- *
- *   common practice of this header: use IBooster and CreateBooster<FMatrixS>
- *
- * \author Tianqi Chen: tianqi.tchen@gmail.com
- */
-#include <vector>
-#include "../utils/xgboost_utils.h"
-#include "../utils/xgboost_fmap.h"
-#include "../utils/xgboost_stream.h"
-#include "../utils/xgboost_config.h"
-#include "xgboost_data.h"
-
-/*! \brief namespace for xboost package */
-namespace xgboost{
-    /*! \brief namespace for boosters */
-    namespace booster{
-        /*! 
-         * \brief interface of a gradient boosting learner 
-         * \tparam FMatrix the feature matrix format that the booster takes
-         */
-        template<typename FMatrix>
-        class InterfaceBooster{
-        public:
-            // interface for model setting and loading
-            // calling procedure:
-            //  (1) booster->SetParam to setting necessary parameters
-            //  (2) if it is first time usage of the model: 
-            //          call booster->InitModel
-            //      else: 
-            //          call booster->LoadModel
-            //  (3) booster->DoBoost to update the model
-            //  (4) booster->Predict to get new prediction
-            /*! 
-             * \brief set parameters from outside 
-             * \param name name of the parameter
-             * \param val  value of the parameter
-             */
-            virtual void SetParam( const char *name, const char *val ) = 0;
-            /*! 
-             * \brief load model from stream
-             * \param fi input stream
-             */
-            virtual void LoadModel( utils::IStream &fi ) = 0;
-            /*! 
-             * \brief save model to stream
-             * \param fo output stream
-             */
-            virtual void SaveModel( utils::IStream &fo ) const = 0;
-            /*!
-             * \brief initialize solver before training, called before training
-             * this function is reserved for solver to allocate necessary space and do other preparation 
-             */
-            virtual void InitModel( void ) = 0;
-        public:
-            /*! 
-             * \brief do gradient boost training for one step, using the information given, 
-             *        Note: content of grad and hess can change after DoBoost
-             * \param grad first order gradient of each instance
-             * \param hess second order gradient of each instance
-             * \param feats features of each instance
-             * \param root_index pre-partitioned root index of each instance, 
-             *          root_index.size() can be 0 which indicates that no pre-partition involved
-             */
-            virtual void DoBoost( std::vector<float> &grad,
-                                  std::vector<float> &hess,
-                                  const FMatrix &feats,
-                                  const std::vector<unsigned> &root_index ) = 0;
-            /*! 
-             * \brief predict the path ids along a trees, for given sparse feature vector. When booster is a tree
-             * \param path the result of path
-             * \param feats feature matrix
-             * \param row_index  row index in the feature matrix
-             * \param root_index root id of current instance, default = 0
-             */
-            virtual void PredPath( std::vector<int> &path, const FMatrix &feats, 
-                                   bst_uint row_index, unsigned root_index = 0 ){
-                utils::Error( "not implemented" );
-            }
-            /*! 
-             * \brief predict values for given sparse feature vector
-             * 
-             *   NOTE: in tree implementation, Sparse Predict is OpenMP threadsafe, but not threadsafe in general,
-             *         dense version of Predict to ensures threadsafety
-             * \param feats feature matrix
-             * \param row_index  row index in the feature matrix
-             * \param root_index root id of current instance, default = 0
-             * \return prediction 
-             */        
-            virtual float Predict( const FMatrix &feats, bst_uint row_index, unsigned root_index = 0 ){                
-                utils::Error( "not implemented" );
-                return 0.0f;
-            }
-            /*! 
-             * \brief predict values for given dense feature vector
-             * \param feat feature vector in dense format
-             * \param funknown indicator that the feature is missing
-             * \param rid root id of current instance, default = 0
-             * \return prediction
-             */
-            virtual float Predict( const std::vector<float> &feat, 
-                                   const std::vector<bool>  &funknown,
-                                   unsigned rid = 0 ){
-                utils::Error( "not implemented" );
-                return 0.0f;
-            }
-            /*! 
-             * \brief print information
-             * \param fo output stream 
-             */        
-            virtual void PrintInfo( FILE *fo ){}
-            /*! 
-             * \brief dump model into text file
-             * \param fo output stream 
-             * \param fmap feature map that may help give interpretations of feature
-             * \param with_stats whether print statistics
-             */
-            virtual void DumpModel( FILE *fo, const utils::FeatMap& fmap, bool with_stats = false ){
-                utils::Error( "not implemented" );                
-            }
-        public:
-            /*! \brief virtual destructor */
-            virtual ~InterfaceBooster( void ){}
-        };
-    };
-    namespace booster{
-        /*! 
-         * \brief this will is the most commonly used booster interface 
-         *  we try to make booster invariant of data structures, but most cases, FMatrixS is what we wnat
-         */
-        typedef InterfaceBooster<FMatrixS> IBooster;
-    };
-};
-
-namespace xgboost{
-    namespace booster{
-        /*! 
-         * \brief create a gradient booster, given type of booster
-         *    normally we use FMatrixS, by calling CreateBooster<FMatrixS>
-         * \param booster_type type of gradient booster, can be used to specify implements
-         * \tparam FMatrix input data type for booster
-         * \return the pointer to the gradient booster created
-         */
-        template<typename FMatrix>
-        inline InterfaceBooster<FMatrix> *CreateBooster( int booster_type );
-    };
-};
-
-// this file includes the template implementations of all boosters
-// the cost of using template is that the user can 'see' all the implementations, which is OK 
-// ignore implementations and focus on the interface:) 
-#include "xgboost-inl.hpp"
-#endif

booster/xgboost_data.h

@@ -1,393 +0,0 @@
-#ifndef XGBOOST_DATA_H
-#define XGBOOST_DATA_H
-
-/*!
- * \file xgboost_data.h
- * \brief the input data structure for gradient boosting
- * \author Tianqi Chen: tianqi.tchen@gmail.com
- */
-
-#include <vector>
-#include <climits>
-#include "../utils/xgboost_utils.h"
-#include "../utils/xgboost_stream.h"
-#include "../utils/xgboost_matrix_csr.h"
-
-namespace xgboost{
-    namespace booster{
-        /*! \brief interger type used in boost */
-        typedef int bst_int;
-        /*! \brief unsigned interger type used in boost */
-        typedef unsigned bst_uint;
-        /*! \brief float type used in boost */
-        typedef float bst_float;
-        /*! \brief debug option for booster */    
-        const bool bst_debug = false;    
-    };
-};
-
-namespace xgboost{
-    namespace booster{
-        /**
-         * \brief This is a interface, defining the way to access features, 
-         *        by column or by row. This interface is used to make implementation
-         *        of booster does not depend on how feature is stored.
-         *
-         *        Why template instead of virtual class: for efficiency
-         *          feature matrix is going to be used by most inner loop of the algorithm
-         *
-         * \tparam Derived type of actual implementation 
-         * \sa FMatrixS: most of time FMatrixS is sufficient, refer to it if you find it confusing
-         */
-        template<typename Derived>
-        struct FMatrix{
-        public:
-            /*! \brief exmaple iterator over one row */            
-            struct RowIter{
-                /*! 
-                 * \brief move to next position 
-                 * \return whether there is element in next position
-                 */
-
-                inline bool Next( void );
-                /*! \return feature index in current position */
-                inline bst_uint  findex( void ) const;
-                /*! \return feature value in current position */
-                inline bst_float fvalue( void ) const;
-            };
-            /*! \brief example iterator over one column */
-            struct ColIter{
-                /*! 
-                 * \brief move to next position 
-                 * \return whether there is element in next position
-                 */
-                inline bool Next( void );
-                /*! \return row index of current position  */
-                inline bst_uint  rindex( void ) const;
-                /*! \return feature value in current position */
-                inline bst_float fvalue( void ) const;
-            };
-            /*! \brief backward iterator over column */
-            struct ColBackIter : public ColIter {};
-        public:
-            /*! 
-             * \brief get number of rows 
-             * \return number of rows
-             */
-            inline size_t NumRow( void ) const;
-            /*! 
-             * \brief get number of columns
-             * \return number of columns
-             */
-            inline size_t NumCol( void ) const;
-            /*!
-             * \brief get row iterator
-             * \param ridx row index
-             * \return row iterator
-             */
-            inline RowIter GetRow( size_t ridx ) const;
-            /*! 
-             * \brief get number of column groups, this ise used together with GetRow( ridx, gid )
-             * \return number of column group
-             */
-            inline unsigned NumColGroup( void ) const{
-                return 1;
-            }
-            /*!
-             * \brief get row iterator, return iterator of specific column group
-             * \param ridx row index
-             * \param gid colmun group id
-             * \return row iterator, only iterates over features of specified column group
-             */
-            inline RowIter GetRow( size_t ridx, unsigned gid ) const;
-
-            /*! \return whether column access is enabled */
-            inline bool HaveColAccess( void ) const;
-            /*!
-             * \brief get column iterator, the columns must be sorted by feature value
-             * \param ridx column index
-             * \return column iterator
-             */
-            inline ColIter GetSortedCol( size_t ridx ) const;
-            /*!
-             * \brief get column backward iterator, starts from biggest fvalue, and iterator back
-             * \param ridx column index
-             * \return reverse column iterator
-             */
-            inline ColBackIter GetReverseSortedCol( size_t ridx ) const;
-        };
-    };
-};
-
-namespace xgboost{
-    namespace booster{
-        /*! 
-         * \brief feature matrix to store training instance, in sparse CSR format
-         */        
-        class FMatrixS: public FMatrix<FMatrixS>{
-        public:
-            /*! \brief one entry in a row */
-            struct REntry{
-                /*! \brief feature index */
-                bst_uint  findex;
-                /*! \brief feature value */
-                bst_float fvalue;
-                /*! \brief constructor */
-                REntry( void ){}
-                /*! \brief constructor */
-                REntry( bst_uint findex, bst_float fvalue ) : findex(findex), fvalue(fvalue){}
-                inline static bool cmp_fvalue( const REntry &a, const REntry &b ){
-                    return a.fvalue < b.fvalue;
-                }
-            };
-            /*! \brief one row of sparse feature matrix */
-            struct Line{
-                /*! \brief array of feature index */
-                const REntry *data_;
-                /*! \brief size of the data */
-                bst_uint len;
-                /*! \brief get k-th element */
-                inline const REntry& operator[]( unsigned i ) const{
-                    return data_[i];
-                }  
-            };
-            /*! \brief row iterator */
-            struct RowIter{
-                const REntry *dptr_, *end_;
-                RowIter( const REntry* dptr, const REntry* end )
-                    :dptr_(dptr),end_(end){}
-                inline bool Next( void ){
-                    if( dptr_ == end_ ) return false;
-                    else{
-                       ++ dptr_; return true;
-                    }
-                }
-                inline bst_uint  findex( void ) const{
-                    return dptr_->findex;
-                }
-                inline bst_float fvalue( void ) const{
-                    return dptr_->fvalue;
-                }
-            };
-            /*! \brief column iterator */
-            struct ColIter: public RowIter{
-                ColIter( const REntry* dptr, const REntry* end )
-                    :RowIter( dptr, end ){}
-                inline bst_uint  rindex( void ) const{
-                    return this->findex();
-                }
-            };
-            /*! \brief reverse column iterator */
-            struct ColBackIter: public ColIter{
-                ColBackIter( const REntry* dptr, const REntry* end )
-                    :ColIter( dptr, end ){}
-                // shadows RowIter::Next
-                inline bool Next( void ){
-                    if( dptr_ == end_ ) return false;
-                    else{
-                       -- dptr_; return true;
-                    }
-                }
-            };
-        public:
-            /*! \brief constructor */
-            FMatrixS( void ){ this->Clear(); }
-            /*!  \brief get number of rows */
-            inline size_t NumRow( void ) const{
-                return row_ptr_.size() - 1;
-            }
-            /*! 
-             * \brief get number of nonzero entries
-             * \return number of nonzero entries
-             */
-            inline size_t NumEntry( void ) const{
-                return row_data_.size();
-            }
-            /*! \brief clear the storage */
-            inline void Clear( void ){
-                row_ptr_.clear();
-                row_ptr_.push_back( 0 );
-                row_data_.clear();
-                col_ptr_.clear();
-                col_data_.clear();
-            }
-            /*! \brief get sparse part of current row */
-            inline Line operator[]( size_t sidx ) const{
-                Line sp;
-                utils::Assert( !bst_debug || sidx < this->NumRow(), "row id exceed bound" );
-                sp.len = static_cast<bst_uint>( row_ptr_[ sidx + 1 ] - row_ptr_[ sidx ] );
-                sp.data_ = &row_data_[ row_ptr_[ sidx ] ];
-                return sp;
-            }
-            /*! 
-             * \brief add a row to the matrix, with data stored in STL container
-             * \param findex feature index
-             * \param fvalue feature value
-             *  \param fstart start bound of feature
-             *  \param fend   end bound range of feature
-             * \return the row id added line
-             */
-            inline size_t AddRow( const std::vector<bst_uint> &findex, 
-                                  const std::vector<bst_float> &fvalue,
-                                  unsigned fstart = 0, unsigned fend = UINT_MAX ){
-                utils::Assert( findex.size() == fvalue.size() );
-                unsigned cnt = 0;
-                for( size_t i = 0; i < findex.size(); i ++ ){
-                    if( findex[i] < fstart || findex[i] >= fend ) continue;
-                    row_data_.push_back( REntry( findex[i], fvalue[i] ) );
-                    cnt ++;
-                }
-                row_ptr_.push_back( row_ptr_.back() + cnt );
-                return row_ptr_.size() - 2;
-            }
-            /*!  \brief get row iterator*/
-            inline RowIter GetRow( size_t ridx ) const{
-                utils::Assert( !bst_debug || ridx < this->NumRow(), "row id exceed bound" );
-                return RowIter( &row_data_[ row_ptr_[ridx] ] - 1, &row_data_[ row_ptr_[ridx+1] ] - 1 );
-            }
-            /*!  \brief get row iterator*/
-            inline RowIter GetRow( size_t ridx, unsigned gid ) const{
-                utils::Assert( gid == 0, "FMatrixS only have 1 column group" );
-                return FMatrixS::GetRow( ridx );
-            }
-        public:
-            /*! \return whether column access is enabled */
-            inline bool HaveColAccess( void ) const{
-                return col_ptr_.size() != 0 && col_data_.size() == row_data_.size();
-            }
-            /*!  \brief get number of colmuns */
-            inline size_t NumCol( void ) const{
-                utils::Assert( this->HaveColAccess() );
-                return col_ptr_.size() - 1;
-            }
-            /*!  \brief get col iterator*/
-            inline ColIter GetSortedCol( size_t cidx ) const{
-                utils::Assert( !bst_debug || cidx < this->NumCol(), "col id exceed bound" );
-                return ColIter( &col_data_[ col_ptr_[cidx] ] - 1, &col_data_[ col_ptr_[cidx+1] ] - 1 );
-            }
-            /*!  \brief get col iterator */
-            inline ColBackIter GetReverseSortedCol( size_t cidx ) const{
-                utils::Assert( !bst_debug || cidx < this->NumCol(), "col id exceed bound" );
-                return ColBackIter( &col_data_[ col_ptr_[cidx+1] ], &col_data_[ col_ptr_[cidx] ] );
-            }
-            /*!
-             * \brief intialize the data so that we have both column and row major
-             *        access, call this whenever we need column access
-             */
-            inline void InitData( void ){
-                utils::SparseCSRMBuilder<REntry> builder( col_ptr_, col_data_ );
-                builder.InitBudget( 0 );
-                for( size_t i = 0; i < this->NumRow(); i ++ ){
-                    for( RowIter it = this->GetRow(i); it.Next(); ){
-                        builder.AddBudget( it.findex() );
-                    }
-                }
-                builder.InitStorage();
-                for( size_t i = 0; i < this->NumRow(); i ++ ){
-                    for( RowIter it = this->GetRow(i); it.Next(); ){
-                        builder.PushElem( it.findex(), REntry( (bst_uint)i, it.fvalue() ) );
-                    }
-                }
-                // sort columns
-                unsigned ncol = static_cast<unsigned>( this->NumCol() );
-                for( unsigned i = 0; i < ncol; i ++ ){
-                    std::sort( &col_data_[ col_ptr_[ i ] ], &col_data_[ col_ptr_[ i+1 ] ], REntry::cmp_fvalue );
-                }
-            }
-            /*!
-             * \brief save data to binary stream 
-             *        note: since we have size_t in ptr, 
-             *              the function is not consistent between 64bit and 32bit machine
-             * \param fo output stream
-             */
-            inline void SaveBinary( utils::IStream &fo ) const{
-                FMatrixS::SaveBinary( fo, row_ptr_, row_data_ );
-                int col_access = this->HaveColAccess() ? 1 : 0;
-                fo.Write( &col_access, sizeof(int) );
-                if( col_access != 0 ){
-                    FMatrixS::SaveBinary( fo, col_ptr_, col_data_ );
-                }
-            }
-            /*!
-             * \brief load data from binary stream 
-             *        note: since we have size_t in ptr, 
-             *              the function is not consistent between 64bit and 32bit machin
-             * \param fi input stream
-             */
-            inline void LoadBinary( utils::IStream &fi ){
-                FMatrixS::LoadBinary( fi, row_ptr_, row_data_ );
-                int col_access;                
-                fi.Read( &col_access, sizeof(int) );
-                if( col_access != 0 ){
-                    FMatrixS::LoadBinary( fi, col_ptr_, col_data_ );
-                }
-            }
-            /*!
-            * \brief load from text file 
-            * \param fi input file pointer
-            */            
-            inline void LoadText( FILE *fi ){
-                this->Clear();
-                int ninst;
-                while( fscanf( fi, "%d", &ninst ) == 1 ){
-                    std::vector<booster::bst_uint>  findex;
-                    std::vector<booster::bst_float> fvalue;
-                    while(  ninst -- ){
-                        unsigned index; float value;
-                        utils::Assert( fscanf( fi, "%u:%f", &index, &value ) == 2, "load Text" );
-                        findex.push_back( index ); fvalue.push_back( value );
-                    }
-                    this->AddRow( findex, fvalue );                    
-                }
-                // initialize column support as well
-                this->InitData();
-            }
-        private:
-            /*!
-             * \brief save data to binary stream 
-             * \param fo output stream
-             * \param ptr pointer data
-             * \param data data content
-             */
-            inline static void SaveBinary( utils::IStream &fo, 
-                                           const std::vector<size_t> &ptr, 
-                                           const std::vector<REntry> &data ){
-                size_t nrow = ptr.size() - 1;
-                fo.Write( &nrow, sizeof(size_t) );
-                fo.Write( &ptr[0], ptr.size() * sizeof(size_t) );
-                if( data.size() != 0 ){
-                    fo.Write( &data[0] , data.size() * sizeof(REntry) );
-                }
-            }
-            /*!
-             * \brief load data from binary stream 
-             * \param fi input stream
-             * \param ptr pointer data
-             * \param data data content
-             */
-            inline static void LoadBinary( utils::IStream &fi,
-                                           std::vector<size_t> &ptr, 
-                                           std::vector<REntry> &data ){
-                size_t nrow;
-                utils::Assert( fi.Read( &nrow, sizeof(size_t) ) != 0, "Load FMatrixS" );
-                ptr.resize( nrow + 1 );
-                utils::Assert( fi.Read( &ptr[0], ptr.size() * sizeof(size_t) ), "Load FMatrixS" );
-
-                data.resize( ptr.back() ); 
-                if( data.size() != 0 ){
-                    utils::Assert( fi.Read( &data[0] , data.size() * sizeof(REntry) ) , "Load FMatrixS" );
-                }
-            }
-        protected:
-            /*! \brief row pointer of CSR sparse storage */
-            std::vector<size_t>  row_ptr_;
-            /*! \brief data in the row */
-            std::vector<REntry>  row_data_;
-            /*! \brief column pointer of CSC format */
-            std::vector<size_t>  col_ptr_;
-            /*! \brief column datas */
-            std::vector<REntry>  col_data_;
-        };        
-    };
-};
-#endif

booster/xgboost_gbmbase.h

@@ -1,385 +0,0 @@
-#ifndef XGBOOST_GBMBASE_H
-#define XGBOOST_GBMBASE_H
-
-#include <cstring>
-#include "xgboost.h"
-#include "xgboost_data.h"
-#include "../utils/xgboost_omp.h"
-#include "../utils/xgboost_config.h"
-/*!
- * \file xgboost_gbmbase.h
- * \brief a base model class, 
- *        that assembles the ensembles of booster together and do model update
- *        this class can be used as base code to create booster variants 
- *
- *        The detailed implementation of boosters should start by using the class
- *        provided by this file
- *        
- * \author Tianqi Chen: tianqi.tchen@gmail.com
- */
-namespace xgboost{
-    namespace booster{
-        /*!
-         * \brief a base model class, 
-         *        that assembles the ensembles of booster together and provide single routines to do prediction buffer and update
-         *        this class can be used as base code to create booster variants 
-         *         *
-         *  relation to xgboost.h:
-         *    (1) xgboost.h provides a interface to a single booster(e.g. a single regression tree )
-         *        while GBMBaseModel builds upon IBooster to build a class that 
-         *        ensembls the boosters together;
-         *    (2) GBMBaseModel provides prediction buffering scheme to speedup training;
-         *    (3) Summary: GBMBaseModel is a standard wrapper for boosting ensembles;
-         *
-         *  Usage of this class, the number index gives calling dependencies:
-         *    (1) model.SetParam to set the parameters
-         *    (2) model.LoadModel to load old models or model.InitModel to create a new model
-         *    (3) model.InitTrainer before calling model.Predict and model.DoBoost
-         *    (4) model.Predict to get predictions given a instance
-         *    (4) model.DoBoost to update the ensembles, add new booster to the model
-         *    (4) model.SaveModel to save learned results 
-         *
-         *  Bufferring: each instance comes with a buffer_index in Predict. 
-         *              when mparam.num_pbuffer != 0, a unique buffer index can be 
-         *              assigned to each instance to buffer previous results of boosters,
-         *              this helps to speedup training, so consider assign buffer_index 
-         *              for each training instances, if buffer_index = -1, the code
-         *              recalculate things from scratch and will still works correctly
-         */
-        class GBMBase{
-        public:
-            /*! \brief number of thread used */
-            GBMBase( void ){}
-            /*! \brief destructor */
-            virtual ~GBMBase( void ){
-                this->FreeSpace();
-            }
-            /*! 
-             * \brief set parameters from outside 
-             * \param name name of the parameter
-             * \param val  value of the parameter
-             */
-            inline void SetParam( const char *name, const char *val ){
-                if( !strncmp( name, "bst:", 4 ) ){
-                    cfg.PushBack( name + 4, val );
-                }
-                if( !strcmp( name, "silent") ){
-                    cfg.PushBack( name, val );
-                }
-                tparam.SetParam( name, val );
-                if( boosters.size() == 0 ) mparam.SetParam( name, val );
-            }
-            /*! 
-             * \brief load model from stream
-             * \param fi input stream
-             */
-            inline void LoadModel( utils::IStream &fi ){
-                if( boosters.size() != 0 ) this->FreeSpace();
-                utils::Assert( fi.Read( &mparam, sizeof(ModelParam) ) != 0 );
-                boosters.resize( mparam.num_boosters );
-                for( size_t i = 0; i < boosters.size(); i ++ ){
-                    boosters[ i ] = booster::CreateBooster<FMatrixS>( mparam.booster_type );
-                    boosters[ i ]->LoadModel( fi );
-                }
-                {// load info 
-                    booster_info.resize( mparam.num_boosters );
-                    if( mparam.num_boosters != 0 ){
-                        utils::Assert( fi.Read( &booster_info[0], sizeof(int)*mparam.num_boosters ) != 0 );
-                    }
-                }
-                if( mparam.num_pbuffer != 0 ){
-                    pred_buffer.resize ( mparam.num_pbuffer );
-                    pred_counter.resize( mparam.num_pbuffer );
-                    utils::Assert( fi.Read( &pred_buffer[0] , pred_buffer.size()*sizeof(float) ) != 0 );
-                    utils::Assert( fi.Read( &pred_counter[0], pred_counter.size()*sizeof(unsigned) ) != 0 );
-                }
-            }
-            /*! 
-             * \brief save model to stream
-             * \param fo output stream
-             */
-            inline void SaveModel( utils::IStream &fo ) const {
-                utils::Assert( mparam.num_boosters == (int)boosters.size() );
-                fo.Write( &mparam, sizeof(ModelParam) );
-                for( size_t i = 0; i < boosters.size(); i ++ ){
-                    boosters[ i ]->SaveModel( fo ); 
-                }
-                if( booster_info.size() != 0 ){
-                    fo.Write( &booster_info[0], sizeof(int) * booster_info.size() );
-                }
-                if( mparam.num_pbuffer != 0 ){
-                    fo.Write( &pred_buffer[0] , pred_buffer.size()*sizeof(float) );
-                    fo.Write( &pred_counter[0], pred_counter.size()*sizeof(unsigned) );
-                }
-            }
-            /*!
-             * \brief initialize the current data storage for model, if the model is used first time, call this function
-             */
-            inline void InitModel( void ){
-                pred_buffer.clear(); pred_counter.clear();
-                pred_buffer.resize ( mparam.num_pbuffer, 0.0 );
-                pred_counter.resize( mparam.num_pbuffer, 0 );
-                utils::Assert( mparam.num_boosters == 0 );
-                utils::Assert( boosters.size() == 0 );
-            }
-            /*!
-             * \brief initialize solver before training, called before training
-             * this function is reserved for solver to allocate necessary space and do other preparation 
-             */            
-            inline void InitTrainer( void ){
-                if( tparam.nthread != 0 ){
-                    omp_set_num_threads( tparam.nthread );
-                }
-                // make sure all the boosters get the latest parameters
-                for( size_t i = 0; i < this->boosters.size(); i ++ ){
-                    this->ConfigBooster( this->boosters[i] );
-                }
-            }
-            /*! 
-             * \brief DumpModel
-             * \param fo text file 
-             * \param fmap feature map that may help give interpretations of feature
-             * \param with_stats whether print statistics
-             */            
-            inline void DumpModel( FILE *fo, const utils::FeatMap& fmap, bool with_stats ){
-                for( size_t i = 0; i < boosters.size(); i ++ ){
-                    fprintf( fo, "booster[%d]\n", (int)i );
-                    boosters[i]->DumpModel( fo, fmap, with_stats );
-                }
-            }
-            /*! 
-             * \brief Dump path of all trees
-             * \param fo text file 
-             * \param data input data
-             */
-            inline void DumpPath( FILE *fo, const FMatrixS &data ){
-                for( size_t i = 0; i < data.NumRow(); ++ i ){
-                    for( size_t j = 0; j < boosters.size(); ++ j ){
-                        if( j != 0 ) fprintf( fo, "\t" );
-                        std::vector<int> path;
-                        boosters[j]->PredPath( path, data, i );
-                        fprintf( fo, "%d", path[0] );
-                        for( size_t k = 1; k < path.size(); ++ k ){
-                            fprintf( fo, ",%d", path[k] );
-                        }
-                    }
-                    fprintf( fo, "\n" );
-                }
-            }
-        public:
-            /*! 
-             * \brief do gradient boost training for one step, using the information given
-             *        Note: content of grad and hess can change after DoBoost
-             * \param grad first order gradient of each instance
-             * \param hess second order gradient of each instance
-             * \param feats features of each instance
-             * \param root_index pre-partitioned root index of each instance, 
-             *          root_index.size() can be 0 which indicates that no pre-partition involved
-             */
-            inline void DoBoost( std::vector<float> &grad,
-                                 std::vector<float> &hess,
-                                 const booster::FMatrixS &feats,
-                                 const std::vector<unsigned> &root_index ) {
-                booster::IBooster *bst = this->GetUpdateBooster();
-                bst->DoBoost( grad, hess, feats, root_index );
-            }
-            /*! 
-             * \brief predict values for given sparse feature vector
-             *   NOTE: in tree implementation, this is only OpenMP threadsafe, but not threadsafe
-             * \param feats feature matrix
-             * \param row_index  row index in the feature matrix
-             * \param buffer_index the buffer index of the current feature line, default -1 means no buffer assigned
-             * \param root_index root id of current instance, default = 0
-             * \return prediction 
-             */
-            inline float Predict( const FMatrixS &feats, bst_uint row_index, int buffer_index = -1, unsigned root_index = 0 ){
-                size_t istart = 0;
-                float  psum = 0.0f;
-
-                // load buffered results if any
-                if( mparam.do_reboost == 0 && buffer_index >= 0 ){
-                    utils::Assert( buffer_index < mparam.num_pbuffer, "buffer index exceed num_pbuffer" );
-                    istart = this->pred_counter[ buffer_index ];
-                    psum   = this->pred_buffer [ buffer_index ];
-                }
-            
-                for( size_t i = istart; i < this->boosters.size(); i ++ ){
-                    psum += this->boosters[ i ]->Predict( feats, row_index, root_index );
-                }                
-                // updated the buffered results
-                if( mparam.do_reboost == 0 && buffer_index >= 0 ){
-                    this->pred_counter[ buffer_index ] = static_cast<unsigned>( boosters.size() );
-                    this->pred_buffer [ buffer_index ] = psum;
-                }
-                return psum;
-            }
-        public:
-            //--------trial code for interactive update an existing booster------
-            //-------- usually not needed, ignore this region ---------
-            /*! 
-             * \brief same as Predict, but removes the prediction of booster to be updated 
-             *        this function must be called once and only once for every data with pbuffer
-             */
-            inline float InteractPredict( const FMatrixS &feats, bst_uint row_index, int buffer_index = -1, unsigned root_index = 0 ){
-                float psum = this->Predict( feats, row_index, buffer_index, root_index );
-                if( tparam.reupdate_booster != -1 ){
-                    const int bid = tparam.reupdate_booster;
-                    utils::Assert( bid >= 0 && bid < (int)boosters.size(), "interact:booster_index exceed existing bound" );
-                    psum -= boosters[ bid ]->Predict( feats, row_index, root_index );
-                    if( mparam.do_reboost == 0 && buffer_index >= 0 ){
-                        this->pred_buffer[ buffer_index ] = psum;
-                    }
-                }
-                return psum;
-            }
-            /*! \brief delete the specified booster */
-            inline void DelteBooster( void ){
-                const int bid = tparam.reupdate_booster;
-                utils::Assert( bid >= 0 && bid < mparam.num_boosters , "must specify booster index for deletion"); 
-                delete boosters[ bid ];
-                for( int i = bid + 1; i < mparam.num_boosters; ++ i ){
-                    boosters[i-1] = boosters[ i ];
-                    booster_info[i-1] = booster_info[ i ];
-                } 
-                boosters.resize( mparam.num_boosters -= 1 );
-                booster_info.resize( boosters.size() );
-            }
-            /*! \brief update the prediction buffer, after booster have been updated */            
-            inline void InteractRePredict( const FMatrixS &feats, bst_uint row_index, int buffer_index = -1, unsigned root_index = 0 ){
-                if( tparam.reupdate_booster != -1 ){
-                    const int bid = tparam.reupdate_booster;
-                    utils::Assert( bid >= 0 && bid < (int)boosters.size(), "interact:booster_index exceed existing bound" );
-                    if( mparam.do_reboost == 0 && buffer_index >= 0 ){
-                        this->pred_buffer[ buffer_index ] += boosters[ bid ]->Predict( feats, row_index, root_index );
-                    }
-                }
-            }
-            //-----------non public fields afterwards-------------
-        protected:
-            /*! \brief free space of the model */
-            inline void FreeSpace( void ){
-                for( size_t i = 0; i < boosters.size(); i ++ ){
-                    delete boosters[i];
-                }
-                boosters.clear(); booster_info.clear(); mparam.num_boosters = 0; 
-            }
-            /*! \brief configure a booster */
-            inline void ConfigBooster( booster::IBooster *bst ){
-                cfg.BeforeFirst();
-                while( cfg.Next() ){
-                    bst->SetParam( cfg.name(), cfg.val() );
-                }
-            }
-            /*! 
-             * \brief get a booster to update 
-             * \return the booster created
-             */
-            inline booster::IBooster *GetUpdateBooster( void ){
-                if( tparam.reupdate_booster != -1 ){
-                    const int bid = tparam.reupdate_booster;
-                    utils::Assert( bid >= 0 && bid < (int)boosters.size(), "interact:booster_index exceed existing bound" );
-                    this->ConfigBooster( boosters[bid] );
-                    return boosters[ bid ];
-                }
-
-                if( mparam.do_reboost == 0 || boosters.size() == 0 ){
-                    mparam.num_boosters += 1;
-                    boosters.push_back( booster::CreateBooster<FMatrixS>( mparam.booster_type ) );
-                    booster_info.push_back( 0 );
-                    this->ConfigBooster( boosters.back() );
-                    boosters.back()->InitModel();                    
-                }else{
-                    this->ConfigBooster( boosters.back() );
-                }
-                return boosters.back();
-            }
-        protected:
-            /*! \brief model parameters */
-            struct ModelParam{
-                /*! \brief number of boosters */
-                int num_boosters;
-                /*! \brief type of tree used */
-                int booster_type;
-                /*! \brief number of root: default 0, means single tree */
-                int num_roots;
-                /*! \brief number of features to be used by boosters */
-                int num_feature;
-                /*! \brief size of predicton buffer allocated for buffering boosting computation */
-                int num_pbuffer;
-                /*! 
-                 * \brief whether we repeatly update a single booster each round: default 0
-                 *        set to 1 for linear booster, so that regularization term can be considered
-                 */
-                int do_reboost;
-                /*! \brief reserved parameters */
-                int reserved[ 32 ];
-                /*! \brief constructor */
-                ModelParam( void ){
-                    num_boosters = 0; 
-                    booster_type = 0;
-                    num_roots = num_feature = 0;                    
-                    do_reboost = 0;
-                    num_pbuffer = 0;
-                    memset( reserved, 0, sizeof( reserved ) );
-                }
-                /*! 
-                 * \brief set parameters from outside 
-                 * \param name name of the parameter
-                 * \param val  value of the parameter
-                 */
-                inline void SetParam( const char *name, const char *val ){
-                    if( !strcmp("booster_type", name ) ){
-                        booster_type = atoi( val );
-                        // linear boost automatically set do reboost
-                        if( booster_type == 1 ) do_reboost = 1;
-                    }
-                    if( !strcmp("num_pbuffer", name ) )      num_pbuffer = atoi( val );
-                    if( !strcmp("do_reboost", name ) )       do_reboost  = atoi( val );
-                    if( !strcmp("bst:num_roots", name ) )    num_roots = atoi( val );
-                    if( !strcmp("bst:num_feature", name ) )  num_feature = atoi( val );
-                }
-            };
-            /*! \brief training parameters */
-            struct TrainParam{
-                /*! \brief number of OpenMP threads */
-                int nthread;
-                /*! 
-                 * \brief index of specific booster to be re-updated, default = -1: update new booster 
-                 *  parameter this is part of trial interactive update mode
-                 */
-                int reupdate_booster;
-                /*! \brief constructor */
-                TrainParam( void ) {
-                    nthread = 1;
-                    reupdate_booster = -1;
-                }
-                /*! 
-                 * \brief set parameters from outside 
-                 * \param name name of the parameter
-                 * \param val  value of the parameter
-                 */                
-                inline void SetParam( const char *name, const char *val ){
-                    if( !strcmp("nthread", name ) )                 nthread = atoi( val );
-                    if( !strcmp("interact:booster_index", name ) )  reupdate_booster = atoi( val );
-                }
-            };
-        protected:
-            /*! \brief model parameters */ 
-            ModelParam mparam;
-            /*! \brief training parameters */ 
-            TrainParam tparam;
-        protected:
-            /*! \brief component boosters */ 
-            std::vector<booster::IBooster*> boosters;
-            /*! \brief some information indicator of the booster, reserved */ 
-            std::vector<int> booster_info;
-            /*! \brief prediction buffer */ 
-            std::vector<float>    pred_buffer;
-            /*! \brief prediction buffer counter, record the progress so fart of the buffer */ 
-            std::vector<unsigned> pred_counter;
-            /*! \brief configurations saved for each booster */
-            utils::ConfigSaver cfg;
-        };
-    };
-};
-#endif

build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# This is a simple script to make xgboost in MAC and Linux
+# Basically, it first try to make with OpenMP, if fails, disable OpenMP and make it again.
+# This will automatically make xgboost for MAC users who don't have OpenMP support.
+# In most cases, type make will give what you want.
+
+# See additional instruction in doc/build.md
+
+
+if make; then
+    echo "Successfully build multi-thread xgboost"
+else
+    echo "-----------------------------"
+    echo "Building multi-thread xgboost failed"
+    echo "Start to build single-thread xgboost"
+    make clean
+    make no_omp=1
+    echo "Successfully build single-thread xgboost"
+    echo "If you want multi-threaded version"
+    echo "See additional instructions in doc/build.md"
+fi

demo/.gitignore

@@ -0,0 +1 @@
+*.libsvm

demo/README.md

@@ -0,0 +1,52 @@
+XGBoost Examples
+====
+This folder contains all the code examples using xgboost. 
+
+* Contribution of examples, benchmarks is more than welcome!
+* If you like to share how you use xgboost to solve your problem, send a pull request:)
+ 
+Features Walkthrough
+====
+This is a list of short codes introducing different functionalities of xgboost and its wrapper.
+* Basic walkthrough of wrappers 
+  [python](guide-python/basic_walkthrough.py)
+  [R](../R-package/demo/basic_walkthrough.R)
+  [Julia](https://github.com/antinucleon/XGBoost.jl/blob/master/demo/basic_walkthrough.jl)
+* Customize loss function, and evaluation metric
+  [python](guide-python/custom_objective.py)
+  [R](../R-package/demo/custom_objective.R)
+  [Julia](https://github.com/antinucleon/XGBoost.jl/blob/master/demo/custom_objective.jl)
+* Boosting from existing prediction
+  [python](guide-python/boost_from_prediction.py)
+  [R](../R-package/demo/boost_from_prediction.R)
+  [Julia](https://github.com/antinucleon/XGBoost.jl/blob/master/demo/boost_from_prediction.jl)
+* Predicting using first n trees 
+  [python](guide-python/predict_first_ntree.py)
+  [R](../R-package/demo/boost_from_prediction.R)
+  [Julia](https://github.com/antinucleon/XGBoost.jl/blob/master/demo/boost_from_prediction.jl)  
+* Generalized Linear Model
+  [python](guide-python/generalized_linear_model.py)
+  [R](../R-package/demo/generalized_linear_model.R)
+  [Julia](https://github.com/antinucleon/XGBoost.jl/blob/master/demo/generalized_linear_model.jl)  
+* Cross validation
+  [python](guide-python/cross_validation.py)
+  [R](../R-package/demo/cross_validation.R)
+  [Julia](https://github.com/antinucleon/XGBoost.jl/blob/master/demo/cross_validation.jl)  
+* Predicting leaf indices
+  [python](guide-python/predict_leaf_indices.py)
+  [R](../R-package/demo/predict_leaf_indices.R)
+
+Basic Examples by Tasks
+====
+Most of examples in this section are based on CLI or python version.
+However, the parameter settings can be applied to all versions
+* [Binary classification](binary_classification)
+* [Multiclass classification](multiclass_classification)
+* [Regression](regression)
+* [Learning to Rank](rank)
+
+Benchmarks
+====
+* [Starter script for Kaggle Higgs Boson](kaggle-higgs)
+* [Kaggle Tradeshift winning solution by daxiongshu](https://github.com/daxiongshu/kaggle-tradeshift-winning-solution) 
+

demo/binary_classification/README

@@ -1,14 +0,0 @@
-Demonstrating how to use XGBoost accomplish binary classification tasks  on UCI mushroom dataset  http://archive.ics.uci.edu/ml/datasets/Mushroom
-
-Run: ./runexp.sh
-
-Format of input: LIBSVM format
-
-Format of ```featmap.txt: <featureid> <featurename> <q or i or int>\n ```:
-  - Feature id must be from 0 to number of features, in sorted order.
-  - i means this feature is binary indicator feature
-  - q means this feature is a quantitative value, such as age, time, can be missing
-  - int means this feature is integer value (when int is hinted, the decision boundary will be integer)
-
-
-Explainations: https://github.com/tqchen/xgboost/wiki/Binary-Classification

demo/binary_classification/README.md

@@ -0,0 +1,172 @@
+Binary Classification
+====
+This is the quick start tutorial for xgboost CLI version. You can also checkout [../../doc/README.md](../../doc/README.md) for links to tutorial in python or R.
+Here we demonstrate how to use XGBoost for a binary classification task. Before getting started, make sure you compile xgboost in the root directory of the project by typing ```make```
+The script runexp.sh can be used to run the demo. Here we use [mushroom dataset](https://archive.ics.uci.edu/ml/datasets/Mushroom) from UCI machine learning repository. 
+
+### Tutorial
+#### Generate Input Data
+XGBoost takes LibSVM format. An example of faked input data is below:
+```
+1 101:1.2 102:0.03
+0 1:2.1 10001:300 10002:400 
+...
+```
+Each line represent a single instance, and in the first line '1' is the instance label,'101' and '102' are feature indices, '1.2' and '0.03' are feature values. In the binary classification case, '1' is used to indicate positive samples, and '0' is used to indicate negative samples. We also support probability values in [0,1] as label, to indicate the probability of the instance being positive.
+
+
+First we will transform the dataset into classic LibSVM format and split the data into training set and test set by running:
+```
+python mapfeat.py
+python mknfold.py agaricus.txt 1
+```
+The two files, 'agaricus.txt.train' and 'agaricus.txt.test' will be used as training set and test set.
+
+#### Training 
+Then we can run the training process:
+```
+../../xgboost mushroom.conf
+```
+
+mushroom.conf is the configuration for both training and testing. Each line containing the [attribute]=[value] configuration:
+
+```conf
+# General Parameters, see comment for each definition
+# can be gbtree or gblinear
+booster = gbtree 
+# choose logistic regression loss function for binary classification
+objective = binary:logistic
+
+# Tree Booster Parameters
+# step size shrinkage
+eta = 1.0 
+# minimum loss reduction required to make a further partition
+gamma = 1.0 
+# minimum sum of instance weight(hessian) needed in a child
+min_child_weight = 1 
+# maximum depth of a tree
+max_depth = 3 
+
+# Task Parameters
+# the number of round to do boosting
+num_round = 2
+# 0 means do not save any model except the final round model
+save_period = 0 
+# The path of training data
+data = "agaricus.txt.train" 
+# The path of validation data, used to monitor training process, here [test] sets name of the validation set
+eval[test] = "agaricus.txt.test" 
+# The path of test data 
+test:data = "agaricus.txt.test"      
+```
+We use the tree booster and logistic regression objective in our setting. This indicates that we accomplish our task using classic gradient boosting regression tree(GBRT), which is a promising method for binary classification.
+
+The parameters shown in the example gives the most common ones that are needed to use xgboost.
+If you are interested in more parameter settings, the complete parameter settings and detailed descriptions are [here](../../doc/parameter.md). Besides putting the parameters in the configuration file, we can set them by passing them as arguments as below:
+
+```
+../../xgboost mushroom.conf max_depth=6
+```
+This means that the parameter max_depth will be set as 6 rather than 3 in the conf file. When you use command line, make sure max_depth=6 is passed in as single argument, i.e. do not contain space in the argument. When a parameter setting is provided in both command line input and  the config file, the command line setting will override the setting in config file.
+
+In this example, we use tree booster for gradient boosting. If you would like to use linear booster for regression, you can keep all the parameters except booster and the tree booster parameters as below:
+```conf 
+# General Parameters
+# choose the linear booster
+booster = gblinear
+...
+
+# Change Tree Booster Parameters into Linear Booster Parameters
+# L2 regularization term on weights, default 0
+lambda = 0.01
+# L1 regularization term on weights, default 0
+f ```agaricus.txt.test.buffer``` exists, and automatically loads from binary buffer if possible, this can speedup training process when you do training many times. You can disable it by setting ```use_buffer=0```.
+  - Buffer file can also be used as standalone input, i.e if buffer file exists, but original agaricus.txt.test was removed, xgboost will still run
+* Deviation from LibSVM input format: xgboost is compatible with LibSVM format, with the following minor differences:
+  - xgboost allows feature index starts from 0
+  - for binary classification, the label is 1 for positive, 0 for negative, instead of +1,-1
+  - the feature indices in each line *do not* need to be sorted
+alpha = 0.01 
+# L2 regularization term on bias, default 0
+lambda_bias = 0.01 
+
+# Regression Parameters
+...
+```
+
+#### Get Predictions 
+After training, we can use the output model to get the prediction of the test data:
+```
+../../xgboost mushroom.conf task=pred model_in=0003.model
+```
+For binary classification, the output predictions are probability confidence scores in [0,1], corresponds to the probability of the label to be positive.
+
+#### Dump Model
+This is a preliminary feature, so far only tree model support text dump. XGBoost can display the tree models in text files and we can scan the model in an easy way:
+```
+../../xgboost mushroom.conf task=dump model_in=0003.model name_dump=dump.raw.txt 
+../../xgboost mushroom.conf task=dump model_in=0003.model fmap=featmap.txt name_dump=dump.nice.txt
+```
+
+In this demo, the tree boosters obtained will be printed in dump.raw.txt and dump.nice.txt, and the latter one is easier to understand because of usage of feature mapping featmap.txt
+
+Format of ```featmap.txt: <featureid> <featurename> <q or i or int>\n ```:
+  - Feature id must be from 0 to number of features, in sorted order.
+  - i means this feature is binary indicator feature
+  - q means this feature is a quantitative value, such as age, time, can be missing
+  - int means this feature is integer value (when int is hinted, the decision boundary will be integer)
+
+#### Monitoring Progress
+When you run training we can find there are messages displayed on screen
+```
+tree train end, 1 roots, 12 extra nodes, 0 pruned nodes ,max_depth=3
+[0]  test-error:0.016139
+boosting round 1, 0 sec elapsed
+
+tree train end, 1 roots, 10 extra nodes, 0 pruned nodes ,max_depth=3
+[1]  test-error:0.000000
+```
+The messages for evaluation are printed into stderr, so if you want only to log the evaluation progress, simply type
+```
+../../xgboost mushroom.conf 2>log.txt
+```
+Then you can find the following content in log.txt
+```
+[0]     test-error:0.016139
+[1]     test-error:0.000000
+```
+We can also monitor both training and test statistics, by adding following lines to configure
+```conf
+eval[test] = "agaricus.txt.test" 
+eval[trainname] = "agaricus.txt.train" 
+```
+Run the command again, we can find the log file becomes
+```
+[0]     test-error:0.016139     trainname-error:0.014433
+[1]     test-error:0.000000     trainname-error:0.001228
+```
+The rule is eval[name-printed-in-log] = filename, then the file will be added to monitoring process, and evaluated each round.
+
+xgboost also support monitoring multiple metrics, suppose we also want to monitor average log-likelihood of each prediction during training, simply add ```eval_metric=logloss``` to configure. Run again, we can find the log file becomes
+```
+[0]     test-error:0.016139     test-negllik:0.029795   trainname-error:0.014433        trainname-negllik:0.027023
+[1]     test-error:0.000000     test-negllik:0.000000   trainname-error:0.001228        trainname-negllik:0.002457
+```
+### Saving Progress Models
+If you want to save model every two round, simply set save_period=2. You will find 0002.model in the current folder. If you want to change the output folder of models, add model_dir=foldername. By default xgboost saves the model of last round.
+
+#### Continue from Existing Model
+If you want to continue boosting from existing model, say 0002.model, use
+```
+../../xgboost mushroom.conf model_in=0002.model num_round=2 model_out=continue.model
+```
+xgboost will load from 0002.model continue boosting for 2 rounds, and save output to continue.model. However, beware that the training and evaluation data specified in mushroom.conf should not change when you use this function.
+#### Use Multi-Threading
+When you are working with a large dataset, you may want to take advantage of parallelism. If your compiler supports OpenMP, xgboost is naturally multi-threaded, to set number of parallel running threads to 10, add ```nthread=10``` to your configuration.
+
+#### Additional Notes
+* What are ```agaricus.txt.test.buffer``` and ```agaricus.txt.train.buffer``` generated during runexp.sh? 
+  - By default xgboost will automatically generate a binary format buffer of input data, with suffix ```buffer```. When next time you run xgboost, it detects i
+Demonstrating how to use XGBoost accomplish binary classification tasks  on UCI mushroom dataset  http://archive.ics.uci.edu/ml/datasets/Mushroom
+
+

demo/binary_classification/mapfeat.py

@@ -1,17 +1,16 @@
 #!/usr/bin/python
-import sys
 
 def loadfmap( fname ):
     fmap = {}
     nmap = {}
-    
+
     for l in open( fname ):
         arr = l.split()
-        if arr[0].find('.') != -1:            
+        if arr[0].find('.') != -1:
             idx = int( arr[0].strip('.') )
-            assert idx not in fmap        
+            assert idx not in fmap
             fmap[ idx ] = {}
-            ftype = arr[1].strip(':')        
+            ftype = arr[1].strip(':')
             content = arr[2]
         else:
             content = arr[0]
@@ -23,8 +22,8 @@ def loadfmap( fname ):
             nmap[ len(nmap) ] = ftype+'='+k
     return fmap, nmap
 
-def write_nmap( fo, nmap ):    
-    for i in xrange( len(nmap) ):
+def write_nmap( fo, nmap ):
+    for i in range( len(nmap) ):
         fo.write('%d\t%s\ti\n' % (i, nmap[i]) )
 
 # start here
@@ -33,7 +32,7 @@ fo = open( 'featmap.txt', 'w' )
 write_nmap( fo, nmap )
 fo.close()
 
-fo = open( 'agaricus.txt', 'w' ) 
+fo = open( 'agaricus.txt', 'w' )
 for l in open( 'agaricus-lepiota.data' ):
     arr = l.split(',')
     if arr[0] == 'p':
@@ -41,10 +40,10 @@ for l in open( 'agaricus-lepiota.data' ):
     else:
         assert arr[0] == 'e'
         fo.write('0')
-    for i in xrange( 1,len(arr) ):
+    for i in range( 1,len(arr) ):
         fo.write( ' %d:1' % fmap[i][arr[i].strip()] )
     fo.write('\n')
 
 fo.close()
 
- 
+

demo/binary_classification/mknfold.py

@@ -3,7 +3,7 @@ import sys
 import random
 
 if len(sys.argv) < 2:
-    print 'Usage:<filename> <k> [nfold = 5]'
+    print ('Usage:<filename> <k> [nfold = 5]')
     exit(0)
 
 random.seed( 10 )

demo/binary_classification/mushroom.conf

@@ -1,18 +1,18 @@
 # General Parameters, see comment for each definition
-# choose the tree booster, 0: tree, 1: linear
-booster_type = 0 
+# choose the booster, can be gbtree or gblinear
+booster = gbtree
 # choose logistic regression loss function for binary classification
-loss_type = 2 
+objective = binary:logistic
 
 # Tree Booster Parameters
 # step size shrinkage
-bst:eta = 1.0 
+eta = 1.0 
 # minimum loss reduction required to make a further partition
-bst:gamma = 1.0 
+gamma = 1.0 
 # minimum sum of instance weight(hessian) needed in a child
-bst:min_child_weight = 1 
+min_child_weight = 1 
 # maximum depth of a tree
-bst:max_depth = 3 
+max_depth = 3 
 
 # Task Parameters
 # the number of round to do boosting
@@ -23,5 +23,7 @@ save_period = 0
 data = "agaricus.txt.train" 
 # The path of validation data, used to monitor training process, here [test] sets name of the validation set
 eval[test] = "agaricus.txt.test" 
+# evaluate on training data as well each round
+eval_train = 1
 # The path of test data 
 test:data = "agaricus.txt.test"      

demo/data/README.md

@@ -0,0 +1,2 @@
+This folder contains processed example dataset used by the demos.
+Copyright of the dataset belongs to the original copyright holder

demo/data/featmap.txt

@@ -0,0 +1,126 @@
+0	cap-shape=bell	i
+1	cap-shape=conical	i
+2	cap-shape=convex	i
+3	cap-shape=flat	i
+4	cap-shape=knobbed	i
+5	cap-shape=sunken	i
+6	cap-surface=fibrous	i
+7	cap-surface=grooves	i
+8	cap-surface=scaly	i
+9	cap-surface=smooth	i
+10	cap-color=brown	i
+11	cap-color=buff	i
+12	cap-color=cinnamon	i
+13	cap-color=gray	i
+14	cap-color=green	i
+15	cap-color=pink	i
+16	cap-color=purple	i
+17	cap-color=red	i
+18	cap-color=white	i
+19	cap-color=yellow	i
+20	bruises?=bruises	i
+21	bruises?=no	i
+22	odor=almond	i
+23	odor=anise	i
+24	odor=creosote	i
+25	odor=fishy	i
+26	odor=foul	i
+27	odor=musty	i
+28	odor=none	i
+29	odor=pungent	i
+30	odor=spicy	i
+31	gill-attachment=attached	i
+32	gill-attachment=descending	i
+33	gill-attachment=free	i
+34	gill-attachment=notched	i
+35	gill-spacing=close	i
+36	gill-spacing=crowded	i
+37	gill-spacing=distant	i
+38	gill-size=broad	i
+39	gill-size=narrow	i
+40	gill-color=black	i
+41	gill-color=brown	i
+42	gill-color=buff	i
+43	gill-color=chocolate	i
+44	gill-color=gray	i
+45	gill-color=green	i
+46	gill-color=orange	i
+47	gill-color=pink	i
+48	gill-color=purple	i
+49	gill-color=red	i
+50	gill-color=white	i
+51	gill-color=yellow	i
+52	stalk-shape=enlarging	i
+53	stalk-shape=tapering	i
+54	stalk-root=bulbous	i
+55	stalk-root=club	i
+56	stalk-root=cup	i
+57	stalk-root=equal	i
+58	stalk-root=rhizomorphs	i
+59	stalk-root=rooted	i
+60	stalk-root=missing	i
+61	stalk-surface-above-ring=fibrous	i
+62	stalk-surface-above-ring=scaly	i
+63	stalk-surface-above-ring=silky	i
+64	stalk-surface-above-ring=smooth	i
+65	stalk-surface-below-ring=fibrous	i
+66	stalk-surface-below-ring=scaly	i
+67	stalk-surface-below-ring=silky	i
+68	stalk-surface-below-ring=smooth	i
+69	stalk-color-above-ring=brown	i
+70	stalk-color-above-ring=buff	i
+71	stalk-color-above-ring=cinnamon	i
+72	stalk-color-above-ring=gray	i
+73	stalk-color-above-ring=orange	i
+74	stalk-color-above-ring=pink	i
+75	stalk-color-above-ring=red	i
+76	stalk-color-above-ring=white	i
+77	stalk-color-above-ring=yellow	i
+78	stalk-color-below-ring=brown	i
+79	stalk-color-below-ring=buff	i
+80	stalk-color-below-ring=cinnamon	i
+81	stalk-color-below-ring=gray	i
+82	stalk-color-below-ring=orange	i
+83	stalk-color-below-ring=pink	i
+84	stalk-color-below-ring=red	i
+85	stalk-color-below-ring=white	i
+86	stalk-color-below-ring=yellow	i
+87	veil-type=partial	i
+88	veil-type=universal	i
+89	veil-color=brown	i
+90	veil-color=orange	i
+91	veil-color=white	i
+92	veil-color=yellow	i
+93	ring-number=none	i
+94	ring-number=one	i
+95	ring-number=two	i
+96	ring-type=cobwebby	i
+97	ring-type=evanescent	i
+98	ring-type=flaring	i
+99	ring-type=large	i
+100	ring-type=none	i
+101	ring-type=pendant	i
+102	ring-type=sheathing	i
+103	ring-type=zone	i
+104	spore-print-color=black	i
+105	spore-print-color=brown	i
+106	spore-print-color=buff	i
+107	spore-print-color=chocolate	i
+108	spore-print-color=green	i
+109	spore-print-color=orange	i
+110	spore-print-color=purple	i
+111	spore-print-color=white	i
+112	spore-print-color=yellow	i
+113	population=abundant	i
+114	population=clustered	i
+115	population=numerous	i
+116	population=scattered	i
+117	population=several	i
+118	population=solitary	i
+119	habitat=grasses	i
+120	habitat=leaves	i
+121	habitat=meadows	i
+122	habitat=paths	i
+123	habitat=urban	i
+124	habitat=waste	i
+125	habitat=woods	i

demo/guide-python/README.md

@@ -0,0 +1,11 @@
+XGBoost Python Feature Walkthrough
+====
+* [Basic walkthrough of wrappers](basic_walkthrough.py) 
+* [Cutomize loss function, and evaluation metric](custom_objective.py)
+* [Boosting from existing prediction](boost_from_prediction.py)
+* [Predicting using first n trees](predict_first_ntree.py)
+* [Generalized Linear Model](generalized_linear_model.py)
+* [Cross validation](cross_validation.py)
+* [Predicting leaf indices](predict_leaf_indices.py)
+* [Sklearn Wrapper](sklearn_example.py)
+* [External Memory](external_memory.py)