fix sometimes python cachelist problem

Compatibility with both Python 2(.7) and 3

.gitignore

@@ -6,19 +6,17 @@
 # Compiled Dynamic libraries
 *.so
 *.dylib
-*.page
+
 # Compiled Static libraries
 *.lai
 *.la
 *.a
 *~
-*.Rcheck
-*.rds
-*.tar.gz
 *txt*
 *conf
 *buffer
 *model
+xgboost
 *pyc
 *train
 *test
@@ -26,21 +24,3 @@
 *rar
 *vali
 *data
-*sdf
-Release
-*exe*
-*exp
-ipch
-*.filters
-*.user
-*log
-Debug
-*suo
-*test*
-.Rhistory
-*.dll
-*i386
-*x64
-*dump
-*save
-*csv

CHANGES.md

@@ -1,22 +0,0 @@
-Change Log
-=====
-
-xgboost-0.1
-=====
-* Initial release
-
-xgboost-0.2x
-=====
-* Python module
-* Weighted samples instances
-* Initial version of pairwise rank
-
-xgboost-0.3
-=====
-* Faster tree construction module
-  - Allows subsample columns during tree construction via ```bst:col_samplebytree=ratio```
-* Support for boosting from initial predictions
-* Experimental version of LambdaRank
-* Linear booster is now parallelized, using parallel coordinated descent.
-* Add [Code Guide](src/README.md) for customizing objective function and evaluation
-* Add R module

Makefile

@@ -1,65 +1,26 @@
 export CC  = gcc
 export CXX = g++
-export LDFLAGS= -pthread -lm 
+export CFLAGS = -Wall -O3 -msse2  -Wno-unknown-pragmas -fopenmp 
-
-export CFLAGS = -Wall -O3 -msse2  -Wno-unknown-pragmas -fPIC -pedantic 
-
-ifeq ($(no_omp),1)
-	CFLAGS += -DDISABLE_OPENMP 
-else 
-	CFLAGS += -fopenmp
-endif
 
 # specify tensor path
 BIN = xgboost
-OBJ = updater.o gbm.o io.o
+OBJ = 
-SLIB = wrapper/libxgboostwrapper.so 
+.PHONY: clean all
 
-.PHONY: clean all python Rpack
+all: $(BIN) $(OBJ)
+export LDFLAGS= -pthread -lm 
 
-all: $(BIN) $(OBJ) $(SLIB) 
+xgboost: regrank/xgboost_regrank_main.cpp regrank/*.h regrank/*.hpp booster/*.h booster/*/*.hpp booster/*.hpp
 
-python: wrapper/libxgboostwrapper.so
-# now the wrapper takes in two files. io and wrapper part
-wrapper/libxgboostwrapper.so: wrapper/xgboost_wrapper.cpp $(OBJ)
-updater.o: src/tree/updater.cpp  src/tree/*.hpp src/*.h src/tree/*.h
-gbm.o: src/gbm/gbm.cpp src/gbm/*.hpp src/gbm/*.h
-io.o: src/io/io.cpp src/io/*.hpp src/utils/*.h src/learner/dmatrix.h src/*.h
-xgboost: src/xgboost_main.cpp src/utils/*.h src/*.h src/learner/*.hpp src/learner/*.h $(OBJ)
-wrapper/libxgboostwrapper.so: wrapper/xgboost_wrapper.cpp src/utils/*.h src/*.h src/learner/*.hpp src/learner/*.h $(OBJ)
 
 $(BIN) : 
 	$(CXX) $(CFLAGS) $(LDFLAGS) -o $@ $(filter %.cpp %.o %.c, $^)
 
-$(SLIB) :
-	$(CXX) $(CFLAGS) -fPIC $(LDFLAGS) -shared -o $@ $(filter %.cpp %.o %.c, $^)
-
 $(OBJ) : 
 	$(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c, $^) )
 
 install:
 	cp -f -r $(BIN)  $(INSTALL_PATH)
 
-Rpack:
-	make clean
-	rm -rf xgboost xgboost*.tar.gz
-	cp -r R-package xgboost
-	rm -rf xgboost/inst/examples/*.buffer
-	rm -rf xgboost/inst/examples/*.model
-	rm -rf xgboost/inst/examples/dump*
-	rm -rf xgboost/src/*.o xgboost/src/*.so xgboost/src/*.dll
-	rm -rf xgboost/demo/*.model xgboost/demo/*.buffer xgboost/demo/*.txt
-	rm -rf xgboost/demo/runall.R
-	cp -r src xgboost/src/src
-	mkdir xgboost/src/wrapper
-	cp  wrapper/xgboost_wrapper.h xgboost/src/wrapper
-	cp  wrapper/xgboost_wrapper.cpp xgboost/src/wrapper
-	cp ./LICENSE xgboost
-	cat R-package/src/Makevars|sed '2s/.*/PKGROOT=./' > xgboost/src/Makevars
-	cat R-package/src/Makevars.win|sed '2s/.*/PKGROOT=./' > xgboost/src/Makevars.win
-	R CMD build xgboost
-	rm -rf xgboost
-	R CMD check --as-cran xgboost*.tar.gz
-
 clean:
-	$(RM) $(OBJ) $(BIN) $(SLIB) *.o  */*.o */*/*.o *~ */*~ */*/*~
+	$(RM) $(OBJ) $(BIN) *~

R-package/DESCRIPTION

@@ -1,24 +0,0 @@
-Package: xgboost
-Type: Package
-Title: eXtreme Gradient Boosting
-Version: 0.3-2
-Date: 2014-08-23
-Author: Tianqi Chen <tianqi.tchen@gmail.com>, Tong He <hetong007@gmail.com>
-Maintainer: Tong He <hetong007@gmail.com>
-Description: This package is a R wrapper of xgboost, which is short for eXtreme
-    Gradient Boosting. It is an efficient and scalable implementation of
-    gradient boosting framework. The package includes efficient linear model
-    solver and tree learning algorithms. The package can automatically do
-    parallel computation with OpenMP, and it can be more than 10 times faster
-    than existing gradient boosting packages such as gbm. It supports various
-    objective functions, including regression, classification and ranking. The
-    package is made to be extensible, so that users are also allowed to define
-    their own objectives easily.
-License: Apache License (== 2.0) | file LICENSE
-URL: https://github.com/tqchen/xgboost
-BugReports: https://github.com/tqchen/xgboost/issues
-Depends:
-    R (>= 2.10)
-Imports:
-    Matrix (>= 1.1-0),
-    methods

R-package/LICENSE

@@ -1,13 +0,0 @@
-Copyright (c) 2014 by Tianqi Chen and Contributors 
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-    
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.

R-package/NAMESPACE

@@ -1,17 +0,0 @@
-# Generated by roxygen2 (4.0.1): do not edit by hand
-
-export(getinfo)
-export(setinfo)
-export(slice)
-export(xgb.DMatrix)
-export(xgb.DMatrix.save)
-export(xgb.cv)
-export(xgb.dump)
-export(xgb.load)
-export(xgb.save)
-export(xgb.train)
-export(xgboost)
-exportMethods(predict)
-import(methods)
-importClassesFrom(Matrix,dgCMatrix)
-importClassesFrom(Matrix,dgeMatrix)

R-package/R/getinfo.xgb.DMatrix.R

@@ -1,41 +0,0 @@
-setClass('xgb.DMatrix')
-
-#' Get information of an xgb.DMatrix object
-#' 
-#' Get information of an xgb.DMatrix object
-#' 
-#' @examples
-#' data(agaricus.train, package='xgboost')
-#' train <- agaricus.train
-#' dtrain <- xgb.DMatrix(train$data, label=train$label)
-#' labels <- getinfo(dtrain, 'label')
-#' setinfo(dtrain, 'label', 1-labels)
-#' labels2 <- getinfo(dtrain, 'label')
-#' stopifnot(all(labels2 == 1-labels))
-#' @rdname getinfo
-#' @export
-#' 
-getinfo <- function(object, ...){
-    UseMethod("getinfo")
-}
-
-#' @param object Object of class "xgb.DMatrix"
-#' @param name the name of the field to get
-#' @param ... other parameters
-#' @rdname getinfo
-#' @method getinfo xgb.DMatrix
-setMethod("getinfo", signature = "xgb.DMatrix", 
-          definition = function(object, name) {
-              if (typeof(name) != "character") {
-                  stop("xgb.getinfo: name must be character")
-              }
-              if (class(object) != "xgb.DMatrix") {
-                  stop("xgb.setinfo: first argument dtrain must be xgb.DMatrix")
-              }
-              if (name != "label" && name != "weight" && name != "base_margin") {
-                  stop(paste("xgb.getinfo: unknown info name", name))
-              }
-              ret <- .Call("XGDMatrixGetInfo_R", object, name, PACKAGE = "xgboost")
-              return(ret)
-          })
-

R-package/R/predict.xgb.Booster.R

@@ -1,42 +0,0 @@
-setClass("xgb.Booster")
-
-#' Predict method for eXtreme Gradient Boosting model
-#' 
-#' Predicted values based on xgboost model object.
-#' 
-#' @param object Object of class "xgb.Boost"
-#' @param newdata takes \code{matrix}, \code{dgCMatrix}, local data file or 
-#'   \code{xgb.DMatrix}. 
-#' @param outputmargin whether the prediction should be shown in the original
-#'   value of sum of functions, when outputmargin=TRUE, the prediction is 
-#'   untransformed margin value. In logistic regression, outputmargin=T will
-#'   output value before logistic transformation.
-#' @param ntreelimit limit number of trees used in prediction, this parameter is
-#'  only valid for gbtree, but not for gblinear. set it to be value bigger 
-#'  than 0. It will use all trees by default.
-#' @examples
-#' data(agaricus.train, package='xgboost')
-#' data(agaricus.test, package='xgboost')
-#' train <- agaricus.train
-#' test <- agaricus.test
-#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
-#'                eta = 1, nround = 2,objective = "binary:logistic")
-#' pred <- predict(bst, test$data)
-#' @export
-#' 
-setMethod("predict", signature = "xgb.Booster", 
-          definition = function(object, newdata, outputmargin = FALSE, ntreelimit = NULL) {
-  if (class(newdata) != "xgb.DMatrix") {
-    newdata <- xgb.DMatrix(newdata)
-  }
-  if (is.null(ntreelimit)) {
-    ntreelimit <- 0
-  } else {
-    if (ntreelimit < 1){
-      stop("predict: ntreelimit must be equal to or greater than 1")
-    }
-  }
-  ret <- .Call("XGBoosterPredict_R", object, newdata, as.integer(outputmargin), as.integer(ntreelimit), PACKAGE = "xgboost")
-  return(ret)
-})
- 

R-package/R/setinfo.xgb.DMatrix.R

@@ -1,29 +0,0 @@
-#' Set information of an xgb.DMatrix object
-#' 
-#' Set information of an xgb.DMatrix object
-#' 
-#' @examples
-#' data(agaricus.train, package='xgboost')
-#' train <- agaricus.train
-#' dtrain <- xgb.DMatrix(train$data, label=train$label)
-#' labels <- getinfo(dtrain, 'label')
-#' setinfo(dtrain, 'label', 1-labels)
-#' labels2 <- getinfo(dtrain, 'label')
-#' stopifnot(all(labels2 == 1-labels))
-#' @rdname setinfo
-#' @export
-#' 
-setinfo <- function(object, ...){
-  UseMethod("setinfo")
-}
-
-#' @param object Object of class "xgb.DMatrix"
-#' @param name the name of the field to get
-#' @param info the specific field of information to set
-#' @param ... other parameters
-#' @rdname setinfo
-#' @method setinfo xgb.DMatrix
-setMethod("setinfo", signature = "xgb.DMatrix", 
-          definition = function(object, name, info) {
-            xgb.setinfo(object, name, info)
-          })

R-package/R/slice.xgb.DMatrix.R

@@ -1,33 +0,0 @@
-setClass('xgb.DMatrix')
-
-#' Get a new DMatrix containing the specified rows of
-#' orginal xgb.DMatrix object
-#'
-#' Get a new DMatrix containing the specified rows of
-#' orginal xgb.DMatrix object
-#' 
-#' @examples
-#' data(agaricus.train, package='xgboost')
-#' train <- agaricus.train
-#' dtrain <- xgb.DMatrix(train$data, label=train$label)
-#' dsub <- slice(dtrain, 1:3)
-#' @rdname slice
-#' @export
-#' 
-slice <- function(object, ...){
-    UseMethod("slice")
-}
-
-#' @param object Object of class "xgb.DMatrix"
-#' @param idxset a integer vector of indices of rows needed
-#' @param ... other parameters
-#' @rdname slice
-#' @method slice xgb.DMatrix
-setMethod("slice", signature = "xgb.DMatrix", 
-          definition = function(object, idxset, ...) {
-              if (class(object) != "xgb.DMatrix") {
-                  stop("slice: first argument dtrain must be xgb.DMatrix")
-              }
-              ret <- .Call("XGDMatrixSliceDMatrix_R", object, idxset, PACKAGE = "xgboost")
-              return(structure(ret, class = "xgb.DMatrix"))
-          })

R-package/R/utils.R

@@ -1,214 +0,0 @@
-#' @importClassesFrom Matrix dgCMatrix dgeMatrix
-#' @import methods
-
-# depends on matrix
-.onLoad <- function(libname, pkgname) {
-  library.dynam("xgboost", pkgname, libname)
-}
-.onUnload <- function(libpath) {
-  library.dynam.unload("xgboost", libpath)
-}
-
-# set information into dmatrix, this mutate dmatrix
-xgb.setinfo <- function(dmat, name, info) {
-  if (class(dmat) != "xgb.DMatrix") {
-    stop("xgb.setinfo: first argument dtrain must be xgb.DMatrix")
-  }
-  if (name == "label") {
-    .Call("XGDMatrixSetInfo_R", dmat, name, as.numeric(info), 
-          PACKAGE = "xgboost")
-    return(TRUE)
-  }
-  if (name == "weight") {
-    .Call("XGDMatrixSetInfo_R", dmat, name, as.numeric(info), 
-          PACKAGE = "xgboost")
-    return(TRUE)
-  }
-  if (name == "base_margin") {
-    .Call("XGDMatrixSetInfo_R", dmat, name, as.numeric(info), 
-          PACKAGE = "xgboost")
-    return(TRUE)
-  }
-  if (name == "group") {
-    .Call("XGDMatrixSetInfo_R", dmat, name, as.integer(info), 
-          PACKAGE = "xgboost")
-    return(TRUE)
-  }
-  stop(paste("xgb.setinfo: unknown info name", name))
-  return(FALSE)
-}
-
-# construct a Booster from cachelist
-xgb.Booster <- function(params = list(), cachelist = list(), modelfile = NULL) {
-  if (typeof(cachelist) != "list") {
-    stop("xgb.Booster: only accepts list of DMatrix as cachelist")
-  }
-  for (dm in cachelist) {
-    if (class(dm) != "xgb.DMatrix") {
-      stop("xgb.Booster: only accepts list of DMatrix as cachelist")
-    }
-  }
-  handle <- .Call("XGBoosterCreate_R", cachelist, PACKAGE = "xgboost")
-  if (length(params) != 0) {
-    for (i in 1:length(params)) {
-      p <- params[i]
-      .Call("XGBoosterSetParam_R", handle, gsub("\\.", "_", names(p)), as.character(p),
-            PACKAGE = "xgboost")
-    }
-  }
-  if (!is.null(modelfile)) {
-    if (typeof(modelfile) != "character") {
-      stop("xgb.Booster: modelfile must be character")
-    }
-    .Call("XGBoosterLoadModel_R", handle, modelfile, PACKAGE = "xgboost")
-  }
-  return(structure(handle, class = "xgb.Booster"))
-}
-
-## ----the following are low level iteratively function, not needed if
-## you do not want to use them ---------------------------------------
-# get dmatrix from data, label
-xgb.get.DMatrix <- function(data, label = NULL) {
-  inClass <- class(data)
-  if (inClass == "dgCMatrix" || inClass == "matrix") {
-    if (is.null(label)) {
-      stop("xgboost: need label when data is a matrix")
-    }
-    dtrain <- xgb.DMatrix(data, label = label)
-  } else {
-    if (!is.null(label)) {
-      warning("xgboost: label will be ignored.")
-    }
-    if (inClass == "character") {
-      dtrain <- xgb.DMatrix(data)
-    } else if (inClass == "xgb.DMatrix") {
-      dtrain <- data
-    } else {
-      stop("xgboost: Invalid input of data")
-    }
-  }
-  return (dtrain)
-}
-xgb.numrow <- function(dmat) {
-  nrow <- .Call("XGDMatrixNumRow_R", dmat, PACKAGE="xgboost")
-  return(nrow)
-}
-# iteratively update booster with customized statistics
-xgb.iter.boost <- function(booster, dtrain, gpair) {
-  if (class(booster) != "xgb.Booster") {
-    stop("xgb.iter.update: first argument must be type xgb.Booster")
-  }
-  if (class(dtrain) != "xgb.DMatrix") {
-    stop("xgb.iter.update: second argument must be type xgb.DMatrix")
-  }
-  .Call("XGBoosterBoostOneIter_R", booster, dtrain, gpair$grad, gpair$hess, 
-        PACKAGE = "xgboost")
-  return(TRUE)
-}
-
-# iteratively update booster with dtrain
-xgb.iter.update <- function(booster, dtrain, iter, obj = NULL) {
-  if (class(booster) != "xgb.Booster") {
-    stop("xgb.iter.update: first argument must be type xgb.Booster")
-  }
-  if (class(dtrain) != "xgb.DMatrix") {
-    stop("xgb.iter.update: second argument must be type xgb.DMatrix")
-  }
-
-  if (is.null(obj)) {
-    .Call("XGBoosterUpdateOneIter_R", booster, as.integer(iter), dtrain, 
-          PACKAGE = "xgboost")
-  } else {
-    pred <- predict(booster, dtrain)
-    gpair <- obj(pred, dtrain)
-    succ <- xgb.iter.boost(booster, dtrain, gpair)
-  }
-  return(TRUE)
-}
-
-# iteratively evaluate one iteration
-xgb.iter.eval <- function(booster, watchlist, iter, feval = NULL) {
-  if (class(booster) != "xgb.Booster") {
-    stop("xgb.eval: first argument must be type xgb.Booster")
-  }
-  if (typeof(watchlist) != "list") {
-    stop("xgb.eval: only accepts list of DMatrix as watchlist")
-  }
-  for (w in watchlist) {
-    if (class(w) != "xgb.DMatrix") {
-      stop("xgb.eval: watch list can only contain xgb.DMatrix")
-    }
-  }
-  if (length(watchlist) != 0) {
-    if (is.null(feval)) {
-      evnames <- list()
-      for (i in 1:length(watchlist)) {
-        w <- watchlist[i]
-        if (length(names(w)) == 0) {
-          stop("xgb.eval: name tag must be presented for every elements in watchlist")
-        }
-        evnames <- append(evnames, names(w))
-      }
-      msg <- .Call("XGBoosterEvalOneIter_R", booster, as.integer(iter), watchlist, 
-                   evnames, PACKAGE = "xgboost")
-    } else {
-      msg <- paste("[", iter, "]", sep="")
-      for (j in 1:length(watchlist)) {
-        w <- watchlist[j]
-        if (length(names(w)) == 0) {
-          stop("xgb.eval: name tag must be presented for every elements in watchlist")
-        }
-        ret <- feval(predict(booster, w[[1]]), w[[1]])
-        msg <- paste(msg, "\t", names(w), "-", ret$metric, ":", ret$value, sep="")
-      }
-    }
-  } else {
-    msg <- ""
-  }
-  return(msg)
-} 
-#------------------------------------------
-# helper functions for cross validation
-#
-xgb.cv.mknfold <- function(dall, nfold, param) {
-  randidx <- sample(1 : xgb.numrow(dall))
-  kstep <- length(randidx) / nfold
-  idset <- list()
-  for (i in 1:nfold) {
-    idset[[i]] <- randidx[ ((i-1) * kstep + 1) : min(i * kstep, length(randidx)) ]
-  }
-  ret <- list()
-  for (k in 1:nfold) {
-    dtest <- slice(dall, idset[[k]])
-    didx = c()
-    for (i in 1:nfold) {
-      if (i != k) {
-        didx <- append(didx, idset[[i]])
-      }
-    }
-    dtrain <- slice(dall, didx)
-    bst <- xgb.Booster(param, list(dtrain, dtest))
-    watchlist = list(train=dtrain, test=dtest)
-    ret[[k]] <- list(dtrain=dtrain, booster=bst, watchlist=watchlist)
-  }
-  return (ret)
-}
-xgb.cv.aggcv <- function(res, showsd = TRUE) {
-  header <- res[[1]]
-  ret <- header[1]
-  for (i in 2:length(header)) {
-    kv <- strsplit(header[i], ":")[[1]]
-    ret <- paste(ret, "\t", kv[1], ":", sep="")
-    stats <- c()
-    stats[1] <- as.numeric(kv[2])    
-    for (j in 2:length(res)) {
-      tkv <- strsplit(res[[j]][i], ":")[[1]]
-      stats[j] <- as.numeric(tkv[2])
-    }
-    ret <- paste(ret, sprintf("%f", mean(stats)), sep="")
-    if (showsd) {
-      ret <- paste(ret, sprintf("+%f", sd(stats)), sep="")
-    }
-  }
-  return (ret)
-}

R-package/R/xgb.DMatrix.R

@@ -1,45 +0,0 @@
-#' Contruct xgb.DMatrix object
-#' 
-#' Contruct xgb.DMatrix object from dense matrix, sparse matrix or local file.
-#' 
-#' @param data a \code{matrix} object, a \code{dgCMatrix} object or a character 
-#'   indicating the data file.
-#' @param info a list of information of the xgb.DMatrix object
-#' @param missing Missing is only used when input is dense matrix, pick a float
-#     value that represents missing value. Sometime a data use 0 or other extreme value to represents missing values.
-#
-#' @param ... other information to pass to \code{info}.
-#' 
-#' @examples
-#' data(agaricus.train, package='xgboost')
-#' train <- agaricus.train
-#' dtrain <- xgb.DMatrix(train$data, label=train$label)
-#' xgb.DMatrix.save(dtrain, 'xgb.DMatrix.data')
-#' dtrain <- xgb.DMatrix('xgb.DMatrix.data')
-#' @export
-#' 
-xgb.DMatrix <- function(data, info = list(), missing = 0, ...) {
-  if (typeof(data) == "character") {
-    handle <- .Call("XGDMatrixCreateFromFile_R", data, as.integer(FALSE), 
-                    PACKAGE = "xgboost")
-  } else if (is.matrix(data)) {
-    handle <- .Call("XGDMatrixCreateFromMat_R", data, missing, 
-                    PACKAGE = "xgboost")
-  } else if (class(data) == "dgCMatrix") {
-    handle <- .Call("XGDMatrixCreateFromCSC_R", data@p, data@i, data@x, 
-                    PACKAGE = "xgboost")
-  } else {
-    stop(paste("xgb.DMatrix: does not support to construct from ", 
-               typeof(data)))
-  }
-  dmat <- structure(handle, class = "xgb.DMatrix")
-  
-  info <- append(info, list(...))
-  if (length(info) == 0) 
-    return(dmat)
-  for (i in 1:length(info)) {
-    p <- info[i]
-    xgb.setinfo(dmat, names(p), p[[1]])
-  }
-  return(dmat)
-} 

R-package/R/xgb.DMatrix.save.R

@@ -1,27 +0,0 @@
-#' Save xgb.DMatrix object to binary file
-#' 
-#' Save xgb.DMatrix object to binary file
-#' 
-#' @param DMatrix the DMatrix object
-#' @param fname the name of the binary file.
-#' 
-#' @examples
-#' data(agaricus.train, package='xgboost')
-#' train <- agaricus.train
-#' dtrain <- xgb.DMatrix(train$data, label=train$label)
-#' xgb.DMatrix.save(dtrain, 'xgb.DMatrix.data')
-#' dtrain <- xgb.DMatrix('xgb.DMatrix.data')
-#' @export
-#' 
-xgb.DMatrix.save <- function(DMatrix, fname) {
-  if (typeof(fname) != "character") {
-    stop("xgb.save: fname must be character")
-  }
-  if (class(DMatrix) == "xgb.DMatrix") {
-    .Call("XGDMatrixSaveBinary_R", DMatrix, fname, as.integer(FALSE), 
-          PACKAGE = "xgboost")
-    return(TRUE)
-  }
-  stop("xgb.DMatrix.save: the input must be xgb.DMatrix")
-  return(FALSE)
-} 

R-package/R/xgb.cv.R

@@ -1,86 +0,0 @@
-#' Cross Validation
-#' 
-#' The cross valudation function of xgboost
-#'
-#' @param params the list of parameters. Commonly used ones are:
-#' \itemize{
-#'   \item \code{objective} objective function, common ones are
-#'   \itemize{
-#'     \item \code{reg:linear} linear regression
-#'     \item \code{binary:logistic} logistic regression for classification
-#'   }
-#'   \item \code{eta} step size of each boosting step
-#'   \item \code{max.depth} maximum depth of the tree
-#'   \item \code{nthread} number of thread used in training, if not set, all threads are used
-#' }
-#'
-#'   See \url{https://github.com/tqchen/xgboost/wiki/Parameters} for 
-#'   further details. See also demo/ for walkthrough example in R.
-#' @param data takes an \code{xgb.DMatrix} as the input.
-#' @param nrounds the max number of iterations
-#' @param nfold number of folds used
-#' @param label option field, when data is Matrix
-#' @param showsd boolean, whether show standard deviation of cross validation
-#' @param metrics, list of evaluation metrics to be used in corss validation,
-#'   when it is not specified, the evaluation metric is chosen according to objective function.
-#'   Possible options are:
-#' \itemize{
-#'   \item \code{error} binary classification error rate
-#'   \item \code{rmse} Rooted mean square error
-#'   \item \code{logloss} negative log-likelihood function
-#'   \item \code{auc} Area under curve
-#'   \item \code{merror} Exact matching error, used to evaluate multi-class classification
-#' }
-#' @param obj customized objective function. Returns gradient and second order 
-#'   gradient with given prediction and dtrain, 
-#' @param feval custimized evaluation function. Returns 
-#'   \code{list(metric='metric-name', value='metric-value')} with given 
-#'   prediction and dtrain,
-#' @param ... other parameters to pass to \code{params}.
-#' 
-#' @details 
-#' This is the cross validation function for xgboost
-#'
-#' Parallelization is automatically enabled if OpenMP is present.
-#' Number of threads can also be manually specified via "nthread" parameter.
-#' 
-#' This function only accepts an \code{xgb.DMatrix} object as the input.
-#'
-#' @examples
-#' data(agaricus.train, package='xgboost')
-#' dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
-#' history <- xgb.cv(data = dtrain, nround=3, nfold = 5, metrics=list("rmse","auc"),
-#'                   "max.depth"=3, "eta"=1, "objective"="binary:logistic")
-#' @export
-#'
-xgb.cv <- function(params=list(), data, nrounds, nfold, label = NULL,
-                   showsd = TRUE, metrics=list(), obj = NULL, feval = NULL, ...) {
-  if (typeof(params) != "list") {
-    stop("xgb.cv: first argument params must be list")
-  }
-  if (nfold <= 1) {
-    stop("nfold must be bigger than 1")
-  }
-  dtrain <- xgb.get.DMatrix(data, label)
-  params <- append(params, list(...))
-  params <- append(params, list(silent=1))
-  for (mc in metrics) {
-    params <- append(params, list("eval_metric"=mc))
-  }
-
-  folds <- xgb.cv.mknfold(dtrain, nfold, params)
-  history <- list()
-  for (i in 1:nrounds) {
-    msg <- list()
-    for (k in 1:nfold) {
-      fd <- folds[[k]]
-      succ <- xgb.iter.update(fd$booster, fd$dtrain, i - 1, obj)      
-      msg[[k]] <- strsplit(xgb.iter.eval(fd$booster, fd$watchlist, i - 1, feval), 
-                           "\t")[[1]]
-    }
-    ret <- xgb.cv.aggcv(msg, showsd)
-    history <- append(history, ret)
-    cat(paste(ret, "\n", sep=""))
-  }
-  return (TRUE)
-}

R-package/R/xgb.dump.R

@@ -1,33 +0,0 @@
-#' Save xgboost model to text file
-#' 
-#' Save a xgboost model to text file. Could be parsed later.
-#' 
-#' @param model the model object.
-#' @param fname the name of the binary file.
-#' @param fmap feature map file representing the type of feature. 
-#'        Detailed description could be found at 
-#'        \url{https://github.com/tqchen/xgboost/wiki/Binary-Classification#dump-model}.
-#'        See demo/ for walkthrough example in R, and
-#'        \url{https://github.com/tqchen/xgboost/blob/master/demo/data/featmap.txt} 
-#'        for example Format.
-#'
-#' @examples
-#' data(agaricus.train, package='xgboost')
-#' data(agaricus.test, package='xgboost')
-#' train <- agaricus.train
-#' test <- agaricus.test
-#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
-#'                eta = 1, nround = 2,objective = "binary:logistic")
-#' xgb.dump(bst, 'xgb.model.dump')
-#' @export
-#' 
-xgb.dump <- function(model, fname, fmap = "") {
-  if (class(model) != "xgb.Booster") {
-    stop("xgb.dump: first argument must be type xgb.Booster")
-  }
-  if (typeof(fname) != "character") {
-    stop("xgb.dump: second argument must be type character")
-  }
-  .Call("XGBoosterDumpModel_R", model, fname, fmap, PACKAGE = "xgboost")
-  return(TRUE)
-} 

R-package/R/xgb.load.R

@@ -1,23 +0,0 @@
-#' Load xgboost model from binary file
-#' 
-#' Load xgboost model from the binary model file
-#' 
-#' @param modelfile the name of the binary file.
-#' 
-#' @examples
-#' data(agaricus.train, package='xgboost')
-#' data(agaricus.test, package='xgboost')
-#' train <- agaricus.train
-#' test <- agaricus.test
-#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
-#'                eta = 1, nround = 2,objective = "binary:logistic")
-#' xgb.save(bst, 'xgb.model')
-#' bst <- xgb.load('xgb.model')
-#' pred <- predict(bst, test$data)
-#' @export
-#' 
-xgb.load <- function(modelfile) {
-  if (is.null(modelfile)) 
-    stop("xgb.load: modelfile cannot be NULL")
-  xgb.Booster(modelfile = modelfile)
-} 

R-package/R/xgb.save.R

@@ -1,31 +0,0 @@
-#' Save xgboost model to binary file
-#' 
-#' Save xgboost model from xgboost or xgb.train
-#' 
-#' @param model the model object.
-#' @param fname the name of the binary file.
-#' 
-#' @examples
-#' data(agaricus.train, package='xgboost')
-#' data(agaricus.test, package='xgboost')
-#' train <- agaricus.train
-#' test <- agaricus.test
-#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
-#'                eta = 1, nround = 2,objective = "binary:logistic")
-#' xgb.save(bst, 'xgb.model')
-#' bst <- xgb.load('xgb.model')
-#' pred <- predict(bst, test$data)
-#' @export
-#' 
-xgb.save <- function(model, fname) {
-  if (typeof(fname) != "character") {
-    stop("xgb.save: fname must be character")
-  }
-  if (class(model) == "xgb.Booster") {
-    .Call("XGBoosterSaveModel_R", model, fname, PACKAGE = "xgboost")
-    return(TRUE)
-  }
-  stop("xgb.save: the input must be xgb.Booster. Use xgb.DMatrix.save to save
-       xgb.DMatrix object.")
-  return(FALSE)
-} 

R-package/R/xgb.train.R

@@ -1,98 +0,0 @@
-#' eXtreme Gradient Boosting Training
-#' 
-#' The training function of xgboost
-#'
-#' @param params the list of parameters. Commonly used ones are:
-#' \itemize{
-#'   \item \code{objective} objective function, common ones are
-#'   \itemize{
-#'     \item \code{reg:linear} linear regression
-#'     \item \code{binary:logistic} logistic regression for classification
-#'   }
-#'   \item \code{eta} step size of each boosting step
-#'   \item \code{max.depth} maximum depth of the tree
-#'   \item \code{nthread} number of thread used in training, if not set, all threads are used
-#' }
-#'
-#'   See \url{https://github.com/tqchen/xgboost/wiki/Parameters} for 
-#'   further details. See also demo/ for walkthrough example in R.
-#' @param data takes an \code{xgb.DMatrix} as the input.
-#' @param nrounds the max number of iterations
-#' @param watchlist what information should be printed when \code{verbose=1} or
-#'   \code{verbose=2}. Watchlist is used to specify validation set monitoring
-#'   during training. For example user can specify
-#'    watchlist=list(validation1=mat1, validation2=mat2) to watch
-#'    the performance of each round's model on mat1 and mat2
-#'
-#' @param obj customized objective function. Returns gradient and second order 
-#'   gradient with given prediction and dtrain, 
-#' @param feval custimized evaluation function. Returns 
-#'   \code{list(metric='metric-name', value='metric-value')} with given 
-#'   prediction and dtrain,
-#' @param verbose If 0, xgboost will stay silent. If 1, xgboost will print 
-#'   information of performance. If 2, xgboost will print information of both
-#'
-#' @param ... other parameters to pass to \code{params}.
-#' 
-#' @details 
-#' This is the training function for xgboost.
-#'
-#' Parallelization is automatically enabled if OpenMP is present.
-#' Number of threads can also be manually specified via "nthread" parameter.
-#' 
-#' This function only accepts an \code{xgb.DMatrix} object as the input.
-#' It supports advanced features such as watchlist, customized objective function,
-#' therefore it is more flexible than \code{\link{xgboost}}.
-#' 
-#' 
-#' @examples
-#' data(agaricus.train, package='xgboost')
-#' dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
-#' dtest <- dtrain
-#' watchlist <- list(eval = dtest, train = dtrain)
-#' param <- list(max.depth = 2, eta = 1, silent = 1)
-#' logregobj <- function(preds, dtrain) {
-#'    labels <- getinfo(dtrain, "label")
-#'    preds <- 1/(1 + exp(-preds))
-#'    grad <- preds - labels
-#'    hess <- preds * (1 - preds)
-#'    return(list(grad = grad, hess = hess))
-#' }
-#' evalerror <- function(preds, dtrain) {
-#'   labels <- getinfo(dtrain, "label")
-#'   err <- as.numeric(sum(labels != (preds > 0)))/length(labels)
-#'   return(list(metric = "error", value = err))
-#' }
-#' bst <- xgb.train(param, dtrain, nround = 2, watchlist, logregobj, evalerror)
-#' @export
-#' 
-xgb.train <- function(params=list(), data, nrounds, watchlist = list(), 
-                      obj = NULL, feval = NULL, verbose = 1, ...) {
-  dtrain <- data
-  if (typeof(params) != "list") {
-    stop("xgb.train: first argument params must be list")
-  }
-  if (class(dtrain) != "xgb.DMatrix") {
-    stop("xgb.train: second argument dtrain must be xgb.DMatrix")
-  }
-  if (verbose > 1) {
-    params <- append(params, list(silent = 0))
-  } else {
-    params <- append(params, list(silent = 1))
-  }
-  if (length(watchlist) != 0 && verbose == 0) {
-    warning('watchlist is provided but verbose=0, no evaluation information will be printed')
-    watchlist <- list()
-  }
-  params = append(params, list(...))
-  
-  bst <- xgb.Booster(params, append(watchlist, dtrain))
-  for (i in 1:nrounds) {
-    succ <- xgb.iter.update(bst, dtrain, i - 1, obj)
-    if (length(watchlist) != 0) {
-      msg <- xgb.iter.eval(bst, watchlist, i - 1, feval)
-      cat(paste(msg, "\n", sep=""))
-    }
-  }
-  return(bst)
-} 

R-package/R/xgboost.R

@@ -1,115 +0,0 @@
-#' eXtreme Gradient Boosting (Tree) library
-#' 
-#' A simple interface for xgboost in R
-#' 
-#' @param data takes \code{matrix}, \code{dgCMatrix}, local data file or 
-#'   \code{xgb.DMatrix}. 
-#' @param label the response variable. User should not set this field,
-#    if data is local data file or  \code{xgb.DMatrix}. 
-#' @param params the list of parameters. Commonly used ones are:
-#' \itemize{
-#'   \item \code{objective} objective function, common ones are
-#'   \itemize{
-#'     \item \code{reg:linear} linear regression
-#'     \item \code{binary:logistic} logistic regression for classification
-#'   }
-#'   \item \code{eta} step size of each boosting step
-#'   \item \code{max.depth} maximum depth of the tree
-#'   \item \code{nthread} number of thread used in training, if not set, all threads are used
-#' }
-#'
-#'   See \url{https://github.com/tqchen/xgboost/wiki/Parameters} for 
-#'   further details. See also demo/ for walkthrough example in R.
-#' @param nrounds the max number of iterations
-#' @param verbose If 0, xgboost will stay silent. If 1, xgboost will print 
-#'   information of performance. If 2, xgboost will print information of both
-#'   performance and construction progress information
-#' @param ... other parameters to pass to \code{params}.
-#' 
-#' @details 
-#' This is the modeling function for xgboost.
-#' 
-#' Parallelization is automatically enabled if OpenMP is present.
-#' Number of threads can also be manually specified via "nthread" parameter
-#' 
-#' @examples
-#' data(agaricus.train, package='xgboost')
-#' data(agaricus.test, package='xgboost')
-#' train <- agaricus.train
-#' test <- agaricus.test
-#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
-#'                eta = 1, nround = 2,objective = "binary:logistic")
-#' pred <- predict(bst, test$data)
-#' 
-#' @export
-#' 
-xgboost <- function(data = NULL, label = NULL, params = list(), nrounds, 
-                    verbose = 1, ...) {
-  dtrain <- xgb.get.DMatrix(data, label)  
-  params <- append(params, list(...))
-  
-  if (verbose > 0) {
-    watchlist <- list(train = dtrain)
-  } else {
-    watchlist <- list()
-  }
-  
-  bst <- xgb.train(params, dtrain, nrounds, watchlist, verbose=verbose)
-  
-  return(bst)
-} 
-
-
-#' Training part from Mushroom Data Set
-#' 
-#' This data set is originally from the Mushroom data set,
-#' UCI Machine Learning Repository.
-#' 
-#' This data set includes the following fields:
-#' 
-#' \itemize{
-#'  \item \code{label} the label for each record
-#'  \item \code{data} a sparse Matrix of \code{dgCMatrix} class, with 127 columns.
-#' }
-#'
-#' @references
-#' https://archive.ics.uci.edu/ml/datasets/Mushroom
-#' 
-#' Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository 
-#' [http://archive.ics.uci.edu/ml]. Irvine, CA: University of California, 
-#' School of Information and Computer Science.
-#' 
-#' @docType data
-#' @keywords datasets
-#' @name agaricus.train
-#' @usage data(agaricus.train)
-#' @format A list containing a label vector, and a dgCMatrix object with 6513 
-#' rows and 127 variables
-NULL
-
-#' Test part from Mushroom Data Set
-#'
-#' This data set is originally from the Mushroom data set,
-#' UCI Machine Learning Repository.
-#' 
-#' This data set includes the following fields:
-#' 
-#' \itemize{
-#'  \item \code{label} the label for each record
-#'  \item \code{data} a sparse Matrix of \code{dgCMatrix} class, with 127 columns.
-#' }
-#'
-#' @references
-#' https://archive.ics.uci.edu/ml/datasets/Mushroom
-#' 
-#' Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository 
-#' [http://archive.ics.uci.edu/ml]. Irvine, CA: University of California, 
-#' School of Information and Computer Science.
-#' 
-#' @docType data
-#' @keywords datasets
-#' @name agaricus.test
-#' @usage data(agaricus.test)
-#' @format A list containing a label vector, and a dgCMatrix object with 1611 
-#' rows and 127 variables
-NULL

R-package/README.md

@@ -1,21 +0,0 @@
-# R package for xgboost.
-
-## Installation
-
-For up-to-date version(which is recommended), please install from github. Windows user will need to install [RTools](http://cran.r-project.org/bin/windows/Rtools/) first.
-
-```r
-require(devtools)
-install_github('xgboost','tqchen',subdir='R-package')
-```
-
-For stable version on CRAN, please run
-
-```r
-install.packages('xgboost')
-```
-
-## Examples
-
-* Please visit [walk through example](https://github.com/tqchen/xgboost/blob/master/R-package/demo).
-* See also the [example scripts](https://github.com/tqchen/xgboost/tree/master/demo/kaggle-higgs) for Kaggle Higgs Challenge, including [speedtest script](https://github.com/tqchen/xgboost/blob/master/demo/kaggle-higgs/speedtest.R) on this dataset.

R-package/demo/00Index

@@ -1,6 +0,0 @@
-basic_walkthrough               Basic feature walkthrough
-custom_objective                Cutomize loss function, and evaluation metric
-boost_from_prediction           Boosting from existing prediction
-predict_first_ntree             Predicting using first n trees
-generalized_linear_model        Generalized Linear Model
-cross_validation                Cross validation

R-package/demo/README.md

@@ -1,17 +0,0 @@
-XGBoost R Feature Walkthrough
-====
-* [Basic walkthrough of wrappers](basic_walkthrough.R) 
-* [Cutomize loss function, and evaluation metric](custom_objective.R)
-* [Boosting from existing prediction](boost_from_prediction.R)
-* [Predicting using first n trees](predict_first_ntree.R)
-* [Generalized Linear Model](generalized_linear_model.R)
-* [Cross validation](cross_validation.R)
-
-Benchmarks
-====
-* [Starter script for Kaggle Higgs Boson](../../demo/kaggle-higgs)
- 
-Notes
-====
-* Contribution of exampls, benchmarks is more than welcomed!
-* If you like to share how you use xgboost to solve your problem, send a pull request:)

R-package/demo/basic_walkthrough.R

@@ -1,93 +0,0 @@
-require(xgboost)
-require(methods)
-# we load in the agaricus dataset
-# In this example, we are aiming to predict whether a mushroom can be eated
-data(agaricus.train, package='xgboost')
-data(agaricus.test, package='xgboost')
-train <- agaricus.train
-test <- agaricus.test
-# the loaded data is stored in sparseMatrix, and label is a numeric vector in {0,1}
-class(train$label)
-class(train$data)
-
-#-------------Basic Training using XGBoost-----------------
-# this is the basic usage of xgboost you can put matrix in data field
-# note: we are puting in sparse matrix here, xgboost naturally handles sparse input
-# use sparse matrix when your feature is sparse(e.g. when you using one-hot encoding vector)
-print("training xgboost with sparseMatrix")
-bst <- xgboost(data = train$data, label = train$label, max.depth = 2, eta = 1, nround = 2,
-               objective = "binary:logistic")
-# alternatively, you can put in dense matrix, i.e. basic R-matrix
-print("training xgboost with Matrix")
-bst <- xgboost(data = as.matrix(train$data), label = train$label, max.depth = 2, eta = 1, nround = 2,
-               objective = "binary:logistic")
-
-# you can also put in xgb.DMatrix object, stores label, data and other meta datas needed for advanced features
-print("training xgboost with xgb.DMatrix")
-dtrain <- xgb.DMatrix(data = train$data, label = train$label)
-bst <- xgboost(data = dtrain, max.depth = 2, eta = 1, nround = 2, objective = "binary:logistic")
-
-# Verbose = 0,1,2
-print ('train xgboost with verbose 0, no message')
-bst <- xgboost(data = dtrain, max.depth = 2, eta = 1, nround = 2,
-               objective = "binary:logistic", verbose = 0)
-print ('train xgboost with verbose 1, print evaluation metric')
-bst <- xgboost(data = dtrain, max.depth = 2, eta = 1, nround = 2,
-               objective = "binary:logistic", verbose = 1)
-print ('train xgboost with verbose 2, also print information about tree')
-bst <- xgboost(data = dtrain, max.depth = 2, eta = 1, nround = 2,
-               objective = "binary:logistic", verbose = 2)
-
-# you can also specify data as file path to a LibSVM format input
-# since we do not have this file with us, the following line is just for illustration
-# bst <- xgboost(data = 'agaricus.train.svm', max.depth = 2, eta = 1, nround = 2,objective = "binary:logistic")
-
-#--------------------basic prediction using xgboost--------------
-# you can do prediction using the following line
-# you can put in Matrix, sparseMatrix, or xgb.DMatrix 
-pred <- predict(bst, test$data)
-err <- mean(as.numeric(pred > 0.5) != test$label)
-print(paste("test-error=", err))
-
-#-------------------save and load models-------------------------
-# save model to binary local file
-xgb.save(bst, "xgboost.model")
-# load binary model to R
-bst2 <- xgb.load("xgboost.model")
-pred2 <- predict(bst2, test$data)
-# pred2 should be identical to pred
-print(paste("sum(abs(pred2-pred))=", sum(abs(pred2-pred))))
-
-#----------------Advanced features --------------
-# to use advanced features, we need to put data in xgb.DMatrix
-dtrain <- xgb.DMatrix(data = train$data, label=train$label)
-dtest <- xgb.DMatrix(data = test$data, label=test$label)
-#---------------Using watchlist----------------
-# watchlist is a list of xgb.DMatrix, each of them tagged with name
-watchlist <- list(train=dtrain, test=dtest)
-# to train with watchlist, use xgb.train, which contains more advanced features
-# watchlist allows us to monitor the evaluation result on all data in the list 
-print ('train xgboost using xgb.train with watchlist')
-bst <- xgb.train(data=dtrain, max.depth=2, eta=1, nround=2, watchlist=watchlist,
-                 objective = "binary:logistic")
-# we can change evaluation metrics, or use multiple evaluation metrics
-print ('train xgboost using xgb.train with watchlist, watch logloss and error')
-bst <- xgb.train(data=dtrain, max.depth=2, eta=1, nround=2, watchlist=watchlist,
-                 eval.metric = "error", eval.metric = "logloss",
-                 objective = "binary:logistic")
-
-# xgb.DMatrix can also be saved using xgb.DMatrix.save
-xgb.DMatrix.save(dtrain, "dtrain.buffer")
-# to load it in, simply call xgb.DMatrix
-dtrain2 <- xgb.DMatrix("dtrain.buffer")
-bst <- xgb.train(data=dtrain2, max.depth=2, eta=1, nround=2, watchlist=watchlist,
-                 objective = "binary:logistic")
-# information can be extracted from xgb.DMatrix using getinfo
-label = getinfo(dtest, "label")
-pred <- predict(bst, dtest)
-err <- as.numeric(sum(as.integer(pred > 0.5) != label))/length(label)
-print(paste("test-error=", err))
-
-# Finally, you can dump the tree you learned using xgb.dump into a text file
-xgb.dump(bst, "dump.raw.txt")
-

R-package/demo/boost_from_prediction.R

@@ -1,26 +0,0 @@
-require(xgboost)
-# load in the agaricus dataset
-data(agaricus.train, package='xgboost')
-data(agaricus.test, package='xgboost')
-dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
-dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label)
-
-watchlist <- list(eval = dtest, train = dtrain)
-###
-# advanced: start from a initial base prediction
-#
-print('start running example to start from a initial prediction')
-# train xgboost for 1 round
-param <- list(max.depth=2,eta=1,silent=1,objective='binary:logistic')
-bst <- xgb.train( param, dtrain, 1, watchlist )
-# Note: we need the margin value instead of transformed prediction in set_base_margin
-# do predict with output_margin=TRUE, will always give you margin values before logistic transformation
-ptrain <- predict(bst, dtrain, outputmargin=TRUE)
-ptest  <- predict(bst, dtest, outputmargin=TRUE)
-# set the base_margin property of dtrain and dtest
-# base margin is the base prediction we will boost from
-setinfo(dtrain, "base_margin", ptrain)
-setinfo(dtest, "base_margin", ptest)
-
-print('this is result of boost from initial prediction')
-bst <- xgb.train( param, dtrain, 1, watchlist )

R-package/demo/cross_validation.R

@@ -1,47 +0,0 @@
-require(xgboost)
-# load in the agaricus dataset
-data(agaricus.train, package='xgboost')
-data(agaricus.test, package='xgboost')
-dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
-dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label)
-
-nround <- 2
-param <- list(max.depth=2,eta=1,silent=1,objective='binary:logistic')
-
-cat('running cross validation\n')
-# do cross validation, this will print result out as
-# [iteration]  metric_name:mean_value+std_value
-# std_value is standard deviation of the metric
-xgb.cv(param, dtrain, nround, nfold=5, metrics={'error'})
-
-cat('running cross validation, disable standard deviation display\n')
-# do cross validation, this will print result out as
-# [iteration]  metric_name:mean_value+std_value
-# std_value is standard deviation of the metric
-xgb.cv(param, dtrain, nround, nfold=5,
-       metrics={'error'}, , showsd = FALSE)
-
-###
-# you can also do cross validation with cutomized loss function
-# See custom_objective.R
-##
-print ('running cross validation, with cutomsized loss function')
-
-logregobj <- function(preds, dtrain) {
-  labels <- getinfo(dtrain, "label")
-  preds <- 1/(1 + exp(-preds))
-  grad <- preds - labels
-  hess <- preds * (1 - preds)
-  return(list(grad = grad, hess = hess))
-}
-evalerror <- function(preds, dtrain) {
-  labels <- getinfo(dtrain, "label")
-  err <- as.numeric(sum(labels != (preds > 0)))/length(labels)
-  return(list(metric = "error", value = err))
-}
-
-param <- list(max.depth=2,eta=1,silent=1)
-# train with customized objective
-xgb.cv(param, dtrain, nround, nfold = 5,
-       obj = logregobj, feval=evalerror)
-

R-package/demo/custom_objective.R

@@ -1,39 +0,0 @@
-require(xgboost)
-# load in the agaricus dataset
-data(agaricus.train, package='xgboost')
-data(agaricus.test, package='xgboost')
-dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
-dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label)
-
-# note: for customized objective function, we leave objective as default
-# note: what we are getting is margin value in prediction
-# you must know what you are doing
-param <- list(max.depth=2,eta=1,silent=1)
-watchlist <- list(eval = dtest, train = dtrain)
-num_round <- 2
-
-# user define objective function, given prediction, return gradient and second order gradient
-# this is loglikelihood loss
-logregobj <- function(preds, dtrain) {
-  labels <- getinfo(dtrain, "label")
-  preds <- 1/(1 + exp(-preds))
-  grad <- preds - labels
-  hess <- preds * (1 - preds)
-  return(list(grad = grad, hess = hess))
-}
-
-# user defined evaluation function, return a pair metric_name, result
-# NOTE: when you do customized loss function, the default prediction value is margin
-# this may make buildin evalution metric not function properly
-# for example, we are doing logistic loss, the prediction is score before logistic transformation
-# the buildin evaluation error assumes input is after logistic transformation
-# Take this in mind when you use the customization, and maybe you need write customized evaluation function
-evalerror <- function(preds, dtrain) {
-  labels <- getinfo(dtrain, "label")
-  err <- as.numeric(sum(labels != (preds > 0)))/length(labels)
-  return(list(metric = "error", value = err))
-}
-print ('start training with user customized objective')
-# training with customized objective, we can also do step by step training
-# simply look at xgboost.py's implementation of train
-bst <- xgb.train(param, dtrain, num_round, watchlist, logregobj, evalerror)

R-package/demo/generalized_linear_model.R

@@ -1,34 +0,0 @@
-require(xgboost)
-# load in the agaricus dataset
-data(agaricus.train, package='xgboost')
-data(agaricus.test, package='xgboost')
-dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
-dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label)
-##
-#  this script demonstrate how to fit generalized linear model in xgboost
-#  basically, we are using linear model, instead of tree for our boosters
-#  you can fit a linear regression, or logistic regression model
-##
-
-# change booster to gblinear, so that we are fitting a linear model
-# alpha is the L1 regularizer 
-# lambda is the L2 regularizer
-# you can also set lambda_bias which is L2 regularizer on the bias term
-param <- list(objective = "binary:logistic", booster = "gblinear",
-              alpha = 0.0001, lambda = 1)
-
-# normally, you do not need to set eta (step_size)
-# XGBoost uses a parallel coordinate descent algorithm (shotgun), 
-# there could be affection on convergence with parallelization on certain cases
-# setting eta to be smaller value, e.g 0.5 can make the optimization more stable
-
-##
-# the rest of settings are the same
-##
-watchlist <- list(eval = dtest, train = dtrain)
-num_round <- 2
-bst <- xgb.train(param, dtrain, num_round, watchlist)
-ypred <- predict(bst, dtest)
-labels <- getinfo(dtest, 'label')
-cat('error of preds=', mean(as.numeric(ypred>0.5)!=labels),'\n')
-

R-package/demo/predict_first_ntree.R

@@ -1,23 +0,0 @@
-require(xgboost)
-# load in the agaricus dataset
-data(agaricus.train, package='xgboost')
-data(agaricus.test, package='xgboost')
-dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
-dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label)
-
-param <- list(max.depth=2,eta=1,silent=1,objective='binary:logistic')
-watchlist <- list(eval = dtest, train = dtrain)
-nround = 2
-
-# training the model for two rounds
-bst = xgb.train(param, dtrain, nround, watchlist)
-cat('start testing prediction from first n trees\n')
-labels <- getinfo(dtest,'label')
-
-### predict using first 1 tree
-ypred1 = predict(bst, dtest, ntreelimit=1)
-# by default, we predict using all the trees
-ypred2 = predict(bst, dtest)
-
-cat('error of ypred1=', mean(as.numeric(ypred1>0.5)!=labels),'\n')
-cat('error of ypred2=', mean(as.numeric(ypred2>0.5)!=labels),'\n')

R-package/demo/runall.R

@@ -1,8 +0,0 @@
-# running all scripts in demo folder
-demo(basic_walkthrough)
-demo(custom_objective)
-demo(boost_from_prediction)
-demo(predict_first_ntree)
-demo(generalized_linear_model)
-demo(cross_validation)
-

R-package/man/agaricus.test.Rd

@@ -1,31 +0,0 @@
-% Generated by roxygen2 (4.0.1): do not edit by hand
-\docType{data}
-\name{agaricus.test}
-\alias{agaricus.test}
-\title{Test part from Mushroom Data Set}
-\format{A list containing a label vector, and a dgCMatrix object with 1611
-rows and 127 variables}
-\usage{
-data(agaricus.test)
-}
-\description{
-This data set is originally from the Mushroom data set,
-UCI Machine Learning Repository.
-}
-\details{
-This data set includes the following fields:
-
-\itemize{
- \item \code{label} the label for each record
- \item \code{data} a sparse Matrix of \code{dgCMatrix} class, with 127 columns.
-}
-}
-\references{
-https://archive.ics.uci.edu/ml/datasets/Mushroom
-
-Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository
-[http://archive.ics.uci.edu/ml]. Irvine, CA: University of California,
-School of Information and Computer Science.
-}
-\keyword{datasets}
-

R-package/man/agaricus.train.Rd

@@ -1,31 +0,0 @@
-% Generated by roxygen2 (4.0.1): do not edit by hand
-\docType{data}
-\name{agaricus.train}
-\alias{agaricus.train}
-\title{Training part from Mushroom Data Set}
-\format{A list containing a label vector, and a dgCMatrix object with 6513
-rows and 127 variables}
-\usage{
-data(agaricus.train)
-}
-\description{
-This data set is originally from the Mushroom data set,
-UCI Machine Learning Repository.
-}
-\details{
-This data set includes the following fields:
-
-\itemize{
- \item \code{label} the label for each record
- \item \code{data} a sparse Matrix of \code{dgCMatrix} class, with 127 columns.
-}
-}
-\references{
-https://archive.ics.uci.edu/ml/datasets/Mushroom
-
-Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository
-[http://archive.ics.uci.edu/ml]. Irvine, CA: University of California,
-School of Information and Computer Science.
-}
-\keyword{datasets}
-

R-package/man/getinfo.Rd

@@ -1,31 +0,0 @@
-% Generated by roxygen2 (4.0.1): do not edit by hand
-\docType{methods}
-\name{getinfo}
-\alias{getinfo}
-\alias{getinfo,xgb.DMatrix-method}
-\title{Get information of an xgb.DMatrix object}
-\usage{
-getinfo(object, ...)
-
-\S4method{getinfo}{xgb.DMatrix}(object, name)
-}
-\arguments{
-\item{object}{Object of class "xgb.DMatrix"}
-
-\item{name}{the name of the field to get}
-
-\item{...}{other parameters}
-}
-\description{
-Get information of an xgb.DMatrix object
-}
-\examples{
-data(agaricus.train, package='xgboost')
-train <- agaricus.train
-dtrain <- xgb.DMatrix(train$data, label=train$label)
-labels <- getinfo(dtrain, 'label')
-setinfo(dtrain, 'label', 1-labels)
-labels2 <- getinfo(dtrain, 'label')
-stopifnot(all(labels2 == 1-labels))
-}
-

R-package/man/predict-xgb.Booster-method.Rd

@@ -1,37 +0,0 @@
-% Generated by roxygen2 (4.0.1): do not edit by hand
-\docType{methods}
-\name{predict,xgb.Booster-method}
-\alias{predict,xgb.Booster-method}
-\title{Predict method for eXtreme Gradient Boosting model}
-\usage{
-\S4method{predict}{xgb.Booster}(object, newdata, outputmargin = FALSE,
-  ntreelimit = NULL)
-}
-\arguments{
-\item{object}{Object of class "xgb.Boost"}
-
-\item{newdata}{takes \code{matrix}, \code{dgCMatrix}, local data file or
-\code{xgb.DMatrix}.}
-
-\item{outputmargin}{whether the prediction should be shown in the original
-value of sum of functions, when outputmargin=TRUE, the prediction is
-untransformed margin value. In logistic regression, outputmargin=T will
-output value before logistic transformation.}
-
-\item{ntreelimit}{limit number of trees used in prediction, this parameter is
-only valid for gbtree, but not for gblinear. set it to be value bigger
-than 0. It will use all trees by default.}
-}
-\description{
-Predicted values based on xgboost model object.
-}
-\examples{
-data(agaricus.train, package='xgboost')
-data(agaricus.test, package='xgboost')
-train <- agaricus.train
-test <- agaricus.test
-bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
-               eta = 1, nround = 2,objective = "binary:logistic")
-pred <- predict(bst, test$data)
-}
-

R-package/man/setinfo.Rd

@@ -1,33 +0,0 @@
-% Generated by roxygen2 (4.0.1): do not edit by hand
-\docType{methods}
-\name{setinfo}
-\alias{setinfo}
-\alias{setinfo,xgb.DMatrix-method}
-\title{Set information of an xgb.DMatrix object}
-\usage{
-setinfo(object, ...)
-
-\S4method{setinfo}{xgb.DMatrix}(object, name, info)
-}
-\arguments{
-\item{object}{Object of class "xgb.DMatrix"}
-
-\item{name}{the name of the field to get}
-
-\item{info}{the specific field of information to set}
-
-\item{...}{other parameters}
-}
-\description{
-Set information of an xgb.DMatrix object
-}
-\examples{
-data(agaricus.train, package='xgboost')
-train <- agaricus.train
-dtrain <- xgb.DMatrix(train$data, label=train$label)
-labels <- getinfo(dtrain, 'label')
-setinfo(dtrain, 'label', 1-labels)
-labels2 <- getinfo(dtrain, 'label')
-stopifnot(all(labels2 == 1-labels))
-}
-

R-package/man/slice.Rd

@@ -1,30 +0,0 @@
-% Generated by roxygen2 (4.0.1): do not edit by hand
-\docType{methods}
-\name{slice}
-\alias{slice}
-\alias{slice,xgb.DMatrix-method}
-\title{Get a new DMatrix containing the specified rows of
-orginal xgb.DMatrix object}
-\usage{
-slice(object, ...)
-
-\S4method{slice}{xgb.DMatrix}(object, idxset, ...)
-}
-\arguments{
-\item{object}{Object of class "xgb.DMatrix"}
-
-\item{idxset}{a integer vector of indices of rows needed}
-
-\item{...}{other parameters}
-}
-\description{
-Get a new DMatrix containing the specified rows of
-orginal xgb.DMatrix object
-}
-\examples{
-data(agaricus.train, package='xgboost')
-train <- agaricus.train
-dtrain <- xgb.DMatrix(train$data, label=train$label)
-dsub <- slice(dtrain, 1:3)
-}
-

R-package/man/xgb.DMatrix.Rd

@@ -1,28 +0,0 @@
-% Generated by roxygen2 (4.0.1): do not edit by hand
-\name{xgb.DMatrix}
-\alias{xgb.DMatrix}
-\title{Contruct xgb.DMatrix object}
-\usage{
-xgb.DMatrix(data, info = list(), missing = 0, ...)
-}
-\arguments{
-\item{data}{a \code{matrix} object, a \code{dgCMatrix} object or a character
-indicating the data file.}
-
-\item{info}{a list of information of the xgb.DMatrix object}
-
-\item{missing}{Missing is only used when input is dense matrix, pick a float}
-
-\item{...}{other information to pass to \code{info}.}
-}
-\description{
-Contruct xgb.DMatrix object from dense matrix, sparse matrix or local file.
-}
-\examples{
-data(agaricus.train, package='xgboost')
-train <- agaricus.train
-dtrain <- xgb.DMatrix(train$data, label=train$label)
-xgb.DMatrix.save(dtrain, 'xgb.DMatrix.data')
-dtrain <- xgb.DMatrix('xgb.DMatrix.data')
-}
-

R-package/man/xgb.DMatrix.save.Rd

@@ -1,23 +0,0 @@
-% Generated by roxygen2 (4.0.1): do not edit by hand
-\name{xgb.DMatrix.save}
-\alias{xgb.DMatrix.save}
-\title{Save xgb.DMatrix object to binary file}
-\usage{
-xgb.DMatrix.save(DMatrix, fname)
-}
-\arguments{
-\item{DMatrix}{the DMatrix object}
-
-\item{fname}{the name of the binary file.}
-}
-\description{
-Save xgb.DMatrix object to binary file
-}
-\examples{
-data(agaricus.train, package='xgboost')
-train <- agaricus.train
-dtrain <- xgb.DMatrix(train$data, label=train$label)
-xgb.DMatrix.save(dtrain, 'xgb.DMatrix.data')
-dtrain <- xgb.DMatrix('xgb.DMatrix.data')
-}
-

R-package/man/xgb.cv.Rd

@@ -1,72 +0,0 @@
-% Generated by roxygen2 (4.0.1): do not edit by hand
-\name{xgb.cv}
-\alias{xgb.cv}
-\title{Cross Validation}
-\usage{
-xgb.cv(params = list(), data, nrounds, nfold, label = NULL, showsd = TRUE,
-  metrics = list(), obj = NULL, feval = NULL, ...)
-}
-\arguments{
-\item{params}{the list of parameters. Commonly used ones are:
-\itemize{
-  \item \code{objective} objective function, common ones are
-  \itemize{
-    \item \code{reg:linear} linear regression
-    \item \code{binary:logistic} logistic regression for classification
-  }
-  \item \code{eta} step size of each boosting step
-  \item \code{max.depth} maximum depth of the tree
-  \item \code{nthread} number of thread used in training, if not set, all threads are used
-}
-
-  See \url{https://github.com/tqchen/xgboost/wiki/Parameters} for
-  further details. See also demo/ for walkthrough example in R.}
-
-\item{data}{takes an \code{xgb.DMatrix} as the input.}
-
-\item{nrounds}{the max number of iterations}
-
-\item{nfold}{number of folds used}
-
-\item{label}{option field, when data is Matrix}
-
-\item{showsd}{boolean, whether show standard deviation of cross validation}
-
-\item{metrics,}{list of evaluation metrics to be used in corss validation,
-  when it is not specified, the evaluation metric is chosen according to objective function.
-  Possible options are:
-\itemize{
-  \item \code{error} binary classification error rate
-  \item \code{rmse} Rooted mean square error
-  \item \code{logloss} negative log-likelihood function
-  \item \code{auc} Area under curve
-  \item \code{merror} Exact matching error, used to evaluate multi-class classification
-}}
-
-\item{obj}{customized objective function. Returns gradient and second order
-gradient with given prediction and dtrain,}
-
-\item{feval}{custimized evaluation function. Returns
-\code{list(metric='metric-name', value='metric-value')} with given
-prediction and dtrain,}
-
-\item{...}{other parameters to pass to \code{params}.}
-}
-\description{
-The cross valudation function of xgboost
-}
-\details{
-This is the cross validation function for xgboost
-
-Parallelization is automatically enabled if OpenMP is present.
-Number of threads can also be manually specified via "nthread" parameter.
-
-This function only accepts an \code{xgb.DMatrix} object as the input.
-}
-\examples{
-data(agaricus.train, package='xgboost')
-dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
-history <- xgb.cv(data = dtrain, nround=3, nfold = 5, metrics=list("rmse","auc"),
-                  "max.depth"=3, "eta"=1, "objective"="binary:logistic")
-}
-

R-package/man/xgb.dump.Rd

@@ -1,32 +0,0 @@
-% Generated by roxygen2 (4.0.1): do not edit by hand
-\name{xgb.dump}
-\alias{xgb.dump}
-\title{Save xgboost model to text file}
-\usage{
-xgb.dump(model, fname, fmap = "")
-}
-\arguments{
-\item{model}{the model object.}
-
-\item{fname}{the name of the binary file.}
-
-\item{fmap}{feature map file representing the type of feature.
-       Detailed description could be found at
-       \url{https://github.com/tqchen/xgboost/wiki/Binary-Classification#dump-model}.
-       See demo/ for walkthrough example in R, and
-       \url{https://github.com/tqchen/xgboost/blob/master/demo/data/featmap.txt}
-       for example Format.}
-}
-\description{
-Save a xgboost model to text file. Could be parsed later.
-}
-\examples{
-data(agaricus.train, package='xgboost')
-data(agaricus.test, package='xgboost')
-train <- agaricus.train
-test <- agaricus.test
-bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
-               eta = 1, nround = 2,objective = "binary:logistic")
-xgb.dump(bst, 'xgb.model.dump')
-}
-

R-package/man/xgb.load.Rd

@@ -1,25 +0,0 @@
-% Generated by roxygen2 (4.0.1): do not edit by hand
-\name{xgb.load}
-\alias{xgb.load}
-\title{Load xgboost model from binary file}
-\usage{
-xgb.load(modelfile)
-}
-\arguments{
-\item{modelfile}{the name of the binary file.}
-}
-\description{
-Load xgboost model from the binary model file
-}
-\examples{
-data(agaricus.train, package='xgboost')
-data(agaricus.test, package='xgboost')
-train <- agaricus.train
-test <- agaricus.test
-bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
-               eta = 1, nround = 2,objective = "binary:logistic")
-xgb.save(bst, 'xgb.model')
-bst <- xgb.load('xgb.model')
-pred <- predict(bst, test$data)
-}
-

R-package/man/xgb.save.Rd

@@ -1,27 +0,0 @@
-% Generated by roxygen2 (4.0.1): do not edit by hand
-\name{xgb.save}
-\alias{xgb.save}
-\title{Save xgboost model to binary file}
-\usage{
-xgb.save(model, fname)
-}
-\arguments{
-\item{model}{the model object.}
-
-\item{fname}{the name of the binary file.}
-}
-\description{
-Save xgboost model from xgboost or xgb.train
-}
-\examples{
-data(agaricus.train, package='xgboost')
-data(agaricus.test, package='xgboost')
-train <- agaricus.train
-test <- agaricus.test
-bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
-               eta = 1, nround = 2,objective = "binary:logistic")
-xgb.save(bst, 'xgb.model')
-bst <- xgb.load('xgb.model')
-pred <- predict(bst, test$data)
-}
-

R-package/man/xgb.train.Rd

@@ -1,80 +0,0 @@
-% Generated by roxygen2 (4.0.1): do not edit by hand
-\name{xgb.train}
-\alias{xgb.train}
-\title{eXtreme Gradient Boosting Training}
-\usage{
-xgb.train(params = list(), data, nrounds, watchlist = list(), obj = NULL,
-  feval = NULL, verbose = 1, ...)
-}
-\arguments{
-\item{params}{the list of parameters. Commonly used ones are:
-\itemize{
-  \item \code{objective} objective function, common ones are
-  \itemize{
-    \item \code{reg:linear} linear regression
-    \item \code{binary:logistic} logistic regression for classification
-  }
-  \item \code{eta} step size of each boosting step
-  \item \code{max.depth} maximum depth of the tree
-  \item \code{nthread} number of thread used in training, if not set, all threads are used
-}
-
-  See \url{https://github.com/tqchen/xgboost/wiki/Parameters} for
-  further details. See also demo/ for walkthrough example in R.}
-
-\item{data}{takes an \code{xgb.DMatrix} as the input.}
-
-\item{nrounds}{the max number of iterations}
-
-\item{watchlist}{what information should be printed when \code{verbose=1} or
-  \code{verbose=2}. Watchlist is used to specify validation set monitoring
-  during training. For example user can specify
-   watchlist=list(validation1=mat1, validation2=mat2) to watch
-   the performance of each round's model on mat1 and mat2}
-
-\item{obj}{customized objective function. Returns gradient and second order
-gradient with given prediction and dtrain,}
-
-\item{feval}{custimized evaluation function. Returns
-\code{list(metric='metric-name', value='metric-value')} with given
-prediction and dtrain,}
-
-\item{verbose}{If 0, xgboost will stay silent. If 1, xgboost will print
-  information of performance. If 2, xgboost will print information of both}
-
-\item{...}{other parameters to pass to \code{params}.}
-}
-\description{
-The training function of xgboost
-}
-\details{
-This is the training function for xgboost.
-
-Parallelization is automatically enabled if OpenMP is present.
-Number of threads can also be manually specified via "nthread" parameter.
-
-This function only accepts an \code{xgb.DMatrix} object as the input.
-It supports advanced features such as watchlist, customized objective function,
-therefore it is more flexible than \code{\link{xgboost}}.
-}
-\examples{
-data(agaricus.train, package='xgboost')
-dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
-dtest <- dtrain
-watchlist <- list(eval = dtest, train = dtrain)
-param <- list(max.depth = 2, eta = 1, silent = 1)
-logregobj <- function(preds, dtrain) {
-   labels <- getinfo(dtrain, "label")
-   preds <- 1/(1 + exp(-preds))
-   grad <- preds - labels
-   hess <- preds * (1 - preds)
-   return(list(grad = grad, hess = hess))
-}
-evalerror <- function(preds, dtrain) {
-  labels <- getinfo(dtrain, "label")
-  err <- as.numeric(sum(labels != (preds > 0)))/length(labels)
-  return(list(metric = "error", value = err))
-}
-bst <- xgb.train(param, dtrain, nround = 2, watchlist, logregobj, evalerror)
-}
-

R-package/man/xgboost.Rd

@@ -1,56 +0,0 @@
-% Generated by roxygen2 (4.0.1): do not edit by hand
-\name{xgboost}
-\alias{xgboost}
-\title{eXtreme Gradient Boosting (Tree) library}
-\usage{
-xgboost(data = NULL, label = NULL, params = list(), nrounds,
-  verbose = 1, ...)
-}
-\arguments{
-\item{data}{takes \code{matrix}, \code{dgCMatrix}, local data file or
-\code{xgb.DMatrix}.}
-
-\item{label}{the response variable. User should not set this field,}
-
-\item{params}{the list of parameters. Commonly used ones are:
-\itemize{
-  \item \code{objective} objective function, common ones are
-  \itemize{
-    \item \code{reg:linear} linear regression
-    \item \code{binary:logistic} logistic regression for classification
-  }
-  \item \code{eta} step size of each boosting step
-  \item \code{max.depth} maximum depth of the tree
-  \item \code{nthread} number of thread used in training, if not set, all threads are used
-}
-
-  See \url{https://github.com/tqchen/xgboost/wiki/Parameters} for
-  further details. See also demo/ for walkthrough example in R.}
-
-\item{nrounds}{the max number of iterations}
-
-\item{verbose}{If 0, xgboost will stay silent. If 1, xgboost will print
-information of performance. If 2, xgboost will print information of both
-performance and construction progress information}
-
-\item{...}{other parameters to pass to \code{params}.}
-}
-\description{
-A simple interface for xgboost in R
-}
-\details{
-This is the modeling function for xgboost.
-
-Parallelization is automatically enabled if OpenMP is present.
-Number of threads can also be manually specified via "nthread" parameter
-}
-\examples{
-data(agaricus.train, package='xgboost')
-data(agaricus.test, package='xgboost')
-train <- agaricus.train
-test <- agaricus.test
-bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
-               eta = 1, nround = 2,objective = "binary:logistic")
-pred <- predict(bst, test$data)
-}
-

R-package/src/Makevars

@@ -1,9 +0,0 @@
-# package root
-PKGROOT=../../
-# _*_ mode: Makefile; _*_
-PKG_CPPFLAGS= -DXGBOOST_CUSTOMIZE_MSG_ -DXGBOOST_CUSTOMIZE_PRNG_ -DXGBOOST_STRICT_CXX98_ -I$(PKGROOT)
-PKG_CXXFLAGS= $(SHLIB_OPENMP_CFLAGS)
-PKG_LIBS = $(SHLIB_OPENMP_CFLAGS)
-OBJECTS= xgboost_R.o xgboost_assert.o $(PKGROOT)/wrapper/xgboost_wrapper.o $(PKGROOT)/src/io/io.o $(PKGROOT)/src/gbm/gbm.o $(PKGROOT)/src/tree/updater.o
-
-

R-package/src/Makevars.win

@@ -1,7 +0,0 @@
-# package root
-PKGROOT=../../
-# _*_ mode: Makefile; _*_
-PKG_CPPFLAGS= -DXGBOOST_CUSTOMIZE_MSG_ -DXGBOOST_CUSTOMIZE_PRNG_ -DXGBOOST_STRICT_CXX98_ -I$(PKGROOT)
-PKG_CXXFLAGS= $(SHLIB_OPENMP_CFLAGS)
-PKG_LIBS = $(SHLIB_OPENMP_CFLAGS)
-OBJECTS= xgboost_R.o xgboost_assert.o $(PKGROOT)/wrapper/xgboost_wrapper.o $(PKGROOT)/src/io/io.o $(PKGROOT)/src/gbm/gbm.o $(PKGROOT)/src/tree/updater.o

R-package/src/xgboost_R.cpp

@@ -1,289 +0,0 @@
-#include <vector>
-#include <string>
-#include <utility>
-#include <cstring>
-#include <cstdio>
-#include "xgboost_R.h"
-#include "wrapper/xgboost_wrapper.h"
-#include "src/utils/utils.h"
-#include "src/utils/omp.h"
-using namespace std;
-using namespace xgboost;
-
-extern "C" {
-  void XGBoostAssert_R(int exp, const char *fmt, ...);
-  void XGBoostCheck_R(int exp, const char *fmt, ...);
-  int XGBoostSPrintf_R(char *buf, size_t size, const char *fmt, ...);
-}
-
-// implements error handling
-namespace xgboost {
-namespace utils {
-extern "C" {
-  void (*Printf)(const char *fmt, ...) = Rprintf;
-  int (*SPrintf)(char *buf, size_t size, const char *fmt, ...) = XGBoostSPrintf_R;
-  void (*Assert)(int exp, const char *fmt, ...) = XGBoostAssert_R;
-  void (*Check)(int exp, const char *fmt, ...) = XGBoostCheck_R;
-  void (*Error)(const char *fmt, ...) = error;
-}
-}  // namespace utils
-
-namespace random {
-void Seed(unsigned seed) {
-  warning("parameter seed is ignored, please set random seed using set.seed");
-}
-double Uniform(void) {
-  return unif_rand();
-}
-double Normal(void) {
-  return norm_rand();
-}
-}  // namespace random
-}  // namespace xgboost
-
-// call before wrapper starts
-inline void _WrapperBegin(void) {
-  GetRNGstate();
-}
-// call after wrapper starts
-inline void _WrapperEnd(void) {
-  PutRNGstate();
-}
-
-extern "C" {
-  void _DMatrixFinalizer(SEXP ext) {    
-    if (R_ExternalPtrAddr(ext) == NULL) return;
-    XGDMatrixFree(R_ExternalPtrAddr(ext));
-    R_ClearExternalPtr(ext);
-  }
-  SEXP XGDMatrixCreateFromFile_R(SEXP fname, SEXP silent) {
-    _WrapperBegin();
-    void *handle = XGDMatrixCreateFromFile(CHAR(asChar(fname)), asInteger(silent));
-    SEXP ret = PROTECT(R_MakeExternalPtr(handle, R_NilValue, R_NilValue));
-    R_RegisterCFinalizerEx(ret, _DMatrixFinalizer, TRUE);
-    UNPROTECT(1);
-    _WrapperEnd();
-    return ret;
-  }
-  SEXP XGDMatrixCreateFromMat_R(SEXP mat, 
-                                SEXP missing) {
-    _WrapperBegin();
-    SEXP dim = getAttrib(mat, R_DimSymbol);
-    int nrow = INTEGER(dim)[0];
-    int ncol = INTEGER(dim)[1];    
-    double *din = REAL(mat);
-    std::vector<float> data(nrow * ncol);
-    #pragma omp parallel for schedule(static)
-    for (int i = 0; i < nrow; ++i) {
-      for (int j = 0; j < ncol; ++j) {
-        data[i * ncol +j] = din[i + nrow * j];
-      }
-    }
-    void *handle = XGDMatrixCreateFromMat(BeginPtr(data), nrow, ncol, asReal(missing));
-    SEXP ret = PROTECT(R_MakeExternalPtr(handle, R_NilValue, R_NilValue));
-    R_RegisterCFinalizerEx(ret, _DMatrixFinalizer, TRUE);
-    UNPROTECT(1);
-    _WrapperEnd();
-    return ret;    
-  }
-  SEXP XGDMatrixCreateFromCSC_R(SEXP indptr,
-                                SEXP indices,
-                                SEXP data) {
-    _WrapperBegin();
-    const int *p_indptr = INTEGER(indptr);
-    const int *p_indices = INTEGER(indices);
-    const double *p_data = REAL(data);
-    int nindptr = length(indptr);
-    int ndata = length(data);
-    std::vector<bst_ulong> col_ptr_(nindptr);
-    std::vector<unsigned> indices_(ndata);
-    std::vector<float> data_(ndata);
-
-    for (int i = 0; i < nindptr; ++i) {
-      col_ptr_[i] = static_cast<bst_ulong>(p_indptr[i]);
-    }
-    #pragma omp parallel for schedule(static)
-    for (int i = 0; i < ndata; ++i) {
-      indices_[i] = static_cast<unsigned>(p_indices[i]);
-      data_[i] = static_cast<float>(p_data[i]);
-    }
-    void *handle = XGDMatrixCreateFromCSC(BeginPtr(col_ptr_), BeginPtr(indices_),
-                                          BeginPtr(data_), nindptr, ndata);
-    SEXP ret = PROTECT(R_MakeExternalPtr(handle, R_NilValue, R_NilValue));
-    R_RegisterCFinalizerEx(ret, _DMatrixFinalizer, TRUE);
-    UNPROTECT(1);
-    _WrapperEnd();
-    return ret;
-  }
-  SEXP XGDMatrixSliceDMatrix_R(SEXP handle, SEXP idxset) {
-    _WrapperBegin();
-    int len = length(idxset);
-    std::vector<int> idxvec(len);
-    for (int i = 0; i < len; ++i) {
-      idxvec[i] = INTEGER(idxset)[i] - 1;
-    }
-    void *res = XGDMatrixSliceDMatrix(R_ExternalPtrAddr(handle), BeginPtr(idxvec), len);
-    SEXP ret = PROTECT(R_MakeExternalPtr(res, R_NilValue, R_NilValue));
-    R_RegisterCFinalizerEx(ret, _DMatrixFinalizer, TRUE);
-    UNPROTECT(1);
-    _WrapperEnd();
-    return ret;        
-  }
-  void XGDMatrixSaveBinary_R(SEXP handle, SEXP fname, SEXP silent) {
-    _WrapperBegin();
-    XGDMatrixSaveBinary(R_ExternalPtrAddr(handle),
-                        CHAR(asChar(fname)), asInteger(silent));
-    _WrapperEnd();
-  }
-  void XGDMatrixSetInfo_R(SEXP handle, SEXP field, SEXP array) {
-    _WrapperBegin();
-    int len = length(array);
-    const char *name = CHAR(asChar(field));
-    if (!strcmp("group", name)) {
-      std::vector<unsigned> vec(len);
-      #pragma omp parallel for schedule(static)      
-      for (int i = 0; i < len; ++i) {
-        vec[i] = static_cast<unsigned>(INTEGER(array)[i]);
-      }
-      XGDMatrixSetGroup(R_ExternalPtrAddr(handle), BeginPtr(vec), len);
-      _WrapperEnd();
-      return;
-    }
-    {
-      std::vector<float> vec(len);
-      #pragma omp parallel for schedule(static)
-      for (int i = 0; i < len; ++i) {
-        vec[i] = REAL(array)[i];
-      }
-      XGDMatrixSetFloatInfo(R_ExternalPtrAddr(handle), 
-                            CHAR(asChar(field)),
-                            BeginPtr(vec), len);
-    }
-    _WrapperEnd();
-  }
-  SEXP XGDMatrixGetInfo_R(SEXP handle, SEXP field) {
-    _WrapperBegin();
-    bst_ulong olen;
-    const float *res = XGDMatrixGetFloatInfo(R_ExternalPtrAddr(handle),
-                                             CHAR(asChar(field)), &olen);
-    SEXP ret = PROTECT(allocVector(REALSXP, olen));
-    for (size_t i = 0; i < olen; ++i) {
-      REAL(ret)[i] = res[i];
-    }
-    UNPROTECT(1);
-    _WrapperEnd();
-    return ret;
-  }
-  SEXP XGDMatrixNumRow_R(SEXP handle) {
-    bst_ulong nrow = XGDMatrixNumRow(R_ExternalPtrAddr(handle));
-    return ScalarInteger(static_cast<int>(nrow));
-  }
-  // functions related to booster
-  void _BoosterFinalizer(SEXP ext) {    
-    if (R_ExternalPtrAddr(ext) == NULL) return;
-    XGBoosterFree(R_ExternalPtrAddr(ext));
-    R_ClearExternalPtr(ext);
-  }
-  SEXP XGBoosterCreate_R(SEXP dmats) {
-    _WrapperBegin();
-    int len = length(dmats);
-    std::vector<void*> dvec;
-    for (int i = 0; i < len; ++i){
-      dvec.push_back(R_ExternalPtrAddr(VECTOR_ELT(dmats, i)));
-    }
-    void *handle = XGBoosterCreate(BeginPtr(dvec), dvec.size());
-    SEXP ret = PROTECT(R_MakeExternalPtr(handle, R_NilValue, R_NilValue));
-    R_RegisterCFinalizerEx(ret, _BoosterFinalizer, TRUE);
-    UNPROTECT(1);
-    _WrapperEnd();
-    return ret;
-  }
-  void XGBoosterSetParam_R(SEXP handle, SEXP name, SEXP val) {
-    _WrapperBegin();
-    XGBoosterSetParam(R_ExternalPtrAddr(handle),
-                      CHAR(asChar(name)),
-                      CHAR(asChar(val)));
-    _WrapperEnd();
-  }
-  void XGBoosterUpdateOneIter_R(SEXP handle, SEXP iter, SEXP dtrain) {
-    _WrapperBegin();
-    XGBoosterUpdateOneIter(R_ExternalPtrAddr(handle),
-                           asInteger(iter),
-                           R_ExternalPtrAddr(dtrain));
-    _WrapperEnd();
-  }
-  void XGBoosterBoostOneIter_R(SEXP handle, SEXP dtrain, SEXP grad, SEXP hess) {
-    _WrapperBegin();
-    utils::Check(length(grad) == length(hess), "gradient and hess must have same length");
-    int len = length(grad);
-    std::vector<float> tgrad(len), thess(len);
-    #pragma omp parallel for schedule(static)
-    for (int j = 0; j < len; ++j) {
-      tgrad[j] = REAL(grad)[j];
-      thess[j] = REAL(hess)[j];
-    }
-    XGBoosterBoostOneIter(R_ExternalPtrAddr(handle),
-                          R_ExternalPtrAddr(dtrain),
-                          BeginPtr(tgrad), BeginPtr(thess), len);
-    _WrapperEnd();
-  }
-  SEXP XGBoosterEvalOneIter_R(SEXP handle, SEXP iter, SEXP dmats, SEXP evnames) {
-    _WrapperBegin();
-    utils::Check(length(dmats) == length(evnames), "dmats and evnams must have same length");
-    int len = length(dmats);
-    std::vector<void*> vec_dmats;
-    std::vector<std::string> vec_names;
-    std::vector<const char*> vec_sptr;
-    for (int i = 0; i < len; ++i) {
-      vec_dmats.push_back(R_ExternalPtrAddr(VECTOR_ELT(dmats, i)));
-      vec_names.push_back(std::string(CHAR(asChar(VECTOR_ELT(evnames, i)))));
-    }
-    for (int i = 0; i < len; ++i) {
-      vec_sptr.push_back(vec_names[i].c_str());
-    }
-    return mkString(XGBoosterEvalOneIter(R_ExternalPtrAddr(handle),
-                                         asInteger(iter),
-                                         BeginPtr(vec_dmats), BeginPtr(vec_sptr), len));
-    _WrapperEnd();
-  }
-  SEXP XGBoosterPredict_R(SEXP handle, SEXP dmat, SEXP output_margin, SEXP ntree_limit) {
-    _WrapperBegin();
-    bst_ulong olen;
-    const float *res = XGBoosterPredict(R_ExternalPtrAddr(handle),
-                                        R_ExternalPtrAddr(dmat),
-                                        asInteger(output_margin),
-                                        asInteger(ntree_limit),
-                                        &olen);
-    SEXP ret = PROTECT(allocVector(REALSXP, olen));
-    for (size_t i = 0; i < olen; ++i) {
-      REAL(ret)[i] = res[i];
-    }
-    UNPROTECT(1);
-    _WrapperEnd();
-    return ret;
-  }
-  void XGBoosterLoadModel_R(SEXP handle, SEXP fname) {
-    _WrapperBegin();
-    XGBoosterLoadModel(R_ExternalPtrAddr(handle), CHAR(asChar(fname)));
-    _WrapperEnd();
-  }
-  void XGBoosterSaveModel_R(SEXP handle, SEXP fname) {
-    _WrapperBegin();
-    XGBoosterSaveModel(R_ExternalPtrAddr(handle), CHAR(asChar(fname)));
-    _WrapperEnd();
-  }
-  void XGBoosterDumpModel_R(SEXP handle, SEXP fname, SEXP fmap) {
-    _WrapperBegin();
-    bst_ulong olen;
-    const char **res = XGBoosterDumpModel(R_ExternalPtrAddr(handle),
-                                          CHAR(asChar(fmap)),
-                                          &olen);
-    FILE *fo = utils::FopenCheck(CHAR(asChar(fname)), "w");
-    for (size_t i = 0; i < olen; ++i) {
-      fprintf(fo, "booster[%u]:\n", static_cast<unsigned>(i));
-      fprintf(fo, "%s", res[i]);
-    }
-    fclose(fo);
-    _WrapperEnd();
-  }
-}

R-package/src/xgboost_R.h

@@ -1,138 +0,0 @@
-#ifndef XGBOOST_WRAPPER_R_H_
-#define XGBOOST_WRAPPER_R_H_
-/*!
- * \file xgboost_wrapper_R.h
- * \author Tianqi Chen
- * \brief R wrapper of xgboost
- */
-extern "C" {
-#include <Rinternals.h>
-#include <R_ext/Random.h>
-}
-
-extern "C" {
-  /*!
-   * \brief load a data matrix 
-   * \param fname name of the content
-   * \param silent whether print messages
-   * \return a loaded data matrix
-   */
-  SEXP XGDMatrixCreateFromFile_R(SEXP fname, SEXP silent);
-  /*!
-   * \brief create matrix content from dense matrix
-   * This assumes the matrix is stored in column major format
-   * \param data R Matrix object
-   * \param missing which value to represent missing value
-   * \return created dmatrix
-   */
-  SEXP XGDMatrixCreateFromMat_R(SEXP mat, 
-                                SEXP missing);
-  /*! 
-   * \brief create a matrix content from CSC format
-   * \param indptr pointer to column headers
-   * \param indices row indices
-   * \param data content of the data
-   * \return created dmatrix
-   */
-  SEXP XGDMatrixCreateFromCSC_R(SEXP indptr,
-                                SEXP indices,
-                                SEXP data);
-  /*!
-   * \brief create a new dmatrix from sliced content of existing matrix
-   * \param handle instance of data matrix to be sliced
-   * \param idxset index set
-   * \return a sliced new matrix
-   */
-  SEXP XGDMatrixSliceDMatrix_R(SEXP handle, SEXP idxset);
-  /*!
-   * \brief load a data matrix into binary file
-   * \param handle a instance of data matrix
-   * \param fname file name
-   * \param silent print statistics when saving
-   */
-  void XGDMatrixSaveBinary_R(SEXP handle, SEXP fname, SEXP silent);
-  /*!
-   * \brief set information to dmatrix
-   * \param handle a instance of data matrix
-   * \param field field name, can be label, weight
-   * \param array pointer to float vector
-   */
-  void XGDMatrixSetInfo_R(SEXP handle, SEXP field, SEXP array);
-  /*!
-   * \brief get info vector from matrix
-   * \param handle a instance of data matrix
-   * \param field field name
-   * \return info vector
-   */  
-  SEXP XGDMatrixGetInfo_R(SEXP handle, SEXP field);
-  /*!
-   * \brief return number of rows
-   * \param handle a instance of data matrix
-   */
-  SEXP XGDMatrixNumRow_R(SEXP handle);
-  /*! 
-   * \brief create xgboost learner 
-   * \param dmats a list of dmatrix handles that will be cached
-   */  
-  SEXP XGBoosterCreate_R(SEXP dmats);
-  /*! 
-   * \brief set parameters 
-   * \param handle handle
-   * \param name  parameter name
-   * \param val value of parameter
-   */
-  void XGBoosterSetParam_R(SEXP handle, SEXP name, SEXP val);
-  /*! 
-   * \brief update the model in one round using dtrain
-   * \param handle handle
-   * \param iter current iteration rounds
-   * \param dtrain training data
-   */
-  void XGBoosterUpdateOneIter_R(SEXP ext, SEXP iter, SEXP dtrain);
-  /*!
-   * \brief update the model, by directly specify gradient and second order gradient,
-   *        this can be used to replace UpdateOneIter, to support customized loss function
-   * \param handle handle
-   * \param dtrain training data
-   * \param grad gradient statistics
-   * \param hess second order gradient statistics
-   */
-  void XGBoosterBoostOneIter_R(SEXP handle, SEXP dtrain, SEXP grad, SEXP hess);
-  /*!
-   * \brief get evaluation statistics for xgboost
-   * \param handle handle
-   * \param iter current iteration rounds
-   * \param dmats list of handles to dmatrices
-   * \param evname name of evaluation
-   * \return the string containing evaluation stati
-   */
-  SEXP XGBoosterEvalOneIter_R(SEXP handle, SEXP iter, SEXP dmats, SEXP evnames);
-  /*!
-   * \brief make prediction based on dmat
-   * \param handle handle
-   * \param dmat data matrix
-   * \param output_margin whether only output raw margin value
-   * \param ntree_limit limit number of trees used in prediction
-   */
-  SEXP XGBoosterPredict_R(SEXP handle, SEXP dmat, SEXP output_margin, SEXP ntree_limit);
-  /*!
-   * \brief load model from existing file
-   * \param handle handle
-   * \param fname file name
-   */
-  void XGBoosterLoadModel_R(SEXP handle, SEXP fname);
-  /*!
-   * \brief save model into existing file
-   * \param handle handle
-   * \param fname file name
-   */    
-  void XGBoosterSaveModel_R(SEXP handle, SEXP fname);
-  /*!
-   * \brief dump model into text file 
-   * \param handle handle
-   * \param fname file name of model that can be dumped into
-   * \param fmap  name to fmap can be empty string
-   */
-  void XGBoosterDumpModel_R(SEXP handle, SEXP fname, SEXP fmap);
-}
-#endif  // XGBOOST_WRAPPER_R_H_

R-package/src/xgboost_assert.c

@@ -1,33 +0,0 @@
-#include <stdio.h>
-#include <stdarg.h>
-#include <Rinternals.h>
-
-// implements error handling
-void XGBoostAssert_R(int exp, const char *fmt, ...) {
-  char buf[1024];
-  if (exp == 0) {
-    va_list args;  
-    va_start(args, fmt);
-    vsprintf(buf, fmt, args);
-    va_end(args);
-    error("AssertError:%s\n", buf);
-  }  
-}
-void XGBoostCheck_R(int exp, const char *fmt, ...) {
-  char buf[1024];
-  if (exp == 0) {
-    va_list args;  
-    va_start(args, fmt);
-    vsprintf(buf, fmt, args);
-    va_end(args);
-    error("%s\n", buf);
-  }
-}
-int XGBoostSPrintf_R(char *buf, size_t size, const char *fmt, ...) {
-  int ret;
-  va_list args;  
-  va_start(args, fmt);
-  ret = vsnprintf(buf, size, fmt, args);
-  va_end(args);
-  return ret;
-}

R-package/vignettes/xgboost.Rnw

@@ -1,216 +0,0 @@
-\documentclass{article}
-\RequirePackage{url}
-\usepackage{hyperref}
-\RequirePackage{amsmath}
-\RequirePackage{natbib}
-\RequirePackage[a4paper,lmargin={1.25in},rmargin={1.25in},tmargin={1in},bmargin={1in}]{geometry}
-
-\makeatletter
-% \VignetteIndexEntry{xgboost: eXtreme Gradient Boosting}
-%\VignetteKeywords{xgboost, gbm, gradient boosting machines}
-%\VignettePackage{xgboost}
-% \VignetteEngine{knitr::knitr}
-\makeatother
-
-\begin{document}
-%\SweaveOpts{concordance=TRUE}
-
-<<knitropts,echo=FALSE,message=FALSE>>=
-if (require('knitr')) opts_chunk$set(fig.width = 5, fig.height = 5, fig.align = 'center', tidy = FALSE, warning = FALSE, cache = TRUE)
-@
-
-%
-<<prelim,echo=FALSE>>=
-xgboost.version = '0.3-0'
-@
-%
-
-    \begin{center}
-    \vspace*{6\baselineskip}
-    \rule{\textwidth}{1.6pt}\vspace*{-\baselineskip}\vspace*{2pt}
-    \rule{\textwidth}{0.4pt}\\[2\baselineskip]
-    {\LARGE \textbf{xgboost: eXtreme Gradient Boosting}}\\[1.2\baselineskip]
-    \rule{\textwidth}{0.4pt}\vspace*{-\baselineskip}\vspace{3.2pt}
-    \rule{\textwidth}{1.6pt}\\[2\baselineskip]
-    {\Large Tianqi Chen, Tong He}\\[\baselineskip]
-    {\large Package Version: \Sexpr{xgboost.version}}\\[\baselineskip]
-    {\large \today}\par
-    \vfill
-    \end{center}
-
-\thispagestyle{empty}
-
-\clearpage
-
-\setcounter{page}{1}
-
-\section{Introduction}
-
-This is an introductory document of using the \verb@xgboost@ package in R. 
-
-\verb@xgboost@ is short for eXtreme Gradient Boosting package. It is an efficient
- and scalable implementation of gradient boosting framework by \citep{friedman2001greedy}. 
-The package includes efficient linear model solver and tree learning algorithm.
-It supports various objective functions, including regression, classification
-and ranking. The package is made to be extendible, so that users are also allowed to define their own objectives easily. It has several features:
-\begin{enumerate}
-    \item{Speed: }{\verb@xgboost@ can automatically do parallel computation on 
-    Windows and Linux, with openmp. It is generally over 10 times faster than
-    \verb@gbm@.}
-    \item{Input Type: }{\verb@xgboost@ takes several types of input data:}
-    \begin{itemize}
-        \item{Dense Matrix: }{R's dense matrix, i.e. \verb@matrix@}
-        \item{Sparse Matrix: }{R's sparse matrix \verb@Matrix::dgCMatrix@}
-        \item{Data File: }{Local data files}
-        \item{xgb.DMatrix: }{\verb@xgboost@'s own class. Recommended.}
-    \end{itemize}
-    \item{Sparsity: }{\verb@xgboost@ accepts sparse input for both tree booster 
-    and linear booster, and is optimized for sparse input.}
-    \item{Customization: }{\verb@xgboost@ supports customized objective function 
-    and evaluation function}
-    \item{Performance: }{\verb@xgboost@ has better performance on several different
-    datasets.}
-\end{enumerate}
-
-
-\section{Example with Mushroom data}
-
-In this section, we will illustrate some common usage of \verb@xgboost@. The 
-Mushroom data is cited from UCI Machine Learning Repository. \citep{Bache+Lichman:2013} 
-
-<<Training and prediction with iris>>=
-library(xgboost)
-data(agaricus.train, package='xgboost')
-data(agaricus.test, package='xgboost')
-train <- agaricus.train
-test <- agaricus.test
-bst <- xgboost(data = train$data, label = train$label, max.depth = 2, eta = 1, 
-               nround = 2, objective = "binary:logistic")
-xgb.save(bst, 'model.save')
-bst = xgb.load('model.save')
-pred <- predict(bst, test$data)
-@
-
-\verb@xgboost@ is the main function to train a \verb@Booster@, i.e. a model.
-\verb@predict@ does prediction on the model.
-
-Here we can save the model to a binary local file, and load it when needed.
-We can't inspect the trees inside. However we have another function to save the
-model in plain text. 
-<<Dump Model>>=
-xgb.dump(bst, 'model.dump')
-@
-
-The output looks like 
-
-\begin{verbatim}
-booster[0]:
-0:[f28<1.00001] yes=1,no=2,missing=2
-  1:[f108<1.00001] yes=3,no=4,missing=4
-    3:leaf=1.85965
-    4:leaf=-1.94071
-  2:[f55<1.00001] yes=5,no=6,missing=6
-    5:leaf=-1.70044
-    6:leaf=1.71218
-booster[1]:
-0:[f59<1.00001] yes=1,no=2,missing=2
-  1:leaf=-6.23624
-  2:[f28<1.00001] yes=3,no=4,missing=4
-    3:leaf=-0.96853
-    4:leaf=0.784718
-\end{verbatim}
-
-It is important to know \verb@xgboost@'s own data type: \verb@xgb.DMatrix@.
-It speeds up \verb@xgboost@, and is needed for advanced features such as 
-training from initial prediction value, weighted training instance. 
-
-We can use \verb@xgb.DMatrix@ to construct an \verb@xgb.DMatrix@ object:
-<<xgb.DMatrix>>=
-dtrain <- xgb.DMatrix(train$data, label = train$label)
-class(dtrain)
-head(getinfo(dtrain,'label'))
-@
-
-We can also save the matrix to a binary file. Then load it simply with 
-\verb@xgb.DMatrix@
-<<save model>>=
-xgb.DMatrix.save(dtrain, 'xgb.DMatrix')
-dtrain = xgb.DMatrix('xgb.DMatrix')
-@
-
-\section{Advanced Examples}
-
-The function \verb@xgboost@ is a simple function with less parameter, in order
-to be R-friendly. The core training function is wrapped in \verb@xgb.train@. It is more flexible than \verb@xgboost@, but it requires users to read the document a bit more carefully.
-
-\verb@xgb.train@ only accept a \verb@xgb.DMatrix@ object as its input, while it supports advanced features as custom objective and evaluation functions.
-
-<<Customized loss function>>=
-logregobj <- function(preds, dtrain) {
-   labels <- getinfo(dtrain, "label")
-   preds <- 1/(1 + exp(-preds))
-   grad <- preds - labels
-   hess <- preds * (1 - preds)
-   return(list(grad = grad, hess = hess))
-}
-
-evalerror <- function(preds, dtrain) {
-  labels <- getinfo(dtrain, "label")
-  err <- sqrt(mean((preds-labels)^2))
-  return(list(metric = "MSE", value = err))
-}
-
-dtest <- xgb.DMatrix(test$data, label = test$label)
-watchlist <- list(eval = dtest, train = dtrain)
-param <- list(max.depth = 2, eta = 1, silent = 1)
-
-bst <- xgb.train(param, dtrain, nround = 2, watchlist, logregobj, evalerror)
-@
-
-The gradient and second order gradient is required for the output of customized 
-objective function. 
-
-We also have \verb@slice@ for row extraction. It is useful in 
-cross-validation.
-
-For a walkthrough demo, please see \verb@R-package/demo/@ for further 
-details.
-
-\section{The Higgs Boson competition}
-
-We have made a demo for \href{http://www.kaggle.com/c/higgs-boson}{the Higgs 
-Boson Machine Learning Challenge}. 
-
-Here are the instructions to make a submission
-\begin{enumerate}
-    \item Download the \href{http://www.kaggle.com/c/higgs-boson/data}{datasets}
-    and extract them to \verb@data/@.
-    \item Run scripts under \verb@xgboost/demo/kaggle-higgs/@: 
-    \href{https://github.com/tqchen/xgboost/blob/master/demo/kaggle-higgs/higgs-train.R}{higgs-train.R} 
-    and \href{https://github.com/tqchen/xgboost/blob/master/demo/kaggle-higgs/higgs-pred.R}{higgs-pred.R}. 
-    The computation will take less than a minute on Intel i7. 
-    \item Go to the \href{http://www.kaggle.com/c/higgs-boson/submissions/attach}{submission page} 
-    and submit your result.
-\end{enumerate}
-
-We provide \href{https://github.com/tqchen/xgboost/blob/master/demo/kaggle-higgs/speedtest.R}{a script}
-to compare the time cost on the higgs dataset with \verb@gbm@ and \verb@xgboost@. 
-The training set contains 350000 records and 30 features. 
-
-\verb@xgboost@ can automatically do parallel computation. On a machine with Intel
-i7-4700MQ and 24GB memories, we found that \verb@xgboost@ costs about 35 seconds, which is about 20 times faster
-than \verb@gbm@. When we limited \verb@xgboost@ to use only one thread, it was 
-still about two times faster than \verb@gbm@. 
-
-Meanwhile, the result from \verb@xgboost@ reaches 
-\href{http://www.kaggle.com/c/higgs-boson/details/evaluation}{3.60@AMS} with a 
-single model. This results stands in the 
-\href{http://www.kaggle.com/c/higgs-boson/leaderboard}{top 30\%} of the 
-competition. 
-
-\bibliographystyle{jss}
-\nocite{*} % list uncited references
-\bibliography{xgboost}
-
-\end{document}
-

R-package/vignettes/xgboost.bib

@@ -1,30 +0,0 @@
-@article{friedman2001greedy,
-    title={Greedy function approximation: a gradient boosting machine},
-    author={Friedman, Jerome H},
-    journal={Annals of Statistics},
-    pages={1189--1232},
-    year={2001},
-    publisher={JSTOR}
-}
-
-@article{friedman2000additive,
-  title={Additive logistic regression: a statistical view of boosting (with discussion and a rejoinder by the authors)},
-  author={Friedman, Jerome and Hastie, Trevor and Tibshirani, Robert and others},
-  journal={The annals of statistics},
-  volume={28},
-  number={2},
-  pages={337--407},
-  year={2000},
-  publisher={Institute of Mathematical Statistics}
-}
-
-
-@misc{
-    Bache+Lichman:2013 ,
-    author = "K. Bache and M. Lichman",
-    year = "2013",
-    title = "{UCI} Machine Learning Repository",
-    url = "http://archive.ics.uci.edu/ml",
-    institution = "University of California, Irvine, School of Information and Computer Sciences" 
-}
-

README.md

@@ -1,24 +1,13 @@
 xgboost: eXtreme Gradient Boosting 
-======
+=======
-An optimized general purpose gradient boosting library. The library is parallelized using OpenMP. It implements machine learning algorithm under gradient boosting framework, including generalized linear model and gradient boosted regression tree. 
+An optimized general purpose gradient boosting (tree) library.
 
 Contributors: https://github.com/tqchen/xgboost/graphs/contributors
 
 Turorial and Documentation: https://github.com/tqchen/xgboost/wiki
  
-Questions and Issues: [https://github.com/tqchen/xgboost/issues](https://github.com/tqchen/xgboost/issues?q=is%3Aissue+label%3Aquestion)
-
-Examples Code: [Learning to use xgboost by examples](demo)
-
-Notes on the Code: [Code Guide](src)
-
-What's New
-=====
-* See the updated [demo folder](demo) for feature walkthrough
-* Thanks to Tong He, the new [R package](R-package) is available
-
 Features
-======
+=======
 * Sparse feature format:
   - Sparse feature format allows easy handling of missing values, and improve computation efficiency.
 * Push the limit on single machine:
@@ -28,25 +17,31 @@ Features
 * Layout of gradient boosting algorithm to support user defined objective
 * Python interface, works with numpy and scipy.sparse matrix
 
+Supported key components
+=======
+* Gradient boosting models: 
+    - regression tree (GBRT)
+    - linear model/lasso
+* Objectives to support tasks: 
+    - regression
+    - classification
+* OpenMP implementation
+
+Planned components
+=======
+* More objective to support tasks: 
+    - ranking
+    - matrix factorization
+    - structured prediction
+
 Build
-=====
+======
-* Run ```bash build.sh``` (you can also type make)
+* Simply type make
 * If your compiler does not come with OpenMP support, it will fire an warning telling you that the code will compile into single thread mode, and you will get single thread xgboost
-* You may get a error: -lgomp is not found
+  - You may get a error: -lgomp is not found, you can remove -fopenmp flag in Makefile to get single thread xgboost, or upgrade your compiler to compile multi-thread version
-  - You can type ```make no_omp=1```, this will get you single thread xgboost
-  - Alternatively, you can upgrade your compiler to compile multi-thread version
-* Windows(VS 2010): see [windows](windows) folder
-  - In principle, you put all the cpp files in the Makefile to the project, and build
 
-Version
+File extension convention
-======
+=======
-* This version xgboost-0.3, the code has been refactored from 0.2x to be cleaner and more flexibility
+* .h are interface, utils and data structures, with detailed comment; 
-* This version of xgboost is not compatible with 0.2x, due to huge amount of changes in code structure
+* .cpp are implementations that will be compiled, with less comment; 
-  - This means the model and buffer file of previous version can not be loaded in xgboost-3.0
+* .hpp are implementations that will be included by .cpp, with less comment
-* For legacy 0.2x code, refer to [Here](https://github.com/tqchen/xgboost/releases/tag/v0.22)
-* Change log in [CHANGES.md](CHANGES.md)
-
-XGBoost in Graphlab Create
-======
-* XGBoost is adopted as part of boosted tree toolkit in Graphlab Create (GLC). Graphlab Create is a powerful python toolkit that allows you to data manipulation, graph processing, hyper-parameter search, and visualization of TeraBytes scale data in one framework. Try the Graphlab Create in http://graphlab.com/products/create/quick-start-guide.html
-* Nice blogpost by Jay Gu using GLC boosted tree to solve kaggle bike sharing challenge: http://blog.graphlab.com/using-gradient-boosted-trees-to-predict-bike-sharing-demand

booster/linear/xgboost_linear.hpp

@@ -0,0 +1,200 @@
+#ifndef XGBOOST_LINEAR_HPP
+#define XGBOOST_LINEAR_HPP
+/*!
+ * \file xgboost_linear.h
+ * \brief Implementation of Linear booster, with L1/L2 regularization: Elastic Net
+ *        the update rule is coordinate descent, require column major format
+ * \author Tianqi Chen: tianqi.tchen@gmail.com
+ */
+#include <vector>
+#include <algorithm>
+
+#include "../xgboost.h"
+#include "../../utils/xgboost_utils.h"
+
+namespace xgboost{
+    namespace booster{
+        /*! \brief linear model, with L1/L2 regularization */
+        template<typename FMatrix>
+        class LinearBooster : public InterfaceBooster<FMatrix>{
+        public:
+            LinearBooster( void ){ silent = 0;}
+            virtual ~LinearBooster( void ){}
+        public:
+            virtual void SetParam( const char *name, const char *val ){
+                if( !strcmp( name, "silent") )  silent = atoi( val );
+                if( model.weight.size() == 0 )  model.param.SetParam( name, val );
+                param.SetParam( name, val );
+            }
+            virtual void LoadModel( utils::IStream &fi ){
+                model.LoadModel( fi );
+            }
+            virtual void SaveModel( utils::IStream &fo ) const{
+                model.SaveModel( fo );
+            }
+            virtual void InitModel( void ){
+                model.InitModel();
+            }
+        public:
+            virtual void DoBoost( std::vector<float> &grad, 
+                                  std::vector<float> &hess,
+                                  const FMatrix &fmat,
+                                  const std::vector<unsigned> &root_index ){
+                utils::Assert( grad.size() < UINT_MAX, "number of instance exceed what we can handle" );
+                this->UpdateWeights( grad, hess, fmat );
+            }
+            inline float Predict( const FMatrix &fmat, bst_uint ridx, unsigned root_index ){
+                float sum = model.bias();
+                for( typename FMatrix::RowIter it = fmat.GetRow(ridx); it.Next(); ){ 
+                    sum += model.weight[ it.findex() ] * it.fvalue();
+                }
+                return sum;
+            }
+            virtual float Predict( const std::vector<float> &feat, 
+                                   const std::vector<bool>  &funknown,
+                                   unsigned rid = 0 ){
+                float sum = model.bias();
+                for( size_t i = 0; i < feat.size(); i ++ ){
+                    if( funknown[i] ) continue;
+                    sum += model.weight[ i ] * feat[ i ];
+                }
+                return sum;
+            }
+            
+        protected:
+            // training parameter
+            struct ParamTrain{
+                /*! \brief learning_rate */
+                float learning_rate;
+                /*! \brief regularization weight for L2 norm */
+                float reg_lambda;
+                /*! \brief regularization weight for L1 norm */
+                float reg_alpha;
+                 /*! \brief regularization weight for L2 norm  in bias */               
+                float reg_lambda_bias;
+                
+                ParamTrain( void ){
+                    reg_alpha = 0.0f; reg_lambda = 0.0f; reg_lambda_bias = 0.0f;
+                    learning_rate = 1.0f;
+                }            
+                inline void SetParam( const char *name, const char *val ){
+                    // sync-names
+                    if( !strcmp( "eta", name ) )    learning_rate = (float)atof( val );
+                    if( !strcmp( "lambda", name ) ) reg_lambda = (float)atof( val );
+                    if( !strcmp( "alpha", name ) )  reg_alpha  = (float)atof( val );
+                    if( !strcmp( "lambda_bias", name ) ) reg_lambda_bias = (float)atof( val );
+                    // real names
+                    if( !strcmp( "learning_rate", name ) ) learning_rate = (float)atof( val );     
+                    if( !strcmp( "reg_lambda", name ) )    reg_lambda = (float)atof( val );
+                    if( !strcmp( "reg_alpha", name ) )     reg_alpha = (float)atof( val );
+                    if( !strcmp( "reg_lambda_bias", name ) )    reg_lambda_bias = (float)atof( val );
+                }
+                // given original weight calculate delta 
+                inline double CalcDelta( double sum_grad, double sum_hess, double w ){
+                    if( sum_hess < 1e-5f ) return 0.0f;
+                    double tmp = w - ( sum_grad + reg_lambda*w )/( sum_hess + reg_lambda );
+                    if ( tmp >=0 ){
+                        return std::max(-( sum_grad + reg_lambda*w + reg_alpha)/(sum_hess+reg_lambda),-w);
+                    }else{
+                        return std::min(-( sum_grad + reg_lambda*w - reg_alpha)/(sum_hess+reg_lambda),-w);
+                    }
+                }
+                // given original weight calculate delta bias
+                inline double CalcDeltaBias( double sum_grad, double sum_hess, double w ){
+                    return - (sum_grad + reg_lambda_bias*w) / (sum_hess + reg_lambda_bias );
+                }
+            };
+            
+            // model for linear booster
+            class Model{
+            public:
+                // model parameter
+                struct Param{
+                    // number of feature dimension
+                    int num_feature;
+                    // reserved field
+                    int reserved[ 32 ];
+                    // constructor
+                    Param( void ){
+                        num_feature = 0;
+                        memset( reserved, 0, sizeof(reserved) );
+                    }
+                    inline void SetParam( const char *name, const char *val ){
+                        if( !strcmp( name, "num_feature" ) )  num_feature = atoi( val );
+                    }
+                };
+            public:
+                Param param;
+                // weight for each of feature, bias is the last one
+                std::vector<float> weight;
+            public:
+                // initialize the model parameter
+                inline void InitModel( void ){
+                    // bias is the last weight
+                    weight.resize( param.num_feature + 1 );
+                    std::fill( weight.begin(), weight.end(), 0.0f );
+                }
+                // save the model to file 
+                inline void SaveModel( utils::IStream &fo ) const{
+                    fo.Write( &param, sizeof(Param) );
+                    fo.Write( &weight[0], sizeof(float) * weight.size() );
+                }
+                // load model from file
+                inline void LoadModel( utils::IStream &fi ){
+                    utils::Assert( fi.Read( &param, sizeof(Param) ) != 0, "Load LinearBooster" );
+                    weight.resize( param.num_feature + 1 );
+                    utils::Assert( fi.Read( &weight[0], sizeof(float) * weight.size() ) != 0, "Load LinearBooster" );
+                }
+                // model bias
+                inline float &bias( void ){
+                    return weight.back();
+                }
+            };
+        private:
+            int silent;
+        protected:
+            Model model;
+            ParamTrain param;
+        protected:
+            // update weights, should work for any FMatrix
+            inline void UpdateWeights( std::vector<float> &grad,                       
+                                       const std::vector<float> &hess,
+                                       const FMatrix &smat ){
+                {// optimize bias
+                    double sum_grad = 0.0, sum_hess = 0.0;
+                    for( size_t i = 0; i < grad.size(); i ++ ){
+                        sum_grad += grad[ i ]; sum_hess += hess[ i ];
+                    }
+                    // remove bias effect
+                    double dw = param.learning_rate * param.CalcDeltaBias( sum_grad, sum_hess, model.bias() );
+                    model.bias() += dw;
+                    // update grad value 
+                    for( size_t i = 0; i < grad.size(); i ++ ){
+                        grad[ i ] += dw * hess[ i ];
+                    }
+                }
+
+                // optimize weight
+                const unsigned nfeat= (unsigned)smat.NumCol();                           
+                for( unsigned i = 0; i < nfeat; i ++ ){
+                    if( !smat.GetSortedCol( i ).Next() ) continue;
+                    double sum_grad = 0.0, sum_hess = 0.0;
+                    for( typename FMatrix::ColIter it = smat.GetSortedCol(i); it.Next(); ){
+                        const float v = it.fvalue();
+                        sum_grad += grad[ it.rindex() ] * v;
+                        sum_hess += hess[ it.rindex() ] * v * v;
+                    }
+                    float w = model.weight[ i ];
+                    double dw = param.learning_rate * param.CalcDelta( sum_grad, sum_hess, w );
+                    model.weight[ i ] += dw;
+                    // update grad value 
+                    for( typename FMatrix::ColIter it = smat.GetSortedCol(i); it.Next(); ){
+                        const float v = it.fvalue();
+                        grad[ it.rindex() ] += hess[ it.rindex() ] * v * dw;
+                    }
+                }
+            }
+        };
+    };
+};
+#endif

booster/tree/xgboost_base_treemaker.hpp

@@ -0,0 +1,147 @@
+#ifndef XGBOOST_BASE_TREEMAKER_HPP
+#define XGBOOST_BASE_TREEMAKER_HPP
+/*!
+ * \file xgboost_base_treemaker.hpp
+ * \brief implementation of base data structure for regression tree maker,
+ *         gives common operations of tree construction steps template 
+ * 
+ * \author Tianqi Chen: tianqi.tchen@gmail.com 
+ */
+#include <vector>
+#include "xgboost_tree_model.h"
+
+namespace xgboost{
+    namespace booster{
+        class BaseTreeMaker{
+        protected:
+            BaseTreeMaker( RegTree &tree,
+                           const TreeParamTrain &param )
+                : tree( tree ), param( param ){}
+        protected:
+            // statistics that is helpful to decide a split
+            struct SplitEntry{
+                /*! \brief loss change after split this node */
+                float  loss_chg;
+                /*! \brief split index */
+                unsigned  sindex;
+                /*! \brief split value */
+                float     split_value;
+                /*! \brief constructor */
+                SplitEntry( void ){
+                    loss_chg = 0.0f;
+                    split_value = 0.0f; sindex = 0;
+                }
+                // This function gives better priority to lower index when loss_chg equals
+                // not the best way, but helps to give consistent result during multi-thread execution
+                inline bool NeedReplace( float loss_chg, unsigned split_index ) const{
+                    if( this->split_index() <= split_index ){
+                        return loss_chg > this->loss_chg; 
+                    }else{
+                        return !(this->loss_chg > loss_chg);
+                    }
+                }
+                inline bool Update( const SplitEntry &e ){
+                    if( this->NeedReplace( e.loss_chg, e.split_index() ) ){
+                        this->loss_chg = e.loss_chg;
+                        this->sindex = e.sindex;
+                        this->split_value = e.split_value;
+                        return true;
+                    } else{
+                        return false;
+                    }
+                }
+                inline bool Update( float loss_chg, unsigned split_index, float split_value, bool default_left ){                    
+                    if( this->NeedReplace( loss_chg, split_index ) ){
+                        this->loss_chg = loss_chg;
+                        if( default_left ) split_index |= (1U << 31);
+                        this->sindex = split_index;
+                        this->split_value = split_value;
+                        return true;
+                    }else{
+                        return false;
+                    }
+                }
+                inline unsigned split_index( void ) const{
+                    return sindex & ( (1U<<31) - 1U );
+                }
+                inline bool default_left( void ) const{
+                    return (sindex >> 31) != 0;
+                }
+            };
+            struct NodeEntry{
+                /*! \brief sum gradient statistics */
+                double sum_grad;
+                /*! \brief sum hessian statistics */
+                double sum_hess;
+                /*! \brief loss of this node, without split */
+                float  root_gain;
+                /*! \brief weight calculated related to current data */
+                float  weight;
+                /*! \brief current best solution */
+                SplitEntry best;
+                NodeEntry( void ){
+                    sum_grad = sum_hess = 0.0;
+                    weight = root_gain = 0.0f;
+                }
+            };
+        private:
+            // try to prune off current leaf, return true if successful
+            inline void TryPruneLeaf( int nid, int depth ){
+                if( tree[ nid ].is_root() ) return;
+                int pid = tree[ nid ].parent();
+                RegTree::NodeStat &s = tree.stat( pid );
+                ++ s.leaf_child_cnt;
+                
+                if( s.leaf_child_cnt >= 2 && param.need_prune( s.loss_chg, depth - 1 ) ){
+                    this->stat_num_pruned += 2;
+                    // need to be pruned
+                    tree.ChangeToLeaf( pid, param.learning_rate * s.base_weight );
+                    // tail recursion
+                    this->TryPruneLeaf( pid, depth - 1 );
+                }
+            }
+        protected:
+            /*! \brief do prunning of a tree */
+            inline int DoPrune( void ){
+                this->stat_num_pruned = 0;
+                // initialize auxiliary statistics
+                for( int nid = 0; nid < tree.param.num_nodes; ++ nid ){
+                    tree.stat( nid ).leaf_child_cnt = 0;
+                    tree.stat( nid ).loss_chg = snode[ nid ].best.loss_chg;
+                    tree.stat( nid ).sum_hess = static_cast<float>( snode[ nid ].sum_hess );
+                }
+                for( int nid = 0; nid < tree.param.num_nodes; ++ nid ){
+                    if( tree[ nid ].is_leaf() ) this->TryPruneLeaf( nid, tree.GetDepth(nid) );
+                }
+                return this->stat_num_pruned;
+            }
+        protected:
+            /*! \brief update queue expand add in new leaves */
+            inline void UpdateQueueExpand( std::vector<int> &qexpand ){
+                std::vector<int> newnodes;
+                for( size_t i = 0; i < qexpand.size(); ++ i ){
+                    const int nid = qexpand[i];
+                    if( !tree[ nid ].is_leaf() ){
+                        newnodes.push_back( tree[nid].cleft() );
+                        newnodes.push_back( tree[nid].cright() );
+                    }
+                }
+                // use new nodes for qexpand
+                qexpand = newnodes;
+            }
+        protected:
+            // local helper tmp data structure
+            // statistics
+            int stat_num_pruned;
+            /*! \brief queue of nodes to be expanded */
+            std::vector<int> qexpand;
+            /*! \brief TreeNode Data: statistics for each constructed node, the derived class must maintain this */
+            std::vector<NodeEntry> snode;
+        protected:
+            // original data that supports tree construction
+            RegTree &tree;
+            const TreeParamTrain &param;
+        };
+    }; // namespace booster
+}; // namespace xgboost
+#endif // XGBOOST_BASE_TREEMAKER_HPP

booster/tree/xgboost_col_treemaker.hpp

@@ -0,0 +1,335 @@
+#ifndef XGBOOST_COL_TREEMAKER_HPP
+#define XGBOOST_COL_TREEMAKER_HPP
+/*!
+ * \file xgboost_col_treemaker.hpp
+ * \brief implementation of regression tree maker,
+ *        use a column based approach, with OpenMP 
+ * \author Tianqi Chen: tianqi.tchen@gmail.com 
+ */
+// use openmp
+#include <vector>
+#include "xgboost_tree_model.h"
+#include "../../utils/xgboost_omp.h"
+#include "../../utils/xgboost_random.h"
+#include "../../utils/xgboost_fmap.h"
+#include "xgboost_base_treemaker.hpp"
+
+namespace xgboost{
+    namespace booster{
+        template<typename FMatrix>
+        class ColTreeMaker : protected BaseTreeMaker{
+        public:
+            ColTreeMaker( RegTree &tree,
+                          const TreeParamTrain &param, 
+                          const std::vector<float> &grad,
+                          const std::vector<float> &hess,
+                          const FMatrix &smat, 
+                          const std::vector<unsigned> &root_index, 
+                          const utils::FeatConstrain  &constrain )
+                : BaseTreeMaker( tree, param ), 
+                  grad(grad), hess(hess), 
+                  smat(smat), root_index(root_index), constrain(constrain) {
+                utils::Assert( grad.size() == hess.size(), "booster:invalid input" );
+                utils::Assert( smat.NumRow() == hess.size(), "booster:invalid input" );
+                utils::Assert( root_index.size() == 0 || root_index.size() == hess.size(), "booster:invalid input" );                
+                utils::Assert( smat.HaveColAccess(), "ColTreeMaker: need column access matrix" );
+            }
+            inline void Make( int& stat_max_depth, int& stat_num_pruned ){
+                this->InitData();
+                this->InitNewNode( this->qexpand );
+                stat_max_depth = 0;
+                
+                for( int depth = 0; depth < param.max_depth; ++ depth ){
+                    this->FindSplit( depth );
+                    this->UpdateQueueExpand( this->qexpand );
+                    this->InitNewNode( this->qexpand );
+                    // if nothing left to be expand, break
+                    if( qexpand.size() == 0 ) break;
+                    stat_max_depth = depth + 1;
+                }
+                // set all the rest expanding nodes to leaf
+                for( size_t i = 0; i < qexpand.size(); ++ i ){
+                    const int nid = qexpand[i];
+                    tree[ nid ].set_leaf( snode[nid].weight * param.learning_rate );                        
+                }
+                // start prunning the tree
+                stat_num_pruned = this->DoPrune();
+            }
+        private:
+            /*! \brief per thread x per node entry to store tmp data */
+            struct ThreadEntry{
+                /*! \brief sum gradient statistics */
+                double sum_grad;
+                /*! \brief sum hessian statistics */
+                double sum_hess;
+                /*! \brief last feature value scanned */
+                float  last_fvalue;
+                /*! \brief current best solution */
+                SplitEntry best;
+                /*! \brief constructor */
+                ThreadEntry( void ){                    
+                    this->ClearStats();
+                }
+                /*! \brief clear statistics */
+                inline void ClearStats( void ){
+                    sum_grad = sum_hess = 0.0;
+                }
+            };
+        private:
+            // make leaf nodes for all qexpand, update node statistics, mark leaf value
+            inline void InitNewNode( const std::vector<int> &qexpand ){
+                {// setup statistics space for each tree node
+                   for( size_t i = 0; i < stemp.size(); ++ i ){
+                        stemp[i].resize( tree.param.num_nodes, ThreadEntry() );
+                   }
+                    snode.resize( tree.param.num_nodes, NodeEntry() );
+                }
+
+                const unsigned ndata = static_cast<unsigned>( position.size() );
+                
+                #pragma omp parallel for schedule( static )
+                for( unsigned i = 0; i < ndata; ++ i ){
+                    const int tid = omp_get_thread_num();
+                    if( position[i] < 0 ) continue; 
+                    stemp[tid][ position[i] ].sum_grad += grad[i];
+                    stemp[tid][ position[i] ].sum_hess += hess[i];
+                }
+
+                for( size_t j = 0; j < qexpand.size(); ++ j ){
+                    const int nid = qexpand[ j ];
+                    double sum_grad = 0.0, sum_hess = 0.0;
+                    for( size_t tid = 0; tid < stemp.size(); tid ++ ){
+                        sum_grad += stemp[tid][nid].sum_grad;
+                        sum_hess += stemp[tid][nid].sum_hess;
+                    }
+                    // update node statistics
+                    snode[nid].sum_grad = sum_grad; 
+                    snode[nid].sum_hess = sum_hess;
+                    snode[nid].root_gain = param.CalcRootGain( sum_grad, sum_hess );
+                    if( !tree[nid].is_root() ){
+                        snode[nid].weight = param.CalcWeight( sum_grad, sum_hess, tree.stat( tree[nid].parent() ).base_weight );
+                        tree.stat(nid).base_weight = snode[nid].weight;
+                    }else{
+                        snode[nid].weight = param.CalcWeight( sum_grad, sum_hess, 0.0f );
+                        tree.stat(nid).base_weight = snode[nid].weight;
+                    }
+                }
+            }
+        private:
+            // enumerate the split values of specific feature
+            template<typename Iter>
+            inline void EnumerateSplit( Iter it, const unsigned fid, std::vector<ThreadEntry> &temp, bool is_forward_search ){
+                // clear all the temp statistics
+                for( size_t j = 0; j < qexpand.size(); ++ j ){
+                    temp[ qexpand[j] ].ClearStats();
+                }
+                
+                while( it.Next() ){
+                    const bst_uint ridx = it.rindex();
+                    const int nid = position[ ridx ];
+                    if( nid < 0 ) continue;
+
+                    const float fvalue = it.fvalue();           
+                    ThreadEntry &e = temp[ nid ];
+
+                    // test if first hit, this is fine, because we set 0 during init
+                    if( e.sum_hess == 0.0 ){
+                        e.sum_grad = grad[ ridx ];
+                        e.sum_hess = hess[ ridx ];
+                        e.last_fvalue = fvalue;
+                    }else{
+                        // try to find a split
+                        if( fabsf(fvalue - e.last_fvalue) > rt_2eps && e.sum_hess >= param.min_child_weight ){
+                            const double csum_hess = snode[ nid ].sum_hess - e.sum_hess;
+                            if( csum_hess >= param.min_child_weight ){
+                                const double csum_grad = snode[nid].sum_grad - e.sum_grad; 
+                                const double loss_chg = 
+                                    + param.CalcGain( e.sum_grad, e.sum_hess, snode[nid].weight ) 
+                                    + param.CalcGain( csum_grad , csum_hess , snode[nid].weight )
+                                    - snode[nid].root_gain;
+                                e.best.Update( loss_chg, fid, (fvalue + e.last_fvalue) * 0.5f, !is_forward_search );
+                            }
+                        }
+                        // update the statistics
+                        e.sum_grad += grad[ ridx ];
+                        e.sum_hess += hess[ ridx ];
+                        e.last_fvalue = fvalue;
+                    }
+                }
+                // finish updating all statistics, check if it is possible to include all sum statistics
+                for( size_t i = 0; i < qexpand.size(); ++ i ){
+                    const int nid = qexpand[ i ];
+                    ThreadEntry &e = temp[ nid ];
+                    const double csum_hess = snode[nid].sum_hess - e.sum_hess;
+
+                    if( e.sum_hess >= param.min_child_weight && csum_hess >= param.min_child_weight ){
+                        const double csum_grad = snode[nid].sum_grad - e.sum_grad; 
+                        const double loss_chg = 
+                            + param.CalcGain( e.sum_grad, e.sum_hess, snode[nid].weight ) 
+                            + param.CalcGain(  csum_grad,  csum_hess, snode[nid].weight )
+                            - snode[nid].root_gain;
+                        const float delta = is_forward_search ? rt_eps:-rt_eps;
+                        e.best.Update( loss_chg, fid, e.last_fvalue + delta, !is_forward_search );
+                    }
+                }
+            }
+
+            // find splits at current level
+            inline void FindSplit( int depth ){
+                const unsigned nsize = static_cast<unsigned>( feat_index.size() );
+                
+                #pragma omp parallel for schedule( dynamic, 1 )
+                for( unsigned i = 0; i < nsize; ++ i ){
+                    const unsigned fid = feat_index[i];
+                    const int tid = omp_get_thread_num();
+                    if( param.need_forward_search() ){
+                        this->EnumerateSplit( smat.GetSortedCol(fid), fid, stemp[tid], true );
+                    }
+                    if( param.need_backward_search() ){
+                        this->EnumerateSplit( smat.GetReverseSortedCol(fid), fid, stemp[tid], false );
+                    }
+                }
+
+                // after this each thread's stemp will get the best candidates, aggregate results
+                for( size_t i = 0; i < qexpand.size(); ++ i ){
+                    const int nid = qexpand[ i ];
+                    NodeEntry &e = snode[ nid ];
+                    for( int tid = 0; tid < this->nthread; ++ tid ){
+                        e.best.Update( stemp[ tid ][ nid ].best );
+                    }
+                    
+                    // now we know the solution in snode[ nid ], set split
+                    if( e.best.loss_chg > rt_eps ){
+                        tree.AddChilds( nid );
+                        tree[ nid ].set_split( e.best.split_index(), e.best.split_value, e.best.default_left() );
+                    } else{
+                        tree[ nid ].set_leaf( e.weight * param.learning_rate );
+                    }  
+                }
+
+                {// reset position 
+                    // step 1, set default direct nodes to default, and leaf nodes to -1, 
+                    const unsigned ndata = static_cast<unsigned>( position.size() );
+                    #pragma omp parallel for schedule( static )
+                    for( unsigned i = 0; i < ndata; ++ i ){
+                        const int nid = position[i];
+                        if( nid >= 0 ){
+                            if( tree[ nid ].is_leaf() ){
+                                position[i] = -1;
+                            }else{
+                                // push to default branch, correct latter
+                                position[i] = tree[nid].default_left() ? tree[nid].cleft(): tree[nid].cright();
+                            }
+                        }
+                    }
+
+                    // step 2, classify the non-default data into right places
+                    std::vector<unsigned> fsplits;
+
+                    for( size_t i = 0; i < qexpand.size(); ++ i ){
+                        const int nid = qexpand[i];
+                        if( !tree[nid].is_leaf() ) fsplits.push_back( tree[nid].split_index() );
+                    }
+                    std::sort( fsplits.begin(), fsplits.end() );
+                    fsplits.resize( std::unique( fsplits.begin(), fsplits.end() ) - fsplits.begin() );
+
+                    const unsigned nfeats = static_cast<unsigned>( fsplits.size() );
+                    #pragma omp parallel for schedule( dynamic, 1 )
+                    for( unsigned i = 0; i < nfeats; ++ i ){
+                        const unsigned fid = fsplits[i];
+                        for( typename FMatrix::ColIter it = smat.GetSortedCol( fid ); it.Next(); ){
+                            const bst_uint ridx = it.rindex();
+                            int nid = position[ ridx ];
+                            if( nid == -1 ) continue;
+                            // go back to parent, correct those who are not default
+                            nid = tree[ nid ].parent();
+                            if( tree[ nid ].split_index() == fid ){
+                                if( it.fvalue() < tree[nid].split_cond() ){
+                                    position[ ridx ] = tree[ nid ].cleft();
+                                }else{
+                                    position[ ridx ] = tree[ nid ].cright();
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        private:
+            // initialize temp data structure
+            inline void InitData( void ){
+                {
+                    position.resize( grad.size() );
+                    if( root_index.size() == 0 ){
+                        std::fill( position.begin(), position.end(), 0 );
+                    }else{
+                        for( size_t i = 0; i < root_index.size(); ++ i ){
+                            position[i] = root_index[i];
+                            utils::Assert( root_index[i] < (unsigned)tree.param.num_roots, "root index exceed setting" );
+                        }
+                    }
+                    // mark delete for the deleted datas
+                    for( size_t i = 0; i < grad.size(); ++ i ){
+                        if( hess[i] < 0.0f ) position[i] = -1;
+                    }
+                    if( param.subsample < 1.0f - 1e-6f ){
+                        for( size_t i = 0; i < grad.size(); ++ i ){
+                            if( hess[i] < 0.0f ) continue;
+                            if( random::SampleBinary( param.subsample) == 0 ){
+                                position[ i ] = -1;
+                            }
+                        }
+                    }
+                }
+                
+                {// initialize feature index
+                    int ncol = static_cast<int>( smat.NumCol() );
+                    for( int i = 0; i < ncol; i ++ ){
+                        if( smat.GetSortedCol(i).Next() && constrain.NotBanned(i) ){
+                            feat_index.push_back( i );
+                        }
+                    }
+                    random::Shuffle( feat_index );
+                }
+                {// setup temp space for each thread
+                    if( param.nthread != 0 ){
+                        omp_set_num_threads( param.nthread );
+                    }
+                    #pragma omp parallel
+                    {
+                        this->nthread = omp_get_num_threads();
+                    }
+
+                    // reserve a small space
+                    stemp.resize( this->nthread, std::vector<ThreadEntry>() );
+                    for( size_t i = 0; i < stemp.size(); ++ i ){
+                        stemp[i].reserve( 256 );
+                    }
+                    snode.reserve( 256 );
+                }
+                
+                {// expand query
+                    qexpand.reserve( 256 ); qexpand.clear();
+                    for( int i = 0; i < tree.param.num_roots; ++ i ){
+                        qexpand.push_back( i );
+                    }
+                }
+            }
+        private:
+            // number of omp thread used during training
+            int nthread;
+            // Per feature: shuffle index of each feature index
+            std::vector<int> feat_index;
+            // Instance Data: current node position in the tree of each instance
+            std::vector<int> position;
+            // PerThread x PerTreeNode: statistics for per thread construction
+            std::vector< std::vector<ThreadEntry> > stemp;
+        private:
+            const std::vector<float> &grad;
+            const std::vector<float> &hess;
+            const FMatrix            &smat;
+            const std::vector<unsigned> &root_index;
+            const utils::FeatConstrain  &constrain;
+        };
+    };
+};
+#endif

booster/tree/xgboost_row_treemaker.hpp

@@ -0,0 +1,386 @@
+#ifndef XGBOOST_ROW_TREEMAKER_HPP
+#define XGBOOST_ROW_TREEMAKER_HPP
+/*!
+ * \file xgboost_row_treemaker.hpp
+ * \brief implementation of regression tree maker,
+ *        use a row based approach
+ * \author Tianqi Chen: tianqi.tchen@gmail.com 
+ */
+// use openmp
+#include <vector>
+#include "xgboost_tree_model.h"
+#include "../../utils/xgboost_omp.h"
+#include "../../utils/xgboost_random.h"
+#include "../../utils/xgboost_fmap.h"
+#include "xgboost_base_treemaker.hpp"
+
+namespace xgboost{
+    namespace booster{
+        template<typename FMatrix>
+        class RowTreeMaker : protected BaseTreeMaker{
+        public:
+            RowTreeMaker( RegTree &tree,
+                          const TreeParamTrain &param, 
+                          const std::vector<float> &grad,
+                          const std::vector<float> &hess,
+                          const FMatrix &smat, 
+                          const std::vector<unsigned> &root_index, 
+                          const utils::FeatConstrain &constrain )
+                : BaseTreeMaker( tree, param ), 
+                  grad(grad), hess(hess), 
+                  smat(smat), root_index(root_index), constrain(constrain) {
+                utils::Assert( grad.size() == hess.size(), "booster:invalid input" );
+                utils::Assert( smat.NumRow() == hess.size(), "booster:invalid input" );
+                utils::Assert( root_index.size() == 0 || root_index.size() == hess.size(), "booster:invalid input" ); 
+                {// setup temp space for each thread
+                    if( param.nthread != 0 ){
+                        omp_set_num_threads( param.nthread );
+                    }
+                    #pragma omp parallel
+                    {
+                        this->nthread = omp_get_num_threads();
+                    }
+                    tmp_rptr.resize( this->nthread, std::vector<size_t>() );
+                    snode.reserve( 256 );
+                }
+            }
+            inline void Make( int& stat_max_depth, int& stat_num_pruned ){
+                this->InitData();
+                this->InitNewNode( this->qexpand );
+                stat_max_depth = 0;
+                
+                for( int depth = 0; depth < param.max_depth; ++ depth ){                                        
+                    this->FindSplit( this->qexpand, depth );
+                    this->UpdateQueueExpand( this->qexpand );
+                    this->InitNewNode( this->qexpand );
+                    // if nothing left to be expand, break
+                    if( qexpand.size() == 0 ) break;
+                    stat_max_depth = depth + 1;
+                }
+                // set all the rest expanding nodes to leaf
+                for( size_t i = 0; i < qexpand.size(); ++ i ){
+                    const int nid = qexpand[i];
+                    tree[ nid ].set_leaf( snode[nid].weight * param.learning_rate );
+                }
+                // start prunning the tree
+                stat_num_pruned = this->DoPrune();
+            }
+            // expand a specific node
+            inline bool Expand( const std::vector<bst_uint> &valid_index, int nid ){
+                if( valid_index.size() == 0 ) return false;
+                this->InitDataExpand( valid_index, nid );
+                this->InitNewNode( this->qexpand );
+                this->FindSplit( nid, tmp_rptr[0] );
+
+                // update node statistics
+                for( size_t i = 0; i < qexpand.size(); ++ i ){
+                    const int nid = qexpand[i];
+                    tree.stat( nid ).loss_chg = snode[ nid ].best.loss_chg;
+                    tree.stat( nid ).sum_hess = static_cast<float>( snode[ nid ].sum_hess );
+                }
+                // change the leaf
+                this->UpdateQueueExpand( this->qexpand );
+                this->InitNewNode( this->qexpand );
+                
+                // set all the rest expanding nodes to leaf
+                for( size_t i = 0; i < qexpand.size(); ++ i ){
+                    const int nid = qexpand[i];
+                    
+                    tree[ nid ].set_leaf( snode[nid].weight * param.learning_rate );
+                    tree.stat( nid ).loss_chg = 0.0f;
+                    tree.stat( nid ).sum_hess = static_cast<float>( snode[ nid ].sum_hess );
+                    tree.param.max_depth = std::max( tree.param.max_depth, tree.GetDepth( nid ) );
+                }
+                if( qexpand.size() != 0 ) {
+                    return true;
+                }else{
+                    return false;
+                }
+            }
+            // collapse specific node
+            inline void Collapse( const std::vector<bst_uint> &valid_index, int nid ){
+                if( valid_index.size() == 0 ) return;
+                this->InitDataExpand( valid_index, nid );
+                this->InitNewNode( this->qexpand );
+                tree.stat( nid ).loss_chg = 0.0f;
+                tree.stat( nid ).sum_hess = static_cast<float>( snode[ nid ].sum_hess );
+                tree.CollapseToLeaf( nid, snode[nid].weight * param.learning_rate );
+            }
+        private:
+            // make leaf nodes for all qexpand, update node statistics, mark leaf value
+            inline void InitNewNode( const std::vector<int> &qexpand ){
+                snode.resize( tree.param.num_nodes, NodeEntry() );
+
+                for( size_t j = 0; j < qexpand.size(); ++j ){
+                    const int nid = qexpand[ j ];
+                    double sum_grad = 0.0, sum_hess = 0.0;
+
+                    for( bst_uint i = node_bound[nid].first; i < node_bound[nid].second; ++i ){
+                        const bst_uint ridx = row_index_set[i];
+                        sum_grad += grad[ridx]; sum_hess += hess[ridx];
+                    }
+                    // update node statistics
+                    snode[nid].sum_grad = sum_grad; 
+                    snode[nid].sum_hess = sum_hess;
+
+                    snode[nid].root_gain = param.CalcRootGain( sum_grad, sum_hess );
+                    if( !tree[nid].is_root() ){
+                        snode[nid].weight = param.CalcWeight( sum_grad, sum_hess, tree.stat( tree[nid].parent() ).base_weight );
+                        tree.stat(nid).base_weight = snode[nid].weight;
+                    }else{
+                        snode[nid].weight = param.CalcWeight( sum_grad, sum_hess, 0.0f );
+                        tree.stat(nid).base_weight = snode[nid].weight;
+                    }
+                }
+            }
+        private:
+            // enumerate the split values of specific feature
+            template<typename Iter>
+            inline void EnumerateSplit( Iter it, SplitEntry &best, const int nid, const unsigned fid, bool is_forward_search ){
+                float last_fvalue = 0.0f;
+                double sum_hess = 0.0, sum_grad = 0.0;
+                const NodeEntry enode = snode[ nid ];
+
+                while( it.Next() ){
+                    const bst_uint ridx = it.rindex();
+                    const float fvalue = it.fvalue();           
+                    
+                    if( sum_hess == 0.0 ){
+                        sum_grad = grad[ ridx ];
+                        sum_hess = hess[ ridx ];
+                        last_fvalue = fvalue;
+                    }else{
+                        // try to find a split
+                        if( fabsf(fvalue - last_fvalue) > rt_2eps && sum_hess >= param.min_child_weight ){
+                            const double csum_hess = enode.sum_hess - sum_hess;
+                            if( csum_hess >= param.min_child_weight ){
+                                const double csum_grad = enode.sum_grad - sum_grad; 
+                                const double loss_chg = 
+                                    + param.CalcGain(  sum_grad,  sum_hess, enode.weight ) 
+                                    + param.CalcGain( csum_grad, csum_hess, enode.weight )
+                                    - enode.root_gain;
+                                best.Update( loss_chg, fid, (fvalue + last_fvalue) * 0.5f, !is_forward_search );
+                            }else{
+                                // the rest part doesn't meet split condition anyway, return 
+                                return;
+                            }
+                        }
+                        // update the statistics
+                        sum_grad += grad[ ridx ];
+                        sum_hess += hess[ ridx ];
+                        last_fvalue = fvalue;
+                    }                    
+                }
+
+                const double csum_hess = enode.sum_hess - sum_hess;
+                if( sum_hess >= param.min_child_weight && csum_hess >= param.min_child_weight ){
+                    const double csum_grad = enode.sum_grad - sum_grad; 
+                    const double loss_chg = 
+                        + param.CalcGain(   sum_grad,   sum_hess, enode.weight ) 
+                        + param.CalcGain(  csum_grad,  csum_hess, enode.weight )
+                        - snode[nid].root_gain;
+                    const float delta = is_forward_search ? rt_eps:-rt_eps;
+                    best.Update( loss_chg, fid, last_fvalue + delta, !is_forward_search );
+                }
+            }
+        private:
+            inline void FindSplit( const std::vector<int> &qexpand, int depth ){
+                int nexpand = (int)qexpand.size();
+                if( depth < 3 ){ 
+                    for( int i = 0; i < nexpand; ++ i ){
+                        this->FindSplit( qexpand[i], tmp_rptr[0] );
+                    }
+                }else{
+                    // if get to enough depth, parallelize over node
+                    #pragma omp parallel for schedule(dynamic,1)
+                    for( int i = 0; i < nexpand; ++ i ){
+                        const int tid = omp_get_thread_num();
+                        utils::Assert( tid < (int)tmp_rptr.size(), "BUG: FindSplit, tid exceed tmp_rptr size" );
+                        this->FindSplit( qexpand[i], tmp_rptr[tid] );
+                    }
+                }
+            }
+        private:
+            inline void MakeSplit( int nid, unsigned gid ){
+                node_bound.resize( tree.param.num_nodes );
+                // re-organize the row_index_set after split on nid
+                const unsigned split_index = tree[nid].split_index();
+                const float    split_value = tree[nid].split_cond();
+
+                std::vector<bst_uint> right;
+                bst_uint top = node_bound[nid].first;
+                for( bst_uint i = node_bound[ nid ].first; i < node_bound[ nid ].second; ++i ){
+                    const bst_uint ridx = row_index_set[i];                    
+                    bool goleft = tree[ nid ].default_left();                    
+                    for( typename FMatrix::RowIter it = smat.GetRow(ridx,gid); it.Next(); ){
+                        if( it.findex() == split_index ){
+                            if( it.fvalue() < split_value ){
+                                goleft = true;  break;
+                            }else{
+                                goleft = false; break;
+                            }
+                        }
+                    }
+                    if( goleft ) {
+                        row_index_set[ top ++ ] = ridx;
+                    }else{
+                        right.push_back( ridx );
+                    }
+                }
+                node_bound[ tree[nid].cleft() ]  = std::make_pair( node_bound[nid].first, top );
+                node_bound[ tree[nid].cright() ] = std::make_pair( top, node_bound[nid].second );
+
+                utils::Assert( node_bound[nid].second - top == (bst_uint)right.size(), "BUG:MakeSplit" );
+                for( size_t i = 0; i < right.size(); ++ i ){
+                    row_index_set[ top ++ ] = right[ i ];
+                }
+            }
+                        
+            // find splits at current level
+            inline void FindSplit( int nid, std::vector<size_t> &tmp_rptr ){
+                if( tmp_rptr.size() == 0 ){
+                    tmp_rptr.resize( tree.param.num_feature + 1, 0 );
+                }
+                const bst_uint begin = node_bound[ nid ].first;
+                const bst_uint end   = node_bound[ nid ].second;
+                const unsigned ncgroup = smat.NumColGroup();
+                unsigned best_group = 0;
+
+                for( unsigned gid = 0; gid < ncgroup; ++gid ){
+                    // records the columns
+                    std::vector<FMatrixS::REntry> centry;
+                    // records the active features
+                    std::vector<size_t>  aclist;
+                    utils::SparseCSRMBuilder<FMatrixS::REntry,true> builder( tmp_rptr, centry, aclist );
+                    builder.InitBudget( tree.param.num_feature );
+                    for( bst_uint i = begin; i < end; ++i ){
+                        const bst_uint ridx = row_index_set[i];
+                        for( typename FMatrix::RowIter it = smat.GetRow(ridx,gid); it.Next(); ){
+                            const bst_uint findex = it.findex();
+                            if( constrain.NotBanned( findex ) ) builder.AddBudget( findex );
+                        }
+                    }
+                    builder.InitStorage();
+                    for( bst_uint i = begin; i < end; ++i ){
+                        const bst_uint ridx = row_index_set[i];
+                        for( typename FMatrix::RowIter it = smat.GetRow(ridx,gid); it.Next(); ){
+                            const bst_uint findex = it.findex();
+                            if( constrain.NotBanned( findex ) ) {
+                                builder.PushElem( findex, FMatrixS::REntry( ridx, it.fvalue() ) );
+                            }
+                        }
+                    }
+                    // --- end of building column major matrix ---                    
+                    // after this point, tmp_rptr and entry is ready to use                    
+                    int naclist = (int)aclist.size();
+                    // best entry for each thread
+                    SplitEntry nbest, tbest;
+                    #pragma omp parallel private(tbest)
+                    { 
+                        #pragma omp for schedule(dynamic,1)
+                        for( int j = 0; j < naclist; ++j ){
+                            bst_uint findex = static_cast<bst_uint>( aclist[j] );
+                            // local sort can be faster when the features are sparse
+                            std::sort( centry.begin() + tmp_rptr[findex], centry.begin() + tmp_rptr[findex+1], FMatrixS::REntry::cmp_fvalue );
+                            if( param.need_forward_search() ){
+                                this->EnumerateSplit( FMatrixS::ColIter( &centry[tmp_rptr[findex]]-1, &centry[tmp_rptr[findex+1]] - 1 ),
+                                                      tbest, nid, findex, true );
+                            }
+                            if( param.need_backward_search() ){
+                                this->EnumerateSplit( FMatrixS::ColBackIter( &centry[tmp_rptr[findex+1]], &centry[tmp_rptr[findex]] ),
+                                                      tbest, nid, findex, false );
+                            }
+                        }
+                        #pragma omp critical 
+                        {
+                            nbest.Update( tbest );
+                        }
+                    }
+                    // if current solution gives the best 
+                    if( snode[nid].best.Update( nbest ) ){
+                        best_group = gid;
+                    }
+                    // cleanup tmp_rptr for next usage
+                    builder.Cleanup();    
+                }
+                
+                // at this point, we already know the best split
+                if( snode[nid].best.loss_chg > rt_eps ){
+                    const SplitEntry &e = snode[nid].best;
+                    tree.AddChilds( nid );
+                    tree[ nid ].set_split( e.split_index(), e.split_value, e.default_left() );
+                    this->MakeSplit( nid, best_group );
+                }else{
+                    tree[ nid ].set_leaf( snode[nid].weight * param.learning_rate );                    
+                }
+            }
+        private:
+            // initialize temp data structure
+            inline void InitData( void ){
+                std::vector<bst_uint> valid_index;
+                for( size_t i = 0; i < grad.size(); ++i ){
+                    if( hess[ i ] < 0.0f ) continue;
+                    if( param.subsample > 1.0f-1e-6f || random::SampleBinary( param.subsample ) != 0 ){
+                        valid_index.push_back( static_cast<bst_uint>(i) );
+                    }
+                }
+                node_bound.resize( tree.param.num_roots );
+
+                if( root_index.size() == 0 ){
+                    row_index_set = valid_index;
+                    // set bound of root node
+                    node_bound[0] = std::make_pair( 0, (bst_uint)row_index_set.size() );
+                }else{                    
+                    std::vector<size_t>   rptr;
+                    utils::SparseCSRMBuilder<bst_uint> builder( rptr, row_index_set );
+                    builder.InitBudget( tree.param.num_roots );
+                    for( size_t i = 0; i < valid_index.size(); ++i ){
+                        const bst_uint rid = valid_index[ i ];
+                        utils::Assert( root_index[ rid ] < (unsigned)tree.param.num_roots, "root id exceed number of roots" );
+                        builder.AddBudget( root_index[ rid ] );
+                    }
+                    builder.InitStorage();
+                    for( size_t i = 0; i < valid_index.size(); ++i ){
+                        const bst_uint rid = valid_index[ i ];
+                        builder.PushElem( root_index[ rid ], rid );
+                    }
+                    for( size_t i = 1; i < rptr.size(); ++ i ){
+                        node_bound[i-1] = std::make_pair( rptr[ i - 1 ], rptr[ i ] );
+                    }
+                }
+
+                {// expand query
+                    qexpand.reserve( 256 ); qexpand.clear();
+                    for( int i = 0; i < tree.param.num_roots; ++ i ){
+                        qexpand.push_back( i );
+                    }
+                }
+            }
+
+            // initialize temp data structure
+            inline void InitDataExpand( const std::vector<bst_uint> &valid_index, int nid ){
+                row_index_set = valid_index;                
+                node_bound.resize( tree.param.num_nodes );
+                node_bound[ nid ] = std::make_pair( 0, (bst_uint)row_index_set.size() );
+             
+                qexpand.clear(); qexpand.push_back( nid );
+            }
+        private:
+            // number of omp thread used during training
+            int nthread;
+            // tmp row pointer, per thread, used for tmp data construction
+            std::vector< std::vector<size_t> > tmp_rptr;
+            // Instance row indexes corresponding to each node
+            std::vector<bst_uint> row_index_set;
+            // lower and upper bound of each nodes' row_index
+            std::vector< std::pair<bst_uint, bst_uint> > node_bound;
+        private:
+            const std::vector<float> &grad;
+            const std::vector<float> &hess;
+            const FMatrix            &smat;
+            const std::vector<unsigned> &root_index;
+            const utils::FeatConstrain  &constrain;
+        };
+    };
+};
+#endif

booster/tree/xgboost_svdf_tree.hpp

@@ -0,0 +1,429 @@
+#ifndef XGBOOST_APEX_TREE_HPP
+#define XGBOOST_APEX_TREE_HPP
+/*!
+ * \file xgboost_svdf_tree.hpp
+ * \brief implementation of regression tree constructor, with layerwise support
+ *        this file is adapted from GBRT implementation in SVDFeature project
+ * \author Tianqi Chen: tqchen@apex.sjtu.edu.cn, tianqi.tchen@gmail.com 
+ */
+#include <algorithm>
+#include "xgboost_tree_model.h"
+#include "../../utils/xgboost_random.h"
+#include "../../utils/xgboost_matrix_csr.h"
+
+namespace xgboost{
+    namespace booster{        
+        inline void assert_sorted( unsigned *idset, int len ){
+            if( !rt_debug || !check_bug ) return;
+            for( int i = 1; i < len; i ++ ){
+                utils::Assert( idset[i-1] < idset[i], "idset not sorted" );
+            }
+        }
+    };
+    
+    namespace booster{                
+        // selecter of rtree to find the suitable candidate
+        class RTSelecter{
+        public:
+            struct Entry{
+                float  loss_chg;
+                size_t start;
+                int    len;
+                unsigned sindex;
+                float    split_value;
+                Entry(){}
+                Entry( float loss_chg, size_t start, int len, unsigned split_index, float split_value, bool default_left ){
+                    this->loss_chg = loss_chg;
+                    this->start    = start;
+                    this->len      = len;
+                    if( default_left ) split_index |= (1U << 31);
+                    this->sindex = split_index;
+                    this->split_value = split_value;
+                }
+                inline unsigned split_index( void ) const{
+                    return sindex & ( (1U<<31) - 1U );
+                }
+                inline bool default_left( void ) const{
+                    return (sindex >> 31) != 0;
+                }
+            };
+        private:
+            Entry best_entry;
+        public:
+            RTSelecter( void ){
+                memset( &best_entry, 0, sizeof(best_entry) );
+                best_entry.loss_chg = 0.0f;
+            }
+            inline void push_back( const Entry &e ){
+                if( e.loss_chg > best_entry.loss_chg ) best_entry = e;
+            }
+            inline const Entry & select( void ){            
+                return best_entry;                
+            }
+        };
+        
+        
+        // updater of rtree, allows the parameters to be stored inside, key solver
+        template<typename FMatrix>
+        class RTreeUpdater{
+        protected:
+            // training task, element of single task
+            struct Task{
+                // node id in tree
+                int nid;
+                // idset pointer, instance id in [idset,idset+len)
+                unsigned *idset;
+                // length of idset
+                unsigned len;            
+                // base_weight of parent
+                float parent_base_weight;
+                Task(){}
+                Task( int nid, unsigned *idset, unsigned len, float pweight = 0.0f ){
+                    this->nid = nid;
+                    this->idset = idset;
+                    this->len = len;
+                    this->parent_base_weight = pweight;
+                }
+            };
+            
+            // sparse column entry
+            struct SCEntry{
+                // feature value 
+                float    fvalue;
+                // row index in grad
+                unsigned rindex;
+                SCEntry(){}
+                SCEntry( float fvalue, unsigned rindex ){
+                    this->fvalue = fvalue; this->rindex = rindex;
+                }
+                inline bool operator<( const SCEntry &p ) const{
+                    return fvalue < p.fvalue;
+                }
+            };
+        private:
+            // training parameter
+            const TreeParamTrain &param;
+            // parameters, reference
+            RegTree &tree;
+            std::vector<float> &grad;
+            std::vector<float> &hess;
+            const FMatrix &smat;
+            const std::vector<unsigned> &group_id;
+        private:
+            // maximum depth up to now
+            int max_depth;
+            // number of nodes being pruned
+            int num_pruned;
+            // stack to store current task
+            std::vector<Task> task_stack;
+            // temporal space for index set
+            std::vector<unsigned> idset;
+        private:
+            // task management: NOTE DFS here
+            inline void add_task( Task tsk ){
+                task_stack.push_back( tsk );
+            }
+            inline bool next_task( Task &tsk ){
+                if( task_stack.size() == 0 ) return false;
+                tsk = task_stack.back(); 
+                task_stack.pop_back();
+                return true;
+            } 
+        private:
+            // try to prune off current leaf, return true if successful
+            inline void try_prune_leaf( int nid, int depth ){
+                if( tree[ nid ].is_root() ) return;
+                int pid = tree[ nid ].parent();
+                RegTree::NodeStat &s = tree.stat( pid );
+                s.leaf_child_cnt ++;
+                
+                if( s.leaf_child_cnt >= 2 && param.need_prune( s.loss_chg, depth - 1 ) ){
+                    // need to be pruned
+                    tree.ChangeToLeaf( pid, param.learning_rate * s.base_weight );
+                    // add statistics to number of nodes pruned
+                    num_pruned += 2;
+                    // tail recursion
+                    this->try_prune_leaf( pid, depth - 1 );
+                }
+            }
+            // make leaf for current node :)
+            inline void make_leaf( Task tsk, double sum_grad, double sum_hess, bool compute ){
+                for( unsigned i = 0; i < tsk.len; i ++ ){
+                    const unsigned ridx = tsk.idset[i];
+                    if( compute ){
+                        sum_grad += grad[ ridx ];
+                        sum_hess += hess[ ridx ];
+                    }
+                }
+                tree.stat( tsk.nid ).sum_hess = static_cast<float>( sum_hess );
+                tree[ tsk.nid ].set_leaf( param.learning_rate * param.CalcWeight( sum_grad, sum_hess, tsk.parent_base_weight ) );
+                this->try_prune_leaf( tsk.nid, tree.GetDepth( tsk.nid ) );
+            }
+        private:
+            // make split for current task, re-arrange positions in idset
+            inline void make_split( Task tsk, const SCEntry *entry, int num, float loss_chg, double sum_hess, double base_weight ){
+                // before split, first prepare statistics
+                RegTree::NodeStat &s = tree.stat( tsk.nid );
+                s.loss_chg = loss_chg; 
+                s.leaf_child_cnt = 0;
+                s.sum_hess = static_cast<float>( sum_hess );
+                s.base_weight = static_cast<float>( base_weight );
+                
+                // add childs to current node
+                tree.AddChilds( tsk.nid );
+                // assert that idset is sorted
+                assert_sorted( tsk.idset, tsk.len );
+                // use merge sort style to get the solution
+                std::vector<unsigned> qset;
+                for( int i = 0; i < num; i ++ ){
+                    qset.push_back( entry[i].rindex );
+                }
+                std::sort( qset.begin(), qset.end() );
+                // do merge sort style, make the other set, remove elements in qset
+                for( unsigned i = 0, top = 0; i < tsk.len; i ++ ){
+                    if( top < qset.size() ){
+                        if( tsk.idset[ i ] != qset[ top ] ){
+                            tsk.idset[ i - top ] = tsk.idset[ i ];
+                        }else{
+                            top ++;
+                        }
+                    }else{
+                        tsk.idset[ i - qset.size() ] = tsk.idset[ i ];
+                    }
+                }
+                // get two parts 
+                RegTree::Node &n = tree[ tsk.nid ];
+                Task def_part( n.default_left() ? n.cleft() : n.cright(), tsk.idset, tsk.len - qset.size(), s.base_weight );
+                Task spl_part( n.default_left() ? n.cright(): n.cleft() , tsk.idset + def_part.len, qset.size(), s.base_weight );  
+                // fill back split part
+                for( unsigned i = 0; i < spl_part.len; i ++ ){
+                    spl_part.idset[ i ] = qset[ i ];
+                }
+                // add tasks to the queue
+                this->add_task( def_part ); 
+                this->add_task( spl_part );
+            }
+            
+            // enumerate split point of the tree
+            inline void enumerate_split( RTSelecter &sglobal, int tlen,
+                                         double rsum_grad, double rsum_hess, double root_gain,
+                                         const SCEntry *entry, size_t start, size_t end, 
+                                         int findex, float parent_base_weight ){
+                // local selecter
+                RTSelecter slocal;
+                
+                if( param.need_forward_search() ){
+                    // forward process, default right
+                    double csum_grad = 0.0, csum_hess = 0.0;
+                    for( size_t j = start; j < end; j ++ ){
+                        const unsigned ridx = entry[ j ].rindex;
+                        csum_grad += grad[ ridx ];
+                        csum_hess += hess[ ridx ];
+                        // check for split
+                        if( j == end - 1 || entry[j].fvalue + rt_2eps < entry[ j + 1 ].fvalue ){
+                            if( csum_hess < param.min_child_weight ) continue;
+                            const double dsum_hess = rsum_hess - csum_hess;
+                            if( dsum_hess < param.min_child_weight ) break;                        
+                            // change of loss 
+                            double loss_chg = 
+                            param.CalcGain( csum_grad, csum_hess, parent_base_weight ) + 
+                                param.CalcGain( rsum_grad - csum_grad, dsum_hess, parent_base_weight ) - root_gain;
+                            
+                            const int clen = static_cast<int>( j + 1 - start );
+                            // add candidate to selecter
+                            slocal.push_back( RTSelecter::Entry( loss_chg, start, clen, findex, 
+                                                                 j == end - 1 ? entry[j].fvalue + rt_eps : 0.5 * (entry[j].fvalue+entry[j+1].fvalue),
+                                                                 false ) );
+                        }
+                    }
+                }
+                
+                if( param.need_backward_search() ){
+                    // backward process, default left
+                    double csum_grad = 0.0, csum_hess = 0.0;
+                    for( size_t j = end; j > start; j -- ){
+                        const unsigned ridx = entry[ j - 1 ].rindex;
+                        csum_grad += grad[ ridx ];
+                        csum_hess += hess[ ridx ];
+                        // check for split
+                        if( j == start + 1 || entry[ j - 2 ].fvalue + rt_2eps < entry[ j - 1 ].fvalue ){
+                            if( csum_hess < param.min_child_weight ) continue;
+                            const double dsum_hess = rsum_hess - csum_hess;
+                            if( dsum_hess < param.min_child_weight ) break;
+                            double loss_chg = param.CalcGain( csum_grad, csum_hess, parent_base_weight ) + 
+                                param.CalcGain( rsum_grad - csum_grad, dsum_hess, parent_base_weight ) - root_gain;
+                            const int clen = static_cast<int>( end - j + 1 );
+                            // add candidate to selecter
+                            slocal.push_back( RTSelecter::Entry( loss_chg, j - 1, clen, findex,
+                                                                 j == start + 1 ? entry[j-1].fvalue - rt_eps : 0.5 * (entry[j-2].fvalue + entry[j-1].fvalue), 
+                                                                 true ) );
+                        }
+                    }
+                }
+                sglobal.push_back( slocal.select() );
+            }
+            
+        private:
+            // temporal storage for expand column major
+            std::vector<size_t>  tmp_rptr;        
+            // find split for current task, another implementation of expand in column major manner
+            // should be more memory frugal, avoid global sorting across feature       
+            inline void expand( Task tsk ){
+                // assert that idset is sorted
+                // if reach maximum depth, make leaf from current node
+                int depth = tree.GetDepth( tsk.nid );
+                // update statistiss
+                if( depth > max_depth ) max_depth = depth; 
+                // if bigger than max depth
+                if( depth >= param.max_depth ){
+                    this->make_leaf( tsk, 0.0, 0.0, true ); return;
+                }
+                // convert to column major CSR format
+                const int nrows = tree.param.num_feature;
+                if( tmp_rptr.size() == 0 ){
+                    // initialize tmp storage in first usage
+                    tmp_rptr.resize( nrows + 1 ); 
+                    std::fill( tmp_rptr.begin(), tmp_rptr.end(), 0 );
+                }
+                // records the columns
+                std::vector<SCEntry> entry;
+                // records the active features
+                std::vector<size_t>  aclist;
+                utils::SparseCSRMBuilder<SCEntry,true> builder( tmp_rptr, entry, aclist );
+                builder.InitBudget( nrows );
+                // statistics of root
+                double rsum_grad = 0.0, rsum_hess = 0.0;            
+                for( unsigned i = 0; i < tsk.len; i ++ ){
+                    const unsigned ridx = tsk.idset[i];
+                    rsum_grad  += grad[ ridx ];
+                    rsum_hess  += hess[ ridx ];
+                    
+                    for( typename FMatrix::RowIter it = smat.GetRow(ridx); it.Next(); ){
+                        builder.AddBudget( it.findex() );
+                    }
+                }
+                
+                // if minimum split weight is not meet
+                if( param.cannot_split( rsum_hess, depth )  ){
+                    this->make_leaf( tsk, rsum_grad, rsum_hess, false ); builder.Cleanup(); return; 
+                }
+                
+                builder.InitStorage();
+                for( unsigned i = 0; i < tsk.len; i ++ ){
+                    const unsigned ridx = tsk.idset[i];
+                    for( typename FMatrix::RowIter it = smat.GetRow(ridx); it.Next(); ){
+                        builder.PushElem( it.findex(), SCEntry( it.fvalue(), ridx ) );
+                    }
+                }
+                // --- end of building column major matrix ---
+                // after this point, tmp_rptr and entry is ready to use
+                
+                // global selecter
+                RTSelecter sglobal;
+                // gain root 
+                const double root_gain = param.CalcRootGain( rsum_grad, rsum_hess );
+                // KEY: layerwise, weight of current node if it is leaf
+                const double base_weight = param.CalcWeight( rsum_grad, rsum_hess, tsk.parent_base_weight );
+                // enumerate feature index
+                for( size_t i = 0; i < aclist.size(); i ++ ){
+                    int findex = static_cast<int>( aclist[i] );                
+                    size_t start = tmp_rptr[ findex ];
+                    size_t end   = tmp_rptr[ findex + 1 ];
+                    utils::Assert( start < end, "bug" );
+                    // local sort can be faster when the features are sparse
+                    std::sort( entry.begin() + start, entry.begin() + end );
+                    // local selecter
+                    this->enumerate_split( sglobal, tsk.len,
+                                           rsum_grad, rsum_hess, root_gain,
+                                           &entry[0], start, end, findex, base_weight );
+                }
+                // Cleanup tmp_rptr for next use
+                builder.Cleanup();
+                // get the best solution
+                const RTSelecter::Entry &e = sglobal.select();
+                // allowed to split
+                if( e.loss_chg > rt_eps ){
+                    // add splits
+                    tree[ tsk.nid ].set_split( e.split_index(), e.split_value, e.default_left() );
+                    // re-arrange idset, push tasks
+                    this->make_split( tsk, &entry[ e.start ], e.len, e.loss_chg, rsum_hess, base_weight ); 
+                }else{
+                    // make leaf if we didn't meet requirement
+                    this->make_leaf( tsk, rsum_grad, rsum_hess, false );
+                }
+            }
+        private:
+            // initialize the tasks
+            inline void init_tasks( size_t ngrads ){
+                // add group partition if necessary
+                if( group_id.size() == 0 ){       
+                    if( param.subsample > 1.0f - 1e-6f ){ 
+                        idset.resize( 0 );
+                        for( size_t i = 0; i < ngrads; i ++ ){
+                            if( hess[i] < 0.0f ) continue;
+                            idset.push_back( (unsigned)i );
+                        } 
+                    }else{
+                        idset.resize( 0 );
+                        for( size_t i = 0; i < ngrads; i ++ ){
+                            if( random::SampleBinary( param.subsample ) != 0 ){
+                                idset.push_back( (unsigned)i );
+                            }
+                        } 
+                    }
+                    this->add_task( Task( 0, &idset[0], idset.size() ) ); return;
+                }
+                
+                utils::Assert( group_id.size() == ngrads, "number of groups must be exact" );            
+                {// new method for grouping, use CSR builder
+                    std::vector<size_t>   rptr;
+                    utils::SparseCSRMBuilder<unsigned> builder( rptr, idset );
+                    builder.InitBudget( tree.param.num_roots );
+                    for( size_t i = 0; i < group_id.size(); i ++ ){
+                        // drop invalid elements
+                        if( hess[ i ] < 0.0f ) continue;
+                        utils::Assert( group_id[ i ] < (unsigned)tree.param.num_roots, 
+                                       "group id exceed number of roots" );
+                        builder.AddBudget( group_id[ i ] );
+                    }
+                    builder.InitStorage();
+                    for( size_t i = 0; i < group_id.size(); i ++ ){
+                        // drop invalid elements
+                        if( hess[ i ] < 0.0f ) continue;
+                        builder.PushElem( group_id[ i ], static_cast<unsigned>(i) );
+                    }
+                    for( size_t i = 1; i < rptr.size(); i ++ ){
+                        const size_t start = rptr[ i - 1 ], end = rptr[ i ];
+                        if( start < end ){
+                            this->add_task( Task( i - 1, &idset[ start ], end - start ) );
+                        }
+                    }
+                }
+            }
+        public:
+            RTreeUpdater( const TreeParamTrain &pparam, 
+                          RegTree &ptree,
+                          std::vector<float> &pgrad,
+                          std::vector<float> &phess,
+                          const FMatrix &psmat, 
+                          const std::vector<unsigned> &pgroup_id ):
+                param( pparam ), tree( ptree ), grad( pgrad ), hess( phess ),
+                smat( psmat ), group_id( pgroup_id ){
+            }
+            inline int do_boost( int &num_pruned ){
+                this->init_tasks( grad.size() );
+                this->max_depth = 0;
+                this->num_pruned = 0;
+                Task tsk;
+                while( this->next_task( tsk ) ){
+                    this->expand( tsk );
+                }
+                num_pruned = this->num_pruned;
+                return max_depth;
+            }
+        };
+    };
+};
+#endif
+
+

booster/tree/xgboost_tree.hpp

@@ -0,0 +1,268 @@
+#ifndef XGBOOST_TREE_HPP
+#define XGBOOST_TREE_HPP
+/*!
+ * \file xgboost_tree.hpp
+ * \brief implementation of regression tree
+ * \author Tianqi Chen: tianqi.tchen@gmail.com 
+ */
+#include "xgboost_tree_model.h"
+
+namespace xgboost{
+    namespace booster{
+        const bool rt_debug = false;
+        // whether to check bugs
+        const bool check_bug = false;
+    
+        const float rt_eps = 1e-5f;
+        const float rt_2eps = rt_eps * 2.0f;
+        
+        inline double sqr( double a ){
+            return a * a;
+        }
+    };
+};
+#include "../../utils/xgboost_fmap.h"
+#include "xgboost_svdf_tree.hpp"
+#include "xgboost_col_treemaker.hpp"
+#include "xgboost_row_treemaker.hpp"
+
+namespace xgboost{
+    namespace booster{
+        // regression tree, construction algorithm is seperated from this class
+        // see RegTreeUpdater
+        template<typename FMatrix>
+        class RegTreeTrainer : public InterfaceBooster<FMatrix>{
+        public:
+            RegTreeTrainer( void ){ 
+                silent = 0; tree_maker = 1; 
+                // interact mode
+                interact_type = 0;
+                interact_node = 0;
+                // normally we won't have more than 64 OpenMP threads
+                threadtemp.resize( 64, ThreadEntry() );
+            }
+            virtual ~RegTreeTrainer( void ){}
+        public:
+            virtual void SetParam( const char *name, const char *val ){
+                if( !strcmp( name, "silent") )      silent = atoi( val );
+                if( !strcmp( name, "tree_maker") )  tree_maker = atoi( val );
+                if( !strncmp( name, "interact:", 9) ){
+                    const char *ename = name + 9;
+                    interact_node = atoi( val );
+                    if( !strcmp( ename, "expand") ) {
+                        interact_type = 1;
+                    }
+                    if( !strcmp( ename, "remove") ) {
+                        interact_type = 2;
+                    }                    
+                }
+                param.SetParam( name, val );
+                constrain.SetParam( name, val );
+                tree.param.SetParam( name, val );
+            }
+            virtual void LoadModel( utils::IStream &fi ){
+                tree.LoadModel( fi );
+            }
+            virtual void SaveModel( utils::IStream &fo ) const{
+                tree.SaveModel( fo );
+            }
+            virtual void InitModel( void ){
+                tree.InitModel();
+            }
+        public:
+            virtual void DoBoost( std::vector<float> &grad, 
+                                  std::vector<float> &hess,
+                                  const FMatrix &smat,
+                                  const std::vector<unsigned> &root_index ){
+                utils::Assert( grad.size() < UINT_MAX, "number of instance exceed what we can handle" );
+
+                // interactive update 
+                if( interact_type != 0 ){
+                    switch( interact_type ){
+                    case 1: this->ExpandNode( grad, hess, smat, root_index, interact_node ); return;
+                    case 2: this->CollapseNode( grad, hess, smat, root_index, interact_node ); return;
+                    default: utils::Error("unknown interact type");
+                    }
+                }
+
+                if( !silent ){
+                    printf( "\nbuild GBRT with %u instances\n", (unsigned)grad.size() );
+                }
+                int num_pruned;
+                switch( tree_maker ){
+                case 0: {
+                    utils::Assert( !constrain.HasConstrain(), "tree maker 0 does not support constrain" );
+                    RTreeUpdater<FMatrix> updater( param, tree, grad, hess, smat, root_index );
+                    tree.param.max_depth = updater.do_boost( num_pruned );
+                    break;
+                }
+                case 1:{
+                    ColTreeMaker<FMatrix> maker( tree, param, grad, hess, smat, root_index, constrain );
+                    maker.Make( tree.param.max_depth, num_pruned );
+                    break;
+                }
+                case 2:{
+                    RowTreeMaker<FMatrix> maker( tree, param, grad, hess, smat, root_index, constrain );
+                    maker.Make( tree.param.max_depth, num_pruned );
+                    break;
+                }                    
+                default: utils::Error("unknown tree maker");
+                }
+                if( !silent ){
+                    printf( "tree train end, %d roots, %d extra nodes, %d pruned nodes ,max_depth=%d\n", 
+                            tree.param.num_roots, tree.num_extra_nodes(), num_pruned, tree.MaxDepth() );
+                }
+            }            
+            virtual float Predict( const FMatrix &fmat, bst_uint ridx, unsigned gid = 0 ){
+                ThreadEntry &e = this->InitTmp();
+                this->PrepareTmp( fmat.GetRow(ridx), e );
+                int pid = this->GetLeafIndex( e.feat, e.funknown, gid );
+                this->DropTmp( fmat.GetRow(ridx), e );
+                return tree[ pid ].leaf_value();          
+            }
+            virtual int GetLeafIndex( const std::vector<float> &feat,
+                                      const std::vector<bool>  &funknown,
+                                      unsigned gid = 0 ){
+                // start from groups that belongs to current data
+                int pid = (int)gid;
+                // tranverse tree
+                while( !tree[ pid ].is_leaf() ){
+                    unsigned split_index = tree[ pid ].split_index();
+                    pid = this->GetNext( pid, feat[ split_index ], funknown[ split_index ] );
+                }
+                return pid;
+            }
+
+            virtual void PredPath( std::vector<int> &path, const FMatrix &fmat, bst_uint ridx, unsigned gid = 0 ){
+                path.clear();
+                ThreadEntry &e = this->InitTmp();
+                this->PrepareTmp( fmat.GetRow(ridx), e );
+                
+                int pid = (int)gid;
+                path.push_back( pid );
+                // tranverse tree
+                while( !tree[ pid ].is_leaf() ){                    
+                    unsigned split_index = tree[ pid ].split_index();
+                    pid = this->GetNext( pid, e.feat[ split_index ], e.funknown[ split_index ] );
+                    path.push_back( pid );
+                }
+                this->DropTmp( fmat.GetRow(ridx), e );
+            }
+            virtual float Predict( const std::vector<float> &feat, 
+                                   const std::vector<bool>  &funknown,
+                                   unsigned gid = 0 ){
+                utils::Assert( feat.size() >= (size_t)tree.param.num_feature,
+                               "input data smaller than num feature" );
+                int pid = this->GetLeafIndex( feat, funknown, gid );
+                return tree[ pid ].leaf_value();
+            }            
+            virtual void DumpModel( FILE *fo, const utils::FeatMap &fmap, bool with_stats ){
+                tree.DumpModel( fo, fmap, with_stats );
+            }
+        private:
+            inline void CollapseNode( std::vector<float> &grad, 
+                                      std::vector<float> &hess,
+                                      const FMatrix &fmat,
+                                      const std::vector<unsigned> &root_index, 
+                                      int nid ){
+                std::vector<bst_uint> valid_index;
+                for( size_t i = 0; i < grad.size(); i ++ ){
+                    ThreadEntry &e = this->InitTmp();
+                    this->PrepareTmp( fmat.GetRow(i), e );
+                    int pid = root_index.size() == 0 ? 0 : (int)root_index[i];
+                    // tranverse tree
+                    while( !tree[ pid ].is_leaf() ){                    
+                        unsigned split_index = tree[ pid ].split_index();
+                        pid = this->GetNext( pid, e.feat[ split_index ], e.funknown[ split_index ] );
+                        if( pid == nid ){
+                            valid_index.push_back( static_cast<bst_uint>(i) );  break;
+                        }
+                    }
+                    this->DropTmp( fmat.GetRow(i), e );
+                }
+                RowTreeMaker<FMatrix> maker( tree, param, grad, hess, fmat, root_index, constrain ); 
+                maker.Collapse( valid_index, nid );
+                if( !silent ){
+                    printf( "tree collapse end, max_depth=%d\n", tree.param.max_depth );
+                }                
+            }
+            inline void ExpandNode( std::vector<float> &grad, 
+                                    std::vector<float> &hess,
+                                    const FMatrix &fmat,
+                                    const std::vector<unsigned> &root_index, 
+                                    int nid ){
+                std::vector<bst_uint> valid_index;
+                for( size_t i = 0; i < grad.size(); i ++ ){
+                    ThreadEntry &e = this->InitTmp();
+                    this->PrepareTmp( fmat.GetRow(i), e );
+                    unsigned rtidx = root_index.size() == 0 ? 0 : root_index[i]; 
+                    int pid = this->GetLeafIndex( e.feat, e.funknown, rtidx );
+                    this->DropTmp( fmat.GetRow(i), e );
+                    if( pid == nid ) valid_index.push_back( static_cast<bst_uint>(i) ); 
+                }
+                RowTreeMaker<FMatrix> maker( tree, param, grad, hess, fmat, root_index, constrain ); 
+                bool success =  maker.Expand( valid_index, nid );
+                if( !silent ){
+                    printf( "tree expand end, success=%d, max_depth=%d\n", (int)success, tree.MaxDepth() );
+                }                
+            }
+        private:
+            // silent 
+            int silent;
+            RegTree tree;
+            TreeParamTrain param;
+        private:
+            // some training parameters
+            // tree maker
+            int tree_maker;
+            // interaction
+            int interact_type;
+            int interact_node;         
+            // feature constrain
+            utils::FeatConstrain  constrain;   
+        private:
+            struct ThreadEntry{
+                std::vector<float> feat;
+                std::vector<bool>  funknown;
+            };
+            std::vector<ThreadEntry> threadtemp;
+        private:
+            inline ThreadEntry& InitTmp( void ){
+                const int tid = omp_get_thread_num();
+                utils::Assert( tid < (int)threadtemp.size(), "RTreeUpdater: threadtemp pool is too small" );
+                ThreadEntry &e = threadtemp[ tid ];
+                if( e.feat.size() != (size_t)tree.param.num_feature ){
+                    e.feat.resize( tree.param.num_feature );
+                    e.funknown.resize( tree.param.num_feature );
+                    std::fill( e.funknown.begin(), e.funknown.end(), true );
+                }
+                return e;
+            }
+            inline void PrepareTmp( typename FMatrix::RowIter it, ThreadEntry &e ){
+                while( it.Next() ){
+                    const bst_uint findex = it.findex();
+                    utils::Assert( findex < (unsigned)tree.param.num_feature , "input feature execeed bound" );
+                    e.funknown[ findex ] = false;
+                    e.feat[ findex ] = it.fvalue();
+                } 
+            }
+            inline void DropTmp( typename FMatrix::RowIter it, ThreadEntry &e ){
+                while( it.Next() ){
+                    e.funknown[ it.findex() ] = true;
+                }
+            }
+
+            inline int GetNext( int pid, float fvalue, bool is_unknown ){
+                float split_value = tree[ pid ].split_cond();
+                if( is_unknown ){ 
+                    return tree[ pid ].cdefault();
+                }else{
+                    if( fvalue < split_value ) return tree[ pid ].cleft();
+                    else return tree[ pid ].cright();
+                }
+            }
+        };
+    };
+};
+
+#endif

booster/tree/xgboost_tree_model.h

@@ -0,0 +1,554 @@
+#ifndef XGBOOST_TREE_MODEL_H
+#define XGBOOST_TREE_MODEL_H
+/*!
+ * \file xgboost_tree_model.h
+ * \brief generic definition of model structure used in tree models
+ *        used to support learning of boosting tree
+ * \author Tianqi Chen: tianqi.tchen@gmail.com
+ */
+#include <cstring>
+#include "../../utils/xgboost_utils.h"
+#include "../../utils/xgboost_stream.h"
+
+namespace xgboost{
+    namespace booster{
+        /*!
+         * \brief template class of TreeModel 
+         * \tparam TSplitCond data type to indicate split condition
+         * \tparam TNodeStat auxiliary statistics of node to help tree building
+         */
+        template<typename TSplitCond,typename TNodeStat>
+        class TreeModel{
+        public:
+            /*! \brief data type to indicate split condition */
+            typedef TNodeStat  NodeStat;
+            /*! \brief auxiliary statistics of node to help tree building */
+            typedef TSplitCond SplitCond;
+        public:
+            /*! \brief parameters of the tree */
+            struct Param{
+                /*! \brief number of start root */
+                int num_roots;
+                /*! \brief total number of nodes */
+                int num_nodes;
+                /*!\brief number of deleted nodes */
+                int num_deleted;
+                /*! \brief maximum depth, this is a statistics of the tree */
+                int max_depth;
+                /*! \brief  number of features used for tree construction */
+                int num_feature;
+                /*! \brief reserved part */
+                int reserved[ 32 ];
+                /*! \brief constructor */
+                Param( void ){
+                    max_depth = 0;
+                    memset( reserved, 0, sizeof( reserved ) );
+                }
+                /*! 
+                 * \brief set parameters from outside 
+                 * \param name name of the parameter
+                 * \param val  value of the parameter
+                 */
+                inline void SetParam( const char *name, const char *val ){
+                    if( !strcmp("num_roots", name ) )    num_roots = atoi( val );
+                    if( !strcmp("num_feature", name ) )  num_feature = atoi( val );
+                }
+            };
+            /*! \brief tree node */
+            class Node{
+            private:
+                friend class TreeModel<TSplitCond,TNodeStat>;
+                /*! 
+                 * \brief in leaf node, we have weights, in non-leaf nodes, 
+                 *        we have split condition 
+                 */
+                union Info{
+                    float leaf_value;
+                    TSplitCond split_cond;
+                };
+            private:
+                // pointer to parent, highest bit is used to indicate whether it's a left child or not 
+                int parent_;
+                // pointer to left, right
+                int cleft_, cright_;
+                // split feature index, left split or right split depends on the highest bit
+                unsigned sindex_;            
+                // extra info
+                Info info_;
+            private:
+                inline void set_parent( int pidx, bool is_left_child = true ){
+                    if( is_left_child ) pidx |= (1U << 31);
+                    this->parent_ = pidx;
+                }
+            public:
+                /*! \brief index of left child */
+                inline int cleft( void ) const{
+                    return this->cleft_;
+                }
+                /*! \brief index of right child */
+                inline int cright( void ) const{
+                    return this->cright_;
+                }
+                /*! \brief index of default child when feature is missing */
+                inline int cdefault( void ) const{
+                    return this->default_left() ? this->cleft() : this->cright();
+                }
+                /*! \brief feature index of split condition */
+                inline unsigned split_index( void ) const{
+                    return sindex_ & ( (1U<<31) - 1U );
+                }
+                /*! \brief when feature is unknown, whether goes to left child */
+                inline bool default_left( void ) const{
+                    return (sindex_ >> 31) != 0;
+                } 
+                /*! \brief whether current node is leaf node */
+                inline bool is_leaf( void ) const{
+                    return cleft_ == -1;
+                }
+                /*! \brief get leaf value of leaf node */
+                inline float leaf_value( void ) const{
+                    return (this->info_).leaf_value;
+                }
+                /*! \brief get split condition of the node */
+                inline TSplitCond split_cond( void ) const{
+                    return (this->info_).split_cond;
+                }
+                /*! \brief get parent of the node */
+                inline int parent( void ) const{
+                    return parent_ & ( (1U << 31) - 1 );
+                } 
+                /*! \brief whether current node is left child */
+                inline bool is_left_child( void ) const{
+                    return ( parent_ & (1U << 31)) != 0;
+                }
+                /*! \brief whether current node is root */
+                inline bool is_root( void ) const{
+                    return parent_ == -1;
+                }
+                /*! 
+                 * \brief set the right child 
+                 * \param nide node id to right child
+                 */
+                inline void set_right_child( int nid ){
+                    this->cright_ = nid;
+                }
+                /*! 
+                 * \brief set split condition of current node 
+                 * \param split_index feature index to split
+                 * \param split_cond  split condition
+                 * \param default_left the default direction when feature is unknown
+                 */
+                inline void set_split( unsigned split_index, TSplitCond split_cond, bool default_left = false ){
+                    if( default_left ) split_index |= (1U << 31);
+                    this->sindex_ = split_index;
+                    (this->info_).split_cond = split_cond;
+                }
+                /*! 
+                 * \brief set the leaf value of the node
+                 * \param value leaf value
+                 * \param right right index, could be used to store 
+                 *        additional information
+                 */
+                inline void set_leaf( float value, int right = -1 ){
+                    (this->info_).leaf_value = value;
+                    this->cleft_ = -1;
+                    this->cright_ = right;
+                }
+            };
+        protected:
+            // vector of nodes
+            std::vector<Node> nodes;
+            // stats of nodes
+            std::vector<TNodeStat> stats;
+        protected:
+            // free node space, used during training process
+            std::vector<int>  deleted_nodes;
+            // allocate a new node, 
+            // !!!!!! NOTE: may cause BUG here, nodes.resize
+            inline int AllocNode( void ){
+                if( param.num_deleted != 0 ){
+                    int nd = deleted_nodes.back();
+                    deleted_nodes.pop_back();
+                    param.num_deleted --;
+                    return nd;
+                }
+                int nd = param.num_nodes ++;
+                nodes.resize( param.num_nodes );
+                stats.resize( param.num_nodes );
+                return nd;
+            }
+            // delete a tree node
+            inline void DeleteNode( int nid ){
+                utils::Assert( nid >= param.num_roots, "can not delete root");
+                deleted_nodes.push_back( nid );
+                nodes[ nid ].set_parent( -1 );
+                param.num_deleted ++;
+            }
+        public:
+            /*! 
+             * \brief change a non leaf node to a leaf node, delete its children
+             * \param rid node id of the node
+             * \param new leaf value
+             */
+            inline void ChangeToLeaf( int rid, float value ){
+                utils::Assert( nodes[ nodes[rid].cleft()  ].is_leaf(), "can not delete a non termial child");
+                utils::Assert( nodes[ nodes[rid].cright() ].is_leaf(), "can not delete a non termial child");
+                this->DeleteNode( nodes[ rid ].cleft() ); 
+                this->DeleteNode( nodes[ rid ].cright() );
+                nodes[ rid ].set_leaf( value );
+            }
+            /*! 
+             * \brief collapse a non leaf node to a leaf node, delete its children
+             * \param rid node id of the node
+             * \param new leaf value
+             */
+            inline void CollapseToLeaf( int rid, float value ){
+                if( nodes[rid].is_leaf() ) return;
+                if( !nodes[ nodes[rid].cleft()  ].is_leaf() ){
+                    CollapseToLeaf( nodes[rid].cleft(), 0.0f );
+                }
+                if( !nodes[ nodes[rid].cright()  ].is_leaf() ){
+                    CollapseToLeaf( nodes[rid].cright(), 0.0f );
+                }
+                this->ChangeToLeaf( rid, value );
+            }
+        public:
+            /*! \brief model parameter */
+            Param param;
+        public:
+            /*! \brief constructor */
+            TreeModel( void ){
+                param.num_nodes = 1;
+                param.num_roots = 1;
+                param.num_deleted = 0;
+                nodes.resize( 1 );
+            }
+            /*! \brief get node given nid */
+            inline Node &operator[]( int nid ){
+                return nodes[ nid ];
+            }
+            /*! \brief get node statistics given nid */
+            inline NodeStat &stat( int nid ){
+                return stats[ nid ];
+            }
+            /*! \brief initialize the model */
+            inline void InitModel( void ){
+                param.num_nodes = param.num_roots;
+                nodes.resize( param.num_nodes );
+                stats.resize( param.num_nodes );
+                for( int i = 0; i < param.num_nodes; i ++ ){
+                    nodes[i].set_leaf( 0.0f );
+                    nodes[i].set_parent( -1 );
+                }
+            }
+            /*! 
+             * \brief load model from stream
+             * \param fi input stream
+             */
+            inline void LoadModel( utils::IStream &fi ){
+                utils::Assert( fi.Read( &param, sizeof(Param) ) > 0, "TreeModel" );
+                nodes.resize( param.num_nodes ); stats.resize( param.num_nodes );
+                utils::Assert( fi.Read( &nodes[0], sizeof(Node) * nodes.size() ) > 0, "TreeModel::Node" );
+                utils::Assert( fi.Read( &stats[0], sizeof(NodeStat) * stats.size() ) > 0, "TreeModel::Node" );
+
+                deleted_nodes.resize( 0 );
+                for( int i = param.num_roots; i < param.num_nodes; i ++ ){
+                    if( nodes[i].is_root() ) deleted_nodes.push_back( i );
+                }
+                utils::Assert( (int)deleted_nodes.size() == param.num_deleted, "number of deleted nodes do not match" );
+            }
+            /*! 
+             * \brief save model to stream
+             * \param fo output stream
+             */
+            inline void SaveModel( utils::IStream &fo ) const{
+                utils::Assert( param.num_nodes == (int)nodes.size() );
+                utils::Assert( param.num_nodes == (int)stats.size() );
+                fo.Write( &param, sizeof(Param) );
+                fo.Write( &nodes[0], sizeof(Node) * nodes.size() );
+                fo.Write( &stats[0], sizeof(NodeStat) * nodes.size() );
+            }
+            /*! 
+             * \brief add child nodes to node
+             * \param nid node id to add childs
+             */
+            inline void AddChilds( int nid ){
+                int pleft  = this->AllocNode();
+                int pright = this->AllocNode();
+                nodes[ nid ].cleft_  = pleft;
+                nodes[ nid ].cright_ = pright;
+                nodes[ nodes[ nid ].cleft()  ].set_parent( nid, true );
+                nodes[ nodes[ nid ].cright() ].set_parent( nid, false );
+            }
+            /*! 
+             * \brief only add a right child to a leaf node 
+             * \param node id to add right child
+             */
+            inline void AddRightChild( int nid ){
+                int pright = this->AllocNode();
+                nodes[ nid ].right  = pright;
+                nodes[ nodes[ nid ].right  ].set_parent( nid, false );
+            }
+            /*!
+             * \brief get current depth
+             * \param nid node id
+             * \param pass_rchild whether right child is not counted in depth
+             */
+            inline int GetDepth( int nid, bool pass_rchild = false ) const{
+                int depth = 0;
+                while( !nodes[ nid ].is_root() ){
+                    if( !pass_rchild || nodes[ nid ].is_left_child() ) depth ++;
+                    nid = nodes[ nid ].parent();
+                }
+                return depth;
+            }
+            /*!
+             * \brief get maximum depth
+             * \param nid node id
+             */
+            inline int MaxDepth( int nid ) const{
+                if( nodes[nid].is_leaf() ) return 0;
+                return std::max( MaxDepth( nodes[nid].cleft() )+1, 
+                                 MaxDepth( nodes[nid].cright() )+1 );
+            }
+            /*!
+             * \brief get maximum depth
+             */
+            inline int MaxDepth( void ){
+                int maxd = 0;
+                for( int i = 0; i < param.num_roots; ++ i ){
+                    maxd = std::max( maxd, MaxDepth( i ) );
+                }
+                return maxd;
+            }
+            /*! \brief number of extra nodes besides the root */
+            inline int num_extra_nodes( void ) const {
+                return param.num_nodes - param.num_roots - param.num_deleted;
+            }
+            /*! \brief dump model to text file  */
+            inline void DumpModel( FILE *fo, const utils::FeatMap& fmap, bool with_stats ){
+                this->Dump( 0, fo, fmap, 0, with_stats );
+            }
+        private:
+            void Dump( int nid, FILE *fo, const utils::FeatMap& fmap, int depth, bool with_stats ){
+                for( int  i = 0;  i < depth; ++ i ){
+                    fprintf( fo, "\t" );
+                }
+                if( nodes[ nid ].is_leaf() ){
+                    fprintf( fo, "%d:leaf=%f ", nid, nodes[ nid ].leaf_value() );
+                    if( with_stats ){
+                        stat( nid ).Print( fo, true );
+                    }
+                    fprintf( fo, "\n" );
+                }else{
+                    // right then left,
+                    TSplitCond cond = nodes[ nid ].split_cond();
+                    const unsigned split_index = nodes[ nid ].split_index();
+
+                    if( split_index < fmap.size() ){
+                        switch( fmap.type(split_index) ){
+                        case utils::FeatMap::kIndicator:{
+                            int nyes = nodes[ nid ].default_left()?nodes[nid].cright():nodes[nid].cleft();
+                            fprintf( fo, "%d:[%s] yes=%d,no=%d", 
+                                     nid, fmap.name( split_index ),
+                                     nyes, nodes[nid].cdefault() );
+                            break;                            
+                        }
+                        case utils::FeatMap::kInteger:{
+                            fprintf( fo, "%d:[%s<%d] yes=%d,no=%d,missing=%d", 
+                                     nid, fmap.name(split_index), int( float(cond)+1.0f), 
+                                     nodes[ nid ].cleft(), nodes[ nid ].cright(),
+                                     nodes[ nid ].cdefault() );
+                            break;
+                        }
+                        case utils::FeatMap::kFloat:
+                        case utils::FeatMap::kQuantitive:{
+                            fprintf( fo, "%d:[%s<%f] yes=%d,no=%d,missing=%d", 
+                                     nid, fmap.name(split_index), float(cond), 
+                                     nodes[ nid ].cleft(), nodes[ nid ].cright(),
+                                     nodes[ nid ].cdefault() );
+                            break;
+                        }
+                        default: utils::Error("unknown fmap type");
+                        }
+                    }else{
+                        fprintf( fo, "%d:[f%u<%f] yes=%d,no=%d,missing=%d", 
+                                 nid, split_index, float(cond), 
+                                 nodes[ nid ].cleft(), nodes[ nid ].cright(),
+                                 nodes[ nid ].cdefault() );
+                    }
+                    if( with_stats ){
+                        fprintf( fo, " ");
+                        stat( nid ).Print( fo, false );
+                    }
+                    fprintf( fo, "\n" );
+                    this->Dump( nodes[ nid ].cleft(), fo, fmap, depth+1, with_stats );
+                    this->Dump( nodes[ nid ].cright(), fo, fmap, depth+1, with_stats );
+                }                
+            } 
+        };
+    };
+    
+    namespace booster{
+        /*! \brief training parameters for regression tree */
+        struct TreeParamTrain{
+            // learning step size for a time
+            float learning_rate;
+            // minimum loss change required for a split
+            float min_split_loss;
+            // maximum depth of a tree
+            int   max_depth;
+            //----- the rest parameters are less important ----
+            // minimum amount of hessian(weight) allowed in a child
+            float min_child_weight;
+            // weight decay parameter used to control leaf fitting
+            float reg_lambda;
+            // reg method
+            int   reg_method;
+            // default direction choice
+            int   default_direction;
+            // whether we want to do subsample
+            float subsample;
+            // whether to use layerwise aware regularization
+            int   use_layerwise;
+            // number of threads to be used for tree construction, if OpenMP is enabled, if equals 0, use system default
+            int nthread;
+            /*! \brief constructor */
+            TreeParamTrain( void ){
+                learning_rate = 0.3f;
+                min_child_weight = 1.0f;
+                max_depth = 6;
+                reg_lambda = 1.0f;
+                reg_method = 2;
+                default_direction = 0;
+                subsample = 1.0f;
+                use_layerwise = 0;
+                nthread = 0;
+            }
+            /*! 
+             * \brief set parameters from outside 
+             * \param name name of the parameter
+             * \param val  value of the parameter
+             */            
+            inline void SetParam( const char *name, const char *val ){
+                // sync-names 
+                if( !strcmp( name, "gamma") )  min_split_loss = (float)atof( val );
+                if( !strcmp( name, "eta") )    learning_rate  = (float)atof( val );
+                if( !strcmp( name, "lambda") ) reg_lambda = (float)atof( val );
+                // normal tree prameters
+                if( !strcmp( name, "learning_rate") )     learning_rate = (float)atof( val );
+                if( !strcmp( name, "min_child_weight") )  min_child_weight = (float)atof( val );
+                if( !strcmp( name, "min_split_loss") )    min_split_loss = (float)atof( val );
+                if( !strcmp( name, "max_depth") )         max_depth = atoi( val );           
+                if( !strcmp( name, "reg_lambda") )        reg_lambda = (float)atof( val );
+                if( !strcmp( name, "reg_method") )        reg_method = (float)atof( val );
+                if( !strcmp( name, "subsample") )         subsample  = (float)atof( val );
+                if( !strcmp( name, "use_layerwise") )     use_layerwise = atoi( val );
+                if( !strcmp( name, "nthread") )           nthread = atoi( val );
+                if( !strcmp( name, "default_direction") ) {
+                    if( !strcmp( val, "learn") )  default_direction = 0;
+                    if( !strcmp( val, "left") )   default_direction = 1;
+                    if( !strcmp( val, "right") )  default_direction = 2;
+                }
+            }
+        protected:
+            // functions for L1 cost
+            static inline double ThresholdL1( double w, double lambda ){
+                if( w > +lambda ) return w - lambda;
+                if( w < -lambda ) return w + lambda;
+                return 0.0;
+            }
+            inline double CalcWeight( double sum_grad, double sum_hess )const{
+                if( sum_hess < min_child_weight ){
+                    return 0.0;
+                }else{
+                    switch( reg_method ){
+                    case 1: return - ThresholdL1( sum_grad, reg_lambda ) / sum_hess;
+                    case 2: return - sum_grad / ( sum_hess + reg_lambda );
+                        // elstic net
+                    case 3: return - ThresholdL1( sum_grad, 0.5 * reg_lambda ) / ( sum_hess + 0.5 * reg_lambda );
+                    default: return - sum_grad / sum_hess;
+                    }
+                }
+            }
+        private:
+            inline static double Sqr( double a ){
+                return a * a;
+            }
+        public:
+            // calculate the cost of loss function
+            inline double CalcGain( double sum_grad, double sum_hess ) const{
+                if( sum_hess < min_child_weight ){
+                    return 0.0;
+                }
+                switch( reg_method ){
+                case 1 : return Sqr( ThresholdL1( sum_grad, reg_lambda ) ) / sum_hess;
+                case 2 : return Sqr( sum_grad ) / ( sum_hess + reg_lambda );
+                    // elstic net
+                case 3 : return Sqr( ThresholdL1( sum_grad, 0.5 * reg_lambda ) ) / ( sum_hess + 0.5 * reg_lambda );
+                default: return Sqr( sum_grad ) / sum_hess;
+                }        
+            }
+            // KEY:layerwise
+            // calculate cost of root
+            inline double CalcRootGain( double sum_grad, double sum_hess ) const{
+                if( use_layerwise == 0 ) return this->CalcGain( sum_grad, sum_hess );
+                else return 0.0;
+            }
+            // KEY:layerwise
+            // calculate the cost after split
+            // base_weight: the base_weight of parent           
+            inline double CalcGain( double sum_grad, double sum_hess, double base_weight ) const{
+                if( use_layerwise == 0 ) return this->CalcGain( sum_grad, sum_hess );
+                else return this->CalcGain( sum_grad + sum_hess * base_weight, sum_hess );
+            }
+            // calculate the weight of leaf
+            inline double CalcWeight( double sum_grad, double sum_hess, double parent_base_weight )const{
+                if( use_layerwise == 0 ) return CalcWeight( sum_grad, sum_hess );
+                else return parent_base_weight + CalcWeight( sum_grad + parent_base_weight * sum_hess, sum_hess );
+            }           
+            /*! \brief whether need forward small to big search: default right */
+            inline bool need_forward_search( void ) const{
+                return this->default_direction != 1;
+            }
+            /*! \brief whether need forward big to small search: default left */
+            inline bool need_backward_search( void ) const{
+                return this->default_direction != 2;
+            }
+            /*! \brief given the loss change, whether we need to invode prunning */
+            inline bool need_prune( double loss_chg, int depth ) const{
+                return loss_chg < this->min_split_loss;
+            }
+            /*! \brief whether we can split with current hessian */
+            inline bool cannot_split( double sum_hess, int depth ) const{
+                return sum_hess < this->min_child_weight * 2.0; 
+            }
+        };
+    };
+    
+    namespace booster{
+        /*! \brief node statistics used in regression tree */
+        struct RTreeNodeStat{
+            /*! \brief loss chg caused by current split */
+            float loss_chg;
+            /*! \brief sum of hessian values, used to measure coverage of data */
+            float sum_hess;
+            /*! \brief weight of current node */
+            float base_weight;
+            /*! \brief number of child that is leaf node known up to now */
+            int   leaf_child_cnt;
+            /*! \brief print information of current stats to fo */
+            inline void Print( FILE *fo, bool is_leaf ) const{
+                if( !is_leaf ){
+                    fprintf( fo, "gain=%f,cover=%f", loss_chg, sum_hess );
+                }else{
+                    fprintf( fo, "cover=%f", sum_hess );
+                }
+            }
+        };
+        /*! \brief most comment structure of regression tree */
+        class RegTree: public TreeModel<bst_float,RTreeNodeStat>{
+        };
+    };
+};
+#endif

booster/xgboost-inl.hpp

@@ -0,0 +1,39 @@
+#ifndef XGBOOST_INL_HPP
+#define XGBOOST_INL_HPP
+/*!
+ * \file xgboost-inl.hpp
+ * \brief bootser implementations
+ * \author Tianqi Chen: tianqi.tchen@gmail.com
+ */
+// implementation of boosters go to here 
+
+// A good design should have minimum functions defined interface, user should only operate on interface
+// I break it a bit, by using template and let user 'see' the implementation
+// The user should pretend that they only can use the interface, and we are all cool
+// I find this is the only way so far I can think of to make boosters invariant of data structure, 
+// while keep everything fast
+#include "xgboost.h"
+#include "../utils/xgboost_utils.h"
+#include "tree/xgboost_tree.hpp"
+#include "linear/xgboost_linear.hpp"
+
+namespace xgboost{
+    namespace booster{
+        /*!
+         * \brief create a gradient booster, given type of booster
+         * \param booster_type type of gradient booster, can be used to specify implements
+         * \tparam FMatrix input data type for booster
+         * \return the pointer to the gradient booster created
+         */
+        template<typename FMatrix>
+        inline InterfaceBooster<FMatrix> *CreateBooster(int booster_type){
+            switch (booster_type){
+            case 0: return new RegTreeTrainer<FMatrix>();
+            case 1: return new LinearBooster<FMatrix>();
+            default: utils::Error("unknown booster_type"); return NULL;
+            }
+        }
+    }; // namespace booster
+}; // namespace xgboost
+
+#endif // XGBOOST_INL_HPP

booster/xgboost.h

@@ -0,0 +1,157 @@
+#ifndef XGBOOST_H
+#define XGBOOST_H
+/*!
+ * \file xgboost.h
+ * \brief the general gradient boosting interface
+ *
+ *   common practice of this header: use IBooster and CreateBooster<FMatrixS>
+ *
+ * \author Tianqi Chen: tianqi.tchen@gmail.com
+ */
+#include <vector>
+#include "../utils/xgboost_utils.h"
+#include "../utils/xgboost_fmap.h"
+#include "../utils/xgboost_stream.h"
+#include "../utils/xgboost_config.h"
+#include "xgboost_data.h"
+
+/*! \brief namespace for xboost package */
+namespace xgboost{
+    /*! \brief namespace for boosters */
+    namespace booster{
+        /*!
+         * \brief interface of a gradient boosting learner
+         * \tparam FMatrix the feature matrix format that the booster takes
+         */
+        template<typename FMatrix>
+        class InterfaceBooster{
+        public:
+            // interface for model setting and loading
+            // calling procedure:
+            //  (1) booster->SetParam to setting necessary parameters
+            //  (2) if it is first time usage of the model: 
+            //          call booster->InitModel
+            //      else: 
+            //          call booster->LoadModel
+            //  (3) booster->DoBoost to update the model
+            //  (4) booster->Predict to get new prediction
+            /*!
+             * \brief set parameters from outside
+             * \param name name of the parameter
+             * \param val  value of the parameter
+             */
+            virtual void SetParam(const char *name, const char *val) = 0;
+            /*!
+             * \brief load model from stream
+             * \param fi input stream
+             */
+            virtual void LoadModel(utils::IStream &fi) = 0;
+            /*!
+             * \brief save model to stream
+             * \param fo output stream
+             */
+            virtual void SaveModel(utils::IStream &fo) const = 0;
+            /*!
+             * \brief initialize solver before training, called before training
+             * this function is reserved for solver to allocate necessary space and do other preparation
+             */
+            virtual void InitModel(void) = 0;
+        public:
+            /*!
+             * \brief do gradient boost training for one step, using the information given,
+             *        Note: content of grad and hess can change after DoBoost
+             * \param grad first order gradient of each instance
+             * \param hess second order gradient of each instance
+             * \param feats features of each instance
+             * \param root_index pre-partitioned root index of each instance,
+             *          root_index.size() can be 0 which indicates that no pre-partition involved
+             */
+            virtual void DoBoost(std::vector<float> &grad,
+                std::vector<float> &hess,
+                const FMatrix &feats,
+                const std::vector<unsigned> &root_index) = 0;
+            /*!
+             * \brief predict the path ids along a trees, for given sparse feature vector. When booster is a tree
+             * \param path the result of path
+             * \param feats feature matrix
+             * \param row_index  row index in the feature matrix
+             * \param root_index root id of current instance, default = 0
+             */
+            virtual void PredPath(std::vector<int> &path, const FMatrix &feats,
+                bst_uint row_index, unsigned root_index = 0){
+                utils::Error("not implemented");
+            }
+            /*!
+             * \brief predict values for given sparse feature vector
+             *
+             *   NOTE: in tree implementation, Sparse Predict is OpenMP threadsafe, but not threadsafe in general,
+             *         dense version of Predict to ensures threadsafety
+             * \param feats feature matrix
+             * \param row_index  row index in the feature matrix
+             * \param root_index root id of current instance, default = 0
+             * \return prediction
+             */
+            virtual float Predict(const FMatrix &feats, bst_uint row_index, unsigned root_index = 0){
+                utils::Error("not implemented");
+                return 0.0f;
+            }
+            /*!
+             * \brief predict values for given dense feature vector
+             * \param feat feature vector in dense format
+             * \param funknown indicator that the feature is missing
+             * \param rid root id of current instance, default = 0
+             * \return prediction
+             */
+            virtual float Predict(const std::vector<float> &feat,
+                const std::vector<bool>  &funknown,
+                unsigned rid = 0){
+                utils::Error("not implemented");
+                return 0.0f;
+            }
+            /*!
+             * \brief print information
+             * \param fo output stream
+             */
+            virtual void PrintInfo(FILE *fo){}
+            /*!
+             * \brief dump model into text file
+             * \param fo output stream
+             * \param fmap feature map that may help give interpretations of feature
+             * \param with_stats whether print statistics
+             */
+            virtual void DumpModel(FILE *fo, const utils::FeatMap& fmap, bool with_stats = false){
+                utils::Error("not implemented");
+            }
+        public:
+            /*! \brief virtual destructor */
+            virtual ~InterfaceBooster(void){}
+        };
+    };
+    namespace booster{
+        /*!
+         * \brief this will is the most commonly used booster interface
+         *  we try to make booster invariant of data structures, but most cases, FMatrixS is what we wnat
+         */
+        typedef InterfaceBooster<FMatrixS> IBooster;
+    };
+};
+
+namespace xgboost{
+    namespace booster{
+        /*!
+         * \brief create a gradient booster, given type of booster
+         *    normally we use FMatrixS, by calling CreateBooster<FMatrixS>
+         * \param booster_type type of gradient booster, can be used to specify implements
+         * \tparam FMatrix input data type for booster
+         * \return the pointer to the gradient booster created
+         */
+        template<typename FMatrix>
+        inline InterfaceBooster<FMatrix> *CreateBooster(int booster_type);
+    };
+};
+
+// this file includes the template implementations of all boosters
+// the cost of using template is that the user can 'see' all the implementations, which is OK 
+// ignore implementations and focus on the interface:) 
+#include "xgboost-inl.hpp"
+#endif

booster/xgboost_data.h

@@ -0,0 +1,396 @@
+#ifndef XGBOOST_DATA_H
+#define XGBOOST_DATA_H
+
+/*!
+ * \file xgboost_data.h
+ * \brief the input data structure for gradient boosting
+ * \author Tianqi Chen: tianqi.tchen@gmail.com
+ */
+
+#include <vector>
+#include <climits>
+#include "../utils/xgboost_utils.h"
+#include "../utils/xgboost_stream.h"
+#include "../utils/xgboost_matrix_csr.h"
+
+namespace xgboost{
+    namespace booster{
+        /*! \brief interger type used in boost */
+        typedef int bst_int;
+        /*! \brief unsigned interger type used in boost */
+        typedef unsigned bst_uint;
+        /*! \brief float type used in boost */
+        typedef float bst_float;
+        /*! \brief debug option for booster */
+        const bool bst_debug = false;
+    };
+};
+
+namespace xgboost{
+    namespace booster{
+        /**
+         * \brief This is a interface, defining the way to access features,
+         *        by column or by row. This interface is used to make implementation
+         *        of booster does not depend on how feature is stored.
+         *
+         *        Why template instead of virtual class: for efficiency
+         *          feature matrix is going to be used by most inner loop of the algorithm
+         *
+         * \tparam Derived type of actual implementation
+         * \sa FMatrixS: most of time FMatrixS is sufficient, refer to it if you find it confusing
+         */
+        template<typename Derived>
+        struct FMatrix{
+        public:
+            /*! \brief exmaple iterator over one row */
+            struct RowIter{
+                /*!
+                 * \brief move to next position
+                 * \return whether there is element in next position
+                 */
+
+                inline bool Next(void);
+                /*! \return feature index in current position */
+                inline bst_uint  findex(void) const;
+                /*! \return feature value in current position */
+                inline bst_float fvalue(void) const;
+            };
+            /*! \brief example iterator over one column */
+            struct ColIter{
+                /*!
+                 * \brief move to next position
+                 * \return whether there is element in next position
+                 */
+                inline bool Next(void);
+                /*! \return row index of current position  */
+                inline bst_uint  rindex(void) const;
+                /*! \return feature value in current position */
+                inline bst_float fvalue(void) const;
+            };
+            /*! \brief backward iterator over column */
+            struct ColBackIter : public ColIter {};
+        public:
+            /*!
+             * \brief get number of rows
+             * \return number of rows
+             */
+            inline size_t NumRow(void) const;
+            /*!
+             * \brief get number of columns
+             * \return number of columns
+             */
+            inline size_t NumCol(void) const;
+            /*!
+             * \brief get row iterator
+             * \param ridx row index
+             * \return row iterator
+             */
+            inline RowIter GetRow(size_t ridx) const;
+            /*!
+             * \brief get number of column groups, this ise used together with GetRow( ridx, gid )
+             * \return number of column group
+             */
+            inline unsigned NumColGroup(void) const{
+                return 1;
+            }
+            /*!
+             * \brief get row iterator, return iterator of specific column group
+             * \param ridx row index
+             * \param gid colmun group id
+             * \return row iterator, only iterates over features of specified column group
+             */
+            inline RowIter GetRow(size_t ridx, unsigned gid) const;
+
+            /*! \return whether column access is enabled */
+            inline bool HaveColAccess(void) const;
+            /*!
+             * \brief get column iterator, the columns must be sorted by feature value
+             * \param ridx column index
+             * \return column iterator
+             */
+            inline ColIter GetSortedCol(size_t ridx) const;
+            /*!
+             * \brief get column backward iterator, starts from biggest fvalue, and iterator back
+             * \param ridx column index
+             * \return reverse column iterator
+             */
+            inline ColBackIter GetReverseSortedCol(size_t ridx) const;
+        };
+    };
+};
+
+namespace xgboost{
+    namespace booster{
+        /*!
+         * \brief feature matrix to store training instance, in sparse CSR format
+         */
+        class FMatrixS : public FMatrix<FMatrixS>{
+        public:
+            /*! \brief one entry in a row */
+            struct REntry{
+                /*! \brief feature index */
+                bst_uint  findex;
+                /*! \brief feature value */
+                bst_float fvalue;
+                /*! \brief constructor */
+                REntry(void){}
+                /*! \brief constructor */
+                REntry(bst_uint findex, bst_float fvalue) : findex(findex), fvalue(fvalue){}
+                inline static bool cmp_fvalue(const REntry &a, const REntry &b){
+                    return a.fvalue < b.fvalue;
+                }
+            };
+            /*! \brief one row of sparse feature matrix */
+            struct Line{
+                /*! \brief array of feature index */
+                const REntry *data_;
+                /*! \brief size of the data */
+                bst_uint len;
+                /*! \brief get k-th element */
+                inline const REntry& operator[](unsigned i) const{
+                    return data_[i];
+                }
+            };
+            /*! \brief row iterator */
+            struct RowIter{
+                const REntry *dptr_, *end_;
+                RowIter(const REntry* dptr, const REntry* end)
+                    :dptr_(dptr), end_(end){}
+                inline bool Next(void){
+                    if (dptr_ == end_) return false;
+                    else{
+                        ++dptr_; return true;
+                    }
+                }
+                inline bst_uint  findex(void) const{
+                    return dptr_->findex;
+                }
+                inline bst_float fvalue(void) const{
+                    return dptr_->fvalue;
+                }
+            };
+            /*! \brief column iterator */
+            struct ColIter : public RowIter{
+                ColIter(const REntry* dptr, const REntry* end)
+                :RowIter(dptr, end){}
+                inline bst_uint  rindex(void) const{
+                    return this->findex();
+                }
+            };
+            /*! \brief reverse column iterator */
+            struct ColBackIter : public ColIter{
+                ColBackIter(const REntry* dptr, const REntry* end)
+                :ColIter(dptr, end){}
+                // shadows RowIter::Next
+                inline bool Next(void){
+                    if (dptr_ == end_) return false;
+                    else{
+                        --dptr_; return true;
+                    }
+                }
+            };
+        public:
+            /*! \brief constructor */
+            FMatrixS(void){ this->Clear(); }
+            /*!  \brief get number of rows */
+            inline size_t NumRow(void) const{
+                return row_ptr_.size() - 1;
+            }
+            /*!
+             * \brief get number of nonzero entries
+             * \return number of nonzero entries
+             */
+            inline size_t NumEntry(void) const{
+                return row_data_.size();
+            }
+            /*! \brief clear the storage */
+            inline void Clear(void){
+                row_ptr_.clear();
+                row_ptr_.push_back(0);
+                row_data_.clear();
+                col_ptr_.clear();
+                col_data_.clear();
+            }
+            /*! \brief get sparse part of current row */
+            inline Line operator[](size_t sidx) const{
+                Line sp;
+                utils::Assert(!bst_debug || sidx < this->NumRow(), "row id exceed bound");
+                sp.len = static_cast<bst_uint>(row_ptr_[sidx + 1] - row_ptr_[sidx]);
+                sp.data_ = &row_data_[row_ptr_[sidx]];
+                return sp;
+            }
+            /*!
+             * \brief add a row to the matrix, with data stored in STL container
+             * \param findex feature index
+             * \param fvalue feature value
+             *  \param fstart start bound of feature
+             *  \param fend   end bound range of feature
+             * \return the row id added line
+             */
+            inline size_t AddRow(const std::vector<bst_uint> &findex,
+                                 const std::vector<bst_float> &fvalue,
+                                 unsigned fstart = 0, unsigned fend = UINT_MAX){
+                utils::Assert(findex.size() == fvalue.size());
+                unsigned cnt = 0;
+                for (size_t i = 0; i < findex.size(); i++){
+                    if (findex[i] < fstart || findex[i] >= fend) continue;
+                    row_data_.push_back(REntry(findex[i], fvalue[i]));
+                    cnt++;
+                }
+                row_ptr_.push_back(row_ptr_.back() + cnt);
+                return row_ptr_.size() - 2;
+            }
+            /*!  \brief get row iterator*/
+            inline RowIter GetRow(size_t ridx) const{
+                utils::Assert(!bst_debug || ridx < this->NumRow(), "row id exceed bound");
+                return RowIter(&row_data_[row_ptr_[ridx]] - 1, &row_data_[row_ptr_[ridx + 1]] - 1);
+            }
+            /*!  \brief get row iterator*/
+            inline RowIter GetRow(size_t ridx, unsigned gid) const{
+                utils::Assert(gid == 0, "FMatrixS only have 1 column group");
+                return FMatrixS::GetRow(ridx);
+            }
+        public:
+            /*! \return whether column access is enabled */
+            inline bool HaveColAccess(void) const{
+                return col_ptr_.size() != 0 && col_data_.size() == row_data_.size();
+            }
+            /*!  \brief get number of colmuns */
+            inline size_t NumCol(void) const{
+                utils::Assert(this->HaveColAccess());
+                return col_ptr_.size() - 1;
+            }
+            /*!  \brief get col iterator*/
+            inline ColIter GetSortedCol(size_t cidx) const{
+                utils::Assert(!bst_debug || cidx < this->NumCol(), "col id exceed bound");
+                return ColIter(&col_data_[col_ptr_[cidx]] - 1, &col_data_[col_ptr_[cidx + 1]] - 1);
+            }
+            /*!  \brief get col iterator */
+            inline ColBackIter GetReverseSortedCol(size_t cidx) const{
+                utils::Assert(!bst_debug || cidx < this->NumCol(), "col id exceed bound");
+                return ColBackIter(&col_data_[col_ptr_[cidx + 1]], &col_data_[col_ptr_[cidx]]);
+            }
+            /*!
+             * \brief intialize the data so that we have both column and row major
+             *        access, call this whenever we need column access
+             */
+            inline void InitData(void){
+                utils::SparseCSRMBuilder<REntry> builder(col_ptr_, col_data_);
+                builder.InitBudget(0);
+                for (size_t i = 0; i < this->NumRow(); i++){
+                    for (RowIter it = this->GetRow(i); it.Next();){
+                        builder.AddBudget(it.findex());
+                    }
+                }
+                builder.InitStorage();
+                for (size_t i = 0; i < this->NumRow(); i++){
+                    for (RowIter it = this->GetRow(i); it.Next();){
+                        builder.PushElem(it.findex(), REntry((bst_uint)i, it.fvalue()));
+                    }
+                }
+                // sort columns
+                unsigned ncol = static_cast<unsigned>(this->NumCol());
+                #pragma omp parallel for schedule(static)
+                for (unsigned i = 0; i < ncol; i++){
+                    std::sort(&col_data_[col_ptr_[i]], &col_data_[col_ptr_[i + 1]], REntry::cmp_fvalue);
+                }
+            }
+            /*!
+             * \brief save data to binary stream
+             *        note: since we have size_t in ptr,
+             *              the function is not consistent between 64bit and 32bit machine
+             * \param fo output stream
+             */
+            inline void SaveBinary(utils::IStream &fo) const{
+                FMatrixS::SaveBinary(fo, row_ptr_, row_data_);
+                int col_access = this->HaveColAccess() ? 1 : 0;
+                fo.Write(&col_access, sizeof(int));
+                if (col_access != 0){
+                    FMatrixS::SaveBinary(fo, col_ptr_, col_data_);
+                }
+            }
+            /*!
+             * \brief load data from binary stream
+             *        note: since we have size_t in ptr,
+             *              the function is not consistent between 64bit and 32bit machin
+             * \param fi input stream
+             */
+            inline void LoadBinary(utils::IStream &fi){
+                FMatrixS::LoadBinary(fi, row_ptr_, row_data_);
+                int col_access;
+                fi.Read(&col_access, sizeof(int));
+                if (col_access != 0){
+                    FMatrixS::LoadBinary(fi, col_ptr_, col_data_);
+                }else{
+                    this->InitData();                    
+                }
+            }
+            /*!
+            * \brief load from text file
+            * \param fi input file pointer
+            */
+            inline void LoadText(FILE *fi){
+                this->Clear();
+                int ninst;
+                while (fscanf(fi, "%d", &ninst) == 1){
+                    std::vector<booster::bst_uint>  findex;
+                    std::vector<booster::bst_float> fvalue;
+                    while (ninst--){
+                        unsigned index; float value;
+                        utils::Assert(fscanf(fi, "%u:%f", &index, &value) == 2, "load Text");
+                        findex.push_back(index); fvalue.push_back(value);
+                    }
+                    this->AddRow(findex, fvalue);
+                }
+                // initialize column support as well
+                this->InitData();
+            }
+        private:
+            /*!
+             * \brief save data to binary stream
+             * \param fo output stream
+             * \param ptr pointer data
+             * \param data data content
+             */
+            inline static void SaveBinary(utils::IStream &fo,
+                const std::vector<size_t> &ptr,
+                const std::vector<REntry> &data){
+                size_t nrow = ptr.size() - 1;
+                fo.Write(&nrow, sizeof(size_t));
+                fo.Write(&ptr[0], ptr.size() * sizeof(size_t));
+                if (data.size() != 0){
+                    fo.Write(&data[0], data.size() * sizeof(REntry));
+                }
+            }
+            /*!
+             * \brief load data from binary stream
+             * \param fi input stream
+             * \param ptr pointer data
+             * \param data data content
+             */
+            inline static void LoadBinary(utils::IStream &fi,
+                std::vector<size_t> &ptr,
+                std::vector<REntry> &data){
+                size_t nrow;
+                utils::Assert(fi.Read(&nrow, sizeof(size_t)) != 0, "Load FMatrixS");
+                ptr.resize(nrow + 1);
+                utils::Assert(fi.Read(&ptr[0], ptr.size() * sizeof(size_t)) != 0, "Load FMatrixS");
+
+                data.resize(ptr.back());
+                if (data.size() != 0){
+                    utils::Assert(fi.Read(&data[0], data.size() * sizeof(REntry)) != 0, "Load FMatrixS");
+                }
+            }
+        public:
+            /*! \brief row pointer of CSR sparse storage */
+            std::vector<size_t>  row_ptr_;
+            /*! \brief data in the row */
+            std::vector<REntry>  row_data_;
+            /*! \brief column pointer of CSC format */
+            std::vector<size_t>  col_ptr_;
+            /*! \brief column datas */
+            std::vector<REntry>  col_data_;
+        };
+    };
+};
+#endif

booster/xgboost_gbmbase.h

@@ -0,0 +1,429 @@
+#ifndef XGBOOST_GBMBASE_H
+#define XGBOOST_GBMBASE_H
+
+#include <cstring>
+#include "xgboost.h"
+#include "xgboost_data.h"
+#include "../utils/xgboost_omp.h"
+#include "../utils/xgboost_config.h"
+/*!
+ * \file xgboost_gbmbase.h
+ * \brief a base model class,
+ *        that assembles the ensembles of booster together and do model update
+ *        this class can be used as base code to create booster variants
+ *
+ *        The detailed implementation of boosters should start by using the class
+ *        provided by this file
+ *
+ * \author Tianqi Chen: tianqi.tchen@gmail.com
+ */
+namespace xgboost{
+    namespace booster{
+        /*!
+         * \brief a base model class,
+         *        that assembles the ensembles of booster together and provide single routines to do prediction buffer and update
+         *        this class can be used as base code to create booster variants
+         *         *
+         *  relation to xgboost.h:
+         *    (1) xgboost.h provides a interface to a single booster(e.g. a single regression tree )
+         *        while GBMBaseModel builds upon IBooster to build a class that
+         *        ensembls the boosters together;
+         *    (2) GBMBaseModel provides prediction buffering scheme to speedup training;
+         *    (3) Summary: GBMBaseModel is a standard wrapper for boosting ensembles;
+         *
+         *  Usage of this class, the number index gives calling dependencies:
+         *    (1) model.SetParam to set the parameters
+         *    (2) model.LoadModel to load old models or model.InitModel to create a new model
+         *    (3) model.InitTrainer before calling model.Predict and model.DoBoost
+         *    (4) model.Predict to get predictions given a instance
+         *    (4) model.DoBoost to update the ensembles, add new booster to the model
+         *    (4) model.SaveModel to save learned results
+         *
+         *  Bufferring: each instance comes with a buffer_index in Predict.
+         *              when mparam.num_pbuffer != 0, a unique buffer index can be
+         *              assigned to each instance to buffer previous results of boosters,
+         *              this helps to speedup training, so consider assign buffer_index
+         *              for each training instances, if buffer_index = -1, the code
+         *              recalculate things from scratch and will still works correctly
+         */
+        class GBMBase{
+        public:
+            /*! \brief number of thread used */
+            GBMBase(void){}
+            /*! \brief destructor */
+            virtual ~GBMBase(void){
+                this->FreeSpace();
+            }
+            /*!
+             * \brief set parameters from outside
+             * \param name name of the parameter
+             * \param val  value of the parameter
+             */
+            inline void SetParam(const char *name, const char *val){
+                if (!strncmp(name, "bst:", 4)){
+                    cfg.PushBack(name + 4, val);
+                }
+                if (!strcmp(name, "silent")){
+                    cfg.PushBack(name, val);
+                }
+                tparam.SetParam(name, val);
+                if (boosters.size() == 0) mparam.SetParam(name, val);
+            }
+            /*!
+             * \brief load model from stream
+             * \param fi input stream
+             */
+            inline void LoadModel(utils::IStream &fi){
+                if (boosters.size() != 0) this->FreeSpace();
+                utils::Assert(fi.Read(&mparam, sizeof(ModelParam)) != 0);
+                boosters.resize(mparam.num_boosters);
+                for (size_t i = 0; i < boosters.size(); i++){
+                    boosters[i] = booster::CreateBooster<FMatrixS>(mparam.booster_type);
+                    boosters[i]->LoadModel(fi);
+                }
+                {// load info 
+                    booster_info.resize(mparam.num_boosters);
+                    if (mparam.num_boosters != 0){
+                        utils::Assert(fi.Read(&booster_info[0], sizeof(int)*mparam.num_boosters) != 0);
+                    }
+                }
+                if (mparam.num_pbuffer != 0){
+                    pred_buffer.resize(mparam.PredBufferSize());
+                    pred_counter.resize(mparam.PredBufferSize());
+                    utils::Assert(fi.Read(&pred_buffer[0], pred_buffer.size()*sizeof(float)) != 0);
+                    utils::Assert(fi.Read(&pred_counter[0], pred_counter.size()*sizeof(unsigned)) != 0);
+                }
+            }
+            /*!
+             * \brief save model to stream
+             * \param fo output stream
+             */
+            inline void SaveModel(utils::IStream &fo) const {
+                utils::Assert(mparam.num_boosters == (int)boosters.size());
+                fo.Write(&mparam, sizeof(ModelParam));
+                for (size_t i = 0; i < boosters.size(); i++){
+                    boosters[i]->SaveModel(fo);
+                }
+                if (booster_info.size() != 0){
+                    fo.Write(&booster_info[0], sizeof(int)* booster_info.size());
+                }
+                if (mparam.num_pbuffer != 0){
+                    fo.Write(&pred_buffer[0], pred_buffer.size()*sizeof(float));
+                    fo.Write(&pred_counter[0], pred_counter.size()*sizeof(unsigned));
+                }
+            }
+            /*!
+             * \brief initialize the current data storage for model, if the model is used first time, call this function
+             */
+            inline void InitModel(void){
+                pred_buffer.clear(); pred_counter.clear();
+                pred_buffer.resize(mparam.PredBufferSize(), 0.0);
+                pred_counter.resize(mparam.PredBufferSize(), 0);
+                utils::Assert(mparam.num_boosters == 0);
+                utils::Assert(boosters.size() == 0);
+            }
+            /*!
+             * \brief initialize solver before training, called before training
+             * this function is reserved for solver to allocate necessary space and do other preparation
+             */
+            inline void InitTrainer(void){
+                if (tparam.nthread != 0){
+                    omp_set_num_threads(tparam.nthread);
+                }
+                if (mparam.num_booster_group == 0) mparam.num_booster_group = 1;
+                // make sure all the boosters get the latest parameters
+                for (size_t i = 0; i < this->boosters.size(); i++){
+                    this->ConfigBooster(this->boosters[i]);
+                }
+            }
+            /*!
+             * \brief DumpModel
+             * \param fo text file
+             * \param fmap feature map that may help give interpretations of feature
+             * \param with_stats whether print statistics
+             */
+            inline void DumpModel(FILE *fo, const utils::FeatMap& fmap, bool with_stats){
+                for (size_t i = 0; i < boosters.size(); i++){
+                    fprintf(fo, "booster[%d]\n", (int)i);
+                    boosters[i]->DumpModel(fo, fmap, with_stats);
+                }
+            }
+            /*!
+             * \brief Dump path of all trees
+             * \param fo text file
+             * \param data input data
+             */
+            inline void DumpPath(FILE *fo, const FMatrixS &data){
+                for (size_t i = 0; i < data.NumRow(); ++i){
+                    for (size_t j = 0; j < boosters.size(); ++j){
+                        if (j != 0) fprintf(fo, "\t");
+                        std::vector<int> path;
+                        boosters[j]->PredPath(path, data, i);
+                        fprintf(fo, "%d", path[0]);
+                        for (size_t k = 1; k < path.size(); ++k){
+                            fprintf(fo, ",%d", path[k]);
+                        }
+                    }
+                    fprintf(fo, "\n");
+                }
+            }
+        public:
+            /*!
+             * \brief do gradient boost training for one step, using the information given
+             *        Note: content of grad and hess can change after DoBoost
+             * \param grad first order gradient of each instance
+             * \param hess second order gradient of each instance
+             * \param feats features of each instance
+             * \param root_index pre-partitioned root index of each instance,
+             *          root_index.size() can be 0 which indicates that no pre-partition involved
+             * \param bst_group which booster group it belongs to, by default, we only have 1 booster group, and leave this parameter as default
+             */
+            inline void DoBoost(std::vector<float> &grad,
+                                std::vector<float> &hess,
+                                const booster::FMatrixS &feats,
+                                const std::vector<unsigned> &root_index,
+                                int bst_group = 0 ) {
+                booster::IBooster *bst = this->GetUpdateBooster( bst_group );
+                bst->DoBoost(grad, hess, feats, root_index);
+            }
+            /*!
+             * \brief predict values for given sparse feature vector
+             *   NOTE: in tree implementation, this is only OpenMP threadsafe, but not threadsafe
+             * \param feats feature matrix
+             * \param row_index  row index in the feature matrix
+             * \param buffer_index the buffer index of the current feature line, default -1 means no buffer assigned
+             * \param root_index root id of current instance, default = 0
+             * \param bst_group booster group index 
+             * \return prediction
+             */
+            inline float Predict(const FMatrixS &feats, bst_uint row_index, 
+                                 int buffer_index = -1, unsigned root_index = 0, int bst_group = 0 ){
+                size_t itop = 0;
+                float  psum = 0.0f;
+                const int bid = mparam.BufferOffset(buffer_index, bst_group);
+
+                // load buffered results if any
+                if (mparam.do_reboost == 0 && bid >= 0){
+                    itop = this->pred_counter[bid];
+                    psum = this->pred_buffer[bid];
+                }
+
+                for (size_t i = itop; i < this->boosters.size(); ++i ){
+                    if( booster_info[i] == bst_group ){
+                        psum += this->boosters[i]->Predict(feats, row_index, root_index);
+                    }
+                }
+                // updated the buffered results
+                if (mparam.do_reboost == 0 && bid >= 0){
+                    this->pred_counter[bid] = static_cast<unsigned>(boosters.size());
+                    this->pred_buffer[bid] = psum;
+                }
+                return psum;
+            }
+            /*! \return number of boosters so far */
+            inline int NumBoosters(void) const{
+                return mparam.num_boosters;
+            }
+            /*! \return number of booster groups */
+            inline int NumBoosterGroup(void) const{
+                if( mparam.num_booster_group == 0 ) return 1;
+                return mparam.num_booster_group;
+            }
+        public:
+            //--------trial code for interactive update an existing booster------
+            //-------- usually not needed, ignore this region ---------
+            /*!
+             * \brief same as Predict, but removes the prediction of booster to be updated
+             *        this function must be called once and only once for every data with pbuffer
+             */
+            inline float InteractPredict(const FMatrixS &feats, bst_uint row_index, 
+                                         int buffer_index = -1, unsigned root_index = 0, int bst_group = 0){
+                float psum = this->Predict(feats, row_index, buffer_index, root_index);
+                if (tparam.reupdate_booster != -1){
+                    const int bid = tparam.reupdate_booster;
+                    utils::Assert(bid >= 0 && bid < (int)boosters.size(), "interact:booster_index exceed existing bound");
+                    if( bst_group == booster_info[bid] ){
+                        psum -= boosters[bid]->Predict(feats, row_index, root_index);
+                    }
+                    if (mparam.do_reboost == 0 && buffer_index >= 0){
+                        this->pred_buffer[mparam.BufferOffset(buffer_index,bst_group)] = psum;
+                    }
+                }
+                return psum;
+            }
+            /*! \brief delete the specified booster */
+            inline void DelteBooster(void){
+                const int bid = tparam.reupdate_booster;
+                utils::Assert(bid >= 0 && bid < mparam.num_boosters, "must specify booster index for deletion");
+                delete boosters[bid];
+                for (int i = bid + 1; i < mparam.num_boosters; ++i){
+                    boosters[i - 1] = boosters[i];
+                    booster_info[i - 1] = booster_info[i];
+                }
+                boosters.resize(mparam.num_boosters -= 1);
+                booster_info.resize(boosters.size());                
+                // update pred counter
+                for( size_t i = 0; i < pred_counter.size(); ++ i ){
+                    if( pred_counter[i] > (unsigned)bid ) pred_counter[i] -= 1;                    
+                }
+            }
+            /*! \brief update the prediction buffer, after booster have been updated */
+            inline void InteractRePredict(const FMatrixS &feats, bst_uint row_index, 
+                                          int buffer_index = -1, unsigned root_index = 0, int bst_group = 0 ){
+                if (tparam.reupdate_booster != -1){
+                    const int bid = tparam.reupdate_booster;
+                    if( booster_info[bid]  != bst_group ) return;
+                    utils::Assert(bid >= 0 && bid < (int)boosters.size(), "interact:booster_index exceed existing bound");
+                    if (mparam.do_reboost == 0 && buffer_index >= 0){
+                        this->pred_buffer[mparam.BufferOffset(buffer_index,bst_group)] += boosters[bid]->Predict(feats, row_index, root_index);
+                    }
+                }
+            }
+            //-----------non public fields afterwards-------------
+        protected:
+            /*! \brief free space of the model */
+            inline void FreeSpace(void){
+                for (size_t i = 0; i < boosters.size(); i++){
+                    delete boosters[i];
+                }
+                boosters.clear(); booster_info.clear(); mparam.num_boosters = 0;
+            }
+            /*! \brief configure a booster */
+            inline void ConfigBooster(booster::IBooster *bst){
+                cfg.BeforeFirst();
+                while (cfg.Next()){
+                    bst->SetParam(cfg.name(), cfg.val());
+                }
+            }
+            /*!
+             * \brief get a booster to update
+             * \return the booster created
+             */
+            inline booster::IBooster *GetUpdateBooster(int bst_group){
+                if (tparam.reupdate_booster != -1){
+                    const int bid = tparam.reupdate_booster;
+                    utils::Assert(bid >= 0 && bid < (int)boosters.size(), "interact:booster_index exceed existing bound");
+                    this->ConfigBooster(boosters[bid]);
+                    utils::Assert( bst_group == booster_info[bid], "booster group must match existing reupdate booster");
+                    return boosters[bid];
+                }
+
+                if (mparam.do_reboost == 0 || boosters.size() == 0){
+                    mparam.num_boosters += 1;
+                    boosters.push_back(booster::CreateBooster<FMatrixS>(mparam.booster_type));
+                    booster_info.push_back(bst_group);
+                    this->ConfigBooster(boosters.back());
+                    boosters.back()->InitModel();
+                }
+                else{
+                    this->ConfigBooster(boosters.back());
+                }
+                return boosters.back();
+            }
+        protected:
+            /*! \brief model parameters */
+            struct ModelParam{
+                /*! \brief number of boosters */
+                int num_boosters;
+                /*! \brief type of tree used */
+                int booster_type;
+                /*! \brief number of root: default 0, means single tree */
+                int num_roots;
+                /*! \brief number of features to be used by boosters */
+                int num_feature;
+                /*! \brief size of predicton buffer allocated for buffering boosting computation */
+                int num_pbuffer;
+                /*!
+                 * \brief whether we repeatly update a single booster each round: default 0
+                 *        set to 1 for linear booster, so that regularization term can be considered
+                 */
+                int do_reboost;
+                /*! 
+                 * \brief number of booster group, how many predictions a single 
+                 *        input instance could corresponds to
+                 */
+                int num_booster_group;
+                /*! \brief reserved parameters */
+                int reserved[31];
+                /*! \brief constructor */
+                ModelParam(void){
+                    num_boosters = 0;
+                    booster_type = 0;
+                    num_roots = num_feature = 0;
+                    do_reboost = 0;
+                    num_pbuffer = 0;
+                    num_booster_group = 1;
+                    memset(reserved, 0, sizeof(reserved));
+                }
+                /*!
+                 * \brief set parameters from outside
+                 * \param name name of the parameter
+                 * \param val  value of the parameter
+                 */
+                inline void SetParam(const char *name, const char *val){
+                    if (!strcmp("booster_type", name)){
+                        booster_type = atoi(val);
+                        // linear boost automatically set do reboost
+                        if (booster_type == 1) do_reboost = 1;
+                    }
+                    if (!strcmp("num_pbuffer", name))       num_pbuffer = atoi(val);
+                    if (!strcmp("do_reboost", name))        do_reboost = atoi(val);
+                    if (!strcmp("num_booster_group", name)) num_booster_group = atoi(val);
+                    if (!strcmp("bst:num_roots", name))     num_roots = atoi(val);
+                    if (!strcmp("bst:num_feature", name))   num_feature = atoi(val);
+                }
+                inline int PredBufferSize(void) const{
+                    if (num_booster_group == 0) return num_pbuffer;
+                    else return num_booster_group * num_pbuffer;
+                }
+                inline int BufferOffset( int buffer_index, int bst_group ) const{
+                    if( buffer_index < 0 ) return -1;
+                    utils::Assert( buffer_index < num_pbuffer, "buffer_indexexceed num_pbuffer" ); 
+                    return buffer_index + num_pbuffer * bst_group;
+                    
+                }
+            };
+            /*! \brief training parameters */
+            struct TrainParam{
+                /*! \brief number of OpenMP threads */
+                int nthread;
+                /*!
+                 * \brief index of specific booster to be re-updated, default = -1: update new booster
+                 *  parameter this is part of trial interactive update mode
+                 */
+                int reupdate_booster;
+                /*! \brief constructor */
+                TrainParam(void) {
+                    nthread = 1;
+                    reupdate_booster = -1;
+                }
+                /*!
+                 * \brief set parameters from outside
+                 * \param name name of the parameter
+                 * \param val  value of the parameter
+                 */
+                inline void SetParam(const char *name, const char *val){
+                    if (!strcmp("nthread", name))                 nthread = atoi(val);
+                    if (!strcmp("interact:booster_index", name))  reupdate_booster = atoi(val);
+                }
+            };
+        protected:
+            /*! \brief model parameters */
+            ModelParam mparam;
+            /*! \brief training parameters */
+            TrainParam tparam;
+        protected:
+            /*! \brief component boosters */
+            std::vector<booster::IBooster*> boosters;
+            /*! \brief some information indicator of the booster, reserved */
+            std::vector<int> booster_info;
+            /*! \brief prediction buffer */
+            std::vector<float>    pred_buffer;
+            /*! \brief prediction buffer counter, record the progress so fart of the buffer */
+            std::vector<unsigned> pred_counter;
+            /*! \brief configurations saved for each booster */
+            utils::ConfigSaver cfg;
+        };
+    };
+};
+#endif

build.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-# this is a simple script to make xgboost in MAC nad Linux
-# basically, it first try to make with OpenMP, if fails, disable OpenMP and make again
-# This will automatically make xgboost for MAC users who do not have openmp support
-# In most cases, type make will give what you want
-if make; then
-    echo "Successfully build multi-thread xgboost"
-else
-    echo "-----------------------------"
-    echo "Building multi-thread xgboost failed"
-    echo "Start to build single-thread xgboost"
-    make clean
-    make no_omp=1
-    echo "Successfully build single-thread xgboost"
-fi

demo/README.md

@@ -1,27 +0,0 @@
-XGBoost Examples
-====
-This folder contains the all example codes using xgboost. 
-
-* Contribution of exampls, benchmarks is more than welcomed!
-* If you like to share how you use xgboost to solve your problem, send a pull request:)
- 
-Features Walkthrough
-====
-This is a list of short codes introducing different functionalities of xgboost and its wrapper.
-* Basic walkthrough of wrappers [python](guide-python/basic_walkthrough.py)
-* Cutomize loss function, and evaluation metric [python](guide-python/custom_objective.py)
-* Boosting from existing prediction [python](guide-python/boost_from_prediction.py)
-* Predicting using first n trees [python](guide-python/predict_first_ntree.py)
-* Generalized Linear Model [python](guide-python/generalized_linear_model.py)
-* Cross validation [python](guide-python/cross_validation.py)
-
-Basic Examples by Tasks
-====
-* [Binary classification](binary_classification)
-* [Multiclass classification](multiclass_classification)
-* [Regression](regression)
-* [Learning to Rank](rank)
-
-Benchmarks
-====
-* [Starter script for Kaggle Higgs Boson](kaggle-higgs)

demo/binary_classification/mushroom.conf

@@ -1,18 +1,18 @@
 # General Parameters, see comment for each definition
-# choose the booster, can be gbtree or gblinear
+# choose the tree booster, 0: tree, 1: linear
-booster = gbtree
+booster_type = 0 
 # choose logistic regression loss function for binary classification
 objective = binary:logistic
 
 # Tree Booster Parameters
 # step size shrinkage
-eta = 1.0 
+bst:eta = 1.0 
 # minimum loss reduction required to make a further partition
-gamma = 1.0 
+bst:gamma = 1.0 
 # minimum sum of instance weight(hessian) needed in a child
-min_child_weight = 1 
+bst:min_child_weight = 1 
 # maximum depth of a tree
-max_depth = 3 
+bst:max_depth = 3 
 
 # Task Parameters
 # the number of round to do boosting

demo/data/README.md

@@ -1,2 +0,0 @@
-This folder contains processed example dataset used by the demos.
-Copyright of the dataset belongs to the original copyright holder

demo/guide-python/README.md

@@ -1,8 +0,0 @@
-XGBoost Python Feature Walkthrough
-====
-* [Basic walkthrough of wrappers](basic_walkthrough.py) 
-* [Cutomize loss function, and evaluation metric](custom_objective.py)
-* [Boosting from existing prediction](boost_from_prediction.py)
-* [Predicting using first n trees](predict_first_ntree.py)
-* [Generalized Linear Model](generalized_linear_model.py)
-* [Cross validation](cross_validation.py)

demo/guide-python/basic_walkthrough.py

@@ -1,76 +0,0 @@
-#!/usr/bin/python
-import sys
-import numpy as np
-import scipy.sparse
-# append the path to xgboost, you may need to change the following line
-# alternatively, you can add the path to PYTHONPATH environment variable
-sys.path.append('../../wrapper')
-import xgboost as xgb
-
-### simple example
-# load file from text file, also binary buffer generated by xgboost
-dtrain = xgb.DMatrix('../data/agaricus.txt.train')
-dtest = xgb.DMatrix('../data/agaricus.txt.test')
-
-# specify parameters via map, definition are same as c++ version
-param = {'max_depth':2, 'eta':1, 'silent':1, 'objective':'binary:logistic' }
-
-# specify validations set to watch performance
-watchlist  = [(dtest,'eval'), (dtrain,'train')]
-num_round = 2
-bst = xgb.train(param, dtrain, num_round, watchlist)
-
-# this is prediction
-preds = bst.predict(dtest)
-labels = dtest.get_label()
-print ('error=%f' % (  sum(1 for i in range(len(preds)) if int(preds[i]>0.5)!=labels[i]) /float(len(preds))))
-bst.save_model('0001.model')
-# dump model
-bst.dump_model('dump.raw.txt')
-# dump model with feature map
-bst.dump_model('dump.nice.txt','../data/featmap.txt')
-
-# save dmatrix into binary buffer
-dtest.save_binary('dtest.buffer')
-bst.save_model('xgb.model')
-# load model and data in 
-bst2 = xgb.Booster(model_file='xgb.model')
-dtest2 = xgb.DMatrix('dtest.buffer')
-preds2 = bst2.predict(dtest2)
-# assert they are the same
-assert np.sum(np.abs(preds2-preds)) == 0
-
-###
-# build dmatrix from scipy.sparse
-print ('start running example of build DMatrix from scipy.sparse CSR Matrix')
-labels = []
-row = []; col = []; dat = []
-i = 0
-for l in open('../data/agaricus.txt.train'):
-    arr = l.split()
-    labels.append( int(arr[0]))
-    for it in arr[1:]:
-        k,v = it.split(':')
-        row.append(i); col.append(int(k)); dat.append(float(v))
-    i += 1
-csr = scipy.sparse.csr_matrix( (dat, (row,col)) )
-dtrain = xgb.DMatrix( csr, label = labels )
-watchlist  = [(dtest,'eval'), (dtrain,'train')]
-bst = xgb.train( param, dtrain, num_round, watchlist )
-
-print ('start running example of build DMatrix from scipy.sparse CSC Matrix')
-# we can also construct from csc matrix
-csc = scipy.sparse.csc_matrix( (dat, (row,col)) )
-dtrain = xgb.DMatrix(csc, label=labels)
-watchlist  = [(dtest,'eval'), (dtrain,'train')]
-bst = xgb.train( param, dtrain, num_round, watchlist )
-
-print ('start running example of build DMatrix from numpy array')
-# NOTE: npymat is numpy array, we will convert it into scipy.sparse.csr_matrix in internal implementation
-# then convert to DMatrix
-npymat = csr.todense()
-dtrain = xgb.DMatrix(npymat, label = labels)
-watchlist  = [(dtest,'eval'), (dtrain,'train')]
-bst = xgb.train( param, dtrain, num_round, watchlist )
-
-

demo/guide-python/boost_from_prediction.py

@@ -1,26 +0,0 @@
-#!/usr/bin/python
-import sys
-import numpy as np
-sys.path.append('../../wrapper')
-import xgboost as xgb
-
-dtrain = xgb.DMatrix('../data/agaricus.txt.train')
-dtest = xgb.DMatrix('../data/agaricus.txt.test')
-watchlist  = [(dtest,'eval'), (dtrain,'train')]
-###
-# advanced: start from a initial base prediction
-#
-print ('start running example to start from a initial prediction')
-# specify parameters via map, definition are same as c++ version
-param = {'max_depth':2, 'eta':1, 'silent':1, 'objective':'binary:logistic' }
-# train xgboost for 1 round
-bst = xgb.train( param, dtrain, 1, watchlist )
-# Note: we need the margin value instead of transformed prediction in set_base_margin
-# do predict with output_margin=True, will always give you margin values before logistic transformation
-ptrain = bst.predict(dtrain, output_margin=True)
-ptest  = bst.predict(dtest, output_margin=True)
-dtrain.set_base_margin(ptrain)
-dtest.set_base_margin(ptest)
-
-print ('this is result of running from initial prediction')
-bst = xgb.train( param, dtrain, 1, watchlist )

demo/guide-python/cross_validation.py

@@ -1,63 +0,0 @@
-#!/usr/bin/python
-import sys
-import numpy as np
-sys.path.append('../../wrapper')
-import xgboost as xgb
-
-### load data in do training
-dtrain = xgb.DMatrix('../data/agaricus.txt.train')
-param = {'max_depth':2, 'eta':1, 'silent':1, 'objective':'binary:logistic'}
-num_round = 2
-
-print ('running cross validation')
-# do cross validation, this will print result out as
-# [iteration]  metric_name:mean_value+std_value
-# std_value is standard deviation of the metric
-xgb.cv(param, dtrain, num_round, nfold=5,
-       metrics={'error'}, seed = 0)
-
-print ('running cross validation, disable standard deviation display')
-# do cross validation, this will print result out as
-# [iteration]  metric_name:mean_value+std_value
-# std_value is standard deviation of the metric
-xgb.cv(param, dtrain, num_round, nfold=5,
-       metrics={'error'}, seed = 0, show_stdv = False)
-
-print ('running cross validation, with preprocessing function')
-# define the preprocessing function
-# used to return the preprocessed training, test data, and parameter
-# we can use this to do weight rescale, etc.
-# as a example, we try to set scale_pos_weight
-def fpreproc(dtrain, dtest, param):
-    label = dtrain.get_label()
-    ratio = float(np.sum(label == 0)) / np.sum(label==1)
-    param['scale_pos_weight'] = ratio
-    return (dtrain, dtest, param)
-
-# do cross validation, for each fold
-# the dtrain, dtest, param will be passed into fpreproc
-# then the return value of fpreproc will be used to generate
-# results of that fold
-xgb.cv(param, dtrain, num_round, nfold=5,
-       metrics={'auc'}, seed = 0, fpreproc = fpreproc)
-
-###
-# you can also do cross validation with cutomized loss function
-# See custom_objective.py
-##
-print ('running cross validation, with cutomsized loss function')
-def logregobj(preds, dtrain):
-    labels = dtrain.get_label()
-    preds = 1.0 / (1.0 + np.exp(-preds))
-    grad = preds - labels
-    hess = preds * (1.0-preds)
-    return grad, hess
-def evalerror(preds, dtrain):
-    labels = dtrain.get_label()
-    return 'error', float(sum(labels != (preds > 0.0))) / len(labels)
-
-param = {'max_depth':2, 'eta':1, 'silent':1} 
-# train with customized objective
-xgb.cv(param, dtrain, num_round, nfold = 5, seed = 0,
-       obj = logregobj, feval=evalerror)
-

demo/guide-python/custom_objective.py

@@ -1,44 +0,0 @@
-#!/usr/bin/python
-import sys
-import numpy as np
-sys.path.append('../../wrapper')
-import xgboost as xgb
-###
-# advanced: cutomsized loss function
-# 
-print ('start running example to used cutomized objective function')
-
-dtrain = xgb.DMatrix('../data/agaricus.txt.train')
-dtest = xgb.DMatrix('../data/agaricus.txt.test')
-
-# note: for customized objective function, we leave objective as default
-# note: what we are getting is margin value in prediction
-# you must know what you are doing
-param = {'max_depth':2, 'eta':1, 'silent':1 }
-watchlist  = [(dtest,'eval'), (dtrain,'train')]
-num_round = 2
-
-# user define objective function, given prediction, return gradient and second order gradient
-# this is loglikelihood loss
-def logregobj(preds, dtrain):
-    labels = dtrain.get_label()
-    preds = 1.0 / (1.0 + np.exp(-preds))
-    grad = preds - labels
-    hess = preds * (1.0-preds)
-    return grad, hess
-
-# user defined evaluation function, return a pair metric_name, result
-# NOTE: when you do customized loss function, the default prediction value is margin
-# this may make buildin evalution metric not function properly
-# for example, we are doing logistic loss, the prediction is score before logistic transformation
-# the buildin evaluation error assumes input is after logistic transformation
-# Take this in mind when you use the customization, and maybe you need write customized evaluation function
-def evalerror(preds, dtrain):
-    labels = dtrain.get_label()
-    # return a pair metric_name, result
-    # since preds are margin(before logistic transformation, cutoff at 0)
-    return 'error', float(sum(labels != (preds > 0.0))) / len(labels)
-
-# training with customized objective, we can also do step by step training
-# simply look at xgboost.py's implementation of train
-bst = xgb.train(param, dtrain, num_round, watchlist, logregobj, evalerror)

demo/guide-python/generalized_linear_model.py

@@ -1,32 +0,0 @@
-#!/usr/bin/python
-import sys
-sys.path.append('../../wrapper')
-import xgboost as xgb
-##
-#  this script demonstrate how to fit generalized linear model in xgboost
-#  basically, we are using linear model, instead of tree for our boosters
-##
-dtrain = xgb.DMatrix('../data/agaricus.txt.train')
-dtest = xgb.DMatrix('../data/agaricus.txt.test')
-# change booster to gblinear, so that we are fitting a linear model
-# alpha is the L1 regularizer 
-# lambda is the L2 regularizer
-# you can also set lambda_bias which is L2 regularizer on the bias term
-param = {'silent':1, 'objective':'binary:logistic', 'booster':'gblinear',
-         'alpha': 0.0001, 'lambda': 1 }
-
-# normally, you do not need to set eta (step_size)
-# XGBoost uses a parallel coordinate descent algorithm (shotgun), 
-# there could be affection on convergence with parallelization on certain cases
-# setting eta to be smaller value, e.g 0.5 can make the optimization more stable
-# param['eta'] = 1 
-
-##
-# the rest of settings are the same
-##
-watchlist  = [(dtest,'eval'), (dtrain,'train')]
-num_round = 4
-bst = xgb.train(param, dtrain, num_round, watchlist)
-preds = bst.predict(dtest)
-labels = dtest.get_label()
-print ('error=%f' % ( sum(1 for i in range(len(preds)) if int(preds[i]>0.5)!=labels[i]) /float(len(preds))))

demo/guide-python/predict_first_ntree.py

@@ -1,22 +0,0 @@
-#!/usr/bin/python
-import sys
-import numpy as np
-sys.path.append('../../wrapper')
-import xgboost as xgb
-
-### load data in do training
-dtrain = xgb.DMatrix('../data/agaricus.txt.train')
-dtest = xgb.DMatrix('../data/agaricus.txt.test')
-param = {'max_depth':2, 'eta':1, 'silent':1, 'objective':'binary:logistic' }
-watchlist  = [(dtest,'eval'), (dtrain,'train')]
-num_round = 3
-bst = xgb.train(param, dtrain, num_round, watchlist)
-
-print ('start testing prediction from first n trees')
-### predict using first 1 tree
-label = dtest.get_label()
-ypred1 = bst.predict(dtest, ntree_limit=1)
-# by default, we predict using all the trees
-ypred2 = bst.predict(dtest)
-print ('error of ypred1=%f' % (np.sum((ypred1>0.5)!=label) /float(len(label))))
-print ('error of ypred2=%f' % (np.sum((ypred2>0.5)!=label) /float(len(label))))

demo/guide-python/runall.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-python basic_walkthrough.py
-python custom_objective.py
-python boost_from_prediction.py
-python generalized_linear_model.py
-python cross_validation.py
-rm -rf *~ *.model *.buffer 

demo/kaggle-higgs/README.md

@@ -7,10 +7,9 @@ This script will achieve about 3.600 AMS score in public leadboard. To get start
 
 1. Compile the XGBoost python lib
 ```bash
-cd ../..
+cd ../../python
 make
 ```
-
 2. Put training.csv test.csv on folder './data' (you can create a symbolic link)
 
 3. Run ./run.sh
@@ -18,9 +17,3 @@ make
 Speed
 =====
 speedtest.py compares xgboost's speed on this dataset with sklearn.GBM
-
-
-Using R module
-=====
-* Alternatively, you can run using R, higgs-train.R and higgs-pred.R. 
-

demo/kaggle-higgs/higgs-cv.py

@@ -1,39 +0,0 @@
-#!/usr/bin/python
-import sys
-import numpy as np
-sys.path.append('../../wrapper')
-import xgboost as xgb
-
-### load data in do training
-train = np.loadtxt('./data/training.csv', delimiter=',', skiprows=1, converters={32: lambda x:int(x=='s'.encode('utf-8')) } )
-label  = train[:,32]
-data   = train[:,1:31]
-weight = train[:,31]
-dtrain = xgb.DMatrix( data, label=label, missing = -999.0, weight=weight )
-param = {'max_depth':6, 'eta':0.1, 'silent':1, 'objective':'binary:logitraw', 'nthread':4}
-num_round = 120
-
-print ('running cross validation, with preprocessing function')
-# define the preprocessing function
-# used to return the preprocessed training, test data, and parameter
-# we can use this to do weight rescale, etc.
-# as a example, we try to set scale_pos_weight
-def fpreproc(dtrain, dtest, param):
-    label = dtrain.get_label()
-    ratio = float(np.sum(label == 0)) / np.sum(label==1)
-    param['scale_pos_weight'] = ratio
-    wtrain = dtrain.get_weight()
-    wtest = dtest.get_weight()
-    sum_weight = sum(wtrain) + sum(wtest)
-    wtrain *= sum_weight / sum(wtrain)
-    wtest *= sum_weight / sum(wtest)
-    dtrain.set_weight(wtrain)
-    dtest.set_weight(wtest)
-    return (dtrain, dtest, param)
-
-# do cross validation, for each fold
-# the dtrain, dtest, param will be passed into fpreproc
-# then the return value of fpreproc will be used to generate
-# results of that fold
-xgb.cv(param, dtrain, num_round, nfold=5,
-       metrics={'ams@0.15', 'auc'}, seed = 0, fpreproc = fpreproc)

demo/kaggle-higgs/higgs-numpy.py

@@ -6,7 +6,7 @@ import sys
 import numpy as np
 # add path of xgboost python module
 code_path = os.path.join(
-    os.path.split(inspect.getfile(inspect.currentframe()))[0], "../../wrapper")
+    os.path.split(inspect.getfile(inspect.currentframe()))[0], "../../python")
 
 sys.path.append(code_path)
 
@@ -42,8 +42,8 @@ param = {}
 param['objective'] = 'binary:logitraw'
 # scale weight of positive examples
 param['scale_pos_weight'] = sum_wneg/sum_wpos
-param['eta'] = 0.1 
+param['bst:eta'] = 0.1 
-param['max_depth'] = 6
+param['bst:max_depth'] = 6
 param['eval_metric'] = 'auc'
 param['silent'] = 1
 param['nthread'] = 16

demo/kaggle-higgs/higgs-pred.R

@@ -1,24 +0,0 @@
-# install xgboost package, see R-package in root folder
-require(xgboost)
-require(methods)
-
-modelfile <- "higgs.model"
-outfile <- "higgs.pred.csv"
-dtest <- read.csv("data/test.csv", header=TRUE)
-data <- as.matrix(dtest[2:31])
-idx <- dtest[[1]]
-
-xgmat <- xgb.DMatrix(data, missing = -999.0)
-bst <- xgb.load(modelfile=modelfile)
-ypred <- predict(bst, xgmat)
-
-rorder <- rank(ypred, ties.method="first")
-
-threshold <- 0.15
-# to be completed
-ntop <- length(rorder) - as.integer(threshold*length(rorder))
-plabel <- ifelse(rorder > ntop, "s", "b")
-outdata <- list("EventId" = idx,
-                "RankOrder" = rorder,
-                "Class" = plabel)
-write.csv(outdata, file = outfile, quote=FALSE, row.names=FALSE)

demo/kaggle-higgs/higgs-pred.py

@@ -3,7 +3,7 @@
 import sys
 import numpy as np
 # add path of xgboost python module
-sys.path.append('../../wrapper/')
+sys.path.append('../../python/')
 import xgboost as xgb
 
 # path to where the data lies
@@ -21,7 +21,8 @@ idx = dtest[:,0]
 
 print ('finish loading from csv ')
 xgmat = xgb.DMatrix( data, missing = -999.0 )
-bst = xgb.Booster({'nthread':16}, model_file = modelfile)
+bst = xgb.Booster({'nthread':16})
+bst.load_model( modelfile )
 ypred = bst.predict( xgmat )
 
 res  = [ ( int(idx[i]), ypred[i] ) for i in range(len(ypred)) ] 
@@ -42,8 +43,7 @@ for k, v in res:
         nhit += 1
     else:
         lb = 'b'        
-    # change output rank order to follow Kaggle convention
+    fo.write('%s,%d,%s\n' % ( k, rorder[k], lb ) )
-    fo.write('%s,%d,%s\n' % ( k,  len(rorder)+1-rorder[k], lb ) )
     ntot += 1
 fo.close()
 

demo/kaggle-higgs/higgs-train.R

@@ -1,33 +0,0 @@
-# install xgboost package, see R-package in root folder
-require(xgboost)
-require(methods)
-
-testsize <- 550000
-
-dtrain <- read.csv("data/training.csv", header=TRUE)
-dtrain[33] <- dtrain[33] == "s"
-label <- as.numeric(dtrain[[33]])
-data <- as.matrix(dtrain[2:31])
-weight <- as.numeric(dtrain[[32]]) * testsize / length(label)
-
-sumwpos <- sum(weight * (label==1.0))
-sumwneg <- sum(weight * (label==0.0))
-print(paste("weight statistics: wpos=", sumwpos, "wneg=", sumwneg, "ratio=", sumwneg / sumwpos))
-
-xgmat <- xgb.DMatrix(data, label = label, weight = weight, missing = -999.0)
-param <- list("objective" = "binary:logitraw",
-              "scale_pos_weight" = sumwneg / sumwpos,
-              "bst:eta" = 0.1,
-              "bst:max_depth" = 6,
-              "eval_metric" = "auc",
-              "eval_metric" = "ams@0.15",
-              "silent" = 1,
-              "nthread" = 16)
-watchlist <- list("train" = xgmat)
-nround = 120
-print ("loading data end, start to boost trees")
-bst = xgb.train(param, xgmat, nround, watchlist );
-# save out model
-xgb.save(bst, "higgs.model")
-print ('finish training')
-

demo/kaggle-higgs/speedtest.R

@@ -1,71 +0,0 @@
-# install xgboost package, see R-package in root folder
-require(xgboost)
-require(gbm)
-require(methods)
-
-testsize <- 550000
-
-dtrain <- read.csv("data/training.csv", header=TRUE, nrows=350001)
-
-# gbm.time = system.time({
-#   gbm.model <- gbm(Label ~ ., data = dtrain[, -c(1,32)], n.trees = 120, 
-#                    interaction.depth = 6, shrinkage = 0.1, bag.fraction = 1,
-#                    verbose = TRUE)
-# })
-# print(gbm.time)
-# Test result: 761.48 secs
-
-dtrain[33] <- dtrain[33] == "s"
-label <- as.numeric(dtrain[[33]])
-data <- as.matrix(dtrain[2:31])
-weight <- as.numeric(dtrain[[32]]) * testsize / length(label)
-
-sumwpos <- sum(weight * (label==1.0))
-sumwneg <- sum(weight * (label==0.0))
-print(paste("weight statistics: wpos=", sumwpos, "wneg=", sumwneg, "ratio=", sumwneg / sumwpos))
-
-xgboost.time = list()
-threads = c(1,2,4,8,16)
-for (i in 1:length(threads)){
-  thread = threads[i]
-  xgboost.time[[i]] = system.time({
-    xgmat <- xgb.DMatrix(data, label = label, weight = weight, missing = -999.0)
-    param <- list("objective" = "binary:logitraw",
-                  "scale_pos_weight" = sumwneg / sumwpos,
-                  "bst:eta" = 0.1,
-                  "bst:max_depth" = 6,
-                  "eval_metric" = "auc",
-                  "eval_metric" = "ams@0.15",
-                  "silent" = 1,
-                  "nthread" = thread)
-    watchlist <- list("train" = xgmat)
-    nround = 120
-    print ("loading data end, start to boost trees")
-    bst = xgb.train(param, xgmat, nround, watchlist );
-    # save out model
-    xgb.save(bst, "higgs.model")
-    print ('finish training')
-  })
-}
-
-xgboost.time
-# [[1]]
-# user  system elapsed 
-# 444.98    1.96  450.22 
-# 
-# [[2]]
-# user  system elapsed 
-# 188.15    0.82  102.41 
-# 
-# [[3]]
-# user  system elapsed 
-# 143.29    0.79   44.18 
-# 
-# [[4]]
-# user  system elapsed 
-# 176.60    1.45   34.04 
-# 
-# [[5]]
-# user  system elapsed 
-# 180.15    2.85   35.26 
-

demo/kaggle-higgs/speedtest.py

@@ -3,7 +3,7 @@
 import sys
 import numpy as np
 # add path of xgboost python module
-sys.path.append('../../wrapper/')
+sys.path.append('../../python/')
 import xgboost as xgb
 from sklearn.ensemble import GradientBoostingClassifier
 import time

demo/multiclass_classification/train.py

@@ -1,7 +1,7 @@
 #! /usr/bin/python
 import sys
 import numpy as np
-sys.path.append('../../wrapper/')
+sys.path.append('../../python/')
 import xgboost as xgb
 
 # label need to be 0 to num_class -1
@@ -25,8 +25,8 @@ param = {}
 # use softmax multi-class classification
 param['objective'] = 'multi:softmax'
 # scale weight of positive examples
-param['eta'] = 0.1
+param['bst:eta'] = 0.1
-param['max_depth'] = 6
+param['bst:max_depth'] = 6
 param['silent'] = 1
 param['nthread'] = 4
 param['num_class'] = 6
@@ -39,12 +39,4 @@ pred = bst.predict( xg_test );
 
 print ('predicting, classification error=%f' % (sum( int(pred[i]) != test_Y[i] for i in range(len(test_Y))) / float(len(test_Y)) ))
 
-# do the same thing again, but output probabilities
-param['objective'] = 'multi:softprob'
-bst = xgb.train(param, xg_train, num_round, watchlist );
-# Note: this convention has been changed since xgboost-unity
-# get prediction, this is in 1D array, need reshape to (ndata, nclass)
-yprob = bst.predict( xg_test ).reshape( test_Y.shape[0], 6 )
-ylabel = np.argmax(yprob, axis=1)
 
-print ('predicting, classification error=%f' % (sum( int(ylabel[i]) != test_Y[i] for i in range(len(test_Y))) / float(len(test_Y)) ))

demo/rank/mq2008.conf

@@ -1,17 +1,19 @@
 # General Parameters, see comment for each definition
+# choose the tree booster, 0: tree, 1: linear
+booster_type = 0 
 
 # specify objective
 objective="rank:pairwise"
 
 # Tree Booster Parameters
 # step size shrinkage
-eta = 0.1 
+bst:eta = 0.1 
 # minimum loss reduction required to make a further partition
-gamma = 1.0 
+bst:gamma = 1.0 
 # minimum sum of instance weight(hessian) needed in a child
-min_child_weight = 0.1
+bst:min_child_weight = 0.1
 # maximum depth of a tree
-max_depth = 6
+bst:max_depth = 6
 
 # Task parameters
 # the number of round to do boosting

demo/regression/machine.conf

@@ -1,19 +1,19 @@
 # General Parameters, see comment for each definition
-# choose the tree booster, can also change to gblinear
+# choose the tree booster, 0: tree, 1: linear
-booster = gbtree
+booster_type = 0 
 # this is the only difference with classification, use reg:linear to do linear classification
 # when labels are in [0,1] we can also use reg:logistic
 objective = reg:linear
 
 # Tree Booster Parameters
 # step size shrinkage
-eta = 1.0 
+bst:eta = 1.0 
 # minimum loss reduction required to make a further partition
-gamma = 1.0 
+bst:gamma = 1.0 
 # minimum sum of instance weight(hessian) needed in a child
-min_child_weight = 1 
+bst:min_child_weight = 1 
 # maximum depth of a tree
-max_depth = 3 
+bst:max_depth = 3 
 
 # Task parameters
 # the number of round to do boosting

python/Makefile

@@ -0,0 +1,26 @@
+export CC  = gcc
+export CXX = g++
+export CFLAGS = -Wall -O3 -msse2  -Wno-unknown-pragmas  -fopenmp
+
+# specify tensor path
+SLIB = libxgboostpy.so
+.PHONY: clean all
+
+all: $(SLIB)
+export LDFLAGS= -pthread -lm 
+
+libxgboostpy.so: xgboost_python.cpp ../regrank/*.h ../booster/*.h ../booster/*/*.hpp ../booster/*.hpp
+
+$(SLIB) :
+	$(CXX) $(CFLAGS) -fPIC $(LDFLAGS) -shared -o $@ $(filter %.cpp %.o %.c, $^)
+$(BIN) : 
+	$(CXX) $(CFLAGS) $(LDFLAGS) -o $@ $(filter %.cpp %.o %.c, $^)
+
+$(OBJ) : 
+	$(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c, $^) )
+
+install:
+	cp -f -r $(BIN)  $(INSTALL_PATH)
+
+clean:
+	$(RM) $(OBJ) $(BIN) $(SLIB) *~

python/README.md

@@ -0,0 +1,3 @@
+python wrapper for xgboost using ctypes
+
+see example for usage

python/example/README.md

@@ -0,0 +1,3 @@
+example to use python xgboost, the data is generated from demo/binary_classification, in libsvm format
+
+for usage: see demo.py and comments in demo.py

python/example/demo.py

@@ -0,0 +1,96 @@
+#!/usr/bin/python
+import sys
+import numpy as np
+import scipy.sparse
+# append the path to xgboost, you may need to change the following line
+sys.path.append('../')
+import xgboost as xgb
+
+### simple example
+# load file from text file, also binary buffer generated by xgboost
+dtrain = xgb.DMatrix('agaricus.txt.train')
+dtest = xgb.DMatrix('agaricus.txt.test')
+
+# specify parameters via map, definition are same as c++ version
+param = {'bst:max_depth':2, 'bst:eta':1, 'silent':1, 'objective':'binary:logistic' }
+
+# specify validations set to watch performance
+evallist  = [(dtest,'eval'), (dtrain,'train')]
+num_round = 2
+bst = xgb.train( param, dtrain, num_round, evallist )
+
+# this is prediction
+preds = bst.predict( dtest )
+labels = dtest.get_label()
+print ('error=%f' % (  sum(1 for i in range(len(preds)) if int(preds[i]>0.5)!=labels[i]) /float(len(preds))))
+bst.save_model('0001.model')
+# dump model
+bst.dump_model('dump.raw.txt')
+# dump model with feature map
+bst.dump_model('dump.raw.txt','featmap.txt')
+
+###
+# build dmatrix in python iteratively
+#
+print ('start running example of build DMatrix in python')
+dtrain = xgb.DMatrix()
+labels = []
+for l in open('agaricus.txt.train'):
+    arr = l.split()
+    labels.append( int(arr[0]))
+    feats = []
+    for it in arr[1:]:
+        k,v = it.split(':')
+        feats.append( (int(k), float(v)) )
+    dtrain.add_row( feats )
+dtrain.set_label( labels )
+evallist  = [(dtest,'eval'), (dtrain,'train')]
+
+bst = xgb.train( param, dtrain, num_round, evallist )
+
+###
+# build dmatrix from scipy.sparse
+print ('start running example of build DMatrix from scipy.sparse')
+labels = []
+row = []; col = []; dat = []
+i = 0
+for l in open('agaricus.txt.train'):
+    arr = l.split()
+    labels.append( int(arr[0]))
+    for it in arr[1:]:
+        k,v = it.split(':')
+        row.append(i); col.append(int(k)); dat.append(float(v))
+    i += 1
+
+csr = scipy.sparse.csr_matrix( (dat, (row,col)) )
+dtrain = xgb.DMatrix( csr )
+dtrain.set_label(labels)
+evallist  = [(dtest,'eval'), (dtrain,'train')]
+bst = xgb.train( param, dtrain, num_round, evallist )
+
+print ('start running example of build DMatrix from numpy array')
+# NOTE: npymat is numpy array, we will convert it into scipy.sparse.csr_matrix in internal implementation,then convert to DMatrix
+npymat = csr.todense()
+dtrain = xgb.DMatrix( npymat )
+dtrain.set_label(labels)
+evallist  = [(dtest,'eval'), (dtrain,'train')]
+bst = xgb.train( param, dtrain, num_round, evallist )
+
+###
+# advanced: cutomsized loss function, set loss_type to 0, so that predict get untransformed score
+# 
+print ('start running example to used cutomized objective function')
+
+# note: set objective= binary:logistic means the prediction will get logistic transformed
+#       in most case, we may want to leave it as default
+param = {'bst:max_depth':2, 'bst:eta':1, 'silent':1, 'objective':'binary:logistic' }
+
+# user define objective function, given prediction, return gradient and second order gradient
+def logregobj( preds, dtrain ):
+    labels = dtrain.get_label()
+    grad = preds - labels
+    hess = preds * (1.0-preds)
+    return grad, hess
+
+# training with customized objective, we can also do step by step training, simply look at xgboost.py's implementation of train
+bst = xgb.train( param, dtrain, num_round, evallist, logregobj )

python/xgboost.py

@@ -0,0 +1,204 @@
+# Author: Tianqi Chen, Bing Xu
+# module for xgboost
+import ctypes 
+import os
+# optinally have scipy sparse, though not necessary
+import numpy
+import numpy.ctypeslib 
+import scipy.sparse as scp
+
+# set this line correctly
+XGBOOST_PATH = os.path.dirname(__file__)+'/libxgboostpy.so'
+
+# entry type of sparse matrix
+class REntry(ctypes.Structure):
+    _fields_ = [("findex", ctypes.c_uint), ("fvalue", ctypes.c_float) ]
+
+# load in xgboost library
+xglib = ctypes.cdll.LoadLibrary(XGBOOST_PATH)
+
+xglib.XGDMatrixCreate.restype = ctypes.c_void_p
+xglib.XGDMatrixNumRow.restype = ctypes.c_ulong
+xglib.XGDMatrixGetLabel.restype =  ctypes.POINTER( ctypes.c_float )
+xglib.XGDMatrixGetWeight.restype =  ctypes.POINTER( ctypes.c_float )
+xglib.XGDMatrixGetRow.restype = ctypes.POINTER( REntry )
+xglib.XGBoosterCreate.restype = ctypes.c_void_p
+xglib.XGBoosterPredict.restype = ctypes.POINTER( ctypes.c_float ) 
+
+def ctypes2numpy( cptr, length ):
+    # convert a ctypes pointer array to numpy
+    assert isinstance( cptr, ctypes.POINTER( ctypes.c_float ) )
+    res = numpy.zeros( length, dtype='float32' )
+    assert ctypes.memmove( res.ctypes.data, cptr, length * res.strides[0] )
+    return res
+
+# data matrix used in xgboost
+class DMatrix:
+    # constructor
+    def __init__(self, data=None, label=None, missing=0.0, weight = None):
+        # force into void_p, mac need to pass things in as void_p
+        self.handle = ctypes.c_void_p( xglib.XGDMatrixCreate() )
+        if data == None:
+            return
+        if isinstance(data,str):
+            xglib.XGDMatrixLoad(self.handle, ctypes.c_char_p(data.encode('utf-8')), 1)             
+        elif isinstance(data,scp.csr_matrix):
+            self.__init_from_csr(data)
+        elif isinstance(data, numpy.ndarray) and len(data.shape) == 2:
+            self.__init_from_npy2d(data, missing)
+        else:
+            try:
+                csr = scp.csr_matrix(data)
+                self.__init_from_csr(csr)
+            except:
+                raise Exception("can not intialize DMatrix from"+str(type(data)))
+        if label != None:
+            self.set_label(label)
+        if weight !=None:
+            self.set_weight(weight)
+
+    # convert data from csr matrix
+    def __init_from_csr(self,csr):
+        assert len(csr.indices) == len(csr.data)
+        xglib.XGDMatrixParseCSR( self.handle, 
+                                 ( ctypes.c_ulong  * len(csr.indptr) )(*csr.indptr),
+                                 ( ctypes.c_uint  * len(csr.indices) )(*csr.indices),
+                                 ( ctypes.c_float * len(csr.data) )(*csr.data),
+                                 len(csr.indptr), len(csr.data) )
+    # convert data from numpy matrix
+    def __init_from_npy2d(self,mat,missing):
+        data = numpy.array( mat.reshape(mat.size), dtype='float32' )
+        xglib.XGDMatrixParseMat( self.handle, 
+                                 data.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), 
+                                 mat.shape[0], mat.shape[1], ctypes.c_float(missing) )
+    # destructor
+    def __del__(self):
+        xglib.XGDMatrixFree(self.handle)    
+    # load data from file 
+    def load(self, fname, silent=True):
+        xglib.XGDMatrixLoad(self.handle, ctypes.c_char_p(fname.encode('utf-8')), int(silent))
+    # load data from file 
+    def save_binary(self, fname, silent=True):
+        xglib.XGDMatrixSaveBinary(self.handle, ctypes.c_char_p(fname.encode('utf-8')), int(silent))
+    # set label of dmatrix
+    def set_label(self, label):
+        xglib.XGDMatrixSetLabel(self.handle, (ctypes.c_float*len(label))(*label), len(label) )
+    # set group size of dmatrix, used for rank
+    def set_group(self, group):
+        xglib.XGDMatrixSetGroup(self.handle, (ctypes.c_uint*len(group))(*group), len(group) )
+    # set weight of each instances
+    def set_weight(self, weight):
+        xglib.XGDMatrixSetWeight(self.handle, (ctypes.c_float*len(weight))(*weight), len(weight) )
+    # get label from dmatrix
+    def get_label(self):
+        length = ctypes.c_ulong()
+        labels = xglib.XGDMatrixGetLabel(self.handle, ctypes.byref(length))
+        return ctypes2numpy( labels, length.value );
+    # get weight from dmatrix
+    def get_weight(self):
+        length = ctypes.c_ulong()
+        weights = xglib.XGDMatrixGetWeight(self.handle, ctypes.byref(length))
+        return ctypes2numpy( weights, length.value );
+    # clear everything
+    def clear(self):
+        xglib.XGDMatrixClear(self.handle)
+    def num_row(self):
+        return xglib.XGDMatrixNumRow(self.handle)
+    # append a row to DMatrix
+    def add_row(self, row):
+        xglib.XGDMatrixAddRow(self.handle, (REntry*len(row))(*row), len(row) )
+    # get n-throw from DMatrix
+    def __getitem__(self, ridx):
+        length = ctypes.c_ulong()
+        row = xglib.XGDMatrixGetRow(self.handle, ridx, ctypes.byref(length) );
+        return [ (int(row[i].findex),row[i].fvalue) for i in range(length.value) ]
+
+class Booster:
+    """learner class """
+    def __init__(self, params={}, cache=[]):
+        """ constructor, param: """    
+        for d in cache:
+            assert isinstance(d,DMatrix)
+        dmats = ( ctypes.c_void_p  * len(cache) )(*[ d.handle for d in cache])
+        self.handle = ctypes.c_void_p( xglib.XGBoosterCreate( dmats, len(cache) ) )
+        self.set_param( params )
+    def __del__(self):
+        xglib.XGBoosterFree(self.handle) 
+    def set_param(self, params, pv=None):
+        if isinstance(params,dict):
+            for k, v in params.items():
+                xglib.XGBoosterSetParam(
+                    self.handle, ctypes.c_char_p(k.encode('utf-8')), 
+                    ctypes.c_char_p(str(v).encode('utf-8')))        
+        elif isinstance(params,str) and pv != None:
+            xglib.XGBoosterSetParam(
+                self.handle, ctypes.c_char_p(params.encode('utf-8')),
+                ctypes.c_char_p(str(pv).encode('utf-8')) )
+        else:
+            for k, v in params:
+                xglib.XGBoosterSetParam(
+                    self.handle, ctypes.c_char_p(k.encode('utf-8')),
+                    ctypes.c_char_p(str(v).encode('utf-8')) )             
+    def update(self, dtrain):
+        """ update """
+        assert isinstance(dtrain, DMatrix)
+        xglib.XGBoosterUpdateOneIter( self.handle, dtrain.handle )
+    def boost(self, dtrain, grad, hess, bst_group = -1):
+        """ update """
+        assert len(grad) == len(hess)
+        assert isinstance(dtrain, DMatrix)
+        xglib.XGBoosterBoostOneIter( self.handle, dtrain.handle,
+                                     (ctypes.c_float*len(grad))(*grad),
+                                     (ctypes.c_float*len(hess))(*hess),
+                                     len(grad), bst_group )
+    def update_interact(self, dtrain, action, booster_index=None):
+        """ beta: update with specified action"""
+        assert isinstance(dtrain, DMatrix)
+        if booster_index != None:
+            self.set_param('interact:booster_index', str(booster_index))
+        xglib.XGBoosterUpdateInteract(
+            self.handle, dtrain.handle, ctypes.c_char_p(str(action)) )
+    def eval_set(self, evals, it = 0):
+        for d in evals:
+            assert isinstance(d[0], DMatrix)
+            assert isinstance(d[1], str)
+        dmats = ( ctypes.c_void_p * len(evals) )(*[ d[0].handle for d in evals])
+        evnames = ( ctypes.c_char_p * len(evals) )(
+            *[ctypes.c_char_p(d[1].encode('utf-8')) for d in evals])
+        xglib.XGBoosterEvalOneIter( self.handle, it, dmats, evnames, len(evals) )
+    def eval(self, mat, name = 'eval', it = 0 ):
+        self.eval_set( [(mat,name)], it)
+    def predict(self, data, bst_group = -1):
+        length = ctypes.c_ulong()
+        preds = xglib.XGBoosterPredict( self.handle, data.handle, ctypes.byref(length), bst_group)
+        return ctypes2numpy( preds, length.value )
+    def save_model(self, fname):
+        """ save model to file """
+        xglib.XGBoosterSaveModel(self.handle, ctypes.c_char_p(fname.encode('utf-8')))
+    def load_model(self, fname):
+        """load model from file"""
+        xglib.XGBoosterLoadModel( self.handle, ctypes.c_char_p(fname.encode('utf-8')) )
+    def dump_model(self, fname, fmap=''):
+        """dump model into text file"""
+        xglib.XGBoosterDumpModel(
+            self.handle, ctypes.c_char_p(fname.encode('utf-8')), 
+            ctypes.c_char_p(fmap.encode('utf-8')))
+
+def train(params, dtrain, num_boost_round = 10, evals = [], obj=None):
+    """ train a booster with given paramaters """
+    bst = Booster(params, [dtrain] )
+    if obj == None:
+        for i in range(num_boost_round):
+            bst.update( dtrain )
+            if len(evals) != 0:
+                bst.eval_set( evals, i )
+    else:
+        # try customized objective function
+        for i in range(num_boost_round):
+            pred = bst.predict( dtrain )
+            grad, hess = obj( pred, dtrain )
+            bst.boost( dtrain, grad, hess )
+            if len(evals) != 0:
+                bst.eval_set( evals, i )        
+    return bst
+

python/xgboost_python.cpp

@@ -0,0 +1,293 @@
+// implementations in ctypes
+#include "xgboost_python.h"
+#include "../regrank/xgboost_regrank.h"
+#include "../regrank/xgboost_regrank_data.h"
+
+namespace xgboost{
+    namespace python{
+        class DMatrix: public regrank::DMatrix{
+        public:
+            // whether column is initialized
+            bool init_col_;
+        public:
+            DMatrix(void){
+                init_col_ = false;
+            }            
+            ~DMatrix(void){}
+        public:            
+            inline void Load(const char *fname, bool silent){
+                this->CacheLoad(fname, silent);
+                init_col_ = this->data.HaveColAccess();
+            }
+            inline void Clear( void ){
+                this->data.Clear();
+                this->info.labels.clear();
+                this->info.weights.clear();
+                this->info.group_ptr.clear();
+            }
+            inline size_t NumRow( void ) const{
+                return this->data.NumRow();
+            }
+            inline void AddRow( const XGEntry *data, size_t len ){
+                xgboost::booster::FMatrixS &mat = this->data;
+                mat.row_data_.resize( mat.row_ptr_.back() + len );
+                memcpy( &mat.row_data_[mat.row_ptr_.back()], data, sizeof(XGEntry)*len );
+                mat.row_ptr_.push_back( mat.row_ptr_.back() + len );
+                init_col_ = false;
+            }
+            inline const XGEntry* GetRow(unsigned ridx, size_t* len) const{
+                const xgboost::booster::FMatrixS &mat = this->data;
+
+                *len = mat.row_ptr_[ridx+1] - mat.row_ptr_[ridx];
+                return &mat.row_data_[ mat.row_ptr_[ridx] ];
+            }
+            inline void ParseCSR( const size_t *indptr,
+                                  const unsigned *indices,
+                                  const float *data,
+                                  size_t nindptr,
+                                  size_t nelem ){
+                xgboost::booster::FMatrixS &mat = this->data;
+                mat.row_ptr_.resize( nindptr );
+                memcpy( &mat.row_ptr_[0], indptr, sizeof(size_t)*nindptr );
+                mat.row_data_.resize( nelem );
+                for( size_t i = 0; i < nelem; ++ i ){
+                    mat.row_data_[i] = XGEntry(indices[i], data[i]);
+                }
+                this->data.InitData();
+                this->init_col_ = true;
+            }
+
+            inline void ParseMat( const float *data,
+                                  size_t nrow,
+                                  size_t ncol,
+                                  float  missing ){
+                xgboost::booster::FMatrixS &mat = this->data;
+                mat.Clear();
+                for( size_t i = 0; i < nrow; ++i, data += ncol ){
+                    size_t nelem = 0;
+                    for( size_t j = 0; j < ncol; ++j ){
+                        if( data[j] != missing ){                           
+                            mat.row_data_.push_back( XGEntry(j, data[j]) );
+                            ++ nelem;
+                        }
+                    }
+                    mat.row_ptr_.push_back( mat.row_ptr_.back() + nelem );
+                }
+                this->data.InitData();
+                this->init_col_ = true;
+            }
+            inline void SetLabel( const float *label, size_t len ){
+                this->info.labels.resize( len );
+                memcpy( &(this->info).labels[0], label, sizeof(float)*len );
+            }
+            inline void SetGroup( const unsigned *group, size_t len ){
+                this->info.group_ptr.resize( len + 1 );
+                this->info.group_ptr[0] = 0;
+                for( size_t i = 0; i < len; ++ i ){
+                    this->info.group_ptr[i+1] = this->info.group_ptr[i]+group[i];
+                }
+            }
+            inline void SetWeight( const float *weight, size_t len ){
+                this->info.weights.resize( len );
+                memcpy( &(this->info).weights[0], weight, sizeof(float)*len );
+            }
+            inline const float* GetLabel( size_t* len ) const{
+                *len = this->info.labels.size();
+                return &(this->info.labels[0]);
+            }
+            inline const float* GetWeight( size_t* len ) const{
+                *len = this->info.weights.size();
+                return &(this->info.weights[0]);
+            }
+            inline void CheckInit(void){
+                if(!init_col_){
+                    this->data.InitData();
+                    init_col_ = true;
+                }
+                utils::Assert( this->data.NumRow() == this->info.labels.size(), "DMatrix: number of labels must match number of rows in matrix");
+            }
+        };
+    
+        class Booster: public xgboost::regrank::RegRankBoostLearner{
+        private:
+            bool init_trainer, init_model;
+        public:
+            Booster(const std::vector<regrank::DMatrix *> mats){
+                silent = 1;
+                init_trainer = false;
+                init_model = false;
+                this->SetCacheData(mats);
+            }
+            inline void CheckInit(void){
+                if( !init_trainer ){
+                    this->InitTrainer(); init_trainer = true;
+                }
+                if( !init_model ){
+                    this->InitModel(); init_model = true;
+                }
+            }
+            inline void LoadModel( const char *fname ){
+                xgboost::regrank::RegRankBoostLearner::LoadModel(fname);
+                this->init_model = true;
+            }
+            const float *Pred( const DMatrix &dmat, size_t *len, int bst_group ){
+                this->CheckInit();
+
+                this->Predict( this->preds_, dmat, bst_group );
+                *len = this->preds_.size();
+                return &this->preds_[0];
+            }
+            inline void BoostOneIter( const DMatrix &train, 
+                                      float *grad, float *hess, size_t len, int bst_group ){
+                this->grad_.resize( len ); this->hess_.resize( len );
+                memcpy( &this->grad_[0], grad, sizeof(float)*len );
+                memcpy( &this->hess_[0], hess, sizeof(float)*len );
+                
+                if( grad_.size() == train.Size() ){
+                    if( bst_group < 0 ) bst_group = 0;
+                    base_gbm.DoBoost(grad_, hess_, train.data, train.info.root_index, bst_group);
+                }else{
+                    utils::Assert( bst_group == -1, "must set bst_group to -1 to support all group boosting" );
+                    int ngroup = base_gbm.NumBoosterGroup();
+                    utils::Assert( grad_.size() == train.Size() * (size_t)ngroup, "BUG: UpdateOneIter: mclass" );
+                    std::vector<float> tgrad( train.Size() ), thess( train.Size() );
+                    for( int g = 0; g < ngroup; ++ g ){
+                        memcpy( &tgrad[0], &grad_[g*tgrad.size()], sizeof(float)*tgrad.size() );
+                        memcpy( &thess[0], &hess_[g*tgrad.size()], sizeof(float)*tgrad.size() );
+                        base_gbm.DoBoost(tgrad, thess, train.data, train.info.root_index, g );
+                    }
+                }                
+            }
+        };
+    };
+};
+
+using namespace xgboost::python;
+
+
+extern "C"{
+    void* XGDMatrixCreate( void ){
+        return new DMatrix();
+    }
+    void XGDMatrixFree( void *handle ){
+        delete static_cast<DMatrix*>(handle);
+    }
+    void XGDMatrixLoad( void *handle, const char *fname, int silent ){
+        static_cast<DMatrix*>(handle)->Load(fname, silent!=0);
+    }
+    void XGDMatrixSaveBinary( void *handle, const char *fname, int silent ){
+        static_cast<DMatrix*>(handle)->SaveBinary(fname, silent!=0);
+    }
+    void XGDMatrixParseCSR( void *handle, 
+                            const size_t *indptr,
+                            const unsigned *indices,
+                            const float *data,
+                            size_t nindptr,
+                            size_t nelem ){
+        static_cast<DMatrix*>(handle)->ParseCSR(indptr, indices, data, nindptr, nelem);
+    }
+    void XGDMatrixParseMat( void *handle, 
+                            const float *data,
+                            size_t nrow,
+                            size_t ncol,
+                            float  missing ){
+      static_cast<DMatrix*>(handle)->ParseMat(data, nrow, ncol, missing);
+    }
+    void XGDMatrixSetLabel( void *handle, const float *label, size_t len ){
+        static_cast<DMatrix*>(handle)->SetLabel(label,len);        
+    }
+    void XGDMatrixSetWeight( void *handle, const float *weight, size_t len ){
+        static_cast<DMatrix*>(handle)->SetWeight(weight,len);        
+    }
+    void XGDMatrixSetGroup( void *handle, const unsigned *group, size_t len ){
+        static_cast<DMatrix*>(handle)->SetGroup(group,len);        
+    }
+    const float* XGDMatrixGetLabel( const void *handle, size_t* len ){
+        return static_cast<const DMatrix*>(handle)->GetLabel(len);
+    }
+    const float* XGDMatrixGetWeight( const void *handle, size_t* len ){
+        return static_cast<const DMatrix*>(handle)->GetWeight(len);
+    }
+    void XGDMatrixClear(void *handle){
+        static_cast<DMatrix*>(handle)->Clear();
+    }
+    void XGDMatrixAddRow( void *handle, const XGEntry *data, size_t len ){
+        static_cast<DMatrix*>(handle)->AddRow(data, len);
+    }
+    size_t XGDMatrixNumRow(const void *handle){
+        return static_cast<const DMatrix*>(handle)->NumRow();
+    }
+    const XGEntry* XGDMatrixGetRow(void *handle, unsigned ridx, size_t* len){
+        return static_cast<DMatrix*>(handle)->GetRow(ridx, len);
+    }
+
+    // xgboost implementation
+    void *XGBoosterCreate( void *dmats[], size_t len ){
+        std::vector<xgboost::regrank::DMatrix*> mats;
+        for( size_t i = 0; i < len; ++i ){
+            DMatrix *dtr = static_cast<DMatrix*>(dmats[i]);
+            dtr->CheckInit();
+            mats.push_back( dtr );
+        }
+        return new Booster( mats );
+    }
+    void XGBoosterFree( void *handle ){
+        delete  static_cast<Booster*>(handle);
+    }
+    void XGBoosterSetParam( void *handle, const char *name, const char *value ){
+        static_cast<Booster*>(handle)->SetParam( name, value );
+    }
+    void XGBoosterUpdateOneIter( void *handle, void *dtrain ){
+        Booster *bst = static_cast<Booster*>(handle);
+        DMatrix *dtr = static_cast<DMatrix*>(dtrain);
+        bst->CheckInit(); dtr->CheckInit(); 
+        bst->UpdateOneIter( *dtr );
+    }    
+    void XGBoosterBoostOneIter( void *handle, void *dtrain, 
+                                float *grad, float *hess, size_t len, int bst_group ){
+        Booster *bst = static_cast<Booster*>(handle);
+        DMatrix *dtr = static_cast<DMatrix*>(dtrain);
+        bst->CheckInit(); dtr->CheckInit(); 
+        bst->BoostOneIter( *dtr, grad, hess, len, bst_group );
+    }      
+    void XGBoosterEvalOneIter( void *handle, int iter, void *dmats[], const char *evnames[], size_t len ){
+        Booster *bst = static_cast<Booster*>(handle);
+        bst->CheckInit();
+
+        std::vector<std::string> names;
+        std::vector<const xgboost::regrank::DMatrix*> mats;
+        for( size_t i = 0; i < len; ++i ){
+            mats.push_back( static_cast<DMatrix*>(dmats[i]) );
+            names.push_back( std::string( evnames[i]) );
+        }
+        bst->EvalOneIter( iter, mats, names, stderr );
+    }
+    const float *XGBoosterPredict( void *handle, void *dmat, size_t *len, int bst_group ){
+        return static_cast<Booster*>(handle)->Pred( *static_cast<DMatrix*>(dmat), len, bst_group );
+    }
+    void XGBoosterLoadModel( void *handle, const char *fname ){        
+        static_cast<Booster*>(handle)->LoadModel( fname );        
+    } 
+    void XGBoosterSaveModel( const void *handle, const char *fname ){
+        static_cast<const Booster*>(handle)->SaveModel( fname );
+    }
+    void XGBoosterDumpModel( void *handle, const char *fname, const char *fmap ){
+        using namespace xgboost::utils;
+        FILE *fo = FopenCheck( fname, "w" );
+        FeatMap featmap; 
+        if( strlen(fmap) != 0 ){ 
+            featmap.LoadText( fmap );
+        }
+        static_cast<Booster*>(handle)->DumpModel( fo, featmap, false );
+        fclose( fo );
+    }
+
+    void XGBoosterUpdateInteract( void *handle, void *dtrain, const char *action ){
+        Booster *bst = static_cast<Booster*>(handle);
+        DMatrix *dtr = static_cast<DMatrix*>(dtrain);        
+        bst->CheckInit(); dtr->CheckInit(); 
+        std::string act( action );
+        bst->UpdateInteract( act, *dtr );
+    }
+};
+

python/xgboost_python.h

@@ -0,0 +1,209 @@
+#ifndef XGBOOST_PYTHON_H
+#define XGBOOST_PYTHON_H
+/*!
+ * \file xgboost_python.h
+ * \author Tianqi Chen
+ * \brief python wrapper for xgboost, using ctypes, 
+ *        hides everything behind functions
+ *      use c style interface
+ */
+#include "../booster/xgboost_data.h"
+extern "C"{
+    /*! \brief type of row entry */
+    typedef xgboost::booster::FMatrixS::REntry XGEntry;
+    
+    /*! 
+     * \brief create a data matrix 
+     * \return a new data matrix
+     */
+    void* XGDMatrixCreate(void);
+    /*! 
+     * \brief free space in data matrix
+     */
+    void XGDMatrixFree(void *handle);
+    /*! 
+     * \brief load a data matrix from text file or buffer(if exists)
+     * \param handle a instance of data matrix
+     * \param fname file name 
+     * \param silent print statistics when loading
+     */
+    void XGDMatrixLoad(void *handle, const char *fname, int silent);
+    /*!
+     * \brief load a data matrix into binary file
+     * \param handle a instance of data matrix
+     * \param fname file name 
+     * \param silent print statistics when saving
+     */
+    void XGDMatrixSaveBinary(void *handle, const char *fname, int silent);
+    /*! 
+     * \brief set matrix content from csr format
+     * \param handle a instance of data matrix
+     * \param indptr pointer to row headers
+     * \param indices findex
+     * \param data    fvalue
+     * \param nindptr number of rows in the matix + 1 
+     * \param nelem number of nonzero elements in the matrix
+     */
+    void XGDMatrixParseCSR( void *handle, 
+                            const size_t *indptr,
+                            const unsigned *indices,
+                            const float *data,
+                            size_t nindptr,
+                            size_t nelem );
+    /*! 
+     * \brief set matrix content from data content
+     * \param handle a instance of data matrix
+     * \param data pointer to the data space
+     * \param nrow number of rows
+     * \param ncol number columns
+     * \param missing which value to represent missing value
+     */
+    void XGDMatrixParseMat( void *handle, 
+                            const float *data,
+                            size_t nrow,
+                            size_t ncol,
+                            float  missing );
+    /*! 
+     * \brief set label of the training matrix
+     * \param handle a instance of data matrix
+     * \param label pointer to label
+     * \param len length of array
+     */    
+    void XGDMatrixSetLabel( void *handle, const float *label, size_t len );        
+    /*! 
+     * \brief set label of the training matrix
+     * \param handle a instance of data matrix
+     * \param group pointer to group size
+     * \param len length of array
+     */    
+    void XGDMatrixSetGroup( void *handle, const unsigned *group, size_t len );        
+    /*! 
+     * \brief set weight of each instacne
+     * \param handle a instance of data matrix
+     * \param weight data pointer to weights
+     * \param len length of array
+     */    
+    void XGDMatrixSetWeight( void *handle, const float *weight, size_t len );        
+    /*! 
+     * \brief get label set from matrix
+     * \param handle a instance of data matrix
+     * \param len used to set result length
+     * \return pointer to the label
+     */
+    const float* XGDMatrixGetLabel( const void *handle, size_t* len );
+    /*! 
+     * \brief get weight set from matrix
+     * \param handle a instance of data matrix
+     * \param len used to set result length
+     * \return pointer to the weight
+     */
+    const float* XGDMatrixGetWeight( const void *handle, size_t* len );
+    /*! 
+     * \brief clear all the records, including feature matrix and label
+     * \param handle a instance of data matrix
+     */
+    void XGDMatrixClear(void *handle);
+    /*! 
+     * \brief return number of rows
+     */    
+    size_t XGDMatrixNumRow(const void *handle);
+    /*! 
+     * \brief add row 
+     * \param handle a instance of data matrix
+     * \param data array of row content 
+     * \param len length of array
+     */
+    void XGDMatrixAddRow(void *handle, const XGEntry *data, size_t len);
+    /*! 
+     * \brief get ridx-th row of sparse matrix
+     * \param handle handle
+     * \param ridx row index 
+     * \param len used to set result length
+     * \reurn pointer to the row
+     */    
+    const XGEntry* XGDMatrixGetRow(void *handle, unsigned ridx, size_t* len);
+    
+    // --- start XGBoost class
+    /*! 
+     * \brief create xgboost learner 
+     * \param dmats matrices that are set to be cached
+     * \param create a booster
+     */
+    void *XGBoosterCreate( void* dmats[], size_t len ); 
+    /*! 
+     * \brief free obj in handle 
+     * \param handle handle to be freed
+     */
+    void XGBoosterFree( void* handle ); 
+    /*! 
+     * \brief set parameters 
+     * \param handle handle
+     * \param name  parameter name
+     * \param val value of parameter
+     */    
+    void XGBoosterSetParam( void *handle, const char *name, const char *value );   
+    /*! 
+     * \brief update the model in one round using dtrain
+     * \param handle handle
+     * \param dtrain training data
+     */        
+    void XGBoosterUpdateOneIter( void *handle, void *dtrain );   
+    
+    /*!
+     * \brief update the model, by directly specify gradient and second order gradient, 
+     *        this can be used to replace UpdateOneIter, to support customized loss function
+     * \param handle handle
+     * \param dtrain training data
+     * \param grad gradient statistics
+     * \param hess second order gradient statistics
+     * \param len length of grad/hess array
+     * \param bst_group boost group we are working at, default = -1
+     */
+    void XGBoosterBoostOneIter( void *handle, void *dtrain, 
+                                float *grad, float *hess, size_t len, int bst_group );   
+    /*! 
+     * \brief print evaluation statistics to stdout for xgboost
+     * \param handle handle
+     * \param iter current iteration rounds
+     * \param dmats pointers to data to be evaluated
+     * \param evnames pointers to names of each data
+     * \param len  length of dmats
+     */        
+    void XGBoosterEvalOneIter( void *handle, int iter, void *dmats[], const char *evnames[], size_t len );   
+    /*! 
+     * \brief make prediction based on dmat
+     * \param handle handle
+     * \param dmat data matrix
+     * \param len used to store length of returning result
+     * \param bst_group booster group, if model contains multiple booster group, default = -1 means predict for all groups 
+     */    
+    const float *XGBoosterPredict( void *handle, void *dmat, size_t *len, int bst_group );
+    /*! 
+     * \brief load model from existing file
+     * \param handle handle
+     * \param fname file name
+     */    
+    void XGBoosterLoadModel( void *handle, const char *fname );
+    /*! 
+     * \brief save model into existing file
+     * \param handle handle
+     * \param fname file name
+     */    
+    void XGBoosterSaveModel( const void *handle, const char *fname );
+    /*! 
+     * \brief dump model into text file
+     * \param handle handle
+     * \param fname file name
+     * \param fmap  name to fmap can be empty string
+     */    
+    void XGBoosterDumpModel( void *handle, const char *fname, const char *fmap );
+    /*! 
+     * \brief interactively update model: beta
+     * \param handle handle
+     * \param dtrain training data
+     * \param action action name
+     */        
+    void XGBoosterUpdateInteract( void *handle, void *dtrain, const char* action );   
+};
+#endif
+

regrank/xgboost_regrank.h

@@ -0,0 +1,385 @@
+#ifndef XGBOOST_REGRANK_H
+#define XGBOOST_REGRANK_H
+/*!
+* \file xgboost_regrank.h
+* \brief class for gradient boosted regression and ranking
+* \author Kailong Chen: chenkl198812@gmail.com, Tianqi Chen: tianqi.tchen@gmail.com
+*/
+#include <cmath>
+#include <cstdlib>
+#include <cstring>
+#include "xgboost_regrank_data.h"
+#include "xgboost_regrank_eval.h"
+#include "xgboost_regrank_obj.h"
+#include "../utils/xgboost_omp.h"
+#include "../booster/xgboost_gbmbase.h"
+#include "../utils/xgboost_utils.h"
+#include "../utils/xgboost_stream.h"
+
+namespace xgboost{
+    namespace regrank{
+        /*! \brief class for gradient boosted regression and ranking */
+        class RegRankBoostLearner{
+        public:
+            /*! \brief constructor */
+            RegRankBoostLearner(void){
+                silent = 0;
+                obj_ = NULL;
+                name_obj_ = "reg:linear";
+            }
+            /*!
+             * \brief a regression booter associated with training and evaluating data
+             * \param mats  array of pointers to matrix whose prediction result need to be cached
+             */
+            RegRankBoostLearner(const std::vector<DMatrix *>& mats){
+                silent = 0;
+                obj_ = NULL;
+                name_obj_ = "reg:linear";
+                this->SetCacheData(mats);
+            }            
+            /*!
+             * \brief add internal cache space for mat, this can speedup prediction for matrix,
+             *        please cache prediction for training and eval data
+             *    warning: if the model is loaded from file from some previous training history
+             *             set cache data must be called with exactly SAME 
+             *             data matrices to continue training otherwise it will cause error
+             * \param mats  array of pointers to matrix whose prediction result need to be cached
+             */          
+            inline void SetCacheData(const std::vector<DMatrix *>& mats){
+                // estimate feature bound
+                int num_feature = 0;
+                // assign buffer index
+                unsigned buffer_size = 0;
+                
+                utils::Assert( cache_.size() == 0, "can only call cache data once" );
+                for( size_t i = 0; i < mats.size(); ++i ){
+                    bool dupilicate = false;
+                    for( size_t j = 0; j < i; ++ j ){
+                        if( mats[i] == mats[j] ) dupilicate = true;
+                    }
+                    if( dupilicate ) continue;
+                    // set mats[i]'s cache learner pointer to this
+                    mats[i]->cache_learner_ptr_ = this;
+                    cache_.push_back( CacheEntry( mats[i], buffer_size, mats[i]->Size() ) );
+                    buffer_size += static_cast<unsigned>(mats[i]->Size());
+                    num_feature = std::max(num_feature, (int)(mats[i]->data.NumCol()));
+                }
+                
+                char str_temp[25];
+                if (num_feature > mparam.num_feature){
+                    mparam.num_feature = num_feature;
+                    sprintf(str_temp, "%d", num_feature);
+                    base_gbm.SetParam("bst:num_feature", str_temp);
+                }
+
+                sprintf(str_temp, "%u", buffer_size);
+                base_gbm.SetParam("num_pbuffer", str_temp);
+                if (!silent){
+                    printf("buffer_size=%u\n", buffer_size);
+                }
+            }
+
+            /*!
+             * \brief set parameters from outside
+             * \param name name of the parameter
+             * \param val  value of the parameter
+             */
+            inline void SetParam(const char *name, const char *val){
+                if (!strcmp(name, "silent"))  silent = atoi(val);
+                if (!strcmp(name, "eval_metric"))  evaluator_.AddEval(val);
+                if (!strcmp(name, "objective") )   name_obj_ = val;
+                if (!strcmp(name, "num_class") )   base_gbm.SetParam("num_booster_group", val );
+                mparam.SetParam(name, val);
+                base_gbm.SetParam(name, val);
+                cfg_.push_back( std::make_pair( std::string(name), std::string(val) ) );
+            }
+            /*!
+            * \brief initialize solver before training, called before training
+            * this function is reserved for solver to allocate necessary space and do other preparation
+            */
+            inline void InitTrainer(void){
+                if( mparam.num_class != 0 ){
+                    if( name_obj_ != "multi:softmax" ){
+                        name_obj_ = "multi:softmax";
+                        printf("auto select objective=softmax to support multi-class classification\n" );
+                    }
+                }
+                base_gbm.InitTrainer();                
+                obj_ = CreateObjFunction( name_obj_.c_str() );
+                for( size_t i = 0; i < cfg_.size(); ++ i ){
+                    obj_->SetParam( cfg_[i].first.c_str(), cfg_[i].second.c_str() );
+                }
+                evaluator_.AddEval( obj_->DefaultEvalMetric() );
+            }
+            /*!
+             * \brief initialize the current data storage for model, if the model is used first time, call this function
+             */
+            inline void InitModel(void){
+                base_gbm.InitModel();
+                mparam.AdjustBase(name_obj_.c_str());
+            }
+            /*!
+             * \brief load model from file 
+             * \param fname file name
+             */
+            inline void LoadModel(const char *fname){
+                utils::FileStream fi(utils::FopenCheck(fname, "rb"));
+                this->LoadModel(fi);
+                fi.Close();                
+            }
+            /*!
+             * \brief load model from stream
+             * \param fi input stream
+             */
+            inline void LoadModel(utils::IStream &fi){
+                base_gbm.LoadModel(fi);
+                utils::Assert(fi.Read(&mparam, sizeof(ModelParam)) != 0);
+            }
+            /*!
+             * \brief DumpModel
+             * \param fo text file
+             * \param fmap feature map that may help give interpretations of feature
+             * \param with_stats whether print statistics as well
+             */
+            inline void DumpModel(FILE *fo, const utils::FeatMap& fmap, bool with_stats){
+                base_gbm.DumpModel(fo, fmap, with_stats);
+            }
+            /*!
+             * \brief Dump path of all trees
+             * \param fo text file
+             * \param data input data
+             */
+            inline void DumpPath(FILE *fo, const DMatrix &data){
+                base_gbm.DumpPath(fo, data.data);
+            }
+            /*!
+            * \brief save model to stream
+            * \param fo output stream
+            */
+            inline void SaveModel(utils::IStream &fo) const{
+                base_gbm.SaveModel(fo);
+                fo.Write(&mparam, sizeof(ModelParam));
+            }
+            /*!
+             * \brief save model into file
+             * \param fname file name
+             */
+            inline void SaveModel(const char *fname) const{
+                utils::FileStream fo(utils::FopenCheck(fname, "wb"));
+                this->SaveModel(fo);
+                fo.Close();                
+            }
+            /*!
+             * \brief update the model for one iteration
+             */
+            inline void UpdateOneIter(const DMatrix &train){
+                this->PredictRaw(preds_, train);
+                obj_->GetGradient(preds_, train.info, base_gbm.NumBoosters(), grad_, hess_);
+                if( grad_.size() == train.Size() ){
+                    base_gbm.DoBoost(grad_, hess_, train.data, train.info.root_index);
+                }else{
+                    int ngroup = base_gbm.NumBoosterGroup();
+                    utils::Assert( grad_.size() == train.Size() * (size_t)ngroup, "BUG: UpdateOneIter: mclass" );
+                    std::vector<float> tgrad( train.Size() ), thess( train.Size() );
+                    for( int g = 0; g < ngroup; ++ g ){
+                        memcpy( &tgrad[0], &grad_[g*tgrad.size()], sizeof(float)*tgrad.size() );
+                        memcpy( &thess[0], &hess_[g*tgrad.size()], sizeof(float)*tgrad.size() );
+                        base_gbm.DoBoost(tgrad, thess, train.data, train.info.root_index, g );
+                    }
+                }
+            }
+            /*!
+             * \brief evaluate the model for specific iteration
+             * \param iter iteration number
+             * \param evals datas i want to evaluate
+             * \param evname name of each dataset
+             * \param fo file to output log
+             */
+            inline void EvalOneIter(int iter,
+                                    const std::vector<const DMatrix*> &evals,
+                                    const std::vector<std::string> &evname,
+                                    FILE *fo=stderr ){
+                fprintf(fo, "[%d]", iter);
+                for (size_t i = 0; i < evals.size(); ++i){
+                    this->PredictRaw(preds_, *evals[i]);
+                    obj_->PredTransform(preds_);
+                    evaluator_.Eval(fo, evname[i].c_str(), preds_, evals[i]->info);
+                }
+                fprintf(fo, "\n");
+                fflush(fo);
+            }
+            /*! 
+             * \brief get prediction
+             * \param storage to store prediction
+             * \param data input data
+             * \param bst_group booster group we are in
+             */
+            inline void Predict(std::vector<float> &preds, const DMatrix &data, int bst_group = -1){
+                this->PredictRaw( preds, data, bst_group );
+                obj_->PredTransform( preds );
+            }            
+        public:
+            /*!
+             * \brief interactive update 
+             * \param action action type 
+             * \parma train training data
+             */
+            inline void UpdateInteract(std::string action, const DMatrix& train){
+                for(size_t i = 0; i < cache_.size(); ++i){
+                    this->InteractPredict(preds_, *cache_[i].mat_);
+                }
+
+                if (action == "remove"){
+                    base_gbm.DelteBooster(); return;
+                }
+
+                obj_->GetGradient(preds_, train.info, base_gbm.NumBoosters(), grad_, hess_);
+                std::vector<unsigned> root_index;
+                base_gbm.DoBoost(grad_, hess_, train.data, root_index);
+
+                for(size_t i = 0; i < cache_.size(); ++i){
+                    this->InteractRePredict(*cache_[i].mat_);
+                }
+            }
+        private:
+            /*! \brief get the transformed predictions, given data */
+            inline void InteractPredict(std::vector<float> &preds, const DMatrix &data){
+                int buffer_offset = this->FindBufferOffset(data);
+                utils::Assert( buffer_offset >=0, "interact mode must cache training data" );
+                preds.resize(data.Size());
+                const unsigned ndata = static_cast<unsigned>(data.Size());
+                #pragma omp parallel for schedule( static )
+                for (unsigned j = 0; j < ndata; ++j){
+                    preds[j] = mparam.base_score + base_gbm.InteractPredict(data.data, j, buffer_offset + j);                    
+                }
+                obj_->PredTransform( preds );
+            }
+            /*! \brief repredict trial */
+            inline void InteractRePredict(const DMatrix &data){
+                int buffer_offset = this->FindBufferOffset(data);
+                utils::Assert( buffer_offset >=0, "interact mode must cache training data" );
+                const unsigned ndata = static_cast<unsigned>(data.Size());
+                #pragma omp parallel for schedule( static )
+                for (unsigned j = 0; j < ndata; ++j){
+                    base_gbm.InteractRePredict(data.data, j, buffer_offset + j);
+                }
+            }
+            /*! \brief get un-transformed prediction*/
+            inline void PredictRaw(std::vector<float> &preds, const DMatrix &data, int bst_group = -1 ){
+                int buffer_offset =  this->FindBufferOffset(data);
+                if( bst_group < 0 ){
+                    int ngroup = base_gbm.NumBoosterGroup();
+                    preds.resize( data.Size() * ngroup );
+                    for( int g = 0; g < ngroup; ++ g ){ 
+                        this->PredictBuffer(&preds[ data.Size() * g ], data, buffer_offset, g );
+                    }
+                }else{
+                    preds.resize( data.Size() );
+                    this->PredictBuffer(&preds[0], data, buffer_offset, bst_group );
+                }
+            }
+            /*! \brief get the un-transformed predictions, given data */
+            inline void PredictBuffer(float *preds, const DMatrix &data, int buffer_offset, int bst_group ){
+                const unsigned ndata = static_cast<unsigned>(data.Size());
+                if( buffer_offset >= 0 ){  
+                    #pragma omp parallel for schedule( static )
+                    for (unsigned j = 0; j < ndata; ++j){
+                        preds[j] = mparam.base_score + base_gbm.Predict(data.data, j, buffer_offset + j, data.info.GetRoot(j), bst_group );
+
+                    }
+                }else
+                    #pragma omp parallel for schedule( static )
+                    for (unsigned j = 0; j < ndata; ++j){
+                        preds[j] = mparam.base_score + base_gbm.Predict(data.data, j, -1, data.info.GetRoot(j), bst_group );
+                    }{
+                }
+            }
+        private:
+            /*! \brief training parameter for regression */
+            struct ModelParam{
+                /* \brief global bias */
+                float base_score;
+                /* \brief type of loss function */
+                int loss_type;
+                /* \brief number of features  */
+                int num_feature;  
+                /* \brief number of class, if it is multi-class classification  */
+                int num_class; 
+                /*! \brief reserved field */
+                int reserved[15];
+                /*! \brief constructor */
+                ModelParam(void){
+                    base_score = 0.5f;
+                    loss_type = -1;
+                    num_feature = 0;
+                    num_class = 0;
+                    memset(reserved, 0, sizeof(reserved));
+                }
+                /*!
+                 * \brief set parameters from outside
+                 * \param name name of the parameter
+                 * \param val  value of the parameter
+                 */
+                inline void SetParam(const char *name, const char *val){
+                    if (!strcmp("base_score", name))  base_score = (float)atof(val);
+                    if (!strcmp("num_class", name))   num_class = atoi(val);
+                    if (!strcmp("loss_type", name))   loss_type = atoi(val);
+                    if (!strcmp("bst:num_feature", name)) num_feature = atoi(val);
+                }
+                /*!
+                * \brief adjust base_score based on loss type and objective function
+                */
+                inline void AdjustBase(const char *obj){
+                    // some tweaks for loss type
+                    if( loss_type == -1 ){
+                        loss_type = 1;
+                        if( !strcmp("reg:linear", obj ) ) loss_type = 0;
+                    }
+                    if (loss_type == 1 || loss_type == 2|| loss_type == 3){
+                        utils::Assert(base_score > 0.0f && base_score < 1.0f, "sigmoid range constrain");
+                        base_score = -logf(1.0f / base_score - 1.0f);
+                    }
+                }
+            };
+        private:
+            struct CacheEntry{
+                const DMatrix *mat_;
+                int   buffer_offset_;
+                size_t num_row_; 
+                CacheEntry(const DMatrix *mat, int buffer_offset, size_t num_row)
+                    :mat_(mat), buffer_offset_(buffer_offset), num_row_(num_row){}
+            };           
+            /*! \brief the entries indicates that we have internal prediction cache */
+            std::vector<CacheEntry> cache_;
+        private:
+            // find internal bufer offset for certain matrix, if not exist, return -1
+            inline int FindBufferOffset(const DMatrix &mat){
+                for(size_t i = 0; i < cache_.size(); ++i){
+                    if( cache_[i].mat_ == &mat && mat.cache_learner_ptr_ == this ) {
+                        if( cache_[i].num_row_ == mat.Size() ){                            
+                            return cache_[i].buffer_offset_; 
+                        }else{
+                            fprintf( stderr, "warning: number of rows in input matrix changed as remembered in cachelist, ignore cached results\n" );
+                            fflush( stderr );
+                        }
+                    }
+                }
+                return -1;
+            } 
+        protected:
+            int silent;
+            EvalSet evaluator_;
+            booster::GBMBase base_gbm;
+            ModelParam   mparam;           
+            // objective fnction
+            IObjFunction *obj_;
+            // name of objective function
+            std::string name_obj_;
+            std::vector< std::pair<std::string, std::string> > cfg_;
+        protected:
+            std::vector<float> grad_, hess_, preds_;
+        };
+    }
+};
+
+#endif

regrank/xgboost_regrank_data.h

@@ -0,0 +1,260 @@
+#ifndef XGBOOST_REGRANK_DATA_H
+#define XGBOOST_REGRANK_DATA_H
+
+/*!
+ * \file xgboost_regrank_data.h
+ * \brief input data structure for regression, binary classification, and rankning.
+ *     Format:
+ *        The data should contain each data instance in each line.
+ *		  The format of line data is as below:
+ *        label <nonzero feature dimension> [feature index:feature value]+
+ *     When using rank, an addtional group file with suffix group must be provided, giving the number of instances in each group
+ *     When using weighted aware classification(regression), an addtional weight file must be provided, giving the weight of each instance
+ * 
+ * \author Kailong Chen: chenkl198812@gmail.com, Tianqi Chen: tianqi.tchen@gmail.com
+ */
+#include <cstdio>
+#include <vector>
+#include <string>
+#include <cstring>
+#include "../booster/xgboost_data.h"
+#include "../utils/xgboost_utils.h"
+#include "../utils/xgboost_stream.h"
+
+namespace xgboost{
+    /*! \brief namespace to handle regression and rank */
+    namespace regrank{
+        /*! \brief data matrix for regression content */
+        struct DMatrix{
+        public:
+            /*! \brief data information besides the features */
+            struct Info{
+                /*! \brief label of each instance */
+                std::vector<float> labels;
+                /*! \brief the index of begin and end of a groupneeded when the learning task is ranking */
+                std::vector<unsigned> group_ptr;
+                /*! \brief weights of each instance, optional */            
+                std::vector<float> weights;
+                /*! \brief specified root index of each instance, can be used for multi task setting*/
+                std::vector<unsigned> root_index;
+                /*! \brief get weight of each instances */
+                inline float GetWeight( size_t i ) const{
+                    if( weights.size() != 0 ) return weights[i];
+                    else return 1.0f;
+                }
+                inline float GetRoot( size_t i ) const{
+                    if( root_index.size() != 0 ) return static_cast<float>(root_index[i]);
+                    else return 0;
+                }
+            };
+        public:
+            /*! \brief feature data content */
+            booster::FMatrixS data;
+            /*! \brief information fields */
+            Info info;
+            /*! 
+             * \brief cache pointer to verify if the data structure is cached in some learner 
+             * this is a bit ugly, we need to have double check verification, so if one side get deleted, 
+             * and some strange re-allocation gets the same pointer we will still be fine
+             */
+            void *cache_learner_ptr_;
+        public:
+            /*! \brief default constructor */
+            DMatrix(void):cache_learner_ptr_(NULL){}
+            /*! \brief get the number of instances */
+            inline size_t Size() const{
+                return data.NumRow();
+            }           
+            /*!
+             * \brief load from text file
+             * \param fname name of text data
+             * \param silent whether print information or not
+             */
+            inline void LoadText(const char* fname, bool silent = false){
+                data.Clear();
+                FILE* file = utils::FopenCheck(fname, "r");
+                float label; bool init = true;
+                char tmp[1024];
+                std::vector<booster::bst_uint> findex;
+                std::vector<booster::bst_float> fvalue;
+                
+                while (fscanf(file, "%s", tmp) == 1){
+                    unsigned index; float value;
+                    if (sscanf(tmp, "%u:%f", &index, &value) == 2){
+                        findex.push_back(index); fvalue.push_back(value);
+                    }
+                    else{
+                        if (!init){
+                            info.labels.push_back(label);
+                            data.AddRow(findex, fvalue);
+                        }
+                        findex.clear(); fvalue.clear();
+                        utils::Assert(sscanf(tmp, "%f", &label) == 1, "invalid format");
+                        init = false;
+                    }
+                }
+            
+                info.labels.push_back(label);
+                data.AddRow(findex, fvalue);
+                // initialize column support as well
+                data.InitData();
+                
+                if (!silent){
+                    printf("%ux%u matrix with %lu entries is loaded from %s\n",
+                           (unsigned)data.NumRow(), (unsigned)data.NumCol(), (unsigned long)data.NumEntry(), fname);
+                }
+                fclose(file);
+                this->TryLoadGroup(fname, silent);
+                this->TryLoadWeight(fname, silent);
+            }
+            /*!
+             * \brief load from binary file
+             * \param fname name of binary data
+             * \param silent whether print information or not
+             * \return whether loading is success
+             */
+            inline bool LoadBinary(const char* fname, bool silent = false){
+                FILE *fp = fopen64(fname, "rb");
+                if (fp == NULL) return false;
+                utils::FileStream fs(fp);
+                data.LoadBinary(fs);
+                info.labels.resize(data.NumRow());
+                utils::Assert(fs.Read(&info.labels[0], sizeof(float)* data.NumRow()) != 0, "DMatrix LoadBinary");
+                {// load in group ptr
+                    unsigned ngptr;
+                    if( fs.Read(&ngptr, sizeof(unsigned) ) != 0 ){
+                        info.group_ptr.resize( ngptr );
+                        if( ngptr != 0 ){
+                            utils::Assert( fs.Read(&info.group_ptr[0], sizeof(unsigned) * ngptr) != 0, "Load group file");
+                            utils::Assert( info.group_ptr.back() == data.NumRow(), "number of group must match number of record" );
+                        }
+                    }
+                }
+                {// load in weight
+                    unsigned nwt;
+                    if( fs.Read(&nwt, sizeof(unsigned) ) != 0 ){
+                        utils::Assert( nwt == 0 || nwt == data.NumRow(), "invalid weight" );
+                        info.weights.resize( nwt );
+                        if( nwt != 0 ){
+                            utils::Assert( fs.Read(&info.weights[0], sizeof(unsigned) * nwt) != 0, "Load weight file");
+                        }
+                    }
+                }
+                fs.Close();
+                
+                if (!silent){
+                    printf("%ux%u matrix with %lu entries is loaded from %s\n",
+                           (unsigned)data.NumRow(), (unsigned)data.NumCol(), (unsigned long)data.NumEntry(), fname);
+                    if( info.group_ptr.size() != 0 ){
+                        printf("data contains %u groups\n", (unsigned)info.group_ptr.size()-1 );
+                    }
+                }
+                return true;
+            }
+            /*!
+             * \brief save to binary file
+             * \param fname name of binary data
+             * \param silent whether print information or not
+             */
+            inline void SaveBinary(const char* fname, bool silent = false){
+                // initialize column support as well
+                data.InitData();
+                
+                utils::FileStream fs(utils::FopenCheck(fname, "wb"));
+                data.SaveBinary(fs);
+                utils::Assert( info.labels.size() == data.NumRow(), "label size is not consistent with feature matrix size" );
+                fs.Write(&info.labels[0], sizeof(float) * data.NumRow());
+                {// write out group ptr
+                    unsigned ngptr = static_cast<unsigned>( info.group_ptr.size() );
+                    fs.Write(&ngptr, sizeof(unsigned) );
+                    if( ngptr != 0 ){
+                        fs.Write(&info.group_ptr[0], sizeof(unsigned) * ngptr);
+                    }
+                }                
+                {// write out weight
+                    unsigned nwt = static_cast<unsigned>( info.weights.size() );
+                    fs.Write( &nwt, sizeof(unsigned) );
+                    if( nwt != 0 ){
+                        fs.Write(&info.weights[0], sizeof(float) * nwt);
+                    }
+                }
+                fs.Close();
+                if (!silent){
+                    printf("%ux%u matrix with %lu entries is saved to %s\n",
+                       (unsigned)data.NumRow(), (unsigned)data.NumCol(), (unsigned long)data.NumEntry(), fname);
+                    if( info.group_ptr.size() != 0 ){
+                        printf("data contains %u groups\n", (unsigned)info.group_ptr.size()-1 );
+                    }
+                }
+            }
+            /*!
+             * \brief cache load data given a file name, if filename ends with .buffer, direct load binary
+             *        otherwise the function will first check if fname + '.buffer' exists,
+             *        if binary buffer exists, it will reads from binary buffer, otherwise, it will load from text file,
+             *        and try to create a buffer file
+             * \param fname name of binary data
+             * \param silent whether print information or not
+             * \param savebuffer whether do save binary buffer if it is text
+             */
+            inline void CacheLoad(const char *fname, bool silent = false, bool savebuffer = true){
+                int len = strlen(fname);
+                if (len > 8 && !strcmp(fname + len - 7, ".buffer")){
+                    if( !this->LoadBinary(fname, silent) ){
+                        fprintf(stderr,"can not open file \"%s\"", fname);
+                        utils::Error("DMatrix::CacheLoad failed");
+                    }
+                    return;
+                }
+                char bname[1024];
+                sprintf(bname, "%s.buffer", fname);
+                if (!this->LoadBinary(bname, silent)){
+                    this->LoadText(fname, silent);
+                    if (savebuffer) this->SaveBinary(bname, silent);
+                }
+            }
+        private:
+            inline bool TryLoadGroup(const char* fname, bool silent = false){
+                std::string name = fname;
+                if (name.length() > 8 && !strcmp(fname + name.length() - 7, ".buffer")){
+                    name.resize( name.length() - 7 );
+                }
+                name += ".group";
+                //if exists group data load it in
+                FILE *fi = fopen64(name.c_str(), "r");
+                if (fi == NULL) return false;                
+                info.group_ptr.push_back(0);
+                unsigned nline;
+                while (fscanf(fi, "%u", &nline) == 1){
+                    info.group_ptr.push_back(info.group_ptr.back()+nline);
+                }
+                if(!silent){
+                    printf("%lu groups are loaded from %s\n", info.group_ptr.size()-1, name.c_str());
+                }
+                fclose(fi);
+                utils::Assert( info.group_ptr.back() == data.NumRow(), "DMatrix: group data does not match the number of rows in feature matrix" );
+                return true;
+            }
+            inline bool TryLoadWeight(const char* fname, bool silent = false){
+                std::string name = fname;
+                if (name.length() > 8 && !strcmp(fname + name.length() - 7, ".buffer")){
+                    name.resize( name.length() - 7 );
+                }
+                name += ".weight";
+                //if exists group data load it in
+                FILE *fi = fopen64(name.c_str(), "r");
+                if (fi == NULL) return false;                
+                float wt;
+                while (fscanf(fi, "%f", &wt) == 1){
+                    info.weights.push_back( wt );
+                }
+                if(!silent){
+                    printf("loading weight from %s\n", name.c_str());
+                }
+                fclose(fi);
+                utils::Assert( info.weights.size() == data.NumRow(), "DMatrix: weight data does not match the number of rows in feature matrix" );
+                return true;
+            }
+        };
+    };
+};
+#endif

regrank/xgboost_regrank_eval.h

@@ -0,0 +1,375 @@
+#ifndef XGBOOST_REGRANK_EVAL_H
+#define XGBOOST_REGRANK_EVAL_H
+/*!
+* \file xgboost_regrank_eval.h
+* \brief evaluation metrics for regression and classification and rank
+* \author Kailong Chen: chenkl198812@gmail.com, Tianqi Chen: tianqi.tchen@gmail.com
+*/
+
+#include <cmath>
+#include <vector>
+#include <algorithm>
+#include "../utils/xgboost_utils.h"
+#include "../utils/xgboost_omp.h"
+#include "../utils/xgboost_random.h"
+#include "xgboost_regrank_data.h"
+#include "xgboost_regrank_utils.h"
+
+namespace xgboost{
+    namespace regrank{
+        /*! \brief evaluator that evaluates the loss metrics */
+        struct IEvaluator{
+            /*!
+             * \brief evaluate a specific metric
+             * \param preds prediction
+             * \param info information, including label etc.
+             */
+            virtual float Eval(const std::vector<float> &preds,
+            const DMatrix::Info &info) const = 0;
+            /*! \return name of metric */
+            virtual const char *Name(void) const = 0;
+            /*! \brief virtual destructor */
+            virtual ~IEvaluator(void){}
+        };
+
+        /*! \brief RMSE */
+        struct EvalRMSE : public IEvaluator{
+            virtual float Eval(const std::vector<float> &preds,
+                               const DMatrix::Info &info) const {
+                utils::Assert( preds.size() == info.labels.size(), "label size predict size not match" );
+                const unsigned ndata = static_cast<unsigned>(preds.size());
+                float sum = 0.0, wsum = 0.0;
+                #pragma omp parallel for reduction(+:sum,wsum) schedule( static )
+                for (unsigned i = 0; i < ndata; ++i){
+                    const float wt = info.GetWeight(i);
+                    const float diff = info.labels[i] - preds[i];
+                    sum += diff*diff * wt;
+                    wsum += wt;
+                }
+                return sqrtf(sum / wsum);
+            }
+            virtual const char *Name(void) const{
+                return "rmse";
+            }
+        };
+
+        /*! \brief Error */
+        struct EvalLogLoss : public IEvaluator{
+            virtual float Eval(const std::vector<float> &preds,
+                               const DMatrix::Info &info) const {
+                utils::Assert( preds.size() == info.labels.size(), "label size predict size not match" );
+                const unsigned ndata = static_cast<unsigned>(preds.size());
+                float sum = 0.0f, wsum = 0.0f;
+                #pragma omp parallel for reduction(+:sum,wsum) schedule( static )
+                for (unsigned i = 0; i < ndata; ++i){
+                    const float y = info.labels[i];
+                    const float py = preds[i];
+                    const float wt = info.GetWeight(i);
+                    sum -= wt * (y * std::log(py) + (1.0f - y)*std::log(1 - py));
+                    wsum += wt;
+                }
+                return sum / wsum;
+            }
+            virtual const char *Name(void) const{
+                return "negllik";
+            }
+        };
+
+        /*! \brief Error */
+        struct EvalError : public IEvaluator{
+            virtual float Eval(const std::vector<float> &preds,
+                               const DMatrix::Info &info) const {
+                const unsigned ndata = static_cast<unsigned>(preds.size());
+                float sum = 0.0f, wsum = 0.0f;
+                #pragma omp parallel for reduction(+:sum,wsum) schedule( static )
+                for (unsigned i = 0; i < ndata; ++i){
+                    const float wt = info.GetWeight(i);                    
+                    if (preds[i] > 0.5f){
+                        if (info.labels[i] < 0.5f) sum += wt;
+                    }
+                    else{
+                        if (info.labels[i] >= 0.5f) sum += wt;
+                    }
+                    wsum += wt;
+                }
+                return sum / wsum;
+            }
+            virtual const char *Name(void) const{
+                return "error";
+            }
+        };
+
+        /*! \brief AMS: also records best threshold */
+        struct EvalAMS : public IEvaluator{
+        public:
+            EvalAMS(const char *name){
+                name_ = name;
+                // note: ams@0 will automatically select which ratio to go
+                utils::Assert( sscanf(name, "ams@%f", &ratio_ ) == 1, "invalid ams format" );
+            }            
+            virtual float Eval(const std::vector<float> &preds,
+                               const DMatrix::Info &info) const {
+                const unsigned ndata = static_cast<unsigned>(preds.size());
+                utils::Assert( info.weights.size() == ndata, "we need weight to evaluate ams");
+                std::vector< std::pair<float, unsigned> > rec(ndata);
+                
+                #pragma omp parallel for schedule( static )                
+                for (unsigned i = 0; i < ndata; ++i){
+                    rec[i] = std::make_pair( preds[i], i );
+                }
+                std::sort( rec.begin(), rec.end(), CmpFirst );
+                unsigned ntop = static_cast<unsigned>( ratio_ * ndata );
+                if( ntop == 0 ) ntop = ndata;
+                const double br = 10.0;
+                unsigned thresindex = 0;
+                double s_tp = 0.0, b_fp = 0.0, tams = 0.0;
+                for (unsigned i = 0; i < ndata-1 && i < ntop; ++i){
+                    const unsigned ridx = rec[i].second;
+                    const float wt = info.weights[ridx];
+                    if( info.labels[ridx] > 0.5f ){
+                        s_tp += wt;
+                    }else{
+                        b_fp += wt;
+                    }
+                    if( rec[i].first != rec[i+1].first ){
+                        double ams = sqrtf( 2*((s_tp+b_fp+br) * log( 1.0 + s_tp/(b_fp+br) ) - s_tp) );
+                        if( tams < ams ){
+                            thresindex = i;
+                            tams = ams;
+                        }
+                    }
+                }
+                if( ntop == ndata ){
+                    fprintf( stderr, "\tams-ratio=%g", float(thresindex)/ndata );
+                    return tams;
+                }else{
+                    return sqrtf( 2*((s_tp+b_fp+br) * log( 1.0 + s_tp/(b_fp+br) ) - s_tp) );
+                }
+            }
+            virtual const char *Name(void) const{
+                return name_.c_str();
+            }
+        private:
+            std::string name_;
+            float ratio_;
+        };
+
+        /*! \brief Error for multi-class classification, need exact match */
+        struct EvalMatchError : public IEvaluator{
+        public:
+            virtual float Eval(const std::vector<float> &preds,
+                               const DMatrix::Info &info) const {
+                const unsigned ndata = static_cast<unsigned>(preds.size());
+                float sum = 0.0f, wsum = 0.0f;
+                #pragma omp parallel for reduction(+:sum,wsum) schedule( static )
+                for (unsigned i = 0; i < ndata; ++i){
+                    const float wt = info.GetWeight(i);
+                    int label = static_cast<int>(info.labels[i]);
+                    if (static_cast<int>(preds[i]) != label ) sum += wt;
+                    wsum += wt;
+                }
+                return sum / wsum;
+            }
+            virtual const char *Name(void) const{
+                return "merror";
+            }
+        };
+
+        /*! \brief Area under curve, for both classification and rank */
+        struct EvalAuc : public IEvaluator{
+            virtual float Eval(const std::vector<float> &preds,
+                               const DMatrix::Info &info) const {
+                utils::Assert( preds.size() == info.labels.size(), "label size predict size not match" );
+                std::vector<unsigned> tgptr(2, 0); tgptr[1] = preds.size();
+                const std::vector<unsigned> &gptr = info.group_ptr.size() == 0 ? tgptr : info.group_ptr;
+                utils::Assert(gptr.back() == preds.size(), "EvalAuc: group structure must match number of prediction");
+                const unsigned ngroup = static_cast<unsigned>(gptr.size() - 1);
+
+                double sum_auc = 0.0f;
+                #pragma omp parallel reduction(+:sum_auc) 
+                {
+                    // each thread takes a local rec
+                    std::vector< std::pair<float, unsigned> > rec;
+                    #pragma omp for schedule(static) 
+                    for (unsigned k = 0; k < ngroup; ++k){
+                        rec.clear();
+                        for (unsigned j = gptr[k]; j < gptr[k + 1]; ++j){
+                            rec.push_back(std::make_pair(preds[j], j));
+                        }
+                        std::sort(rec.begin(), rec.end(), CmpFirst);
+                        // calculate AUC
+                        double sum_pospair = 0.0;
+                        double sum_npos = 0.0, sum_nneg = 0.0, buf_pos = 0.0, buf_neg = 0.0;
+                        for (size_t j = 0; j < rec.size(); ++j){
+                            const float wt = info.GetWeight(rec[j].second);
+                            const float ctr = info.labels[rec[j].second];
+                            // keep bucketing predictions in same bucket
+                            if (j != 0 && rec[j].first != rec[j - 1].first){
+                                sum_pospair += buf_neg * (sum_npos + buf_pos *0.5);
+                                sum_npos += buf_pos; sum_nneg += buf_neg;
+                                buf_neg = buf_pos = 0.0f;
+                            }
+                            buf_pos += ctr * wt; buf_neg += (1.0f - ctr) * wt;
+                        }
+                        sum_pospair += buf_neg * (sum_npos + buf_pos *0.5);
+                        sum_npos += buf_pos; sum_nneg += buf_neg;
+                        // 
+                        utils::Assert(sum_npos > 0.0 && sum_nneg > 0.0, "the dataset only contains pos or neg samples");
+                        // this is the AUC
+                        sum_auc += sum_pospair / (sum_npos*sum_nneg);
+                    }
+                }
+                // return average AUC over list
+                return static_cast<float>(sum_auc) / ngroup;
+            }
+            virtual const char *Name(void) const{
+                return "auc";
+            }
+        };
+
+        /*! \brief Evaluate rank list */          
+        struct EvalRankList : public IEvaluator{
+        public:
+            virtual float Eval(const std::vector<float> &preds,
+                               const DMatrix::Info &info) const {
+                utils::Assert( preds.size() == info.labels.size(), "label size predict size not match" );
+                const std::vector<unsigned> &gptr = info.group_ptr;
+                utils::Assert(gptr.size() != 0, "must specify group when constructing rank file");
+                utils::Assert( gptr.back() == preds.size(), "EvalRanklist: group structure must match number of prediction");
+                const unsigned ngroup = static_cast<unsigned>(gptr.size() - 1);
+
+                double sum_metric = 0.0f;
+                #pragma omp parallel reduction(+:sum_metric) 
+                {
+                    // each thread takes a local rec
+                    std::vector< std::pair<float, unsigned> > rec;
+                    #pragma omp for schedule(static) 
+                    for (unsigned k = 0; k < ngroup; ++k){
+                        rec.clear();
+                        for (unsigned j = gptr[k]; j < gptr[k + 1]; ++j){
+                            rec.push_back(std::make_pair(preds[j], (int)info.labels[j]));
+                        }
+                        sum_metric += this->EvalMetric( rec );                        
+                    }
+                }
+                return static_cast<float>(sum_metric) / ngroup;
+            }
+            virtual const char *Name(void) const{
+                return name_.c_str();
+            }
+        protected:
+            EvalRankList(const char *name){
+                name_ = name;
+                if( sscanf(name, "%*[^@]@%u", &topn_) != 1 ){
+                    topn_ = UINT_MAX;
+                }
+            }
+            /*! \return evaluation metric, given the pair_sort record, (pred,label) */
+            virtual float EvalMetric( std::vector< std::pair<float, unsigned> > &pair_sort ) const = 0;
+        protected:
+            unsigned topn_;
+            std::string name_;
+        };
+        
+        /*! \brief Precison at N, for both classification and rank */
+        struct EvalPrecision : public EvalRankList{
+        public:
+            EvalPrecision(const char *name):EvalRankList(name){}
+        protected:
+            virtual float EvalMetric( std::vector< std::pair<float, unsigned> > &rec ) const {
+                // calculate Preicsion
+                std::sort(rec.begin(), rec.end(), CmpFirst);
+                unsigned nhit = 0;
+                for (size_t j = 0; j < rec.size() && j < this->topn_; ++j){
+                    nhit += (rec[j].second != 0 );
+                }
+                return static_cast<float>( nhit ) / topn_;
+            }
+        };
+
+
+        /*! \brief NDCG */
+        struct EvalNDCG : public EvalRankList{
+        public:
+            EvalNDCG(const char *name):EvalRankList(name){}
+        protected:
+            inline float CalcDCG( const std::vector< std::pair<float,unsigned> > &rec ) const {
+                double sumdcg = 0.0;
+                for( size_t i = 0; i < rec.size() && i < this->topn_; i ++ ){
+                    const unsigned rel = rec[i].second;
+                    if( rel != 0 ){ 
+                        sumdcg += logf(2.0f) * ((1<<rel)-1) / logf( i + 2 );
+                    }
+                }
+                return static_cast<float>(sumdcg);
+            }
+            virtual float EvalMetric( std::vector< std::pair<float, unsigned> > &rec ) const {
+                std::sort(rec.begin(), rec.end(), CmpSecond);
+                float idcg = this->CalcDCG(rec);
+                std::sort(rec.begin(), rec.end(), CmpFirst);
+                float dcg = this->CalcDCG(rec);
+                if( idcg == 0.0f ) return 0.0f;
+                else return dcg/idcg;
+            }
+        };
+
+        /*! \brief Precison at N, for both classification and rank */
+        struct EvalMAP : public EvalRankList{
+        public:
+            EvalMAP(const char *name):EvalRankList(name){}
+        protected:
+            virtual float EvalMetric( std::vector< std::pair<float, unsigned> > &rec ) const {
+                std::sort(rec.begin(), rec.end(), CmpFirst);
+                unsigned nhits = 0;
+                double sumap = 0.0;
+                for( size_t i = 0; i < rec.size(); ++i){
+                    if( rec[i].second != 0 ){
+                        nhits += 1;
+                        if( i < this->topn_ ){
+                            sumap += static_cast<float>(nhits) / (i+1);
+                        }
+                    }
+                }
+                if (nhits != 0) sumap /= nhits;
+                return static_cast<float>(sumap);                
+            }
+        };
+    };
+
+    namespace regrank{
+        /*! \brief a set of evaluators */
+        struct EvalSet{
+        public:
+            inline void AddEval(const char *name){
+                for (size_t i = 0; i < evals_.size(); ++i){
+                    if (!strcmp(name, evals_[i]->Name())) return;
+                }
+                if (!strcmp(name, "rmse"))    evals_.push_back(new EvalRMSE());
+                if (!strcmp(name, "error"))   evals_.push_back(new EvalError());
+                if (!strcmp(name, "merror"))   evals_.push_back(new EvalMatchError());
+                if (!strcmp(name, "logloss")) evals_.push_back(new EvalLogLoss());
+                if (!strcmp(name, "auc"))    evals_.push_back(new EvalAuc());
+                if (!strncmp(name, "ams@",4))  evals_.push_back(new EvalAMS(name));
+                if (!strncmp(name, "pre@", 4)) evals_.push_back(new EvalPrecision(name));
+                if (!strncmp(name, "map", 3))   evals_.push_back(new EvalMAP(name));
+                if (!strncmp(name, "ndcg", 3))  evals_.push_back(new EvalNDCG(name));
+            }
+            ~EvalSet(){
+                for (size_t i = 0; i < evals_.size(); ++i){
+                    delete evals_[i];
+                }
+            }
+            inline void Eval(FILE *fo, const char *evname,
+                const std::vector<float> &preds,
+                const DMatrix::Info &info) const{
+                for (size_t i = 0; i < evals_.size(); ++i){
+                    float res = evals_[i]->Eval(preds, info);
+                    fprintf(fo, "\t%s-%s:%f", evname, evals_[i]->Name(), res);
+                }
+            }
+        private:
+            std::vector<const IEvaluator*> evals_;
+        };
+    };
+};
+#endif