Update README.md

Fixing Configuration Type for Win32/Debug.

Thanks Giulio!

.gitignore

@@ -6,8 +6,41 @@
 # Compiled Dynamic libraries
 *.so
 *.dylib
-
+*.page
 # Compiled Static libraries
 *.lai
 *.la
 *.a
+*~
+*.Rcheck
+*.rds
+*.tar.gz
+*txt*
+*conf
+*buffer
+*model
+*pyc
+*train
+*test
+*group
+*rar
+*vali
+*data
+*sdf
+Release
+*exe*
+*exp
+ipch
+*.filters
+*.user
+*log
+Debug
+*suo
+*test*
+.Rhistory
+*.dll
+*i386
+*x64
+*dump
+*save
+*csv

CHANGES.md

@@ -0,0 +1,22 @@
+Change Log
+=====
+
+xgboost-0.1
+=====
+* Initial release
+
+xgboost-0.2x
+=====
+* Python module
+* Weighted samples instances
+* Initial version of pairwise rank
+
+xgboost-0.3
+=====
+* Faster tree construction module
+  - Allows subsample columns during tree construction via ```bst:col_samplebytree=ratio```
+* Support for boosting from initial predictions
+* Experimental version of LambdaRank
+* Linear booster is now parallelized, using parallel coordinated descent.
+* Add [Code Guide](src/README.md) for customizing objective function and evaluation
+* Add R module

LICENSE

@@ -1,202 +1,13 @@
-Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
+Copyright (c) 2014 by Tianqi Chen and Contributors 
 
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright {yyyy} {name of copyright owner}
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    
+   http://www.apache.org/licenses/LICENSE-2.0
 
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

Makefile

@@ -0,0 +1,65 @@
+export CC  = gcc
+export CXX = g++
+export LDFLAGS= -pthread -lm 
+
+export CFLAGS = -Wall -O3 -msse2  -Wno-unknown-pragmas -fPIC -pedantic 
+
+ifeq ($(no_omp),1)
+	CFLAGS += -DDISABLE_OPENMP 
+else 
+	CFLAGS += -fopenmp
+endif
+
+# specify tensor path
+BIN = xgboost
+OBJ = updater.o gbm.o io.o
+SLIB = wrapper/libxgboostwrapper.so 
+
+.PHONY: clean all python Rpack
+
+all: $(BIN) $(OBJ) $(SLIB) 
+
+python: wrapper/libxgboostwrapper.so
+# now the wrapper takes in two files. io and wrapper part
+wrapper/libxgboostwrapper.so: wrapper/xgboost_wrapper.cpp $(OBJ)
+updater.o: src/tree/updater.cpp  src/tree/*.hpp src/*.h src/tree/*.h
+gbm.o: src/gbm/gbm.cpp src/gbm/*.hpp src/gbm/*.h
+io.o: src/io/io.cpp src/io/*.hpp src/utils/*.h src/learner/dmatrix.h src/*.h
+xgboost: src/xgboost_main.cpp src/utils/*.h src/*.h src/learner/*.hpp src/learner/*.h $(OBJ)
+wrapper/libxgboostwrapper.so: wrapper/xgboost_wrapper.cpp src/utils/*.h src/*.h src/learner/*.hpp src/learner/*.h $(OBJ)
+
+$(BIN) : 
+	$(CXX) $(CFLAGS) $(LDFLAGS) -o $@ $(filter %.cpp %.o %.c, $^)
+
+$(SLIB) :
+	$(CXX) $(CFLAGS) -fPIC $(LDFLAGS) -shared -o $@ $(filter %.cpp %.o %.c, $^)
+
+$(OBJ) : 
+	$(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c, $^) )
+
+install:
+	cp -f -r $(BIN)  $(INSTALL_PATH)
+
+Rpack:
+	make clean
+	rm -rf xgboost xgboost*.tar.gz
+	cp -r R-package xgboost
+	rm -rf xgboost/inst/examples/*.buffer
+	rm -rf xgboost/inst/examples/*.model
+	rm -rf xgboost/inst/examples/dump*
+	rm -rf xgboost/src/*.o xgboost/src/*.so xgboost/src/*.dll
+	rm -rf xgboost/demo/*.model xgboost/demo/*.buffer xgboost/demo/*.txt
+	rm -rf xgboost/demo/runall.R
+	cp -r src xgboost/src/src
+	mkdir xgboost/src/wrapper
+	cp  wrapper/xgboost_wrapper.h xgboost/src/wrapper
+	cp  wrapper/xgboost_wrapper.cpp xgboost/src/wrapper
+	cp ./LICENSE xgboost
+	cat R-package/src/Makevars|sed '2s/.*/PKGROOT=./' > xgboost/src/Makevars
+	cat R-package/src/Makevars.win|sed '2s/.*/PKGROOT=./' > xgboost/src/Makevars.win
+	R CMD build xgboost
+	rm -rf xgboost
+	R CMD check --as-cran xgboost*.tar.gz
+
+clean:
+	$(RM) $(OBJ) $(BIN) $(SLIB) *.o  */*.o */*/*.o *~ */*~ */*/*~

R-package/DESCRIPTION

@@ -0,0 +1,24 @@
+Package: xgboost
+Type: Package
+Title: eXtreme Gradient Boosting
+Version: 0.3-2
+Date: 2014-08-23
+Author: Tianqi Chen <tianqi.tchen@gmail.com>, Tong He <hetong007@gmail.com>
+Maintainer: Tong He <hetong007@gmail.com>
+Description: This package is a R wrapper of xgboost, which is short for eXtreme
+    Gradient Boosting. It is an efficient and scalable implementation of
+    gradient boosting framework. The package includes efficient linear model
+    solver and tree learning algorithms. The package can automatically do
+    parallel computation with OpenMP, and it can be more than 10 times faster
+    than existing gradient boosting packages such as gbm. It supports various
+    objective functions, including regression, classification and ranking. The
+    package is made to be extensible, so that users are also allowed to define
+    their own objectives easily.
+License: Apache License (== 2.0) | file LICENSE
+URL: https://github.com/tqchen/xgboost
+BugReports: https://github.com/tqchen/xgboost/issues
+Depends:
+    R (>= 2.10)
+Imports:
+    Matrix (>= 1.1-0),
+    methods

R-package/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2014 by Tianqi Chen and Contributors 
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

R-package/NAMESPACE

@@ -0,0 +1,17 @@
+# Generated by roxygen2 (4.0.1): do not edit by hand
+
+export(getinfo)
+export(setinfo)
+export(slice)
+export(xgb.DMatrix)
+export(xgb.DMatrix.save)
+export(xgb.cv)
+export(xgb.dump)
+export(xgb.load)
+export(xgb.save)
+export(xgb.train)
+export(xgboost)
+exportMethods(predict)
+import(methods)
+importClassesFrom(Matrix,dgCMatrix)
+importClassesFrom(Matrix,dgeMatrix)

R-package/R/getinfo.xgb.DMatrix.R

@@ -0,0 +1,41 @@
+setClass('xgb.DMatrix')
+
+#' Get information of an xgb.DMatrix object
+#' 
+#' Get information of an xgb.DMatrix object
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' train <- agaricus.train
+#' dtrain <- xgb.DMatrix(train$data, label=train$label)
+#' labels <- getinfo(dtrain, 'label')
+#' setinfo(dtrain, 'label', 1-labels)
+#' labels2 <- getinfo(dtrain, 'label')
+#' stopifnot(all(labels2 == 1-labels))
+#' @rdname getinfo
+#' @export
+#' 
+getinfo <- function(object, ...){
+    UseMethod("getinfo")
+}
+
+#' @param object Object of class "xgb.DMatrix"
+#' @param name the name of the field to get
+#' @param ... other parameters
+#' @rdname getinfo
+#' @method getinfo xgb.DMatrix
+setMethod("getinfo", signature = "xgb.DMatrix", 
+          definition = function(object, name) {
+              if (typeof(name) != "character") {
+                  stop("xgb.getinfo: name must be character")
+              }
+              if (class(object) != "xgb.DMatrix") {
+                  stop("xgb.setinfo: first argument dtrain must be xgb.DMatrix")
+              }
+              if (name != "label" && name != "weight" && name != "base_margin") {
+                  stop(paste("xgb.getinfo: unknown info name", name))
+              }
+              ret <- .Call("XGDMatrixGetInfo_R", object, name, PACKAGE = "xgboost")
+              return(ret)
+          })
+

R-package/R/predict.xgb.Booster.R

@@ -0,0 +1,42 @@
+setClass("xgb.Booster")
+
+#' Predict method for eXtreme Gradient Boosting model
+#' 
+#' Predicted values based on xgboost model object.
+#' 
+#' @param object Object of class "xgb.Boost"
+#' @param newdata takes \code{matrix}, \code{dgCMatrix}, local data file or 
+#'   \code{xgb.DMatrix}. 
+#' @param outputmargin whether the prediction should be shown in the original
+#'   value of sum of functions, when outputmargin=TRUE, the prediction is 
+#'   untransformed margin value. In logistic regression, outputmargin=T will
+#'   output value before logistic transformation.
+#' @param ntreelimit limit number of trees used in prediction, this parameter is
+#'  only valid for gbtree, but not for gblinear. set it to be value bigger 
+#'  than 0. It will use all trees by default.
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' data(agaricus.test, package='xgboost')
+#' train <- agaricus.train
+#' test <- agaricus.test
+#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
+#'                eta = 1, nround = 2,objective = "binary:logistic")
+#' pred <- predict(bst, test$data)
+#' @export
+#' 
+setMethod("predict", signature = "xgb.Booster", 
+          definition = function(object, newdata, outputmargin = FALSE, ntreelimit = NULL) {
+  if (class(newdata) != "xgb.DMatrix") {
+    newdata <- xgb.DMatrix(newdata)
+  }
+  if (is.null(ntreelimit)) {
+    ntreelimit <- 0
+  } else {
+    if (ntreelimit < 1){
+      stop("predict: ntreelimit must be equal to or greater than 1")
+    }
+  }
+  ret <- .Call("XGBoosterPredict_R", object, newdata, as.integer(outputmargin), as.integer(ntreelimit), PACKAGE = "xgboost")
+  return(ret)
+})
+ 

R-package/R/setinfo.xgb.DMatrix.R

@@ -0,0 +1,29 @@
+#' Set information of an xgb.DMatrix object
+#' 
+#' Set information of an xgb.DMatrix object
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' train <- agaricus.train
+#' dtrain <- xgb.DMatrix(train$data, label=train$label)
+#' labels <- getinfo(dtrain, 'label')
+#' setinfo(dtrain, 'label', 1-labels)
+#' labels2 <- getinfo(dtrain, 'label')
+#' stopifnot(all(labels2 == 1-labels))
+#' @rdname setinfo
+#' @export
+#' 
+setinfo <- function(object, ...){
+  UseMethod("setinfo")
+}
+
+#' @param object Object of class "xgb.DMatrix"
+#' @param name the name of the field to get
+#' @param info the specific field of information to set
+#' @param ... other parameters
+#' @rdname setinfo
+#' @method setinfo xgb.DMatrix
+setMethod("setinfo", signature = "xgb.DMatrix", 
+          definition = function(object, name, info) {
+            xgb.setinfo(object, name, info)
+          })

R-package/R/slice.xgb.DMatrix.R

@@ -0,0 +1,33 @@
+setClass('xgb.DMatrix')
+
+#' Get a new DMatrix containing the specified rows of
+#' orginal xgb.DMatrix object
+#'
+#' Get a new DMatrix containing the specified rows of
+#' orginal xgb.DMatrix object
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' train <- agaricus.train
+#' dtrain <- xgb.DMatrix(train$data, label=train$label)
+#' dsub <- slice(dtrain, 1:3)
+#' @rdname slice
+#' @export
+#' 
+slice <- function(object, ...){
+    UseMethod("slice")
+}
+
+#' @param object Object of class "xgb.DMatrix"
+#' @param idxset a integer vector of indices of rows needed
+#' @param ... other parameters
+#' @rdname slice
+#' @method slice xgb.DMatrix
+setMethod("slice", signature = "xgb.DMatrix", 
+          definition = function(object, idxset, ...) {
+              if (class(object) != "xgb.DMatrix") {
+                  stop("slice: first argument dtrain must be xgb.DMatrix")
+              }
+              ret <- .Call("XGDMatrixSliceDMatrix_R", object, idxset, PACKAGE = "xgboost")
+              return(structure(ret, class = "xgb.DMatrix"))
+          })

R-package/R/utils.R

@@ -0,0 +1,214 @@
+#' @importClassesFrom Matrix dgCMatrix dgeMatrix
+#' @import methods
+
+# depends on matrix
+.onLoad <- function(libname, pkgname) {
+  library.dynam("xgboost", pkgname, libname)
+}
+.onUnload <- function(libpath) {
+  library.dynam.unload("xgboost", libpath)
+}
+
+# set information into dmatrix, this mutate dmatrix
+xgb.setinfo <- function(dmat, name, info) {
+  if (class(dmat) != "xgb.DMatrix") {
+    stop("xgb.setinfo: first argument dtrain must be xgb.DMatrix")
+  }
+  if (name == "label") {
+    .Call("XGDMatrixSetInfo_R", dmat, name, as.numeric(info), 
+          PACKAGE = "xgboost")
+    return(TRUE)
+  }
+  if (name == "weight") {
+    .Call("XGDMatrixSetInfo_R", dmat, name, as.numeric(info), 
+          PACKAGE = "xgboost")
+    return(TRUE)
+  }
+  if (name == "base_margin") {
+    .Call("XGDMatrixSetInfo_R", dmat, name, as.numeric(info), 
+          PACKAGE = "xgboost")
+    return(TRUE)
+  }
+  if (name == "group") {
+    .Call("XGDMatrixSetInfo_R", dmat, name, as.integer(info), 
+          PACKAGE = "xgboost")
+    return(TRUE)
+  }
+  stop(paste("xgb.setinfo: unknown info name", name))
+  return(FALSE)
+}
+
+# construct a Booster from cachelist
+xgb.Booster <- function(params = list(), cachelist = list(), modelfile = NULL) {
+  if (typeof(cachelist) != "list") {
+    stop("xgb.Booster: only accepts list of DMatrix as cachelist")
+  }
+  for (dm in cachelist) {
+    if (class(dm) != "xgb.DMatrix") {
+      stop("xgb.Booster: only accepts list of DMatrix as cachelist")
+    }
+  }
+  handle <- .Call("XGBoosterCreate_R", cachelist, PACKAGE = "xgboost")
+  if (length(params) != 0) {
+    for (i in 1:length(params)) {
+      p <- params[i]
+      .Call("XGBoosterSetParam_R", handle, gsub("\\.", "_", names(p)), as.character(p),
+            PACKAGE = "xgboost")
+    }
+  }
+  if (!is.null(modelfile)) {
+    if (typeof(modelfile) != "character") {
+      stop("xgb.Booster: modelfile must be character")
+    }
+    .Call("XGBoosterLoadModel_R", handle, modelfile, PACKAGE = "xgboost")
+  }
+  return(structure(handle, class = "xgb.Booster"))
+}
+
+## ----the following are low level iteratively function, not needed if
+## you do not want to use them ---------------------------------------
+# get dmatrix from data, label
+xgb.get.DMatrix <- function(data, label = NULL) {
+  inClass <- class(data)
+  if (inClass == "dgCMatrix" || inClass == "matrix") {
+    if (is.null(label)) {
+      stop("xgboost: need label when data is a matrix")
+    }
+    dtrain <- xgb.DMatrix(data, label = label)
+  } else {
+    if (!is.null(label)) {
+      warning("xgboost: label will be ignored.")
+    }
+    if (inClass == "character") {
+      dtrain <- xgb.DMatrix(data)
+    } else if (inClass == "xgb.DMatrix") {
+      dtrain <- data
+    } else {
+      stop("xgboost: Invalid input of data")
+    }
+  }
+  return (dtrain)
+}
+xgb.numrow <- function(dmat) {
+  nrow <- .Call("XGDMatrixNumRow_R", dmat, PACKAGE="xgboost")
+  return(nrow)
+}
+# iteratively update booster with customized statistics
+xgb.iter.boost <- function(booster, dtrain, gpair) {
+  if (class(booster) != "xgb.Booster") {
+    stop("xgb.iter.update: first argument must be type xgb.Booster")
+  }
+  if (class(dtrain) != "xgb.DMatrix") {
+    stop("xgb.iter.update: second argument must be type xgb.DMatrix")
+  }
+  .Call("XGBoosterBoostOneIter_R", booster, dtrain, gpair$grad, gpair$hess, 
+        PACKAGE = "xgboost")
+  return(TRUE)
+}
+
+# iteratively update booster with dtrain
+xgb.iter.update <- function(booster, dtrain, iter, obj = NULL) {
+  if (class(booster) != "xgb.Booster") {
+    stop("xgb.iter.update: first argument must be type xgb.Booster")
+  }
+  if (class(dtrain) != "xgb.DMatrix") {
+    stop("xgb.iter.update: second argument must be type xgb.DMatrix")
+  }
+
+  if (is.null(obj)) {
+    .Call("XGBoosterUpdateOneIter_R", booster, as.integer(iter), dtrain, 
+          PACKAGE = "xgboost")
+  } else {
+    pred <- predict(booster, dtrain)
+    gpair <- obj(pred, dtrain)
+    succ <- xgb.iter.boost(booster, dtrain, gpair)
+  }
+  return(TRUE)
+}
+
+# iteratively evaluate one iteration
+xgb.iter.eval <- function(booster, watchlist, iter, feval = NULL) {
+  if (class(booster) != "xgb.Booster") {
+    stop("xgb.eval: first argument must be type xgb.Booster")
+  }
+  if (typeof(watchlist) != "list") {
+    stop("xgb.eval: only accepts list of DMatrix as watchlist")
+  }
+  for (w in watchlist) {
+    if (class(w) != "xgb.DMatrix") {
+      stop("xgb.eval: watch list can only contain xgb.DMatrix")
+    }
+  }
+  if (length(watchlist) != 0) {
+    if (is.null(feval)) {
+      evnames <- list()
+      for (i in 1:length(watchlist)) {
+        w <- watchlist[i]
+        if (length(names(w)) == 0) {
+          stop("xgb.eval: name tag must be presented for every elements in watchlist")
+        }
+        evnames <- append(evnames, names(w))
+      }
+      msg <- .Call("XGBoosterEvalOneIter_R", booster, as.integer(iter), watchlist, 
+                   evnames, PACKAGE = "xgboost")
+    } else {
+      msg <- paste("[", iter, "]", sep="")
+      for (j in 1:length(watchlist)) {
+        w <- watchlist[j]
+        if (length(names(w)) == 0) {
+          stop("xgb.eval: name tag must be presented for every elements in watchlist")
+        }
+        ret <- feval(predict(booster, w[[1]]), w[[1]])
+        msg <- paste(msg, "\t", names(w), "-", ret$metric, ":", ret$value, sep="")
+      }
+    }
+  } else {
+    msg <- ""
+  }
+  return(msg)
+} 
+#------------------------------------------
+# helper functions for cross validation
+#
+xgb.cv.mknfold <- function(dall, nfold, param) {
+  randidx <- sample(1 : xgb.numrow(dall))
+  kstep <- length(randidx) / nfold
+  idset <- list()
+  for (i in 1:nfold) {
+    idset[[i]] <- randidx[ ((i-1) * kstep + 1) : min(i * kstep, length(randidx)) ]
+  }
+  ret <- list()
+  for (k in 1:nfold) {
+    dtest <- slice(dall, idset[[k]])
+    didx = c()
+    for (i in 1:nfold) {
+      if (i != k) {
+        didx <- append(didx, idset[[i]])
+      }
+    }
+    dtrain <- slice(dall, didx)
+    bst <- xgb.Booster(param, list(dtrain, dtest))
+    watchlist = list(train=dtrain, test=dtest)
+    ret[[k]] <- list(dtrain=dtrain, booster=bst, watchlist=watchlist)
+  }
+  return (ret)
+}
+xgb.cv.aggcv <- function(res, showsd = TRUE) {
+  header <- res[[1]]
+  ret <- header[1]
+  for (i in 2:length(header)) {
+    kv <- strsplit(header[i], ":")[[1]]
+    ret <- paste(ret, "\t", kv[1], ":", sep="")
+    stats <- c()
+    stats[1] <- as.numeric(kv[2])    
+    for (j in 2:length(res)) {
+      tkv <- strsplit(res[[j]][i], ":")[[1]]
+      stats[j] <- as.numeric(tkv[2])
+    }
+    ret <- paste(ret, sprintf("%f", mean(stats)), sep="")
+    if (showsd) {
+      ret <- paste(ret, sprintf("+%f", sd(stats)), sep="")
+    }
+  }
+  return (ret)
+}

R-package/R/xgb.DMatrix.R

@@ -0,0 +1,45 @@
+#' Contruct xgb.DMatrix object
+#' 
+#' Contruct xgb.DMatrix object from dense matrix, sparse matrix or local file.
+#' 
+#' @param data a \code{matrix} object, a \code{dgCMatrix} object or a character 
+#'   indicating the data file.
+#' @param info a list of information of the xgb.DMatrix object
+#' @param missing Missing is only used when input is dense matrix, pick a float
+#     value that represents missing value. Sometime a data use 0 or other extreme value to represents missing values.
+#
+#' @param ... other information to pass to \code{info}.
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' train <- agaricus.train
+#' dtrain <- xgb.DMatrix(train$data, label=train$label)
+#' xgb.DMatrix.save(dtrain, 'xgb.DMatrix.data')
+#' dtrain <- xgb.DMatrix('xgb.DMatrix.data')
+#' @export
+#' 
+xgb.DMatrix <- function(data, info = list(), missing = 0, ...) {
+  if (typeof(data) == "character") {
+    handle <- .Call("XGDMatrixCreateFromFile_R", data, as.integer(FALSE), 
+                    PACKAGE = "xgboost")
+  } else if (is.matrix(data)) {
+    handle <- .Call("XGDMatrixCreateFromMat_R", data, missing, 
+                    PACKAGE = "xgboost")
+  } else if (class(data) == "dgCMatrix") {
+    handle <- .Call("XGDMatrixCreateFromCSC_R", data@p, data@i, data@x, 
+                    PACKAGE = "xgboost")
+  } else {
+    stop(paste("xgb.DMatrix: does not support to construct from ", 
+               typeof(data)))
+  }
+  dmat <- structure(handle, class = "xgb.DMatrix")
+  
+  info <- append(info, list(...))
+  if (length(info) == 0) 
+    return(dmat)
+  for (i in 1:length(info)) {
+    p <- info[i]
+    xgb.setinfo(dmat, names(p), p[[1]])
+  }
+  return(dmat)
+} 

R-package/R/xgb.DMatrix.save.R

@@ -0,0 +1,27 @@
+#' Save xgb.DMatrix object to binary file
+#' 
+#' Save xgb.DMatrix object to binary file
+#' 
+#' @param DMatrix the DMatrix object
+#' @param fname the name of the binary file.
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' train <- agaricus.train
+#' dtrain <- xgb.DMatrix(train$data, label=train$label)
+#' xgb.DMatrix.save(dtrain, 'xgb.DMatrix.data')
+#' dtrain <- xgb.DMatrix('xgb.DMatrix.data')
+#' @export
+#' 
+xgb.DMatrix.save <- function(DMatrix, fname) {
+  if (typeof(fname) != "character") {
+    stop("xgb.save: fname must be character")
+  }
+  if (class(DMatrix) == "xgb.DMatrix") {
+    .Call("XGDMatrixSaveBinary_R", DMatrix, fname, as.integer(FALSE), 
+          PACKAGE = "xgboost")
+    return(TRUE)
+  }
+  stop("xgb.DMatrix.save: the input must be xgb.DMatrix")
+  return(FALSE)
+} 

R-package/R/xgb.cv.R

@@ -0,0 +1,86 @@
+#' Cross Validation
+#' 
+#' The cross valudation function of xgboost
+#'
+#' @param params the list of parameters. Commonly used ones are:
+#' \itemize{
+#'   \item \code{objective} objective function, common ones are
+#'   \itemize{
+#'     \item \code{reg:linear} linear regression
+#'     \item \code{binary:logistic} logistic regression for classification
+#'   }
+#'   \item \code{eta} step size of each boosting step
+#'   \item \code{max.depth} maximum depth of the tree
+#'   \item \code{nthread} number of thread used in training, if not set, all threads are used
+#' }
+#'
+#'   See \url{https://github.com/tqchen/xgboost/wiki/Parameters} for 
+#'   further details. See also demo/ for walkthrough example in R.
+#' @param data takes an \code{xgb.DMatrix} as the input.
+#' @param nrounds the max number of iterations
+#' @param nfold number of folds used
+#' @param label option field, when data is Matrix
+#' @param showsd boolean, whether show standard deviation of cross validation
+#' @param metrics, list of evaluation metrics to be used in corss validation,
+#'   when it is not specified, the evaluation metric is chosen according to objective function.
+#'   Possible options are:
+#' \itemize{
+#'   \item \code{error} binary classification error rate
+#'   \item \code{rmse} Rooted mean square error
+#'   \item \code{logloss} negative log-likelihood function
+#'   \item \code{auc} Area under curve
+#'   \item \code{merror} Exact matching error, used to evaluate multi-class classification
+#' }
+#' @param obj customized objective function. Returns gradient and second order 
+#'   gradient with given prediction and dtrain, 
+#' @param feval custimized evaluation function. Returns 
+#'   \code{list(metric='metric-name', value='metric-value')} with given 
+#'   prediction and dtrain,
+#' @param ... other parameters to pass to \code{params}.
+#' 
+#' @details 
+#' This is the cross validation function for xgboost
+#'
+#' Parallelization is automatically enabled if OpenMP is present.
+#' Number of threads can also be manually specified via "nthread" parameter.
+#' 
+#' This function only accepts an \code{xgb.DMatrix} object as the input.
+#'
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+#' history <- xgb.cv(data = dtrain, nround=3, nfold = 5, metrics=list("rmse","auc"),
+#'                   "max.depth"=3, "eta"=1, "objective"="binary:logistic")
+#' @export
+#'
+xgb.cv <- function(params=list(), data, nrounds, nfold, label = NULL,
+                   showsd = TRUE, metrics=list(), obj = NULL, feval = NULL, ...) {
+  if (typeof(params) != "list") {
+    stop("xgb.cv: first argument params must be list")
+  }
+  if (nfold <= 1) {
+    stop("nfold must be bigger than 1")
+  }
+  dtrain <- xgb.get.DMatrix(data, label)
+  params <- append(params, list(...))
+  params <- append(params, list(silent=1))
+  for (mc in metrics) {
+    params <- append(params, list("eval_metric"=mc))
+  }
+
+  folds <- xgb.cv.mknfold(dtrain, nfold, params)
+  history <- list()
+  for (i in 1:nrounds) {
+    msg <- list()
+    for (k in 1:nfold) {
+      fd <- folds[[k]]
+      succ <- xgb.iter.update(fd$booster, fd$dtrain, i - 1, obj)      
+      msg[[k]] <- strsplit(xgb.iter.eval(fd$booster, fd$watchlist, i - 1, feval), 
+                           "\t")[[1]]
+    }
+    ret <- xgb.cv.aggcv(msg, showsd)
+    history <- append(history, ret)
+    cat(paste(ret, "\n", sep=""))
+  }
+  return (TRUE)
+}

R-package/R/xgb.dump.R

@@ -0,0 +1,33 @@
+#' Save xgboost model to text file
+#' 
+#' Save a xgboost model to text file. Could be parsed later.
+#' 
+#' @param model the model object.
+#' @param fname the name of the binary file.
+#' @param fmap feature map file representing the type of feature. 
+#'        Detailed description could be found at 
+#'        \url{https://github.com/tqchen/xgboost/wiki/Binary-Classification#dump-model}.
+#'        See demo/ for walkthrough example in R, and
+#'        \url{https://github.com/tqchen/xgboost/blob/master/demo/data/featmap.txt} 
+#'        for example Format.
+#'
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' data(agaricus.test, package='xgboost')
+#' train <- agaricus.train
+#' test <- agaricus.test
+#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
+#'                eta = 1, nround = 2,objective = "binary:logistic")
+#' xgb.dump(bst, 'xgb.model.dump')
+#' @export
+#' 
+xgb.dump <- function(model, fname, fmap = "") {
+  if (class(model) != "xgb.Booster") {
+    stop("xgb.dump: first argument must be type xgb.Booster")
+  }
+  if (typeof(fname) != "character") {
+    stop("xgb.dump: second argument must be type character")
+  }
+  .Call("XGBoosterDumpModel_R", model, fname, fmap, PACKAGE = "xgboost")
+  return(TRUE)
+} 

R-package/R/xgb.load.R

@@ -0,0 +1,23 @@
+#' Load xgboost model from binary file
+#' 
+#' Load xgboost model from the binary model file
+#' 
+#' @param modelfile the name of the binary file.
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' data(agaricus.test, package='xgboost')
+#' train <- agaricus.train
+#' test <- agaricus.test
+#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
+#'                eta = 1, nround = 2,objective = "binary:logistic")
+#' xgb.save(bst, 'xgb.model')
+#' bst <- xgb.load('xgb.model')
+#' pred <- predict(bst, test$data)
+#' @export
+#' 
+xgb.load <- function(modelfile) {
+  if (is.null(modelfile)) 
+    stop("xgb.load: modelfile cannot be NULL")
+  xgb.Booster(modelfile = modelfile)
+} 

R-package/R/xgb.save.R

@@ -0,0 +1,31 @@
+#' Save xgboost model to binary file
+#' 
+#' Save xgboost model from xgboost or xgb.train
+#' 
+#' @param model the model object.
+#' @param fname the name of the binary file.
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' data(agaricus.test, package='xgboost')
+#' train <- agaricus.train
+#' test <- agaricus.test
+#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
+#'                eta = 1, nround = 2,objective = "binary:logistic")
+#' xgb.save(bst, 'xgb.model')
+#' bst <- xgb.load('xgb.model')
+#' pred <- predict(bst, test$data)
+#' @export
+#' 
+xgb.save <- function(model, fname) {
+  if (typeof(fname) != "character") {
+    stop("xgb.save: fname must be character")
+  }
+  if (class(model) == "xgb.Booster") {
+    .Call("XGBoosterSaveModel_R", model, fname, PACKAGE = "xgboost")
+    return(TRUE)
+  }
+  stop("xgb.save: the input must be xgb.Booster. Use xgb.DMatrix.save to save
+       xgb.DMatrix object.")
+  return(FALSE)
+} 

R-package/R/xgb.train.R

@@ -0,0 +1,98 @@
+#' eXtreme Gradient Boosting Training
+#' 
+#' The training function of xgboost
+#'
+#' @param params the list of parameters. Commonly used ones are:
+#' \itemize{
+#'   \item \code{objective} objective function, common ones are
+#'   \itemize{
+#'     \item \code{reg:linear} linear regression
+#'     \item \code{binary:logistic} logistic regression for classification
+#'   }
+#'   \item \code{eta} step size of each boosting step
+#'   \item \code{max.depth} maximum depth of the tree
+#'   \item \code{nthread} number of thread used in training, if not set, all threads are used
+#' }
+#'
+#'   See \url{https://github.com/tqchen/xgboost/wiki/Parameters} for 
+#'   further details. See also demo/ for walkthrough example in R.
+#' @param data takes an \code{xgb.DMatrix} as the input.
+#' @param nrounds the max number of iterations
+#' @param watchlist what information should be printed when \code{verbose=1} or
+#'   \code{verbose=2}. Watchlist is used to specify validation set monitoring
+#'   during training. For example user can specify
+#'    watchlist=list(validation1=mat1, validation2=mat2) to watch
+#'    the performance of each round's model on mat1 and mat2
+#'
+#' @param obj customized objective function. Returns gradient and second order 
+#'   gradient with given prediction and dtrain, 
+#' @param feval custimized evaluation function. Returns 
+#'   \code{list(metric='metric-name', value='metric-value')} with given 
+#'   prediction and dtrain,
+#' @param verbose If 0, xgboost will stay silent. If 1, xgboost will print 
+#'   information of performance. If 2, xgboost will print information of both
+#'
+#' @param ... other parameters to pass to \code{params}.
+#' 
+#' @details 
+#' This is the training function for xgboost.
+#'
+#' Parallelization is automatically enabled if OpenMP is present.
+#' Number of threads can also be manually specified via "nthread" parameter.
+#' 
+#' This function only accepts an \code{xgb.DMatrix} object as the input.
+#' It supports advanced features such as watchlist, customized objective function,
+#' therefore it is more flexible than \code{\link{xgboost}}.
+#' 
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+#' dtest <- dtrain
+#' watchlist <- list(eval = dtest, train = dtrain)
+#' param <- list(max.depth = 2, eta = 1, silent = 1)
+#' logregobj <- function(preds, dtrain) {
+#'    labels <- getinfo(dtrain, "label")
+#'    preds <- 1/(1 + exp(-preds))
+#'    grad <- preds - labels
+#'    hess <- preds * (1 - preds)
+#'    return(list(grad = grad, hess = hess))
+#' }
+#' evalerror <- function(preds, dtrain) {
+#'   labels <- getinfo(dtrain, "label")
+#'   err <- as.numeric(sum(labels != (preds > 0)))/length(labels)
+#'   return(list(metric = "error", value = err))
+#' }
+#' bst <- xgb.train(param, dtrain, nround = 2, watchlist, logregobj, evalerror)
+#' @export
+#' 
+xgb.train <- function(params=list(), data, nrounds, watchlist = list(), 
+                      obj = NULL, feval = NULL, verbose = 1, ...) {
+  dtrain <- data
+  if (typeof(params) != "list") {
+    stop("xgb.train: first argument params must be list")
+  }
+  if (class(dtrain) != "xgb.DMatrix") {
+    stop("xgb.train: second argument dtrain must be xgb.DMatrix")
+  }
+  if (verbose > 1) {
+    params <- append(params, list(silent = 0))
+  } else {
+    params <- append(params, list(silent = 1))
+  }
+  if (length(watchlist) != 0 && verbose == 0) {
+    warning('watchlist is provided but verbose=0, no evaluation information will be printed')
+    watchlist <- list()
+  }
+  params = append(params, list(...))
+  
+  bst <- xgb.Booster(params, append(watchlist, dtrain))
+  for (i in 1:nrounds) {
+    succ <- xgb.iter.update(bst, dtrain, i - 1, obj)
+    if (length(watchlist) != 0) {
+      msg <- xgb.iter.eval(bst, watchlist, i - 1, feval)
+      cat(paste(msg, "\n", sep=""))
+    }
+  }
+  return(bst)
+} 

R-package/R/xgboost.R

@@ -0,0 +1,115 @@
+#' eXtreme Gradient Boosting (Tree) library
+#' 
+#' A simple interface for xgboost in R
+#' 
+#' @param data takes \code{matrix}, \code{dgCMatrix}, local data file or 
+#'   \code{xgb.DMatrix}. 
+#' @param label the response variable. User should not set this field,
+#    if data is local data file or  \code{xgb.DMatrix}. 
+#' @param params the list of parameters. Commonly used ones are:
+#' \itemize{
+#'   \item \code{objective} objective function, common ones are
+#'   \itemize{
+#'     \item \code{reg:linear} linear regression
+#'     \item \code{binary:logistic} logistic regression for classification
+#'   }
+#'   \item \code{eta} step size of each boosting step
+#'   \item \code{max.depth} maximum depth of the tree
+#'   \item \code{nthread} number of thread used in training, if not set, all threads are used
+#' }
+#'
+#'   See \url{https://github.com/tqchen/xgboost/wiki/Parameters} for 
+#'   further details. See also demo/ for walkthrough example in R.
+#' @param nrounds the max number of iterations
+#' @param verbose If 0, xgboost will stay silent. If 1, xgboost will print 
+#'   information of performance. If 2, xgboost will print information of both
+#'   performance and construction progress information
+#' @param ... other parameters to pass to \code{params}.
+#' 
+#' @details 
+#' This is the modeling function for xgboost.
+#' 
+#' Parallelization is automatically enabled if OpenMP is present.
+#' Number of threads can also be manually specified via "nthread" parameter
+#' 
+#' @examples
+#' data(agaricus.train, package='xgboost')
+#' data(agaricus.test, package='xgboost')
+#' train <- agaricus.train
+#' test <- agaricus.test
+#' bst <- xgboost(data = train$data, label = train$label, max.depth = 2, 
+#'                eta = 1, nround = 2,objective = "binary:logistic")
+#' pred <- predict(bst, test$data)
+#' 
+#' @export
+#' 
+xgboost <- function(data = NULL, label = NULL, params = list(), nrounds, 
+                    verbose = 1, ...) {
+  dtrain <- xgb.get.DMatrix(data, label)  
+  params <- append(params, list(...))
+  
+  if (verbose > 0) {
+    watchlist <- list(train = dtrain)
+  } else {
+    watchlist <- list()
+  }
+  
+  bst <- xgb.train(params, dtrain, nrounds, watchlist, verbose=verbose)
+  
+  return(bst)
+} 
+
+
+#' Training part from Mushroom Data Set
+#' 
+#' This data set is originally from the Mushroom data set,
+#' UCI Machine Learning Repository.
+#' 
+#' This data set includes the following fields:
+#' 
+#' \itemize{
+#'  \item \code{label} the label for each record
+#'  \item \code{data} a sparse Matrix of \code{dgCMatrix} class, with 127 columns.
+#' }
+#'
+#' @references
+#' https://archive.ics.uci.edu/ml/datasets/Mushroom
+#' 
+#' Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository 
+#' [http://archive.ics.uci.edu/ml]. Irvine, CA: University of California, 
+#' School of Information and Computer Science.
+#' 
+#' @docType data
+#' @keywords datasets
+#' @name agaricus.train
+#' @usage data(agaricus.train)
+#' @format A list containing a label vector, and a dgCMatrix object with 6513 
+#' rows and 127 variables
+NULL
+
+#' Test part from Mushroom Data Set
+#'
+#' This data set is originally from the Mushroom data set,
+#' UCI Machine Learning Repository.
+#' 
+#' This data set includes the following fields:
+#' 
+#' \itemize{
+#'  \item \code{label} the label for each record
+#'  \item \code{data} a sparse Matrix of \code{dgCMatrix} class, with 127 columns.
+#' }
+#'
+#' @references
+#' https://archive.ics.uci.edu/ml/datasets/Mushroom
+#' 
+#' Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository 
+#' [http://archive.ics.uci.edu/ml]. Irvine, CA: University of California, 
+#' School of Information and Computer Science.
+#' 
+#' @docType data
+#' @keywords datasets
+#' @name agaricus.test
+#' @usage data(agaricus.test)
+#' @format A list containing a label vector, and a dgCMatrix object with 1611 
+#' rows and 127 variables
+NULL

R-package/README.md

@@ -0,0 +1,21 @@
+# R package for xgboost.
+
+## Installation
+
+For up-to-date version(which is recommended), please install from github. Windows user will need to install [RTools](http://cran.r-project.org/bin/windows/Rtools/) first.
+
+```r
+require(devtools)
+install_github('xgboost','tqchen',subdir='R-package')
+```
+
+For stable version on CRAN, please run
+
+```r
+install.packages('xgboost')
+```
+
+## Examples
+
+* Please visit [walk through example](https://github.com/tqchen/xgboost/blob/master/R-package/demo).
+* See also the [example scripts](https://github.com/tqchen/xgboost/tree/master/demo/kaggle-higgs) for Kaggle Higgs Challenge, including [speedtest script](https://github.com/tqchen/xgboost/blob/master/demo/kaggle-higgs/speedtest.R) on this dataset.

R-package/demo/00Index

@@ -0,0 +1,6 @@
+basic_walkthrough               Basic feature walkthrough
+custom_objective                Cutomize loss function, and evaluation metric
+boost_from_prediction           Boosting from existing prediction
+predict_first_ntree             Predicting using first n trees
+generalized_linear_model        Generalized Linear Model
+cross_validation                Cross validation

R-package/demo/README.md

@@ -0,0 +1,17 @@
+XGBoost R Feature Walkthrough
+====
+* [Basic walkthrough of wrappers](basic_walkthrough.R) 
+* [Cutomize loss function, and evaluation metric](custom_objective.R)
+* [Boosting from existing prediction](boost_from_prediction.R)
+* [Predicting using first n trees](predict_first_ntree.R)
+* [Generalized Linear Model](generalized_linear_model.R)
+* [Cross validation](cross_validation.R)
+
+Benchmarks
+====
+* [Starter script for Kaggle Higgs Boson](../../demo/kaggle-higgs)
+ 
+Notes
+====
+* Contribution of exampls, benchmarks is more than welcomed!
+* If you like to share how you use xgboost to solve your problem, send a pull request:)

R-package/demo/basic_walkthrough.R

@@ -0,0 +1,93 @@
+require(xgboost)
+require(methods)
+# we load in the agaricus dataset
+# In this example, we are aiming to predict whether a mushroom can be eated
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+train <- agaricus.train
+test <- agaricus.test
+# the loaded data is stored in sparseMatrix, and label is a numeric vector in {0,1}
+class(train$label)
+class(train$data)
+
+#-------------Basic Training using XGBoost-----------------
+# this is the basic usage of xgboost you can put matrix in data field
+# note: we are puting in sparse matrix here, xgboost naturally handles sparse input
+# use sparse matrix when your feature is sparse(e.g. when you using one-hot encoding vector)
+print("training xgboost with sparseMatrix")
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2, eta = 1, nround = 2,
+               objective = "binary:logistic")
+# alternatively, you can put in dense matrix, i.e. basic R-matrix
+print("training xgboost with Matrix")
+bst <- xgboost(data = as.matrix(train$data), label = train$label, max.depth = 2, eta = 1, nround = 2,
+               objective = "binary:logistic")
+
+# you can also put in xgb.DMatrix object, stores label, data and other meta datas needed for advanced features
+print("training xgboost with xgb.DMatrix")
+dtrain <- xgb.DMatrix(data = train$data, label = train$label)
+bst <- xgboost(data = dtrain, max.depth = 2, eta = 1, nround = 2, objective = "binary:logistic")
+
+# Verbose = 0,1,2
+print ('train xgboost with verbose 0, no message')
+bst <- xgboost(data = dtrain, max.depth = 2, eta = 1, nround = 2,
+               objective = "binary:logistic", verbose = 0)
+print ('train xgboost with verbose 1, print evaluation metric')
+bst <- xgboost(data = dtrain, max.depth = 2, eta = 1, nround = 2,
+               objective = "binary:logistic", verbose = 1)
+print ('train xgboost with verbose 2, also print information about tree')
+bst <- xgboost(data = dtrain, max.depth = 2, eta = 1, nround = 2,
+               objective = "binary:logistic", verbose = 2)
+
+# you can also specify data as file path to a LibSVM format input
+# since we do not have this file with us, the following line is just for illustration
+# bst <- xgboost(data = 'agaricus.train.svm', max.depth = 2, eta = 1, nround = 2,objective = "binary:logistic")
+
+#--------------------basic prediction using xgboost--------------
+# you can do prediction using the following line
+# you can put in Matrix, sparseMatrix, or xgb.DMatrix 
+pred <- predict(bst, test$data)
+err <- mean(as.numeric(pred > 0.5) != test$label)
+print(paste("test-error=", err))
+
+#-------------------save and load models-------------------------
+# save model to binary local file
+xgb.save(bst, "xgboost.model")
+# load binary model to R
+bst2 <- xgb.load("xgboost.model")
+pred2 <- predict(bst2, test$data)
+# pred2 should be identical to pred
+print(paste("sum(abs(pred2-pred))=", sum(abs(pred2-pred))))
+
+#----------------Advanced features --------------
+# to use advanced features, we need to put data in xgb.DMatrix
+dtrain <- xgb.DMatrix(data = train$data, label=train$label)
+dtest <- xgb.DMatrix(data = test$data, label=test$label)
+#---------------Using watchlist----------------
+# watchlist is a list of xgb.DMatrix, each of them tagged with name
+watchlist <- list(train=dtrain, test=dtest)
+# to train with watchlist, use xgb.train, which contains more advanced features
+# watchlist allows us to monitor the evaluation result on all data in the list 
+print ('train xgboost using xgb.train with watchlist')
+bst <- xgb.train(data=dtrain, max.depth=2, eta=1, nround=2, watchlist=watchlist,
+                 objective = "binary:logistic")
+# we can change evaluation metrics, or use multiple evaluation metrics
+print ('train xgboost using xgb.train with watchlist, watch logloss and error')
+bst <- xgb.train(data=dtrain, max.depth=2, eta=1, nround=2, watchlist=watchlist,
+                 eval.metric = "error", eval.metric = "logloss",
+                 objective = "binary:logistic")
+
+# xgb.DMatrix can also be saved using xgb.DMatrix.save
+xgb.DMatrix.save(dtrain, "dtrain.buffer")
+# to load it in, simply call xgb.DMatrix
+dtrain2 <- xgb.DMatrix("dtrain.buffer")
+bst <- xgb.train(data=dtrain2, max.depth=2, eta=1, nround=2, watchlist=watchlist,
+                 objective = "binary:logistic")
+# information can be extracted from xgb.DMatrix using getinfo
+label = getinfo(dtest, "label")
+pred <- predict(bst, dtest)
+err <- as.numeric(sum(as.integer(pred > 0.5) != label))/length(label)
+print(paste("test-error=", err))
+
+# Finally, you can dump the tree you learned using xgb.dump into a text file
+xgb.dump(bst, "dump.raw.txt")
+

R-package/demo/boost_from_prediction.R

@@ -0,0 +1,26 @@
+require(xgboost)
+# load in the agaricus dataset
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label)
+
+watchlist <- list(eval = dtest, train = dtrain)
+###
+# advanced: start from a initial base prediction
+#
+print('start running example to start from a initial prediction')
+# train xgboost for 1 round
+param <- list(max.depth=2,eta=1,silent=1,objective='binary:logistic')
+bst <- xgb.train( param, dtrain, 1, watchlist )
+# Note: we need the margin value instead of transformed prediction in set_base_margin
+# do predict with output_margin=TRUE, will always give you margin values before logistic transformation
+ptrain <- predict(bst, dtrain, outputmargin=TRUE)
+ptest  <- predict(bst, dtest, outputmargin=TRUE)
+# set the base_margin property of dtrain and dtest
+# base margin is the base prediction we will boost from
+setinfo(dtrain, "base_margin", ptrain)
+setinfo(dtest, "base_margin", ptest)
+
+print('this is result of boost from initial prediction')
+bst <- xgb.train( param, dtrain, 1, watchlist )

R-package/demo/cross_validation.R

@@ -0,0 +1,47 @@
+require(xgboost)
+# load in the agaricus dataset
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label)
+
+nround <- 2
+param <- list(max.depth=2,eta=1,silent=1,objective='binary:logistic')
+
+cat('running cross validation\n')
+# do cross validation, this will print result out as
+# [iteration]  metric_name:mean_value+std_value
+# std_value is standard deviation of the metric
+xgb.cv(param, dtrain, nround, nfold=5, metrics={'error'})
+
+cat('running cross validation, disable standard deviation display\n')
+# do cross validation, this will print result out as
+# [iteration]  metric_name:mean_value+std_value
+# std_value is standard deviation of the metric
+xgb.cv(param, dtrain, nround, nfold=5,
+       metrics={'error'}, , showsd = FALSE)
+
+###
+# you can also do cross validation with cutomized loss function
+# See custom_objective.R
+##
+print ('running cross validation, with cutomsized loss function')
+
+logregobj <- function(preds, dtrain) {
+  labels <- getinfo(dtrain, "label")
+  preds <- 1/(1 + exp(-preds))
+  grad <- preds - labels
+  hess <- preds * (1 - preds)
+  return(list(grad = grad, hess = hess))
+}
+evalerror <- function(preds, dtrain) {
+  labels <- getinfo(dtrain, "label")
+  err <- as.numeric(sum(labels != (preds > 0)))/length(labels)
+  return(list(metric = "error", value = err))
+}
+
+param <- list(max.depth=2,eta=1,silent=1)
+# train with customized objective
+xgb.cv(param, dtrain, nround, nfold = 5,
+       obj = logregobj, feval=evalerror)
+

R-package/demo/custom_objective.R

@@ -0,0 +1,39 @@
+require(xgboost)
+# load in the agaricus dataset
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label)
+
+# note: for customized objective function, we leave objective as default
+# note: what we are getting is margin value in prediction
+# you must know what you are doing
+param <- list(max.depth=2,eta=1,silent=1)
+watchlist <- list(eval = dtest, train = dtrain)
+num_round <- 2
+
+# user define objective function, given prediction, return gradient and second order gradient
+# this is loglikelihood loss
+logregobj <- function(preds, dtrain) {
+  labels <- getinfo(dtrain, "label")
+  preds <- 1/(1 + exp(-preds))
+  grad <- preds - labels
+  hess <- preds * (1 - preds)
+  return(list(grad = grad, hess = hess))
+}
+
+# user defined evaluation function, return a pair metric_name, result
+# NOTE: when you do customized loss function, the default prediction value is margin
+# this may make buildin evalution metric not function properly
+# for example, we are doing logistic loss, the prediction is score before logistic transformation
+# the buildin evaluation error assumes input is after logistic transformation
+# Take this in mind when you use the customization, and maybe you need write customized evaluation function
+evalerror <- function(preds, dtrain) {
+  labels <- getinfo(dtrain, "label")
+  err <- as.numeric(sum(labels != (preds > 0)))/length(labels)
+  return(list(metric = "error", value = err))
+}
+print ('start training with user customized objective')
+# training with customized objective, we can also do step by step training
+# simply look at xgboost.py's implementation of train
+bst <- xgb.train(param, dtrain, num_round, watchlist, logregobj, evalerror)

R-package/demo/generalized_linear_model.R

@@ -0,0 +1,34 @@
+require(xgboost)
+# load in the agaricus dataset
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label)
+##
+#  this script demonstrate how to fit generalized linear model in xgboost
+#  basically, we are using linear model, instead of tree for our boosters
+#  you can fit a linear regression, or logistic regression model
+##
+
+# change booster to gblinear, so that we are fitting a linear model
+# alpha is the L1 regularizer 
+# lambda is the L2 regularizer
+# you can also set lambda_bias which is L2 regularizer on the bias term
+param <- list(objective = "binary:logistic", booster = "gblinear",
+              alpha = 0.0001, lambda = 1)
+
+# normally, you do not need to set eta (step_size)
+# XGBoost uses a parallel coordinate descent algorithm (shotgun), 
+# there could be affection on convergence with parallelization on certain cases
+# setting eta to be smaller value, e.g 0.5 can make the optimization more stable
+
+##
+# the rest of settings are the same
+##
+watchlist <- list(eval = dtest, train = dtrain)
+num_round <- 2
+bst <- xgb.train(param, dtrain, num_round, watchlist)
+ypred <- predict(bst, dtest)
+labels <- getinfo(dtest, 'label')
+cat('error of preds=', mean(as.numeric(ypred>0.5)!=labels),'\n')
+

R-package/demo/predict_first_ntree.R

@@ -0,0 +1,23 @@
+require(xgboost)
+# load in the agaricus dataset
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label)
+
+param <- list(max.depth=2,eta=1,silent=1,objective='binary:logistic')
+watchlist <- list(eval = dtest, train = dtrain)
+nround = 2
+
+# training the model for two rounds
+bst = xgb.train(param, dtrain, nround, watchlist)
+cat('start testing prediction from first n trees\n')
+labels <- getinfo(dtest,'label')
+
+### predict using first 1 tree
+ypred1 = predict(bst, dtest, ntreelimit=1)
+# by default, we predict using all the trees
+ypred2 = predict(bst, dtest)
+
+cat('error of ypred1=', mean(as.numeric(ypred1>0.5)!=labels),'\n')
+cat('error of ypred2=', mean(as.numeric(ypred2>0.5)!=labels),'\n')

R-package/demo/runall.R

@@ -0,0 +1,8 @@
+# running all scripts in demo folder
+demo(basic_walkthrough)
+demo(custom_objective)
+demo(boost_from_prediction)
+demo(predict_first_ntree)
+demo(generalized_linear_model)
+demo(cross_validation)
+

R-package/man/agaricus.test.Rd

@@ -0,0 +1,31 @@
+% Generated by roxygen2 (4.0.1): do not edit by hand
+\docType{data}
+\name{agaricus.test}
+\alias{agaricus.test}
+\title{Test part from Mushroom Data Set}
+\format{A list containing a label vector, and a dgCMatrix object with 1611
+rows and 127 variables}
+\usage{
+data(agaricus.test)
+}
+\description{
+This data set is originally from the Mushroom data set,
+UCI Machine Learning Repository.
+}
+\details{
+This data set includes the following fields:
+
+\itemize{
+ \item \code{label} the label for each record
+ \item \code{data} a sparse Matrix of \code{dgCMatrix} class, with 127 columns.
+}
+}
+\references{
+https://archive.ics.uci.edu/ml/datasets/Mushroom
+
+Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository
+[http://archive.ics.uci.edu/ml]. Irvine, CA: University of California,
+School of Information and Computer Science.
+}
+\keyword{datasets}
+

R-package/man/agaricus.train.Rd

@@ -0,0 +1,31 @@
+% Generated by roxygen2 (4.0.1): do not edit by hand
+\docType{data}
+\name{agaricus.train}
+\alias{agaricus.train}
+\title{Training part from Mushroom Data Set}
+\format{A list containing a label vector, and a dgCMatrix object with 6513
+rows and 127 variables}
+\usage{
+data(agaricus.train)
+}
+\description{
+This data set is originally from the Mushroom data set,
+UCI Machine Learning Repository.
+}
+\details{
+This data set includes the following fields:
+
+\itemize{
+ \item \code{label} the label for each record
+ \item \code{data} a sparse Matrix of \code{dgCMatrix} class, with 127 columns.
+}
+}
+\references{
+https://archive.ics.uci.edu/ml/datasets/Mushroom
+
+Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository
+[http://archive.ics.uci.edu/ml]. Irvine, CA: University of California,
+School of Information and Computer Science.
+}
+\keyword{datasets}
+

R-package/man/getinfo.Rd

@@ -0,0 +1,31 @@
+% Generated by roxygen2 (4.0.1): do not edit by hand
+\docType{methods}
+\name{getinfo}
+\alias{getinfo}
+\alias{getinfo,xgb.DMatrix-method}
+\title{Get information of an xgb.DMatrix object}
+\usage{
+getinfo(object, ...)
+
+\S4method{getinfo}{xgb.DMatrix}(object, name)
+}
+\arguments{
+\item{object}{Object of class "xgb.DMatrix"}
+
+\item{name}{the name of the field to get}
+
+\item{...}{other parameters}
+}
+\description{
+Get information of an xgb.DMatrix object
+}
+\examples{
+data(agaricus.train, package='xgboost')
+train <- agaricus.train
+dtrain <- xgb.DMatrix(train$data, label=train$label)
+labels <- getinfo(dtrain, 'label')
+setinfo(dtrain, 'label', 1-labels)
+labels2 <- getinfo(dtrain, 'label')
+stopifnot(all(labels2 == 1-labels))
+}
+

R-package/man/predict-xgb.Booster-method.Rd

@@ -0,0 +1,37 @@
+% Generated by roxygen2 (4.0.1): do not edit by hand
+\docType{methods}
+\name{predict,xgb.Booster-method}
+\alias{predict,xgb.Booster-method}
+\title{Predict method for eXtreme Gradient Boosting model}
+\usage{
+\S4method{predict}{xgb.Booster}(object, newdata, outputmargin = FALSE,
+  ntreelimit = NULL)
+}
+\arguments{
+\item{object}{Object of class "xgb.Boost"}
+
+\item{newdata}{takes \code{matrix}, \code{dgCMatrix}, local data file or
+\code{xgb.DMatrix}.}
+
+\item{outputmargin}{whether the prediction should be shown in the original
+value of sum of functions, when outputmargin=TRUE, the prediction is
+untransformed margin value. In logistic regression, outputmargin=T will
+output value before logistic transformation.}
+
+\item{ntreelimit}{limit number of trees used in prediction, this parameter is
+only valid for gbtree, but not for gblinear. set it to be value bigger
+than 0. It will use all trees by default.}
+}
+\description{
+Predicted values based on xgboost model object.
+}
+\examples{
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+train <- agaricus.train
+test <- agaricus.test
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
+               eta = 1, nround = 2,objective = "binary:logistic")
+pred <- predict(bst, test$data)
+}
+

R-package/man/setinfo.Rd

@@ -0,0 +1,33 @@
+% Generated by roxygen2 (4.0.1): do not edit by hand
+\docType{methods}
+\name{setinfo}
+\alias{setinfo}
+\alias{setinfo,xgb.DMatrix-method}
+\title{Set information of an xgb.DMatrix object}
+\usage{
+setinfo(object, ...)
+
+\S4method{setinfo}{xgb.DMatrix}(object, name, info)
+}
+\arguments{
+\item{object}{Object of class "xgb.DMatrix"}
+
+\item{name}{the name of the field to get}
+
+\item{info}{the specific field of information to set}
+
+\item{...}{other parameters}
+}
+\description{
+Set information of an xgb.DMatrix object
+}
+\examples{
+data(agaricus.train, package='xgboost')
+train <- agaricus.train
+dtrain <- xgb.DMatrix(train$data, label=train$label)
+labels <- getinfo(dtrain, 'label')
+setinfo(dtrain, 'label', 1-labels)
+labels2 <- getinfo(dtrain, 'label')
+stopifnot(all(labels2 == 1-labels))
+}
+

R-package/man/slice.Rd

@@ -0,0 +1,30 @@
+% Generated by roxygen2 (4.0.1): do not edit by hand
+\docType{methods}
+\name{slice}
+\alias{slice}
+\alias{slice,xgb.DMatrix-method}
+\title{Get a new DMatrix containing the specified rows of
+orginal xgb.DMatrix object}
+\usage{
+slice(object, ...)
+
+\S4method{slice}{xgb.DMatrix}(object, idxset, ...)
+}
+\arguments{
+\item{object}{Object of class "xgb.DMatrix"}
+
+\item{idxset}{a integer vector of indices of rows needed}
+
+\item{...}{other parameters}
+}
+\description{
+Get a new DMatrix containing the specified rows of
+orginal xgb.DMatrix object
+}
+\examples{
+data(agaricus.train, package='xgboost')
+train <- agaricus.train
+dtrain <- xgb.DMatrix(train$data, label=train$label)
+dsub <- slice(dtrain, 1:3)
+}
+

R-package/man/xgb.DMatrix.Rd

@@ -0,0 +1,28 @@
+% Generated by roxygen2 (4.0.1): do not edit by hand
+\name{xgb.DMatrix}
+\alias{xgb.DMatrix}
+\title{Contruct xgb.DMatrix object}
+\usage{
+xgb.DMatrix(data, info = list(), missing = 0, ...)
+}
+\arguments{
+\item{data}{a \code{matrix} object, a \code{dgCMatrix} object or a character
+indicating the data file.}
+
+\item{info}{a list of information of the xgb.DMatrix object}
+
+\item{missing}{Missing is only used when input is dense matrix, pick a float}
+
+\item{...}{other information to pass to \code{info}.}
+}
+\description{
+Contruct xgb.DMatrix object from dense matrix, sparse matrix or local file.
+}
+\examples{
+data(agaricus.train, package='xgboost')
+train <- agaricus.train
+dtrain <- xgb.DMatrix(train$data, label=train$label)
+xgb.DMatrix.save(dtrain, 'xgb.DMatrix.data')
+dtrain <- xgb.DMatrix('xgb.DMatrix.data')
+}
+

R-package/man/xgb.DMatrix.save.Rd

@@ -0,0 +1,23 @@
+% Generated by roxygen2 (4.0.1): do not edit by hand
+\name{xgb.DMatrix.save}
+\alias{xgb.DMatrix.save}
+\title{Save xgb.DMatrix object to binary file}
+\usage{
+xgb.DMatrix.save(DMatrix, fname)
+}
+\arguments{
+\item{DMatrix}{the DMatrix object}
+
+\item{fname}{the name of the binary file.}
+}
+\description{
+Save xgb.DMatrix object to binary file
+}
+\examples{
+data(agaricus.train, package='xgboost')
+train <- agaricus.train
+dtrain <- xgb.DMatrix(train$data, label=train$label)
+xgb.DMatrix.save(dtrain, 'xgb.DMatrix.data')
+dtrain <- xgb.DMatrix('xgb.DMatrix.data')
+}
+

R-package/man/xgb.cv.Rd

@@ -0,0 +1,72 @@
+% Generated by roxygen2 (4.0.1): do not edit by hand
+\name{xgb.cv}
+\alias{xgb.cv}
+\title{Cross Validation}
+\usage{
+xgb.cv(params = list(), data, nrounds, nfold, label = NULL, showsd = TRUE,
+  metrics = list(), obj = NULL, feval = NULL, ...)
+}
+\arguments{
+\item{params}{the list of parameters. Commonly used ones are:
+\itemize{
+  \item \code{objective} objective function, common ones are
+  \itemize{
+    \item \code{reg:linear} linear regression
+    \item \code{binary:logistic} logistic regression for classification
+  }
+  \item \code{eta} step size of each boosting step
+  \item \code{max.depth} maximum depth of the tree
+  \item \code{nthread} number of thread used in training, if not set, all threads are used
+}
+
+  See \url{https://github.com/tqchen/xgboost/wiki/Parameters} for
+  further details. See also demo/ for walkthrough example in R.}
+
+\item{data}{takes an \code{xgb.DMatrix} as the input.}
+
+\item{nrounds}{the max number of iterations}
+
+\item{nfold}{number of folds used}
+
+\item{label}{option field, when data is Matrix}
+
+\item{showsd}{boolean, whether show standard deviation of cross validation}
+
+\item{metrics,}{list of evaluation metrics to be used in corss validation,
+  when it is not specified, the evaluation metric is chosen according to objective function.
+  Possible options are:
+\itemize{
+  \item \code{error} binary classification error rate
+  \item \code{rmse} Rooted mean square error
+  \item \code{logloss} negative log-likelihood function
+  \item \code{auc} Area under curve
+  \item \code{merror} Exact matching error, used to evaluate multi-class classification
+}}
+
+\item{obj}{customized objective function. Returns gradient and second order
+gradient with given prediction and dtrain,}
+
+\item{feval}{custimized evaluation function. Returns
+\code{list(metric='metric-name', value='metric-value')} with given
+prediction and dtrain,}
+
+\item{...}{other parameters to pass to \code{params}.}
+}
+\description{
+The cross valudation function of xgboost
+}
+\details{
+This is the cross validation function for xgboost
+
+Parallelization is automatically enabled if OpenMP is present.
+Number of threads can also be manually specified via "nthread" parameter.
+
+This function only accepts an \code{xgb.DMatrix} object as the input.
+}
+\examples{
+data(agaricus.train, package='xgboost')
+dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+history <- xgb.cv(data = dtrain, nround=3, nfold = 5, metrics=list("rmse","auc"),
+                  "max.depth"=3, "eta"=1, "objective"="binary:logistic")
+}
+

R-package/man/xgb.dump.Rd

@@ -0,0 +1,32 @@
+% Generated by roxygen2 (4.0.1): do not edit by hand
+\name{xgb.dump}
+\alias{xgb.dump}
+\title{Save xgboost model to text file}
+\usage{
+xgb.dump(model, fname, fmap = "")
+}
+\arguments{
+\item{model}{the model object.}
+
+\item{fname}{the name of the binary file.}
+
+\item{fmap}{feature map file representing the type of feature.
+       Detailed description could be found at
+       \url{https://github.com/tqchen/xgboost/wiki/Binary-Classification#dump-model}.
+       See demo/ for walkthrough example in R, and
+       \url{https://github.com/tqchen/xgboost/blob/master/demo/data/featmap.txt}
+       for example Format.}
+}
+\description{
+Save a xgboost model to text file. Could be parsed later.
+}
+\examples{
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+train <- agaricus.train
+test <- agaricus.test
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
+               eta = 1, nround = 2,objective = "binary:logistic")
+xgb.dump(bst, 'xgb.model.dump')
+}
+

R-package/man/xgb.load.Rd

@@ -0,0 +1,25 @@
+% Generated by roxygen2 (4.0.1): do not edit by hand
+\name{xgb.load}
+\alias{xgb.load}
+\title{Load xgboost model from binary file}
+\usage{
+xgb.load(modelfile)
+}
+\arguments{
+\item{modelfile}{the name of the binary file.}
+}
+\description{
+Load xgboost model from the binary model file
+}
+\examples{
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+train <- agaricus.train
+test <- agaricus.test
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
+               eta = 1, nround = 2,objective = "binary:logistic")
+xgb.save(bst, 'xgb.model')
+bst <- xgb.load('xgb.model')
+pred <- predict(bst, test$data)
+}
+

R-package/man/xgb.save.Rd

@@ -0,0 +1,27 @@
+% Generated by roxygen2 (4.0.1): do not edit by hand
+\name{xgb.save}
+\alias{xgb.save}
+\title{Save xgboost model to binary file}
+\usage{
+xgb.save(model, fname)
+}
+\arguments{
+\item{model}{the model object.}
+
+\item{fname}{the name of the binary file.}
+}
+\description{
+Save xgboost model from xgboost or xgb.train
+}
+\examples{
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+train <- agaricus.train
+test <- agaricus.test
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
+               eta = 1, nround = 2,objective = "binary:logistic")
+xgb.save(bst, 'xgb.model')
+bst <- xgb.load('xgb.model')
+pred <- predict(bst, test$data)
+}
+

R-package/man/xgb.train.Rd

@@ -0,0 +1,80 @@
+% Generated by roxygen2 (4.0.1): do not edit by hand
+\name{xgb.train}
+\alias{xgb.train}
+\title{eXtreme Gradient Boosting Training}
+\usage{
+xgb.train(params = list(), data, nrounds, watchlist = list(), obj = NULL,
+  feval = NULL, verbose = 1, ...)
+}
+\arguments{
+\item{params}{the list of parameters. Commonly used ones are:
+\itemize{
+  \item \code{objective} objective function, common ones are
+  \itemize{
+    \item \code{reg:linear} linear regression
+    \item \code{binary:logistic} logistic regression for classification
+  }
+  \item \code{eta} step size of each boosting step
+  \item \code{max.depth} maximum depth of the tree
+  \item \code{nthread} number of thread used in training, if not set, all threads are used
+}
+
+  See \url{https://github.com/tqchen/xgboost/wiki/Parameters} for
+  further details. See also demo/ for walkthrough example in R.}
+
+\item{data}{takes an \code{xgb.DMatrix} as the input.}
+
+\item{nrounds}{the max number of iterations}
+
+\item{watchlist}{what information should be printed when \code{verbose=1} or
+  \code{verbose=2}. Watchlist is used to specify validation set monitoring
+  during training. For example user can specify
+   watchlist=list(validation1=mat1, validation2=mat2) to watch
+   the performance of each round's model on mat1 and mat2}
+
+\item{obj}{customized objective function. Returns gradient and second order
+gradient with given prediction and dtrain,}
+
+\item{feval}{custimized evaluation function. Returns
+\code{list(metric='metric-name', value='metric-value')} with given
+prediction and dtrain,}
+
+\item{verbose}{If 0, xgboost will stay silent. If 1, xgboost will print
+  information of performance. If 2, xgboost will print information of both}
+
+\item{...}{other parameters to pass to \code{params}.}
+}
+\description{
+The training function of xgboost
+}
+\details{
+This is the training function for xgboost.
+
+Parallelization is automatically enabled if OpenMP is present.
+Number of threads can also be manually specified via "nthread" parameter.
+
+This function only accepts an \code{xgb.DMatrix} object as the input.
+It supports advanced features such as watchlist, customized objective function,
+therefore it is more flexible than \code{\link{xgboost}}.
+}
+\examples{
+data(agaricus.train, package='xgboost')
+dtrain <- xgb.DMatrix(agaricus.train$data, label = agaricus.train$label)
+dtest <- dtrain
+watchlist <- list(eval = dtest, train = dtrain)
+param <- list(max.depth = 2, eta = 1, silent = 1)
+logregobj <- function(preds, dtrain) {
+   labels <- getinfo(dtrain, "label")
+   preds <- 1/(1 + exp(-preds))
+   grad <- preds - labels
+   hess <- preds * (1 - preds)
+   return(list(grad = grad, hess = hess))
+}
+evalerror <- function(preds, dtrain) {
+  labels <- getinfo(dtrain, "label")
+  err <- as.numeric(sum(labels != (preds > 0)))/length(labels)
+  return(list(metric = "error", value = err))
+}
+bst <- xgb.train(param, dtrain, nround = 2, watchlist, logregobj, evalerror)
+}
+

R-package/man/xgboost.Rd

@@ -0,0 +1,56 @@
+% Generated by roxygen2 (4.0.1): do not edit by hand
+\name{xgboost}
+\alias{xgboost}
+\title{eXtreme Gradient Boosting (Tree) library}
+\usage{
+xgboost(data = NULL, label = NULL, params = list(), nrounds,
+  verbose = 1, ...)
+}
+\arguments{
+\item{data}{takes \code{matrix}, \code{dgCMatrix}, local data file or
+\code{xgb.DMatrix}.}
+
+\item{label}{the response variable. User should not set this field,}
+
+\item{params}{the list of parameters. Commonly used ones are:
+\itemize{
+  \item \code{objective} objective function, common ones are
+  \itemize{
+    \item \code{reg:linear} linear regression
+    \item \code{binary:logistic} logistic regression for classification
+  }
+  \item \code{eta} step size of each boosting step
+  \item \code{max.depth} maximum depth of the tree
+  \item \code{nthread} number of thread used in training, if not set, all threads are used
+}
+
+  See \url{https://github.com/tqchen/xgboost/wiki/Parameters} for
+  further details. See also demo/ for walkthrough example in R.}
+
+\item{nrounds}{the max number of iterations}
+
+\item{verbose}{If 0, xgboost will stay silent. If 1, xgboost will print
+information of performance. If 2, xgboost will print information of both
+performance and construction progress information}
+
+\item{...}{other parameters to pass to \code{params}.}
+}
+\description{
+A simple interface for xgboost in R
+}
+\details{
+This is the modeling function for xgboost.
+
+Parallelization is automatically enabled if OpenMP is present.
+Number of threads can also be manually specified via "nthread" parameter
+}
+\examples{
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+train <- agaricus.train
+test <- agaricus.test
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2,
+               eta = 1, nround = 2,objective = "binary:logistic")
+pred <- predict(bst, test$data)
+}
+

R-package/src/Makevars

@@ -0,0 +1,9 @@
+# package root
+PKGROOT=../../
+# _*_ mode: Makefile; _*_
+PKG_CPPFLAGS= -DXGBOOST_CUSTOMIZE_MSG_ -DXGBOOST_CUSTOMIZE_PRNG_ -DXGBOOST_STRICT_CXX98_ -I$(PKGROOT)
+PKG_CXXFLAGS= $(SHLIB_OPENMP_CFLAGS)
+PKG_LIBS = $(SHLIB_OPENMP_CFLAGS)
+OBJECTS= xgboost_R.o xgboost_assert.o $(PKGROOT)/wrapper/xgboost_wrapper.o $(PKGROOT)/src/io/io.o $(PKGROOT)/src/gbm/gbm.o $(PKGROOT)/src/tree/updater.o
+
+

R-package/src/Makevars.win

@@ -0,0 +1,7 @@
+# package root
+PKGROOT=../../
+# _*_ mode: Makefile; _*_
+PKG_CPPFLAGS= -DXGBOOST_CUSTOMIZE_MSG_ -DXGBOOST_CUSTOMIZE_PRNG_ -DXGBOOST_STRICT_CXX98_ -I$(PKGROOT)
+PKG_CXXFLAGS= $(SHLIB_OPENMP_CFLAGS)
+PKG_LIBS = $(SHLIB_OPENMP_CFLAGS)
+OBJECTS= xgboost_R.o xgboost_assert.o $(PKGROOT)/wrapper/xgboost_wrapper.o $(PKGROOT)/src/io/io.o $(PKGROOT)/src/gbm/gbm.o $(PKGROOT)/src/tree/updater.o

R-package/src/xgboost_R.cpp

@@ -0,0 +1,289 @@
+#include <vector>
+#include <string>
+#include <utility>
+#include <cstring>
+#include <cstdio>
+#include "xgboost_R.h"
+#include "wrapper/xgboost_wrapper.h"
+#include "src/utils/utils.h"
+#include "src/utils/omp.h"
+using namespace std;
+using namespace xgboost;
+
+extern "C" {
+  void XGBoostAssert_R(int exp, const char *fmt, ...);
+  void XGBoostCheck_R(int exp, const char *fmt, ...);
+  int XGBoostSPrintf_R(char *buf, size_t size, const char *fmt, ...);
+}
+
+// implements error handling
+namespace xgboost {
+namespace utils {
+extern "C" {
+  void (*Printf)(const char *fmt, ...) = Rprintf;
+  int (*SPrintf)(char *buf, size_t size, const char *fmt, ...) = XGBoostSPrintf_R;
+  void (*Assert)(int exp, const char *fmt, ...) = XGBoostAssert_R;
+  void (*Check)(int exp, const char *fmt, ...) = XGBoostCheck_R;
+  void (*Error)(const char *fmt, ...) = error;
+}
+}  // namespace utils
+
+namespace random {
+void Seed(unsigned seed) {
+  warning("parameter seed is ignored, please set random seed using set.seed");
+}
+double Uniform(void) {
+  return unif_rand();
+}
+double Normal(void) {
+  return norm_rand();
+}
+}  // namespace random
+}  // namespace xgboost
+
+// call before wrapper starts
+inline void _WrapperBegin(void) {
+  GetRNGstate();
+}
+// call after wrapper starts
+inline void _WrapperEnd(void) {
+  PutRNGstate();
+}
+
+extern "C" {
+  void _DMatrixFinalizer(SEXP ext) {    
+    if (R_ExternalPtrAddr(ext) == NULL) return;
+    XGDMatrixFree(R_ExternalPtrAddr(ext));
+    R_ClearExternalPtr(ext);
+  }
+  SEXP XGDMatrixCreateFromFile_R(SEXP fname, SEXP silent) {
+    _WrapperBegin();
+    void *handle = XGDMatrixCreateFromFile(CHAR(asChar(fname)), asInteger(silent));
+    SEXP ret = PROTECT(R_MakeExternalPtr(handle, R_NilValue, R_NilValue));
+    R_RegisterCFinalizerEx(ret, _DMatrixFinalizer, TRUE);
+    UNPROTECT(1);
+    _WrapperEnd();
+    return ret;
+  }
+  SEXP XGDMatrixCreateFromMat_R(SEXP mat, 
+                                SEXP missing) {
+    _WrapperBegin();
+    SEXP dim = getAttrib(mat, R_DimSymbol);
+    int nrow = INTEGER(dim)[0];
+    int ncol = INTEGER(dim)[1];    
+    double *din = REAL(mat);
+    std::vector<float> data(nrow * ncol);
+    #pragma omp parallel for schedule(static)
+    for (int i = 0; i < nrow; ++i) {
+      for (int j = 0; j < ncol; ++j) {
+        data[i * ncol +j] = din[i + nrow * j];
+      }
+    }
+    void *handle = XGDMatrixCreateFromMat(BeginPtr(data), nrow, ncol, asReal(missing));
+    SEXP ret = PROTECT(R_MakeExternalPtr(handle, R_NilValue, R_NilValue));
+    R_RegisterCFinalizerEx(ret, _DMatrixFinalizer, TRUE);
+    UNPROTECT(1);
+    _WrapperEnd();
+    return ret;    
+  }
+  SEXP XGDMatrixCreateFromCSC_R(SEXP indptr,
+                                SEXP indices,
+                                SEXP data) {
+    _WrapperBegin();
+    const int *p_indptr = INTEGER(indptr);
+    const int *p_indices = INTEGER(indices);
+    const double *p_data = REAL(data);
+    int nindptr = length(indptr);
+    int ndata = length(data);
+    std::vector<bst_ulong> col_ptr_(nindptr);
+    std::vector<unsigned> indices_(ndata);
+    std::vector<float> data_(ndata);
+
+    for (int i = 0; i < nindptr; ++i) {
+      col_ptr_[i] = static_cast<bst_ulong>(p_indptr[i]);
+    }
+    #pragma omp parallel for schedule(static)
+    for (int i = 0; i < ndata; ++i) {
+      indices_[i] = static_cast<unsigned>(p_indices[i]);
+      data_[i] = static_cast<float>(p_data[i]);
+    }
+    void *handle = XGDMatrixCreateFromCSC(BeginPtr(col_ptr_), BeginPtr(indices_),
+                                          BeginPtr(data_), nindptr, ndata);
+    SEXP ret = PROTECT(R_MakeExternalPtr(handle, R_NilValue, R_NilValue));
+    R_RegisterCFinalizerEx(ret, _DMatrixFinalizer, TRUE);
+    UNPROTECT(1);
+    _WrapperEnd();
+    return ret;
+  }
+  SEXP XGDMatrixSliceDMatrix_R(SEXP handle, SEXP idxset) {
+    _WrapperBegin();
+    int len = length(idxset);
+    std::vector<int> idxvec(len);
+    for (int i = 0; i < len; ++i) {
+      idxvec[i] = INTEGER(idxset)[i] - 1;
+    }
+    void *res = XGDMatrixSliceDMatrix(R_ExternalPtrAddr(handle), BeginPtr(idxvec), len);
+    SEXP ret = PROTECT(R_MakeExternalPtr(res, R_NilValue, R_NilValue));
+    R_RegisterCFinalizerEx(ret, _DMatrixFinalizer, TRUE);
+    UNPROTECT(1);
+    _WrapperEnd();
+    return ret;        
+  }
+  void XGDMatrixSaveBinary_R(SEXP handle, SEXP fname, SEXP silent) {
+    _WrapperBegin();
+    XGDMatrixSaveBinary(R_ExternalPtrAddr(handle),
+                        CHAR(asChar(fname)), asInteger(silent));
+    _WrapperEnd();
+  }
+  void XGDMatrixSetInfo_R(SEXP handle, SEXP field, SEXP array) {
+    _WrapperBegin();
+    int len = length(array);
+    const char *name = CHAR(asChar(field));
+    if (!strcmp("group", name)) {
+      std::vector<unsigned> vec(len);
+      #pragma omp parallel for schedule(static)      
+      for (int i = 0; i < len; ++i) {
+        vec[i] = static_cast<unsigned>(INTEGER(array)[i]);
+      }
+      XGDMatrixSetGroup(R_ExternalPtrAddr(handle), BeginPtr(vec), len);
+      _WrapperEnd();
+      return;
+    }
+    {
+      std::vector<float> vec(len);
+      #pragma omp parallel for schedule(static)
+      for (int i = 0; i < len; ++i) {
+        vec[i] = REAL(array)[i];
+      }
+      XGDMatrixSetFloatInfo(R_ExternalPtrAddr(handle), 
+                            CHAR(asChar(field)),
+                            BeginPtr(vec), len);
+    }
+    _WrapperEnd();
+  }
+  SEXP XGDMatrixGetInfo_R(SEXP handle, SEXP field) {
+    _WrapperBegin();
+    bst_ulong olen;
+    const float *res = XGDMatrixGetFloatInfo(R_ExternalPtrAddr(handle),
+                                             CHAR(asChar(field)), &olen);
+    SEXP ret = PROTECT(allocVector(REALSXP, olen));
+    for (size_t i = 0; i < olen; ++i) {
+      REAL(ret)[i] = res[i];
+    }
+    UNPROTECT(1);
+    _WrapperEnd();
+    return ret;
+  }
+  SEXP XGDMatrixNumRow_R(SEXP handle) {
+    bst_ulong nrow = XGDMatrixNumRow(R_ExternalPtrAddr(handle));
+    return ScalarInteger(static_cast<int>(nrow));
+  }
+  // functions related to booster
+  void _BoosterFinalizer(SEXP ext) {    
+    if (R_ExternalPtrAddr(ext) == NULL) return;
+    XGBoosterFree(R_ExternalPtrAddr(ext));
+    R_ClearExternalPtr(ext);
+  }
+  SEXP XGBoosterCreate_R(SEXP dmats) {
+    _WrapperBegin();
+    int len = length(dmats);
+    std::vector<void*> dvec;
+    for (int i = 0; i < len; ++i){
+      dvec.push_back(R_ExternalPtrAddr(VECTOR_ELT(dmats, i)));
+    }
+    void *handle = XGBoosterCreate(BeginPtr(dvec), dvec.size());
+    SEXP ret = PROTECT(R_MakeExternalPtr(handle, R_NilValue, R_NilValue));
+    R_RegisterCFinalizerEx(ret, _BoosterFinalizer, TRUE);
+    UNPROTECT(1);
+    _WrapperEnd();
+    return ret;
+  }
+  void XGBoosterSetParam_R(SEXP handle, SEXP name, SEXP val) {
+    _WrapperBegin();
+    XGBoosterSetParam(R_ExternalPtrAddr(handle),
+                      CHAR(asChar(name)),
+                      CHAR(asChar(val)));
+    _WrapperEnd();
+  }
+  void XGBoosterUpdateOneIter_R(SEXP handle, SEXP iter, SEXP dtrain) {
+    _WrapperBegin();
+    XGBoosterUpdateOneIter(R_ExternalPtrAddr(handle),
+                           asInteger(iter),
+                           R_ExternalPtrAddr(dtrain));
+    _WrapperEnd();
+  }
+  void XGBoosterBoostOneIter_R(SEXP handle, SEXP dtrain, SEXP grad, SEXP hess) {
+    _WrapperBegin();
+    utils::Check(length(grad) == length(hess), "gradient and hess must have same length");
+    int len = length(grad);
+    std::vector<float> tgrad(len), thess(len);
+    #pragma omp parallel for schedule(static)
+    for (int j = 0; j < len; ++j) {
+      tgrad[j] = REAL(grad)[j];
+      thess[j] = REAL(hess)[j];
+    }
+    XGBoosterBoostOneIter(R_ExternalPtrAddr(handle),
+                          R_ExternalPtrAddr(dtrain),
+                          BeginPtr(tgrad), BeginPtr(thess), len);
+    _WrapperEnd();
+  }
+  SEXP XGBoosterEvalOneIter_R(SEXP handle, SEXP iter, SEXP dmats, SEXP evnames) {
+    _WrapperBegin();
+    utils::Check(length(dmats) == length(evnames), "dmats and evnams must have same length");
+    int len = length(dmats);
+    std::vector<void*> vec_dmats;
+    std::vector<std::string> vec_names;
+    std::vector<const char*> vec_sptr;
+    for (int i = 0; i < len; ++i) {
+      vec_dmats.push_back(R_ExternalPtrAddr(VECTOR_ELT(dmats, i)));
+      vec_names.push_back(std::string(CHAR(asChar(VECTOR_ELT(evnames, i)))));
+    }
+    for (int i = 0; i < len; ++i) {
+      vec_sptr.push_back(vec_names[i].c_str());
+    }
+    return mkString(XGBoosterEvalOneIter(R_ExternalPtrAddr(handle),
+                                         asInteger(iter),
+                                         BeginPtr(vec_dmats), BeginPtr(vec_sptr), len));
+    _WrapperEnd();
+  }
+  SEXP XGBoosterPredict_R(SEXP handle, SEXP dmat, SEXP output_margin, SEXP ntree_limit) {
+    _WrapperBegin();
+    bst_ulong olen;
+    const float *res = XGBoosterPredict(R_ExternalPtrAddr(handle),
+                                        R_ExternalPtrAddr(dmat),
+                                        asInteger(output_margin),
+                                        asInteger(ntree_limit),
+                                        &olen);
+    SEXP ret = PROTECT(allocVector(REALSXP, olen));
+    for (size_t i = 0; i < olen; ++i) {
+      REAL(ret)[i] = res[i];
+    }
+    UNPROTECT(1);
+    _WrapperEnd();
+    return ret;
+  }
+  void XGBoosterLoadModel_R(SEXP handle, SEXP fname) {
+    _WrapperBegin();
+    XGBoosterLoadModel(R_ExternalPtrAddr(handle), CHAR(asChar(fname)));
+    _WrapperEnd();
+  }
+  void XGBoosterSaveModel_R(SEXP handle, SEXP fname) {
+    _WrapperBegin();
+    XGBoosterSaveModel(R_ExternalPtrAddr(handle), CHAR(asChar(fname)));
+    _WrapperEnd();
+  }
+  void XGBoosterDumpModel_R(SEXP handle, SEXP fname, SEXP fmap) {
+    _WrapperBegin();
+    bst_ulong olen;
+    const char **res = XGBoosterDumpModel(R_ExternalPtrAddr(handle),
+                                          CHAR(asChar(fmap)),
+                                          &olen);
+    FILE *fo = utils::FopenCheck(CHAR(asChar(fname)), "w");
+    for (size_t i = 0; i < olen; ++i) {
+      fprintf(fo, "booster[%u]:\n", static_cast<unsigned>(i));
+      fprintf(fo, "%s", res[i]);
+    }
+    fclose(fo);
+    _WrapperEnd();
+  }
+}

R-package/src/xgboost_R.h

@@ -0,0 +1,138 @@
+#ifndef XGBOOST_WRAPPER_R_H_
+#define XGBOOST_WRAPPER_R_H_
+/*!
+ * \file xgboost_wrapper_R.h
+ * \author Tianqi Chen
+ * \brief R wrapper of xgboost
+ */
+extern "C" {
+#include <Rinternals.h>
+#include <R_ext/Random.h>
+}
+
+extern "C" {
+  /*!
+   * \brief load a data matrix 
+   * \param fname name of the content
+   * \param silent whether print messages
+   * \return a loaded data matrix
+   */
+  SEXP XGDMatrixCreateFromFile_R(SEXP fname, SEXP silent);
+  /*!
+   * \brief create matrix content from dense matrix
+   * This assumes the matrix is stored in column major format
+   * \param data R Matrix object
+   * \param missing which value to represent missing value
+   * \return created dmatrix
+   */
+  SEXP XGDMatrixCreateFromMat_R(SEXP mat, 
+                                SEXP missing);
+  /*! 
+   * \brief create a matrix content from CSC format
+   * \param indptr pointer to column headers
+   * \param indices row indices
+   * \param data content of the data
+   * \return created dmatrix
+   */
+  SEXP XGDMatrixCreateFromCSC_R(SEXP indptr,
+                                SEXP indices,
+                                SEXP data);
+  /*!
+   * \brief create a new dmatrix from sliced content of existing matrix
+   * \param handle instance of data matrix to be sliced
+   * \param idxset index set
+   * \return a sliced new matrix
+   */
+  SEXP XGDMatrixSliceDMatrix_R(SEXP handle, SEXP idxset);
+  /*!
+   * \brief load a data matrix into binary file
+   * \param handle a instance of data matrix
+   * \param fname file name
+   * \param silent print statistics when saving
+   */
+  void XGDMatrixSaveBinary_R(SEXP handle, SEXP fname, SEXP silent);
+  /*!
+   * \brief set information to dmatrix
+   * \param handle a instance of data matrix
+   * \param field field name, can be label, weight
+   * \param array pointer to float vector
+   */
+  void XGDMatrixSetInfo_R(SEXP handle, SEXP field, SEXP array);
+  /*!
+   * \brief get info vector from matrix
+   * \param handle a instance of data matrix
+   * \param field field name
+   * \return info vector
+   */  
+  SEXP XGDMatrixGetInfo_R(SEXP handle, SEXP field);
+  /*!
+   * \brief return number of rows
+   * \param handle a instance of data matrix
+   */
+  SEXP XGDMatrixNumRow_R(SEXP handle);
+  /*! 
+   * \brief create xgboost learner 
+   * \param dmats a list of dmatrix handles that will be cached
+   */  
+  SEXP XGBoosterCreate_R(SEXP dmats);
+  /*! 
+   * \brief set parameters 
+   * \param handle handle
+   * \param name  parameter name
+   * \param val value of parameter
+   */
+  void XGBoosterSetParam_R(SEXP handle, SEXP name, SEXP val);
+  /*! 
+   * \brief update the model in one round using dtrain
+   * \param handle handle
+   * \param iter current iteration rounds
+   * \param dtrain training data
+   */
+  void XGBoosterUpdateOneIter_R(SEXP ext, SEXP iter, SEXP dtrain);
+  /*!
+   * \brief update the model, by directly specify gradient and second order gradient,
+   *        this can be used to replace UpdateOneIter, to support customized loss function
+   * \param handle handle
+   * \param dtrain training data
+   * \param grad gradient statistics
+   * \param hess second order gradient statistics
+   */
+  void XGBoosterBoostOneIter_R(SEXP handle, SEXP dtrain, SEXP grad, SEXP hess);
+  /*!
+   * \brief get evaluation statistics for xgboost
+   * \param handle handle
+   * \param iter current iteration rounds
+   * \param dmats list of handles to dmatrices
+   * \param evname name of evaluation
+   * \return the string containing evaluation stati
+   */
+  SEXP XGBoosterEvalOneIter_R(SEXP handle, SEXP iter, SEXP dmats, SEXP evnames);
+  /*!
+   * \brief make prediction based on dmat
+   * \param handle handle
+   * \param dmat data matrix
+   * \param output_margin whether only output raw margin value
+   * \param ntree_limit limit number of trees used in prediction
+   */
+  SEXP XGBoosterPredict_R(SEXP handle, SEXP dmat, SEXP output_margin, SEXP ntree_limit);
+  /*!
+   * \brief load model from existing file
+   * \param handle handle
+   * \param fname file name
+   */
+  void XGBoosterLoadModel_R(SEXP handle, SEXP fname);
+  /*!
+   * \brief save model into existing file
+   * \param handle handle
+   * \param fname file name
+   */    
+  void XGBoosterSaveModel_R(SEXP handle, SEXP fname);
+  /*!
+   * \brief dump model into text file 
+   * \param handle handle
+   * \param fname file name of model that can be dumped into
+   * \param fmap  name to fmap can be empty string
+   */
+  void XGBoosterDumpModel_R(SEXP handle, SEXP fname, SEXP fmap);
+}
+#endif  // XGBOOST_WRAPPER_R_H_

R-package/src/xgboost_assert.c

@@ -0,0 +1,33 @@
+#include <stdio.h>
+#include <stdarg.h>
+#include <Rinternals.h>
+
+// implements error handling
+void XGBoostAssert_R(int exp, const char *fmt, ...) {
+  char buf[1024];
+  if (exp == 0) {
+    va_list args;  
+    va_start(args, fmt);
+    vsprintf(buf, fmt, args);
+    va_end(args);
+    error("AssertError:%s\n", buf);
+  }  
+}
+void XGBoostCheck_R(int exp, const char *fmt, ...) {
+  char buf[1024];
+  if (exp == 0) {
+    va_list args;  
+    va_start(args, fmt);
+    vsprintf(buf, fmt, args);
+    va_end(args);
+    error("%s\n", buf);
+  }
+}
+int XGBoostSPrintf_R(char *buf, size_t size, const char *fmt, ...) {
+  int ret;
+  va_list args;  
+  va_start(args, fmt);
+  ret = vsnprintf(buf, size, fmt, args);
+  va_end(args);
+  return ret;
+}

R-package/vignettes/xgboost.Rnw

@@ -0,0 +1,216 @@
+\documentclass{article}
+\RequirePackage{url}
+\usepackage{hyperref}
+\RequirePackage{amsmath}
+\RequirePackage{natbib}
+\RequirePackage[a4paper,lmargin={1.25in},rmargin={1.25in},tmargin={1in},bmargin={1in}]{geometry}
+
+\makeatletter
+% \VignetteIndexEntry{xgboost: eXtreme Gradient Boosting}
+%\VignetteKeywords{xgboost, gbm, gradient boosting machines}
+%\VignettePackage{xgboost}
+% \VignetteEngine{knitr::knitr}
+\makeatother
+
+\begin{document}
+%\SweaveOpts{concordance=TRUE}
+
+<<knitropts,echo=FALSE,message=FALSE>>=
+if (require('knitr')) opts_chunk$set(fig.width = 5, fig.height = 5, fig.align = 'center', tidy = FALSE, warning = FALSE, cache = TRUE)
+@
+
+%
+<<prelim,echo=FALSE>>=
+xgboost.version = '0.3-0'
+@
+%
+
+    \begin{center}
+    \vspace*{6\baselineskip}
+    \rule{\textwidth}{1.6pt}\vspace*{-\baselineskip}\vspace*{2pt}
+    \rule{\textwidth}{0.4pt}\\[2\baselineskip]
+    {\LARGE \textbf{xgboost: eXtreme Gradient Boosting}}\\[1.2\baselineskip]
+    \rule{\textwidth}{0.4pt}\vspace*{-\baselineskip}\vspace{3.2pt}
+    \rule{\textwidth}{1.6pt}\\[2\baselineskip]
+    {\Large Tianqi Chen, Tong He}\\[\baselineskip]
+    {\large Package Version: \Sexpr{xgboost.version}}\\[\baselineskip]
+    {\large \today}\par
+    \vfill
+    \end{center}
+
+\thispagestyle{empty}
+
+\clearpage
+
+\setcounter{page}{1}
+
+\section{Introduction}
+
+This is an introductory document of using the \verb@xgboost@ package in R. 
+
+\verb@xgboost@ is short for eXtreme Gradient Boosting package. It is an efficient
+ and scalable implementation of gradient boosting framework by \citep{friedman2001greedy}. 
+The package includes efficient linear model solver and tree learning algorithm.
+It supports various objective functions, including regression, classification
+and ranking. The package is made to be extendible, so that users are also allowed to define their own objectives easily. It has several features:
+\begin{enumerate}
+    \item{Speed: }{\verb@xgboost@ can automatically do parallel computation on 
+    Windows and Linux, with openmp. It is generally over 10 times faster than
+    \verb@gbm@.}
+    \item{Input Type: }{\verb@xgboost@ takes several types of input data:}
+    \begin{itemize}
+        \item{Dense Matrix: }{R's dense matrix, i.e. \verb@matrix@}
+        \item{Sparse Matrix: }{R's sparse matrix \verb@Matrix::dgCMatrix@}
+        \item{Data File: }{Local data files}
+        \item{xgb.DMatrix: }{\verb@xgboost@'s own class. Recommended.}
+    \end{itemize}
+    \item{Sparsity: }{\verb@xgboost@ accepts sparse input for both tree booster 
+    and linear booster, and is optimized for sparse input.}
+    \item{Customization: }{\verb@xgboost@ supports customized objective function 
+    and evaluation function}
+    \item{Performance: }{\verb@xgboost@ has better performance on several different
+    datasets.}
+\end{enumerate}
+
+
+\section{Example with Mushroom data}
+
+In this section, we will illustrate some common usage of \verb@xgboost@. The 
+Mushroom data is cited from UCI Machine Learning Repository. \citep{Bache+Lichman:2013} 
+
+<<Training and prediction with iris>>=
+library(xgboost)
+data(agaricus.train, package='xgboost')
+data(agaricus.test, package='xgboost')
+train <- agaricus.train
+test <- agaricus.test
+bst <- xgboost(data = train$data, label = train$label, max.depth = 2, eta = 1, 
+               nround = 2, objective = "binary:logistic")
+xgb.save(bst, 'model.save')
+bst = xgb.load('model.save')
+pred <- predict(bst, test$data)
+@
+
+\verb@xgboost@ is the main function to train a \verb@Booster@, i.e. a model.
+\verb@predict@ does prediction on the model.
+
+Here we can save the model to a binary local file, and load it when needed.
+We can't inspect the trees inside. However we have another function to save the
+model in plain text. 
+<<Dump Model>>=
+xgb.dump(bst, 'model.dump')
+@
+
+The output looks like 
+
+\begin{verbatim}
+booster[0]:
+0:[f28<1.00001] yes=1,no=2,missing=2
+  1:[f108<1.00001] yes=3,no=4,missing=4
+    3:leaf=1.85965
+    4:leaf=-1.94071
+  2:[f55<1.00001] yes=5,no=6,missing=6
+    5:leaf=-1.70044
+    6:leaf=1.71218
+booster[1]:
+0:[f59<1.00001] yes=1,no=2,missing=2
+  1:leaf=-6.23624
+  2:[f28<1.00001] yes=3,no=4,missing=4
+    3:leaf=-0.96853
+    4:leaf=0.784718
+\end{verbatim}
+
+It is important to know \verb@xgboost@'s own data type: \verb@xgb.DMatrix@.
+It speeds up \verb@xgboost@, and is needed for advanced features such as 
+training from initial prediction value, weighted training instance. 
+
+We can use \verb@xgb.DMatrix@ to construct an \verb@xgb.DMatrix@ object:
+<<xgb.DMatrix>>=
+dtrain <- xgb.DMatrix(train$data, label = train$label)
+class(dtrain)
+head(getinfo(dtrain,'label'))
+@
+
+We can also save the matrix to a binary file. Then load it simply with 
+\verb@xgb.DMatrix@
+<<save model>>=
+xgb.DMatrix.save(dtrain, 'xgb.DMatrix')
+dtrain = xgb.DMatrix('xgb.DMatrix')
+@
+
+\section{Advanced Examples}
+
+The function \verb@xgboost@ is a simple function with less parameter, in order
+to be R-friendly. The core training function is wrapped in \verb@xgb.train@. It is more flexible than \verb@xgboost@, but it requires users to read the document a bit more carefully.
+
+\verb@xgb.train@ only accept a \verb@xgb.DMatrix@ object as its input, while it supports advanced features as custom objective and evaluation functions.
+
+<<Customized loss function>>=
+logregobj <- function(preds, dtrain) {
+   labels <- getinfo(dtrain, "label")
+   preds <- 1/(1 + exp(-preds))
+   grad <- preds - labels
+   hess <- preds * (1 - preds)
+   return(list(grad = grad, hess = hess))
+}
+
+evalerror <- function(preds, dtrain) {
+  labels <- getinfo(dtrain, "label")
+  err <- sqrt(mean((preds-labels)^2))
+  return(list(metric = "MSE", value = err))
+}
+
+dtest <- xgb.DMatrix(test$data, label = test$label)
+watchlist <- list(eval = dtest, train = dtrain)
+param <- list(max.depth = 2, eta = 1, silent = 1)
+
+bst <- xgb.train(param, dtrain, nround = 2, watchlist, logregobj, evalerror)
+@
+
+The gradient and second order gradient is required for the output of customized 
+objective function. 
+
+We also have \verb@slice@ for row extraction. It is useful in 
+cross-validation.
+
+For a walkthrough demo, please see \verb@R-package/demo/@ for further 
+details.
+
+\section{The Higgs Boson competition}
+
+We have made a demo for \href{http://www.kaggle.com/c/higgs-boson}{the Higgs 
+Boson Machine Learning Challenge}. 
+
+Here are the instructions to make a submission
+\begin{enumerate}
+    \item Download the \href{http://www.kaggle.com/c/higgs-boson/data}{datasets}
+    and extract them to \verb@data/@.
+    \item Run scripts under \verb@xgboost/demo/kaggle-higgs/@: 
+    \href{https://github.com/tqchen/xgboost/blob/master/demo/kaggle-higgs/higgs-train.R}{higgs-train.R} 
+    and \href{https://github.com/tqchen/xgboost/blob/master/demo/kaggle-higgs/higgs-pred.R}{higgs-pred.R}. 
+    The computation will take less than a minute on Intel i7. 
+    \item Go to the \href{http://www.kaggle.com/c/higgs-boson/submissions/attach}{submission page} 
+    and submit your result.
+\end{enumerate}
+
+We provide \href{https://github.com/tqchen/xgboost/blob/master/demo/kaggle-higgs/speedtest.R}{a script}
+to compare the time cost on the higgs dataset with \verb@gbm@ and \verb@xgboost@. 
+The training set contains 350000 records and 30 features. 
+
+\verb@xgboost@ can automatically do parallel computation. On a machine with Intel
+i7-4700MQ and 24GB memories, we found that \verb@xgboost@ costs about 35 seconds, which is about 20 times faster
+than \verb@gbm@. When we limited \verb@xgboost@ to use only one thread, it was 
+still about two times faster than \verb@gbm@. 
+
+Meanwhile, the result from \verb@xgboost@ reaches 
+\href{http://www.kaggle.com/c/higgs-boson/details/evaluation}{3.60@AMS} with a 
+single model. This results stands in the 
+\href{http://www.kaggle.com/c/higgs-boson/leaderboard}{top 30\%} of the 
+competition. 
+
+\bibliographystyle{jss}
+\nocite{*} % list uncited references
+\bibliography{xgboost}
+
+\end{document}
+

R-package/vignettes/xgboost.bib

@@ -0,0 +1,30 @@
+@article{friedman2001greedy,
+    title={Greedy function approximation: a gradient boosting machine},
+    author={Friedman, Jerome H},
+    journal={Annals of Statistics},
+    pages={1189--1232},
+    year={2001},
+    publisher={JSTOR}
+}
+
+@article{friedman2000additive,
+  title={Additive logistic regression: a statistical view of boosting (with discussion and a rejoinder by the authors)},
+  author={Friedman, Jerome and Hastie, Trevor and Tibshirani, Robert and others},
+  journal={The annals of statistics},
+  volume={28},
+  number={2},
+  pages={337--407},
+  year={2000},
+  publisher={Institute of Mathematical Statistics}
+}
+
+
+@misc{
+    Bache+Lichman:2013 ,
+    author = "K. Bache and M. Lichman",
+    year = "2013",
+    title = "{UCI} Machine Learning Repository",
+    url = "http://archive.ics.uci.edu/ml",
+    institution = "University of California, Irvine, School of Information and Computer Sciences" 
+}
+

README.md

@@ -1,4 +1,52 @@
-xgboost
-=======
+xgboost: eXtreme Gradient Boosting 
+======
+An optimized general purpose gradient boosting library. The library is parallelized using OpenMP. It implements machine learning algorithm under gradient boosting framework, including generalized linear model and gradient boosted regression tree. 
 
-General Purpose Gradient Boosting Library
+Contributors: https://github.com/tqchen/xgboost/graphs/contributors
+
+Turorial and Documentation: https://github.com/tqchen/xgboost/wiki
+
+Questions and Issues: [https://github.com/tqchen/xgboost/issues](https://github.com/tqchen/xgboost/issues?q=is%3Aissue+label%3Aquestion)
+
+Examples Code: [Learning to use xgboost by examples](demo)
+
+Notes on the Code: [Code Guide](src)
+
+What's New
+=====
+* See the updated [demo folder](demo) for feature walkthrough
+* Thanks to Tong He, the new [R package](R-package) is available
+
+Features
+======
+* Sparse feature format:
+  - Sparse feature format allows easy handling of missing values, and improve computation efficiency.
+* Push the limit on single machine:
+  - Efficient implementation that optimizes memory and computation.
+* Speed: XGBoost is very fast
+  - IN [demo/higgs/speedtest.py](demo/kaggle-higgs/speedtest.py), kaggle higgs data it is faster(on our machine 20 times faster using 4 threads) than sklearn.ensemble.GradientBoostingClassifier
+* Layout of gradient boosting algorithm to support user defined objective
+* Python interface, works with numpy and scipy.sparse matrix
+
+Build
+=====
+* Run ```bash build.sh``` (you can also type make)
+* If your compiler does not come with OpenMP support, it will fire an warning telling you that the code will compile into single thread mode, and you will get single thread xgboost
+* You may get a error: -lgomp is not found
+  - You can type ```make no_omp=1```, this will get you single thread xgboost
+  - Alternatively, you can upgrade your compiler to compile multi-thread version
+* Windows(VS 2010): see [windows](windows) folder
+  - In principle, you put all the cpp files in the Makefile to the project, and build
+
+Version
+======
+* This version xgboost-0.3, the code has been refactored from 0.2x to be cleaner and more flexibility
+* This version of xgboost is not compatible with 0.2x, due to huge amount of changes in code structure
+  - This means the model and buffer file of previous version can not be loaded in xgboost-3.0
+* For legacy 0.2x code, refer to [Here](https://github.com/tqchen/xgboost/releases/tag/v0.22)
+* Change log in [CHANGES.md](CHANGES.md)
+
+XGBoost in Graphlab Create
+======
+* XGBoost is adopted as part of boosted tree toolkit in Graphlab Create (GLC). Graphlab Create is a powerful python toolkit that allows you to data manipulation, graph processing, hyper-parameter search, and visualization of TeraBytes scale data in one framework. Try the Graphlab Create in http://graphlab.com/products/create/quick-start-guide.html
+* Nice blogpost by Jay Gu using GLC boosted tree to solve kaggle bike sharing challenge: http://blog.graphlab.com/using-gradient-boosted-trees-to-predict-bike-sharing-demand

build.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# this is a simple script to make xgboost in MAC nad Linux
+# basically, it first try to make with OpenMP, if fails, disable OpenMP and make again
+# This will automatically make xgboost for MAC users who do not have openmp support
+# In most cases, type make will give what you want
+if make; then
+    echo "Successfully build multi-thread xgboost"
+else
+    echo "-----------------------------"
+    echo "Building multi-thread xgboost failed"
+    echo "Start to build single-thread xgboost"
+    make clean
+    make no_omp=1
+    echo "Successfully build single-thread xgboost"
+fi

demo/README.md

@@ -0,0 +1,27 @@
+XGBoost Examples
+====
+This folder contains the all example codes using xgboost. 
+
+* Contribution of exampls, benchmarks is more than welcomed!
+* If you like to share how you use xgboost to solve your problem, send a pull request:)
+ 
+Features Walkthrough
+====
+This is a list of short codes introducing different functionalities of xgboost and its wrapper.
+* Basic walkthrough of wrappers [python](guide-python/basic_walkthrough.py)
+* Cutomize loss function, and evaluation metric [python](guide-python/custom_objective.py)
+* Boosting from existing prediction [python](guide-python/boost_from_prediction.py)
+* Predicting using first n trees [python](guide-python/predict_first_ntree.py)
+* Generalized Linear Model [python](guide-python/generalized_linear_model.py)
+* Cross validation [python](guide-python/cross_validation.py)
+
+Basic Examples by Tasks
+====
+* [Binary classification](binary_classification)
+* [Multiclass classification](multiclass_classification)
+* [Regression](regression)
+* [Learning to Rank](rank)
+
+Benchmarks
+====
+* [Starter script for Kaggle Higgs Boson](kaggle-higgs)

demo/binary_classification/README

@@ -0,0 +1,14 @@
+Demonstrating how to use XGBoost accomplish binary classification tasks  on UCI mushroom dataset  http://archive.ics.uci.edu/ml/datasets/Mushroom
+
+Run: ./runexp.sh
+
+Format of input: LIBSVM format
+
+Format of ```featmap.txt: <featureid> <featurename> <q or i or int>\n ```:
+  - Feature id must be from 0 to number of features, in sorted order.
+  - i means this feature is binary indicator feature
+  - q means this feature is a quantitative value, such as age, time, can be missing
+  - int means this feature is integer value (when int is hinted, the decision boundary will be integer)
+
+
+Explainations: https://github.com/tqchen/xgboost/wiki/Binary-Classification

demo/binary_classification/agaricus-lepiota.fmap

@@ -0,0 +1,32 @@
+     1. cap-shape:                bell=b,conical=c,convex=x,flat=f,knobbed=k,sunken=s
+     2. cap-surface:              fibrous=f,grooves=g,scaly=y,smooth=s
+     3. cap-color:                brown=n,buff=b,cinnamon=c,gray=g,green=r,pink=p,purple=u,red=e,white=w,yellow=y
+     4. bruises?:                 bruises=t,no=f
+     5. odor:                     almond=a,anise=l,creosote=c,fishy=y,foul=f,
+                                  musty=m,none=n,pungent=p,spicy=s
+     6. gill-attachment:          attached=a,descending=d,free=f,notched=n
+     7. gill-spacing:             close=c,crowded=w,distant=d
+     8. gill-size:                broad=b,narrow=n
+     9. gill-color:               black=k,brown=n,buff=b,chocolate=h,gray=g,
+                                  green=r,orange=o,pink=p,purple=u,red=e,
+                                  white=w,yellow=y
+    10. stalk-shape:              enlarging=e,tapering=t
+    11. stalk-root:               bulbous=b,club=c,cup=u,equal=e,
+                                  rhizomorphs=z,rooted=r,missing=?
+    12. stalk-surface-above-ring: fibrous=f,scaly=y,silky=k,smooth=s
+    13. stalk-surface-below-ring: fibrous=f,scaly=y,silky=k,smooth=s
+    14. stalk-color-above-ring:   brown=n,buff=b,cinnamon=c,gray=g,orange=o,
+                                  pink=p,red=e,white=w,yellow=y
+    15. stalk-color-below-ring:   brown=n,buff=b,cinnamon=c,gray=g,orange=o,
+                                  pink=p,red=e,white=w,yellow=y
+    16. veil-type:                partial=p,universal=u
+    17. veil-color:               brown=n,orange=o,white=w,yellow=y
+    18. ring-number:              none=n,one=o,two=t
+    19. ring-type:                cobwebby=c,evanescent=e,flaring=f,large=l,
+                                  none=n,pendant=p,sheathing=s,zone=z
+    20. spore-print-color:        black=k,brown=n,buff=b,chocolate=h,green=r,
+                                  orange=o,purple=u,white=w,yellow=y
+    21. population:               abundant=a,clustered=c,numerous=n,
+                                  scattered=s,several=v,solitary=y
+    22. habitat:                  grasses=g,leaves=l,meadows=m,paths=p,
+                                  urban=u,waste=w,woods=d

demo/binary_classification/agaricus-lepiota.names

@@ -0,0 +1,148 @@
+1. Title: Mushroom Database
+
+2. Sources: 
+    (a) Mushroom records drawn from The Audubon Society Field Guide to North
+        American Mushrooms (1981). G. H. Lincoff (Pres.), New York: Alfred
+        A. Knopf
+    (b) Donor: Jeff Schlimmer (Jeffrey.Schlimmer@a.gp.cs.cmu.edu)
+    (c) Date: 27 April 1987
+
+3. Past Usage:
+    1. Schlimmer,J.S. (1987). Concept Acquisition Through Representational
+       Adjustment (Technical Report 87-19).  Doctoral disseration, Department
+       of Information and Computer Science, University of California, Irvine.
+       --- STAGGER: asymptoted to 95% classification accuracy after reviewing
+           1000 instances.
+    2. Iba,W., Wogulis,J., & Langley,P. (1988).  Trading off Simplicity
+       and Coverage in Incremental Concept Learning. In Proceedings of 
+       the 5th International Conference on Machine Learning, 73-79.
+       Ann Arbor, Michigan: Morgan Kaufmann.  
+       -- approximately the same results with their HILLARY algorithm    
+    3. In the following references a set of rules (given below) were
+	learned for this data set which may serve as a point of
+	comparison for other researchers.
+
+	Duch W, Adamczak R, Grabczewski K (1996) Extraction of logical rules
+	from training data using backpropagation networks, in: Proc. of the
+	The 1st Online Workshop on Soft Computing, 19-30.Aug.1996, pp. 25-30,
+	available on-line at: http://www.bioele.nuee.nagoya-u.ac.jp/wsc1/
+
+	Duch W, Adamczak R, Grabczewski K, Ishikawa M, Ueda H, Extraction of
+	crisp logical rules using constrained backpropagation networks -
+	comparison of two new approaches, in: Proc. of the European Symposium
+	on Artificial Neural Networks (ESANN'97), Bruge, Belgium 16-18.4.1997,
+	pp. xx-xx
+
+	Wlodzislaw Duch, Department of Computer Methods, Nicholas Copernicus
+	University, 87-100 Torun, Grudziadzka 5, Poland
+	e-mail: duch@phys.uni.torun.pl
+	WWW     http://www.phys.uni.torun.pl/kmk/
+	
+	Date: Mon, 17 Feb 1997 13:47:40 +0100
+	From: Wlodzislaw Duch <duch@phys.uni.torun.pl>
+	Organization: Dept. of Computer Methods, UMK
+
+	I have attached a file containing logical rules for mushrooms.
+	It should be helpful for other people since only in the last year I
+	have seen about 10 papers analyzing this dataset and obtaining quite
+	complex rules. We will try to contribute other results later.
+
+	With best regards, Wlodek Duch
+	________________________________________________________________
+
+	Logical rules for the mushroom data sets.
+
+	Logical rules given below seem to be the simplest possible for the
+	mushroom dataset and therefore should be treated as benchmark results.
+
+	Disjunctive rules for poisonous mushrooms, from most general
+	to most specific:
+
+	P_1) odor=NOT(almond.OR.anise.OR.none)
+	     120 poisonous cases missed, 98.52% accuracy
+
+	P_2) spore-print-color=green
+	     48 cases missed, 99.41% accuracy
+         
+	P_3) odor=none.AND.stalk-surface-below-ring=scaly.AND.
+	          (stalk-color-above-ring=NOT.brown) 
+	     8 cases missed, 99.90% accuracy
+         
+	P_4) habitat=leaves.AND.cap-color=white
+	         100% accuracy     
+
+	Rule P_4) may also be
+
+	P_4') population=clustered.AND.cap_color=white
+
+	These rule involve 6 attributes (out of 22). Rules for edible
+	mushrooms are obtained as negation of the rules given above, for
+	example the rule:
+
+	odor=(almond.OR.anise.OR.none).AND.spore-print-color=NOT.green
+
+	gives 48 errors, or 99.41% accuracy on the whole dataset.
+
+	Several slightly more complex variations on these rules exist,
+	involving other attributes, such as gill_size, gill_spacing,
+	stalk_surface_above_ring, but the rules given above are the simplest
+	we have found.
+
+
+4. Relevant Information:
+    This data set includes descriptions of hypothetical samples
+    corresponding to 23 species of gilled mushrooms in the Agaricus and
+    Lepiota Family (pp. 500-525).  Each species is identified as
+    definitely edible, definitely poisonous, or of unknown edibility and
+    not recommended.  This latter class was combined with the poisonous
+    one.  The Guide clearly states that there is no simple rule for
+    determining the edibility of a mushroom; no rule like ``leaflets
+    three, let it be'' for Poisonous Oak and Ivy.
+
+5. Number of Instances: 8124
+
+6. Number of Attributes: 22 (all nominally valued)
+
+7. Attribute Information: (classes: edible=e, poisonous=p)
+     1. cap-shape:                bell=b,conical=c,convex=x,flat=f,
+                                  knobbed=k,sunken=s
+     2. cap-surface:              fibrous=f,grooves=g,scaly=y,smooth=s
+     3. cap-color:                brown=n,buff=b,cinnamon=c,gray=g,green=r,
+                                  pink=p,purple=u,red=e,white=w,yellow=y
+     4. bruises?:                 bruises=t,no=f
+     5. odor:                     almond=a,anise=l,creosote=c,fishy=y,foul=f,
+                                  musty=m,none=n,pungent=p,spicy=s
+     6. gill-attachment:          attached=a,descending=d,free=f,notched=n
+     7. gill-spacing:             close=c,crowded=w,distant=d
+     8. gill-size:                broad=b,narrow=n
+     9. gill-color:               black=k,brown=n,buff=b,chocolate=h,gray=g,
+                                  green=r,orange=o,pink=p,purple=u,red=e,
+                                  white=w,yellow=y
+    10. stalk-shape:              enlarging=e,tapering=t
+    11. stalk-root:               bulbous=b,club=c,cup=u,equal=e,
+                                  rhizomorphs=z,rooted=r,missing=?
+    12. stalk-surface-above-ring: fibrous=f,scaly=y,silky=k,smooth=s
+    13. stalk-surface-below-ring: fibrous=f,scaly=y,silky=k,smooth=s
+    14. stalk-color-above-ring:   brown=n,buff=b,cinnamon=c,gray=g,orange=o,
+                                  pink=p,red=e,white=w,yellow=y
+    15. stalk-color-below-ring:   brown=n,buff=b,cinnamon=c,gray=g,orange=o,
+                                  pink=p,red=e,white=w,yellow=y
+    16. veil-type:                partial=p,universal=u
+    17. veil-color:               brown=n,orange=o,white=w,yellow=y
+    18. ring-number:              none=n,one=o,two=t
+    19. ring-type:                cobwebby=c,evanescent=e,flaring=f,large=l,
+                                  none=n,pendant=p,sheathing=s,zone=z
+    20. spore-print-color:        black=k,brown=n,buff=b,chocolate=h,green=r,
+                                  orange=o,purple=u,white=w,yellow=y
+    21. population:               abundant=a,clustered=c,numerous=n,
+                                  scattered=s,several=v,solitary=y
+    22. habitat:                  grasses=g,leaves=l,meadows=m,paths=p,
+                                  urban=u,waste=w,woods=d
+
+8. Missing Attribute Values: 2480 of them (denoted by "?"), all for
+   attribute #11.
+
+9. Class Distribution: 
+    --    edible: 4208 (51.8%)
+    -- poisonous: 3916 (48.2%)
+    --     total: 8124 instances

demo/binary_classification/mapfeat.py

@@ -0,0 +1,50 @@
+#!/usr/bin/python
+import sys
+
+def loadfmap( fname ):
+    fmap = {}
+    nmap = {}
+    
+    for l in open( fname ):
+        arr = l.split()
+        if arr[0].find('.') != -1:            
+            idx = int( arr[0].strip('.') )
+            assert idx not in fmap        
+            fmap[ idx ] = {}
+            ftype = arr[1].strip(':')        
+            content = arr[2]
+        else:
+            content = arr[0]
+        for it in content.split(','):
+            if it.strip() == '':
+                continue
+            k , v = it.split('=')
+            fmap[ idx ][ v ] = len(nmap)
+            nmap[ len(nmap) ] = ftype+'='+k
+    return fmap, nmap
+
+def write_nmap( fo, nmap ):    
+    for i in range( len(nmap) ):
+        fo.write('%d\t%s\ti\n' % (i, nmap[i]) )
+
+# start here
+fmap, nmap = loadfmap( 'agaricus-lepiota.fmap' )
+fo = open( 'featmap.txt', 'w' )
+write_nmap( fo, nmap )
+fo.close()
+
+fo = open( 'agaricus.txt', 'w' ) 
+for l in open( 'agaricus-lepiota.data' ):
+    arr = l.split(',')
+    if arr[0] == 'p':
+        fo.write('1')
+    else:
+        assert arr[0] == 'e'
+        fo.write('0')
+    for i in range( 1,len(arr) ):
+        fo.write( ' %d:1' % fmap[i][arr[i].strip()] )
+    fo.write('\n')
+
+fo.close()
+
+ 

demo/binary_classification/mknfold.py

@@ -0,0 +1,29 @@
+#!/usr/bin/python
+import sys
+import random
+
+if len(sys.argv) < 2:
+    print ('Usage:<filename> <k> [nfold = 5]')
+    exit(0)
+
+random.seed( 10 )
+
+k = int( sys.argv[2] )
+if len(sys.argv) > 3:
+    nfold = int( sys.argv[3] )
+else:
+    nfold = 5
+
+fi = open( sys.argv[1], 'r' )
+ftr = open( sys.argv[1]+'.train', 'w' )
+fte = open( sys.argv[1]+'.test', 'w' )
+for l in fi:
+    if random.randint( 1 , nfold ) == k:
+        fte.write( l )
+    else:
+        ftr.write( l )
+
+fi.close()
+ftr.close()
+fte.close()
+

demo/binary_classification/mushroom.conf

@@ -0,0 +1,29 @@
+# General Parameters, see comment for each definition
+# choose the booster, can be gbtree or gblinear
+booster = gbtree
+# choose logistic regression loss function for binary classification
+objective = binary:logistic
+
+# Tree Booster Parameters
+# step size shrinkage
+eta = 1.0 
+# minimum loss reduction required to make a further partition
+gamma = 1.0 
+# minimum sum of instance weight(hessian) needed in a child
+min_child_weight = 1 
+# maximum depth of a tree
+max_depth = 3 
+
+# Task Parameters
+# the number of round to do boosting
+num_round = 2
+# 0 means do not save any model except the final round model
+save_period = 0 
+# The path of training data
+data = "agaricus.txt.train" 
+# The path of validation data, used to monitor training process, here [test] sets name of the validation set
+eval[test] = "agaricus.txt.test" 
+# evaluate on training data as well each round
+eval_train = 1
+# The path of test data 
+test:data = "agaricus.txt.test"      

demo/binary_classification/runexp.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# map feature using indicator encoding, also produce featmap.txt
+python mapfeat.py
+# split train and test
+python mknfold.py agaricus.txt 1
+# training and output the models
+../../xgboost mushroom.conf
+# output prediction task=pred 
+../../xgboost mushroom.conf task=pred model_in=0002.model
+# print the boosters of 00002.model in dump.raw.txt
+../../xgboost mushroom.conf task=dump model_in=0002.model name_dump=dump.raw.txt 
+# use the feature map in printing for better visualization
+../../xgboost mushroom.conf task=dump model_in=0002.model fmap=featmap.txt name_dump=dump.nice.txt
+cat dump.nice.txt
+

demo/data/README.md

@@ -0,0 +1,2 @@
+This folder contains processed example dataset used by the demos.
+Copyright of the dataset belongs to the original copyright holder

demo/data/featmap.txt

@@ -0,0 +1,126 @@
+0	cap-shape=bell	i
+1	cap-shape=conical	i
+2	cap-shape=convex	i
+3	cap-shape=flat	i
+4	cap-shape=knobbed	i
+5	cap-shape=sunken	i
+6	cap-surface=fibrous	i
+7	cap-surface=grooves	i
+8	cap-surface=scaly	i
+9	cap-surface=smooth	i
+10	cap-color=brown	i
+11	cap-color=buff	i
+12	cap-color=cinnamon	i
+13	cap-color=gray	i
+14	cap-color=green	i
+15	cap-color=pink	i
+16	cap-color=purple	i
+17	cap-color=red	i
+18	cap-color=white	i
+19	cap-color=yellow	i
+20	bruises?=bruises	i
+21	bruises?=no	i
+22	odor=almond	i
+23	odor=anise	i
+24	odor=creosote	i
+25	odor=fishy	i
+26	odor=foul	i
+27	odor=musty	i
+28	odor=none	i
+29	odor=pungent	i
+30	odor=spicy	i
+31	gill-attachment=attached	i
+32	gill-attachment=descending	i
+33	gill-attachment=free	i
+34	gill-attachment=notched	i
+35	gill-spacing=close	i
+36	gill-spacing=crowded	i
+37	gill-spacing=distant	i
+38	gill-size=broad	i
+39	gill-size=narrow	i
+40	gill-color=black	i
+41	gill-color=brown	i
+42	gill-color=buff	i
+43	gill-color=chocolate	i
+44	gill-color=gray	i
+45	gill-color=green	i
+46	gill-color=orange	i
+47	gill-color=pink	i
+48	gill-color=purple	i
+49	gill-color=red	i
+50	gill-color=white	i
+51	gill-color=yellow	i
+52	stalk-shape=enlarging	i
+53	stalk-shape=tapering	i
+54	stalk-root=bulbous	i
+55	stalk-root=club	i
+56	stalk-root=cup	i
+57	stalk-root=equal	i
+58	stalk-root=rhizomorphs	i
+59	stalk-root=rooted	i
+60	stalk-root=missing	i
+61	stalk-surface-above-ring=fibrous	i
+62	stalk-surface-above-ring=scaly	i
+63	stalk-surface-above-ring=silky	i
+64	stalk-surface-above-ring=smooth	i
+65	stalk-surface-below-ring=fibrous	i
+66	stalk-surface-below-ring=scaly	i
+67	stalk-surface-below-ring=silky	i
+68	stalk-surface-below-ring=smooth	i
+69	stalk-color-above-ring=brown	i
+70	stalk-color-above-ring=buff	i
+71	stalk-color-above-ring=cinnamon	i
+72	stalk-color-above-ring=gray	i
+73	stalk-color-above-ring=orange	i
+74	stalk-color-above-ring=pink	i
+75	stalk-color-above-ring=red	i
+76	stalk-color-above-ring=white	i
+77	stalk-color-above-ring=yellow	i
+78	stalk-color-below-ring=brown	i
+79	stalk-color-below-ring=buff	i
+80	stalk-color-below-ring=cinnamon	i
+81	stalk-color-below-ring=gray	i
+82	stalk-color-below-ring=orange	i
+83	stalk-color-below-ring=pink	i
+84	stalk-color-below-ring=red	i
+85	stalk-color-below-ring=white	i
+86	stalk-color-below-ring=yellow	i
+87	veil-type=partial	i
+88	veil-type=universal	i
+89	veil-color=brown	i
+90	veil-color=orange	i
+91	veil-color=white	i
+92	veil-color=yellow	i
+93	ring-number=none	i
+94	ring-number=one	i
+95	ring-number=two	i
+96	ring-type=cobwebby	i
+97	ring-type=evanescent	i
+98	ring-type=flaring	i
+99	ring-type=large	i
+100	ring-type=none	i
+101	ring-type=pendant	i
+102	ring-type=sheathing	i
+103	ring-type=zone	i
+104	spore-print-color=black	i
+105	spore-print-color=brown	i
+106	spore-print-color=buff	i
+107	spore-print-color=chocolate	i
+108	spore-print-color=green	i
+109	spore-print-color=orange	i
+110	spore-print-color=purple	i
+111	spore-print-color=white	i
+112	spore-print-color=yellow	i
+113	population=abundant	i
+114	population=clustered	i
+115	population=numerous	i
+116	population=scattered	i
+117	population=several	i
+118	population=solitary	i
+119	habitat=grasses	i
+120	habitat=leaves	i
+121	habitat=meadows	i
+122	habitat=paths	i
+123	habitat=urban	i
+124	habitat=waste	i
+125	habitat=woods	i

demo/guide-python/README.md

@@ -0,0 +1,8 @@
+XGBoost Python Feature Walkthrough
+====
+* [Basic walkthrough of wrappers](basic_walkthrough.py) 
+* [Cutomize loss function, and evaluation metric](custom_objective.py)
+* [Boosting from existing prediction](boost_from_prediction.py)
+* [Predicting using first n trees](predict_first_ntree.py)
+* [Generalized Linear Model](generalized_linear_model.py)
+* [Cross validation](cross_validation.py)

demo/guide-python/basic_walkthrough.py

@@ -0,0 +1,76 @@
+#!/usr/bin/python
+import sys
+import numpy as np
+import scipy.sparse
+# append the path to xgboost, you may need to change the following line
+# alternatively, you can add the path to PYTHONPATH environment variable
+sys.path.append('../../wrapper')
+import xgboost as xgb
+
+### simple example
+# load file from text file, also binary buffer generated by xgboost
+dtrain = xgb.DMatrix('../data/agaricus.txt.train')
+dtest = xgb.DMatrix('../data/agaricus.txt.test')
+
+# specify parameters via map, definition are same as c++ version
+param = {'max_depth':2, 'eta':1, 'silent':1, 'objective':'binary:logistic' }
+
+# specify validations set to watch performance
+watchlist  = [(dtest,'eval'), (dtrain,'train')]
+num_round = 2
+bst = xgb.train(param, dtrain, num_round, watchlist)
+
+# this is prediction
+preds = bst.predict(dtest)
+labels = dtest.get_label()
+print ('error=%f' % (  sum(1 for i in range(len(preds)) if int(preds[i]>0.5)!=labels[i]) /float(len(preds))))
+bst.save_model('0001.model')
+# dump model
+bst.dump_model('dump.raw.txt')
+# dump model with feature map
+bst.dump_model('dump.nice.txt','../data/featmap.txt')
+
+# save dmatrix into binary buffer
+dtest.save_binary('dtest.buffer')
+bst.save_model('xgb.model')
+# load model and data in 
+bst2 = xgb.Booster(model_file='xgb.model')
+dtest2 = xgb.DMatrix('dtest.buffer')
+preds2 = bst2.predict(dtest2)
+# assert they are the same
+assert np.sum(np.abs(preds2-preds)) == 0
+
+###
+# build dmatrix from scipy.sparse
+print ('start running example of build DMatrix from scipy.sparse CSR Matrix')
+labels = []
+row = []; col = []; dat = []
+i = 0
+for l in open('../data/agaricus.txt.train'):
+    arr = l.split()
+    labels.append( int(arr[0]))
+    for it in arr[1:]:
+        k,v = it.split(':')
+        row.append(i); col.append(int(k)); dat.append(float(v))
+    i += 1
+csr = scipy.sparse.csr_matrix( (dat, (row,col)) )
+dtrain = xgb.DMatrix( csr, label = labels )
+watchlist  = [(dtest,'eval'), (dtrain,'train')]
+bst = xgb.train( param, dtrain, num_round, watchlist )
+
+print ('start running example of build DMatrix from scipy.sparse CSC Matrix')
+# we can also construct from csc matrix
+csc = scipy.sparse.csc_matrix( (dat, (row,col)) )
+dtrain = xgb.DMatrix(csc, label=labels)
+watchlist  = [(dtest,'eval'), (dtrain,'train')]
+bst = xgb.train( param, dtrain, num_round, watchlist )
+
+print ('start running example of build DMatrix from numpy array')
+# NOTE: npymat is numpy array, we will convert it into scipy.sparse.csr_matrix in internal implementation
+# then convert to DMatrix
+npymat = csr.todense()
+dtrain = xgb.DMatrix(npymat, label = labels)
+watchlist  = [(dtest,'eval'), (dtrain,'train')]
+bst = xgb.train( param, dtrain, num_round, watchlist )
+
+

demo/guide-python/boost_from_prediction.py

@@ -0,0 +1,26 @@
+#!/usr/bin/python
+import sys
+import numpy as np
+sys.path.append('../../wrapper')
+import xgboost as xgb
+
+dtrain = xgb.DMatrix('../data/agaricus.txt.train')
+dtest = xgb.DMatrix('../data/agaricus.txt.test')
+watchlist  = [(dtest,'eval'), (dtrain,'train')]
+###
+# advanced: start from a initial base prediction
+#
+print ('start running example to start from a initial prediction')
+# specify parameters via map, definition are same as c++ version
+param = {'max_depth':2, 'eta':1, 'silent':1, 'objective':'binary:logistic' }
+# train xgboost for 1 round
+bst = xgb.train( param, dtrain, 1, watchlist )
+# Note: we need the margin value instead of transformed prediction in set_base_margin
+# do predict with output_margin=True, will always give you margin values before logistic transformation
+ptrain = bst.predict(dtrain, output_margin=True)
+ptest  = bst.predict(dtest, output_margin=True)
+dtrain.set_base_margin(ptrain)
+dtest.set_base_margin(ptest)
+
+print ('this is result of running from initial prediction')
+bst = xgb.train( param, dtrain, 1, watchlist )

demo/guide-python/cross_validation.py

@@ -0,0 +1,63 @@
+#!/usr/bin/python
+import sys
+import numpy as np
+sys.path.append('../../wrapper')
+import xgboost as xgb
+
+### load data in do training
+dtrain = xgb.DMatrix('../data/agaricus.txt.train')
+param = {'max_depth':2, 'eta':1, 'silent':1, 'objective':'binary:logistic'}
+num_round = 2
+
+print ('running cross validation')
+# do cross validation, this will print result out as
+# [iteration]  metric_name:mean_value+std_value
+# std_value is standard deviation of the metric
+xgb.cv(param, dtrain, num_round, nfold=5,
+       metrics={'error'}, seed = 0)
+
+print ('running cross validation, disable standard deviation display')
+# do cross validation, this will print result out as
+# [iteration]  metric_name:mean_value+std_value
+# std_value is standard deviation of the metric
+xgb.cv(param, dtrain, num_round, nfold=5,
+       metrics={'error'}, seed = 0, show_stdv = False)
+
+print ('running cross validation, with preprocessing function')
+# define the preprocessing function
+# used to return the preprocessed training, test data, and parameter
+# we can use this to do weight rescale, etc.
+# as a example, we try to set scale_pos_weight
+def fpreproc(dtrain, dtest, param):
+    label = dtrain.get_label()
+    ratio = float(np.sum(label == 0)) / np.sum(label==1)
+    param['scale_pos_weight'] = ratio
+    return (dtrain, dtest, param)
+
+# do cross validation, for each fold
+# the dtrain, dtest, param will be passed into fpreproc
+# then the return value of fpreproc will be used to generate
+# results of that fold
+xgb.cv(param, dtrain, num_round, nfold=5,
+       metrics={'auc'}, seed = 0, fpreproc = fpreproc)
+
+###
+# you can also do cross validation with cutomized loss function
+# See custom_objective.py
+##
+print ('running cross validation, with cutomsized loss function')
+def logregobj(preds, dtrain):
+    labels = dtrain.get_label()
+    preds = 1.0 / (1.0 + np.exp(-preds))
+    grad = preds - labels
+    hess = preds * (1.0-preds)
+    return grad, hess
+def evalerror(preds, dtrain):
+    labels = dtrain.get_label()
+    return 'error', float(sum(labels != (preds > 0.0))) / len(labels)
+
+param = {'max_depth':2, 'eta':1, 'silent':1} 
+# train with customized objective
+xgb.cv(param, dtrain, num_round, nfold = 5, seed = 0,
+       obj = logregobj, feval=evalerror)
+

demo/guide-python/custom_objective.py

@@ -0,0 +1,44 @@
+#!/usr/bin/python
+import sys
+import numpy as np
+sys.path.append('../../wrapper')
+import xgboost as xgb
+###
+# advanced: cutomsized loss function
+# 
+print ('start running example to used cutomized objective function')
+
+dtrain = xgb.DMatrix('../data/agaricus.txt.train')
+dtest = xgb.DMatrix('../data/agaricus.txt.test')
+
+# note: for customized objective function, we leave objective as default
+# note: what we are getting is margin value in prediction
+# you must know what you are doing
+param = {'max_depth':2, 'eta':1, 'silent':1 }
+watchlist  = [(dtest,'eval'), (dtrain,'train')]
+num_round = 2
+
+# user define objective function, given prediction, return gradient and second order gradient
+# this is loglikelihood loss
+def logregobj(preds, dtrain):
+    labels = dtrain.get_label()
+    preds = 1.0 / (1.0 + np.exp(-preds))
+    grad = preds - labels
+    hess = preds * (1.0-preds)
+    return grad, hess
+
+# user defined evaluation function, return a pair metric_name, result
+# NOTE: when you do customized loss function, the default prediction value is margin
+# this may make buildin evalution metric not function properly
+# for example, we are doing logistic loss, the prediction is score before logistic transformation
+# the buildin evaluation error assumes input is after logistic transformation
+# Take this in mind when you use the customization, and maybe you need write customized evaluation function
+def evalerror(preds, dtrain):
+    labels = dtrain.get_label()
+    # return a pair metric_name, result
+    # since preds are margin(before logistic transformation, cutoff at 0)
+    return 'error', float(sum(labels != (preds > 0.0))) / len(labels)
+
+# training with customized objective, we can also do step by step training
+# simply look at xgboost.py's implementation of train
+bst = xgb.train(param, dtrain, num_round, watchlist, logregobj, evalerror)

demo/guide-python/generalized_linear_model.py

@@ -0,0 +1,32 @@
+#!/usr/bin/python
+import sys
+sys.path.append('../../wrapper')
+import xgboost as xgb
+##
+#  this script demonstrate how to fit generalized linear model in xgboost
+#  basically, we are using linear model, instead of tree for our boosters
+##
+dtrain = xgb.DMatrix('../data/agaricus.txt.train')
+dtest = xgb.DMatrix('../data/agaricus.txt.test')
+# change booster to gblinear, so that we are fitting a linear model
+# alpha is the L1 regularizer 
+# lambda is the L2 regularizer
+# you can also set lambda_bias which is L2 regularizer on the bias term
+param = {'silent':1, 'objective':'binary:logistic', 'booster':'gblinear',
+         'alpha': 0.0001, 'lambda': 1 }
+
+# normally, you do not need to set eta (step_size)
+# XGBoost uses a parallel coordinate descent algorithm (shotgun), 
+# there could be affection on convergence with parallelization on certain cases
+# setting eta to be smaller value, e.g 0.5 can make the optimization more stable
+# param['eta'] = 1 
+
+##
+# the rest of settings are the same
+##
+watchlist  = [(dtest,'eval'), (dtrain,'train')]
+num_round = 4
+bst = xgb.train(param, dtrain, num_round, watchlist)
+preds = bst.predict(dtest)
+labels = dtest.get_label()
+print ('error=%f' % ( sum(1 for i in range(len(preds)) if int(preds[i]>0.5)!=labels[i]) /float(len(preds))))

demo/guide-python/predict_first_ntree.py

@@ -0,0 +1,22 @@
+#!/usr/bin/python
+import sys
+import numpy as np
+sys.path.append('../../wrapper')
+import xgboost as xgb
+
+### load data in do training
+dtrain = xgb.DMatrix('../data/agaricus.txt.train')
+dtest = xgb.DMatrix('../data/agaricus.txt.test')
+param = {'max_depth':2, 'eta':1, 'silent':1, 'objective':'binary:logistic' }
+watchlist  = [(dtest,'eval'), (dtrain,'train')]
+num_round = 3
+bst = xgb.train(param, dtrain, num_round, watchlist)
+
+print ('start testing prediction from first n trees')
+### predict using first 1 tree
+label = dtest.get_label()
+ypred1 = bst.predict(dtest, ntree_limit=1)
+# by default, we predict using all the trees
+ypred2 = bst.predict(dtest)
+print ('error of ypred1=%f' % (np.sum((ypred1>0.5)!=label) /float(len(label))))
+print ('error of ypred2=%f' % (np.sum((ypred2>0.5)!=label) /float(len(label))))

demo/guide-python/runall.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+python basic_walkthrough.py
+python custom_objective.py
+python boost_from_prediction.py
+python generalized_linear_model.py
+python cross_validation.py
+rm -rf *~ *.model *.buffer 

demo/kaggle-higgs/README.md

@@ -0,0 +1,26 @@
+Guide for Kaggle Higgs Challenge
+=====
+
+This is the folder giving example of how to use XGBoost Python Module  to run Kaggle Higgs competition
+
+This script will achieve about 3.600 AMS score in public leadboard. To get start, you need do following step:
+
+1. Compile the XGBoost python lib
+```bash
+cd ../..
+make
+```
+
+2. Put training.csv test.csv on folder './data' (you can create a symbolic link)
+
+3. Run ./run.sh
+
+Speed
+=====
+speedtest.py compares xgboost's speed on this dataset with sklearn.GBM
+
+
+Using R module
+=====
+* Alternatively, you can run using R, higgs-train.R and higgs-pred.R. 
+

demo/kaggle-higgs/higgs-cv.py

@@ -0,0 +1,39 @@
+#!/usr/bin/python
+import sys
+import numpy as np
+sys.path.append('../../wrapper')
+import xgboost as xgb
+
+### load data in do training
+train = np.loadtxt('./data/training.csv', delimiter=',', skiprows=1, converters={32: lambda x:int(x=='s'.encode('utf-8')) } )
+label  = train[:,32]
+data   = train[:,1:31]
+weight = train[:,31]
+dtrain = xgb.DMatrix( data, label=label, missing = -999.0, weight=weight )
+param = {'max_depth':6, 'eta':0.1, 'silent':1, 'objective':'binary:logitraw', 'nthread':4}
+num_round = 120
+
+print ('running cross validation, with preprocessing function')
+# define the preprocessing function
+# used to return the preprocessed training, test data, and parameter
+# we can use this to do weight rescale, etc.
+# as a example, we try to set scale_pos_weight
+def fpreproc(dtrain, dtest, param):
+    label = dtrain.get_label()
+    ratio = float(np.sum(label == 0)) / np.sum(label==1)
+    param['scale_pos_weight'] = ratio
+    wtrain = dtrain.get_weight()
+    wtest = dtest.get_weight()
+    sum_weight = sum(wtrain) + sum(wtest)
+    wtrain *= sum_weight / sum(wtrain)
+    wtest *= sum_weight / sum(wtest)
+    dtrain.set_weight(wtrain)
+    dtest.set_weight(wtest)
+    return (dtrain, dtest, param)
+
+# do cross validation, for each fold
+# the dtrain, dtest, param will be passed into fpreproc
+# then the return value of fpreproc will be used to generate
+# results of that fold
+xgb.cv(param, dtrain, num_round, nfold=5,
+       metrics={'ams@0.15', 'auc'}, seed = 0, fpreproc = fpreproc)

demo/kaggle-higgs/higgs-numpy.py

@@ -0,0 +1,62 @@
+#!/usr/bin/python
+# this is the example script to use xgboost to train 
+import inspect
+import os
+import sys
+import numpy as np
+# add path of xgboost python module
+code_path = os.path.join(
+    os.path.split(inspect.getfile(inspect.currentframe()))[0], "../../wrapper")
+
+sys.path.append(code_path)
+
+import xgboost as xgb
+
+test_size = 550000
+
+# path to where the data lies
+dpath = 'data'
+
+# load in training data, directly use numpy
+dtrain = np.loadtxt( dpath+'/training.csv', delimiter=',', skiprows=1, converters={32: lambda x:int(x=='s'.encode('utf-8')) } )
+print ('finish loading from csv ')
+
+label  = dtrain[:,32]
+data   = dtrain[:,1:31]
+# rescale weight to make it same as test set
+weight = dtrain[:,31] * float(test_size) / len(label)
+
+sum_wpos = sum( weight[i] for i in range(len(label)) if label[i] == 1.0  )
+sum_wneg = sum( weight[i] for i in range(len(label)) if label[i] == 0.0  )
+
+# print weight statistics 
+print ('weight statistics: wpos=%g, wneg=%g, ratio=%g' % ( sum_wpos, sum_wneg, sum_wneg/sum_wpos ))
+
+# construct xgboost.DMatrix from numpy array, treat -999.0 as missing value
+xgmat = xgb.DMatrix( data, label=label, missing = -999.0, weight=weight )
+
+# setup parameters for xgboost
+param = {}
+# use logistic regression loss, use raw prediction before logistic transformation
+# since we only need the rank
+param['objective'] = 'binary:logitraw'
+# scale weight of positive examples
+param['scale_pos_weight'] = sum_wneg/sum_wpos
+param['eta'] = 0.1 
+param['max_depth'] = 6
+param['eval_metric'] = 'auc'
+param['silent'] = 1
+param['nthread'] = 16
+
+# you can directly throw param in, though we want to watch multiple metrics here 
+plst = list(param.items())+[('eval_metric', 'ams@0.15')]
+
+watchlist = [ (xgmat,'train') ]
+# boost 120 tres
+num_round = 120
+print ('loading data end, start to boost trees')
+bst = xgb.train( plst, xgmat, num_round, watchlist );
+# save out model
+bst.save_model('higgs.model')
+
+print ('finish training')

demo/kaggle-higgs/higgs-pred.R

@@ -0,0 +1,24 @@
+# install xgboost package, see R-package in root folder
+require(xgboost)
+require(methods)
+
+modelfile <- "higgs.model"
+outfile <- "higgs.pred.csv"
+dtest <- read.csv("data/test.csv", header=TRUE)
+data <- as.matrix(dtest[2:31])
+idx <- dtest[[1]]
+
+xgmat <- xgb.DMatrix(data, missing = -999.0)
+bst <- xgb.load(modelfile=modelfile)
+ypred <- predict(bst, xgmat)
+
+rorder <- rank(ypred, ties.method="first")
+
+threshold <- 0.15
+# to be completed
+ntop <- length(rorder) - as.integer(threshold*length(rorder))
+plabel <- ifelse(rorder > ntop, "s", "b")
+outdata <- list("EventId" = idx,
+                "RankOrder" = rorder,
+                "Class" = plabel)
+write.csv(outdata, file = outfile, quote=FALSE, row.names=FALSE)

demo/kaggle-higgs/higgs-pred.py

@@ -0,0 +1,53 @@
+#!/usr/bin/python
+# make prediction 
+import sys
+import numpy as np
+# add path of xgboost python module
+sys.path.append('../../wrapper/')
+import xgboost as xgb
+
+# path to where the data lies
+dpath = 'data'
+
+modelfile = 'higgs.model'
+outfile = 'higgs.pred.csv'
+# make top 15% as positive 
+threshold_ratio = 0.15
+
+# load in training data, directly use numpy
+dtest = np.loadtxt( dpath+'/test.csv', delimiter=',', skiprows=1 )
+data   = dtest[:,1:31]
+idx = dtest[:,0]
+
+print ('finish loading from csv ')
+xgmat = xgb.DMatrix( data, missing = -999.0 )
+bst = xgb.Booster({'nthread':16}, model_file = modelfile)
+ypred = bst.predict( xgmat )
+
+res  = [ ( int(idx[i]), ypred[i] ) for i in range(len(ypred)) ] 
+
+rorder = {}
+for k, v in sorted( res, key = lambda x:-x[1] ):
+    rorder[ k ] = len(rorder) + 1
+
+# write out predictions
+ntop = int( threshold_ratio * len(rorder ) )
+fo = open(outfile, 'w')
+nhit = 0
+ntot = 0
+fo.write('EventId,RankOrder,Class\n')
+for k, v in res:        
+    if rorder[k] <= ntop:
+        lb = 's'
+        nhit += 1
+    else:
+        lb = 'b'        
+    # change output rank order to follow Kaggle convention
+    fo.write('%s,%d,%s\n' % ( k,  len(rorder)+1-rorder[k], lb ) )
+    ntot += 1
+fo.close()
+
+print ('finished writing into prediction file')
+
+
+

demo/kaggle-higgs/higgs-train.R

@@ -0,0 +1,33 @@
+# install xgboost package, see R-package in root folder
+require(xgboost)
+require(methods)
+
+testsize <- 550000
+
+dtrain <- read.csv("data/training.csv", header=TRUE)
+dtrain[33] <- dtrain[33] == "s"
+label <- as.numeric(dtrain[[33]])
+data <- as.matrix(dtrain[2:31])
+weight <- as.numeric(dtrain[[32]]) * testsize / length(label)
+
+sumwpos <- sum(weight * (label==1.0))
+sumwneg <- sum(weight * (label==0.0))
+print(paste("weight statistics: wpos=", sumwpos, "wneg=", sumwneg, "ratio=", sumwneg / sumwpos))
+
+xgmat <- xgb.DMatrix(data, label = label, weight = weight, missing = -999.0)
+param <- list("objective" = "binary:logitraw",
+              "scale_pos_weight" = sumwneg / sumwpos,
+              "bst:eta" = 0.1,
+              "bst:max_depth" = 6,
+              "eval_metric" = "auc",
+              "eval_metric" = "ams@0.15",
+              "silent" = 1,
+              "nthread" = 16)
+watchlist <- list("train" = xgmat)
+nround = 120
+print ("loading data end, start to boost trees")
+bst = xgb.train(param, xgmat, nround, watchlist );
+# save out model
+xgb.save(bst, "higgs.model")
+print ('finish training')
+

demo/kaggle-higgs/run.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+python -u higgs-numpy.py
+ret=$?
+if [[ $ret != 0 ]]; then
+    echo "ERROR in higgs-numpy.py"
+    exit $ret
+fi
+python -u higgs-pred.py
+ret=$?
+if [[ $ret != 0 ]]; then
+    echo "ERROR in higgs-pred.py"
+    exit $ret
+fi

demo/kaggle-higgs/speedtest.R

@@ -0,0 +1,71 @@
+# install xgboost package, see R-package in root folder
+require(xgboost)
+require(gbm)
+require(methods)
+
+testsize <- 550000
+
+dtrain <- read.csv("data/training.csv", header=TRUE, nrows=350001)
+
+# gbm.time = system.time({
+#   gbm.model <- gbm(Label ~ ., data = dtrain[, -c(1,32)], n.trees = 120, 
+#                    interaction.depth = 6, shrinkage = 0.1, bag.fraction = 1,
+#                    verbose = TRUE)
+# })
+# print(gbm.time)
+# Test result: 761.48 secs
+
+dtrain[33] <- dtrain[33] == "s"
+label <- as.numeric(dtrain[[33]])
+data <- as.matrix(dtrain[2:31])
+weight <- as.numeric(dtrain[[32]]) * testsize / length(label)
+
+sumwpos <- sum(weight * (label==1.0))
+sumwneg <- sum(weight * (label==0.0))
+print(paste("weight statistics: wpos=", sumwpos, "wneg=", sumwneg, "ratio=", sumwneg / sumwpos))
+
+xgboost.time = list()
+threads = c(1,2,4,8,16)
+for (i in 1:length(threads)){
+  thread = threads[i]
+  xgboost.time[[i]] = system.time({
+    xgmat <- xgb.DMatrix(data, label = label, weight = weight, missing = -999.0)
+    param <- list("objective" = "binary:logitraw",
+                  "scale_pos_weight" = sumwneg / sumwpos,
+                  "bst:eta" = 0.1,
+                  "bst:max_depth" = 6,
+                  "eval_metric" = "auc",
+                  "eval_metric" = "ams@0.15",
+                  "silent" = 1,
+                  "nthread" = thread)
+    watchlist <- list("train" = xgmat)
+    nround = 120
+    print ("loading data end, start to boost trees")
+    bst = xgb.train(param, xgmat, nround, watchlist );
+    # save out model
+    xgb.save(bst, "higgs.model")
+    print ('finish training')
+  })
+}
+
+xgboost.time
+# [[1]]
+# user  system elapsed 
+# 444.98    1.96  450.22 
+# 
+# [[2]]
+# user  system elapsed 
+# 188.15    0.82  102.41 
+# 
+# [[3]]
+# user  system elapsed 
+# 143.29    0.79   44.18 
+# 
+# [[4]]
+# user  system elapsed 
+# 176.60    1.45   34.04 
+# 
+# [[5]]
+# user  system elapsed 
+# 180.15    2.85   35.26 
+

demo/kaggle-higgs/speedtest.py

@@ -0,0 +1,66 @@
+#!/usr/bin/python
+# this is the example script to use xgboost to train
+import sys
+import numpy as np
+# add path of xgboost python module
+sys.path.append('../../wrapper/')
+import xgboost as xgb
+from sklearn.ensemble import GradientBoostingClassifier
+import time
+test_size = 550000
+
+# path to where the data lies
+dpath = 'data'
+
+# load in training data, directly use numpy
+dtrain = np.loadtxt( dpath+'/training.csv', delimiter=',', skiprows=1, converters={32: lambda x:int(x=='s') } )
+print ('finish loading from csv ')
+
+label  = dtrain[:,32]
+data   = dtrain[:,1:31]
+# rescale weight to make it same as test set
+weight = dtrain[:,31] * float(test_size) / len(label)
+
+sum_wpos = sum( weight[i] for i in range(len(label)) if label[i] == 1.0  )
+sum_wneg = sum( weight[i] for i in range(len(label)) if label[i] == 0.0  )
+
+# print weight statistics
+print ('weight statistics: wpos=%g, wneg=%g, ratio=%g' % ( sum_wpos, sum_wneg, sum_wneg/sum_wpos ))
+
+# construct xgboost.DMatrix from numpy array, treat -999.0 as missing value
+xgmat = xgb.DMatrix( data, label=label, missing = -999.0, weight=weight )
+
+# setup parameters for xgboost
+param = {}
+# use logistic regression loss
+param['objective'] = 'binary:logitraw'
+# scale weight of positive examples
+param['scale_pos_weight'] = sum_wneg/sum_wpos
+param['bst:eta'] = 0.1
+param['bst:max_depth'] = 6
+param['eval_metric'] = 'auc'
+param['silent'] = 1
+param['nthread'] = 4
+
+plst = param.items()+[('eval_metric', 'ams@0.15')]
+
+watchlist = [ (xgmat,'train') ]
+# boost 10 tres
+num_round = 10
+print ('loading data end, start to boost trees')
+print ("training GBM from sklearn")
+tmp = time.time()
+gbm = GradientBoostingClassifier(n_estimators=num_round, max_depth=6, verbose=2)
+gbm.fit(data, label)
+print ("sklearn.GBM costs: %s seconds" % str(time.time() - tmp))
+#raw_input()
+print ("training xgboost")
+threads = [1, 2, 4, 16]
+for i in threads:
+    param['nthread'] = i
+    tmp = time.time()
+    plst = param.items()+[('eval_metric', 'ams@0.15')]
+    bst = xgb.train( plst, xgmat, num_round, watchlist );
+    print ("XGBoost with %d thread costs: %s seconds" % (i, str(time.time() - tmp)))
+
+print ('finish training')

demo/multiclass_classification/README.md

@@ -0,0 +1,10 @@
+Demonstrating how to use XGBoost accomplish Multi-Class classification task on [UCI Dermatology dataset](https://archive.ics.uci.edu/ml/datasets/Dermatology)
+
+Make sure you make make xgboost python module in ../../python
+
+1. Run runexp.sh
+```bash
+./runexp.sh
+```
+
+Explainations can be found in [wiki](https://github.com/tqchen/xgboost/wiki)

demo/multiclass_classification/runexp.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+if [ -f dermatology.data ]
+then
+    echo "use existing data to run multi class classification"
+else
+    echo "getting data from uci, make sure you are connected to internet"
+    wget https://archive.ics.uci.edu/ml/machine-learning-databases/dermatology/dermatology.data
+fi
+python train.py

demo/multiclass_classification/train.py

@@ -0,0 +1,50 @@
+#! /usr/bin/python
+import sys
+import numpy as np
+sys.path.append('../../wrapper/')
+import xgboost as xgb
+
+# label need to be 0 to num_class -1
+data = np.loadtxt('./dermatology.data', delimiter=',',converters={33: lambda x:int(x == '?'), 34: lambda x:int(x)-1 } )
+sz = data.shape
+
+train = data[:int(sz[0] * 0.7), :]
+test = data[int(sz[0] * 0.7):, :]
+
+train_X = train[:,0:33]
+train_Y = train[:, 34]
+
+
+test_X = test[:,0:33]
+test_Y = test[:, 34]
+
+xg_train = xgb.DMatrix( train_X, label=train_Y)
+xg_test = xgb.DMatrix(test_X, label=test_Y)
+# setup parameters for xgboost
+param = {}
+# use softmax multi-class classification
+param['objective'] = 'multi:softmax'
+# scale weight of positive examples
+param['eta'] = 0.1
+param['max_depth'] = 6
+param['silent'] = 1
+param['nthread'] = 4
+param['num_class'] = 6
+
+watchlist = [ (xg_train,'train'), (xg_test, 'test') ]
+num_round = 5
+bst = xgb.train(param, xg_train, num_round, watchlist );
+# get prediction
+pred = bst.predict( xg_test );
+
+print ('predicting, classification error=%f' % (sum( int(pred[i]) != test_Y[i] for i in range(len(test_Y))) / float(len(test_Y)) ))
+
+# do the same thing again, but output probabilities
+param['objective'] = 'multi:softprob'
+bst = xgb.train(param, xg_train, num_round, watchlist );
+# Note: this convention has been changed since xgboost-unity
+# get prediction, this is in 1D array, need reshape to (ndata, nclass)
+yprob = bst.predict( xg_test ).reshape( test_Y.shape[0], 6 )
+ylabel = np.argmax(yprob, axis=1)
+
+print ('predicting, classification error=%f' % (sum( int(ylabel[i]) != test_Y[i] for i in range(len(test_Y))) / float(len(test_Y)) ))

demo/rank/README

@@ -0,0 +1,13 @@
+Instructions:
+The dataset for ranking demo is from LETOR04 MQ2008 fold1, 
+You can use the following command to run the example
+
+
+Get the data: ./wgetdata.sh
+Run the example: ./runexp.sh
+
+
+
+
+
+

demo/rank/mq2008.conf

@@ -0,0 +1,28 @@
+# General Parameters, see comment for each definition
+
+# specify objective
+objective="rank:pairwise"
+
+# Tree Booster Parameters
+# step size shrinkage
+eta = 0.1 
+# minimum loss reduction required to make a further partition
+gamma = 1.0 
+# minimum sum of instance weight(hessian) needed in a child
+min_child_weight = 0.1
+# maximum depth of a tree
+max_depth = 6
+
+# Task parameters
+# the number of round to do boosting
+num_round = 4
+# 0 means do not save any model except the final round model
+save_period = 0 
+# The path of training data
+data = "mq2008.train" 
+# The path of validation data, used to monitor training process, here [test] sets name of the validation set
+eval[test] = "mq2008.vali" 
+# The path of test data 
+test:data = "mq2008.test"      
+
+

demo/rank/runexp.sh

@@ -0,0 +1,11 @@
+python trans_data.py train.txt mq2008.train mq2008.train.group
+
+python trans_data.py test.txt mq2008.test mq2008.test.group
+
+python trans_data.py vali.txt mq2008.vali mq2008.vali.group
+
+../../xgboost mq2008.conf
+
+../../xgboost mq2008.conf task=pred model_in=0004.model
+
+

demo/rank/trans_data.py

@@ -0,0 +1,41 @@
+import sys
+
+def save_data(group_data,output_feature,output_group):
+    if len(group_data) == 0:
+        return
+
+    output_group.write(str(len(group_data))+"\n")
+    for data in group_data:
+        # only include nonzero features
+        feats = [ p for p in data[2:] if float(p.split(':')[1]) != 0.0 ]        
+        output_feature.write(data[0] + " " + " ".join(feats) + "\n")
+
+if __name__ == "__main__":
+    if len(sys.argv) != 4:
+        print ("Usage: python trans_data.py [Ranksvm Format Input] [Output Feature File] [Output Group File]")
+        sys.exit(0)
+
+    fi = open(sys.argv[1])
+    output_feature = open(sys.argv[2],"w")
+    output_group = open(sys.argv[3],"w")
+    
+    group_data = []
+    group = ""
+    for line in fi:
+        if not line:
+            break
+        if "#" in line:
+            line = line[:line.index("#")]
+        splits = line.strip().split(" ")
+        if splits[1] != group:
+            save_data(group_data,output_feature,output_group)
+            group_data = []
+        group = splits[1]
+        group_data.append(splits)
+
+    save_data(group_data,output_feature,output_group)
+
+    fi.close()
+    output_feature.close()
+    output_group.close()
+

demo/rank/wgetdata.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+wget http://research.microsoft.com/en-us/um/beijing/projects/letor/LETOR4.0/Data/MQ2008.rar
+unrar x MQ2008.rar
+mv -f MQ2008/Fold1/*.txt .

demo/regression/README

@@ -0,0 +1,13 @@
+Demonstrating how to use XGBoost accomplish regression tasks on computer hardware dataset https://archive.ics.uci.edu/ml/datasets/Computer+Hardware
+
+Run: ./runexp.sh
+
+Format of input: LIBSVM format
+
+Format of ```featmap.txt: <featureid> <featurename> <q or i or int>\n ```:
+  - Feature id must be from 0 to number of features, in sorted order.
+  - i means this feature is binary indicator feature
+  - q means this feature is a quantitative value, such as age, time, can be missing
+  - int means this feature is integer value (when int is hinted, the decision boundary will be integer)
+
+Explainations: https://github.com/tqchen/xgboost/wiki/Regression

demo/regression/machine.conf

@@ -0,0 +1,30 @@
+# General Parameters, see comment for each definition
+# choose the tree booster, can also change to gblinear
+booster = gbtree
+# this is the only difference with classification, use reg:linear to do linear classification
+# when labels are in [0,1] we can also use reg:logistic
+objective = reg:linear
+
+# Tree Booster Parameters
+# step size shrinkage
+eta = 1.0 
+# minimum loss reduction required to make a further partition
+gamma = 1.0 
+# minimum sum of instance weight(hessian) needed in a child
+min_child_weight = 1 
+# maximum depth of a tree
+max_depth = 3 
+
+# Task parameters
+# the number of round to do boosting
+num_round = 2
+# 0 means do not save any model except the final round model
+save_period = 0 
+# The path of training data
+data = "machine.txt.train" 
+# The path of validation data, used to monitor training process, here [test] sets name of the validation set
+eval[test] = "machine.txt.test" 
+# The path of test data 
+test:data = "machine.txt.test"      
+
+

demo/regression/machine.data

@@ -0,0 +1,209 @@
+adviser,32/60,125,256,6000,256,16,128,198,199
+amdahl,470v/7,29,8000,32000,32,8,32,269,253
+amdahl,470v/7a,29,8000,32000,32,8,32,220,253
+amdahl,470v/7b,29,8000,32000,32,8,32,172,253
+amdahl,470v/7c,29,8000,16000,32,8,16,132,132
+amdahl,470v/b,26,8000,32000,64,8,32,318,290
+amdahl,580-5840,23,16000,32000,64,16,32,367,381
+amdahl,580-5850,23,16000,32000,64,16,32,489,381
+amdahl,580-5860,23,16000,64000,64,16,32,636,749
+amdahl,580-5880,23,32000,64000,128,32,64,1144,1238
+apollo,dn320,400,1000,3000,0,1,2,38,23
+apollo,dn420,400,512,3500,4,1,6,40,24
+basf,7/65,60,2000,8000,65,1,8,92,70
+basf,7/68,50,4000,16000,65,1,8,138,117
+bti,5000,350,64,64,0,1,4,10,15
+bti,8000,200,512,16000,0,4,32,35,64
+burroughs,b1955,167,524,2000,8,4,15,19,23
+burroughs,b2900,143,512,5000,0,7,32,28,29
+burroughs,b2925,143,1000,2000,0,5,16,31,22
+burroughs,b4955,110,5000,5000,142,8,64,120,124
+burroughs,b5900,143,1500,6300,0,5,32,30,35
+burroughs,b5920,143,3100,6200,0,5,20,33,39
+burroughs,b6900,143,2300,6200,0,6,64,61,40
+burroughs,b6925,110,3100,6200,0,6,64,76,45
+c.r.d,68/10-80,320,128,6000,0,1,12,23,28
+c.r.d,universe:2203t,320,512,2000,4,1,3,69,21
+c.r.d,universe:68,320,256,6000,0,1,6,33,28
+c.r.d,universe:68/05,320,256,3000,4,1,3,27,22
+c.r.d,universe:68/137,320,512,5000,4,1,5,77,28
+c.r.d,universe:68/37,320,256,5000,4,1,6,27,27
+cdc,cyber:170/750,25,1310,2620,131,12,24,274,102
+cdc,cyber:170/760,25,1310,2620,131,12,24,368,102
+cdc,cyber:170/815,50,2620,10480,30,12,24,32,74
+cdc,cyber:170/825,50,2620,10480,30,12,24,63,74
+cdc,cyber:170/835,56,5240,20970,30,12,24,106,138
+cdc,cyber:170/845,64,5240,20970,30,12,24,208,136
+cdc,omega:480-i,50,500,2000,8,1,4,20,23
+cdc,omega:480-ii,50,1000,4000,8,1,5,29,29
+cdc,omega:480-iii,50,2000,8000,8,1,5,71,44
+cambex,1636-1,50,1000,4000,8,3,5,26,30
+cambex,1636-10,50,1000,8000,8,3,5,36,41
+cambex,1641-1,50,2000,16000,8,3,5,40,74
+cambex,1641-11,50,2000,16000,8,3,6,52,74
+cambex,1651-1,50,2000,16000,8,3,6,60,74
+dec,decsys:10:1091,133,1000,12000,9,3,12,72,54
+dec,decsys:20:2060,133,1000,8000,9,3,12,72,41
+dec,microvax-1,810,512,512,8,1,1,18,18
+dec,vax:11/730,810,1000,5000,0,1,1,20,28
+dec,vax:11/750,320,512,8000,4,1,5,40,36
+dec,vax:11/780,200,512,8000,8,1,8,62,38
+dg,eclipse:c/350,700,384,8000,0,1,1,24,34
+dg,eclipse:m/600,700,256,2000,0,1,1,24,19
+dg,eclipse:mv/10000,140,1000,16000,16,1,3,138,72
+dg,eclipse:mv/4000,200,1000,8000,0,1,2,36,36
+dg,eclipse:mv/6000,110,1000,4000,16,1,2,26,30
+dg,eclipse:mv/8000,110,1000,12000,16,1,2,60,56
+dg,eclipse:mv/8000-ii,220,1000,8000,16,1,2,71,42
+formation,f4000/100,800,256,8000,0,1,4,12,34
+formation,f4000/200,800,256,8000,0,1,4,14,34
+formation,f4000/200ap,800,256,8000,0,1,4,20,34
+formation,f4000/300,800,256,8000,0,1,4,16,34
+formation,f4000/300ap,800,256,8000,0,1,4,22,34
+four-phase,2000/260,125,512,1000,0,8,20,36,19
+gould,concept:32/8705,75,2000,8000,64,1,38,144,75
+gould,concept:32/8750,75,2000,16000,64,1,38,144,113
+gould,concept:32/8780,75,2000,16000,128,1,38,259,157
+hp,3000/30,90,256,1000,0,3,10,17,18
+hp,3000/40,105,256,2000,0,3,10,26,20
+hp,3000/44,105,1000,4000,0,3,24,32,28
+hp,3000/48,105,2000,4000,8,3,19,32,33
+hp,3000/64,75,2000,8000,8,3,24,62,47
+hp,3000/88,75,3000,8000,8,3,48,64,54
+hp,3000/iii,175,256,2000,0,3,24,22,20
+harris,100,300,768,3000,0,6,24,36,23
+harris,300,300,768,3000,6,6,24,44,25
+harris,500,300,768,12000,6,6,24,50,52
+harris,600,300,768,4500,0,1,24,45,27
+harris,700,300,384,12000,6,1,24,53,50
+harris,80,300,192,768,6,6,24,36,18
+harris,800,180,768,12000,6,1,31,84,53
+honeywell,dps:6/35,330,1000,3000,0,2,4,16,23
+honeywell,dps:6/92,300,1000,4000,8,3,64,38,30
+honeywell,dps:6/96,300,1000,16000,8,2,112,38,73
+honeywell,dps:7/35,330,1000,2000,0,1,2,16,20
+honeywell,dps:7/45,330,1000,4000,0,3,6,22,25
+honeywell,dps:7/55,140,2000,4000,0,3,6,29,28
+honeywell,dps:7/65,140,2000,4000,0,4,8,40,29
+honeywell,dps:8/44,140,2000,4000,8,1,20,35,32
+honeywell,dps:8/49,140,2000,32000,32,1,20,134,175
+honeywell,dps:8/50,140,2000,8000,32,1,54,66,57
+honeywell,dps:8/52,140,2000,32000,32,1,54,141,181
+honeywell,dps:8/62,140,2000,32000,32,1,54,189,181
+honeywell,dps:8/20,140,2000,4000,8,1,20,22,32
+ibm,3033:s,57,4000,16000,1,6,12,132,82
+ibm,3033:u,57,4000,24000,64,12,16,237,171
+ibm,3081,26,16000,32000,64,16,24,465,361
+ibm,3081:d,26,16000,32000,64,8,24,465,350
+ibm,3083:b,26,8000,32000,0,8,24,277,220
+ibm,3083:e,26,8000,16000,0,8,16,185,113
+ibm,370/125-2,480,96,512,0,1,1,6,15
+ibm,370/148,203,1000,2000,0,1,5,24,21
+ibm,370/158-3,115,512,6000,16,1,6,45,35
+ibm,38/3,1100,512,1500,0,1,1,7,18
+ibm,38/4,1100,768,2000,0,1,1,13,20
+ibm,38/5,600,768,2000,0,1,1,16,20
+ibm,38/7,400,2000,4000,0,1,1,32,28
+ibm,38/8,400,4000,8000,0,1,1,32,45
+ibm,4321,900,1000,1000,0,1,2,11,18
+ibm,4331-1,900,512,1000,0,1,2,11,17
+ibm,4331-11,900,1000,4000,4,1,2,18,26
+ibm,4331-2,900,1000,4000,8,1,2,22,28
+ibm,4341,900,2000,4000,0,3,6,37,28
+ibm,4341-1,225,2000,4000,8,3,6,40,31
+ibm,4341-10,225,2000,4000,8,3,6,34,31
+ibm,4341-11,180,2000,8000,8,1,6,50,42
+ibm,4341-12,185,2000,16000,16,1,6,76,76
+ibm,4341-2,180,2000,16000,16,1,6,66,76
+ibm,4341-9,225,1000,4000,2,3,6,24,26
+ibm,4361-4,25,2000,12000,8,1,4,49,59
+ibm,4361-5,25,2000,12000,16,3,5,66,65
+ibm,4381-1,17,4000,16000,8,6,12,100,101
+ibm,4381-2,17,4000,16000,32,6,12,133,116
+ibm,8130-a,1500,768,1000,0,0,0,12,18
+ibm,8130-b,1500,768,2000,0,0,0,18,20
+ibm,8140,800,768,2000,0,0,0,20,20
+ipl,4436,50,2000,4000,0,3,6,27,30
+ipl,4443,50,2000,8000,8,3,6,45,44
+ipl,4445,50,2000,8000,8,1,6,56,44
+ipl,4446,50,2000,16000,24,1,6,70,82
+ipl,4460,50,2000,16000,24,1,6,80,82
+ipl,4480,50,8000,16000,48,1,10,136,128
+magnuson,m80/30,100,1000,8000,0,2,6,16,37
+magnuson,m80/31,100,1000,8000,24,2,6,26,46
+magnuson,m80/32,100,1000,8000,24,3,6,32,46
+magnuson,m80/42,50,2000,16000,12,3,16,45,80
+magnuson,m80/43,50,2000,16000,24,6,16,54,88
+magnuson,m80/44,50,2000,16000,24,6,16,65,88
+microdata,seq.ms/3200,150,512,4000,0,8,128,30,33
+nas,as/3000,115,2000,8000,16,1,3,50,46
+nas,as/3000-n,115,2000,4000,2,1,5,40,29
+nas,as/5000,92,2000,8000,32,1,6,62,53
+nas,as/5000-e,92,2000,8000,32,1,6,60,53
+nas,as/5000-n,92,2000,8000,4,1,6,50,41
+nas,as/6130,75,4000,16000,16,1,6,66,86
+nas,as/6150,60,4000,16000,32,1,6,86,95
+nas,as/6620,60,2000,16000,64,5,8,74,107
+nas,as/6630,60,4000,16000,64,5,8,93,117
+nas,as/6650,50,4000,16000,64,5,10,111,119
+nas,as/7000,72,4000,16000,64,8,16,143,120
+nas,as/7000-n,72,2000,8000,16,6,8,105,48
+nas,as/8040,40,8000,16000,32,8,16,214,126
+nas,as/8050,40,8000,32000,64,8,24,277,266
+nas,as/8060,35,8000,32000,64,8,24,370,270
+nas,as/9000-dpc,38,16000,32000,128,16,32,510,426
+nas,as/9000-n,48,4000,24000,32,8,24,214,151
+nas,as/9040,38,8000,32000,64,8,24,326,267
+nas,as/9060,30,16000,32000,256,16,24,510,603
+ncr,v8535:ii,112,1000,1000,0,1,4,8,19
+ncr,v8545:ii,84,1000,2000,0,1,6,12,21
+ncr,v8555:ii,56,1000,4000,0,1,6,17,26
+ncr,v8565:ii,56,2000,6000,0,1,8,21,35
+ncr,v8565:ii-e,56,2000,8000,0,1,8,24,41
+ncr,v8575:ii,56,4000,8000,0,1,8,34,47
+ncr,v8585:ii,56,4000,12000,0,1,8,42,62
+ncr,v8595:ii,56,4000,16000,0,1,8,46,78
+ncr,v8635,38,4000,8000,32,16,32,51,80
+ncr,v8650,38,4000,8000,32,16,32,116,80
+ncr,v8655,38,8000,16000,64,4,8,100,142
+ncr,v8665,38,8000,24000,160,4,8,140,281
+ncr,v8670,38,4000,16000,128,16,32,212,190
+nixdorf,8890/30,200,1000,2000,0,1,2,25,21
+nixdorf,8890/50,200,1000,4000,0,1,4,30,25
+nixdorf,8890/70,200,2000,8000,64,1,5,41,67
+perkin-elmer,3205,250,512,4000,0,1,7,25,24
+perkin-elmer,3210,250,512,4000,0,4,7,50,24
+perkin-elmer,3230,250,1000,16000,1,1,8,50,64
+prime,50-2250,160,512,4000,2,1,5,30,25
+prime,50-250-ii,160,512,2000,2,3,8,32,20
+prime,50-550-ii,160,1000,4000,8,1,14,38,29
+prime,50-750-ii,160,1000,8000,16,1,14,60,43
+prime,50-850-ii,160,2000,8000,32,1,13,109,53
+siemens,7.521,240,512,1000,8,1,3,6,19
+siemens,7.531,240,512,2000,8,1,5,11,22
+siemens,7.536,105,2000,4000,8,3,8,22,31
+siemens,7.541,105,2000,6000,16,6,16,33,41
+siemens,7.551,105,2000,8000,16,4,14,58,47
+siemens,7.561,52,4000,16000,32,4,12,130,99
+siemens,7.865-2,70,4000,12000,8,6,8,75,67
+siemens,7.870-2,59,4000,12000,32,6,12,113,81
+siemens,7.872-2,59,8000,16000,64,12,24,188,149
+siemens,7.875-2,26,8000,24000,32,8,16,173,183
+siemens,7.880-2,26,8000,32000,64,12,16,248,275
+siemens,7.881-2,26,8000,32000,128,24,32,405,382
+sperry,1100/61-h1,116,2000,8000,32,5,28,70,56
+sperry,1100/81,50,2000,32000,24,6,26,114,182
+sperry,1100/82,50,2000,32000,48,26,52,208,227
+sperry,1100/83,50,2000,32000,112,52,104,307,341
+sperry,1100/84,50,4000,32000,112,52,104,397,360
+sperry,1100/93,30,8000,64000,96,12,176,915,919
+sperry,1100/94,30,8000,64000,128,12,176,1150,978
+sperry,80/3,180,262,4000,0,1,3,12,24
+sperry,80/4,180,512,4000,0,1,3,14,24
+sperry,80/5,180,262,4000,0,1,3,18,24
+sperry,80/6,180,512,4000,0,1,3,21,24
+sperry,80/8,124,1000,8000,0,1,8,42,37
+sperry,90/80-model-3,98,1000,8000,32,2,8,46,50
+sratus,32,125,2000,8000,0,2,14,52,41
+wang,vs-100,480,512,8000,32,0,0,67,47
+wang,vs-90,480,1000,4000,0,0,0,45,25

demo/regression/machine.names

@@ -0,0 +1,72 @@
+1. Title: Relative CPU Performance Data 
+
+2. Source Information
+   -- Creators: Phillip Ein-Dor and Jacob Feldmesser
+     -- Ein-Dor: Faculty of Management; Tel Aviv University; Ramat-Aviv; 
+        Tel Aviv, 69978; Israel
+   -- Donor: David W. Aha (aha@ics.uci.edu) (714) 856-8779   
+   -- Date: October, 1987
+ 
+3. Past Usage:
+    1. Ein-Dor and Feldmesser (CACM 4/87, pp 308-317)
+       -- Results: 
+          -- linear regression prediction of relative cpu performance
+          -- Recorded 34% average deviation from actual values 
+    2. Kibler,D. & Aha,D. (1988).  Instance-Based Prediction of
+       Real-Valued Attributes.  In Proceedings of the CSCSI (Canadian
+       AI) Conference.
+       -- Results:
+          -- instance-based prediction of relative cpu performance
+          -- similar results; no transformations required
+    - Predicted attribute: cpu relative performance (numeric)
+
+4. Relevant Information:
+   -- The estimated relative performance values were estimated by the authors
+      using a linear regression method.  See their article (pp 308-313) for
+      more details on how the relative performance values were set.
+
+5. Number of Instances: 209 
+
+6. Number of Attributes: 10 (6 predictive attributes, 2 non-predictive, 
+                             1 goal field, and the linear regression's guess)
+
+7. Attribute Information:
+   1. vendor name: 30 
+      (adviser, amdahl,apollo, basf, bti, burroughs, c.r.d, cambex, cdc, dec, 
+       dg, formation, four-phase, gould, honeywell, hp, ibm, ipl, magnuson, 
+       microdata, nas, ncr, nixdorf, perkin-elmer, prime, siemens, sperry, 
+       sratus, wang)
+   2. Model Name: many unique symbols
+   3. MYCT: machine cycle time in nanoseconds (integer)
+   4. MMIN: minimum main memory in kilobytes (integer)
+   5. MMAX: maximum main memory in kilobytes (integer)
+   6. CACH: cache memory in kilobytes (integer)
+   7. CHMIN: minimum channels in units (integer)
+   8. CHMAX: maximum channels in units (integer)
+   9. PRP: published relative performance (integer)
+  10. ERP: estimated relative performance from the original article (integer)
+
+8. Missing Attribute Values: None
+
+9. Class Distribution: the class value (PRP) is continuously valued.
+   PRP Value Range:   Number of Instances in Range:
+   0-20               31
+   21-100             121
+   101-200            27
+   201-300            13
+   301-400            7
+   401-500            4
+   501-600            2
+   above 600          4
+
+Summary Statistics:
+	   Min  Max   Mean    SD      PRP Correlation
+   MCYT:   17   1500  203.8   260.3   -0.3071
+   MMIN:   64   32000 2868.0  3878.7   0.7949
+   MMAX:   64   64000 11796.1 11726.6  0.8630
+   CACH:   0    256   25.2    40.6     0.6626
+   CHMIN:  0    52    4.7     6.8      0.6089
+   CHMAX:  0    176   18.2    26.0     0.6052
+   PRP:    6    1150  105.6   160.8    1.0000
+   ERP:   15    1238  99.3    154.8    0.9665
+

demo/regression/mapfeat.py

@@ -0,0 +1,32 @@
+#!/usr/bin/python
+import sys
+
+fo = open( 'machine.txt', 'w' ) 
+cnt = 6
+fmap = {}
+for l in open( 'machine.data' ):
+    arr = l.split(',')
+    fo.write(arr[8])
+    for i in range( 0,6 ):
+        fo.write( ' %d:%s' %(i,arr[i+2]) )
+    
+    if arr[0] not in fmap:
+        fmap[arr[0]] = cnt
+        cnt += 1
+    
+    fo.write( ' %d:1' % fmap[arr[0]] )	
+    fo.write('\n')
+
+fo.close()
+
+# create feature map for machine data
+fo = open('featmap.txt', 'w')
+# list from machine.names
+names = ['vendor','MYCT', 'MMIN', 'MMAX', 'CACH', 'CHMIN', 'CHMAX', 'PRP', 'ERP' ]; 
+
+for i in range(0,6):
+    fo.write( '%d\t%s\tint\n' % (i, names[i+1]))
+
+for v, k in sorted( fmap.items(), key = lambda x:x[1] ):
+    fo.write( '%d\tvendor=%s\ti\n' % (k, v))
+fo.close()

demo/regression/mknfold.py

@@ -0,0 +1,29 @@
+#!/usr/bin/python
+import sys
+import random
+
+if len(sys.argv) < 2:
+    print ('Usage:<filename> <k> [nfold = 5]')
+    exit(0)
+
+random.seed( 10 )
+
+k = int( sys.argv[2] )
+if len(sys.argv) > 3:
+    nfold = int( sys.argv[3] )
+else:
+    nfold = 5
+
+fi = open( sys.argv[1], 'r' )
+ftr = open( sys.argv[1]+'.train', 'w' )
+fte = open( sys.argv[1]+'.test', 'w' )
+for l in fi:
+    if random.randint( 1 , nfold ) == k:
+        fte.write( l )
+    else:
+        ftr.write( l )
+
+fi.close()
+ftr.close()
+fte.close()
+

demo/regression/runexp.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# map the data to features. For convenience we only use 7 original attributes and encode them as features in a trivial way 
+python mapfeat.py
+# split train and test
+python mknfold.py machine.txt 1
+# training and output the models
+../../xgboost machine.conf
+# output predictions of test data
+../../xgboost machine.conf task=pred model_in=0002.model
+# print the boosters of 0002.model in dump.raw.txt
+../../xgboost machine.conf task=dump model_in=0002.model name_dump=dump.raw.txt
+# print the boosters of 0002.model in dump.nice.txt with feature map
+../../xgboost machine.conf task=dump model_in=0002.model fmap=featmap.txt name_dump=dump.nice.txt 
+
+# cat the result
+cat dump.nice.txt