From 2159d18f0ba449673de49c70bf7588d0f6a25586 Mon Sep 17 00:00:00 2001 From: Tianqi Chen Date: Fri, 13 Mar 2015 23:23:23 -0700 Subject: [PATCH 1/6] Update param.h --- src/tree/param.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tree/param.h b/src/tree/param.h index 2c2362095..41abba90f 100644 --- a/src/tree/param.h +++ b/src/tree/param.h @@ -50,6 +50,7 @@ struct TrainParam{ /*! \brief constructor */ TrainParam(void) { learning_rate = 0.3f; + min_split_loss = 0.0f; min_child_weight = 1.0f; max_depth = 6; reg_lambda = 1.0f; From 8386c2b7fa9a044debb37240baba885c50dd38c2 Mon Sep 17 00:00:00 2001 From: tqchen Date: Fri, 13 Mar 2015 23:49:56 -0700 Subject: [PATCH 2/6] check r --- R-package/src/xgboost_R.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/R-package/src/xgboost_R.cpp b/R-package/src/xgboost_R.cpp index b887c60ca..315b6c2c1 100644 --- a/R-package/src/xgboost_R.cpp +++ b/R-package/src/xgboost_R.cpp @@ -293,7 +293,9 @@ extern "C" { const char *raw = XGBoosterGetModelRaw(R_ExternalPtrAddr(handle), &olen); _WrapperEnd(); SEXP ret = PROTECT(allocVector(RAWSXP, olen)); - memcpy(RAW(ret), raw, olen); + if (olen != 0) { + memcpy(RAW(ret), raw, olen); + } UNPROTECT(1); return ret; } From 1e001f7cf3fff55dca04699be1c99ee1c73c0275 Mon Sep 17 00:00:00 2001 From: Tong He Date: Mon, 16 Mar 2015 23:20:31 -0700 Subject: [PATCH 3/6] add length check --- R-package/R/utils.R | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/R-package/R/utils.R b/R-package/R/utils.R index bff6dd0e8..db46f83bb 100644 --- a/R-package/R/utils.R +++ b/R-package/R/utils.R @@ -15,11 +15,15 @@ xgb.setinfo <- function(dmat, name, info) { stop("xgb.setinfo: first argument dtrain must be xgb.DMatrix") } if (name == "label") { + if (length(info)!=xgb.numrow(dmat)) + stop("The length of labels must equal to the number of rows in the input data") .Call("XGDMatrixSetInfo_R", dmat, name, as.numeric(info), PACKAGE = "xgboost") return(TRUE) } if (name == "weight") { + if (length(info)!=xgb.numrow(dmat)) + stop("The length of weights must equal to the number of rows in the input data") .Call("XGDMatrixSetInfo_R", dmat, name, as.numeric(info), PACKAGE = "xgboost") return(TRUE) From a146f0c5e1166bf1857238c76074768c4bd840ab Mon Sep 17 00:00:00 2001 From: Tong He Date: Mon, 16 Mar 2015 23:23:22 -0700 Subject: [PATCH 4/6] Update utils.R --- R-package/R/utils.R | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/R-package/R/utils.R b/R-package/R/utils.R index db46f83bb..ba3123a21 100644 --- a/R-package/R/utils.R +++ b/R-package/R/utils.R @@ -29,11 +29,15 @@ xgb.setinfo <- function(dmat, name, info) { return(TRUE) } if (name == "base_margin") { + if (length(info)!=xgb.numrow(dmat)) + stop("The length of base margin must equal to the number of rows in the input data") .Call("XGDMatrixSetInfo_R", dmat, name, as.numeric(info), PACKAGE = "xgboost") return(TRUE) } if (name == "group") { + if (length(info)!=xgb.numrow(dmat)) + stop("The length of groups must equal to the number of rows in the input data") .Call("XGDMatrixSetInfo_R", dmat, name, as.integer(info), PACKAGE = "xgboost") return(TRUE) From e1538ae615896d1527895a8a972b522f24389de1 Mon Sep 17 00:00:00 2001 From: tqchen Date: Thu, 19 Mar 2015 11:34:38 -0700 Subject: [PATCH 5/6] add new evaluation metric mlogloss for multi-class classification logloss --- src/learner/evaluation-inl.hpp | 81 +++++++++++++++++++++++++++++++--- src/learner/evaluation.h | 1 + src/learner/helper_utils.h | 15 +++++-- src/learner/objective-inl.hpp | 2 +- 4 files changed, 88 insertions(+), 11 deletions(-) diff --git a/src/learner/evaluation-inl.hpp b/src/learner/evaluation-inl.hpp index 60a8da8f1..64ad40849 100644 --- a/src/learner/evaluation-inl.hpp +++ b/src/learner/evaluation-inl.hpp @@ -27,8 +27,9 @@ struct EvalEWiseBase : public IEvaluator { const MetaInfo &info, bool distributed) const { utils::Check(info.labels.size() != 0, "label set cannot be empty"); - utils::Check(preds.size() % info.labels.size() == 0, - "label and prediction size not match"); + utils::Check(preds.size() == info.labels.size(), + "label and prediction size not match"\ + "hint: use merror or mlogloss for multi-class classification"); const bst_omp_uint ndata = static_cast(info.labels.size()); @@ -50,7 +51,6 @@ struct EvalEWiseBase : public IEvaluator { * get evaluation result from one row * \param label label of current instance * \param pred prediction value of current instance - * \param weight weight of current instance */ inline static float EvalRow(float label, float pred); /*! @@ -98,15 +98,84 @@ struct EvalError : public EvalEWiseBase { } }; +/*! + * \brief base class of multi-class evaluation + * \tparam Derived the name of subclass + */ +template +struct EvalMClassBase : public IEvaluator { + virtual float Eval(const std::vector &preds, + const MetaInfo &info, + bool distributed) const { + utils::Check(info.labels.size() != 0, "label set cannot be empty"); + utils::Check(preds.size() % info.labels.size() == 0, + "label and prediction size not match"); + const size_t nclass = preds.size() / info.labels.size(); + const bst_omp_uint ndata = static_cast(info.labels.size()); + + float sum = 0.0, wsum = 0.0; + #pragma omp parallel for reduction(+: sum, wsum) schedule(static) + for (bst_omp_uint i = 0; i < ndata; ++i) { + const float wt = info.GetWeight(i); + sum += Derived::EvalRow(info.labels[i], + BeginPtr(preds) + i * nclass, + nclass) * wt; + wsum += wt; + } + float dat[2]; dat[0] = sum, dat[1] = wsum; + if (distributed) { + rabit::Allreduce(dat, 2); + } + return Derived::GetFinal(dat[0], dat[1]); + } + /*! + * \brief to be implemented by subclass, + * get evaluation result from one row + * \param label label of current instance + * \param pred prediction value of current instance + * \param nclass number of class in the prediction + */ + inline static float EvalRow(float label, + const float *pred, + size_t nclass); + /*! + * \brief to be overide by subclas, final trasnformation + * \param esum the sum statistics returned by EvalRow + * \param wsum sum of weight + */ + inline static float GetFinal(float esum, float wsum) { + return esum / wsum; + } +}; /*! \brief match error */ -struct EvalMatchError : public EvalEWiseBase { +struct EvalMatchError : public EvalMClassBase { virtual const char *Name(void) const { return "merror"; } - inline static float EvalRow(float label, float pred) { - return static_cast(pred) != static_cast(label); + inline static float EvalRow(float label, + const float *pred, + size_t nclass) { + return FindMaxIndex(pred, nclass) != static_cast(label); } }; +/*! \brief match error */ +struct EvalMultiLogLoss : public EvalMClassBase { + virtual const char *Name(void) const { + return "mlogloss"; + } + inline static float EvalRow(float label, + const float *pred, + size_t nclass) { + size_t k = static_cast(label); + utils::Check(k < nclass, "mlogloss: label must be in [0, num_class)"); + if (pred[k] > eps) { + return -std::log(pred[k]); + } else { + return -std::log(eps); + } + } + const static float eps = 1e-16; +}; /*! \brief ctest */ struct EvalCTest: public IEvaluator { diff --git a/src/learner/evaluation.h b/src/learner/evaluation.h index 4d59e270a..146f757a2 100644 --- a/src/learner/evaluation.h +++ b/src/learner/evaluation.h @@ -45,6 +45,7 @@ inline IEvaluator* CreateEvaluator(const char *name) { if (!strcmp(name, "error")) return new EvalError(); if (!strcmp(name, "merror")) return new EvalMatchError(); if (!strcmp(name, "logloss")) return new EvalLogLoss(); + if (!strcmp(name, "mlogloss")) return new EvalMultiLogLoss(); if (!strcmp(name, "auc")) return new EvalAuc(); if (!strncmp(name, "ams@", 4)) return new EvalAMS(name); if (!strncmp(name, "pre@", 4)) return new EvalPrecision(name); diff --git a/src/learner/helper_utils.h b/src/learner/helper_utils.h index aa1e66bbc..ac1ec745e 100644 --- a/src/learner/helper_utils.h +++ b/src/learner/helper_utils.h @@ -27,21 +27,28 @@ inline static void Softmax(std::vector* p_rec) { rec[i] /= static_cast(wsum); } } -// simple helper function to do softmax -inline static int FindMaxIndex(const std::vector& rec) { + +inline static int FindMaxIndex(const float *rec, size_t size) { size_t mxid = 0; - for (size_t i = 1; i < rec.size(); ++i) { - if (rec[i] > rec[mxid] + 1e-6f) { + for (size_t i = 1; i < size; ++i) { + if (rec[i] > rec[mxid]) { mxid = i; } } return static_cast(mxid); } +// simple helper function to do softmax +inline static int FindMaxIndex(const std::vector& rec) { + return FindMaxIndex(BeginPtr(rec), rec.size()); +} + + inline static bool CmpFirst(const std::pair &a, const std::pair &b) { return a.first > b.first; } + inline static bool CmpSecond(const std::pair &a, const std::pair &b) { return a.second > b.second; diff --git a/src/learner/objective-inl.hpp b/src/learner/objective-inl.hpp index 9887e7a05..4d6ae602a 100644 --- a/src/learner/objective-inl.hpp +++ b/src/learner/objective-inl.hpp @@ -225,7 +225,7 @@ class SoftmaxMultiClassObj : public IObjFunction { this->Transform(io_preds, output_prob); } virtual void EvalTransform(std::vector *io_preds) { - this->Transform(io_preds, 0); + this->Transform(io_preds, 1); } virtual const char* DefaultEvalMetric(void) const { return "merror"; From 360cc7118d6863ade97a585da37315b3f03c5aeb Mon Sep 17 00:00:00 2001 From: tqchen Date: Thu, 19 Mar 2015 11:53:55 -0700 Subject: [PATCH 6/6] fix cxx11 --- src/learner/evaluation-inl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/learner/evaluation-inl.hpp b/src/learner/evaluation-inl.hpp index 64ad40849..ab68a6ec6 100644 --- a/src/learner/evaluation-inl.hpp +++ b/src/learner/evaluation-inl.hpp @@ -166,6 +166,7 @@ struct EvalMultiLogLoss : public EvalMClassBase { inline static float EvalRow(float label, const float *pred, size_t nclass) { + const float eps = 1e-16f; size_t k = static_cast(label); utils::Check(k < nclass, "mlogloss: label must be in [0, num_class)"); if (pred[k] > eps) { @@ -173,8 +174,7 @@ struct EvalMultiLogLoss : public EvalMClassBase { } else { return -std::log(eps); } - } - const static float eps = 1e-16; + } }; /*! \brief ctest */