changes to reg booster

Makefile

@@ -3,7 +3,7 @@ export CXX = g++
 export CFLAGS = -Wall -O3 -msse2 
 
 # specify tensor path
-BIN = 
+BIN = xgboost
 OBJ = xgboost.o
 .PHONY: clean all
 
@@ -11,6 +11,7 @@ all: $(BIN) $(OBJ)
 export LDFLAGS= -pthread -lm 
 
 xgboost.o: booster/xgboost.h booster/xgboost_data.h booster/xgboost.cpp booster/*/*.hpp booster/*/*.h
+xgboost: regression/xgboost_reg_main.cpp xgboost.o
 
 $(BIN) : 
 	$(CXX) $(CFLAGS) $(LDFLAGS) -o $@ $(filter %.cpp %.o %.c, $^)

booster/xgboost_data.h

@@ -8,6 +8,7 @@
  */
 
 #include <vector>
+#include <climits>
 #include "../utils/xgboost_utils.h"
 #include "../utils/xgboost_stream.h"
 

regression/xgboost_reg.h

@@ -6,6 +6,8 @@
 * \author Kailong Chen: chenkl198812@gmail.com, Tianqi Chen: tianqi.tchen@gmail.com
 */
 #include <cmath>
+#include <cstdlib>
+#include <cstring>
 #include "xgboost_regdata.h"
 #include "../booster/xgboost_gbmbase.h"
 #include "../utils/xgboost_utils.h"
@@ -16,11 +18,8 @@ namespace xgboost{
         /*! \brief class for gradient boosted regression */
         class RegBoostLearner{            
         public:
-
+            /*! \brief constructor */
-            RegBoostLearner(bool silent = false){
+            RegBoostLearner( void ){}
-                this->silent = silent;
-            }
-
             /*! 
             * \brief a regression booter associated with training and evaluating data 
             * \param train pointer to the training data
@@ -28,10 +27,9 @@ namespace xgboost{
             * \param evname name of evaluation data, used print statistics
             */
             RegBoostLearner( const DMatrix *train,
-                std::vector<const DMatrix *> evals,
+                             const std::vector<DMatrix *> &evals,
-                std::vector<std::string> evname, bool silent = false ){
+                             const std::vector<std::string> &evname ){
-                    this->silent = silent;
+                this->SetData(train,evals,evname);
-                    SetData(train,evals,evname);
             }
 
             /*! 
@@ -40,23 +38,22 @@ namespace xgboost{
             * \param evals array of evaluating data
             * \param evname name of evaluation data, used print statistics
             */
-            inline void SetData(const DMatrix *train,
+            inline void SetData( const DMatrix *train,
-                std::vector<const DMatrix *> evals,
+                                 const std::vector<DMatrix *> &evals,
-                std::vector<std::string> evname){
+                                 const std::vector<std::string> &evname ){
-                    this->train_ = train;
+                this->train_ = train;
-                    this->evals_ = evals;
+                this->evals_ = evals;
-                    this->evname_ = evname; 
+                this->evname_ = evname; 
-                    //assign buffer index
+                //assign buffer index
-                    int buffer_size = (*train).size();
+                unsigned buffer_size = static_cast<unsigned>( train->Size() );
-                    for(int i = 0; i < evals.size(); i++){
-                        buffer_size += (*evals[i]).size();
-                    }
-                    char str[25];
-                    _itoa(buffer_size,str,10);
-                    base_model.SetParam("num_pbuffer",str);
-                    base_model.SetParam("num_pbuffer",str);
-            }
                 
+                for( size_t i = 0; i < evals.size(); ++ i ){
+                    buffer_size += static_cast<unsigned>( evals[i]->Size() );
+                }
+                char snum_pbuffer[25];
+                printf( snum_pbuffer, "%u", buffer_size );
+                base_model.SetParam( "num_pbuffer",snum_pbuffer );
+            }
             /*! 
             * \brief set parameters from outside 
             * \param name name of the parameter
@@ -72,17 +69,14 @@ namespace xgboost{
             */
             inline void InitTrainer( void ){
                 base_model.InitTrainer();
-                InitModel();
-                mparam.AdjustBase();
             } 
-
             /*!
             * \brief initialize the current data storage for model, if the model is used first time, call this function
             */
             inline void InitModel( void ){
                 base_model.InitModel();
+                mparam.AdjustBase();
             }
-
             /*! 
             * \brief load model from stream
             * \param fi input stream
@@ -99,57 +93,78 @@ namespace xgboost{
                 fo.Write( &mparam, sizeof(ModelParam) );
                 base_model.SaveModel( fo );	
             } 
-
             /*! 
-            * \brief update the model for one iteration
+             * \brief update the model for one iteration
-            * \param iteration the number of updating iteration 
+             * \param iteration iteration number
-            */           
+             */
-            inline void UpdateOneIter( int iteration ){
+            inline void UpdateOneIter( int iter ){
-                std::vector<float> grad,hess,preds;
+                std::vector<float> grad, hess, preds;
-                std::vector<unsigned> root_index;
+                this->Predict( preds, *train_, 0 );
-                booster::FMatrixS::Image train_image((*train_).data);
+                this->GetGradient( preds, train_->labels, grad, hess );
-                Predict(preds,*train_,0);
-                Gradient(preds,(*train_).labels,grad,hess);
-                base_model.DoBoost(grad,hess,train_image,root_index);
-                int buffer_index_offset = (*train_).size();
-                float loss = 0.0;
-                for(int i = 0; i < evals_.size();i++){
-                    Predict(preds, *evals_[i], buffer_index_offset);
-                    loss = mparam.Loss(preds,(*evals_[i]).labels);
-                    if(!silent){
-                        printf("The loss of %s data set in %d the \
-                               iteration is %f",evname_[i].c_str(),&iteration,&loss);
-                    }
-                    buffer_index_offset += (*evals_[i]).size();
-                }
 
+                std::vector<unsigned> root_index;
+                booster::FMatrixS::Image train_image( train_->data );                
+                base_model.DoBoost(grad,hess,train_image,root_index);                
+            }
+            /*! 
+             * \brief evaluate the model for specific iteration
+             * \param iter iteration number
+             * \param fo file to output log
+             */            
+            inline void EvalOneIter( int iter, FILE *fo = stderr ){
+                std::vector<float> preds;
+                fprintf( fo, "[%d]", iter );
+                int buffer_offset = static_cast<int>( train_->Size() );
+
+                for(size_t i = 0; i < evals_.size();i++){
+                    this->Predict(preds, *evals_[i], buffer_offset);
+                    this->Eval( fo, evname_[i].c_str(), preds, (*evals_[i]).labels );
+                    buffer_offset += static_cast<int>( evals_[i]->Size() );
+                }
+                fprintf( fo,"\n" );
             }
 
+            /*! \brief get prediction, without buffering */
+            inline void Predict( std::vector<float> &preds, const DMatrix &data ){
+                preds.resize( data.Size() );
+                for( size_t j = 0; j < data.Size(); j++ ){
+                    preds[j] = mparam.PredTransform
+                        ( mparam.base_score + base_model.Predict( data.data[j], -1 ) );
+                }
+            }
+        private:
+            /*! \brief print evaluation results */
+            inline void Eval( FILE *fo, const char *evname,
+                              const std::vector<float> &preds, 
+                              const std::vector<float> &labels ){
+                const float loss = mparam.Loss( preds, labels );
+                fprintf( fo, "\t%s:%f", evname, loss );
+            }
             /*! \brief get the transformed predictions, given data */
-            inline void Predict( std::vector<float> &preds, const DMatrix &data,int buffer_index_offset = 0 ){
+            inline void Predict( std::vector<float> &preds, const DMatrix &data, unsigned buffer_offset ){
-                int data_size = data.size();
+                preds.resize( data.Size() );
-                preds.resize(data_size);
+                for( size_t j = 0; j < data.Size(); j++ ){
-                for(int j = 0; j < data_size; j++){
+                    preds[j] = mparam.PredTransform
-                    preds[j] = mparam.PredTransform(mparam.base_score + 
+                        ( mparam.base_score + base_model.Predict( data.data[j], buffer_offset + j ) );
-                        base_model.Predict(data.data[j],buffer_index_offset + j));
+                }
+            }
+
+            /*! \brief get the first order and second order gradient, given the transformed predictions and labels */
+            inline void GetGradient( const std::vector<float> &preds, 
+                                     const std::vector<float> &labels, 
+                                     std::vector<float> &grad,
+                                     std::vector<float> &hess ){
+                grad.clear(); hess.clear();
+                for( size_t j = 0; j < preds.size(); j++ ){
+                    grad.push_back( mparam.FirstOrderGradient (preds[j],labels[j]) );
+                    hess.push_back( mparam.SecondOrderGradient(preds[j],labels[j]) );
                 }
             }
 
         private:
-            /*! \brief get the first order and second order gradient, given the transformed predictions and labels*/
+            enum LossType{
-            inline void Gradient(const std::vector<float> &preds, const std::vector<float> &labels, std::vector<float> &grad,
+                kLinearSquare = 0,
-                std::vector<float> &hess){
+                kLogisticNeglik = 1,
-                    grad.clear(); 
-                    hess.clear();
-                    for(int j = 0; j < preds.size(); j++){
-                        grad.push_back(mparam.FirstOrderGradient(preds[j],labels[j]));
-                        hess.push_back(mparam.SecondOrderGradient(preds[j],labels[j]));
-                    }
-            }
-
-            enum LOSS_TYPE_LIST{
-                LINEAR_SQUARE,
-                LOGISTIC_NEGLOGLIKELIHOOD,
             };
 
             /*! \brief training parameter for regression */
@@ -181,6 +196,20 @@ namespace xgboost{
                         base_score = - logf( 1.0f / base_score - 1.0f );
                     }
                 }
+
+                /*! 
+                * \brief transform the linear sum to prediction 
+                * \param x linear sum of boosting ensemble
+                * \return transformed prediction
+                */
+                inline float PredTransform( float x ){
+                    switch( loss_type ){                        
+                    case kLinearSquare: return x;
+                    case kLogisticNeglik: return 1.0f/(1.0f + expf(-x));
+                    default: utils::Error("unknown loss_type"); return 0.0f;
+                    }
+                }
+
                 /*! 
                 * \brief calculate first order gradient of loss, given transformed prediction
                 * \param predt transformed prediction
@@ -189,7 +218,7 @@ namespace xgboost{
                 */
                 inline float FirstOrderGradient( float predt, float label ) const{
                     switch( loss_type ){                        
-                    case LINEAR_SQUARE: return predt - label;
+                    case kLinearSquare: return predt - label;
                     case 1: return predt - label;
                     default: utils::Error("unknown loss_type"); return 0.0f;
                     }
@@ -202,8 +231,8 @@ namespace xgboost{
                 */
                 inline float SecondOrderGradient( float predt, float label ) const{
                     switch( loss_type ){                        
-                    case LINEAR_SQUARE: return 1.0f;
+                    case kLinearSquare: return 1.0f;
-                    case LOGISTIC_NEGLOGLIKELIHOOD: return predt * ( 1 - predt );
+                    case kLogisticNeglik: return predt * ( 1 - predt );
                     default: utils::Error("unknown loss_type"); return 0.0f;
                     }
                 }
@@ -216,8 +245,8 @@ namespace xgboost{
                 */
                 inline float Loss(const std::vector<float> &preds, const std::vector<float> &labels) const{
                     switch( loss_type ){
-                    case LINEAR_SQUARE: return SquareLoss(preds,labels);
+                    case kLinearSquare: return SquareLoss(preds,labels);
-                    case LOGISTIC_NEGLOGLIKELIHOOD: return NegLoglikelihoodLoss(preds,labels);
+                    case kLogisticNeglik: return NegLoglikelihoodLoss(preds,labels);
                     default: utils::Error("unknown loss_type"); return 0.0f;
                     }
                 }
@@ -230,8 +259,10 @@ namespace xgboost{
                 */
                 inline float SquareLoss(const std::vector<float> &preds, const std::vector<float> &labels) const{
                     float ans = 0.0;
-                    for(int i = 0; i < preds.size(); i++)
+                    for(size_t i = 0; i < preds.size(); i++){
-                        ans += pow(preds[i] - labels[i], 2);
+                        float dif = preds[i] - labels[i];
+                        ans += dif * dif;
+                    }
                     return ans;
                 }
 
@@ -243,34 +274,18 @@ namespace xgboost{
                 */
                 inline float NegLoglikelihoodLoss(const std::vector<float> &preds, const std::vector<float> &labels) const{
                     float ans = 0.0;
-                    for(int i = 0; i < preds.size(); i++)
+                    for(size_t i = 0; i < preds.size(); i++)
-                        ans -= labels[i] * log(preds[i]) + ( 1 - labels[i] ) * log(1 - preds[i]);
+                        ans -= labels[i] * logf(preds[i]) + ( 1 - labels[i] ) * logf(1 - preds[i]);
                     return ans;
                 }
-
-
-                /*! 
-                * \brief transform the linear sum to prediction 
-                * \param x linear sum of boosting ensemble
-                * \return transformed prediction
-                */
-                inline float PredTransform( float x ){
-                    switch( loss_type ){                        
-                    case LINEAR_SQUARE: return x;
-                    case LOGISTIC_NEGLOGLIKELIHOOD: return 1.0f/(1.0f + expf(-x));
-                    default: utils::Error("unknown loss_type"); return 0.0f;
-                    }
-                }
-
-
             };
         private:            
             booster::GBMBaseModel base_model;
             ModelParam   mparam;
             const DMatrix *train_;
-            std::vector<const DMatrix *> evals_;
+            std::vector<DMatrix *> evals_;
             std::vector<std::string> evname_;
-            bool silent;
+            std::vector<unsigned> buffer_index_;
         };
     }
 };

regression/xgboost_reg_main.cpp

@@ -1,15 +1,180 @@
-#include"xgboost_reg_train.h"
+#define _CRT_SECURE_NO_WARNINGS
-#include"xgboost_reg_test.h"
+#define _CRT_SECURE_NO_DEPRECATE
-using namespace xgboost::regression;
 
-int main(int argc, char *argv[]){
+#include <ctime>
-    //char* config_path = argv[1];
+#include <string>
-    //bool silent = ( atoi(argv[2]) == 1 );
+#include <cstring>
-    char* config_path = "c:\\cygwin64\\home\\chen\\github\\xgboost\\demo\\regression\\reg.conf";
+#include "xgboost_reg.h"
-    bool silent = false;
+#include "../utils/xgboost_random.h"
-    RegBoostTrain train;
+#include "../utils/xgboost_config.h"
-    train.train(config_path,false);
 
-    RegBoostTest test;
+namespace xgboost{
-    test.test(config_path,false);
+    namespace regression{
+        /*!
+        * \brief wrapping the training process of the gradient boosting regression model,
+        *   given the configuation
+        * \author Kailong Chen: chenkl198812@gmail.com, Tianqi Chen: tianqi.chen@gmail.com
+        */
+        class RegBoostTask{            
+        public:
+            inline int Run( int argc, char *argv[] ){
+                if( argc < 2 ){
+                    printf("Usage: <config>\n"); 
+                    return 0;
+                }
+                utils::ConfigIterator itr( argv[1] );
+                while( itr.Next() ){
+                    this->SetParam( itr.name(), itr.val() );
+                }
+                for( int i = 2; i < argc; i ++ ){
+                    char name[256], val[256];
+                    if( sscanf( argv[i], "%[^=]=%s", name, val ) == 2 ){
+                        this->SetParam( name, val );
+                    }
+                }
+                this->InitData();
+                this->InitLearner();
+                if( !strcmp( task.c_str(), "test") ){
+                    this->TaskTest();
+                }else{
+                    this->TaskTrain();
+                }
+                return 0;
+            }
+            inline void SetParam( const char *name, const char *val ){
+                if( !strcmp("silent", name ) )       silent = atoi( val );
+                if( !strcmp("seed", name ) )         random::Seed( atoi(val) );
+                if( !strcmp("num_round", name ) )    num_round = atoi( val );
+                if( !strcmp("save_period", name ) )  save_period = atoi( val );
+                if( !strcmp("task", name ) )         task = val;
+                if( !strcmp("data",  name ) )        train_path = val;
+                if( !strcmp("test:data",  name ) )   test_path  = val;
+                if( !strcmp("model_in",  name ) )    model_in   = val;
+                if( !strcmp("model_dir", name ) )    model_dir_path = val;
+                if( !strncmp("eval[",  name, 5 ) ) {
+                    char evname[ 256 ];
+                    utils::Assert( sscanf( name, "eval[%[^]]", evname ) == 1, "must specify evaluation name for display");
+                    eval_data_names.push_back( std::string( evname ) );
+                    eval_data_paths.push_back( std::string( val ) );
+                }
+                cfg.PushBack( name, val );
+            }
+        public:
+            RegBoostTask( void ){
+                // default parameters
+                silent = 0;
+                num_round = 10;
+                save_period = 0;
+                task = "train";                
+                model_in = "NULL";
+                name_pred = "pred.txt";
+                model_dir_path = "./";
+            }
+            ~RegBoostTask( void ){
+                for( size_t i = 0; i < deval.size(); i ++ ){
+                    delete deval[i];
+                }
+            }
+        private:
+            inline void InitData( void ){
+                if( !strcmp( task.c_str(), "test") ){
+                    data.CacheLoad( test_path.c_str() );
+                }else{
+                    // training 
+                    data.CacheLoad( train_path.c_str() );
+                    utils::Assert( eval_data_names.size() == eval_data_paths.size() );
+                    for( size_t i = 0; i < eval_data_names.size(); ++ i ){
+                        deval.push_back( new DMatrix() );
+                        deval.back()->CacheLoad( eval_data_paths[i].c_str() );
+                    }
+                }
+                learner.SetData( &data, deval, eval_data_names );
+            }
+            inline void InitLearner( void ){
+                cfg.BeforeFirst();
+                while( cfg.Next() ){
+                    learner.SetParam( cfg.name(), cfg.val() );
+                }
+                if( strcmp( model_in.c_str(), "NULL" ) != 0 ){
+                    utils::Assert( !strcmp( task.c_str(), "train"), "model_in not specified" );
+                    utils::FileStream fi( utils::FopenCheck( model_in.c_str(), "rb") );
+                    learner.LoadModel( fi );
+                    fi.Close();
+                }else{
+                    learner.InitModel();
+                }
+                learner.InitTrainer();
+            }
+            inline void TaskTrain( void ){
+                const time_t start    = time( NULL );
+                unsigned long elapsed = 0;
+                for( int i = 0; i < num_round; ++ i ){
+                    elapsed = (unsigned long)(time(NULL) - start); 
+                    if( !silent ) printf("boosting round %d, %lu sec elapsed\n", i , elapsed );
+                    learner.UpdateOneIter( i );
+                    learner.EvalOneIter( i );
+                    if( save_period != 0 && (i+1) % save_period == 0 ){
+                        SaveModel( i );
+                    }
+                    elapsed = (unsigned long)(time(NULL) - start); 
+                }
+                // always save final round
+                if( num_round % save_period != 0 ){
+                    SaveModel( num_round );
+                }
+                if( !silent ){
+                    printf("\nupdating end, %lu sec in all\n", elapsed );
+                }
+            }
+            inline void SaveModel( int i ){
+                char fname[256];
+                sprintf( fname ,"%s/%04d.model", model_dir_path.c_str(), i+1 );
+                utils::FileStream fo( utils::FopenCheck( fname, "wb" ) );
+                learner.SaveModel( fo );
+                fo.Close();
+            }
+            inline void TaskTest( void ){
+                std::vector<float> preds;
+                learner.Predict( preds, data );
+                FILE *fo = utils::FopenCheck( name_pred.c_str(), "w" );
+                for( size_t i = 0; i < preds.size(); i ++ ){
+                    fprintf( fo, "%f\n", preds[i] );
+                }
+                fclose( fo );                
+            }
+        private:
+            /* \brief whether silent */
+            int silent;
+            /* \brief number of boosting iterations */
+            int num_round;            
+            /* \brief the period to save the model, 0 means only save the final round model */
+            int save_period;            
+            /* \brief the path of training/test data set */
+            std::string train_path, test_path;
+            /* \brief the path of test model file, or file to restart training */
+            std::string model_in;
+            /* \brief the path of directory containing the saved models */
+            std::string model_dir_path;
+            /* \brief task to perform */
+            std::string task;
+            /* \brief name of predict file */
+            std::string name_pred;
+            /* \brief the paths of validation data sets */
+            std::vector<std::string> eval_data_paths;            
+            /* \brief the names of the evaluation data used in output log */
+            std::vector<std::string> eval_data_names;
+            /*! \brief saves configurations */
+            utils::ConfigSaver cfg;
+        private:
+            DMatrix data;
+            std::vector<DMatrix*> deval;
+            RegBoostLearner learner;
+        };
+    };
+};
+
+int main( int argc, char *argv[] ){
+    xgboost::random::Seed( 0 );
+    xgboost::regression::RegBoostTask tsk;
+    return tsk.Run( argc, argv );
 }

regression/xgboost_reg_test.h

@@ -27,7 +27,7 @@ namespace xgboost{
             * \param silent whether to print feedback messages
             */
             void test(char* config_path,bool silent = false){
-                reg_boost_learner = new xgboost::regression::RegBoostLearner(silent);
+                reg_boost_learner = new xgboost::regression::RegBoostLearner();
                 ConfigIterator config_itr(config_path);
                 //Get the training data and validation data paths, config the Learner
                 while (config_itr.Next()){
@@ -42,10 +42,11 @@ namespace xgboost{
                 reg_boost_learner->InitModel();
                 char model_path[256];
                 std::vector<float> preds;
-                for(int i = 0; i < test_param.test_paths.size(); i++){
+                for(size_t i = 0; i < test_param.test_paths.size(); i++){
                     xgboost::regression::DMatrix test_data;
                     test_data.LoadText(test_param.test_paths[i].c_str());
                     sprintf(model_path,"%s/final.model",test_param.model_dir_path);
+                    // BUG: model need to be rb
                     FileStream fin(fopen(model_path,"r"));
                     reg_boost_learner->LoadModel(fin);
                     fin.Close();

regression/xgboost_reg_train.h

@@ -1,13 +1,13 @@
 #ifndef _XGBOOST_REG_TRAIN_H_
 #define _XGBOOST_REG_TRAIN_H_
 
-#include<iostream>
+#include <iostream>
-#include<string>
+#include <string>
-#include<fstream>
+#include <fstream>
-#include"../utils/xgboost_config.h"
+#include "../utils/xgboost_config.h"
-#include"xgboost_reg.h"
+#include "xgboost_reg.h"
-#include"xgboost_regdata.h"
+#include "xgboost_regdata.h"
-#include"../utils/xgboost_string.h"
+#include "../utils/xgboost_string.h"
 
 using namespace xgboost::utils;
 
@@ -28,7 +28,8 @@ namespace xgboost{
             * \param silent whether to print feedback messages
             */
             void train(char* config_path,bool silent = false){
-                reg_boost_learner = new xgboost::regression::RegBoostLearner(silent);
+                reg_boost_learner = new xgboost::regression::RegBoostLearner();
+                
                 ConfigIterator config_itr(config_path);
                 //Get the training data and validation data paths, config the Learner
                 while (config_itr.Next()){
@@ -38,14 +39,14 @@ namespace xgboost{
                 }
 
                 Assert(train_param.validation_data_paths.size() == train_param.validation_data_names.size(),
-                    "The number of validation paths is not the same as the number of validation data set names");
+                       "The number of validation paths is not the same as the number of validation data set names");
 
                 //Load Data
                 xgboost::regression::DMatrix train;
                 printf("%s",train_param.train_path);
                 train.LoadText(train_param.train_path);
                 std::vector<const xgboost::regression::DMatrix*> evals;
-                for(int i = 0; i < train_param.validation_data_paths.size(); i++){
+                for(size_t i = 0; i < train_param.validation_data_paths.size(); i++){
                     xgboost::regression::DMatrix eval;
                     eval.LoadText(train_param.validation_data_paths[i].c_str());
                     evals.push_back(&eval);
@@ -58,7 +59,7 @@ namespace xgboost{
                 for(int i = 1; i <= train_param.boost_iterations; i++){
                     reg_boost_learner->UpdateOneIter(i);
                     if(train_param.save_period != 0 && i % train_param.save_period == 0){
-                        sscanf(suffix,"%d.model",i);
+                        sprintf(suffix,"%d.model",i);
                         SaveModel(suffix);
                     }
                 }

regression/xgboost_regdata.h

@@ -31,12 +31,10 @@ namespace xgboost{
             /*! \brief default constructor */
             DMatrix( void ){}
 
-
             /*! \brief get the number of instances */
-            inline int size() const{
+            inline size_t Size() const{
                 return labels.size();
             }
-
             /*! 
             * \brief load from text file 
             * \param fname name of text data