lambda rank added

rank/xgboost_rank.h

@@ -0,0 +1,295 @@
+#ifndef XGBOOST_RANK_H
+#define XGBOOST_RANK_H
+/*!
+* \file xgboost_rank.h
+* \brief class for gradient boosting ranking
+* \author Kailong Chen: chenkl198812@gmail.com, Tianqi Chen: tianqi.tchen@gmail.com
+*/
+#include <cmath>
+#include <cstdlib>
+#include <vector>
+#include "xgboost_sample.h"
+#include "xgboost_rank_eval.h"
+#include "../base/xgboost_data_instance.h"
+#include "../utils/xgboost_omp.h"
+#include "../booster/xgboost_gbmbase.h"
+#include "../utils/xgboost_utils.h"
+#include "../utils/xgboost_stream.h"
+#include "../base/xgboost_learner.h"
+
+namespace xgboost {
+    namespace rank {
+        /*! \brief class for gradient boosted regression */
+        class RankBoostLearner :public base::BoostLearner{
+        public:
+            /*! \brief constructor */
+            RankBoostLearner(void) {
+                BoostLearner();
+            }
+            /*!
+            * \brief a rank booster associated with training and evaluating data
+            * \param train pointer to the training data
+            * \param evals array of evaluating data
+            * \param evname name of evaluation data, used print statistics
+            */
+            RankBoostLearner(const base::DMatrix *train,
+                const std::vector<base::DMatrix *> &evals,
+                const std::vector<std::string> &evname) {
+
+                BoostLearner(train, evals, evname);
+            }
+
+            /*!
+            * \brief initialize solver before training, called before training
+            * this function is reserved for solver to allocate necessary space
+            * and do other preparation
+            */
+            inline void InitTrainer(void) {
+                BoostLearner::InitTrainer();
+                if (mparam.loss_type == PAIRWISE) {
+                    evaluator_.AddEval("PAIR");
+                }
+                else if (mparam.loss_type == MAP) {
+                    evaluator_.AddEval("MAP");
+                }
+                else {
+                    evaluator_.AddEval("NDCG");
+                }
+                evaluator_.Init();
+            }
+
+            void EvalOneIter(int iter, FILE *fo = stderr) {
+                fprintf(fo, "[%d]", iter);
+                int buffer_offset = static_cast<int>(train_->Size());
+
+                for (size_t i = 0; i < evals_.size(); ++i) {
+                    std::vector<float> &preds = this->eval_preds_[i];
+                    this->PredictBuffer(preds, *evals_[i], buffer_offset);
+                    evaluator_.Eval(fo, evname_[i].c_str(), preds, (*evals_[i]).labels, (*evals_[i]).group_index);
+                    buffer_offset += static_cast<int>(evals_[i]->Size());
+                }
+                fprintf(fo, "\n");
+            }
+
+            virtual inline void SetParam(const char *name, const char *val){
+                BoostLearner::SetParam(name,val);
+	        if (!strcmp(name, "eval_metric"))  evaluator_.AddEval(val);
+                if (!strcmp(name, "rank:sampler"))  sampler.AssignSampler(atoi(val));
+            }
+            
+	private:
+	    inline std::vector< Triple<float,float,int> > GetSortedTuple(const std::vector<float> &preds,
+                const std::vector<float> &labels,
+                const std::vector<int> &group_index,
+		int group){
+		std::vector< Triple<float,float,int> > sorted_triple;
+		for(int j = group_index[group]; j < group_index[group+1]; j++){
+		      sorted_triple.push_back(Triple<float,float,int>(preds[j],labels[j],j));
+		}
+                std::sort(sorted_triple.begin(),sorted_triple.end(),Triplef1Comparer);
+		return sorted_triple;
+	    }
+	    
+	    inline std::vector<int> GetIndexMap(std::vector< Triple<float,float,int> > sorted_triple,int start){
+		std::vector<int> index_remap;
+		index_remap.resize(sorted_triple.size());
+		for(int i = 0; i < sorted_triple.size(); i++){
+		  index_remap[sorted_triple[i].f3_-start] = i;
+		}
+		return index_remap;
+	    }
+	    
+	    inline float GetLambdaMAP(const std::vector< Triple<float,float,int> > sorted_triple,
+		int index1,int index2,
+		std::vector< Quadruple<float,float,float,float> > map_acc){
+	        if(index1 > index2) std::swap(index1,index2);
+		float original = map_acc[index2].f1_;
+		if(index1 != 0) original -= map_acc[index1 - 1].f1_;
+		float changed = 0;
+		if(sorted_triple[index1].f2_ < sorted_triple[index2].f2_){
+		  changed += map_acc[index2 - 1].f3_ - map_acc[index1].f3_;
+		  changed += (map_acc[index1].f4_ + 1.0f)/(index1 + 1);
+		}else{
+		  changed += map_acc[index2 - 1].f2_ - map_acc[index1].f2_;
+		  changed += map_acc[index2].f4_/(index2 + 1);
+		}
+		float ans = (changed - original)/(map_acc[map_acc.size() - 1].f4_);
+		if(ans < 0) ans = -ans;
+		return ans;
+	    }
+	    
+	    inline float GetLambdaNDCG(const std::vector< Triple<float,float,int> > sorted_triple,
+		int index1,
+		int index2,float IDCG){
+	        float original = pow(2,sorted_triple[index1].f2_)/log(index1+2)
+		                + pow(2,sorted_triple[index2].f2_)/log(index2+2);
+	        float changed = pow(2,sorted_triple[index2].f2_)/log(index1+2)
+		                + pow(2,sorted_triple[index1].f2_)/log(index2+2);
+		float ans = (original - changed)/IDCG;
+		if(ans < 0) ans = -ans;
+		return ans;
+	    }
+	    
+	    
+	    inline float GetIDCG(const std::vector< Triple<float,float,int> > sorted_triple){
+		std::vector<float> labels;
+		for(int i = 0; i < sorted_triple.size(); i++){
+		  labels.push_back(sorted_triple[i].f2_);
+		}
+		
+		std::sort(labels.begin(),labels.end(),std::greater<float>());
+		return EvalNDCG::DCG(labels);
+	    }
+	    
+	    inline std::vector< Quadruple<float,float,float,float> > GetMAPAcc(const std::vector< Triple<float,float,int> > sorted_triple){
+		std::vector< Quadruple<float,float,float,float> > map_acc;
+		float hit = 0,acc1 = 0,acc2 = 0,acc3 = 0;
+		for(int i = 0; i < sorted_triple.size(); i++){
+		    if(sorted_triple[i].f2_ == 1) {
+		      hit++;
+		      acc1 += hit /( i + 1 );
+		      acc2 += (hit - 1)/(i+1);
+		      acc3 += (hit + 1)/(i+1);
+		    }
+		    map_acc.push_back(Quadruple<float,float,float,float>(acc1,acc2,acc3,hit));
+		}
+		return map_acc;
+	      
+	    }
+	    
+	    inline void GetGroupGradient(const std::vector<float> &preds,
+                const std::vector<float> &labels,
+                const std::vector<int> &group_index,
+                std::vector<float> &grad,
+                std::vector<float> &hess,
+		const std::vector< Triple<float,float,int> > sorted_triple,
+		const std::vector<int> index_remap,
+		const sample::Pairs& pairs,
+	        int group){
+	        bool j_better;
+                float IDCG, pred_diff, pred_diff_exp, delta;
+		float first_order_gradient, second_order_gradient;
+                std::vector< Quadruple<float,float,float,float> > map_acc;
+		
+		if(mparam.loss_type == NDCG){
+		  IDCG = GetIDCG(sorted_triple);
+		}else if(mparam.loss_type == MAP){
+		  map_acc = GetMAPAcc(sorted_triple);
+		}
+		
+		for (int j = group_index[group]; j < group_index[group + 1]; j++){
+		    std::vector<int> pair_instance = pairs.GetPairs(j);
+                    for (int k = 0; k < pair_instance.size(); k++){
+			j_better = labels[j] > labels[pair_instance[k]];
+                        if (j_better){
+			    switch(mparam.loss_type){
+			      case PAIRWISE: delta = 1.0;break;
+			      case MAP: delta = GetLambdaMAP(sorted_triple,index_remap[j - group_index[group]],index_remap[pair_instance[k]-group_index[group]],map_acc);break;
+			      case NDCG: delta = GetLambdaNDCG(sorted_triple,index_remap[j - group_index[group]],index_remap[pair_instance[k]-group_index[group]],IDCG);break;
+			      default: utils::Error("Cannot find the specified loss type");
+			    }
+		        
+			    pred_diff = preds[preds[j] - pair_instance[k]];
+			    pred_diff_exp = j_better ? expf(-pred_diff) : expf(pred_diff);
+			    first_order_gradient = delta * FirstOrderGradient(pred_diff_exp);
+			    second_order_gradient = 2 * delta * SecondOrderGradient(pred_diff_exp);
+			    hess[j] += second_order_gradient;
+			    grad[j] += first_order_gradient;
+			    hess[pair_instance[k]] += second_order_gradient;
+			    grad[pair_instance[k]] += -first_order_gradient;
+			}
+		    }
+                }
+	    }
+	public:
+            /*! \brief get the first order and second order gradient, given the 
+	     *         intransformed predictions and labels */
+            inline void GetGradient(const std::vector<float> &preds,
+                const std::vector<float> &labels,
+                const std::vector<int> &group_index,
+                std::vector<float> &grad,
+                std::vector<float> &hess) {
+                grad.resize(preds.size());
+                hess.resize(preds.size());
+                for (int i = 0; i < group_index.size() - 1; i++){
+		    sample::Pairs pairs = sampler.GenPairs(preds, labels, group_index[i], group_index[i + 1]);
+                    //pairs.GetPairs()
+		    std::vector< Triple<float,float,int> > sorted_triple = GetSortedTuple(preds,labels,group_index,i);
+		    std::vector<int> index_remap = GetIndexMap(sorted_triple,group_index[i]);
+		    GetGroupGradient(preds,labels,group_index,
+			grad,hess,sorted_triple,index_remap,pairs,i);
+                }
+            }
+
+            inline void UpdateInteract(std::string action) {
+		this->InteractPredict(preds_, *train_, 0);
+
+                int buffer_offset = static_cast<int>(train_->Size());
+                for (size_t i = 0; i < evals_.size(); ++i){
+                    std::vector<float> &preds = this->eval_preds_[i];
+                    this->InteractPredict(preds, *evals_[i], buffer_offset);
+                    buffer_offset += static_cast<int>(evals_[i]->Size());
+                }
+
+                if (action == "remove"){
+                    base_gbm.DelteBooster(); return;
+                }
+
+                this->GetGradient(preds_, train_->labels,train_->group_index, grad_, hess_);
+                std::vector<unsigned> root_index;
+                base_gbm.DoBoost(grad_, hess_, train_->data, root_index);
+
+                this->InteractRePredict(*train_, 0);
+                buffer_offset = static_cast<int>(train_->Size());
+                for (size_t i = 0; i < evals_.size(); ++i){
+                    this->InteractRePredict(*evals_[i], buffer_offset);
+                    buffer_offset += static_cast<int>(evals_[i]->Size());
+                }
+            }
+            
+            
+            
+        private:
+            enum LossType {
+                PAIRWISE = 0,
+                MAP = 1,
+                NDCG = 2
+            };
+
+
+
+            /*!
+            * \brief calculate first order gradient of pairwise loss function(f(x) = ln(1+exp(-x)),
+            * given the exponential of the difference of intransformed pair predictions
+            * \param the intransformed prediction of positive instance
+            * \param the intransformed prediction of negative instance
+            * \return first order gradient
+            */
+            inline float FirstOrderGradient(float pred_diff_exp) const {
+                return -pred_diff_exp / (1 + pred_diff_exp);
+            }
+
+            /*!
+            * \brief calculate second order gradient of pairwise loss function(f(x) = ln(1+exp(-x)),
+            * given the exponential of the difference of intransformed pair predictions
+            * \param the intransformed prediction of positive instance
+            * \param the intransformed prediction of negative instance
+            * \return second order gradient
+            */
+            inline float SecondOrderGradient(float pred_diff_exp) const {
+                return pred_diff_exp / pow(1 + pred_diff_exp, 2);
+            }
+
+        private:
+            RankEvalSet evaluator_;
+            sample::PairSamplerWrapper sampler;
+        };
+    };
+};
+
+#endif
+
+
+
+
+

rank/xgboost_rank_eval.h

@@ -0,0 +1,237 @@
+#ifndef XGBOOST_RANK_EVAL_H
+#define XGBOOST_RANK_EVAL_H
+/*!
+* \file xgboost_rank_eval.h
+* \brief evaluation metrics for ranking
+* \author Kailong Chen: chenkl198812@gmail.com, Tianqi Chen: tianqi.tchen@gmail.com
+*/
+
+#include <cmath>
+#include <vector>
+#include <algorithm>
+#include "../utils/xgboost_utils.h"
+#include "../utils/xgboost_omp.h"
+
+namespace xgboost {
+    namespace rank {
+        /*! \brief evaluator that evaluates the loss metrics */
+        class IRankEvaluator {
+        public:
+            /*!
+             * \brief evaluate a specific metric
+             * \param preds prediction
+             * \param labels label
+             */
+            virtual float Eval(const std::vector<float> &preds,
+                const std::vector<float> &labels,
+                const std::vector<int> &group_index) const = 0;
+            /*! \return name of metric */
+            virtual const char *Name(void) const = 0;
+        };
+
+        class Pair{
+        public:
+            float key_;
+            float value_;
+
+            Pair(float key, float value):key_(key),value_(value){
+            }
+        };
+
+        bool PairKeyComparer(const Pair &a, const Pair &b){  
+	  return a.key_ < b.key_;
+        }
+
+        bool PairValueComparer(const Pair &a, const Pair &b){
+            return a.value_ < b.value_;
+        }
+
+        template<typename T1,typename T2,typename T3>
+        class Triple{
+	public:
+	  T1 f1_;
+	  T2 f2_;
+	  T3 f3_;
+	  Triple(T1 f1,T2 f2,T3 f3):f1_(f1),f2_(f2),f3_(f3){
+	    
+	  }
+	};
+	
+	template<typename T1,typename T2,typename T3,typename T4>
+        class Quadruple{
+	public:
+	  T1 f1_;
+	  T2 f2_;
+	  T3 f3_;
+	  T4 f4_;
+	  Quadruple(T1 f1,T2 f2,T3 f3,T4 f4):f1_(f1),f2_(f2),f3_(f3),f4_(f4){
+	    
+	  }
+	};
+	
+	bool Triplef1Comparer(const Triple<float,float,int> &a, const Triple<float,float,int> &b){  
+	  return a.f1_< b.f1_;
+        }
+        
+        /*! \brief Mean Average Precision */
+        class EvalMAP : public IRankEvaluator {
+        public:
+            float Eval(const std::vector<float> &preds,
+                const std::vector<float> &labels,
+                const std::vector<int> &group_index) const {
+		if (group_index.size() <= 1) return 0;
+                float acc = 0;
+                std::vector<Pair> pairs_sort;
+                for (int i = 0; i < group_index.size() - 1; i++){
+                    for (int j = group_index[i]; j < group_index[i + 1]; j++){
+                        Pair pair(preds[j], labels[j]);
+                        pairs_sort.push_back(pair);
+                    }
+                    acc += average_precision(pairs_sort);
+                }
+                return acc / (group_index.size() - 1);
+            }
+            
+	    
+
+            virtual const char *Name(void) const {
+                return "MAP";
+            }
+	private:
+            float average_precision(std::vector<Pair> pairs_sort) const{
+
+                std::sort(pairs_sort.begin(), pairs_sort.end(), PairKeyComparer);
+                float hits = 0;
+                float average_precision = 0;
+                for (int j = 0; j < pairs_sort.size(); j++){
+                    if (pairs_sort[j].value_ == 1){
+                        hits++;
+                        average_precision += hits / (j + 1);
+                    }
+                }
+                if (hits != 0) average_precision /= hits;
+                return average_precision;
+            }
+        };
+
+
+        class EvalPair : public IRankEvaluator{
+        public:
+            float Eval(const std::vector<float> &preds,
+                const std::vector<float> &labels,
+                const std::vector<int> &group_index) const {
+		if (group_index.size() <= 1) return 0;
+                float acc = 0;
+                for (int i = 0; i < group_index.size() - 1; i++){
+                    acc += Count_Inversion(preds,labels,
+			group_index[i],group_index[i+1]);
+                }
+                return acc / (group_index.size() - 1);	  
+	    }
+
+            const char *Name(void) const {
+                return "PAIR";
+            }
+	private:
+	    float Count_Inversion(const std::vector<float> &preds,
+	      const std::vector<float> &labels,int begin,int end
+	    ) const{
+	      float ans = 0;
+	      for(int i = begin; i < end; i++){
+		for(int j = i + 1; j < end; j++){
+		  if(preds[i] > preds[j] && labels[i] < labels[j])
+		    ans++;
+		}
+	      }
+	      return ans;
+	    }
+        };
+
+        /*! \brief Normalized DCG */
+        class EvalNDCG : public IRankEvaluator {
+        public:
+            float Eval(const std::vector<float> &preds,
+                const std::vector<float> &labels,
+                const std::vector<int> &group_index) const {
+                if (group_index.size() <= 1) return 0;
+                float acc = 0;
+                std::vector<Pair> pairs_sort;
+                for (int i = 0; i < group_index.size() - 1; i++){
+                    for (int j = group_index[i]; j < group_index[i + 1]; j++){
+                        Pair pair(preds[j], labels[j]);
+                        pairs_sort.push_back(pair);
+                    }
+                    acc += NDCG(pairs_sort);
+                }
+                return acc / (group_index.size() - 1);
+            }
+            
+            static float DCG(const std::vector<float> &labels){
+		float ans = 0.0;
+                for (int i = 0; i < labels.size(); i++){
+                    ans += (pow(2,labels[i]) - 1 ) / log(i + 2);
+                }
+                return ans;
+	    }
+	    
+            virtual const char *Name(void) const {
+                return "NDCG";
+            }
+            
+	  private:
+            float NDCG(std::vector<Pair> pairs_sort) const{
+                std::sort(pairs_sort.begin(), pairs_sort.end(), PairKeyComparer);
+                float dcg = DCG(pairs_sort);
+                std::sort(pairs_sort.begin(), pairs_sort.end(), PairValueComparer);
+                float IDCG = DCG(pairs_sort);
+                if (IDCG == 0) return 0;
+                return dcg / IDCG;
+            }
+
+            float DCG(std::vector<Pair> pairs_sort) const{
+                std::vector<float> labels;
+	        for (int i = 1; i < pairs_sort.size(); i++){
+		  labels.push_back(pairs_sort[i].value_);
+		}
+                return DCG(labels);
+            }
+
+            
+        };
+
+    };
+
+    namespace rank {
+        /*! \brief a set of evaluators */
+        class RankEvalSet {
+        public:
+            inline void AddEval(const char *name) {
+                if (!strcmp(name, "PAIR")) evals_.push_back(&pair_);
+                if (!strcmp(name, "MAP")) evals_.push_back(&map_);
+                if (!strcmp(name, "NDCG")) evals_.push_back(&ndcg_);
+            }
+
+            inline void Init(void) {
+                std::sort(evals_.begin(), evals_.end());
+                evals_.resize(std::unique(evals_.begin(), evals_.end()) - evals_.begin());
+            }
+
+            inline void Eval(FILE *fo, const char *evname,
+                const std::vector<float> &preds,
+                const std::vector<float> &labels,
+                const std::vector<int> &group_index) const {
+                for (size_t i = 0; i < evals_.size(); ++i) {
+                    float res = evals_[i]->Eval(preds, labels, group_index);
+                    fprintf(fo, "\t%s-%s:%f", evname, evals_[i]->Name(), res);
+                }
+            }
+
+        private:
+            EvalPair pair_;
+            EvalMAP map_;
+            EvalNDCG ndcg_;
+            std::vector<const IRankEvaluator*> evals_;
+        };
+    };
+};
+#endif

rank/xgboost_rank_main.cpp

@@ -0,0 +1,22 @@
+#define _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_DEPRECATE
+#include <ctime>
+#include <string>
+#include <cstring>
+#include "../base/xgboost_learner.h"
+#include "../utils/xgboost_fmap.h"
+#include "../utils/xgboost_random.h"
+#include "../utils/xgboost_config.h"
+#include "../base/xgboost_learner.h"
+#include "../base/xgboost_boost_task.h"
+#include "xgboost_rank.h"
+#include "../regression/xgboost_reg.h"
+#include "../regression/xgboost_reg_main.cpp"
+#include "../base/xgboost_data_instance.h"
+
+int main(int argc, char *argv[]) {    
+  xgboost::random::Seed(0);
+  xgboost::base::BoostTask rank_tsk;
+  rank_tsk.SetLearner(new xgboost::rank::RankBoostLearner);
+  return rank_tsk.Run(argc, argv);
+}

rank/xgboost_sample.h

@@ -0,0 +1,128 @@
+#ifndef _XGBOOST_SAMPLE_H_
+#define _XGBOOST_SAMPLE_H_
+
+#include <vector>
+#include"../utils/xgboost_utils.h"
+
+namespace xgboost {
+    namespace rank {
+        namespace sample {
+
+            /*
+             * \brief the data structure to maintain the sample pairs
+             */
+            struct Pairs {
+
+                /*
+                 * \brief constructor given the start and end offset of the sampling group
+                 *        in overall instances
+                 * \param start the begin index of the group
+                 * \param end the end index of the group
+                 */
+                Pairs(int start, int end) :start_(start), end_(end){
+                    for (int i = start; i < end; i++){
+                        std::vector<int> v;
+                        pairs_.push_back(v);
+                    }
+                }
+                /*
+                 * \brief retrieve the related pair information of an data instances
+                 * \param index, the index of retrieved instance
+                 * \return the index of instances paired
+                 */
+                std::vector<int> GetPairs(int index) const{
+                    utils::Assert(index >= start_ && index < end_, "The query index out of sampling bound");
+                    return pairs_[index - start_];
+                }
+
+                /*
+                 * \brief add in a sampled pair
+                 * \param index the index of the instance to sample a friend
+                 * \param paired_index the index of the instance sampled as a friend
+                 */
+                void push(int index, int paired_index){
+                    pairs_[index - start_].push_back(paired_index);
+                }
+
+                std::vector< std::vector<int> > pairs_;
+                int start_;
+                int end_;
+            };
+
+            /*
+             * \brief the interface of pair sampler
+             */
+            struct IPairSampler {
+                /*
+                 * \brief Generate sample pairs given the predcions, labels, the start and the end index
+                 *        of a specified group
+                 * \param preds, the predictions of all data instances
+                 * \param labels, the labels of all data instances
+                 * \param start, the start index of a specified group
+                 * \param end, the end index of a specified group
+                 * \return the generated pairs
+                 */
+                virtual Pairs GenPairs(const std::vector<float> &preds,
+                const std::vector<float> &labels,
+                int start, int end) = 0;
+
+            };
+
+            enum{
+                BINARY_LINEAR_SAMPLER
+            };
+
+            /*! \brief A simple pair sampler when the rank relevence scale is binary
+             *         for each positive instance, we will pick a negative
+             *         instance and add in a pair. When using binary linear sampler,
+             *         we should guarantee the labels are 0 or 1
+             */
+            struct BinaryLinearSampler :public IPairSampler{
+                virtual Pairs GenPairs(const std::vector<float> &preds,
+                const std::vector<float> &labels,
+                int start, int end) {
+                    Pairs pairs(start, end);
+                    int pointer = 0, last_pointer = 0, index = start, interval = end - start;
+                    for (int i = start; i < end; i++){
+                        if (labels[i] == 1){
+                            while (true){
+                                index = (++pointer) % interval + start;
+                                if (labels[index] == 0) break;
+                                if (pointer - last_pointer > interval) return pairs;
+                            }
+                            pairs.push(i, index);
+                            pairs.push(index, i);
+                            last_pointer = pointer;
+                        }
+                    }
+                    return pairs;
+                }
+            };
+
+
+            /*! \brief Pair Sampler Wrapper*/
+            struct PairSamplerWrapper{
+            public:
+                inline void AssignSampler(int sampler_index){
+
+                    switch (sampler_index){
+                    case BINARY_LINEAR_SAMPLER:sampler_ = &binary_linear_sampler; break;
+
+                    default:utils::Error("Cannot find the specified sampler");
+                    }
+                }
+
+                Pairs GenPairs(const std::vector<float> &preds,
+                    const std::vector<float> &labels,
+                    int start, int end){
+		    utils::Assert(sampler_ != NULL,"Not config the sampler yet. Add rank:sampler in the config file\n");
+                    return sampler_->GenPairs(preds, labels, start, end);
+                }
+            private:
+                BinaryLinearSampler binary_linear_sampler;
+                IPairSampler *sampler_;
+            };
+        }
+    }
+}
+#endif