#ifndef XGBOOST_REGRANK_DATA_H #define XGBOOST_REGRANK_DATA_H /*! * \file xgboost_regrank_data.h * \brief input data structure for regression, binary classification, and rankning. * Format: * The data should contain each data instance in each line. * The format of line data is as below: * label [feature index:feature value]+ * When using rank, an addtional group file with suffix group must be provided, giving the number of instances in each group * When using weighted aware classification(regression), an addtional weight file must be provided, giving the weight of each instance * * \author Kailong Chen: chenkl198812@gmail.com, Tianqi Chen: tianqi.tchen@gmail.com */ #include #include #include "../booster/xgboost_data.h" #include "../utils/xgboost_utils.h" #include "../utils/xgboost_stream.h" namespace xgboost{ /*! \brief namespace to handle regression and rank */ namespace regrank{ /*! \brief data matrix for regression content */ struct DMatrix{ public: /*! \brief data information besides the features */ struct Info{ /*! \brief label of each instance */ std::vector labels; /*! \brief the index of begin and end of a groupneeded when the learning task is ranking */ std::vector group_ptr; /*! \brief weights of each instance, optional */ std::vector weights; /*! \brief get weight of each instances */ inline float GetWeight( size_t i ) const{ if( weights.size() != 0 ) return weights[i]; else return 1.0f; } }; public: /*! \brief feature data content */ booster::FMatrixS data; /*! \brief information fields */ Info info; public: /*! \brief default constructor */ DMatrix(void){} /*! \brief get the number of instances */ inline size_t Size() const{ return info.labels.size(); } /*! * \brief load from text file * \param fname name of text data * \param silent whether print information or not */ inline void LoadText(const char* fname, bool silent = false){ data.Clear(); FILE* file = utils::FopenCheck(fname, "r"); float label; bool init = true; char tmp[1024]; std::vector findex; std::vector fvalue; while (fscanf(file, "%s", tmp) == 1){ unsigned index; float value; if (sscanf(tmp, "%u:%f", &index, &value) == 2){ findex.push_back(index); fvalue.push_back(value); } else{ if (!init){ info.labels.push_back(label); data.AddRow(findex, fvalue); } findex.clear(); fvalue.clear(); utils::Assert(sscanf(tmp, "%f", &label) == 1, "invalid format"); init = false; } } info.labels.push_back(label); data.AddRow(findex, fvalue); // initialize column support as well data.InitData(); if (!silent){ printf("%ux%u matrix with %lu entries is loaded from %s\n", (unsigned)data.NumRow(), (unsigned)data.NumCol(), (unsigned long)data.NumEntry(), fname); } fclose(file); this->TryLoadGroup(fname, silent); this->TryLoadWeight(fname, silent); } /*! * \brief load from binary file * \param fname name of binary data * \param silent whether print information or not * \return whether loading is success */ inline bool LoadBinary(const char* fname, bool silent = false){ FILE *fp = fopen64(fname, "rb"); if (fp == NULL) return false; utils::FileStream fs(fp); data.LoadBinary(fs); info.labels.resize(data.NumRow()); utils::Assert(fs.Read(&info.labels[0], sizeof(float)* data.NumRow()) != 0, "DMatrix LoadBinary"); fs.Close(); // initialize column support as well data.InitData(); if (!silent){ printf("%ux%u matrix with %lu entries is loaded from %s\n", (unsigned)data.NumRow(), (unsigned)data.NumCol(), (unsigned long)data.NumEntry(), fname); } this->TryLoadGroup(fname, silent); this->TryLoadWeight(fname, silent); return true; } /*! * \brief save to binary file * \param fname name of binary data * \param silent whether print information or not */ inline void SaveBinary(const char* fname, bool silent = false){ // initialize column support as well data.InitData(); utils::FileStream fs(utils::FopenCheck(fname, "wb")); data.SaveBinary(fs); fs.Write(&info.labels[0], sizeof(float)* data.NumRow()); fs.Close(); if (!silent){ printf("%ux%u matrix with %lu entries is saved to %s\n", (unsigned)data.NumRow(), (unsigned)data.NumCol(), (unsigned long)data.NumEntry(), fname); } } /*! * \brief cache load data given a file name, if filename ends with .buffer, direct load binary * otherwise the function will first check if fname + '.buffer' exists, * if binary buffer exists, it will reads from binary buffer, otherwise, it will load from text file, * and try to create a buffer file * \param fname name of binary data * \param silent whether print information or not * \param savebuffer whether do save binary buffer if it is text */ inline void CacheLoad(const char *fname, bool silent = false, bool savebuffer = true){ int len = strlen(fname); if (len > 8 && !strcmp(fname + len - 7, ".buffer")){ this->LoadBinary(fname, silent); return; } char bname[1024]; sprintf(bname, "%s.buffer", fname); if (!this->LoadBinary(bname, silent)){ this->LoadText(fname, silent); if (savebuffer) this->SaveBinary(bname, silent); } } private: inline bool TryLoadGroup(const char* fname, bool silent = false){ std::string name = fname; if (name.length() > 8 && !strcmp(fname + name.length() - 7, ".buffer")){ name.resize( name.length() - 7 ); } name += ".group"; //if exists group data load it in FILE *fi = fopen64(name.c_str(), "r"); if (fi == NULL) return false; info.group_ptr.push_back(0); unsigned nline; while (fscanf(fi, "%u", &nline) == 1){ info.group_ptr.push_back(info.group_ptr.back()+nline); } if(!silent){ printf("%lu groups are loaded from %s\n", info.group_ptr.size()-1, name.c_str()); } fclose(fi); utils::Assert( info.group_ptr.back() == data.NumRow(), "DMatrix: group data does not match the number of rows in feature matrix" ); return true; } inline bool TryLoadWeight(const char* fname, bool silent = false){ std::string name = fname; if (name.length() > 8 && !strcmp(fname + name.length() - 7, ".buffer")){ name.resize( name.length() - 7 ); } name += ".weight"; //if exists group data load it in FILE *fi = fopen64(name.c_str(), "r"); if (fi == NULL) return false; float wt; while (fscanf(fi, "%f", &wt) == 1){ info.weights.push_back( wt ); } if(!silent){ printf("loading weight from %s\n", name.c_str()); } fclose(fi); utils::Assert( info.weights.size() == data.NumRow(), "DMatrix: weight data does not match the number of rows in feature matrix" ); return true; } }; }; }; #endif