half ram support

src/io/io.cpp

@@ -66,10 +66,16 @@ DataMatrix* LoadDataMatrix(const char *fname,
       dmat->LoadBinary(fs, silent, cache_file);
       fs.Close();
       return dmat;
-    } else {   
-      DMatrixPage *dmat = new DMatrixPage();
-      dmat->LoadText(fname, cache_file, false, loadsplit);
-      return dmat;
+    } else {
+      if (fname[0] == '!') {
+        DMatrixHalfRAM *dmat = new DMatrixHalfRAM();
+        dmat->LoadText(fname + 1, cache_file, false, loadsplit);
+        return dmat;
+      } else {
+        DMatrixPage *dmat = new DMatrixPage();
+        dmat->LoadText(fname, cache_file, false, loadsplit);
+        return dmat;
+      }
     }
   }
 }

src/io/page_dmatrix-inl.hpp

@@ -224,6 +224,25 @@ class DMatrixPage : public DMatrixPageBase<0xffffab02> {
   /*! \brief the real fmatrix */
   FMatrixPage *fmat_;
 };
+
+// mix of FMatrix S and DMatrix
+// cost half of ram usually as DMatrixSimple
+class DMatrixHalfRAM : public DMatrixPageBase<0xffffab03> {
+ public:
+  DMatrixHalfRAM(void) {
+    fmat_ = new FMatrixS(iter_);
+  }
+  virtual ~DMatrixHalfRAM(void) {
+    delete fmat_;
+  }
+  virtual IFMatrix *fmat(void) const {
+    return fmat_;
+  }
+  virtual void set_cache_file(const std::string &cache_file) {
+  }
+  /*! \brief the real fmatrix */
+  IFMatrix *fmat_;
+};
 }  // namespace io
 }  // namespace xgboost
 #endif  // XGBOOST_IO_PAGE_ROW_ITER_INL_HPP_

src/learner/learner-inl.hpp

@@ -260,9 +260,9 @@ class BoostLearner : public rabit::Serializable {
     std::vector<bool> enabled(ncol, true);    
     // initialize column access
     p_train->fmat()->InitColAccess(enabled, prob_buffer_row);
-    const int kMagicSimple = 0xffffab01;
-    // check, if it is not DMatrix simple, then use hist maker
-    if (p_train->magic != kMagicSimple) {
+    const int kMagicPage = 0xffffab02;
+    // check, if it is DMatrixPage, then use hist maker
+    if (p_train->magic == kMagicPage) {
       this->SetParam("updater", "grow_histmaker,prune");
     }
   }