Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor io, initial hdfs file access need test

Browse Source
This commit is contained in:
tqchen
2015-03-06 22:17:27 -08:00
parent 19be870562
commit 88ce76767e
9 changed files with 352 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
rabit-learn/linear/linear.cc
View File

@@ -1,6 +1,5 @@
#include "./linear.h"
#include "../io/io.h"
#include "../io/base64.h"
namespace rabit {
namespace linear {
@@ -74,37 +73,37 @@ class LinearObjFunction : public solver::IObjFunction<float> {
printf("Finishing writing to %s\n", name_pred.c_str());
}
inline void LoadModel(const char *fname) {
io::FileStream fi(fname, "rb");
IStream *fi = io::CreateStream(fname, "r");
std::string header; header.resize(4);
// check header for different binary encode
// can be base64 or binary
utils::Check(fi.Read(&header[0], 4) != 0, "invalid model");
utils::Check(fi->Read(&header[0], 4) != 0, "invalid model");
// base64 format
if (header == "bs64") {
io::Base64InStream bsin(&fi);
io::Base64InStream bsin(fi);
bsin.InitPosition();
model.Load(bsin);
return;
} else if (header == "binf") {
model.Load(fi);
return;
model.Load(*fi);
} else {
utils::Error("invalid model file");
}
delete fi;
}
inline void SaveModel(const char *fname,
const float *wptr,
bool save_base64 = false) {
io::FileStream fo(fname, "wb");
IStream *fo = io::CreateStream(fname, "w");
if (save_base64 != 0 || !strcmp(fname, "stdout")) {
fo.Write("bs64\t", 5);
io::Base64OutStream bout(&fo);
fo->Write("bs64\t", 5);
io::Base64OutStream bout(fo);
model.Save(bout, wptr);
bout.Finish('\n');
} else {
fo.Write("binf", 4);
model.Save(fo, wptr);
fo->Write("binf", 4);
model.Save(*fo, wptr);
}
delete fo;
}
inline void LoadData(const char *fname) {
dtrain.Load(fname);