Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

nice fix, start check local check

Browse Source
This commit is contained in:
tqchen 2014-12-18 18:39:24 -08:00
parent 3f22596e3c
commit dbd05a65b5
2 changed files with 29 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
src/allreduce_robust.cc
View File

@ -151,18 +151,21 @@ int AllreduceRobust::LoadCheckPoint(utils::ISerializable *global_model,
resbuf.Clear(); seq_counter = 0;
// load from buffer
utils::MemoryBufferStream fs(&global_checkpoint);
fs.Read(&version_number, sizeof(version_number));
if (version_number == 0) return version_number;
global_model->Load(fs);
if (global_checkpoint.length() == 0) {
version_number = 0;
} else {
utils::Assert(fs.Read(&version_number, sizeof(version_number)) != 0, "read in version number");
global_model->Load(fs);
}
// run another phase of check ack, if recovered from data
utils::Assert(RecoverExec(NULL, 0, ActionSummary::kCheckAck, ActionSummary::kSpecialOp),
"check ack must return true");
return version_number;
} else {
// reset result buffer
resbuf.Clear(); seq_counter = 0;
resbuf.Clear(); seq_counter = 0; version_number = 0;
// nothing loaded, a fresh start, everyone init model
return false;
return version_number;
}
}
/*!
@ -185,8 +188,8 @@ void AllreduceRobust::CheckPoint(const utils::ISerializable *global_model,
const utils::ISerializable *local_model) {
if (num_local_replica == 0) {
utils::Check(local_model == NULL, "need to set num_local_replica larger than 1 to checkpoint local_model");
}
if (num_local_replica != 0) {
}
if (num_local_replica != 0) {
while (true) {
if (RecoverExec(NULL, 0, 0, ActionSummary::kLocalCheckPoint)) break;
// save model model to new version place
@ -516,7 +519,7 @@ AllreduceRobust::TryRecoverData(RecoverType role,
int recv_link,
const std::vector<bool> &req_in) {
RefLinkVector &links = tree_links;
// no need to run recovery for zero size message
// no need to run recovery for zero size messages
if (links.size() == 0 || size == 0) return kSuccess;
utils::Assert(req_in.size() == links.size(), "TryRecoverData");
const int nlink = static_cast<int>(links.size());
@ -542,7 +545,7 @@ AllreduceRobust::TryRecoverData(RecoverType role,
if (i == recv_link && links[i].size_read != size) {
selecter.WatchRead(links[i].sock);
finished = false;
}
}
if (req_in[i] && links[i].size_write != size) {
if (role == kHaveData ||
(role == kPassData && links[recv_link].size_read != links[i].size_write)) {
@ -620,16 +623,16 @@ AllreduceRobust::ReturnType AllreduceRobust::TryLoadCheckPoint(bool requester) {
// check in local data
RecoverType role = requester ? kRequestData : kHaveData;
ReturnType succ;
if (false) {
if (num_local_replica != 0) {
if (requester) {
// clear existing history, if any, before load
local_rptr[local_chkpt_version].clear();
local_chkpt[local_chkpt_version].clear();
}
// recover local checkpoint
//succ = TryRecoverLocalState(&local_rptr[local_chkpt_version],
//m&local_chkpt[local_chkpt_version]);
//if (succ != kSuccess) return succ;
succ = TryRecoverLocalState(&local_rptr[local_chkpt_version],
&local_chkpt[local_chkpt_version]);
if (succ != kSuccess) return succ;
int nlocal = std::max(static_cast<int>(local_rptr[local_chkpt_version].size()) - 1, 0);
// check if everyone is OK
unsigned state = 0;
@ -818,8 +821,7 @@ AllreduceRobust::TryRecoverLocalState(std::vector<size_t> *p_local_rptr,
utils::Assert(chkpt.length() == 0, "local chkpt space inconsistent");
}
const int n = num_local_replica;
utils::LogPrintf("[%d] backward!!\n", rabit::GetRank());
if(false){// backward passing, passing state in backward direction of the ring
{// backward passing, passing state in backward direction of the ring
const int nlocal = static_cast<int>(rptr.size() - 1);
utils::Assert(nlocal <= n + 1, "invalid local replica");
std::vector<int> msg_back(n + 1);
@ -872,8 +874,6 @@ AllreduceRobust::TryRecoverLocalState(std::vector<size_t> *p_local_rptr,
rptr.resize(nlocal + 1); chkpt.resize(rptr.back()); return succ;
}
}
utils::LogPrintf("[%d] FORward!!\n", rabit::GetRank());
{// forward passing, passing state in forward direction of the ring
const int nlocal = static_cast<int>(rptr.size() - 1);
utils::Assert(nlocal <= n + 1, "invalid local replica");
@ -937,7 +937,6 @@ AllreduceRobust::TryRecoverLocalState(std::vector<size_t> *p_local_rptr,
rptr.resize(nlocal + 1); chkpt.resize(rptr.back()); return succ;
}
}
utils::LogPrintf("[%d] Finished!!\n", rabit::GetRank());
return kSuccess;
}
/*!

23
test/test_model_recover.cpp
View File

@ -26,6 +26,7 @@ class Model : public rabit::utils::ISerializable {
fo.Write(data);
}
virtual void InitModel(size_t n) {
data.clear();
data.resize(n, 1.0f);
}
};
@ -40,15 +41,15 @@ inline void TestMax(test::Mock &mock, Model *model, int ntrial, int iter) {
ndata[i] = (i * (rank+1)) % z + model->data[i];
}
mock.Allreduce<op::Max>(&ndata[0], ndata.size());
if (ntrial == iter && rank == 3) {
// exit(-1);
if (ntrial == 0 && rank == 3) {
exit(-1);
}
for (size_t i = 0; i < ndata.size(); ++i) {
float rmax = (i * 1) % z + model->data[i];
for (int r = 0; r < nproc; ++r) {
rmax = std::max(rmax, (float)((i * (r+1)) % z) + model->data[i]);
}
utils::Check(rmax == ndata[i], "[%d] TestMax check failure", rank);
utils::Check(rmax == ndata[i], "[%d] TestMax check failurem i=%lu, rmax=%f, ndata=%f", rank, i, rmax, ndata[i]);
}
model->data = ndata;
}
@ -62,12 +63,12 @@ inline void TestSum(test::Mock &mock, Model *model, int ntrial, int iter) {
for (size_t i = 0; i < ndata.size(); ++i) {
ndata[i] = (i * (rank+1)) % z + model->data[i];
}
mock.Allreduce<op::Sum>(&ndata[0], ndata.size());
if (ntrial == iter && rank == 0) {
if (iter == 0 && ntrial==0 && rank == 0) {
throw MockException();
}
mock.Allreduce<op::Sum>(&ndata[0], ndata.size());
for (size_t i = 0; i < ndata.size(); ++i) {
float rsum = model->data[i] * nproc;
for (int r = 0; r < nproc; ++r) {
@ -125,11 +126,11 @@ int main(int argc, char *argv[]) {
for (int r = iter; r < 3; ++r) {
TestMax(mock, &model, ntrial, r);
utils::LogPrintf("[%d] !!!TestMax pass, iter=%d\n", rank, r);
int step = std::max(nproc / 3, 1);
for (int i = 0; i < nproc; i += step) {
TestBcast(mock, n, i, ntrial);
}
utils::LogPrintf("[%d] !!!TestBcast pass, iter=%d\n", rank, r);
//int step = std::max(nproc / 3, 1);
//for (int i = 0; i < nproc; i += step) {
//TestBcast(mock, n, i, ntrial);
//}
//utils::LogPrintf("[%d] !!!TestBcast pass, iter=%d\n", rank, r);
TestSum(mock, &model, ntrial, r);
utils::LogPrintf("[%d] !!!TestSum pass, iter=%d\n", rank, r);
rabit::CheckPoint(&model);