Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

pass local model recover test

Browse Source
This commit is contained in:
tqchen 2014-12-18 18:53:58 -08:00
parent dbd05a65b5
commit c8faed0b54
2 changed files with 19 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/allreduce_robust.cc
View File

@ -141,12 +141,17 @@ int AllreduceRobust::LoadCheckPoint(utils::ISerializable *global_model,
} }
// check if we succesful // check if we succesful
if (RecoverExec(NULL, 0, ActionSummary::kLoadCheck, ActionSummary::kSpecialOp)) { if (RecoverExec(NULL, 0, ActionSummary::kLoadCheck, ActionSummary::kSpecialOp)) {
int nlocal = std::max(static_cast<int>(local_rptr[local_chkpt_version].size()) - 1, 0);
if (local_model != NULL) { if (local_model != NULL) {
// load in local model if (nlocal == num_local_replica + 1) {
utils::MemoryFixSizeBuffer fs(BeginPtr(local_chkpt[local_chkpt_version]), // load in local model
local_rptr[local_chkpt_version][1]); utils::MemoryFixSizeBuffer fs(BeginPtr(local_chkpt[local_chkpt_version]),
local_model->Load(fs); local_rptr[local_chkpt_version][1]);
} local_model->Load(fs);
} else {
utils::Assert(nlocal == 0, "[%d] local model inconsistent, nlocal=%d", rank, nlocal);
}
}
// reset result buffer // reset result buffer
resbuf.Clear(); seq_counter = 0; resbuf.Clear(); seq_counter = 0;
// load from buffer // load from buffer
@ -156,6 +161,8 @@ int AllreduceRobust::LoadCheckPoint(utils::ISerializable *global_model,
} else { } else {
utils::Assert(fs.Read(&version_number, sizeof(version_number)) != 0, "read in version number"); utils::Assert(fs.Read(&version_number, sizeof(version_number)) != 0, "read in version number");
global_model->Load(fs); global_model->Load(fs);
utils::Assert(local_model == NULL || nlocal == num_local_replica + 1,
"local model inconsistent, nlocal=%d", nlocal);
} }
// run another phase of check ack, if recovered from data // run another phase of check ack, if recovered from data
utils::Assert(RecoverExec(NULL, 0, ActionSummary::kCheckAck, ActionSummary::kSpecialOp), utils::Assert(RecoverExec(NULL, 0, ActionSummary::kCheckAck, ActionSummary::kSpecialOp),

14
test/test_local_recover.cpp
View File

@ -26,6 +26,7 @@ class Model : public rabit::utils::ISerializable {
fo.Write(data); fo.Write(data);
} }
virtual void InitModel(size_t n, float v) { virtual void InitModel(size_t n, float v) {
data.clear();
data.resize(n, v); data.resize(n, v);
} }
}; };
@ -34,13 +35,13 @@ inline void TestMax(test::Mock &mock, Model *model, Model *local, int ntrial, in
int rank = rabit::GetRank(); int rank = rabit::GetRank();
int nproc = rabit::GetWorldSize(); int nproc = rabit::GetWorldSize();
const int z = iter + 111; const int z = iter + 111;
std::vector<float> ndata(model->data.size()); std::vector<float> ndata(model->data.size());
for (size_t i = 0; i < ndata.size(); ++i) { for (size_t i = 0; i < ndata.size(); ++i) {
ndata[i] = (i * (rank+1)) % z + local->data[i]; ndata[i] = (i * (rank+1)) % z + local->data[i];
} }
mock.Allreduce<op::Max>(&ndata[0], ndata.size()); mock.Allreduce<op::Max>(&ndata[0], ndata.size());
if (ntrial == iter && rank == 3) { if (ntrial == iter && rank == 1) {
throw MockException(); throw MockException();
} }
for (size_t i = 0; i < ndata.size(); ++i) { for (size_t i = 0; i < ndata.size(); ++i) {
@ -66,11 +67,10 @@ inline void TestSum(test::Mock &mock, Model *model, Model *local, int ntrial, in
for (size_t i = 0; i < ndata.size(); ++i) { for (size_t i = 0; i < ndata.size(); ++i) {
ndata[i] = (i * (rank+1)) % z + local->data[i]; ndata[i] = (i * (rank+1)) % z + local->data[i];
} }
mock.Allreduce<op::Sum>(&ndata[0], ndata.size());
if (ntrial == iter && rank == 0) { if (ntrial == iter && rank == 0) {
exit(-1); throw MockException();
} }
mock.Allreduce<op::Sum>(&ndata[0], ndata.size());
for (size_t i = 0; i < ndata.size(); ++i) { for (size_t i = 0; i < ndata.size(); ++i) {
float rsum = 0.0f; float rsum = 0.0f;
@ -135,9 +135,9 @@ int main(int argc, char *argv[]) {
utils::LogPrintf("[%d] !!!TestMax pass, iter=%d\n", rank, r); utils::LogPrintf("[%d] !!!TestMax pass, iter=%d\n", rank, r);
int step = std::max(nproc / 3, 1); int step = std::max(nproc / 3, 1);
for (int i = 0; i < nproc; i += step) { for (int i = 0; i < nproc; i += step) {
TestBcast(mock, n, i, ntrial); //TestBcast(mock, n, i, ntrial);
} }
utils::LogPrintf("[%d] !!!TestBcast pass, iter=%d\n", rank, r); //utils::LogPrintf("[%d] !!!TestBcast pass, iter=%d\n", rank, r);
TestSum(mock, &model, &local, ntrial, r); TestSum(mock, &model, &local, ntrial, r);
utils::LogPrintf("[%d] !!!TestSum pass, iter=%d\n", rank, r); utils::LogPrintf("[%d] !!!TestSum pass, iter=%d\n", rank, r);
rabit::CheckPoint(&model, &local); rabit::CheckPoint(&model, &local);