Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add auto caching of python in hadoop script, mock test module to python, with checkpt

Browse Source
This commit is contained in:
tqchen
2015-01-13 14:29:10 -08:00
parent 877fc42e40
commit 3419cf9aa7
10 changed files with 104 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
test/Makefile
View File

@@ -8,13 +8,15 @@ export CFLAGS = -Wall -O3 -msse2 -Wno-unknown-pragmas -fPIC -I../include -std=
BIN = speed_test test_model_recover test_local_recover
OBJ = $(RABIT_OBJ) speed_test.o test_model_recover.o test_local_recover.o
MPIBIN = speed_test.mpi
.PHONY: clean all lib
.PHONY: clean all lib mpi
all: $(BIN) $(MPIBIN)
lib:
cd ..;make;cd -
mpi:
cd ..;make mpi;cd -
# programs
speed_test.o: speed_test.cpp ../include/*.h lib
speed_test.o: speed_test.cpp ../include/*.h lib mpi
test_model_recover.o: test_model_recover.cpp ../include/*.h lib
test_local_recover.o: test_local_recover.cpp ../include/*.h lib

6
test/test.mk
View File

@@ -9,12 +9,6 @@ endif
.PHONY: model_recover local_recover speed
local_recover:
../tracker/rabit_demo.py -n $(nslave) test_local_recover $(ndata) rabit_local_replica=1
local_recover_10_10k:
../tracker/rabit_demo.py -n 10 test_local_recover 10000 rabit_local_replica=1
# this experiment test recovery with actually process exit, use keepalive to keep program alive
model_recover_10_10k:
../tracker/rabit_demo.py -n 10 test_model_recover 10000 mock=0,0,1,0 mock=1,1,1,0

75
test/test_local_recover.cpp
View File

@@ -5,28 +5,8 @@
#include <cstdio>
#include <cstdlib>
#include <cmath>
#include "./mock.h"
using namespace rabit;
namespace rabit {
namespace test {
inline void CallBegin(const char *fun, int ntrial, int iter) {
int rank = rabit::GetRank();
if (!strcmp(fun, "Allreduce::Sum")) {
if (ntrial == iter && rank == 0) throw MockException();
}
if (!strcmp(fun, "Allreduce::Max")) {
if (ntrial == iter && rank == 3) throw MockException();
}
}
inline void CallEnd(const char *fun, int ntrial, int iter) {
int rank = rabit::GetRank();
if (!strcmp(fun, "Allreduce::Bcast")) {
if (ntrial == iter && rand() % 10 == rank) throw MockException();
}
}
}
}
// dummy model
class Model : public rabit::ISerializable {
@@ -52,8 +32,6 @@ inline void TestMax(Model *model, Model *local, int ntrial, int iter) {
int nproc = rabit::GetWorldSize();
const int z = iter + 111;
std::vector<float> ndata(model->data.size());
test::CallBegin("Allreduce::Max", ntrial, iter);
rabit::Allreduce<op::Max>(&ndata[0], ndata.size(),
[&]() {
// use lambda expression to prepare the data
@@ -61,7 +39,6 @@ inline void TestMax(Model *model, Model *local, int ntrial, int iter) {
ndata[i] = (i * (rank+1)) % z + local->data[i];
}
});
test::CallEnd("Allreduce::Max", ntrial, iter);
for (size_t i = 0; i < ndata.size(); ++i) {
float rmax = (i * 1) % z + model->data[i];
@@ -86,9 +63,7 @@ inline void TestSum(Model *model, Model *local, int ntrial, int iter) {
for (size_t i = 0; i < ndata.size(); ++i) {
ndata[i] = (i * (rank+1)) % z + local->data[i];
}
test::CallBegin("Allreduce::Sum", ntrial, iter);
Allreduce<op::Sum>(&ndata[0], ndata.size());
test::CallEnd("Allreduce::Sum", ntrial, iter);
for (size_t i = 0; i < ndata.size(); ++i) {
float rsum = 0.0f;
@@ -113,13 +88,9 @@ inline void TestBcast(size_t n, int root, int ntrial, int iter) {
std::string res;
if (root == rank) {
res = s;
test::CallBegin("Broadcast", ntrial, iter);
rabit::Broadcast(&res, root);
test::CallBegin("Broadcast", ntrial, iter);
} else {
test::CallBegin("Broadcast", ntrial, iter);
rabit::Broadcast(&res, root);
test::CallEnd("Broadcast", ntrial, iter);
}
utils::Check(res == s, "[%d] TestBcast fail", rank);
}
@@ -141,34 +112,26 @@ int main(int argc, char *argv[]) {
int n;
if (sscanf(argv[i], "repeat=%d", &n) == 1) ntrial = n;
}
while (true) {
try {
int iter = rabit::LoadCheckPoint(&model, &local);
if (iter == 0) {
model.InitModel(n, 1.0f);
local.InitModel(n, 1.0f + rank);
printf("[%d] reload-trail=%d, init iter=%d\n", rank, ntrial, iter);
} else {
printf("[%d] reload-trail=%d, init iter=%d\n", rank, ntrial, iter);
}
for (int r = iter; r < 3; ++r) {
TestMax(&model, &local, ntrial, r);
printf("[%d] !!!TestMax pass, iter=%d\n", rank, r);
int step = std::max(nproc / 3, 1);
for (int i = 0; i < nproc; i += step) {
TestBcast(n, i, ntrial, r);
}
printf("[%d] !!!TestBcast pass, iter=%d\n", rank, r);
TestSum(&model, &local, ntrial, r);
printf("[%d] !!!TestSum pass, iter=%d\n", rank, r);
rabit::CheckPoint(&model, &local);
printf("[%d] !!!CheckPont pass, iter=%d\n", rank, r);
}
break;
} catch (MockException &e) {
rabit::engine::GetEngine()->InitAfterException();
++ntrial;
int iter = rabit::LoadCheckPoint(&model, &local);
if (iter == 0) {
model.InitModel(n, 1.0f);
local.InitModel(n, 1.0f + rank);
printf("[%d] reload-trail=%d, init iter=%d\n", rank, ntrial, iter);
} else {
printf("[%d] reload-trail=%d, init iter=%d\n", rank, ntrial, iter);
}
for (int r = iter; r < 3; ++r) {
TestMax(&model, &local, ntrial, r);
printf("[%d] !!!TestMax pass, iter=%d\n", rank, r);
int step = std::max(nproc / 3, 1);
for (int i = 0; i < nproc; i += step) {
TestBcast(n, i, ntrial, r);
}
printf("[%d] !!!TestBcast pass, iter=%d\n", rank, r);
TestSum(&model, &local, ntrial, r);
printf("[%d] !!!TestSum pass, iter=%d\n", rank, r);
rabit::CheckPoint(&model, &local);
printf("[%d] !!!CheckPont pass, iter=%d\n", rank, r);
}
rabit::Finalize();
return 0;

75
test/test_model_recover.cpp
View File

@@ -5,30 +5,7 @@
#include <cstdio>
#include <cstdlib>
#include <cmath>
#include "./mock.h"
using namespace rabit;
namespace rabit {
namespace test {
inline void CallBegin(const char *fun, int ntrial, int iter) {
return;
int rank = rabit::GetRank();
if (!strcmp(fun, "Allreduce::Sum")) {
if (ntrial == iter && rank == 0) exit(-1);
}
if (!strcmp(fun, "Allreduce::Max")) {
if (ntrial == iter && rank == 3) exit(-1);
}
}
inline void CallEnd(const char *fun, int ntrial, int iter) {
return;
int rank = rabit::GetRank();
if (!strcmp(fun, "Allreduce::Bcast")) {
if (ntrial == iter && rand() % 10 == rank) exit(-1);
}
}
}
}
// dummy model
class Model : public rabit::ISerializable {
@@ -58,9 +35,7 @@ inline void TestMax(Model *model, int ntrial, int iter) {
for (size_t i = 0; i < ndata.size(); ++i) {
ndata[i] = (i * (rank+1)) % z + model->data[i];
}
test::CallBegin("Allreduce::Max", ntrial, iter);
rabit::Allreduce<op::Max>(&ndata[0], ndata.size());
test::CallEnd("Allreduce::Max", ntrial, iter);
for (size_t i = 0; i < ndata.size(); ++i) {
float rmax = (i * 1) % z + model->data[i];
@@ -81,9 +56,7 @@ inline void TestSum(Model *model, int ntrial, int iter) {
for (size_t i = 0; i < ndata.size(); ++i) {
ndata[i] = (i * (rank+1)) % z + model->data[i];
}
test::CallBegin("Allreduce::Sum", ntrial, iter);
Allreduce<op::Sum>(&ndata[0], ndata.size());
test::CallEnd("Allreduce::Sum", ntrial, iter);
for (size_t i = 0; i < ndata.size(); ++i) {
float rsum = model->data[i] * nproc;
@@ -105,13 +78,9 @@ inline void TestBcast(size_t n, int root, int ntrial, int iter) {
std::string res;
if (root == rank) {
res = s;
test::CallBegin("Broadcast", ntrial, iter);
rabit::Broadcast(&res, root);
test::CallBegin("Broadcast", ntrial, iter);
} else {
test::CallBegin("Broadcast", ntrial, iter);
rabit::Broadcast(&res, root);
test::CallEnd("Broadcast", ntrial, iter);
}
utils::Check(res == s, "[%d] TestBcast fail", rank);
}
@@ -133,33 +102,25 @@ int main(int argc, char *argv[]) {
int n;
if (sscanf(argv[i], "rabit_num_trial=%d", &n) == 1) ntrial = n;
}
while (true) {
try {
int iter = rabit::LoadCheckPoint(&model);
if (iter == 0) {
model.InitModel(n);
printf("[%d] reload-trail=%d, init iter=%d\n", rank, ntrial, iter);
} else {
printf("[%d] reload-trail=%d, init iter=%d\n", rank, ntrial, iter);
}
for (int r = iter; r < 3; ++r) {
TestMax(&model, ntrial, r);
printf("[%d] !!!TestMax pass, iter=%d\n", rank, r);
int step = std::max(nproc / 3, 1);
for (int i = 0; i < nproc; i += step) {
TestBcast(n, i, ntrial, r);
}
printf("[%d] !!!TestBcast pass, iter=%d\n", rank, r);
TestSum(&model, ntrial, r);
printf("[%d] !!!TestSum pass, iter=%d\n", rank, r);
rabit::CheckPoint(&model);
printf("[%d] !!!CheckPont pass, iter=%d\n", rank, r);
}
break;
} catch (MockException &e) {
rabit::engine::GetEngine()->InitAfterException();
++ntrial;
int iter = rabit::LoadCheckPoint(&model);
if (iter == 0) {
model.InitModel(n);
printf("[%d] reload-trail=%d, init iter=%d\n", rank, ntrial, iter);
} else {
printf("[%d] reload-trail=%d, init iter=%d\n", rank, ntrial, iter);
}
for (int r = iter; r < 3; ++r) {
TestMax(&model, ntrial, r);
printf("[%d] !!!TestMax pass, iter=%d\n", rank, r);
int step = std::max(nproc / 3, 1);
for (int i = 0; i < nproc; i += step) {
TestBcast(n, i, ntrial, r);
}
printf("[%d] !!!TestBcast pass, iter=%d\n", rank, r);
TestSum(&model, ntrial, r);
printf("[%d] !!!TestSum pass, iter=%d\n", rank, r);
rabit::CheckPoint(&model);
printf("[%d] !!!CheckPont pass, iter=%d\n", rank, r);
}
rabit::Finalize();
return 0;