From 925d014271d42d0ac031f4dcf1019b648c23d0f0 Mon Sep 17 00:00:00 2001 From: tqchen Date: Sat, 20 Dec 2014 16:19:54 -0800 Subject: [PATCH] change file structure --- Makefile | 14 +++--- include/README.md | 7 +++ {src => include}/rabit.h | 10 ++--- {src => include/rabit}/engine.h | 3 +- {src => include/rabit}/io.h | 2 +- {src => include/rabit}/rabit-inl.h | 1 + {src => include/rabit}/timer.h | 1 + {src => include/rabit}/utils.h | 0 .../rabit_serializable.h | 6 +-- src/README.md | 6 +++ src/allreduce_base.h | 4 +- src/allreduce_robust.cc | 7 +-- src/allreduce_robust.h | 2 +- src/engine.cc | 2 +- src/engine_empty.cc | 3 +- src/engine_mpi.cc | 4 +- src/socket.h | 2 +- test/Makefile | 31 +++++-------- test/config.h | 2 +- test/speed_test.cpp | 4 +- test/test_local_recover.cpp | 2 +- test/test_model_recover.cpp | 2 +- toolkit/Makefile | 2 +- toolkit/kmeans.cpp | 43 ++++++++++--------- 24 files changed, 84 insertions(+), 76 deletions(-) create mode 100644 include/README.md rename {src => include}/rabit.h (98%) rename {src => include/rabit}/engine.h (99%) rename {src => include/rabit}/io.h (99%) rename {src => include/rabit}/rabit-inl.h (99%) rename {src => include/rabit}/timer.h (88%) rename {src => include/rabit}/utils.h (100%) rename src/serializable.h => include/rabit_serializable.h (96%) create mode 100644 src/README.md diff --git a/Makefile b/Makefile index a591600a8..2b614f3e1 100644 --- a/Makefile +++ b/Makefile @@ -2,23 +2,23 @@ export CC = gcc export CXX = g++ export MPICXX = mpicxx export LDFLAGS= -export CFLAGS = -Wall -O3 -msse2 -Wno-unknown-pragmas -fPIC -I../src +export CFLAGS = -Wall -O3 -msse2 -Wno-unknown-pragmas -fPIC -Iinclude BPATH=lib # objectives that makes up rabit library MPIOBJ= $(BPATH)/engine_mpi.o OBJ= $(BPATH)/allreduce_base.o $(BPATH)/allreduce_robust.o $(BPATH)/engine.o $(BPATH)/engine_empty.o ALIB= lib/librabit.a lib/librabit_mpi.a lib/librabit_empty.a - +HEADERS=src/*.h include/*.h include/rabit/*.h .PHONY: clean all all: $(ALIB) -$(BPATH)/allreduce_base.o: src/allreduce_base.cc src/*.h -$(BPATH)/engine.o: src/engine.cc src/*.h -$(BPATH)/allreduce_robust.o: src/allreduce_robust.cc src/*.h -$(BPATH)/engine_mpi.o: src/engine_mpi.cc src/*.h -$(BPATH)/engine_empty.o: src/engine_empty.cc src/*.h +$(BPATH)/allreduce_base.o: src/allreduce_base.cc $(HEADERS) +$(BPATH)/engine.o: src/engine.cc $(HEADERS) +$(BPATH)/allreduce_robust.o: src/allreduce_robust.cc $(HEADERS) +$(BPATH)/engine_mpi.o: src/engine_mpi.cc $(HEADERS) +$(BPATH)/engine_empty.o: src/engine_empty.cc $(HEADERS) lib/librabit.a: $(BPATH)/allreduce_base.o $(BPATH)/allreduce_robust.o $(BPATH)/engine.o lib/librabit_empty.a: $(BPATH)/engine_empty.o diff --git a/include/README.md b/include/README.md new file mode 100644 index 000000000..1481f2cdb --- /dev/null +++ b/include/README.md @@ -0,0 +1,7 @@ +Library Header Files of Rabit +==== +* This folder contains all the header needed to use rabit libary +* To use it, add include to the search path of the compiler +* User only need to know [rabit.h](rabit.h) and [rabit_serializable.h](rabit_serializable.h) to use the library +* Folder [rabit](rabit) contains headers for internal engine and implementation of template +* Not all .h files in the projects are contained in include, .h files that are internally used by library remains at [src](../src) diff --git a/src/rabit.h b/include/rabit.h similarity index 98% rename from src/rabit.h rename to include/rabit.h index 834c21fd1..e3f9edfb3 100644 --- a/src/rabit.h +++ b/include/rabit.h @@ -16,11 +16,11 @@ #include #endif // C++11 // contains definition of ISerializable -#include "./serializable.h" +#include "./rabit_serializable.h" // engine definition of rabit, defines internal implementation -// to use rabit interface, there is no need to read engine.h rabit.h and serializable.h -// is suffice to use the interface -#include "./engine.h" +// to use rabit interface, there is no need to read engine.h +// rabit.h and serializable.h are suffice to use the interface +#include "./rabit/engine.h" /*! \brief namespace of rabit */ namespace rabit { @@ -282,5 +282,5 @@ class SerializeReducer { }; } // namespace rabit // implementation of template functions -#include "./rabit-inl.h" +#include "./rabit/rabit-inl.h" #endif // RABIT_ALLREDUCE_H diff --git a/src/engine.h b/include/rabit/engine.h similarity index 99% rename from src/engine.h rename to include/rabit/engine.h index 03cd8e44a..ce2f85b66 100644 --- a/src/engine.h +++ b/include/rabit/engine.h @@ -5,7 +5,7 @@ */ #ifndef RABIT_ENGINE_H #define RABIT_ENGINE_H -#include "./serializable.h" +#include "../rabit_serializable.h" namespace MPI { /*! \brief MPI data type just to be compatible with MPI reduce function*/ @@ -222,3 +222,4 @@ class ReduceHandle { } // namespace engine } // namespace rabit #endif // RABIT_ENGINE_H + diff --git a/src/io.h b/include/rabit/io.h similarity index 99% rename from src/io.h rename to include/rabit/io.h index 699a93c83..44d0a0505 100644 --- a/src/io.h +++ b/include/rabit/io.h @@ -5,7 +5,7 @@ #include #include #include "./utils.h" -#include "./serializable.h" +#include "../rabit_serializable.h" /*! * \file io.h * \brief utilities that implements different serializable interface diff --git a/src/rabit-inl.h b/include/rabit/rabit-inl.h similarity index 99% rename from src/rabit-inl.h rename to include/rabit/rabit-inl.h index 20ee39720..55f60cf44 100644 --- a/src/rabit-inl.h +++ b/include/rabit/rabit-inl.h @@ -9,6 +9,7 @@ // use engine for implementation #include "./io.h" #include "./utils.h" +#include "../rabit.h" namespace rabit { namespace engine { diff --git a/src/timer.h b/include/rabit/timer.h similarity index 88% rename from src/timer.h rename to include/rabit/timer.h index c0c83c1c8..bf9f48383 100644 --- a/src/timer.h +++ b/include/rabit/timer.h @@ -10,6 +10,7 @@ namespace rabit { namespace utils { +// TODO not net cross platform, avoid to use this in most places /*! * \brief return time in seconds */ diff --git a/src/utils.h b/include/rabit/utils.h similarity index 100% rename from src/utils.h rename to include/rabit/utils.h diff --git a/src/serializable.h b/include/rabit_serializable.h similarity index 96% rename from src/serializable.h rename to include/rabit_serializable.h index a269dc1c7..eabc03f81 100644 --- a/src/serializable.h +++ b/include/rabit_serializable.h @@ -1,8 +1,8 @@ -#ifndef RABIT_SERIALIZABLE_H -#define RABIT_SERIALIZABLE_H +#ifndef RABIT_RABIT_SERIALIZABLE_H +#define RABIT_RABIT_SERIALIZABLE_H #include #include -#include "./utils.h" +#include "./rabit/utils.h" /*! * \file serializable.h * \brief defines serializable interface of rabit diff --git a/src/README.md b/src/README.md new file mode 100644 index 000000000..5e55d9210 --- /dev/null +++ b/src/README.md @@ -0,0 +1,6 @@ +Source Files of Rabit +==== +* This folder contains the source files of rabit library +* The library headers are in folder [include](../include) +* The .h files in this folder are internal header files that are only used by rabit and will not be seen by users + diff --git a/src/allreduce_base.h b/src/allreduce_base.h index 8bcc76781..f26b5ec45 100644 --- a/src/allreduce_base.h +++ b/src/allreduce_base.h @@ -13,9 +13,9 @@ #include #include -#include "./utils.h" +#include +#include #include "./socket.h" -#include "./engine.h" namespace MPI { // MPI data type to be compatible with existing MPI interface diff --git a/src/allreduce_robust.cc b/src/allreduce_robust.cc index 7afd3546d..fe8013cb6 100644 --- a/src/allreduce_robust.cc +++ b/src/allreduce_robust.cc @@ -9,10 +9,11 @@ #define NOMINMAX #include #include -#include "./io.h" -#include "./utils.h" +#include +#include +#include +#include #include "./allreduce_robust.h" -#include "./rabit.h" namespace rabit { namespace engine { diff --git a/src/allreduce_robust.h b/src/allreduce_robust.h index fd85e4828..d178e391a 100644 --- a/src/allreduce_robust.h +++ b/src/allreduce_robust.h @@ -10,7 +10,7 @@ #ifndef RABIT_ALLREDUCE_ROBUST_H #define RABIT_ALLREDUCE_ROBUST_H #include -#include "./engine.h" +#include #include "./allreduce_base.h" namespace rabit { diff --git a/src/engine.cc b/src/engine.cc index d6d6c92b6..efcb8616d 100644 --- a/src/engine.cc +++ b/src/engine.cc @@ -9,7 +9,7 @@ #define _CRT_SECURE_NO_DEPRECATE #define NOMINMAX -#include "./engine.h" +#include #include "./allreduce_base.h" #include "./allreduce_robust.h" diff --git a/src/engine_empty.cc b/src/engine_empty.cc index be37e3a7a..ff838717e 100644 --- a/src/engine_empty.cc +++ b/src/engine_empty.cc @@ -9,7 +9,8 @@ #define _CRT_SECURE_NO_DEPRECATE #define NOMINMAX -#include "./engine.h" +#include + namespace rabit { namespace engine { /*! \brief EmptyEngine */ diff --git a/src/engine_mpi.cc b/src/engine_mpi.cc index 46867d3cc..d8a30cbbc 100644 --- a/src/engine_mpi.cc +++ b/src/engine_mpi.cc @@ -9,8 +9,8 @@ #define _CRT_SECURE_NO_DEPRECATE #define NOMINMAX #include -#include "./engine.h" -#include "./utils.h" +#include +#include #include namespace rabit { diff --git a/src/socket.h b/src/socket.h index 65516690d..899ab03a7 100644 --- a/src/socket.h +++ b/src/socket.h @@ -21,7 +21,7 @@ #endif #include #include -#include "./utils.h" +#include #if defined(_WIN32) typedef int ssize_t; diff --git a/test/Makefile b/test/Makefile index 18d876b2e..3b5729b11 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,30 +1,22 @@ export CC = gcc export CXX = g++ export MPICXX = mpicxx -export LDFLAGS= -pthread -lm -lrt -export CFLAGS = -Wall -O3 -msse2 -Wno-unknown-pragmas -fPIC -I../src -std=c++11 +export LDFLAGS= -pthread -lm -lrt -L../lib +export CFLAGS = -Wall -O3 -msse2 -Wno-unknown-pragmas -fPIC -I../include -std=c++11 # specify tensor path BIN = speed_test test_model_recover test_local_recover -# objectives that makes up rabit library -RABIT_OBJ = allreduce_base.o allreduce_robust.o engine.o -MPIOBJ = engine_mpi.o - OBJ = $(RABIT_OBJ) speed_test.o test_model_recover.o test_local_recover.o MPIBIN = speed_test.mpi -.PHONY: clean all +.PHONY: clean all lib all: $(BIN) $(MPIBIN) -# the rabit library -allreduce_base.o: ../src/allreduce_base.cc ../src/*.h -engine.o: ../src/engine.cc ../src/*.h -engine_mpi.o: ../src/engine_mpi.cc -allreduce_robust.o: ../src/allreduce_robust.cc ../src/*.h - +lib: + cd ..;make;cd - # programs -speed_test.o: speed_test.cpp ../src/*.h -test_model_recover.o: test_model_recover.cpp ../src/*.h -test_local_recover.o: test_local_recover.cpp ../src/*.h +speed_test.o: speed_test.cpp ../include/*.h lib +test_model_recover.o: test_model_recover.cpp ../include/*.h lib +test_local_recover.o: test_local_recover.cpp ../include/*.h lib # we can link against MPI version to get use MPI speed_test: speed_test.o $(RABIT_OBJ) @@ -33,16 +25,13 @@ test_model_recover: test_model_recover.o $(RABIT_OBJ) test_local_recover: test_local_recover.o $(RABIT_OBJ) $(BIN) : - $(CXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c %.cc, $^) $(LDFLAGS) + $(CXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c %.cc, $^) $(LDFLAGS) -lrabit $(OBJ) : $(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c %.cc, $^) ) $(MPIBIN) : - $(MPICXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c %.cc, $^) $(LDFLAGS) - -$(MPIOBJ) : - $(MPICXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c %.cc, $^) ) + $(MPICXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c %.cc, $^) $(LDFLAGS) -lrabit_mpi clean: $(RM) $(OBJ) $(BIN) $(MPIBIN) $(MPIOBJ) *~ ../src/*~ diff --git a/test/config.h b/test/config.h index 146948adc..467e8f63e 100644 --- a/test/config.h +++ b/test/config.h @@ -10,7 +10,7 @@ #include #include #include -#include "./utils.h" +#include "./rabit/utils.h" namespace rabit { namespace utils { diff --git a/test/speed_test.cpp b/test/speed_test.cpp index e716731fd..68891bd31 100644 --- a/test/speed_test.cpp +++ b/test/speed_test.cpp @@ -1,7 +1,7 @@ // This program is used to test the speed of rabit API #include -#include -#include +#include +#include #include #include #include diff --git a/test/test_local_recover.cpp b/test/test_local_recover.cpp index d473345b3..e278a38ca 100644 --- a/test/test_local_recover.cpp +++ b/test/test_local_recover.cpp @@ -1,7 +1,7 @@ // this is a test case to test whether rabit can recover model when // facing an exception #include -#include +#include #include #include #include diff --git a/test/test_model_recover.cpp b/test/test_model_recover.cpp index 117acef09..f3693fa24 100644 --- a/test/test_model_recover.cpp +++ b/test/test_model_recover.cpp @@ -1,7 +1,7 @@ // this is a test case to test whether rabit can recover model when // facing an exception #include -#include +#include #include #include #include diff --git a/toolkit/Makefile b/toolkit/Makefile index 429264ad3..a194ccd08 100644 --- a/toolkit/Makefile +++ b/toolkit/Makefile @@ -2,7 +2,7 @@ export CC = gcc export CXX = g++ export MPICXX = mpicxx export LDFLAGS= -pthread -lm -L../lib -export CFLAGS = -Wall -O3 -msse2 -Wno-unknown-pragmas -fPIC -I../src -std=c++11 +export CFLAGS = -Wall -O3 -msse2 -Wno-unknown-pragmas -fPIC -I../include -std=c++11 # specify tensor path BIN = kmeans diff --git a/toolkit/kmeans.cpp b/toolkit/kmeans.cpp index bbd5067af..3a55a0427 100644 --- a/toolkit/kmeans.cpp +++ b/toolkit/kmeans.cpp @@ -1,7 +1,7 @@ // this is a test case to test whether rabit can recover model when // facing an exception #include -#include +#include #include "./toolkit_util.h" #include @@ -105,32 +105,33 @@ int main(int argc, char *argv[]) { model.InitModel(num_cluster, data.feat_dim); InitCentroids(data, &model.centroids); model.Normalize(); - utils::LogPrintf("[%d] start at %s\n", - rabit::GetRank(), rabit::GetProcessorName().c_str()); + rabit::TrackerPrintf("[%d] start at %s\n", + rabit::GetRank(), rabit::GetProcessorName().c_str()); } else { - utils::LogPrintf("[%d] restart iter=%d\n", rabit::GetRank(), iter); + rabit::TrackerPrintf("[%d] restart iter=%d\n", rabit::GetRank(), iter); } const unsigned num_feat = data.feat_dim; // matrix to store the result Matrix temp; - for (int r = iter; r < max_iter; ++r) { + for (int r = iter; r < max_iter; ++r) { temp.Init(num_cluster, num_feat + 1, 0.0f); - // call allreduce - rabit::Allreduce(&temp.data[0], temp.data.size(), [&]() { - // lambda function used to calculate the data if necessary - // this function may not be called when the result can be directly recovered - const size_t ndata = data.NumRow(); - for (size_t i = 0; i < ndata; ++i) { - SparseMat::Vector v = data[i]; - size_t k = GetCluster(model.centroids, v); - // temp[k] += v - for (size_t j = 0; j < v.length; ++j) { - temp[k][v[j].findex] += v[j].fvalue; - } - // use last column to record counts - temp[k][num_feat] += 1.0f; + auto lazy_get_centroid = [&]() { + // lambda function used to calculate the data if necessary + // this function may not be called when the result can be directly recovered + const size_t ndata = data.NumRow(); + for (size_t i = 0; i < ndata; ++i) { + SparseMat::Vector v = data[i]; + size_t k = GetCluster(model.centroids, v); + // temp[k] += v + for (size_t j = 0; j < v.length; ++j) { + temp[k][v[j].findex] += v[j].fvalue; } - }); + // use last column to record counts + temp[k][num_feat] += 1.0f; + } + }; + // call allreduce + rabit::Allreduce(&temp.data[0], temp.data.size(), lazy_get_centroid); // set number for (int k = 0; k < num_cluster; ++k) { float cnt = temp[k][num_feat]; @@ -146,7 +147,7 @@ int main(int argc, char *argv[]) { if (rabit::GetRank() == 0) { model.centroids.Print(argv[4]); } - utils::LogPrintf("[%d] Time taken: %f seconds\n", rabit::GetRank(), static_cast(clock() - tStart) / CLOCKS_PER_SEC); + rabit::TrackerPrintf("[%d] Time taken: %f seconds\n", rabit::GetRank(), static_cast(clock() - tStart) / CLOCKS_PER_SEC); rabit::Finalize(); return 0; }