Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

change file structure

Browse Source
This commit is contained in:
tqchen 2014-12-20 16:19:54 -08:00
parent 77d74f6c0d
commit 925d014271
24 changed files with 84 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Makefile
View File

@ -2,23 +2,23 @@ export CC = gcc
export CXX = g++
export MPICXX = mpicxx
export LDFLAGS=
export CFLAGS = -Wall -O3 -msse2 -Wno-unknown-pragmas -fPIC -I../src
export CFLAGS = -Wall -O3 -msse2 -Wno-unknown-pragmas -fPIC -Iinclude
BPATH=lib
# objectives that makes up rabit library
MPIOBJ= $(BPATH)/engine_mpi.o
OBJ= $(BPATH)/allreduce_base.o $(BPATH)/allreduce_robust.o $(BPATH)/engine.o $(BPATH)/engine_empty.o
ALIB= lib/librabit.a lib/librabit_mpi.a lib/librabit_empty.a
HEADERS=src/*.h include/*.h include/rabit/*.h
.PHONY: clean all
all: $(ALIB)
$(BPATH)/allreduce_base.o: src/allreduce_base.cc src/*.h
$(BPATH)/engine.o: src/engine.cc src/*.h
$(BPATH)/allreduce_robust.o: src/allreduce_robust.cc src/*.h
$(BPATH)/engine_mpi.o: src/engine_mpi.cc src/*.h
$(BPATH)/engine_empty.o: src/engine_empty.cc src/*.h
$(BPATH)/allreduce_base.o: src/allreduce_base.cc $(HEADERS)
$(BPATH)/engine.o: src/engine.cc $(HEADERS)
$(BPATH)/allreduce_robust.o: src/allreduce_robust.cc $(HEADERS)
$(BPATH)/engine_mpi.o: src/engine_mpi.cc $(HEADERS)
$(BPATH)/engine_empty.o: src/engine_empty.cc $(HEADERS)
lib/librabit.a: $(BPATH)/allreduce_base.o $(BPATH)/allreduce_robust.o $(BPATH)/engine.o
lib/librabit_empty.a: $(BPATH)/engine_empty.o

7
include/README.md Normal file
View File

@ -0,0 +1,7 @@
Library Header Files of Rabit
====
* This folder contains all the header needed to use rabit libary
* To use it, add include to the search path of the compiler
* User only need to know [rabit.h](rabit.h) and [rabit_serializable.h](rabit_serializable.h) to use the library
* Folder [rabit](rabit) contains headers for internal engine and implementation of template
* Not all .h files in the projects are contained in include, .h files that are internally used by library remains at [src](../src)

10
src/rabit.h → include/rabit.h
View File

@ -16,11 +16,11 @@
#include <functional>
#endif // C++11
// contains definition of ISerializable
#include "./serializable.h"
#include "./rabit_serializable.h"
// engine definition of rabit, defines internal implementation
// to use rabit interface, there is no need to read engine.h rabit.h and serializable.h
// is suffice to use the interface
#include "./engine.h"
// to use rabit interface, there is no need to read engine.h
// rabit.h and serializable.h are suffice to use the interface
#include "./rabit/engine.h"
/*! \brief namespace of rabit */
namespace rabit {
@ -282,5 +282,5 @@ class SerializeReducer {
};
} // namespace rabit
// implementation of template functions
#include "./rabit-inl.h"
#include "./rabit/rabit-inl.h"
#endif // RABIT_ALLREDUCE_H

3
src/engine.h → include/rabit/engine.h
View File

@ -5,7 +5,7 @@
*/
#ifndef RABIT_ENGINE_H
#define RABIT_ENGINE_H
#include "./serializable.h"
#include "../rabit_serializable.h"
namespace MPI {
/*! \brief MPI data type just to be compatible with MPI reduce function*/
@ -222,3 +222,4 @@ class ReduceHandle {
} // namespace engine
} // namespace rabit
#endif // RABIT_ENGINE_H

2
src/io.h → include/rabit/io.h
View File

@ -5,7 +5,7 @@
#include <cstring>
#include <string>
#include "./utils.h"
#include "./serializable.h"
#include "../rabit_serializable.h"
/*!
* \file io.h
* \brief utilities that implements different serializable interface

1
src/rabit-inl.h → include/rabit/rabit-inl.h
View File

@ -9,6 +9,7 @@
// use engine for implementation
#include "./io.h"
#include "./utils.h"
#include "../rabit.h"
namespace rabit {
namespace engine {

1
src/timer.h → include/rabit/timer.h
View File

@ -10,6 +10,7 @@
namespace rabit {
namespace utils {
// TODO not net cross platform, avoid to use this in most places
/*!
* \brief return time in seconds
*/

0
src/utils.h → include/rabit/utils.h
View File

6
src/serializable.h → include/rabit_serializable.h
View File

@ -1,8 +1,8 @@
#ifndef RABIT_SERIALIZABLE_H
#define RABIT_SERIALIZABLE_H
#ifndef RABIT_RABIT_SERIALIZABLE_H
#define RABIT_RABIT_SERIALIZABLE_H
#include <vector>
#include <string>
#include "./utils.h"
#include "./rabit/utils.h"
/*!
* \file serializable.h
* \brief defines serializable interface of rabit

6
src/README.md Normal file
View File

@ -0,0 +1,6 @@
Source Files of Rabit
====
* This folder contains the source files of rabit library
* The library headers are in folder [include](../include)
* The .h files in this folder are internal header files that are only used by rabit and will not be seen by users

4
src/allreduce_base.h
View File

@ -13,9 +13,9 @@
#include <vector>
#include <string>
#include "./utils.h"
#include <rabit/utils.h>
#include <rabit/engine.h>
#include "./socket.h"
#include "./engine.h"
namespace MPI {
// MPI data type to be compatible with existing MPI interface

7
src/allreduce_robust.cc
View File

@ -9,10 +9,11 @@
#define NOMINMAX
#include <limits>
#include <utility>
#include "./io.h"
#include "./utils.h"
#include <rabit/io.h>
#include <rabit/utils.h>
#include <rabit/engine.h>
#include <rabit/rabit-inl.h>
#include "./allreduce_robust.h"
#include "./rabit.h"
namespace rabit {
namespace engine {

2
src/allreduce_robust.h
View File

@ -10,7 +10,7 @@
#ifndef RABIT_ALLREDUCE_ROBUST_H
#define RABIT_ALLREDUCE_ROBUST_H
#include <vector>
#include "./engine.h"
#include <rabit/engine.h>
#include "./allreduce_base.h"
namespace rabit {

2
src/engine.cc
View File

@ -9,7 +9,7 @@
#define _CRT_SECURE_NO_DEPRECATE
#define NOMINMAX
#include "./engine.h"
#include <rabit/engine.h>
#include "./allreduce_base.h"
#include "./allreduce_robust.h"

3
src/engine_empty.cc
View File

@ -9,7 +9,8 @@
#define _CRT_SECURE_NO_DEPRECATE
#define NOMINMAX
#include "./engine.h"
#include <rabit/engine.h>
namespace rabit {
namespace engine {
/*! \brief EmptyEngine */

4
src/engine_mpi.cc
View File

@ -9,8 +9,8 @@
#define _CRT_SECURE_NO_DEPRECATE
#define NOMINMAX
#include <cstdio>
#include "./engine.h"
#include "./utils.h"
#include <rabit/engine.h>
#include <rabit/utils.h>
#include <mpi.h>
namespace rabit {

2
src/socket.h
View File

@ -21,7 +21,7 @@
#endif
#include <string>
#include <cstring>
#include "./utils.h"
#include <rabit/utils.h>
#if defined(_WIN32)
typedef int ssize_t;

31
test/Makefile
View File

@ -1,30 +1,22 @@
export CC = gcc
export CXX = g++
export MPICXX = mpicxx
export LDFLAGS= -pthread -lm -lrt
export CFLAGS = -Wall -O3 -msse2 -Wno-unknown-pragmas -fPIC -I../src -std=c++11
export LDFLAGS= -pthread -lm -lrt -L../lib
export CFLAGS = -Wall -O3 -msse2 -Wno-unknown-pragmas -fPIC -I../include -std=c++11
# specify tensor path
BIN = speed_test test_model_recover test_local_recover
# objectives that makes up rabit library
RABIT_OBJ = allreduce_base.o allreduce_robust.o engine.o
MPIOBJ = engine_mpi.o
OBJ = $(RABIT_OBJ) speed_test.o test_model_recover.o test_local_recover.o
MPIBIN = speed_test.mpi
.PHONY: clean all
.PHONY: clean all lib
all: $(BIN) $(MPIBIN)
# the rabit library
allreduce_base.o: ../src/allreduce_base.cc ../src/*.h
engine.o: ../src/engine.cc ../src/*.h
engine_mpi.o: ../src/engine_mpi.cc
allreduce_robust.o: ../src/allreduce_robust.cc ../src/*.h
lib:
cd ..;make;cd -
# programs
speed_test.o: speed_test.cpp ../src/*.h
test_model_recover.o: test_model_recover.cpp ../src/*.h
test_local_recover.o: test_local_recover.cpp ../src/*.h
speed_test.o: speed_test.cpp ../include/*.h lib
test_model_recover.o: test_model_recover.cpp ../include/*.h lib
test_local_recover.o: test_local_recover.cpp ../include/*.h lib
# we can link against MPI version to get use MPI
speed_test: speed_test.o $(RABIT_OBJ)
@ -33,16 +25,13 @@ test_model_recover: test_model_recover.o $(RABIT_OBJ)
test_local_recover: test_local_recover.o $(RABIT_OBJ)
$(BIN) :
$(CXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c %.cc, $^) $(LDFLAGS)
$(CXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c %.cc, $^) $(LDFLAGS) -lrabit
$(OBJ) :
$(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c %.cc, $^) )
$(MPIBIN) :
$(MPICXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c %.cc, $^) $(LDFLAGS)
$(MPIOBJ) :
$(MPICXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c %.cc, $^) )
$(MPICXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c %.cc, $^) $(LDFLAGS) -lrabit_mpi
clean:
$(RM) $(OBJ) $(BIN) $(MPIBIN) $(MPIOBJ) *~ ../src/*~

2
test/config.h
View File

@ -10,7 +10,7 @@
#include <string>
#include <istream>
#include <fstream>
#include "./utils.h"
#include "./rabit/utils.h"
namespace rabit {
namespace utils {

4
test/speed_test.cpp
View File

@ -1,7 +1,7 @@
// This program is used to test the speed of rabit API
#include <rabit.h>
#include <utils.h>
#include <timer.h>
#include <rabit/utils.h>
#include <rabit/timer.h>
#include <cstdio>
#include <cstdlib>
#include <cmath>

2
test/test_local_recover.cpp
View File

@ -1,7 +1,7 @@
// this is a test case to test whether rabit can recover model when
// facing an exception
#include <rabit.h>
#include <utils.h>
#include <rabit/utils.h>
#include <cstdio>
#include <cstdlib>
#include <cmath>

2
test/test_model_recover.cpp
View File

@ -1,7 +1,7 @@
// this is a test case to test whether rabit can recover model when
// facing an exception
#include <rabit.h>
#include <utils.h>
#include <rabit/utils.h>
#include <cstdio>
#include <cstdlib>
#include <cmath>

2
toolkit/Makefile
View File

@ -2,7 +2,7 @@ export CC = gcc
export CXX = g++
export MPICXX = mpicxx
export LDFLAGS= -pthread -lm -L../lib
export CFLAGS = -Wall -O3 -msse2 -Wno-unknown-pragmas -fPIC -I../src -std=c++11
export CFLAGS = -Wall -O3 -msse2 -Wno-unknown-pragmas -fPIC -I../include -std=c++11
# specify tensor path
BIN = kmeans

43
toolkit/kmeans.cpp
View File

@ -1,7 +1,7 @@
// this is a test case to test whether rabit can recover model when
// facing an exception
#include <rabit.h>
#include <utils.h>
#include <rabit/utils.h>
#include "./toolkit_util.h"
#include <time.h>
@ -105,32 +105,33 @@ int main(int argc, char *argv[]) {
model.InitModel(num_cluster, data.feat_dim);
InitCentroids(data, &model.centroids);
model.Normalize();
utils::LogPrintf("[%d] start at %s\n",
rabit::GetRank(), rabit::GetProcessorName().c_str());
rabit::TrackerPrintf("[%d] start at %s\n",
rabit::GetRank(), rabit::GetProcessorName().c_str());
} else {
utils::LogPrintf("[%d] restart iter=%d\n", rabit::GetRank(), iter);
rabit::TrackerPrintf("[%d] restart iter=%d\n", rabit::GetRank(), iter);
}
const unsigned num_feat = data.feat_dim;
// matrix to store the result
Matrix temp;
for (int r = iter; r < max_iter; ++r) {
for (int r = iter; r < max_iter; ++r) {
temp.Init(num_cluster, num_feat + 1, 0.0f);
// call allreduce
rabit::Allreduce<op::Sum>(&temp.data[0], temp.data.size(), [&]() {
// lambda function used to calculate the data if necessary
// this function may not be called when the result can be directly recovered
const size_t ndata = data.NumRow();
for (size_t i = 0; i < ndata; ++i) {
SparseMat::Vector v = data[i];
size_t k = GetCluster(model.centroids, v);
// temp[k] += v
for (size_t j = 0; j < v.length; ++j) {
temp[k][v[j].findex] += v[j].fvalue;
}
// use last column to record counts
temp[k][num_feat] += 1.0f;
auto lazy_get_centroid = [&]() {
// lambda function used to calculate the data if necessary
// this function may not be called when the result can be directly recovered
const size_t ndata = data.NumRow();
for (size_t i = 0; i < ndata; ++i) {
SparseMat::Vector v = data[i];
size_t k = GetCluster(model.centroids, v);
// temp[k] += v
for (size_t j = 0; j < v.length; ++j) {
temp[k][v[j].findex] += v[j].fvalue;
}
});
// use last column to record counts
temp[k][num_feat] += 1.0f;
}
};
// call allreduce
rabit::Allreduce<op::Sum>(&temp.data[0], temp.data.size(), lazy_get_centroid);
// set number
for (int k = 0; k < num_cluster; ++k) {
float cnt = temp[k][num_feat];
@ -146,7 +147,7 @@ int main(int argc, char *argv[]) {
if (rabit::GetRank() == 0) {
model.centroids.Print(argv[4]);
}
utils::LogPrintf("[%d] Time taken: %f seconds\n", rabit::GetRank(), static_cast<float>(clock() - tStart) / CLOCKS_PER_SEC);
rabit::TrackerPrintf("[%d] Time taken: %f seconds\n", rabit::GetRank(), static_cast<float>(clock() - tStart) / CLOCKS_PER_SEC);
rabit::Finalize();
return 0;
}