Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix stdin input

Browse Source
This commit is contained in:
tqchen 2015-03-08 12:22:11 -07:00
parent 3258bcf531
commit 2fbda812bc
5 changed files with 16 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
rabit-learn/io/hdfs-inl.h
View File

@ -78,7 +78,7 @@ class HDFSStream : public utils::ISeekStream {
}
inline void Close(void) {
if (fp_ != NULL) {
if (hdfsCloseFile(fs_, fp_) == 0) {
if (hdfsCloseFile(fs_, fp_) == -1) {
int errsv = errno;
utils::Error("HDFSStream.Close Error:%s", strerror(errsv));
}

6
rabit-learn/io/line_split-inl.h
View File

@ -140,8 +140,10 @@ class LineSplitBase : public InputSplit {
class SingleFileSplit : public InputSplit {
public:
explicit SingleFileSplit(const char *fname) {
if (!strcmp(fname, "stdin")) {
use_stdin_ = true;
if (!strcmp(fname, "stdin")) {
#ifndef RABIT_STRICT_CXX98_
use_stdin_ = true; fp_ = stdin;
#endif
}
if (!use_stdin_) {
fp_ = utils::FopenCheck(fname, "r");

12
rabit-learn/linear/run-hadoop.sh
View File

@ -6,15 +6,15 @@ then
fi
# put the local training file to HDFS
hadoop fs -rm -r -f $2/data
#hadoop fs -rm -r -f $2/data
hadoop fs -rm -r -f $2/mushroom.linear.model
hadoop fs -mkdir $2/data
hadoop fs -put ../data/agaricus.txt.train $2/data
#hadoop fs -mkdir $2/data
#hadoop fs -put ../data/agaricus.txt.train $2/data
# submit to hadoop
../../tracker/rabit_hadoop.py --host_ip ip -n $1 -i $2/data/agaricus.txt.train -o $2/mushroom.linear.model linear.rabit stdin model_out=stdout "${*:3}"
../../tracker/rabit_hadoop_streaming.py -n $1 --vcores 1 -i $2/data/agaricus.txt.train -o $2/mushroom.linear.model linear.rabit stdin model_out=stdout "${*:3}"
# get the final model file
hadoop fs -get $2/mushroom.linear.model/part-00000 ./linear.model
#hadoop fs -get $2/mushroom.linear.model/part-00000 ./linear.model
./linear.rabit ../data/agaricus.txt.test task=pred model_in=linear.model
#./linear.rabit ../data/agaricus.txt.test task=pred model_in=linear.model

6
rabit-learn/make/common.mk
View File

@ -1,12 +1,12 @@
# this is the common build script for rabit programs
# you do not have to use it
export LDFLAGS= -pthread -lm -L../../lib -lrt
export LDFLAGS= -L../../lib -pthread -lm -lrt
export CFLAGS = -Wall -msse2 -Wno-unknown-pragmas -fPIC -I../../include
# setup opencv
ifeq ($(USE_HDFS),1)
CFLAGS+= -DRABIT_USE_HDFS=1 -I$(LIBHDFS_INCLUDE) -I$(JAVA_HOME)/include
LDFLAGS+= -L$(HDFS_HOME)/lib/native -lhdfs
CFLAGS+= -DRABIT_USE_HDFS=1 -I$(HADOOP_HDFS_HOME)/include -I$(JAVA_HOME)/include
LDFLAGS+= -L$(HADOOP_HDFS_HOME)/lib/native -L$(LIBJVM) -lhdfs -ljvm
else
CFLAGS+= -DRABIT_USE_HDFS=0
endif

4
rabit-learn/make/config.mk
View File

@ -17,5 +17,5 @@ export MPICXX = mpicxx
# whether use HDFS support during compile
USE_HDFS = 1
# home of hadoop
HDFS_HOME = $(HADOOP_HDFS_HOME)
# path to libjvm.so
LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server