diff --git a/rabit-learn/io/hdfs-inl.h b/rabit-learn/io/hdfs-inl.h index d50d0578a..7b4abae73 100644 --- a/rabit-learn/io/hdfs-inl.h +++ b/rabit-learn/io/hdfs-inl.h @@ -78,7 +78,7 @@ class HDFSStream : public utils::ISeekStream { } inline void Close(void) { if (fp_ != NULL) { - if (hdfsCloseFile(fs_, fp_) == 0) { + if (hdfsCloseFile(fs_, fp_) == -1) { int errsv = errno; utils::Error("HDFSStream.Close Error:%s", strerror(errsv)); } diff --git a/rabit-learn/io/line_split-inl.h b/rabit-learn/io/line_split-inl.h index 4d66b344c..7ef322137 100644 --- a/rabit-learn/io/line_split-inl.h +++ b/rabit-learn/io/line_split-inl.h @@ -140,8 +140,10 @@ class LineSplitBase : public InputSplit { class SingleFileSplit : public InputSplit { public: explicit SingleFileSplit(const char *fname) { - if (!strcmp(fname, "stdin")) { - use_stdin_ = true; + if (!strcmp(fname, "stdin")) { +#ifndef RABIT_STRICT_CXX98_ + use_stdin_ = true; fp_ = stdin; +#endif } if (!use_stdin_) { fp_ = utils::FopenCheck(fname, "r"); diff --git a/rabit-learn/linear/run-hadoop.sh b/rabit-learn/linear/run-hadoop.sh index c090f3966..924a4998f 100755 --- a/rabit-learn/linear/run-hadoop.sh +++ b/rabit-learn/linear/run-hadoop.sh @@ -6,15 +6,15 @@ then fi # put the local training file to HDFS -hadoop fs -rm -r -f $2/data +#hadoop fs -rm -r -f $2/data hadoop fs -rm -r -f $2/mushroom.linear.model -hadoop fs -mkdir $2/data -hadoop fs -put ../data/agaricus.txt.train $2/data +#hadoop fs -mkdir $2/data +#hadoop fs -put ../data/agaricus.txt.train $2/data # submit to hadoop -../../tracker/rabit_hadoop.py --host_ip ip -n $1 -i $2/data/agaricus.txt.train -o $2/mushroom.linear.model linear.rabit stdin model_out=stdout "${*:3}" +../../tracker/rabit_hadoop_streaming.py -n $1 --vcores 1 -i $2/data/agaricus.txt.train -o $2/mushroom.linear.model linear.rabit stdin model_out=stdout "${*:3}" # get the final model file -hadoop fs -get $2/mushroom.linear.model/part-00000 ./linear.model +#hadoop fs -get $2/mushroom.linear.model/part-00000 ./linear.model -./linear.rabit ../data/agaricus.txt.test task=pred model_in=linear.model +#./linear.rabit ../data/agaricus.txt.test task=pred model_in=linear.model diff --git a/rabit-learn/make/common.mk b/rabit-learn/make/common.mk index 7ab25bf34..3431b95f6 100644 --- a/rabit-learn/make/common.mk +++ b/rabit-learn/make/common.mk @@ -1,12 +1,12 @@ # this is the common build script for rabit programs # you do not have to use it -export LDFLAGS= -pthread -lm -L../../lib -lrt +export LDFLAGS= -L../../lib -pthread -lm -lrt export CFLAGS = -Wall -msse2 -Wno-unknown-pragmas -fPIC -I../../include # setup opencv ifeq ($(USE_HDFS),1) - CFLAGS+= -DRABIT_USE_HDFS=1 -I$(LIBHDFS_INCLUDE) -I$(JAVA_HOME)/include - LDFLAGS+= -L$(HDFS_HOME)/lib/native -lhdfs + CFLAGS+= -DRABIT_USE_HDFS=1 -I$(HADOOP_HDFS_HOME)/include -I$(JAVA_HOME)/include + LDFLAGS+= -L$(HADOOP_HDFS_HOME)/lib/native -L$(LIBJVM) -lhdfs -ljvm else CFLAGS+= -DRABIT_USE_HDFS=0 endif diff --git a/rabit-learn/make/config.mk b/rabit-learn/make/config.mk index 9324ca438..936564e05 100644 --- a/rabit-learn/make/config.mk +++ b/rabit-learn/make/config.mk @@ -17,5 +17,5 @@ export MPICXX = mpicxx # whether use HDFS support during compile USE_HDFS = 1 -# home of hadoop -HDFS_HOME = $(HADOOP_HDFS_HOME) +# path to libjvm.so +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server