From 0111a14aef2ecd2bbc98ec8e0b4111b01c8b52d6 Mon Sep 17 00:00:00 2001 From: chenshuaihua Date: Sun, 11 Jan 2015 23:57:52 +0800 Subject: [PATCH] yarn script --- multi-node/hadoop/run_yarn_mushroom.sh | 29 ++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 multi-node/hadoop/run_yarn_mushroom.sh diff --git a/multi-node/hadoop/run_yarn_mushroom.sh b/multi-node/hadoop/run_yarn_mushroom.sh new file mode 100644 index 000000000..07ac291d1 --- /dev/null +++ b/multi-node/hadoop/run_yarn_mushroom.sh @@ -0,0 +1,29 @@ +#!/bin/bash +if [ "$#" -lt 3 ]; +then + echo "Usage: " + exit -1 +fi + +# put the local training file to HDFS +hadoop fs -mkdir $3/data +hadoop fs -put ../../demo/data/agaricus.txt.train $3/data + + +python ../../rabit/tracker/rabit_yarn.py -nw $1 -nt $2 -f ../../demo/data/agaricus.txt.test \ + -i $3/data/agaricus.txt.train -o $3/model ../../xgboost mushroom.hadoop.conf nthread=$2 dsplit=row + + + +# get the final model file +hadoop fs -get $3/model/part-00000 ./final.model + +# output prediction task=pred +../../xgboost mushroom.hadoop.conf task=pred model_in=final.model \ + test:data=../../demo/data/agaricus.txt.test +# print the boosters of final.model in dump.raw.txt +../../xgboost mushroom.hadoop.conf task=dump model_in=final.model name_dump=dump.raw.txt +# use the feature map in printing for better visualization +../../xgboost mushroom.hadoop.conf task=dump model_in=final.model \ +fmap=../../demo/data/featmap.txt name_dump=dump.nice.txt +cat dump.nice.txt