Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add file command

Browse Source
This commit is contained in:
tqchen 2014-12-22 03:48:14 -08:00
parent ab7492dbc2
commit d82a6ed811
2 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
toolkit/kmeans_hadoop.sh
View File

@ -6,4 +6,4 @@ then
fi fi
#set path to hadoop streaming jar here #set path to hadoop streaming jar here
STREAMING_JAR= STREAMING_JAR=
python ../tracker/rabit_hadoop.py -hs $STREAMING_JAR -s $1 -i $2 -m kmeans.rabit --args "stdin "$3" "$4" stdout" -o $5 python ../tracker/rabit_hadoop.py -hs $STREAMING_JAR -s $1 -i $2 -m kmeans.rabit --args "stdin "$3" "$4" stdout" -o $5 --file kmeans.rabit

4
tracker/rabit_hadoop.py
View File

@ -27,6 +27,7 @@ parser.add_argument('-i', '--input', required=True)
parser.add_argument('-o', '--output', required=True) parser.add_argument('-o', '--output', required=True)
parser.add_argument('-m', '--mapper', required=True) parser.add_argument('-m', '--mapper', required=True)
parser.add_argument('-a', '--args', required=True) parser.add_argument('-a', '--args', required=True)
parser.add_argument('-f', '--file', required=True)
args = parser.parse_args() args = parser.parse_args()
if hadoop_binary != None: if hadoop_binary != None:
@ -38,7 +39,8 @@ def hadoop_streaming(nslaves, slave_args):
cmd = '%s jar %s -D mapred.map.tasks=%d' % (args.hadoop_binary, args.hadoop_streaming_jar, nslaves) cmd = '%s jar %s -D mapred.map.tasks=%d' % (args.hadoop_binary, args.hadoop_streaming_jar, nslaves)
cmd += ' -input %s -output %s' % (args.input, args.output) cmd += ' -input %s -output %s' % (args.input, args.output)
cmd += ' -mapper \"%s %s %s\" -reducer \"/bin/cat\" ' % (args.mapper, args.args, ' '.join(slave_args)) cmd += ' -mapper \"%s %s %s\" -reducer \"/bin/cat\" ' % (args.mapper, args.args, ' '.join(slave_args))
cmd += ' -file %s' % (args.mapper) for f in args.file.split('#'):
cmd += ' -file %s' % (f)
print cmd print cmd
subprocess.check_call(cmd, shell = True) subprocess.check_call(cmd, shell = True)