Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add kmeans hadoop

Browse Source
This commit is contained in:
tqchen 2014-12-22 00:31:01 -08:00
parent dcb6e22a9e
commit 5fe3c58b4a
2 changed files with 11 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
toolkit/kmeans_hadoop.sh Executable file
View File

@ -0,0 +1,9 @@
#!/bin/bash
if [ "$#" -lt 5 ];
then
echo "Usage: <nslaves> <input_data> <ncluster> <max_iteration> <output>"
exit -1
fi
python ../rabit_hadoop.py -s $1 -i $2 -m kmeans --args "stdin "$3" "$4" stdout" -o $5

3
tracker/rabit_hadoop.py
View File

@ -35,7 +35,8 @@ if hadoop_streaming_jar != None:
args.hadoop_streaming_jar = hadoop_streaming_jar args.hadoop_streaming_jar = hadoop_streaming_jar
def hadoop_streaming(nslaves, slave_args): def hadoop_streaming(nslaves, slave_args):
cmd = '%s jar %s -input %s -output %s -mapper \"%s stdin %d %d stdout %s\" -reducer \"/bin/cat\" -file %s -D mapred.map.tasks=%d' % (args.hadoop_binary, args.hadoop_streaming_jar, args.input, args.output, args.mapper, args.nclusters, args.iterations, ' '.join(slave_args), args.mapper, nslaves) cmd = '%s jar %s -input %s -output %s -mapper \"%s %s %s\" -reducer \"/bin/cat\" -file %s -D mapred.map.tasks=%d'
% (args.hadoop_binary, args.hadoop_streaming_jar, args.input, args.output, args.mapper, args.args, ' '.join(slave_args), args.mapper, nslaves)
print cmd print cmd
subprocess.check_call(cmd, shell = True) subprocess.check_call(cmd, shell = True)