Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix bugs

Browse Source
This commit is contained in:
Boliang Chen 2015-01-11 00:14:37 +08:00
parent 7f5cb3aa0e
commit d986693fbd
1 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
tracker/rabit_hadoop.py
View File

@ -12,7 +12,7 @@ import rabit_tracker as tracker
#!!! Set path to hadoop and hadoop streaming jar here #!!! Set path to hadoop and hadoop streaming jar here
hadoop_binary = 'hadoop' hadoop_binary = 'hadoop'
hadoop_streaming_jar = None hadoop_streaming_jar = None
# code # code
hadoop_home = os.getenv('HADOOP_HOME') hadoop_home = os.getenv('HADOOP_HOME')
@ -67,7 +67,7 @@ if args.jobname is None:
def hadoop_streaming(nworker, worker_args): def hadoop_streaming(nworker, worker_args):
cmd = '%s jar %s -D mapred.map.tasks=%d' % (args.hadoop_binary, args.hadoop_streaming_jar, nworker) cmd = '%s jar %s -D mapred.map.tasks=%d' % (args.hadoop_binary, args.hadoop_streaming_jar, nworker)
cmd += ' -D mapred.job.name=%d' % (a) cmd += ' -D mapred.job.name=%s' % (args.jobname)
cmd += ' -input %s -output %s' % (args.input, args.output) cmd += ' -input %s -output %s' % (args.input, args.output)
cmd += ' -mapper \"%s\" -reducer \"/bin/cat\" ' % (' '.join(args.command + worker_args)) cmd += ' -mapper \"%s\" -reducer \"/bin/cat\" ' % (' '.join(args.command + worker_args))
fset = set() fset = set()
@ -75,9 +75,10 @@ def hadoop_streaming(nworker, worker_args):
for f in args.command: for f in args.command:
if os.path.exists(f): if os.path.exists(f):
fset.add(f) fset.add(f)
for flst in args.files: if args.files != None:
for f in flst.split('#'): for flst in args.files:
fset.add(f) for f in flst.split('#'):
fset.add(f)
for f in fset: for f in fset:
cmd += ' -file %s' % f cmd += ' -file %s' % f
print cmd print cmd