Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1 from cblsjtu/master

Browse Source 
fix several bugs
This commit is contained in:
Tianqi Chen 2015-01-10 09:29:55 -08:00
commit 9d34d2e036
1 changed files with 9 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
tracker/rabit_hadoop.py
View File

@ -21,11 +21,11 @@ if hadoop_home != None:
hadoop_binary = hadoop_home + '/bin/hadoop' hadoop_binary = hadoop_home + '/bin/hadoop'
assert os.path.exists(hadoop_binary), "HADDOP_HOME does not contain the hadoop binary" assert os.path.exists(hadoop_binary), "HADDOP_HOME does not contain the hadoop binary"
if hadoop_streaming_jar == None: if hadoop_streaming_jar == None:
hadoop_streaming_jar = hadoop_home + '/lib/hadoop-streaming.har' hadoop_streaming_jar = hadoop_home + '/lib/hadoop-streaming.jar'
assert os.path.exists(hadoop_streaming_jar), "HADDOP_HOME does not contain the haddop streaming jar" assert os.path.exists(hadoop_streaming_jar), "HADDOP_HOME does not contain the haddop streaming jar"
if hadoop_binary == None or hadoop_streaming_jar == None: if hadoop_binary == None or hadoop_streaming_jar == None:
print 'Warning: Cannot auto-detect path to hadoop and streaming jar, need to set them via arguments -hs and -hb' print 'Warning: Cannot auto-detect path to hadoop and hadoop-streaming jar, need to set them via arguments -hs and -hb'
print '\tTo enable auto-detection, you can set enviroment variable HADOOP_HOME or modify rabit_hadoop.py line 14' print '\tTo enable auto-detection, you can set enviroment variable HADOOP_HOME or modify rabit_hadoop.py line 14'
parser = argparse.ArgumentParser(description='Rabit script to submit rabit jobs using Hadoop Streaming') parser = argparse.ArgumentParser(description='Rabit script to submit rabit jobs using Hadoop Streaming')
@ -53,10 +53,10 @@ else:
help="path-to-hadoop binary folder") help="path-to-hadoop binary folder")
if hadoop_streaming_jar == None: if hadoop_streaming_jar == None:
parser.add_argument('-jar', '--hadoop_streaming_jar', required = True, parser.add_argument('-hs', '--hadoop_streaming_jar', required = True,
help='path-to hadoop streamimg jar file') help='path-to hadoop streamimg jar file')
else: else:
parser.add_argument('-jar', '--hadoop_streaming_jar', default = hadoop_streaming_jar, parser.add_argument('-hs', '--hadoop_streaming_jar', default = hadoop_streaming_jar,
help='path-to hadoop streamimg jar file') help='path-to hadoop streamimg jar file')
parser.add_argument('command', nargs='+', parser.add_argument('command', nargs='+',
help = 'command for rabit program') help = 'command for rabit program')
@ -67,7 +67,7 @@ if args.jobname is None:
def hadoop_streaming(nworker, worker_args): def hadoop_streaming(nworker, worker_args):
cmd = '%s jar %s -D mapred.map.tasks=%d' % (args.hadoop_binary, args.hadoop_streaming_jar, nworker) cmd = '%s jar %s -D mapred.map.tasks=%d' % (args.hadoop_binary, args.hadoop_streaming_jar, nworker)
cmd += ' -D mapred.job.name=%d' % (a) cmd += ' -D mapred.job.name=%s' % (args.jobname)
cmd += ' -input %s -output %s' % (args.input, args.output) cmd += ' -input %s -output %s' % (args.input, args.output)
cmd += ' -mapper \"%s\" -reducer \"/bin/cat\" ' % (' '.join(args.command + worker_args)) cmd += ' -mapper \"%s\" -reducer \"/bin/cat\" ' % (' '.join(args.command + worker_args))
fset = set() fset = set()
@ -75,9 +75,10 @@ def hadoop_streaming(nworker, worker_args):
for f in args.command: for f in args.command:
if os.path.exists(f): if os.path.exists(f):
fset.add(f) fset.add(f)
for flst in args.files: if args.files != None:
for f in flst.split('#'): for flst in args.files:
fset.add(f) for f in flst.split('#'):
fset.add(f)
for f in fset: for f in fset:
cmd += ' -file %s' % f cmd += ' -file %s' % f
print cmd print cmd