Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

yarn script

Browse Source
This commit is contained in:
chenshuaihua 2015-01-11 23:32:14 +08:00
parent 981f69ff55
commit 00323f462a
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
tracker/rabit_yarn.py
View File

@ -74,12 +74,14 @@ args = parser.parse_args()
if args.jobname is None: if args.jobname is None:
args.jobname = ('Rabit(nworker=%d):' % args.nworker) + args.command[0].split('/')[-1]; args.jobname = ('Rabit(nworker=%d):' % args.nworker) + args.command[0].split('/')[-1];
def hadoop_streaming(nworker, slave_args): def hadoop_streaming(nworker, slave_args):
cmd = '%s jar %s -D mapreduce.job.maps=%d' % (args.hadoop_binary, args.hadoop_streaming_jar, nworker) cmd = '%s jar %s -D mapreduce.job.maps=%d' % (args.hadoop_binary, args.hadoop_streaming_jar, nworker)
cmd += ' -D mapreduce.job.name=%s' % (args.jobname) cmd += ' -D mapreduce.job.name' % (args.jobname)
cmd += ' -D mapreduce.map.cpu.vcores=%d' % (args.nthread) cmd += ' -D mapreduce.task.timeout=%d' % (args.timeout)
cmd += ' -D mapreduce.map.memory.mb=%d' % (args.memory_mb) cmd += ' -D mapreduce.map.memory.mb=%d' % (args.memory_mb)
cmd += ' -D mapreduce.task.timeout=%d' % (args.timeout) cmd += ' -D mapreduce.map.cpu.vcores=%d' % (args.nthread)
cmd += ' -input %s -output %s' % (args.input, args.output) cmd += ' -input %s -output %s' % (args.input, args.output)
cmd += ' -mapper \"%s\" -reducer \"/bin/cat\" ' % (' '.join(args.command + slave_args)) cmd += ' -mapper \"%s\" -reducer \"/bin/cat\" ' % (' '.join(args.command + slave_args))
fset = set() fset = set()