From bfc3f61010d9b8acdc9b875f44501c90b2f42976 Mon Sep 17 00:00:00 2001 From: tqchen Date: Sun, 11 Jan 2015 11:15:12 -0800 Subject: [PATCH] minor --- tracker/rabit_hadoop.py | 5 +++-- tracker/rabit_yarn.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tracker/rabit_hadoop.py b/tracker/rabit_hadoop.py index fdd949c2c..8ecb2c04d 100755 --- a/tracker/rabit_hadoop.py +++ b/tracker/rabit_hadoop.py @@ -108,7 +108,8 @@ def hadoop_streaming(nworker, worker_args, yarn = False): cmd += ' -D%s=%d' % (kmap['nworker'], nworker) cmd += ' -D%s=%s' % (kmap['jobname'], args.jobname) if args.nthread != -1: - assert kmap['nthread'] is not None, "nthread can only be set in Yarn cluster, it is highly recommended to " + assert kmap['nthread'] is not None, 'nthread can only be set in Yarn(Hadoop 2.x) cluster'\ + 'it is recommended to use Yarn to submit rabit jobs' cmd += ' -D%s=%d' % (kmap['ntread'], args.nthread) cmd += ' -D%s=%d' % (kmap['timeout'], args.timeout) if args.memory_mb != -1: @@ -126,5 +127,5 @@ def hadoop_streaming(nworker, worker_args, yarn = False): subprocess.check_call(cmd, shell = True) if __name__ == 'main': - fun_submit = lambda nworker, worker_args: hadoop_streaming(nworker, worker_args, False) + fun_submit = lambda nworker, worker_args: hadoop_streaming(nworker, worker_args, yarn=False) tracker.submit(args.nworker, [], fun_submit = fun_submit, verbose = args.verbose) diff --git a/tracker/rabit_yarn.py b/tracker/rabit_yarn.py index 3bf231cae..ed80595eb 100755 --- a/tracker/rabit_yarn.py +++ b/tracker/rabit_yarn.py @@ -6,5 +6,5 @@ submit the rabit process as mappers of MapReduce import rabit_hadoop if __name__ == 'main': - fun_submit = lambda nworker, worker_args: hadoop_streaming(nworker, worker_args, True) + fun_submit = lambda nworker, worker_args: hadoop_streaming(nworker, worker_args, yarn=True) tracker.submit(args.nworker, [], fun_submit = fun_submit, verbose = args.verbose)