From f203d13efca7dd7e717544c6b9ed2d2743f14536 Mon Sep 17 00:00:00 2001 From: nachocano Date: Sat, 6 Dec 2014 11:59:16 -0800 Subject: [PATCH 1/8] speed runner --- test/speed_runner.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 test/speed_runner.py diff --git a/test/speed_runner.py b/test/speed_runner.py new file mode 100644 index 000000000..eba97c681 --- /dev/null +++ b/test/speed_runner.py @@ -0,0 +1,24 @@ +import os +import argparse + +def main(): + parser = argparse.ArgumentParser(description='TODO') + parser.add_argument('-h', '--host_dir', required=True) + parser.add_argument('-s', '--submit_script', required=True) + args = parser.parse_args() + + ndata = [10^4, 10^5, 10^6, 10^7, 10^8] + nrepeat = [10^2, 10^3, 10^4, 10^5] + + machines = [2,4,8,16,31] + + for data in ndata: + for repeat in nrepeat: + for machine in machines: + host_file = os.path.join(args.host_dir, 'host%d' % machine) + cmd = 'python %s %d %s %d %d' % (args.submit_script, machine, host_file, data, repeat) + print 'data=%d, repeat=%d, machine=%d' % (data, repeat, machine) + os.system(cmd) + +if __name__ == "__main__": + main() \ No newline at end of file From 8f0d7d1d3ef793777f238be789e742f8d68166ef Mon Sep 17 00:00:00 2001 From: nachocano Date: Sat, 6 Dec 2014 12:01:05 -0800 Subject: [PATCH 2/8] changing to -ho not to conflict with help --- test/speed_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/speed_runner.py b/test/speed_runner.py index eba97c681..94d4f2d87 100644 --- a/test/speed_runner.py +++ b/test/speed_runner.py @@ -3,7 +3,7 @@ import argparse def main(): parser = argparse.ArgumentParser(description='TODO') - parser.add_argument('-h', '--host_dir', required=True) + parser.add_argument('-ho', '--host_dir', required=True) parser.add_argument('-s', '--submit_script', required=True) args = parser.parse_args() From e0053c62e1e18bfef85c9e4db55be3d6e4b909df Mon Sep 17 00:00:00 2001 From: nachocano Date: Sat, 6 Dec 2014 12:05:08 -0800 Subject: [PATCH 3/8] adding executable --- test/speed_runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/speed_runner.py b/test/speed_runner.py index 94d4f2d87..8dcae7f9a 100644 --- a/test/speed_runner.py +++ b/test/speed_runner.py @@ -5,6 +5,7 @@ def main(): parser = argparse.ArgumentParser(description='TODO') parser.add_argument('-ho', '--host_dir', required=True) parser.add_argument('-s', '--submit_script', required=True) + parser.add_argument('-ex', '--executable', required=True) args = parser.parse_args() ndata = [10^4, 10^5, 10^6, 10^7, 10^8] @@ -16,7 +17,7 @@ def main(): for repeat in nrepeat: for machine in machines: host_file = os.path.join(args.host_dir, 'host%d' % machine) - cmd = 'python %s %d %s %d %d' % (args.submit_script, machine, host_file, data, repeat) + cmd = 'python %s %d %s %s %d %d' % (args.submit_script, machine, host_file, args.executable, data, repeat) print 'data=%d, repeat=%d, machine=%d' % (data, repeat, machine) os.system(cmd) From 9ed59e71f632eec793a82ddd7c94d3df93caabec Mon Sep 17 00:00:00 2001 From: nachocano Date: Sat, 6 Dec 2014 12:09:40 -0800 Subject: [PATCH 4/8] speed runner --- test/speed_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/speed_runner.py b/test/speed_runner.py index 8dcae7f9a..8e6bfa404 100644 --- a/test/speed_runner.py +++ b/test/speed_runner.py @@ -16,7 +16,7 @@ def main(): for data in ndata: for repeat in nrepeat: for machine in machines: - host_file = os.path.join(args.host_dir, 'host%d' % machine) + host_file = os.path.join(args.host_dir, 'hosts%d' % machine) cmd = 'python %s %d %s %s %d %d' % (args.submit_script, machine, host_file, args.executable, data, repeat) print 'data=%d, repeat=%d, machine=%d' % (data, repeat, machine) os.system(cmd) From 52d472c209054149975f891dfe6e9372ec1ebc51 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 6 Dec 2014 20:30:35 +0000 Subject: [PATCH 5/8] using hostfile --- submit_mpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submit_mpi.py b/submit_mpi.py index 83c97175e..3a65ec440 100755 --- a/submit_mpi.py +++ b/submit_mpi.py @@ -24,7 +24,7 @@ def mpi_submit(nslave, args): args arguments to launch each job this usually includes the parameters of master_uri and parameters passed into submit """ - cmd = ' '.join(['mpirun -n %d -machinefile %s' % (nslave, args[0])] + args[1:]) + cmd = ' '.join(['mpirun -n %d --hostfile %s' % (nslave, args[0])] + args[1:]) print cmd subprocess.check_call(cmd, shell = True) From 659b9cd517313757597484d7f3e0301929a019a9 Mon Sep 17 00:00:00 2001 From: nachocano Date: Sat, 6 Dec 2014 15:14:14 -0800 Subject: [PATCH 6/8] changing number of repetitions --- test/speed_runner.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/test/speed_runner.py b/test/speed_runner.py index 8e6bfa404..a8951fc66 100644 --- a/test/speed_runner.py +++ b/test/speed_runner.py @@ -8,18 +8,17 @@ def main(): parser.add_argument('-ex', '--executable', required=True) args = parser.parse_args() - ndata = [10^4, 10^5, 10^6, 10^7, 10^8] - nrepeat = [10^2, 10^3, 10^4, 10^5] + ndata = [10**4, 10**5, 10**6, 10**7] + nrepeat = [10**4, 10**3, 10**2, 10] machines = [2,4,8,16,31] - for data in ndata: - for repeat in nrepeat: - for machine in machines: - host_file = os.path.join(args.host_dir, 'hosts%d' % machine) - cmd = 'python %s %d %s %s %d %d' % (args.submit_script, machine, host_file, args.executable, data, repeat) - print 'data=%d, repeat=%d, machine=%d' % (data, repeat, machine) - os.system(cmd) + for i, data in enumerate(ndata): + for machine in machines: + host_file = os.path.join(args.host_dir, 'hosts%d' % machine) + cmd = 'python %s %d %s %s %d %d' % (args.submit_script, machine, host_file, args.executable, data, nrepeat[i]) + print 'data=%d, repeat=%d, machine=%d' % (data, nrepeat[i], machine) + os.system(cmd) if __name__ == "__main__": main() \ No newline at end of file From fcf2f0a03d8c928af3f142a10e2a1f8e135853af Mon Sep 17 00:00:00 2001 From: nachocano Date: Sat, 6 Dec 2014 15:22:29 -0800 Subject: [PATCH 7/8] to stderr --- test/speed_runner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/speed_runner.py b/test/speed_runner.py index a8951fc66..a5c459d82 100644 --- a/test/speed_runner.py +++ b/test/speed_runner.py @@ -1,5 +1,6 @@ import os import argparse +import sys def main(): parser = argparse.ArgumentParser(description='TODO') @@ -17,7 +18,8 @@ def main(): for machine in machines: host_file = os.path.join(args.host_dir, 'hosts%d' % machine) cmd = 'python %s %d %s %s %d %d' % (args.submit_script, machine, host_file, args.executable, data, nrepeat[i]) - print 'data=%d, repeat=%d, machine=%d' % (data, nrepeat[i], machine) + sys.stderr.write('data=%d, repeat=%d, machine=%d\n' % (data, nrepeat[i], machine)) + sys.stderr.flush() os.system(cmd) if __name__ == "__main__": From 20b03e781c127450ace0d779f0c65a027311d9ab Mon Sep 17 00:00:00 2001 From: nachocano Date: Sat, 6 Dec 2014 15:37:09 -0800 Subject: [PATCH 8/8] to run all executables --- test/speed_runner.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/test/speed_runner.py b/test/speed_runner.py index a5c459d82..7331c9075 100644 --- a/test/speed_runner.py +++ b/test/speed_runner.py @@ -6,7 +6,8 @@ def main(): parser = argparse.ArgumentParser(description='TODO') parser.add_argument('-ho', '--host_dir', required=True) parser.add_argument('-s', '--submit_script', required=True) - parser.add_argument('-ex', '--executable', required=True) + parser.add_argument('-rex', '--rabit_exec', required=True) + parser.add_argument('-mpi', '--mpi_exec', required=True) args = parser.parse_args() ndata = [10**4, 10**5, 10**6, 10**7] @@ -14,13 +15,20 @@ def main(): machines = [2,4,8,16,31] - for i, data in enumerate(ndata): - for machine in machines: - host_file = os.path.join(args.host_dir, 'hosts%d' % machine) - cmd = 'python %s %d %s %s %d %d' % (args.submit_script, machine, host_file, args.executable, data, nrepeat[i]) - sys.stderr.write('data=%d, repeat=%d, machine=%d\n' % (data, nrepeat[i], machine)) - sys.stderr.flush() - os.system(cmd) + executables = [args.rabit_exec, args.mpi_exec] + + for executable in executables: + sys.stderr.write('Executable %s' % executable) + sys.stderr.flush() + for i, data in enumerate(ndata): + for machine in machines: + host_file = os.path.join(args.host_dir, 'hosts%d' % machine) + cmd = 'python %s %d %s %s %d %d' % (args.submit_script, machine, host_file, executable, data, nrepeat[i]) + sys.stderr.write('data=%d, repeat=%d, machine=%d\n' % (data, nrepeat[i], machine)) + sys.stderr.flush() + os.system(cmd) + sys.stderr.write('\n') + sys.stderr.flush() if __name__ == "__main__": main() \ No newline at end of file