Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

new tracker

Browse Source
This commit is contained in:
tqchen 2015-03-02 07:32:25 +00:00
parent 75c647cd84
commit 4fa054e26e
2 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
rabit-learn/linear/run-hadoop.sh
View File

@ -12,7 +12,7 @@ hadoop fs -mkdir $2/data
hadoop fs -put ../data/agaricus.txt.train $2/data hadoop fs -put ../data/agaricus.txt.train $2/data
# submit to hadoop # submit to hadoop
../../tracker/rabit_hadoop.py -n $1 -i $2/data/agaricus.txt.train -o $2/mushroom.linear.model linear.rabit stdin model_out=stdout "${*:3}" ../../tracker/rabit_hadoop.py --host_ip ip -n $1 -i $2/data/agaricus.txt.train -o $2/mushroom.linear.model linear.rabit stdin model_out=stdout "${*:3}"
# get the final model file # get the final model file
hadoop fs -get $2/mushroom.linear.model/part-00000 ./linear.model hadoop fs -get $2/mushroom.linear.model/part-00000 ./linear.model

10
tracker/rabit_tracker.py
View File

@ -141,11 +141,11 @@ class Tracker:
def slave_args(self): def slave_args(self):
if self.hostIP == 'auto': if self.hostIP == 'auto':
host = socket.gethostname() host = socket.gethostname()
elif self.hostIP = 'ip': elif self.hostIP == 'ip':
host = socket.gethostbyname(socket.getfqdn()) host = socket.gethostbyname(socket.getfqdn())
else: else:
host = hostIP host = self.hostIP
return ['rabit_tracker_uri=%s' % hostIP, return ['rabit_tracker_uri=%s' % host,
'rabit_tracker_port=%s' % self.port] 'rabit_tracker_port=%s' % self.port]
def get_neighbor(self, rank, nslave): def get_neighbor(self, rank, nslave):
rank = rank + 1 rank = rank + 1
@ -261,8 +261,8 @@ class Tracker:
wait_conn[rank] = s wait_conn[rank] = s
self.log_print('@tracker All nodes finishes job', 2) self.log_print('@tracker All nodes finishes job', 2)
def submit(nslave, args, fun_submit, verbose): def submit(nslave, args, fun_submit, verbose, hostIP):
master = Tracker(verbose = verbose) master = Tracker(verbose = verbose, hostIP = hostIP)
submit_thread = Thread(target = fun_submit, args = (nslave, args + master.slave_args())) submit_thread = Thread(target = fun_submit, args = (nslave, args + master.slave_args()))
submit_thread.daemon = True submit_thread.daemon = True
submit_thread.start() submit_thread.start()