minmum fix

yarn/run_hdfs_prog.py

@@ -33,6 +33,13 @@ lpath.append('%s/jre/lib/amd64/server' % java_home)
 
 env = os.environ.copy()
 env['CLASSPATH'] = '${CLASSPATH}:' + (':'.join(cpath))
+
+# setup hdfs options
+if 'rabit_hdfs_opts' in env:
+    env['LIBHDFS_OPTS'] = env['rabit_hdfs_opts']
+elif 'LIBHDFS_OPTS' not in env:
+    env['LIBHDFS_OPTS'] = '--Xmx128m'
+
 env['LD_LIBRARY_PATH'] = '${LD_LIBRARY_PATH}:' + (':'.join(lpath)) 
 ret = subprocess.call(args = sys.argv[1:], env = env)
 sys.exit(ret)

yarn/src/org/apache/hadoop/yarn/rabit/ApplicationMaster.java

@@ -60,7 +60,7 @@ public class ApplicationMaster {
     // total number of tasks
     private int numTasks = 1;
     // maximum number of attempts to try in each task
-    private int maxNumAttempt = 1;
+    private int maxNumAttempt = 3;
     // command to launch
     private String command = "";
 
@@ -340,6 +340,9 @@ public class ApplicationMaster {
             if (e.getKey().startsWith("rabit_")) {
                 env.put(e.getKey(), e.getValue());
             }
+            if (e.getKey() == "LIBHDFS_OPTS") {
+                env.put(e.getKey(), e.getValue());
+            }
         }
         env.put("rabit_task_id", String.valueOf(task.taskId));
         env.put("rabit_num_trial", String.valueOf(task.attemptCounter));
@@ -480,7 +483,7 @@ public class ApplicationMaster {
                 } catch (Exception e) {
                 }
                 LOG.info("[Rabit] Task " + r.taskId + " exited with status "
-                        + exstatus);
+                         + exstatus + " Diagnostics:"+ s.getDiagnostics());
                 failed.add(s.getContainerId());
             }
         }

yarn/src/org/apache/hadoop/yarn/rabit/Client.java

@@ -136,6 +136,9 @@ public class Client {
             if (e.getKey().startsWith("rabit_")) {
                 env.put(e.getKey(), e.getValue());
             }
+            if (e.getKey() == "LIBHDFS_OPTS") {
+                env.put(e.getKey(), e.getValue());
+            }
         }
         LOG.debug(env);
         return env;