Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

final attempt

Browse Source
This commit is contained in:
tqchen 2015-03-13 00:04:51 -07:00
parent ed3bee84c2
commit e5a9e31d13
2 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
yarn/src/org/apache/hadoop/yarn/rabit/ApplicationMaster.java
View File

@ -344,7 +344,7 @@ public class ApplicationMaster {
} }
env.put("rabit_task_id", String.valueOf(task.taskId)); env.put("rabit_task_id", String.valueOf(task.taskId));
env.put("rabit_num_trial", String.valueOf(task.attemptCounter)); env.put("rabit_num_trial", String.valueOf(task.attemptCounter));
// ctx.setUser(userName);
ctx.setEnvironment(env); ctx.setEnvironment(env);
synchronized (this) { synchronized (this) {
assert (!this.runningTasks.containsKey(container.getId())); assert (!this.runningTasks.containsKey(container.getId()));

11
yarn/src/org/apache/hadoop/yarn/rabit/Client.java
View File

@ -9,6 +9,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationReport;
@ -43,6 +44,8 @@ public class Client {
private String appArgs = ""; private String appArgs = "";
// HDFS Path to store temporal result // HDFS Path to store temporal result
private String tempdir = "/tmp"; private String tempdir = "/tmp";
// user name
private String userName = "";
// job name // job name
private String jobName = ""; private String jobName = "";
/** /**
@ -53,6 +56,7 @@ public class Client {
conf.addResource(new Path(System.getenv("HADOOP_CONF_DIR") +"/core-site.xml")); conf.addResource(new Path(System.getenv("HADOOP_CONF_DIR") +"/core-site.xml"));
conf.addResource(new Path(System.getenv("HADOOP_CONF_DIR") +"/hdfs-site.xml")); conf.addResource(new Path(System.getenv("HADOOP_CONF_DIR") +"/hdfs-site.xml"));
dfs = FileSystem.get(conf); dfs = FileSystem.get(conf);
userName = UserGroupInformation.getCurrentUser().getShortUserName();
} }
/** /**
@ -182,6 +186,7 @@ public class Client {
.newRecord(ContainerLaunchContext.class); .newRecord(ContainerLaunchContext.class);
ApplicationSubmissionContext appContext = app ApplicationSubmissionContext appContext = app
.getApplicationSubmissionContext(); .getApplicationSubmissionContext();
// Submit application // Submit application
ApplicationId appId = appContext.getApplicationId(); ApplicationId appId = appContext.getApplicationId();
// setup cache-files and environment variables // setup cache-files and environment variables
@ -200,13 +205,13 @@ public class Client {
Resource capability = Records.newRecord(Resource.class); Resource capability = Records.newRecord(Resource.class);
capability.setMemory(256); capability.setMemory(256);
capability.setVirtualCores(1); capability.setVirtualCores(1);
LOG.info("jobname=" + this.jobName); LOG.info("jobname=" + this.jobName + ",username=" + this.userName);
appContext.setApplicationName(jobName + ":RABIT-YARN"); appContext.setApplicationName(jobName + ":RABIT-YARN");
appContext.setAMContainerSpec(amContainer); appContext.setAMContainerSpec(amContainer);
appContext.setResource(capability); appContext.setResource(capability);
appContext.setQueue("default"); appContext.setQueue("default");
//appContext.setUser(userName);
LOG.info("Submitting application " + appId); LOG.info("Submitting application " + appId);
yarnClient.submitApplication(appContext); yarnClient.submitApplication(appContext);