diff --git a/subtree/rabit/yarn/src/org/apache/hadoop/yarn/rabit/ApplicationMaster.java b/subtree/rabit/yarn/src/org/apache/hadoop/yarn/rabit/ApplicationMaster.java index f75f4ae6d..47432aa26 100644 --- a/subtree/rabit/yarn/src/org/apache/hadoop/yarn/rabit/ApplicationMaster.java +++ b/subtree/rabit/yarn/src/org/apache/hadoop/yarn/rabit/ApplicationMaster.java @@ -15,6 +15,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -36,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; import org.apache.hadoop.yarn.client.api.async.NMClientAsync; import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; /** @@ -66,6 +68,10 @@ public class ApplicationMaster { // username private String userName = ""; + // user credentials + private Credentials credentials = null; + // security tokens + private ByteBuffer securityTokens = null; // application tracker hostname private String appHostName = ""; // tracker URL to do @@ -99,8 +105,12 @@ public class ApplicationMaster { private ApplicationMaster() throws IOException { dfs = FileSystem.get(conf); + userName = UserGroupInformation.getCurrentUser().getShortUserName(); + credentials = UserGroupInformation.getCurrentUser().getCredentials(); + DataOutputBuffer buffer = new DataOutputBuffer(); + this.credentials.writeTokenStorageToStream(buffer); + this.securityTokens = ByteBuffer.wrap(buffer.getData()); } - /** * get integer argument from environment variable * @@ -132,7 +142,7 @@ public class ApplicationMaster { * @param args */ private void initArgs(String args[]) throws IOException { - LOG.info("Invoke initArgs"); + LOG.info("Start AM as user=" + this.userName); // get user name userName = UserGroupInformation.getCurrentUser().getShortUserName(); // cached maps @@ -183,7 +193,7 @@ public class ApplicationMaster { RegisterApplicationMasterResponse response = this.rmClient .registerApplicationMaster(this.appHostName, this.appTrackerPort, this.appTrackerUrl); - + boolean success = false; String diagnostics = ""; try { @@ -274,13 +284,6 @@ public class ApplicationMaster { task.containerRequest = null; ContainerLaunchContext ctx = Records .newRecord(ContainerLaunchContext.class); - String hadoop = "hadoop"; - if (System.getenv("HADOOP_HOME") != null) { - hadoop = "${HADOOP_HOME}/bin/hadoop"; - } else if (System.getenv("HADOOP_PREFIX") != null) { - hadoop = "${HADOOP_PREFIX}/bin/hadoop"; - } - String cmd = // use this to setup CLASSPATH correctly for libhdfs this.command + " 1>" @@ -288,6 +291,7 @@ public class ApplicationMaster { + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"; ctx.setCommands(Collections.singletonList(cmd)); + ctx.setTokens(this.securityTokens); LOG.info(workerResources); ctx.setLocalResources(this.workerResources); // setup environment variables diff --git a/subtree/rabit/yarn/src/org/apache/hadoop/yarn/rabit/Client.java b/subtree/rabit/yarn/src/org/apache/hadoop/yarn/rabit/Client.java index 7c6de9fb3..9dbdc2619 100644 --- a/subtree/rabit/yarn/src/org/apache/hadoop/yarn/rabit/Client.java +++ b/subtree/rabit/yarn/src/org/apache/hadoop/yarn/rabit/Client.java @@ -1,5 +1,6 @@ package org.apache.hadoop.yarn.rabit; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.Map; @@ -9,7 +10,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -47,6 +50,8 @@ public class Client { private String tempdir = "/tmp"; // user name private String userName = ""; + // user credentials + private Credentials credentials = null; // job name private String jobName = ""; // queue @@ -60,10 +65,22 @@ public class Client { conf.addResource(new Path(System.getenv("HADOOP_CONF_DIR") +"/hdfs-site.xml")); dfs = FileSystem.get(conf); userName = UserGroupInformation.getCurrentUser().getShortUserName(); + credentials = UserGroupInformation.getCurrentUser().getCredentials(); } /** - * ge + * setup security token given current user + * @return the ByeBuffer containing the security tokens + * @throws IOException + */ + private ByteBuffer setupTokens() throws IOException { + DataOutputBuffer buffer = new DataOutputBuffer(); + this.credentials.writeTokenStorageToStream(buffer); + return ByteBuffer.wrap(buffer.getData()); + } + + /** + * setup all the cached files * * @param fmaps * the file maps @@ -194,9 +211,10 @@ public class Client { .newRecord(ContainerLaunchContext.class); ApplicationSubmissionContext appContext = app .getApplicationSubmissionContext(); - // Submit application ApplicationId appId = appContext.getApplicationId(); + // setup security token + amContainer.setTokens(this.setupTokens()); // setup cache-files and environment variables amContainer.setLocalResources(this.setupCacheFiles(appId)); amContainer.setEnvironment(this.getEnvironment());