diff --git a/analysis/depristo/s3GATKReport/GATKPolicy.txt b/analysis/depristo/s3GATKReport/GATKPolicy.txt new file mode 100644 index 000000000..d68b40765 --- /dev/null +++ b/analysis/depristo/s3GATKReport/GATKPolicy.txt @@ -0,0 +1,12 @@ +{ + "Statement": [ + { + "Sid": "Stmt1296439478068", + "Action": [ + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": "arn:aws:s3:::GATK_Run_Reports/*" + } + ] +} diff --git a/analysis/depristo/s3GATKReport/GATK_cred.txt b/analysis/depristo/s3GATKReport/GATK_cred.txt new file mode 100644 index 000000000..3491d6072 --- /dev/null +++ b/analysis/depristo/s3GATKReport/GATK_cred.txt @@ -0,0 +1,2 @@ +AKIAJXU7VIHBPDW4TDSQ +uQLTduhK6Gy8mbOycpoZIxr8ZoVj1SQaglTWjpYA diff --git a/analysis/depristo/s3GATKReport/GroupPolicy.txt b/analysis/depristo/s3GATKReport/GroupPolicy.txt new file mode 100644 index 000000000..eb6d3bdaa --- /dev/null +++ b/analysis/depristo/s3GATKReport/GroupPolicy.txt @@ -0,0 +1,8 @@ +{ + "Statement":[{ + "Effect":"Allow", + "Action":"*", + "Resource":"*" + } + ] +} diff --git a/analysis/depristo/s3GATKReport/setupS3User.csh b/analysis/depristo/s3GATKReport/setupS3User.csh new file mode 100755 index 000000000..081ba6c28 --- /dev/null +++ b/analysis/depristo/s3GATKReport/setupS3User.csh @@ -0,0 +1,45 @@ +#!/bin/tcsh + +# download CLI tools +# http://aws.amazon.com/developertools/AWS-Identity-and-Access-Management/4143 + +setenv JAVA_HOME /usr/ +setenv AWS_IAM_HOME ~/Downloads/IAMCli-1.1.0 +setenv PATH $AWS_IAM_HOME/bin:$PATH +setenv AWS_CREDENTIAL_FILE /Users/depristo/Desktop/broadLocal/GATK/trunk/account-key + +setenv CREATE_GROUPS false +setenv CREATE_GATK_USER false +setenv UPDATE_USER_KEYS false +setenv UPDATE_USER_POLICY true + +# Create the administrators group: +# we aren't actually using this, in fact +if ( $CREATE_GROUPS == true ) then +iam-groupcreate -g Admins +iam-grouplistbypath +iam-groupuploadpolicy -g Admins -p AdminsGroupPolicy -f GroupPolicy.txt +iam-grouplistpolicies -g Admins +endif + +# Create the GATK user -- uncomment if the GATK user needs to be created from scratch +# update the secret key +if $CREATE_GATK_USER == true then +iam-usercreate -u GATK -k -v > GATK_cred.txt +endif + +# the user access and secret keys are in the GATK source file GATKRunReport.java +# and must be updated to be the most current ones +if $UPDATE_USER_KEYS == true then +iam-userdelkey -u GATK -k $1 # $1 -> current access key +iam-useraddkey -u GATK > GATK_cred.txt +cat GATK_cred.txt +endif + +echo "GATK user policies" +if $UPDATE_USER_POLICY == true then +echo "Deleting policy" +iam-userdelpolicy -u GATK -p GATKRunReportUploading +iam-useruploadpolicy -u GATK -p GATKRunReportUploading -f GATKPolicy.txt +endif +iam-userlistpolicies -u GATK -v diff --git a/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java b/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java index 152765cc1..d976c1efe 100755 --- a/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java +++ b/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java @@ -86,11 +86,6 @@ public class GATKArgumentCollection { @Argument(fullName = "phone_home", shortName = "et", doc="What kind of GATK run report should we generate? Standard is the default, can be verbose or NO_ET so nothing is posted to the run repository", required = false) public GATKRunReport.PhoneHomeOption phoneHomeType = GATKRunReport.PhoneHomeOption.STANDARD; - @Element(required = false) - @Argument(fullName = "S3SecretKey", shortName = "s3sk", doc="Secret key to be used for AWS S3 interactions", required = false) - public String S3SecretKey = null; - - @ElementList(required = false) @Argument(fullName = "read_filter", shortName = "rf", doc = "Specify filtration criteria to apply to each read individually.", required = false) public List readFilters = new ArrayList(); @@ -400,10 +395,6 @@ public class GATKArgumentCollection { if (other.phoneHomeType != this.phoneHomeType) { return false; } - if ((other.S3SecretKey == null && this.S3SecretKey != null) || - (other.S3SecretKey != null && !other.S3SecretKey.equals(this.S3SecretKey))) { - return false; - } if (BTIMergeRule != other.BTIMergeRule) return false; diff --git a/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java b/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java index 6abeb5643..23a8a58aa 100644 --- a/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java +++ b/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java @@ -159,8 +159,6 @@ public class GATKRunReport { // todo md5 all filenames // todo size of filenames - private String S3SecretKey = null; - public enum PhoneHomeOption { NO_ET, STANDARD, @@ -228,8 +226,6 @@ public class GATKRunReport { // if there was an exception, capture it this.mException = e == null ? null : new ExceptionToXML(e); - - this.S3SecretKey = engine.getArguments().S3SecretKey; } public String getID() { @@ -331,51 +327,47 @@ public class GATKRunReport { private void postReportToAWSS3() { // modifying example code from http://jets3t.s3.amazonaws.com/toolkit/code-samples.html - if ( S3SecretKey == null ) - exceptDuringRunReport("Cannot upload run reports to AWS S3 without providing a secret key on the command line"); - else { - this.hostName = resolveHostname(); // we want to fill in the host name - File localFile = postReportToLocalDisk(new File("./")); - logger.info("Generating GATK report to AWS S3 based on local file " + localFile); - if ( localFile != null ) { - try { - // we succeeded in creating the local file + this.hostName = resolveHostname(); // we want to fill in the host name + File localFile = postReportToLocalDisk(new File("./")); + logger.info("Generating GATK report to AWS S3 based on local file " + localFile); + if ( localFile != null ) { + try { + // we succeeded in creating the local file - // Your Amazon Web Services (AWS) login credentials are required to manage S3 accounts. These credentials - // are stored in an AWSCredentials object: - String awsAccessKey = "AKIAJQQEIHTAHSM333EQ"; - AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, S3SecretKey); + // Your Amazon Web Services (AWS) login credentials are required to manage S3 accounts. These credentials + // are stored in an AWSCredentials object: - // To communicate with S3, create a class that implements an S3Service. We will use the REST/HTTP - // implementation based on HttpClient, as this is the most robust implementation provided with JetS3t. - S3Service s3Service = new RestS3Service(awsCredentials); + // IAM GATK user credentials -- only right is to PutObject into GATK_Run_Report bucket + String awsAccessKey = "AKIAJXU7VIHBPDW4TDSQ"; // GATK AWS user + String awsSecretKey = "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"; // GATK AWS user + AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey); - // grab the reports bucket - S3Bucket reportsBucket = s3Service.getBucket(REPORT_BUCKET_NAME); - logger.info("Uploading to bucket: " + reportsBucket); + // To communicate with S3, create a class that implements an S3Service. We will use the REST/HTTP + // implementation based on HttpClient, as this is the most robust implementation provided with JetS3t. + S3Service s3Service = new RestS3Service(awsCredentials); - // Create an S3Object based on a file, with Content-Length set automatically and - // Content-Type set based on the file's extension (using the Mimetypes utility class) - S3Object fileObject = new S3Object(localFile); - logger.info("Created S3Object" + fileObject); - logger.info("Uploading " + localFile + " to AWS bucket"); - s3Service.putObject(reportsBucket, fileObject); - //logger.info("Done. File hash value: " + fileObject.getMd5HashAsHex()); - } catch ( S3ServiceException e ) { - exceptDuringRunReport("S3 exception occurred", e); - } catch ( NoSuchAlgorithmException e ) { - exceptDuringRunReport("Couldn't calculate MD5", e); - } catch ( IOException e ) { - exceptDuringRunReport("Couldn't read report file", e); - } finally { - localFile.delete(); - } + // Create an S3Object based on a file, with Content-Length set automatically and + // Content-Type set based on the file's extension (using the Mimetypes utility class) + S3Object fileObject = new S3Object(localFile); + logger.info("Created S3Object" + fileObject); + logger.info("Uploading " + localFile + " to AWS bucket"); + S3Object s3Object = s3Service.putObject(REPORT_BUCKET_NAME, fileObject); + logger.info("Uploaded: " + s3Object); + } catch ( S3ServiceException e ) { + exceptDuringRunReport("S3 exception occurred", e); + } catch ( NoSuchAlgorithmException e ) { + exceptDuringRunReport("Couldn't calculate MD5", e); + } catch ( IOException e ) { + exceptDuringRunReport("Couldn't read report file", e); + } finally { + localFile.delete(); } } } private void exceptDuringRunReport(String msg, Throwable e) { - logger.warn("An occurred during GATK run reporting [everything is fine, but no report could be generated]. Message is: " + msg + ". Error message is: " + e.getMessage() + ". Stack track follows" + e.getStackTrace()); + logger.warn("An occurred during GATK run reporting [everything is fine, but no report could be generated]. Message is: " + msg + ". Error message is: " + e.getMessage() + ". Stack track follows"); + e.printStackTrace(); } private void exceptDuringRunReport(String msg) {