From b70733133260bc76cdd1cfcb6efdc89107f0f005 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Wed, 30 Jan 2013 16:41:23 -0500 Subject: [PATCH] Encrypt GATK AWS keys using the GATK private key, and decrypt as needed as a resource when uploading to AWS logs -- Has the overall effect that the GATK user AWS keys are no longer visible in the gatk source as plain text. This will stop AWS from emailing me (they crawl the web looking for keys) -- Added utility EncryptAWSKeys that takes as command line arguments the GATK user AWS access and secret keys, encrypts them with the GATK private key, and writes out the resulting file to resources in phonehome. -- GATKRunReport now decrypts as needed these keys using the GATK public key as resources in the GATK bundle -- Refactored the essential function of Resource (reading the resource) from IOUtils into the class itself. Now how to get the data in the resouce is straightforward -- Refactored md5 calculation code from a byte[] into Utils. Added unit tests -- Committing the encrypted AWS keys -- #resolves https://jira.broadinstitute.org/browse/GSA-730 --- build.xml | 3 + .../sting/gatk/phonehome/GATKRunReport.java | 59 ++++++++++++++++-- .../sting/gatk/phonehome/GATK_AWS_access.key | 2 + .../sting/gatk/phonehome/GATK_AWS_secret.key | Bin 0 -> 256 bytes .../org/broadinstitute/sting/utils/Utils.java | 28 +++++++++ .../sting/utils/io/IOUtils.java | 12 +--- .../sting/utils/io/Resource.java | 24 +++++++ .../test/org/broadinstitute/sting/MD5DB.java | 9 +-- .../gatk/phonehome/GATKRunReportUnitTest.java | 52 +++++++++++++++ .../sting/utils/UtilsUnitTest.java | 15 +++++ 10 files changed, 181 insertions(+), 23 deletions(-) create mode 100644 public/java/src/org/broadinstitute/sting/gatk/phonehome/GATK_AWS_access.key create mode 100644 public/java/src/org/broadinstitute/sting/gatk/phonehome/GATK_AWS_secret.key create mode 100644 public/java/test/org/broadinstitute/sting/gatk/phonehome/GATKRunReportUnitTest.java diff --git a/build.xml b/build.xml index e92e41c10..1e88bb400 100644 --- a/build.xml +++ b/build.xml @@ -708,6 +708,9 @@ + + + diff --git a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java index c2a064dbe..839ebcfa1 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java +++ b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java @@ -31,8 +31,11 @@ import org.broadinstitute.sting.gatk.CommandLineGATK; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.gatk.walkers.Walker; import org.broadinstitute.sting.utils.Utils; +import org.broadinstitute.sting.utils.crypt.CryptUtils; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; +import org.broadinstitute.sting.utils.io.IOUtils; +import org.broadinstitute.sting.utils.io.Resource; import org.broadinstitute.sting.utils.threading.ThreadEfficiencyMonitor; import org.jets3t.service.S3Service; import org.jets3t.service.S3ServiceException; @@ -48,6 +51,7 @@ import org.simpleframework.xml.stream.HyphenStyle; import java.io.*; import java.security.NoSuchAlgorithmException; +import java.security.PublicKey; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -309,6 +313,51 @@ public class GATKRunReport { } } + /** + * Decrypts encrypted AWS key from encryptedKeySource + * @param encryptedKeySource a file containing an encrypted AWS key + * @return a decrypted AWS key as a String + */ + public static String decryptAWSKey(final File encryptedKeySource) throws FileNotFoundException { + return decryptAWSKey(new FileInputStream(encryptedKeySource)); + } + + /** + * @see #decryptAWSKey(java.io.File) but with input from an inputstream + */ + private static String decryptAWSKey(final InputStream encryptedKeySource) { + final PublicKey key = CryptUtils.loadGATKDistributedPublicKey(); + final byte[] fromDisk = IOUtils.readStreamIntoByteArray(encryptedKeySource); + final byte[] decrypted = CryptUtils.decryptData(fromDisk, key); + return new String(decrypted); + } + + /** + * Get the decrypted AWS key sorted in the resource directories of name + * @param name the name of the file containing the needed AWS key + * @return a non-null GATK + */ + private static String getAWSKey(final String name) { + final Resource resource = new Resource(name, GATKRunReport.class); + return decryptAWSKey(resource.getResourceContentsAsStream()); + } + + /** + * Get the AWS access key for the GATK user + * @return a non-null AWS access key for the GATK user + */ + protected static String getAWSAccessKey() { + return getAWSKey("GATK_AWS_access.key"); + } + + /** + * Get the AWS secret key for the GATK user + * @return a non-null AWS secret key for the GATK user + */ + protected static String getAWSSecretKey() { + return getAWSKey("GATK_AWS_secret.key"); + } + private class S3PutRunnable implements Runnable { public AtomicBoolean isSuccess; @@ -331,17 +380,17 @@ public class GATKRunReport { // are stored in an AWSCredentials object: // IAM GATK user credentials -- only right is to PutObject into GATK_Run_Report bucket - String awsAccessKey = "AKIAJXU7VIHBPDW4TDSQ"; // GATK AWS user - String awsSecretKey = "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"; // GATK AWS user - AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey); + final String awsAccessKey = getAWSAccessKey(); // GATK AWS user + final String awsSecretKey = getAWSSecretKey(); // GATK AWS user + final AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey); // To communicate with S3, create a class that implements an S3Service. We will use the REST/HTTP // implementation based on HttpClient, as this is the most robust implementation provided with JetS3t. - S3Service s3Service = new RestS3Service(awsCredentials); + final S3Service s3Service = new RestS3Service(awsCredentials); // Create an S3Object based on a file, with Content-Length set automatically and // Content-Type set based on the file's extension (using the Mimetypes utility class) - S3Object fileObject = new S3Object(key, report); + final S3Object fileObject = new S3Object(key, report); //logger.info("Created S3Object" + fileObject); //logger.info("Uploading " + localFile + " to AWS bucket"); s3Object = s3Service.putObject(REPORT_BUCKET_NAME, fileObject); diff --git a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATK_AWS_access.key b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATK_AWS_access.key new file mode 100644 index 000000000..45242c3cd --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATK_AWS_access.key @@ -0,0 +1,2 @@ +BwX~[uGe,툉)s/sg1LQaG%$R݊{xqPz׊J}\{(BK&ܶ,`@`oX +%/`m֑Ȥ3h3rQ v1aW?uHsaz޿M˦?Uzhk#+xĄMk X)MıՕ ip-?jH@d+HR|bF \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATK_AWS_secret.key b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATK_AWS_secret.key new file mode 100644 index 0000000000000000000000000000000000000000..b86a25036e84ae8a1373776e52f1ef04a135bfd0 GIT binary patch literal 256 zcmV+b0ssClQ_lh#EL zNy%zD4$cZr3OHzad*T{?JVK|@Tkm)?rp1pN33JU-fi{b*XQH?g&z%d%(ha|KT`EjN zBSe){t0~(z%*aiOc6s(*c)DT0;m|PNz6Iq-NLK3b$Te4ViCd4fXKs%s(Pj` clazz = resource.getRelativeClass(); - InputStream inputStream = null; + InputStream inputStream = resource.getResourceContentsAsStream(); OutputStream outputStream = null; try { - if (clazz == null) { - inputStream = ClassLoader.getSystemResourceAsStream(path); - if (inputStream == null) - throw new IllegalArgumentException("Resource not found: " + path); - } else { - inputStream = clazz.getResourceAsStream(path); - if (inputStream == null) - throw new IllegalArgumentException("Resource not found relative to " + clazz + ": " + path); - } outputStream = FileUtils.openOutputStream(file); org.apache.commons.io.IOUtils.copy(inputStream, outputStream); } catch (IOException e) { diff --git a/public/java/src/org/broadinstitute/sting/utils/io/Resource.java b/public/java/src/org/broadinstitute/sting/utils/io/Resource.java index 1f181a826..85ca5ce1c 100644 --- a/public/java/src/org/broadinstitute/sting/utils/io/Resource.java +++ b/public/java/src/org/broadinstitute/sting/utils/io/Resource.java @@ -26,6 +26,7 @@ package org.broadinstitute.sting.utils.io; import java.io.File; +import java.io.InputStream; /** * Stores a resource by path and a relative class. @@ -64,4 +65,27 @@ public class Resource { File.separator, path); } + + /** + * Get the contents of this resource as an InputStream + * @throws IllegalArgumentException if resource cannot be read + * @return an input stream that will read the contents of this resource + */ + public InputStream getResourceContentsAsStream() { + final Class clazz = getRelativeClass(); + + final InputStream inputStream; + if (clazz == null) { + inputStream = ClassLoader.getSystemResourceAsStream(path); + if (inputStream == null) + throw new IllegalArgumentException("Resource not found: " + path); + } else { + inputStream = clazz.getResourceAsStream(path); + if (inputStream == null) + throw new IllegalArgumentException("Resource not found relative to " + clazz + ": " + path); + + } + + return inputStream; + } } diff --git a/public/java/test/org/broadinstitute/sting/MD5DB.java b/public/java/test/org/broadinstitute/sting/MD5DB.java index aed98b78a..2b0d52a11 100644 --- a/public/java/test/org/broadinstitute/sting/MD5DB.java +++ b/public/java/test/org/broadinstitute/sting/MD5DB.java @@ -28,11 +28,10 @@ package org.broadinstitute.sting; import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import org.broadinstitute.sting.gatk.walkers.diffengine.DiffEngine; +import org.broadinstitute.sting.utils.Utils; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import java.io.*; -import java.math.BigInteger; -import java.security.MessageDigest; import java.util.Arrays; /** @@ -252,11 +251,7 @@ public class MD5DB { */ public String testFileMD5(final String name, final File resultsFile, final String expectedMD5, final boolean parameterize) { try { - byte[] bytesOfMessage = getBytesFromFile(resultsFile); - byte[] thedigest = MessageDigest.getInstance("MD5").digest(bytesOfMessage); - BigInteger bigInt = new BigInteger(1, thedigest); - String filemd5sum = bigInt.toString(16); - while (filemd5sum.length() < 32) filemd5sum = "0" + filemd5sum; // pad to length 32 + final String filemd5sum = Utils.calcMD5(getBytesFromFile(resultsFile)); // // copy md5 to integrationtests diff --git a/public/java/test/org/broadinstitute/sting/gatk/phonehome/GATKRunReportUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/phonehome/GATKRunReportUnitTest.java new file mode 100644 index 000000000..03f19968c --- /dev/null +++ b/public/java/test/org/broadinstitute/sting/gatk/phonehome/GATKRunReportUnitTest.java @@ -0,0 +1,52 @@ +/* +* Copyright (c) 2012 The Broad Institute +* +* Permission is hereby granted, free of charge, to any person +* obtaining a copy of this software and associated documentation +* files (the "Software"), to deal in the Software without +* restriction, including without limitation the rights to use, +* copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following +* conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +* THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +package org.broadinstitute.sting.gatk.phonehome; + +import junit.framework.Assert; +import org.broadinstitute.sting.BaseTest; +import org.broadinstitute.sting.utils.Utils; +import org.testng.annotations.Test; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +public class GATKRunReportUnitTest extends BaseTest { + @Test + public void testAccessKey() throws Exception { + testAWSKey(GATKRunReport.getAWSAccessKey(), "c0f0afa1ff5ba41d9bf216cfcdbf26bf"); + } + + @Test + public void testSecretKey() throws Exception { + testAWSKey(GATKRunReport.getAWSSecretKey(), "db2f13b3a7c98ad24e28783733ec4a62"); + } + + private void testAWSKey(final String accessKey, final String expectedMD5) throws Exception { + Assert.assertNotNull(accessKey, "AccessKey should not be null"); + final String actualmd5 = Utils.calcMD5(accessKey); + Assert.assertEquals(actualmd5, expectedMD5); + } +} diff --git a/public/java/test/org/broadinstitute/sting/utils/UtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/UtilsUnitTest.java index fc10f1102..5d6ecd0f9 100644 --- a/public/java/test/org/broadinstitute/sting/utils/UtilsUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/UtilsUnitTest.java @@ -25,10 +25,13 @@ package org.broadinstitute.sting.utils; +import org.apache.commons.io.FileUtils; +import org.broadinstitute.sting.utils.io.IOUtils; import org.testng.Assert; import org.broadinstitute.sting.BaseTest; import org.testng.annotations.Test; +import java.io.File; import java.util.LinkedHashMap; import java.util.Map; @@ -135,4 +138,16 @@ public class UtilsUnitTest extends BaseTest { actual = Utils.escapeExpressions(" one two 'three four' "); Assert.assertEquals(actual, expected); } + + @Test + public void testCalcMD5() throws Exception { + final File source = new File(publicTestDir + "exampleFASTA.fasta"); + final String sourceMD5 = "36880691cf9e4178216f7b52e8d85fbe"; + + final byte[] sourceBytes = IOUtils.readFileIntoByteArray(source); + Assert.assertEquals(Utils.calcMD5(sourceBytes), sourceMD5); + + final String sourceString = FileUtils.readFileToString(source); + Assert.assertEquals(Utils.calcMD5(sourceString), sourceMD5); + } }