diff --git a/build.xml b/build.xml index e92e41c10..1e88bb400 100644 --- a/build.xml +++ b/build.xml @@ -708,6 +708,9 @@ + + + diff --git a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java index c2a064dbe..839ebcfa1 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java +++ b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java @@ -31,8 +31,11 @@ import org.broadinstitute.sting.gatk.CommandLineGATK; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.gatk.walkers.Walker; import org.broadinstitute.sting.utils.Utils; +import org.broadinstitute.sting.utils.crypt.CryptUtils; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; +import org.broadinstitute.sting.utils.io.IOUtils; +import org.broadinstitute.sting.utils.io.Resource; import org.broadinstitute.sting.utils.threading.ThreadEfficiencyMonitor; import org.jets3t.service.S3Service; import org.jets3t.service.S3ServiceException; @@ -48,6 +51,7 @@ import org.simpleframework.xml.stream.HyphenStyle; import java.io.*; import java.security.NoSuchAlgorithmException; +import java.security.PublicKey; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -309,6 +313,51 @@ public class GATKRunReport { } } + /** + * Decrypts encrypted AWS key from encryptedKeySource + * @param encryptedKeySource a file containing an encrypted AWS key + * @return a decrypted AWS key as a String + */ + public static String decryptAWSKey(final File encryptedKeySource) throws FileNotFoundException { + return decryptAWSKey(new FileInputStream(encryptedKeySource)); + } + + /** + * @see #decryptAWSKey(java.io.File) but with input from an inputstream + */ + private static String decryptAWSKey(final InputStream encryptedKeySource) { + final PublicKey key = CryptUtils.loadGATKDistributedPublicKey(); + final byte[] fromDisk = IOUtils.readStreamIntoByteArray(encryptedKeySource); + final byte[] decrypted = CryptUtils.decryptData(fromDisk, key); + return new String(decrypted); + } + + /** + * Get the decrypted AWS key sorted in the resource directories of name + * @param name the name of the file containing the needed AWS key + * @return a non-null GATK + */ + private static String getAWSKey(final String name) { + final Resource resource = new Resource(name, GATKRunReport.class); + return decryptAWSKey(resource.getResourceContentsAsStream()); + } + + /** + * Get the AWS access key for the GATK user + * @return a non-null AWS access key for the GATK user + */ + protected static String getAWSAccessKey() { + return getAWSKey("GATK_AWS_access.key"); + } + + /** + * Get the AWS secret key for the GATK user + * @return a non-null AWS secret key for the GATK user + */ + protected static String getAWSSecretKey() { + return getAWSKey("GATK_AWS_secret.key"); + } + private class S3PutRunnable implements Runnable { public AtomicBoolean isSuccess; @@ -331,17 +380,17 @@ public class GATKRunReport { // are stored in an AWSCredentials object: // IAM GATK user credentials -- only right is to PutObject into GATK_Run_Report bucket - String awsAccessKey = "AKIAJXU7VIHBPDW4TDSQ"; // GATK AWS user - String awsSecretKey = "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"; // GATK AWS user - AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey); + final String awsAccessKey = getAWSAccessKey(); // GATK AWS user + final String awsSecretKey = getAWSSecretKey(); // GATK AWS user + final AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey); // To communicate with S3, create a class that implements an S3Service. We will use the REST/HTTP // implementation based on HttpClient, as this is the most robust implementation provided with JetS3t. - S3Service s3Service = new RestS3Service(awsCredentials); + final S3Service s3Service = new RestS3Service(awsCredentials); // Create an S3Object based on a file, with Content-Length set automatically and // Content-Type set based on the file's extension (using the Mimetypes utility class) - S3Object fileObject = new S3Object(key, report); + final S3Object fileObject = new S3Object(key, report); //logger.info("Created S3Object" + fileObject); //logger.info("Uploading " + localFile + " to AWS bucket"); s3Object = s3Service.putObject(REPORT_BUCKET_NAME, fileObject); diff --git a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATK_AWS_access.key b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATK_AWS_access.key new file mode 100644 index 000000000..45242c3cd --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATK_AWS_access.key @@ -0,0 +1,2 @@ +BwX~[uGe,툉)s/sg1LQaG%$R݊{xqPz׊J}\{(BK&ܶ,`@`oX +%/`m֑Ȥ3h3rQ v1aW?uHsaz޿M˦?Uzhk#+xĄMk X)MıՕ ip-?jH@d+HR|bF \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATK_AWS_secret.key b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATK_AWS_secret.key new file mode 100644 index 000000000..b86a25036 Binary files /dev/null and b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATK_AWS_secret.key differ diff --git a/public/java/src/org/broadinstitute/sting/utils/Utils.java b/public/java/src/org/broadinstitute/sting/utils/Utils.java index 3f5dffa68..eb287abd8 100644 --- a/public/java/src/org/broadinstitute/sting/utils/Utils.java +++ b/public/java/src/org/broadinstitute/sting/utils/Utils.java @@ -25,6 +25,7 @@ package org.broadinstitute.sting.utils; +import com.google.java.contract.Ensures; import com.google.java.contract.Requires; import net.sf.samtools.SAMFileHeader; import net.sf.samtools.SAMProgramRecord; @@ -34,7 +35,10 @@ import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.gatk.io.StingSAMFileWriter; import org.broadinstitute.sting.utils.text.TextFormattingUtils; +import java.math.BigInteger; import java.net.InetAddress; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.*; /** @@ -911,4 +915,28 @@ public class Utils { return subLists; } + /** + * @see #calcMD5(byte[]) + */ + public static String calcMD5(final String s) throws NoSuchAlgorithmException { + return calcMD5(s.getBytes()); + } + + /** + * Calculate the md5 for bytes, and return the result as a 32 character string + * + * @param bytes the bytes to calculate the md5 of + * @return the md5 of bytes, as a 32-character long string + * @throws NoSuchAlgorithmException + */ + @Ensures({"result != null", "result.length() == 32"}) + public static String calcMD5(final byte[] bytes) throws NoSuchAlgorithmException { + if ( bytes == null ) throw new IllegalArgumentException("bytes cannot be null"); + final byte[] thedigest = MessageDigest.getInstance("MD5").digest(bytes); + final BigInteger bigInt = new BigInteger(1, thedigest); + + String md5String = bigInt.toString(16); + while (md5String.length() < 32) md5String = "0" + md5String; // pad to length 32 + return md5String; + } } diff --git a/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java b/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java index c25ecb49b..59466dad3 100644 --- a/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java @@ -359,19 +359,9 @@ public class IOUtils { */ public static void writeResource(Resource resource, File file) { String path = resource.getPath(); - Class clazz = resource.getRelativeClass(); - InputStream inputStream = null; + InputStream inputStream = resource.getResourceContentsAsStream(); OutputStream outputStream = null; try { - if (clazz == null) { - inputStream = ClassLoader.getSystemResourceAsStream(path); - if (inputStream == null) - throw new IllegalArgumentException("Resource not found: " + path); - } else { - inputStream = clazz.getResourceAsStream(path); - if (inputStream == null) - throw new IllegalArgumentException("Resource not found relative to " + clazz + ": " + path); - } outputStream = FileUtils.openOutputStream(file); org.apache.commons.io.IOUtils.copy(inputStream, outputStream); } catch (IOException e) { diff --git a/public/java/src/org/broadinstitute/sting/utils/io/Resource.java b/public/java/src/org/broadinstitute/sting/utils/io/Resource.java index 1f181a826..85ca5ce1c 100644 --- a/public/java/src/org/broadinstitute/sting/utils/io/Resource.java +++ b/public/java/src/org/broadinstitute/sting/utils/io/Resource.java @@ -26,6 +26,7 @@ package org.broadinstitute.sting.utils.io; import java.io.File; +import java.io.InputStream; /** * Stores a resource by path and a relative class. @@ -64,4 +65,27 @@ public class Resource { File.separator, path); } + + /** + * Get the contents of this resource as an InputStream + * @throws IllegalArgumentException if resource cannot be read + * @return an input stream that will read the contents of this resource + */ + public InputStream getResourceContentsAsStream() { + final Class clazz = getRelativeClass(); + + final InputStream inputStream; + if (clazz == null) { + inputStream = ClassLoader.getSystemResourceAsStream(path); + if (inputStream == null) + throw new IllegalArgumentException("Resource not found: " + path); + } else { + inputStream = clazz.getResourceAsStream(path); + if (inputStream == null) + throw new IllegalArgumentException("Resource not found relative to " + clazz + ": " + path); + + } + + return inputStream; + } } diff --git a/public/java/test/org/broadinstitute/sting/MD5DB.java b/public/java/test/org/broadinstitute/sting/MD5DB.java index aed98b78a..2b0d52a11 100644 --- a/public/java/test/org/broadinstitute/sting/MD5DB.java +++ b/public/java/test/org/broadinstitute/sting/MD5DB.java @@ -28,11 +28,10 @@ package org.broadinstitute.sting; import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import org.broadinstitute.sting.gatk.walkers.diffengine.DiffEngine; +import org.broadinstitute.sting.utils.Utils; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import java.io.*; -import java.math.BigInteger; -import java.security.MessageDigest; import java.util.Arrays; /** @@ -252,11 +251,7 @@ public class MD5DB { */ public String testFileMD5(final String name, final File resultsFile, final String expectedMD5, final boolean parameterize) { try { - byte[] bytesOfMessage = getBytesFromFile(resultsFile); - byte[] thedigest = MessageDigest.getInstance("MD5").digest(bytesOfMessage); - BigInteger bigInt = new BigInteger(1, thedigest); - String filemd5sum = bigInt.toString(16); - while (filemd5sum.length() < 32) filemd5sum = "0" + filemd5sum; // pad to length 32 + final String filemd5sum = Utils.calcMD5(getBytesFromFile(resultsFile)); // // copy md5 to integrationtests diff --git a/public/java/test/org/broadinstitute/sting/gatk/phonehome/GATKRunReportUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/phonehome/GATKRunReportUnitTest.java new file mode 100644 index 000000000..03f19968c --- /dev/null +++ b/public/java/test/org/broadinstitute/sting/gatk/phonehome/GATKRunReportUnitTest.java @@ -0,0 +1,52 @@ +/* +* Copyright (c) 2012 The Broad Institute +* +* Permission is hereby granted, free of charge, to any person +* obtaining a copy of this software and associated documentation +* files (the "Software"), to deal in the Software without +* restriction, including without limitation the rights to use, +* copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following +* conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +* THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +package org.broadinstitute.sting.gatk.phonehome; + +import junit.framework.Assert; +import org.broadinstitute.sting.BaseTest; +import org.broadinstitute.sting.utils.Utils; +import org.testng.annotations.Test; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +public class GATKRunReportUnitTest extends BaseTest { + @Test + public void testAccessKey() throws Exception { + testAWSKey(GATKRunReport.getAWSAccessKey(), "c0f0afa1ff5ba41d9bf216cfcdbf26bf"); + } + + @Test + public void testSecretKey() throws Exception { + testAWSKey(GATKRunReport.getAWSSecretKey(), "db2f13b3a7c98ad24e28783733ec4a62"); + } + + private void testAWSKey(final String accessKey, final String expectedMD5) throws Exception { + Assert.assertNotNull(accessKey, "AccessKey should not be null"); + final String actualmd5 = Utils.calcMD5(accessKey); + Assert.assertEquals(actualmd5, expectedMD5); + } +} diff --git a/public/java/test/org/broadinstitute/sting/utils/UtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/UtilsUnitTest.java index fc10f1102..5d6ecd0f9 100644 --- a/public/java/test/org/broadinstitute/sting/utils/UtilsUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/UtilsUnitTest.java @@ -25,10 +25,13 @@ package org.broadinstitute.sting.utils; +import org.apache.commons.io.FileUtils; +import org.broadinstitute.sting.utils.io.IOUtils; import org.testng.Assert; import org.broadinstitute.sting.BaseTest; import org.testng.annotations.Test; +import java.io.File; import java.util.LinkedHashMap; import java.util.Map; @@ -135,4 +138,16 @@ public class UtilsUnitTest extends BaseTest { actual = Utils.escapeExpressions(" one two 'three four' "); Assert.assertEquals(actual, expected); } + + @Test + public void testCalcMD5() throws Exception { + final File source = new File(publicTestDir + "exampleFASTA.fasta"); + final String sourceMD5 = "36880691cf9e4178216f7b52e8d85fbe"; + + final byte[] sourceBytes = IOUtils.readFileIntoByteArray(source); + Assert.assertEquals(Utils.calcMD5(sourceBytes), sourceMD5); + + final String sourceString = FileUtils.readFileToString(source); + Assert.assertEquals(Utils.calcMD5(sourceString), sourceMD5); + } }