zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Encrypt GATK AWS keys using the GATK private key, and decrypt as needed as a resource when uploading to AWS logs 

-- Has the overall effect that the GATK user AWS keys are no longer visible in the gatk source as plain text.  This will stop AWS from emailing me (they crawl the web looking for keys)
-- Added utility EncryptAWSKeys that takes as command line arguments the GATK user AWS access and secret keys, encrypts them with the GATK private key, and writes out the resulting file to resources in phonehome.
-- GATKRunReport now decrypts as needed these keys using the GATK public key as resources in the GATK bundle
-- Refactored the essential function of Resource (reading the resource) from IOUtils into the class itself.  Now how to get the data in the resouce is straightforward
-- Refactored md5 calculation code from a byte[] into Utils.  Added unit tests
-- Committing the encrypted AWS keys
-- #resolves https://jira.broadinstitute.org/browse/GSA-730
This commit is contained in:
Mark DePristo 2013-01-30 16:41:23 -05:00
parent 5f4a063def
commit b707331332
10 changed files with 181 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
build.xml
View File

@ -708,6 +708,9 @@
<include name="**/sting/gatk/**/*.R"/>
<include name="**/sting/alignment/**/*.R"/>
</fileset>
<fileset dir="${java.public.source.dir}">
<include name="**/phonehome/*.key"/>
</fileset>
<fileset dir="${key.dir}">
<include name="**/*.key"/>
</fileset>

59
public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java
View File

@ -31,8 +31,11 @@ import org.broadinstitute.sting.gatk.CommandLineGATK;
import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
import org.broadinstitute.sting.gatk.walkers.Walker;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.crypt.CryptUtils;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import org.broadinstitute.sting.utils.exceptions.UserException;
import org.broadinstitute.sting.utils.io.IOUtils;
import org.broadinstitute.sting.utils.io.Resource;
import org.broadinstitute.sting.utils.threading.ThreadEfficiencyMonitor;
import org.jets3t.service.S3Service;
import org.jets3t.service.S3ServiceException;
@ -48,6 +51,7 @@ import org.simpleframework.xml.stream.HyphenStyle;
import java.io.*;
import java.security.NoSuchAlgorithmException;
import java.security.PublicKey;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@ -309,6 +313,51 @@ public class GATKRunReport {
}
}
/**
* Decrypts encrypted AWS key from encryptedKeySource
* @param encryptedKeySource a file containing an encrypted AWS key
* @return a decrypted AWS key as a String
*/
public static String decryptAWSKey(final File encryptedKeySource) throws FileNotFoundException {
return decryptAWSKey(new FileInputStream(encryptedKeySource));
}
/**
* @see #decryptAWSKey(java.io.File) but with input from an inputstream
*/
private static String decryptAWSKey(final InputStream encryptedKeySource) {
final PublicKey key = CryptUtils.loadGATKDistributedPublicKey();
final byte[] fromDisk = IOUtils.readStreamIntoByteArray(encryptedKeySource);
final byte[] decrypted = CryptUtils.decryptData(fromDisk, key);
return new String(decrypted);
}
/**
* Get the decrypted AWS key sorted in the resource directories of name
* @param name the name of the file containing the needed AWS key
* @return a non-null GATK
*/
private static String getAWSKey(final String name) {
final Resource resource = new Resource(name, GATKRunReport.class);
return decryptAWSKey(resource.getResourceContentsAsStream());
}
/**
* Get the AWS access key for the GATK user
* @return a non-null AWS access key for the GATK user
*/
protected static String getAWSAccessKey() {
return getAWSKey("GATK_AWS_access.key");
}
/**
* Get the AWS secret key for the GATK user
* @return a non-null AWS secret key for the GATK user
*/
protected static String getAWSSecretKey() {
return getAWSKey("GATK_AWS_secret.key");
}
private class S3PutRunnable implements Runnable {
public AtomicBoolean isSuccess;
@ -331,17 +380,17 @@ public class GATKRunReport {
// are stored in an AWSCredentials object:
// IAM GATK user credentials -- only right is to PutObject into GATK_Run_Report bucket
String awsAccessKey = "AKIAJXU7VIHBPDW4TDSQ"; // GATK AWS user
String awsSecretKey = "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"; // GATK AWS user
AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey);
final String awsAccessKey = getAWSAccessKey(); // GATK AWS user
final String awsSecretKey = getAWSSecretKey(); // GATK AWS user
final AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey);
// To communicate with S3, create a class that implements an S3Service. We will use the REST/HTTP
// implementation based on HttpClient, as this is the most robust implementation provided with JetS3t.
S3Service s3Service = new RestS3Service(awsCredentials);
final S3Service s3Service = new RestS3Service(awsCredentials);
// Create an S3Object based on a file, with Content-Length set automatically and
// Content-Type set based on the file's extension (using the Mimetypes utility class)
S3Object fileObject = new S3Object(key, report);
final S3Object fileObject = new S3Object(key, report);
//logger.info("Created S3Object" + fileObject);
//logger.info("Uploading " + localFile + " to AWS bucket");
s3Object = s3Service.putObject(REPORT_BUCKET_NAME, fileObject);

2
public/java/src/org/broadinstitute/sting/gatk/phonehome/GATK_AWS_access.key 100644
View File

@ -0,0 +1,2 @@
B<EFBFBD>w“לX~¨[u<><1E>G¢£›װרֵ¦e‰קב‡,םˆ‰)<29>s­/sg1µב<C2B5>L„±ƒQ¹<>aG%$·R<C2B7><52>ל<D79C>{x<>qPz׳<7A><D7B3>”<>J®¬}”ב\¸–´”{(<28>הגֽׁB<D6BD>K’&<26>¶™<C2B6><E284A2>ץַ,’`¥³ק”@`oX
%/`mגײ‘ָ₪3ְhר3<D7A8>אrQ÷ v1<14>a×W•<57>?ˆ‘uH¼<48>ײױַ<D7B1>״זsנֵג<06>aּz¬<7A>¿­ֵ<10>Mֶֻ¦?מUzhקל·k#+<2B>xִ„M<E2809E>םk“ ¦©X<C2A9>²)÷‎°Mִ±<D6B4>ױ• ַipע-מ?jך<6A>תHµ@d+HR|<7C>bע³F

BIN
public/java/src/org/broadinstitute/sting/gatk/phonehome/GATK_AWS_secret.key 100644
View File

Binary file not shown.

28
public/java/src/org/broadinstitute/sting/utils/Utils.java
View File

@ -25,6 +25,7 @@
package org.broadinstitute.sting.utils;
import com.google.java.contract.Ensures;
import com.google.java.contract.Requires;
import net.sf.samtools.SAMFileHeader;
import net.sf.samtools.SAMProgramRecord;
@ -34,7 +35,10 @@ import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
import org.broadinstitute.sting.gatk.io.StingSAMFileWriter;
import org.broadinstitute.sting.utils.text.TextFormattingUtils;
import java.math.BigInteger;
import java.net.InetAddress;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.*;
/**
@ -911,4 +915,28 @@ public class Utils {
return subLists;
}
/**
* @see #calcMD5(byte[])
*/
public static String calcMD5(final String s) throws NoSuchAlgorithmException {
return calcMD5(s.getBytes());
}
/**
* Calculate the md5 for bytes, and return the result as a 32 character string
*
* @param bytes the bytes to calculate the md5 of
* @return the md5 of bytes, as a 32-character long string
* @throws NoSuchAlgorithmException
*/
@Ensures({"result != null", "result.length() == 32"})
public static String calcMD5(final byte[] bytes) throws NoSuchAlgorithmException {
if ( bytes == null ) throw new IllegalArgumentException("bytes cannot be null");
final byte[] thedigest = MessageDigest.getInstance("MD5").digest(bytes);
final BigInteger bigInt = new BigInteger(1, thedigest);
String md5String = bigInt.toString(16);
while (md5String.length() < 32) md5String = "0" + md5String; // pad to length 32
return md5String;
}
}

12
public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java
View File

@ -359,19 +359,9 @@ public class IOUtils {
*/
public static void writeResource(Resource resource, File file) {
String path = resource.getPath();
Class<?> clazz = resource.getRelativeClass();
InputStream inputStream = null;
InputStream inputStream = resource.getResourceContentsAsStream();
OutputStream outputStream = null;
try {
if (clazz == null) {
inputStream = ClassLoader.getSystemResourceAsStream(path);
if (inputStream == null)
throw new IllegalArgumentException("Resource not found: " + path);
} else {
inputStream = clazz.getResourceAsStream(path);
if (inputStream == null)
throw new IllegalArgumentException("Resource not found relative to " + clazz + ": " + path);
}
outputStream = FileUtils.openOutputStream(file);
org.apache.commons.io.IOUtils.copy(inputStream, outputStream);
} catch (IOException e) {

24
public/java/src/org/broadinstitute/sting/utils/io/Resource.java
View File

@ -26,6 +26,7 @@
package org.broadinstitute.sting.utils.io;
import java.io.File;
import java.io.InputStream;
/**
* Stores a resource by path and a relative class.
@ -64,4 +65,27 @@ public class Resource {
File.separator,
path);
}
/**
* Get the contents of this resource as an InputStream
* @throws IllegalArgumentException if resource cannot be read
* @return an input stream that will read the contents of this resource
*/
public InputStream getResourceContentsAsStream() {
final Class<?> clazz = getRelativeClass();
final InputStream inputStream;
if (clazz == null) {
inputStream = ClassLoader.getSystemResourceAsStream(path);
if (inputStream == null)
throw new IllegalArgumentException("Resource not found: " + path);
} else {
inputStream = clazz.getResourceAsStream(path);
if (inputStream == null)
throw new IllegalArgumentException("Resource not found relative to " + clazz + ": " + path);
}
return inputStream;
}
}

9
public/java/test/org/broadinstitute/sting/MD5DB.java
View File

@ -28,11 +28,10 @@ package org.broadinstitute.sting;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.broadinstitute.sting.gatk.walkers.diffengine.DiffEngine;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import java.io.*;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.util.Arrays;
/**
@ -252,11 +251,7 @@ public class MD5DB {
*/
public String testFileMD5(final String name, final File resultsFile, final String expectedMD5, final boolean parameterize) {
try {
byte[] bytesOfMessage = getBytesFromFile(resultsFile);
byte[] thedigest = MessageDigest.getInstance("MD5").digest(bytesOfMessage);
BigInteger bigInt = new BigInteger(1, thedigest);
String filemd5sum = bigInt.toString(16);
while (filemd5sum.length() < 32) filemd5sum = "0" + filemd5sum; // pad to length 32
final String filemd5sum = Utils.calcMD5(getBytesFromFile(resultsFile));
//
// copy md5 to integrationtests

52
public/java/test/org/broadinstitute/sting/gatk/phonehome/GATKRunReportUnitTest.java 100644
View File

@ -0,0 +1,52 @@
/*
* Copyright (c) 2012 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
* THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.phonehome;
import junit.framework.Assert;
import org.broadinstitute.sting.BaseTest;
import org.broadinstitute.sting.utils.Utils;
import org.testng.annotations.Test;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
public class GATKRunReportUnitTest extends BaseTest {
@Test
public void testAccessKey() throws Exception {
testAWSKey(GATKRunReport.getAWSAccessKey(), "c0f0afa1ff5ba41d9bf216cfcdbf26bf");
}
@Test
public void testSecretKey() throws Exception {
testAWSKey(GATKRunReport.getAWSSecretKey(), "db2f13b3a7c98ad24e28783733ec4a62");
}
private void testAWSKey(final String accessKey, final String expectedMD5) throws Exception {
Assert.assertNotNull(accessKey, "AccessKey should not be null");
final String actualmd5 = Utils.calcMD5(accessKey);
Assert.assertEquals(actualmd5, expectedMD5);
}
}

15
public/java/test/org/broadinstitute/sting/utils/UtilsUnitTest.java
View File

@ -25,10 +25,13 @@
package org.broadinstitute.sting.utils;
import org.apache.commons.io.FileUtils;
import org.broadinstitute.sting.utils.io.IOUtils;
import org.testng.Assert;
import org.broadinstitute.sting.BaseTest;
import org.testng.annotations.Test;
import java.io.File;
import java.util.LinkedHashMap;
import java.util.Map;
@ -135,4 +138,16 @@ public class UtilsUnitTest extends BaseTest {
actual = Utils.escapeExpressions(" one two 'three four' ");
Assert.assertEquals(actual, expected);
}
@Test
public void testCalcMD5() throws Exception {
final File source = new File(publicTestDir + "exampleFASTA.fasta");
final String sourceMD5 = "36880691cf9e4178216f7b52e8d85fbe";
final byte[] sourceBytes = IOUtils.readFileIntoByteArray(source);
Assert.assertEquals(Utils.calcMD5(sourceBytes), sourceMD5);
final String sourceString = FileUtils.readFileToString(source);
Assert.assertEquals(Utils.calcMD5(sourceString), sourceMD5);
}
}