zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:broadinstitute/gsa-unstable

This commit is contained in:
Ryan Poplin 2013-01-30 17:12:28 -05:00
parent bb29bd7df7 b707331332
commit 438c98035b
10 changed files with 181 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
build.xml
View File

@ -708,6 +708,9 @@
<include name="**/sting/gatk/**/*.R"/>
<include name="**/sting/alignment/**/*.R"/>
</fileset>
<fileset dir="${java.public.source.dir}">
<include name="**/phonehome/*.key"/>
</fileset>
<fileset dir="${key.dir}">
<include name="**/*.key"/>
</fileset>

59
public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java
View File

@ -31,8 +31,11 @@ import org.broadinstitute.sting.gatk.CommandLineGATK;
import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
import org.broadinstitute.sting.gatk.walkers.Walker;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.crypt.CryptUtils;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import org.broadinstitute.sting.utils.exceptions.UserException;
import org.broadinstitute.sting.utils.io.IOUtils;
import org.broadinstitute.sting.utils.io.Resource;
import org.broadinstitute.sting.utils.threading.ThreadEfficiencyMonitor;
import org.jets3t.service.S3Service;
import org.jets3t.service.S3ServiceException;
@ -48,6 +51,7 @@ import org.simpleframework.xml.stream.HyphenStyle;
import java.io.*;
import java.security.NoSuchAlgorithmException;
import java.security.PublicKey;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@ -309,6 +313,51 @@ public class GATKRunReport {
}
}
/**
* Decrypts encrypted AWS key from encryptedKeySource
* @param encryptedKeySource a file containing an encrypted AWS key
* @return a decrypted AWS key as a String
*/
public static String decryptAWSKey(final File encryptedKeySource) throws FileNotFoundException {
return decryptAWSKey(new FileInputStream(encryptedKeySource));
}
/**
* @see #decryptAWSKey(java.io.File) but with input from an inputstream
*/
private static String decryptAWSKey(final InputStream encryptedKeySource) {
final PublicKey key = CryptUtils.loadGATKDistributedPublicKey();
final byte[] fromDisk = IOUtils.readStreamIntoByteArray(encryptedKeySource);
final byte[] decrypted = CryptUtils.decryptData(fromDisk, key);
return new String(decrypted);
}
/**
* Get the decrypted AWS key sorted in the resource directories of name
* @param name the name of the file containing the needed AWS key
* @return a non-null GATK
*/
private static String getAWSKey(final String name) {
final Resource resource = new Resource(name, GATKRunReport.class);
return decryptAWSKey(resource.getResourceContentsAsStream());
}
/**
* Get the AWS access key for the GATK user
* @return a non-null AWS access key for the GATK user
*/
protected static String getAWSAccessKey() {
return getAWSKey("GATK_AWS_access.key");
}
/**
* Get the AWS secret key for the GATK user
* @return a non-null AWS secret key for the GATK user
*/
protected static String getAWSSecretKey() {
return getAWSKey("GATK_AWS_secret.key");
}
private class S3PutRunnable implements Runnable {
public AtomicBoolean isSuccess;
@ -331,17 +380,17 @@ public class GATKRunReport {
// are stored in an AWSCredentials object:
// IAM GATK user credentials -- only right is to PutObject into GATK_Run_Report bucket
String awsAccessKey = "AKIAJXU7VIHBPDW4TDSQ"; // GATK AWS user
String awsSecretKey = "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"; // GATK AWS user
AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey);
final String awsAccessKey = getAWSAccessKey(); // GATK AWS user
final String awsSecretKey = getAWSSecretKey(); // GATK AWS user
final AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey);
// To communicate with S3, create a class that implements an S3Service. We will use the REST/HTTP
// implementation based on HttpClient, as this is the most robust implementation provided with JetS3t.
S3Service s3Service = new RestS3Service(awsCredentials);
final S3Service s3Service = new RestS3Service(awsCredentials);
// Create an S3Object based on a file, with Content-Length set automatically and
// Content-Type set based on the file's extension (using the Mimetypes utility class)
S3Object fileObject = new S3Object(key, report);
final S3Object fileObject = new S3Object(key, report);
//logger.info("Created S3Object" + fileObject);
//logger.info("Uploading " + localFile + " to AWS bucket");
s3Object = s3Service.putObject(REPORT_BUCKET_NAME, fileObject);

2
public/java/src/org/broadinstitute/sting/gatk/phonehome/GATK_AWS_access.key 100644
View File

@ -0,0 +1,2 @@
B<EFBFBD>w“לX~¨[u<><1E>G¢£›װרֵ¦e‰קב‡,םˆ‰)<29>s­/sg1µב<C2B5>L„±ƒQ¹<>aG%$·R<C2B7><52>ל<D79C>{x<>qPz׳<7A><D7B3>”<>J®¬}”ב\¸–´”{(<28>הגֽׁB<D6BD>K’&<26>¶™<C2B6><E284A2>ץַ,’`¥³ק”@`oX
%/`mגײ‘ָ₪3ְhר3<D7A8>אrQ÷ v1<14>a×W•<57>?ˆ‘uH¼<48>ײױַ<D7B1>״זsנֵג<06>aּz¬<7A>¿­ֵ<10>Mֶֻ¦?מUzhקל·k#+<2B>xִ„M<E2809E>םk“ ¦©X<C2A9>²)÷‎°Mִ±<D6B4>ױ• ַipע-מ?jך<6A>תHµ@d+HR|<7C>bע³F

BIN
public/java/src/org/broadinstitute/sting/gatk/phonehome/GATK_AWS_secret.key 100644
View File

Binary file not shown.

28
public/java/src/org/broadinstitute/sting/utils/Utils.java
View File

@ -25,6 +25,7 @@
package org.broadinstitute.sting.utils;
import com.google.java.contract.Ensures;
import com.google.java.contract.Requires;
import net.sf.samtools.SAMFileHeader;
import net.sf.samtools.SAMProgramRecord;
@ -34,7 +35,10 @@ import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
import org.broadinstitute.sting.gatk.io.StingSAMFileWriter;
import org.broadinstitute.sting.utils.text.TextFormattingUtils;
import java.math.BigInteger;
import java.net.InetAddress;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.*;
/**
@ -911,4 +915,28 @@ public class Utils {
return subLists;
}
/**
* @see #calcMD5(byte[])
*/
public static String calcMD5(final String s) throws NoSuchAlgorithmException {
return calcMD5(s.getBytes());
}
/**
* Calculate the md5 for bytes, and return the result as a 32 character string
*
* @param bytes the bytes to calculate the md5 of
* @return the md5 of bytes, as a 32-character long string
* @throws NoSuchAlgorithmException
*/
@Ensures({"result != null", "result.length() == 32"})
public static String calcMD5(final byte[] bytes) throws NoSuchAlgorithmException {
if ( bytes == null ) throw new IllegalArgumentException("bytes cannot be null");
final byte[] thedigest = MessageDigest.getInstance("MD5").digest(bytes);
final BigInteger bigInt = new BigInteger(1, thedigest);
String md5String = bigInt.toString(16);
while (md5String.length() < 32) md5String = "0" + md5String; // pad to length 32
return md5String;
}
}

12
public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java
View File

@ -359,19 +359,9 @@ public class IOUtils {
*/
public static void writeResource(Resource resource, File file) {
String path = resource.getPath();
Class<?> clazz = resource.getRelativeClass();
InputStream inputStream = null;
InputStream inputStream = resource.getResourceContentsAsStream();
OutputStream outputStream = null;
try {
if (clazz == null) {
inputStream = ClassLoader.getSystemResourceAsStream(path);
if (inputStream == null)
throw new IllegalArgumentException("Resource not found: " + path);
} else {
inputStream = clazz.getResourceAsStream(path);
if (inputStream == null)
throw new IllegalArgumentException("Resource not found relative to " + clazz + ": " + path);
}
outputStream = FileUtils.openOutputStream(file);
org.apache.commons.io.IOUtils.copy(inputStream, outputStream);
} catch (IOException e) {

24
public/java/src/org/broadinstitute/sting/utils/io/Resource.java
View File

@ -26,6 +26,7 @@
package org.broadinstitute.sting.utils.io;
import java.io.File;
import java.io.InputStream;
/**
* Stores a resource by path and a relative class.
@ -64,4 +65,27 @@ public class Resource {
File.separator,
path);
}
/**
* Get the contents of this resource as an InputStream
* @throws IllegalArgumentException if resource cannot be read
* @return an input stream that will read the contents of this resource
*/
public InputStream getResourceContentsAsStream() {
final Class<?> clazz = getRelativeClass();
final InputStream inputStream;
if (clazz == null) {
inputStream = ClassLoader.getSystemResourceAsStream(path);
if (inputStream == null)
throw new IllegalArgumentException("Resource not found: " + path);
} else {
inputStream = clazz.getResourceAsStream(path);
if (inputStream == null)
throw new IllegalArgumentException("Resource not found relative to " + clazz + ": " + path);
}
return inputStream;
}
}

9
public/java/test/org/broadinstitute/sting/MD5DB.java
View File

@ -28,11 +28,10 @@ package org.broadinstitute.sting;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.broadinstitute.sting.gatk.walkers.diffengine.DiffEngine;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import java.io.*;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.util.Arrays;
/**
@ -252,11 +251,7 @@ public class MD5DB {
*/
public String testFileMD5(final String name, final File resultsFile, final String expectedMD5, final boolean parameterize) {
try {
byte[] bytesOfMessage = getBytesFromFile(resultsFile);
byte[] thedigest = MessageDigest.getInstance("MD5").digest(bytesOfMessage);
BigInteger bigInt = new BigInteger(1, thedigest);
String filemd5sum = bigInt.toString(16);
while (filemd5sum.length() < 32) filemd5sum = "0" + filemd5sum; // pad to length 32
final String filemd5sum = Utils.calcMD5(getBytesFromFile(resultsFile));
//
// copy md5 to integrationtests

52
public/java/test/org/broadinstitute/sting/gatk/phonehome/GATKRunReportUnitTest.java 100644
View File

@ -0,0 +1,52 @@
/*
* Copyright (c) 2012 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
* THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.phonehome;
import junit.framework.Assert;
import org.broadinstitute.sting.BaseTest;
import org.broadinstitute.sting.utils.Utils;
import org.testng.annotations.Test;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
public class GATKRunReportUnitTest extends BaseTest {
@Test
public void testAccessKey() throws Exception {
testAWSKey(GATKRunReport.getAWSAccessKey(), "c0f0afa1ff5ba41d9bf216cfcdbf26bf");
}
@Test
public void testSecretKey() throws Exception {
testAWSKey(GATKRunReport.getAWSSecretKey(), "db2f13b3a7c98ad24e28783733ec4a62");
}
private void testAWSKey(final String accessKey, final String expectedMD5) throws Exception {
Assert.assertNotNull(accessKey, "AccessKey should not be null");
final String actualmd5 = Utils.calcMD5(accessKey);
Assert.assertEquals(actualmd5, expectedMD5);
}
}

15
public/java/test/org/broadinstitute/sting/utils/UtilsUnitTest.java
View File

@ -25,10 +25,13 @@
package org.broadinstitute.sting.utils;
import org.apache.commons.io.FileUtils;
import org.broadinstitute.sting.utils.io.IOUtils;
import org.testng.Assert;
import org.broadinstitute.sting.BaseTest;
import org.testng.annotations.Test;
import java.io.File;
import java.util.LinkedHashMap;
import java.util.Map;
@ -135,4 +138,16 @@ public class UtilsUnitTest extends BaseTest {
actual = Utils.escapeExpressions(" one two 'three four' ");
Assert.assertEquals(actual, expected);
}
@Test
public void testCalcMD5() throws Exception {
final File source = new File(publicTestDir + "exampleFASTA.fasta");
final String sourceMD5 = "36880691cf9e4178216f7b52e8d85fbe";
final byte[] sourceBytes = IOUtils.readFileIntoByteArray(source);
Assert.assertEquals(Utils.calcMD5(sourceBytes), sourceMD5);
final String sourceString = FileUtils.readFileToString(source);
Assert.assertEquals(Utils.calcMD5(sourceString), sourceMD5);
}
}