zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove Phone Home

This commit is contained in:
Geraldine Van der Auwera 2016-05-27 15:02:43 -04:00
parent 15f3f902f7
commit 0769c8ae3e
7 changed files with 0 additions and 1403 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/CommandLineExecutable.java
View File

@ -34,15 +34,10 @@ import org.broadinstitute.gatk.engine.filters.ReadFilter;
import org.broadinstitute.gatk.engine.io.stubs.OutputStreamArgumentTypeDescriptor;
import org.broadinstitute.gatk.engine.io.stubs.SAMFileWriterArgumentTypeDescriptor;
import org.broadinstitute.gatk.engine.io.stubs.VCFWriterArgumentTypeDescriptor;
import org.broadinstitute.gatk.engine.phonehome.GATKRunReport;
import org.broadinstitute.gatk.utils.refdata.utils.RMDTriplet;
import org.broadinstitute.gatk.engine.walkers.Walker;
import org.broadinstitute.gatk.engine.crypt.CryptUtils;
import org.broadinstitute.gatk.engine.crypt.GATKKey;
import org.broadinstitute.gatk.utils.exceptions.UserException;
import org.broadinstitute.gatk.utils.text.ListFileUtils;
import java.security.PublicKey;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@ -85,9 +80,6 @@ public abstract class CommandLineExecutable extends CommandLineProgram {
Walker<?,?> walker = engine.getWalkerByName(getAnalysisName());
try {
// Make sure a valid GATK user key is present, if required.
authorizeGATKRun();
engine.setArguments(getArgumentCollection());
// File lists can require a bit of additional expansion. Set these explicitly by the engine.
@ -119,9 +111,7 @@ public abstract class CommandLineExecutable extends CommandLineProgram {
}
engine.execute();
generateGATKRunReport(walker);
} catch ( Exception e ) {
generateGATKRunReport(walker, e);
throw e;
}
@ -129,53 +119,6 @@ public abstract class CommandLineExecutable extends CommandLineProgram {
return 0;
}
/**
* Authorizes this run of the GATK by checking for a valid GATK user key, if required.
* Currently, a key is required only if running with the -et NO_ET or -et STDOUT options.
*/
private void authorizeGATKRun() {
if ( getArgumentCollection().phoneHomeType == GATKRunReport.PhoneHomeOption.NO_ET ||
getArgumentCollection().phoneHomeType == GATKRunReport.PhoneHomeOption.STDOUT ) {
if ( getArgumentCollection().gatkKeyFile == null ) {
throw new UserException("Running with the -et NO_ET or -et STDOUT option requires a GATK Key file. " +
"Please see " + UserException.PHONE_HOME_DOCS_URL +
" for more information and instructions on how to obtain a key.");
}
else {
PublicKey gatkPublicKey = CryptUtils.loadGATKDistributedPublicKey();
GATKKey gatkUserKey = new GATKKey(gatkPublicKey, getArgumentCollection().gatkKeyFile);
if ( ! gatkUserKey.isValid() ) {
throw new UserException.KeySignatureVerificationException(getArgumentCollection().gatkKeyFile);
}
}
}
}
/**
* Generate the GATK run report for this walker using the current GATKEngine, if -et is enabled.
* This report will be written to either STDOUT or to the run repository, depending on the options
* for -et.
*
* @param e the exception, can be null if no exception occurred
*/
private void generateGATKRunReport(Walker<?,?> walker, Exception e) {
if ( getArgumentCollection().phoneHomeType != GATKRunReport.PhoneHomeOption.NO_ET ) {
GATKRunReport report = new GATKRunReport(walker, e, engine, getArgumentCollection().phoneHomeType );
report.postReport(getArgumentCollection().phoneHomeType);
}
}
/**
* Convenience method for fully parameterized generateGATKRunReport when an exception has
* not occurred
*
* @param walker
*/
private void generateGATKRunReport(Walker<?,?> walker) {
generateGATKRunReport(walker, null);
}
/**
* Subclasses of CommandLinePrograms can provide their own types of command-line arguments.
* @return A collection of type descriptors generating implementation-dependent placeholders.

4
public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/GenomeAnalysisEngine.java
View File

@ -49,7 +49,6 @@ import org.broadinstitute.gatk.engine.io.OutputTracker;
import org.broadinstitute.gatk.engine.io.stubs.Stub;
import org.broadinstitute.gatk.engine.iterators.ReadTransformer;
import org.broadinstitute.gatk.engine.iterators.ReadTransformersMode;
import org.broadinstitute.gatk.engine.phonehome.GATKRunReport;
import org.broadinstitute.gatk.utils.io.ReferenceBacked;
import org.broadinstitute.gatk.utils.refdata.tracks.IndexDictionaryUtils;
import org.broadinstitute.gatk.utils.refdata.tracks.RMDTrackBuilder;
@ -248,9 +247,6 @@ public class GenomeAnalysisEngine {
* @return the value of this traversal.
*/
public Object execute() {
// first thing is to make sure the AWS keys can be decrypted
GATKRunReport.checkAWSAreValid();
//HeapSizeMonitor monitor = new HeapSizeMonitor();
//monitor.start();
setStartTime(new java.util.Date());

37
public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/arguments/GATKArgumentCollection.java
View File

@ -31,7 +31,6 @@ import org.broadinstitute.gatk.utils.commandline.*;
import org.broadinstitute.gatk.engine.GenomeAnalysisEngine;
import org.broadinstitute.gatk.utils.downsampling.DownsampleType;
import org.broadinstitute.gatk.utils.downsampling.DownsamplingMethod;
import org.broadinstitute.gatk.engine.phonehome.GATKRunReport;
import org.broadinstitute.gatk.engine.samples.PedigreeValidationType;
import org.broadinstitute.gatk.utils.QualityUtils;
import org.broadinstitute.gatk.utils.baq.BAQ;
@ -74,42 +73,6 @@ public class GATKArgumentCollection {
@Argument(fullName = "read_buffer_size", shortName = "rbs", doc="Number of reads per SAM file to buffer in memory", required = false, minValue = 0)
public Integer readBufferSize = null;
// --------------------------------------------------------------------------------------------------------------
//
// GATKRunReport options
//
// --------------------------------------------------------------------------------------------------------------
/**
* By default, GATK generates a run report that is uploaded to a cloud-based service. This report contains basic
* statistics about the run (which tool was used, whether the run was successful etc.) that help us for debugging
* and development. Up to version 3.3-0 the run report contains a record of the username and hostname associated
* with the run, but it does **NOT** contain any information that could be used to identify patient data.
* Nevertheless, if your data is subject to stringent confidentiality clauses (no outside communication) or if your
* run environment is not connected to the internet, you can disable the reporting system by seeting this option to
* "NO_ET". You will also need to request a key using the online request form on our website (see FAQs).
*/
@Argument(fullName = "phone_home", shortName = "et", doc="Run reporting mode", required = false)
public GATKRunReport.PhoneHomeOption phoneHomeType = GATKRunReport.PhoneHomeOption.AWS;
/**
* Please see the "phone_home" argument below and the online documentation FAQs for more details on the key system
* and how to request a key.
*/
@Argument(fullName = "gatk_key", shortName = "K", doc="GATK key file required to run with -et NO_ET", required = false)
public File gatkKeyFile = null;
/**
* The GATKRunReport supports tagging GATK runs with an arbitrary tag that can be
* used to group together runs during later analysis (as of GATK 2.2) . One use of this capability is to tag
* runs as GATK performance tests, so that the performance of the GATK over time can be assessed from the logs
* directly.
*
* Note that the tags do not conform to any ontology, so you are free to use any tags that you might find
* meaningful.
*/
@Argument(fullName = "tag", shortName = "tag", doc="Tag to identify this GATK run as part of a group of runs", required = false)
public String tag = "NA";
// --------------------------------------------------------------------------------------------------------------
//
// General features

786
public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/phonehome/GATKRunReport.java
View File

@ -1,786 +0,0 @@
/*
* Copyright 2012-2016 Broad Institute, Inc.
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
* THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.gatk.engine.phonehome;
import com.google.java.contract.Ensures;
import com.google.java.contract.Requires;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.broadinstitute.gatk.engine.CommandLineGATK;
import org.broadinstitute.gatk.engine.GenomeAnalysisEngine;
import org.broadinstitute.gatk.engine.walkers.Walker;
import org.broadinstitute.gatk.utils.Utils;
import org.broadinstitute.gatk.engine.crypt.CryptUtils;
import org.broadinstitute.gatk.utils.exceptions.ReviewedGATKException;
import org.broadinstitute.gatk.utils.io.IOUtils;
import org.broadinstitute.gatk.utils.io.Resource;
import org.broadinstitute.gatk.utils.threading.ThreadEfficiencyMonitor;
import org.jets3t.service.S3Service;
import org.jets3t.service.S3ServiceException;
import org.jets3t.service.impl.rest.httpclient.RestS3Service;
import org.jets3t.service.model.S3Object;
import org.jets3t.service.security.AWSCredentials;
import org.simpleframework.xml.Element;
import org.simpleframework.xml.Serializer;
import org.simpleframework.xml.core.Persister;
import java.io.*;
import java.security.NoSuchAlgorithmException;
import java.security.PublicKey;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
/**
* A detailed description of a GATK run, and error if applicable. Simply create a GATKRunReport
* with the constructor, providing the walker that was run and the fully instantiated GenomeAnalysisEngine
* <b>after the run finishes</b> and the GATKRunReport will collect all of the report information
* into this object. Call postReport to write out the report, as an XML document, to either STDOUT,
* a file (in which case the output is gzipped), or with no arguments the report will be posted to the
* GATK run report database.
*
* @author depristo
* @since 2010
*/
public class GATKRunReport {
protected static final String REPORT_BUCKET_NAME = "broad.gsa.gatk.run.reports";
protected static final String TEST_REPORT_BUCKET_NAME = "broad.gsa.gatk.run.reports.test";
protected final static String AWS_ACCESS_KEY_MD5 = "34d4a26eb2062b3f06e833b28f9a38c6";
protected final static String AWS_SECRET_KEY_MD5 = "83f2332eec99ef1d7425d5dc5d4b514a";
private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy/MM/dd HH.mm.ss");
/**
* our log
*/
protected static final Logger logger = Logger.getLogger(GATKRunReport.class);
/**
* Default value for the number of milliseconds before an S3 put operation is timed-out.
* Can be overridden via a constructor argument.
*/
private static final long S3_DEFAULT_PUT_TIME_OUT_IN_MILLISECONDS = 30 * 1000;
/**
* Number of milliseconds before an S3 put operation is timed-out.
*/
private long s3PutTimeOutInMilliseconds = S3_DEFAULT_PUT_TIME_OUT_IN_MILLISECONDS;
// -----------------------------------------------------------------
// elements captured for the report
// -----------------------------------------------------------------
@Element(required = false, name = "id")
private String id;
@Element(required = false, name = "exception")
private GATKRunReportException mException;
@Element(required = true, name = "start-time")
private String startTime = "ND";
@Element(required = true, name = "end-time")
private String endTime;
@Element(required = true, name = "run-time")
private long runTime = 0;
@Element(required = true, name = "walker-name")
private String walkerName;
@Element(required = true, name = "svn-version")
private String svnVersion;
@Element(required = true, name = "total-memory")
private long totalMemory;
@Element(required = true, name = "max-memory")
private long maxMemory;
@Element(required = true, name = "user-name")
private String userName;
@Element(required = true, name = "host-name")
private String hostName;
@Element(required = true, name = "java")
private String javaVersion;
@Element(required = true, name = "machine")
private String machine;
@Element(required = true, name = "iterations")
private long nIterations;
@Element(required = true, name = "tag")
private String tag;
@Element(required = true, name = "num-threads")
private int numThreads;
@Element(required = true, name = "percent-time-running")
private String percentTimeRunning;
@Element(required = true, name = "percent-time-waiting")
private String percentTimeWaiting;
@Element(required = true, name = "percent-time-blocking")
private String percentTimeBlocking;
@Element(required = true, name = "percent-time-waiting-for-io")
private String percentTimeWaitingForIO;
/** The error message, if one occurred, or null if none did */
public String errorMessage = null;
/** The error that occurred, if one did, or null if none did */
public Throwable errorThrown = null;
/**
* How should the GATK report its usage?
*/
public enum PhoneHomeOption {
/** Disable phone home */
NO_ET,
/** Forces the report to go to S3 */
AWS,
/** Force output to STDOUT. For debugging only */
STDOUT
}
/**
* To allow us to deserial reports from XML
*/
private GATKRunReport() { }
/**
* Read a GATKRunReport from the serialized XML representation in String reportAsXML
* @param stream an input stream containing a serialized XML report
* @return a reconstituted GATKRunReport from reportAsXML
* @throws Exception if parsing fails for any reason
*/
@Ensures("result != null")
protected static GATKRunReport deserializeReport(final InputStream stream) throws Exception {
final Serializer serializer = new Persister();
return serializer.read(GATKRunReport.class, stream);
}
/**
* Create a new GATKRunReport from a report on S3
*
* Assumes that s3Object has already been written to S3, and this function merely
* fetches it from S3 and deserializes it. The access keys must have permission to
* GetObject from S3.
*
* @param downloaderAccessKey AWS access key with permission to GetObject from bucketName
* @param downloaderSecretKey AWS secret key with permission to GetObject from bucketName
* @param bucketName the name of the bucket holding the report
* @param s3Object the s3Object we wrote to S3 in bucketName that we want to get back and decode
* @return a deserialized report derived from s3://bucketName/s3Object.getName()
* @throws Exception
*/
@Ensures("result != null")
protected static GATKRunReport deserializeReport(final String downloaderAccessKey,
final String downloaderSecretKey,
final String bucketName,
final S3Object s3Object) throws Exception {
final S3Service s3Service = initializeAWSService(downloaderAccessKey, downloaderSecretKey);
// Retrieve the whole data object we created previously
final S3Object objectComplete = s3Service.getObject(bucketName, s3Object.getName());
// Read the data from the object's DataInputStream using a loop, and print it out.
return deserializeReport(new GZIPInputStream(objectComplete.getDataInputStream()));
}
/**
* Create a new RunReport and population all of the fields with values from the walker and engine.
* Allows the S3 put timeout to be explicitly set.
*
* @param walker the GATK walker that we ran
* @param e the exception caused by running this walker, or null if we completed successfully
* @param engine the GAE we used to run the walker, so we can fetch runtime, args, etc
* @param type the GATK phone home setting
* @param s3PutTimeOutInMilliseconds number of milliseconds to wait before timing out an S3 put operation
*/
public GATKRunReport(final Walker<?,?> walker, final Exception e, final GenomeAnalysisEngine engine, final PhoneHomeOption type,
final long s3PutTimeOutInMilliseconds) {
this(walker, e, engine, type);
this.s3PutTimeOutInMilliseconds = s3PutTimeOutInMilliseconds;
}
/**
* Create a new RunReport and population all of the fields with values from the walker and engine.
* Leaves the S3 put timeout set to the default value of S3_DEFAULT_PUT_TIME_OUT_IN_MILLISECONDS.
*
* @param walker the GATK walker that we ran
* @param e the exception caused by running this walker, or null if we completed successfully
* @param engine the GAE we used to run the walker, so we can fetch runtime, args, etc
* @param type the GATK phone home setting
*/
public GATKRunReport(final Walker<?,?> walker, final Exception e, final GenomeAnalysisEngine engine, final PhoneHomeOption type) {
if ( type == PhoneHomeOption.NO_ET )
throw new ReviewedGATKException("Trying to create a run report when type is NO_ET!");
logger.debug("Aggregating data for run report");
// what did we run?
id = org.apache.commons.lang.RandomStringUtils.randomAlphanumeric(32);
walkerName = engine.getWalkerName(walker.getClass());
svnVersion = CommandLineGATK.getVersionNumber();
// runtime performance metrics
Date end = new java.util.Date();
endTime = DATE_FORMAT.format(end);
if ( engine.getStartTime() != null ) { // made it this far during initialization
startTime = DATE_FORMAT.format(engine.getStartTime());
runTime = (end.getTime() - engine.getStartTime().getTime()) / 1000L; // difference in seconds
}
// deal with memory usage
Runtime.getRuntime().gc(); // call GC so totalMemory is ~ used memory
maxMemory = Runtime.getRuntime().maxMemory();
totalMemory = Runtime.getRuntime().totalMemory();
// we can only do some operations if an error hasn't occurred
if ( engine.getCumulativeMetrics() != null ) {
// it's possible we aborted so early that these data structures arent initialized
nIterations = engine.getCumulativeMetrics().getNumIterations();
}
tag = engine.getArguments().tag;
// user and hostname -- information about the runner of the GATK
userName = System.getProperty("user.name");
hostName = Utils.resolveHostname();
// basic java information
javaVersion = Utils.join("-", Arrays.asList(System.getProperty("java.vendor"), System.getProperty("java.version")));
machine = Utils.join("-", Arrays.asList(System.getProperty("os.name"), System.getProperty("os.arch")));
// if there was an exception, capture it
this.mException = e == null ? null : new GATKRunReportException(e);
numThreads = engine.getTotalNumberOfThreads();
percentTimeRunning = getThreadEfficiencyPercent(engine, ThreadEfficiencyMonitor.State.USER_CPU);
percentTimeBlocking = getThreadEfficiencyPercent(engine, ThreadEfficiencyMonitor.State.BLOCKING);
percentTimeWaiting = getThreadEfficiencyPercent(engine, ThreadEfficiencyMonitor.State.WAITING);
percentTimeWaitingForIO = getThreadEfficiencyPercent(engine, ThreadEfficiencyMonitor.State.WAITING_FOR_IO);
}
/**
* Get the random alpha-numeric ID of this GATKRunReport
* @return a non-null string ID
*/
@Ensures("result != null")
public String getID() {
return id;
}
/**
* Return a string representing the percent of time the GATK spent in state, if possible. Otherwise return NA
*
* @param engine the GATK engine whose threading efficiency info we will use
* @param state the state whose occupancy we wish to know
* @return a string representation of the percent occupancy of state, or NA is not possible
*/
@Requires({"engine != null", "state != null"})
@Ensures("result != null")
private String getThreadEfficiencyPercent(final GenomeAnalysisEngine engine, final ThreadEfficiencyMonitor.State state) {
final ThreadEfficiencyMonitor tem = engine.getThreadEfficiencyMonitor();
return tem == null ? "NA" : String.format("%.2f", tem.getStatePercent(state));
}
/**
* Get a filename (no path) appropriate for this report
*
* @return a non-null string filename
*/
@Ensures("result != null")
protected String getReportFileName() {
return getID() + ".report.xml.gz";
}
// ---------------------------------------------------------------------------
//
// Main public interface method for posting reports
//
// ---------------------------------------------------------------------------
/**
* Post this GATK report to the destination implied by the PhoneHomeOption type
*
* Guaranteed to never throw an exception (exception noted below) and to return
* with a reasonable (~10 seconds) time regardless of successful writing of the report.
*
* @throws IllegalArgumentException if type == null
* @param type the type of phoning home we want to do
* @return true if a report was successfully written, false otherwise
*/
public boolean postReport(final PhoneHomeOption type) {
if ( type == null ) throw new IllegalArgumentException("type cannot be null");
logger.debug("Posting report of type " + type);
switch (type) {
case NO_ET: // don't do anything
return false;
case AWS:
wentToAWS = true;
return postReportToAWSS3() != null;
case STDOUT:
return postReportToStream(System.out);
default:
exceptDuringRunReport("BUG: unexpected PhoneHomeOption ");
return false;
}
}
// ---------------------------------------------------------------------------
//
// Code for sending reports to local files
//
// ---------------------------------------------------------------------------
/**
* Write an XML representation of this report to the stream, throwing a GATKException if the marshalling
* fails for any reason.
*
* @param stream an output stream to write the report to
*/
@Requires("stream != null")
protected boolean postReportToStream(final OutputStream stream) {
final Serializer serializer = new Persister();
try {
serializer.write(this, stream);
return true;
} catch (Exception e) {
return false;
}
}
// ---------------------------------------------------------------------------
//
// Code for sending reports to s3
//
// ---------------------------------------------------------------------------
/**
* Get the name of the S3 bucket where we should upload this report
*
* @return the string name of the s3 bucket
*/
@Ensures("result != null")
protected String getS3ReportBucket() {
return s3ReportBucket;
}
/**
* Decrypts encrypted AWS key from encryptedKeySource
* @param encryptedKeySource a file containing an encrypted AWS key
* @return a decrypted AWS key as a String
*/
@Ensures("result != null")
public static String decryptAWSKey(final File encryptedKeySource) throws FileNotFoundException {
if ( encryptedKeySource == null ) throw new IllegalArgumentException("encryptedKeySource cannot be null");
return decryptAWSKey(new FileInputStream(encryptedKeySource));
}
/**
* @see #decryptAWSKey(java.io.File) but with input from an inputstream
*/
@Requires("encryptedKeySource != null")
@Ensures("result != null")
private static String decryptAWSKey(final InputStream encryptedKeySource) {
final PublicKey key = CryptUtils.loadGATKDistributedPublicKey();
final byte[] fromDisk = IOUtils.readStreamIntoByteArray(encryptedKeySource);
final byte[] decrypted = CryptUtils.decryptData(fromDisk, key);
return new String(decrypted);
}
/**
* Get the decrypted AWS key sorted in the resource directories of name
* @param name the name of the file containing the needed AWS key
* @return a non-null GATK
*/
@Requires("name != null")
@Ensures("result != null")
private static String getAWSKey(final String name) {
final Resource resource = new Resource(name, GATKRunReport.class);
return decryptAWSKey(resource.getResourceContentsAsStream());
}
/**
* Get the AWS access key for the GATK user
* @return a non-null AWS access key for the GATK user
*/
@Ensures("result != null")
protected static String getAWSUploadAccessKey() {
return getAWSKey("resources/GATK_AWS_access.key");
}
/**
* Get the AWS secret key for the GATK user
* @return a non-null AWS secret key for the GATK user
*/
@Ensures("result != null")
protected static String getAWSUploadSecretKey() {
return getAWSKey("resources/GATK_AWS_secret.key");
}
/**
* Check that the AWS keys can be decrypted and are what we expect them to be
*
* @throws ReviewedGATKException if anything goes wrong
*/
public static void checkAWSAreValid() {
try {
final String accessKeyMD5 = Utils.calcMD5(getAWSUploadAccessKey());
final String secretKeyMD5 = Utils.calcMD5(getAWSUploadSecretKey());
if ( ! AWS_ACCESS_KEY_MD5.equals(accessKeyMD5) ) {
throw new ReviewedGATKException("Invalid AWS access key found, expected MD5 " + AWS_ACCESS_KEY_MD5 + " but got " + accessKeyMD5);
}
if ( ! AWS_SECRET_KEY_MD5.equals(secretKeyMD5) ) {
throw new ReviewedGATKException("Invalid AWS secret key found, expected MD5 " + AWS_SECRET_KEY_MD5 + " but got " + secretKeyMD5);
}
} catch ( Exception e ) {
throw new ReviewedGATKException("Couldn't decrypt AWS keys, something is wrong with the GATK distribution");
}
}
/**
* Get an initialized S3Service for use in communicating with AWS/s3
*
* @param awsAccessKey our AWS access key to use
* @param awsSecretKey our AWS secret key to use
* @return an initialized S3Service object that can be immediately used to interact with S3
* @throws S3ServiceException
*/
@Requires({"awsAccessKey != null", "awsSecretKey != null"})
@Ensures("result != null")
protected static S3Service initializeAWSService(final String awsAccessKey, final String awsSecretKey) throws S3ServiceException {
// To communicate with S3, create a class that implements an S3Service. We will use the REST/HTTP
// implementation based on HttpClient, as this is the most robust implementation provided with JetS3t.
final AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey);
return new RestS3Service(awsCredentials);
}
/**
* A runnable that pushes this GATKReport up to s3.
*
* Should be run in a separate thread so we can time it out if something is taking too long
*/
private class S3PutRunnable implements Runnable {
/** Was the upload operation successful? */
public final AtomicBoolean isSuccess;
/** The name of this report */
private final String filename;
/** The contents of this report */
private final byte[] contents;
/** The s3Object that we created to upload, or null if it failed */
public S3Object s3Object = null;
@Requires({"filename != null", "contents != null"})
public S3PutRunnable(final String filename, final byte[] contents){
this.isSuccess = new AtomicBoolean();
this.filename = filename;
this.contents = contents;
}
public void run() {
try {
switch ( awsMode ) {
case FAIL_WITH_EXCEPTION:
throw new IllegalStateException("We are throwing an exception for testing purposes");
case TIMEOUT:
try {
Thread.sleep(s3PutTimeOutInMilliseconds * 100);
} catch ( InterruptedException e ) {
// supposed to be empty
}
break;
case NORMAL:
// IAM GATK user credentials -- only right is to PutObject into broad.gsa.gatk.run.reports bucket
final S3Service s3Service = initializeAWSService(getAWSUploadAccessKey(), getAWSUploadSecretKey());
// Create an S3Object based on a file, with Content-Length set automatically and
// Content-Type set based on the file's extension (using the Mimetypes utility class)
final S3Object fileObject = new S3Object(filename, contents);
//logger.info("Created S3Object" + fileObject);
//logger.info("Uploading " + localFile + " to AWS bucket");
s3Object = s3Service.putObject(getS3ReportBucket(), fileObject);
isSuccess.set(true);
break;
default:
throw new IllegalStateException("Unexpected AWS exception");
}
} catch ( S3ServiceException e ) {
exceptDuringRunReport("S3 exception occurred", e);
} catch ( NoSuchAlgorithmException e ) {
exceptDuringRunReport("Couldn't calculate MD5", e);
} catch ( IOException e ) {
exceptDuringRunReport("Couldn't read report file", e);
} catch ( Exception e ) {
exceptDuringRunReport("An unexpected exception occurred during posting", e);
}
}
}
/**
* Post this GATK report to the AWS s3 GATK_Run_Report log
*
* @return the s3Object pointing to our pushed report, or null if we failed to push
*/
protected S3Object postReportToAWSS3() {
// modifying example code from http://jets3t.s3.amazonaws.com/toolkit/code-samples.html
this.hostName = Utils.resolveHostname(); // we want to fill in the host name
final String key = getReportFileName();
logger.debug("Generating GATK report to AWS S3 with key " + key);
try {
// create an byte output stream so we can capture the output as a byte[]
final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(8096);
final OutputStream outputStream = new GZIPOutputStream(byteStream);
postReportToStream(outputStream);
outputStream.close();
final byte[] report = byteStream.toByteArray();
// stop us from printing the annoying, and meaningless, mime types warning
final Logger mimeTypeLogger = Logger.getLogger(org.jets3t.service.utils.Mimetypes.class);
mimeTypeLogger.setLevel(Level.FATAL);
// Set the S3 upload on its own thread with timeout:
final S3PutRunnable s3run = new S3PutRunnable(key,report);
final Thread s3thread = new Thread(s3run);
s3thread.setDaemon(true);
s3thread.setName("S3Put-Thread");
s3thread.start();
s3thread.join(s3PutTimeOutInMilliseconds);
if(s3thread.isAlive()){
s3thread.interrupt();
exceptDuringRunReport("Run statistics report upload to AWS S3 timed-out");
} else if(s3run.isSuccess.get()) {
logger.info("Uploaded run statistics report to AWS S3");
logger.debug("Uploaded to AWS: " + s3run.s3Object);
return s3run.s3Object;
} else {
// an exception occurred, the thread should have already invoked the exceptDuringRunReport function
}
} catch ( IOException e ) {
exceptDuringRunReport("Couldn't read report file", e);
} catch ( InterruptedException e) {
exceptDuringRunReport("Run statistics report upload interrupted", e);
}
return null;
}
// ---------------------------------------------------------------------------
//
// Error handling code
//
// ---------------------------------------------------------------------------
/**
* Note that an exception occurred during creating or writing this report
* @param msg the message to print
* @param e the exception that occurred
*/
@Ensures("exceptionOccurredDuringPost()")
private void exceptDuringRunReport(final String msg, final Throwable e) {
this.errorMessage = msg;
this.errorThrown = e;
logger.debug("A problem occurred during GATK run reporting [*** everything is fine, but no report could be generated; please do not post this to the support forum ***]. Message is: " + msg + ". Error message is: " + e.getMessage());
}
/**
* Note that an exception occurred during creating or writing this report
* @param msg the message to print
*/
@Ensures("exceptionOccurredDuringPost()")
private void exceptDuringRunReport(final String msg) {
this.errorMessage = msg;
logger.debug("A problem occurred during GATK run reporting [*** everything is fine, but no report could be generated; please do not post this to the support forum ***]. Message is " + msg);
}
/**
* Did an error occur during the posting of this run report?
* @return true if so, false if not
*/
public boolean exceptionOccurredDuringPost() {
return getErrorMessage() != null;
}
/**
* If an error occurred during posting of this report, retrieve the message of the error that occurred, or null if
* no error occurred
* @return a string describing the error that occurred, or null if none did
*/
public String getErrorMessage() {
return errorMessage;
}
/**
* Get the throwable that caused the exception during posting of this message, or null if none was available
*
* Note that getting a null valuable from this function doesn't not imply that no error occurred. Some
* errors that occurred many not have generated a throwable.
*
* @return the Throwable that caused the error, or null if no error occurred or was not caused by a throwable
*/
public Throwable getErrorThrown() {
return errorThrown;
}
/**
* Helper method to format the exception that occurred during posting, or a string saying none occurred
* @return a non-null string
*/
@Ensures("result != null")
protected String formatError() {
return exceptionOccurredDuringPost()
? String.format("Exception message=%s with cause=%s", getErrorMessage(), getErrorThrown())
: "No exception occurred";
}
// ---------------------------------------------------------------------------
//
// Equals and hashcode -- purely for comparing reports for testing
//
// ---------------------------------------------------------------------------
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
GATKRunReport that = (GATKRunReport) o;
if (maxMemory != that.maxMemory) return false;
if (nIterations != that.nIterations) return false;
if (numThreads != that.numThreads) return false;
if (runTime != that.runTime) return false;
if (totalMemory != that.totalMemory) return false;
if (endTime != null ? !endTime.equals(that.endTime) : that.endTime != null) return false;
if (hostName != null ? !hostName.equals(that.hostName) : that.hostName != null) return false;
if (id != null ? !id.equals(that.id) : that.id != null) return false;
if (javaVersion != null ? !javaVersion.equals(that.javaVersion) : that.javaVersion != null) return false;
if (mException != null ? !mException.equals(that.mException) : that.mException != null) return false;
if (machine != null ? !machine.equals(that.machine) : that.machine != null) return false;
if (percentTimeBlocking != null ? !percentTimeBlocking.equals(that.percentTimeBlocking) : that.percentTimeBlocking != null)
return false;
if (percentTimeRunning != null ? !percentTimeRunning.equals(that.percentTimeRunning) : that.percentTimeRunning != null)
return false;
if (percentTimeWaiting != null ? !percentTimeWaiting.equals(that.percentTimeWaiting) : that.percentTimeWaiting != null)
return false;
if (percentTimeWaitingForIO != null ? !percentTimeWaitingForIO.equals(that.percentTimeWaitingForIO) : that.percentTimeWaitingForIO != null)
return false;
if (startTime != null ? !startTime.equals(that.startTime) : that.startTime != null) return false;
if (svnVersion != null ? !svnVersion.equals(that.svnVersion) : that.svnVersion != null) return false;
if (tag != null ? !tag.equals(that.tag) : that.tag != null) return false;
if (userName != null ? !userName.equals(that.userName) : that.userName != null) return false;
if (walkerName != null ? !walkerName.equals(that.walkerName) : that.walkerName != null) return false;
return true;
}
@Override
public int hashCode() {
int result = id != null ? id.hashCode() : 0;
result = 31 * result + (mException != null ? mException.hashCode() : 0);
result = 31 * result + (startTime != null ? startTime.hashCode() : 0);
result = 31 * result + (endTime != null ? endTime.hashCode() : 0);
result = 31 * result + (int) (runTime ^ (runTime >>> 32));
result = 31 * result + (walkerName != null ? walkerName.hashCode() : 0);
result = 31 * result + (svnVersion != null ? svnVersion.hashCode() : 0);
result = 31 * result + (int) (totalMemory ^ (totalMemory >>> 32));
result = 31 * result + (int) (maxMemory ^ (maxMemory >>> 32));
result = 31 * result + (userName != null ? userName.hashCode() : 0);
result = 31 * result + (hostName != null ? hostName.hashCode() : 0);
result = 31 * result + (javaVersion != null ? javaVersion.hashCode() : 0);
result = 31 * result + (machine != null ? machine.hashCode() : 0);
result = 31 * result + (int) (nIterations ^ (nIterations >>> 32));
result = 31 * result + (tag != null ? tag.hashCode() : 0);
result = 31 * result + numThreads;
result = 31 * result + (percentTimeRunning != null ? percentTimeRunning.hashCode() : 0);
result = 31 * result + (percentTimeWaiting != null ? percentTimeWaiting.hashCode() : 0);
result = 31 * result + (percentTimeBlocking != null ? percentTimeBlocking.hashCode() : 0);
result = 31 * result + (percentTimeWaitingForIO != null ? percentTimeWaitingForIO.hashCode() : 0);
return result;
}
// ---------------------------------------------------------------------------
//
// Code specifically for testing the GATKRunReport
//
// ---------------------------------------------------------------------------
/**
* Enum specifying how the S3 uploader should behave. Must be normal by default. Purely for testing purposes
*/
protected enum AWSMode {
NORMAL, // write normally to AWS
FAIL_WITH_EXCEPTION, // artificially fail during writing
TIMEOUT // sleep, so we time out
}
/** Our AWS mode */
private AWSMode awsMode = AWSMode.NORMAL;
/** The bucket were we send the GATK report on AWS/s3 */
private String s3ReportBucket = REPORT_BUCKET_NAME;
/** Did we send the report to AWS? */
private boolean wentToAWS = false;
/**
* Send the report to the AWS test bucket -- for testing only
*/
protected void sendAWSToTestBucket() {
s3ReportBucket = TEST_REPORT_BUCKET_NAME;
}
/**
* Has the report been written to AWS?
*
* Does not imply anything about the success of the send, just that it was attempted
*
* @return true if the report has been sent to AWS, false otherwise
*/
protected boolean wentToAWS() {
return wentToAWS;
}
/**
* Purely for testing purposes. Tells the AWS uploader whether to actually upload or simulate errors
* @param mode what we want to do
*/
@Requires("mode != null")
protected void setAwsMode(final AWSMode mode) {
this.awsMode = mode;
}
}

157
public/gatk-engine/src/test/java/org/broadinstitute/gatk/engine/crypt/GATKKeyIntegrationTest.java
View File

@ -1,157 +0,0 @@
/*
* Copyright 2012-2016 Broad Institute, Inc.
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
* THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.gatk.engine.crypt;
import org.broadinstitute.gatk.engine.walkers.WalkerTest;
import org.broadinstitute.gatk.engine.phonehome.GATKRunReport;
import org.broadinstitute.gatk.utils.exceptions.UserException;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import java.util.Arrays;
public class GATKKeyIntegrationTest extends WalkerTest {
public static final String BASE_COMMAND = String.format("-T TestPrintReadsWalker -R %s -I %s -o %%s",
publicTestDir + "exampleFASTA.fasta",
publicTestDir + "exampleBAM.bam");
public static final String MD5_UPON_SUCCESSFUL_RUN = "462656ec9632f8c21ee534d35093c3f8";
private void runGATKKeyTest ( String testName, String etArg, String keyArg, Class expectedException, String md5 ) {
String command = BASE_COMMAND + String.format(" %s %s", etArg, keyArg);
WalkerTestSpec spec = expectedException != null ?
new WalkerTestSpec(command, 1, expectedException) :
new WalkerTestSpec(command, 1, Arrays.asList(md5));
spec.disableImplicitArgs(); // Turn off automatic inclusion of -et/-K args by WalkerTest
executeTest(testName, spec);
}
@Test
public void testValidKeyNoET() {
runGATKKeyTest("testValidKeyNoET",
"-et " + GATKRunReport.PhoneHomeOption.NO_ET,
"-K " + keysDataLocation + "valid.key",
null,
MD5_UPON_SUCCESSFUL_RUN);
}
@Test
public void testValidKeyETStdout() {
runGATKKeyTest("testValidKeyETStdout",
"-et " + GATKRunReport.PhoneHomeOption.STDOUT,
"-K " + keysDataLocation + "valid.key",
null,
MD5_UPON_SUCCESSFUL_RUN);
}
@Test
public void testValidKeyETStandard() {
runGATKKeyTest("testValidKeyETStandard",
"",
"-K " + keysDataLocation + "valid.key",
null,
MD5_UPON_SUCCESSFUL_RUN);
}
@Test
public void testNoKeyNoET() {
runGATKKeyTest("testNoKeyNoET",
"-et " + GATKRunReport.PhoneHomeOption.NO_ET,
"",
UserException.class,
null);
}
@Test
public void testNoKeyETStdout() {
runGATKKeyTest("testNoKeyETStdout",
"-et " + GATKRunReport.PhoneHomeOption.STDOUT,
"",
UserException.class,
null);
}
@Test
public void testNoKeyETStandard() {
runGATKKeyTest("testNoKeyETStandard",
"",
"",
null,
MD5_UPON_SUCCESSFUL_RUN);
}
@Test
public void testRevokedKey() {
runGATKKeyTest("testRevokedKey",
"-et " + GATKRunReport.PhoneHomeOption.NO_ET,
"-K " + keysDataLocation + "revoked.key",
UserException.KeySignatureVerificationException.class,
null);
}
@DataProvider(name = "CorruptKeyTestData")
public Object[][] corruptKeyDataProvider() {
return new Object[][] {
{ "corrupt_empty.key", UserException.UnreadableKeyException.class },
{ "corrupt_single_byte_file.key", UserException.UnreadableKeyException.class },
{ "corrupt_random_contents.key", UserException.UnreadableKeyException.class },
{ "corrupt_single_byte_deletion.key", UserException.UnreadableKeyException.class },
{ "corrupt_single_byte_insertion.key", UserException.UnreadableKeyException.class },
{ "corrupt_single_byte_change.key", UserException.UnreadableKeyException.class },
{ "corrupt_multi_byte_deletion.key", UserException.UnreadableKeyException.class },
{ "corrupt_multi_byte_insertion.key", UserException.UnreadableKeyException.class },
{ "corrupt_multi_byte_change.key", UserException.UnreadableKeyException.class },
{ "corrupt_bad_isize_field.key", UserException.UnreadableKeyException.class },
{ "corrupt_bad_crc.key", UserException.UnreadableKeyException.class },
{ "corrupt_no_email_address.key", UserException.UnreadableKeyException.class },
{ "corrupt_no_sectional_delimiter.key", UserException.UnreadableKeyException.class },
{ "corrupt_no_signature.key", UserException.UnreadableKeyException.class },
{ "corrupt_bad_signature.key", UserException.KeySignatureVerificationException.class },
{ "corrupt_non_gzipped_valid_key.key", UserException.UnreadableKeyException.class }
};
}
@Test(dataProvider = "CorruptKeyTestData")
public void testCorruptKey ( String corruptKeyName, Class expectedException ) {
runGATKKeyTest(String.format("testCorruptKey (%s)", corruptKeyName),
"-et " + GATKRunReport.PhoneHomeOption.NO_ET,
"-K " + keysDataLocation + corruptKeyName,
expectedException,
null);
}
@Test
public void testCorruptButNonRequiredKey() {
runGATKKeyTest("testCorruptButNonRequiredKey",
"",
"-K " + keysDataLocation + "corrupt_random_contents.key",
null,
MD5_UPON_SUCCESSFUL_RUN);
}
}

358
public/gatk-engine/src/test/java/org/broadinstitute/gatk/engine/phonehome/GATKRunReportUnitTest.java
View File

@ -1,358 +0,0 @@
/*
* Copyright 2012-2016 Broad Institute, Inc.
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
* THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.gatk.engine.phonehome;
import org.broadinstitute.gatk.engine.walkers.*;
import org.broadinstitute.gatk.utils.BaseTest;
import org.broadinstitute.gatk.engine.GenomeAnalysisEngine;
import org.broadinstitute.gatk.engine.arguments.GATKArgumentCollection;
import org.broadinstitute.gatk.utils.contexts.AlignmentContext;
import org.broadinstitute.gatk.utils.contexts.ReferenceContext;
import org.broadinstitute.gatk.utils.refdata.RefMetaDataTracker;
import org.broadinstitute.gatk.utils.Utils;
import org.broadinstitute.gatk.utils.activeregion.ActiveRegion;
import org.broadinstitute.gatk.utils.activeregion.ActivityProfileState;
import org.broadinstitute.gatk.utils.exceptions.ReviewedGATKException;
import org.broadinstitute.gatk.utils.exceptions.UserException;
import org.broadinstitute.gatk.utils.sam.GATKSAMRecord;
import org.jets3t.service.S3Service;
import org.jets3t.service.S3ServiceException;
import org.jets3t.service.ServiceException;
import org.jets3t.service.model.S3Object;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
public class GATKRunReportUnitTest extends BaseTest {
private final static boolean DEBUG = false;
private static final long S3_PUT_TIMEOUT_IN_MILLISECONDS_FOR_TESTING = 30 * 1000;
private static final String AWS_DOWNLOADER_CREDENTIALS_PROPERTIES_FILE = privateTestDir + "phonehome/awsDownloaderCredentials.properties";
private Walker walker;
private Exception exception;
private GenomeAnalysisEngine engine;
private String downloaderAccessKey;
private String downloaderSecretKey;
@BeforeClass
public void setup() throws Exception {
walker = new RunReportDummyReadWalker();
exception = new IllegalArgumentException("javaException");
engine = new GenomeAnalysisEngine();
engine.setArguments(new GATKArgumentCollection());
Properties awsProperties = new Properties();
awsProperties.load(new FileInputStream(AWS_DOWNLOADER_CREDENTIALS_PROPERTIES_FILE));
downloaderAccessKey = awsProperties.getProperty("accessKey");
downloaderSecretKey = awsProperties.getProperty("secretKey");
}
@Test(enabled = ! DEBUG)
public void testAWSKeysAreValid() {
// throws an exception if they aren't
GATKRunReport.checkAWSAreValid();
}
@Test(enabled = ! DEBUG)
public void testAccessKey() throws Exception {
testAWSKey(GATKRunReport.getAWSUploadAccessKey(), GATKRunReport.AWS_ACCESS_KEY_MD5);
}
@Test(enabled = ! DEBUG)
public void testSecretKey() throws Exception {
testAWSKey(GATKRunReport.getAWSUploadSecretKey(), GATKRunReport.AWS_SECRET_KEY_MD5);
}
private void testAWSKey(final String accessKey, final String expectedMD5) throws Exception {
Assert.assertNotNull(accessKey, "AccessKey should not be null");
final String actualmd5 = Utils.calcMD5(accessKey);
Assert.assertEquals(actualmd5, expectedMD5);
}
@DataProvider(name = "GATKReportCreationTest")
public Object[][] makeGATKReportCreationTest() {
List<Object[]> tests = new ArrayList<Object[]>();
final Walker readWalker = new RunReportDummyReadWalker();
final Walker lociWalker = new RunReportDummyLocusWalker();
final Walker rodWalker = new RunReportDummyRodWalker();
final Walker artWalker = new RunReportDummyActiveRegionWalker();
final Exception noException = null;
final Exception javaException = new IllegalArgumentException("javaException");
final Exception stingException = new ReviewedGATKException("GATKException");
final Exception userException = new UserException("userException");
final GenomeAnalysisEngine engine = new GenomeAnalysisEngine();
engine.setArguments(new GATKArgumentCollection());
for ( final Walker walker : Arrays.asList(readWalker, lociWalker, rodWalker, artWalker) ) {
for ( final Exception exception : Arrays.asList(noException, javaException, stingException, userException) ) {
tests.add(new Object[]{walker, exception, engine});
}
}
return tests.toArray(new Object[][]{});
}
@Test(enabled = !DEBUG, dataProvider = "GATKReportCreationTest")
public void testGATKReportCreationReadingAndWriting(final Walker walker, final Exception exception, final GenomeAnalysisEngine engine) throws Exception {
final GATKRunReport report = new GATKRunReport(walker, exception, engine, GATKRunReport.PhoneHomeOption.STDOUT);
final ByteArrayOutputStream captureStream = new ByteArrayOutputStream();
final boolean succeeded = report.postReportToStream(captureStream);
Assert.assertTrue(succeeded, "Failed to write report to stream");
Assert.assertFalse(report.exceptionOccurredDuringPost(), "Post succeeded but report says it failed");
Assert.assertNull(report.getErrorMessage(), "Post succeeded but there was an error message");
Assert.assertNull(report.getErrorThrown(), "Post succeeded but there was an error message");
final InputStream readStream = new ByteArrayInputStream(captureStream.toByteArray());
GATKRunReport deserialized = null;
try {
deserialized = GATKRunReport.deserializeReport(readStream);
} catch ( Exception e ) {
final String reportString = new String(captureStream.toByteArray());
Assert.fail("Failed to deserialize GATK report " + reportString + " with exception " + e);
}
if ( deserialized != null )
Assert.assertEquals(report, deserialized);
}
@DataProvider(name = "GATKAWSReportMode")
public Object[][] makeGATKAWSReportMode() {
List<Object[]> tests = new ArrayList<Object[]>();
for ( final GATKRunReport.AWSMode mode : GATKRunReport.AWSMode.values() ) {
tests.add(new Object[]{mode});
}
return tests.toArray(new Object[][]{});
}
// Will fail with timeout if AWS time out isn't working
// Will fail with exception if AWS doesn't protect itself from errors
@Test(enabled = ! DEBUG, dataProvider = "GATKAWSReportMode", timeOut = S3_PUT_TIMEOUT_IN_MILLISECONDS_FOR_TESTING * 2)
public void testAWS(final GATKRunReport.AWSMode awsMode) {
logger.warn("Starting testAWS mode=" + awsMode);
// Use a shorter timeout than usual when we're testing GATKRunReport.AWSMode.TIMEOUT
final long thisTestS3Timeout = awsMode == GATKRunReport.AWSMode.TIMEOUT ? 30 * 1000 : S3_PUT_TIMEOUT_IN_MILLISECONDS_FOR_TESTING;
final GATKRunReport report = new GATKRunReport(walker, exception, engine, GATKRunReport.PhoneHomeOption.AWS, thisTestS3Timeout);
report.sendAWSToTestBucket();
report.setAwsMode(awsMode);
final S3Object s3Object = report.postReportToAWSS3();
if ( awsMode == GATKRunReport.AWSMode.NORMAL ) {
Assert.assertNotNull(s3Object, "Upload to AWS failed, s3Object was null. error was " + report.formatError());
Assert.assertFalse(report.exceptionOccurredDuringPost(), "The upload should have succeeded but the report says it didn't. Error was " + report.formatError());
Assert.assertNull(report.getErrorMessage(), "Report succeeded but an error message was found");
Assert.assertNull(report.getErrorThrown(), "Report succeeded but an thrown error was found");
try {
final GATKRunReport deserialized = GATKRunReport.deserializeReport(downloaderAccessKey, downloaderSecretKey, report.getS3ReportBucket(), s3Object);
Assert.assertEquals(report, deserialized);
deleteFromS3(report);
} catch ( Exception e ) {
Assert.fail("Failed to read, deserialize, or delete GATK report " + s3Object.getName() + " with exception " + e);
}
} else {
Assert.assertNull(s3Object, "AWS upload should have failed for mode " + awsMode + " but got non-null s3 object back " + s3Object + " error was " + report.formatError());
Assert.assertTrue(report.exceptionOccurredDuringPost(), "S3 object was null but the report says that the upload succeeded");
Assert.assertNotNull(report.getErrorMessage(), "Report succeeded but an error message wasn't found");
if ( awsMode == GATKRunReport.AWSMode.FAIL_WITH_EXCEPTION )
Assert.assertNotNull(report.getErrorThrown());
}
}
private void deleteFromS3(final GATKRunReport report) throws Exception {
final S3Service s3Service = GATKRunReport.initializeAWSService(downloaderAccessKey, downloaderSecretKey);
// Retrieve the whole data object we created previously
s3Service.deleteObject(report.getS3ReportBucket(), report.getReportFileName());
}
@DataProvider(name = "PostReportByType")
public Object[][] makePostReportByType() {
List<Object[]> tests = new ArrayList<Object[]>();
for ( final GATKRunReport.PhoneHomeOption et : GATKRunReport.PhoneHomeOption.values() ) {
tests.add(new Object[]{et});
}
return tests.toArray(new Object[][]{});
}
@Test(enabled = ! DEBUG, dataProvider = "PostReportByType", timeOut = S3_PUT_TIMEOUT_IN_MILLISECONDS_FOR_TESTING * 2)
public void testPostReportByType(final GATKRunReport.PhoneHomeOption type) {
final GATKRunReport report = new GATKRunReport(walker, exception, engine, GATKRunReport.PhoneHomeOption.AWS, S3_PUT_TIMEOUT_IN_MILLISECONDS_FOR_TESTING);
Assert.assertFalse(report.exceptionOccurredDuringPost(), "An exception occurred during posting the report");
final boolean succeeded = report.postReport(type);
if ( type == GATKRunReport.PhoneHomeOption.NO_ET )
Assert.assertFalse(succeeded, "NO_ET option shouldn't write a report");
else {
Assert.assertTrue(succeeded, "Any non NO_ET option should succeed in writing a report");
if ( type == GATKRunReport.PhoneHomeOption.STDOUT ) {
// nothing to do
} else {
// must have gone to AWS
try {
Assert.assertTrue(report.wentToAWS(), "The report should have gone to AWS but the report says it wasn't");
deleteFromS3(report);
} catch ( Exception e ) {
Assert.fail("Failed delete GATK report " + report.getReportFileName() + " with exception " + e);
}
}
}
}
public interface S3Op {
public void apply() throws ServiceException;
}
// Will fail with timeout if AWS time out isn't working
// Will fail with exception if AWS doesn't protect itself from errors
@Test(timeOut = S3_PUT_TIMEOUT_IN_MILLISECONDS_FOR_TESTING * 2)
public void testAWSPublicKeyHasAccessControls() throws Exception {
final GATKRunReport report = new GATKRunReport(walker, exception, engine, GATKRunReport.PhoneHomeOption.AWS, S3_PUT_TIMEOUT_IN_MILLISECONDS_FOR_TESTING);
report.sendAWSToTestBucket();
final S3Object s3Object = report.postReportToAWSS3();
Assert.assertNotNull(s3Object, "Upload to AWS failed, s3Object was null. error was " + report.formatError());
// create a service with the public key, and make sure it cannot list or delete
final S3Service s3Service = GATKRunReport.initializeAWSService(GATKRunReport.getAWSUploadAccessKey(), GATKRunReport.getAWSUploadSecretKey());
assertOperationNotAllowed("listAllBuckets", new S3Op() {
@Override
public void apply() throws S3ServiceException {
s3Service.listAllBuckets();
}
});
assertOperationNotAllowed("listBucket", new S3Op() {
@Override
public void apply() throws S3ServiceException { s3Service.listObjects(report.getS3ReportBucket()); }
});
assertOperationNotAllowed("createBucket", new S3Op() {
@Override
public void apply() throws S3ServiceException { s3Service.createBucket("ShouldNotCreate"); }
});
assertOperationNotAllowed("deleteObject", new S3Op() {
@Override
public void apply() throws ServiceException { s3Service.deleteObject(report.getS3ReportBucket(), report.getReportFileName()); }
});
}
private void assertOperationNotAllowed(final String name, final S3Op op) {
try {
op.apply();
// only gets here if the operation was successful
Assert.fail("Operation " + name + " ran successfully but we expected to it fail");
} catch ( ServiceException e ) {
Assert.assertEquals(e.getErrorCode(), "AccessDenied");
}
}
class RunReportDummyReadWalker extends ReadWalker<Integer, Integer> {
@Override
public Integer map(ReferenceContext ref, GATKSAMRecord read, RefMetaDataTracker metaDataTracker) {
return 0;
}
@Override
public Integer reduceInit() {
return 0;
}
@Override
public Integer reduce(Integer value, Integer sum) {
return 0;
}
}
class RunReportDummyLocusWalker extends LocusWalker<Integer, Integer> {
@Override
public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
return 0;
}
@Override
public Integer reduceInit() {
return 0;
}
@Override
public Integer reduce(Integer value, Integer sum) {
return 0;
}
}
class RunReportDummyRodWalker extends RodWalker<Integer, Integer> {
@Override
public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
return 0;
}
@Override
public Integer reduceInit() {
return 0;
}
@Override
public Integer reduce(Integer value, Integer sum) {
return 0;
}
}
class RunReportDummyActiveRegionWalker extends ActiveRegionWalker<Integer, Integer> {
@Override
public ActivityProfileState isActive(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
return new ActivityProfileState(ref.getLocus(), 0.0);
}
@Override
public Integer map(ActiveRegion activeRegion, RefMetaDataTracker metaDataTracker) {
return 0;
}
@Override
public Integer reduceInit() {
return 0;
}
@Override
public Integer reduce(Integer value, Integer sum) {
return 0;
}
}
}

4
public/gatk-engine/src/test/java/org/broadinstitute/gatk/engine/walkers/WalkerTest.java
View File

@ -34,7 +34,6 @@ import org.apache.commons.lang.StringUtils;
import org.broadinstitute.gatk.engine.CommandLineExecutable;
import org.broadinstitute.gatk.engine.CommandLineGATK;
import org.broadinstitute.gatk.engine.crypt.CryptUtils;
import org.broadinstitute.gatk.engine.phonehome.GATKRunReport;
import org.broadinstitute.gatk.utils.BaseTest;
import org.broadinstitute.gatk.utils.MD5DB;
import org.broadinstitute.gatk.utils.MD5Mismatch;
@ -225,9 +224,6 @@ public class WalkerTest extends BaseTest {
public String getArgsWithImplicitArgs() {
String args = this.args;
if ( includeImplicitArgs ) {
args = args + (ENABLE_PHONE_HOME_FOR_TESTS ?
String.format(" -et %s ", GATKRunReport.PhoneHomeOption.AWS) :
String.format(" -et %s -K %s ", GATKRunReport.PhoneHomeOption.NO_ET, gatkKeyFile));
if ( includeShadowBCF && GENERATE_SHADOW_BCF )
args = args + " --generateShadowBCF ";
if ( ! ENABLE_AUTO_INDEX_CREATION_AND_LOCKING_FOR_TESTS )