diff --git a/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/CommandLineExecutable.java b/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/CommandLineExecutable.java index 56cfac40b..73dc0749c 100644 --- a/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/CommandLineExecutable.java +++ b/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/CommandLineExecutable.java @@ -34,15 +34,10 @@ import org.broadinstitute.gatk.engine.filters.ReadFilter; import org.broadinstitute.gatk.engine.io.stubs.OutputStreamArgumentTypeDescriptor; import org.broadinstitute.gatk.engine.io.stubs.SAMFileWriterArgumentTypeDescriptor; import org.broadinstitute.gatk.engine.io.stubs.VCFWriterArgumentTypeDescriptor; -import org.broadinstitute.gatk.engine.phonehome.GATKRunReport; import org.broadinstitute.gatk.utils.refdata.utils.RMDTriplet; import org.broadinstitute.gatk.engine.walkers.Walker; -import org.broadinstitute.gatk.engine.crypt.CryptUtils; -import org.broadinstitute.gatk.engine.crypt.GATKKey; -import org.broadinstitute.gatk.utils.exceptions.UserException; import org.broadinstitute.gatk.utils.text.ListFileUtils; -import java.security.PublicKey; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -85,9 +80,6 @@ public abstract class CommandLineExecutable extends CommandLineProgram { Walker walker = engine.getWalkerByName(getAnalysisName()); try { - // Make sure a valid GATK user key is present, if required. - authorizeGATKRun(); - engine.setArguments(getArgumentCollection()); // File lists can require a bit of additional expansion. Set these explicitly by the engine. @@ -119,9 +111,7 @@ public abstract class CommandLineExecutable extends CommandLineProgram { } engine.execute(); - generateGATKRunReport(walker); } catch ( Exception e ) { - generateGATKRunReport(walker, e); throw e; } @@ -129,53 +119,6 @@ public abstract class CommandLineExecutable extends CommandLineProgram { return 0; } - /** - * Authorizes this run of the GATK by checking for a valid GATK user key, if required. - * Currently, a key is required only if running with the -et NO_ET or -et STDOUT options. - */ - private void authorizeGATKRun() { - if ( getArgumentCollection().phoneHomeType == GATKRunReport.PhoneHomeOption.NO_ET || - getArgumentCollection().phoneHomeType == GATKRunReport.PhoneHomeOption.STDOUT ) { - if ( getArgumentCollection().gatkKeyFile == null ) { - throw new UserException("Running with the -et NO_ET or -et STDOUT option requires a GATK Key file. " + - "Please see " + UserException.PHONE_HOME_DOCS_URL + - " for more information and instructions on how to obtain a key."); - } - else { - PublicKey gatkPublicKey = CryptUtils.loadGATKDistributedPublicKey(); - GATKKey gatkUserKey = new GATKKey(gatkPublicKey, getArgumentCollection().gatkKeyFile); - - if ( ! gatkUserKey.isValid() ) { - throw new UserException.KeySignatureVerificationException(getArgumentCollection().gatkKeyFile); - } - } - } - } - - /** - * Generate the GATK run report for this walker using the current GATKEngine, if -et is enabled. - * This report will be written to either STDOUT or to the run repository, depending on the options - * for -et. - * - * @param e the exception, can be null if no exception occurred - */ - private void generateGATKRunReport(Walker walker, Exception e) { - if ( getArgumentCollection().phoneHomeType != GATKRunReport.PhoneHomeOption.NO_ET ) { - GATKRunReport report = new GATKRunReport(walker, e, engine, getArgumentCollection().phoneHomeType ); - report.postReport(getArgumentCollection().phoneHomeType); - } - } - - /** - * Convenience method for fully parameterized generateGATKRunReport when an exception has - * not occurred - * - * @param walker - */ - private void generateGATKRunReport(Walker walker) { - generateGATKRunReport(walker, null); - } - /** * Subclasses of CommandLinePrograms can provide their own types of command-line arguments. * @return A collection of type descriptors generating implementation-dependent placeholders. diff --git a/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/GenomeAnalysisEngine.java b/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/GenomeAnalysisEngine.java index 9f5ccaf93..469e5491d 100644 --- a/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/GenomeAnalysisEngine.java +++ b/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/GenomeAnalysisEngine.java @@ -49,7 +49,6 @@ import org.broadinstitute.gatk.engine.io.OutputTracker; import org.broadinstitute.gatk.engine.io.stubs.Stub; import org.broadinstitute.gatk.engine.iterators.ReadTransformer; import org.broadinstitute.gatk.engine.iterators.ReadTransformersMode; -import org.broadinstitute.gatk.engine.phonehome.GATKRunReport; import org.broadinstitute.gatk.utils.io.ReferenceBacked; import org.broadinstitute.gatk.utils.refdata.tracks.IndexDictionaryUtils; import org.broadinstitute.gatk.utils.refdata.tracks.RMDTrackBuilder; @@ -248,9 +247,6 @@ public class GenomeAnalysisEngine { * @return the value of this traversal. */ public Object execute() { - // first thing is to make sure the AWS keys can be decrypted - GATKRunReport.checkAWSAreValid(); - //HeapSizeMonitor monitor = new HeapSizeMonitor(); //monitor.start(); setStartTime(new java.util.Date()); diff --git a/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/arguments/GATKArgumentCollection.java b/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/arguments/GATKArgumentCollection.java index 9b74d9c57..1b4548d38 100644 --- a/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/arguments/GATKArgumentCollection.java +++ b/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/arguments/GATKArgumentCollection.java @@ -31,7 +31,6 @@ import org.broadinstitute.gatk.utils.commandline.*; import org.broadinstitute.gatk.engine.GenomeAnalysisEngine; import org.broadinstitute.gatk.utils.downsampling.DownsampleType; import org.broadinstitute.gatk.utils.downsampling.DownsamplingMethod; -import org.broadinstitute.gatk.engine.phonehome.GATKRunReport; import org.broadinstitute.gatk.engine.samples.PedigreeValidationType; import org.broadinstitute.gatk.utils.QualityUtils; import org.broadinstitute.gatk.utils.baq.BAQ; @@ -74,42 +73,6 @@ public class GATKArgumentCollection { @Argument(fullName = "read_buffer_size", shortName = "rbs", doc="Number of reads per SAM file to buffer in memory", required = false, minValue = 0) public Integer readBufferSize = null; - // -------------------------------------------------------------------------------------------------------------- - // - // GATKRunReport options - // - // -------------------------------------------------------------------------------------------------------------- - - /** - * By default, GATK generates a run report that is uploaded to a cloud-based service. This report contains basic - * statistics about the run (which tool was used, whether the run was successful etc.) that help us for debugging - * and development. Up to version 3.3-0 the run report contains a record of the username and hostname associated - * with the run, but it does **NOT** contain any information that could be used to identify patient data. - * Nevertheless, if your data is subject to stringent confidentiality clauses (no outside communication) or if your - * run environment is not connected to the internet, you can disable the reporting system by seeting this option to - * "NO_ET". You will also need to request a key using the online request form on our website (see FAQs). - */ - @Argument(fullName = "phone_home", shortName = "et", doc="Run reporting mode", required = false) - public GATKRunReport.PhoneHomeOption phoneHomeType = GATKRunReport.PhoneHomeOption.AWS; - /** - * Please see the "phone_home" argument below and the online documentation FAQs for more details on the key system - * and how to request a key. - */ - @Argument(fullName = "gatk_key", shortName = "K", doc="GATK key file required to run with -et NO_ET", required = false) - public File gatkKeyFile = null; - - /** - * The GATKRunReport supports tagging GATK runs with an arbitrary tag that can be - * used to group together runs during later analysis (as of GATK 2.2) . One use of this capability is to tag - * runs as GATK performance tests, so that the performance of the GATK over time can be assessed from the logs - * directly. - * - * Note that the tags do not conform to any ontology, so you are free to use any tags that you might find - * meaningful. - */ - @Argument(fullName = "tag", shortName = "tag", doc="Tag to identify this GATK run as part of a group of runs", required = false) - public String tag = "NA"; - // -------------------------------------------------------------------------------------------------------------- // // General features diff --git a/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/phonehome/GATKRunReport.java b/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/phonehome/GATKRunReport.java deleted file mode 100644 index 0186823a5..000000000 --- a/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/phonehome/GATKRunReport.java +++ /dev/null @@ -1,786 +0,0 @@ -/* -* Copyright 2012-2016 Broad Institute, Inc. -* -* Permission is hereby granted, free of charge, to any person -* obtaining a copy of this software and associated documentation -* files (the "Software"), to deal in the Software without -* restriction, including without limitation the rights to use, -* copy, modify, merge, publish, distribute, sublicense, and/or sell -* copies of the Software, and to permit persons to whom the -* Software is furnished to do so, subject to the following -* conditions: -* -* The above copyright notice and this permission notice shall be -* included in all copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR -* THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -package org.broadinstitute.gatk.engine.phonehome; - -import com.google.java.contract.Ensures; -import com.google.java.contract.Requires; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; -import org.broadinstitute.gatk.engine.CommandLineGATK; -import org.broadinstitute.gatk.engine.GenomeAnalysisEngine; -import org.broadinstitute.gatk.engine.walkers.Walker; -import org.broadinstitute.gatk.utils.Utils; -import org.broadinstitute.gatk.engine.crypt.CryptUtils; -import org.broadinstitute.gatk.utils.exceptions.ReviewedGATKException; -import org.broadinstitute.gatk.utils.io.IOUtils; -import org.broadinstitute.gatk.utils.io.Resource; -import org.broadinstitute.gatk.utils.threading.ThreadEfficiencyMonitor; -import org.jets3t.service.S3Service; -import org.jets3t.service.S3ServiceException; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.model.S3Object; -import org.jets3t.service.security.AWSCredentials; -import org.simpleframework.xml.Element; -import org.simpleframework.xml.Serializer; -import org.simpleframework.xml.core.Persister; - -import java.io.*; -import java.security.NoSuchAlgorithmException; -import java.security.PublicKey; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Date; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; - - -/** - * A detailed description of a GATK run, and error if applicable. Simply create a GATKRunReport - * with the constructor, providing the walker that was run and the fully instantiated GenomeAnalysisEngine - * after the run finishes and the GATKRunReport will collect all of the report information - * into this object. Call postReport to write out the report, as an XML document, to either STDOUT, - * a file (in which case the output is gzipped), or with no arguments the report will be posted to the - * GATK run report database. - * - * @author depristo - * @since 2010 - */ -public class GATKRunReport { - protected static final String REPORT_BUCKET_NAME = "broad.gsa.gatk.run.reports"; - protected static final String TEST_REPORT_BUCKET_NAME = "broad.gsa.gatk.run.reports.test"; - protected final static String AWS_ACCESS_KEY_MD5 = "34d4a26eb2062b3f06e833b28f9a38c6"; - protected final static String AWS_SECRET_KEY_MD5 = "83f2332eec99ef1d7425d5dc5d4b514a"; - - private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy/MM/dd HH.mm.ss"); - - /** - * our log - */ - protected static final Logger logger = Logger.getLogger(GATKRunReport.class); - - /** - * Default value for the number of milliseconds before an S3 put operation is timed-out. - * Can be overridden via a constructor argument. - */ - private static final long S3_DEFAULT_PUT_TIME_OUT_IN_MILLISECONDS = 30 * 1000; - - /** - * Number of milliseconds before an S3 put operation is timed-out. - */ - private long s3PutTimeOutInMilliseconds = S3_DEFAULT_PUT_TIME_OUT_IN_MILLISECONDS; - - // ----------------------------------------------------------------- - // elements captured for the report - // ----------------------------------------------------------------- - - @Element(required = false, name = "id") - private String id; - - @Element(required = false, name = "exception") - private GATKRunReportException mException; - - @Element(required = true, name = "start-time") - private String startTime = "ND"; - - @Element(required = true, name = "end-time") - private String endTime; - - @Element(required = true, name = "run-time") - private long runTime = 0; - - @Element(required = true, name = "walker-name") - private String walkerName; - - @Element(required = true, name = "svn-version") - private String svnVersion; - - @Element(required = true, name = "total-memory") - private long totalMemory; - - @Element(required = true, name = "max-memory") - private long maxMemory; - - @Element(required = true, name = "user-name") - private String userName; - - @Element(required = true, name = "host-name") - private String hostName; - - @Element(required = true, name = "java") - private String javaVersion; - - @Element(required = true, name = "machine") - private String machine; - - @Element(required = true, name = "iterations") - private long nIterations; - - @Element(required = true, name = "tag") - private String tag; - - @Element(required = true, name = "num-threads") - private int numThreads; - @Element(required = true, name = "percent-time-running") - private String percentTimeRunning; - @Element(required = true, name = "percent-time-waiting") - private String percentTimeWaiting; - @Element(required = true, name = "percent-time-blocking") - private String percentTimeBlocking; - @Element(required = true, name = "percent-time-waiting-for-io") - private String percentTimeWaitingForIO; - - /** The error message, if one occurred, or null if none did */ - public String errorMessage = null; - /** The error that occurred, if one did, or null if none did */ - public Throwable errorThrown = null; - - /** - * How should the GATK report its usage? - */ - public enum PhoneHomeOption { - /** Disable phone home */ - NO_ET, - /** Forces the report to go to S3 */ - AWS, - /** Force output to STDOUT. For debugging only */ - STDOUT - } - - /** - * To allow us to deserial reports from XML - */ - private GATKRunReport() { } - - /** - * Read a GATKRunReport from the serialized XML representation in String reportAsXML - * @param stream an input stream containing a serialized XML report - * @return a reconstituted GATKRunReport from reportAsXML - * @throws Exception if parsing fails for any reason - */ - @Ensures("result != null") - protected static GATKRunReport deserializeReport(final InputStream stream) throws Exception { - final Serializer serializer = new Persister(); - return serializer.read(GATKRunReport.class, stream); - } - - /** - * Create a new GATKRunReport from a report on S3 - * - * Assumes that s3Object has already been written to S3, and this function merely - * fetches it from S3 and deserializes it. The access keys must have permission to - * GetObject from S3. - * - * @param downloaderAccessKey AWS access key with permission to GetObject from bucketName - * @param downloaderSecretKey AWS secret key with permission to GetObject from bucketName - * @param bucketName the name of the bucket holding the report - * @param s3Object the s3Object we wrote to S3 in bucketName that we want to get back and decode - * @return a deserialized report derived from s3://bucketName/s3Object.getName() - * @throws Exception - */ - @Ensures("result != null") - protected static GATKRunReport deserializeReport(final String downloaderAccessKey, - final String downloaderSecretKey, - final String bucketName, - final S3Object s3Object) throws Exception { - final S3Service s3Service = initializeAWSService(downloaderAccessKey, downloaderSecretKey); - - // Retrieve the whole data object we created previously - final S3Object objectComplete = s3Service.getObject(bucketName, s3Object.getName()); - - // Read the data from the object's DataInputStream using a loop, and print it out. - return deserializeReport(new GZIPInputStream(objectComplete.getDataInputStream())); - } - - /** - * Create a new RunReport and population all of the fields with values from the walker and engine. - * Allows the S3 put timeout to be explicitly set. - * - * @param walker the GATK walker that we ran - * @param e the exception caused by running this walker, or null if we completed successfully - * @param engine the GAE we used to run the walker, so we can fetch runtime, args, etc - * @param type the GATK phone home setting - * @param s3PutTimeOutInMilliseconds number of milliseconds to wait before timing out an S3 put operation - */ - public GATKRunReport(final Walker walker, final Exception e, final GenomeAnalysisEngine engine, final PhoneHomeOption type, - final long s3PutTimeOutInMilliseconds) { - this(walker, e, engine, type); - this.s3PutTimeOutInMilliseconds = s3PutTimeOutInMilliseconds; - } - - /** - * Create a new RunReport and population all of the fields with values from the walker and engine. - * Leaves the S3 put timeout set to the default value of S3_DEFAULT_PUT_TIME_OUT_IN_MILLISECONDS. - * - * @param walker the GATK walker that we ran - * @param e the exception caused by running this walker, or null if we completed successfully - * @param engine the GAE we used to run the walker, so we can fetch runtime, args, etc - * @param type the GATK phone home setting - */ - public GATKRunReport(final Walker walker, final Exception e, final GenomeAnalysisEngine engine, final PhoneHomeOption type) { - if ( type == PhoneHomeOption.NO_ET ) - throw new ReviewedGATKException("Trying to create a run report when type is NO_ET!"); - - logger.debug("Aggregating data for run report"); - - // what did we run? - id = org.apache.commons.lang.RandomStringUtils.randomAlphanumeric(32); - walkerName = engine.getWalkerName(walker.getClass()); - svnVersion = CommandLineGATK.getVersionNumber(); - - // runtime performance metrics - Date end = new java.util.Date(); - endTime = DATE_FORMAT.format(end); - if ( engine.getStartTime() != null ) { // made it this far during initialization - startTime = DATE_FORMAT.format(engine.getStartTime()); - runTime = (end.getTime() - engine.getStartTime().getTime()) / 1000L; // difference in seconds - } - - // deal with memory usage - Runtime.getRuntime().gc(); // call GC so totalMemory is ~ used memory - maxMemory = Runtime.getRuntime().maxMemory(); - totalMemory = Runtime.getRuntime().totalMemory(); - - // we can only do some operations if an error hasn't occurred - if ( engine.getCumulativeMetrics() != null ) { - // it's possible we aborted so early that these data structures arent initialized - nIterations = engine.getCumulativeMetrics().getNumIterations(); - } - - tag = engine.getArguments().tag; - - // user and hostname -- information about the runner of the GATK - userName = System.getProperty("user.name"); - hostName = Utils.resolveHostname(); - - // basic java information - javaVersion = Utils.join("-", Arrays.asList(System.getProperty("java.vendor"), System.getProperty("java.version"))); - machine = Utils.join("-", Arrays.asList(System.getProperty("os.name"), System.getProperty("os.arch"))); - - // if there was an exception, capture it - this.mException = e == null ? null : new GATKRunReportException(e); - - numThreads = engine.getTotalNumberOfThreads(); - percentTimeRunning = getThreadEfficiencyPercent(engine, ThreadEfficiencyMonitor.State.USER_CPU); - percentTimeBlocking = getThreadEfficiencyPercent(engine, ThreadEfficiencyMonitor.State.BLOCKING); - percentTimeWaiting = getThreadEfficiencyPercent(engine, ThreadEfficiencyMonitor.State.WAITING); - percentTimeWaitingForIO = getThreadEfficiencyPercent(engine, ThreadEfficiencyMonitor.State.WAITING_FOR_IO); - } - - /** - * Get the random alpha-numeric ID of this GATKRunReport - * @return a non-null string ID - */ - @Ensures("result != null") - public String getID() { - return id; - } - - /** - * Return a string representing the percent of time the GATK spent in state, if possible. Otherwise return NA - * - * @param engine the GATK engine whose threading efficiency info we will use - * @param state the state whose occupancy we wish to know - * @return a string representation of the percent occupancy of state, or NA is not possible - */ - @Requires({"engine != null", "state != null"}) - @Ensures("result != null") - private String getThreadEfficiencyPercent(final GenomeAnalysisEngine engine, final ThreadEfficiencyMonitor.State state) { - final ThreadEfficiencyMonitor tem = engine.getThreadEfficiencyMonitor(); - return tem == null ? "NA" : String.format("%.2f", tem.getStatePercent(state)); - } - - /** - * Get a filename (no path) appropriate for this report - * - * @return a non-null string filename - */ - @Ensures("result != null") - protected String getReportFileName() { - return getID() + ".report.xml.gz"; - } - - // --------------------------------------------------------------------------- - // - // Main public interface method for posting reports - // - // --------------------------------------------------------------------------- - - /** - * Post this GATK report to the destination implied by the PhoneHomeOption type - * - * Guaranteed to never throw an exception (exception noted below) and to return - * with a reasonable (~10 seconds) time regardless of successful writing of the report. - * - * @throws IllegalArgumentException if type == null - * @param type the type of phoning home we want to do - * @return true if a report was successfully written, false otherwise - */ - public boolean postReport(final PhoneHomeOption type) { - if ( type == null ) throw new IllegalArgumentException("type cannot be null"); - - logger.debug("Posting report of type " + type); - switch (type) { - case NO_ET: // don't do anything - return false; - case AWS: - wentToAWS = true; - return postReportToAWSS3() != null; - case STDOUT: - return postReportToStream(System.out); - default: - exceptDuringRunReport("BUG: unexpected PhoneHomeOption "); - return false; - } - } - - // --------------------------------------------------------------------------- - // - // Code for sending reports to local files - // - // --------------------------------------------------------------------------- - - /** - * Write an XML representation of this report to the stream, throwing a GATKException if the marshalling - * fails for any reason. - * - * @param stream an output stream to write the report to - */ - @Requires("stream != null") - protected boolean postReportToStream(final OutputStream stream) { - final Serializer serializer = new Persister(); - try { - serializer.write(this, stream); - return true; - } catch (Exception e) { - return false; - } - } - - // --------------------------------------------------------------------------- - // - // Code for sending reports to s3 - // - // --------------------------------------------------------------------------- - - /** - * Get the name of the S3 bucket where we should upload this report - * - * @return the string name of the s3 bucket - */ - @Ensures("result != null") - protected String getS3ReportBucket() { - return s3ReportBucket; - } - - /** - * Decrypts encrypted AWS key from encryptedKeySource - * @param encryptedKeySource a file containing an encrypted AWS key - * @return a decrypted AWS key as a String - */ - @Ensures("result != null") - public static String decryptAWSKey(final File encryptedKeySource) throws FileNotFoundException { - if ( encryptedKeySource == null ) throw new IllegalArgumentException("encryptedKeySource cannot be null"); - return decryptAWSKey(new FileInputStream(encryptedKeySource)); - } - - /** - * @see #decryptAWSKey(java.io.File) but with input from an inputstream - */ - @Requires("encryptedKeySource != null") - @Ensures("result != null") - private static String decryptAWSKey(final InputStream encryptedKeySource) { - final PublicKey key = CryptUtils.loadGATKDistributedPublicKey(); - final byte[] fromDisk = IOUtils.readStreamIntoByteArray(encryptedKeySource); - final byte[] decrypted = CryptUtils.decryptData(fromDisk, key); - return new String(decrypted); - } - - /** - * Get the decrypted AWS key sorted in the resource directories of name - * @param name the name of the file containing the needed AWS key - * @return a non-null GATK - */ - @Requires("name != null") - @Ensures("result != null") - private static String getAWSKey(final String name) { - final Resource resource = new Resource(name, GATKRunReport.class); - return decryptAWSKey(resource.getResourceContentsAsStream()); - } - - /** - * Get the AWS access key for the GATK user - * @return a non-null AWS access key for the GATK user - */ - @Ensures("result != null") - protected static String getAWSUploadAccessKey() { - return getAWSKey("resources/GATK_AWS_access.key"); - } - - /** - * Get the AWS secret key for the GATK user - * @return a non-null AWS secret key for the GATK user - */ - @Ensures("result != null") - protected static String getAWSUploadSecretKey() { - return getAWSKey("resources/GATK_AWS_secret.key"); - } - - /** - * Check that the AWS keys can be decrypted and are what we expect them to be - * - * @throws ReviewedGATKException if anything goes wrong - */ - public static void checkAWSAreValid() { - try { - final String accessKeyMD5 = Utils.calcMD5(getAWSUploadAccessKey()); - final String secretKeyMD5 = Utils.calcMD5(getAWSUploadSecretKey()); - - if ( ! AWS_ACCESS_KEY_MD5.equals(accessKeyMD5) ) { - throw new ReviewedGATKException("Invalid AWS access key found, expected MD5 " + AWS_ACCESS_KEY_MD5 + " but got " + accessKeyMD5); - } - if ( ! AWS_SECRET_KEY_MD5.equals(secretKeyMD5) ) { - throw new ReviewedGATKException("Invalid AWS secret key found, expected MD5 " + AWS_SECRET_KEY_MD5 + " but got " + secretKeyMD5); - } - - } catch ( Exception e ) { - throw new ReviewedGATKException("Couldn't decrypt AWS keys, something is wrong with the GATK distribution"); - } - } - - /** - * Get an initialized S3Service for use in communicating with AWS/s3 - * - * @param awsAccessKey our AWS access key to use - * @param awsSecretKey our AWS secret key to use - * @return an initialized S3Service object that can be immediately used to interact with S3 - * @throws S3ServiceException - */ - @Requires({"awsAccessKey != null", "awsSecretKey != null"}) - @Ensures("result != null") - protected static S3Service initializeAWSService(final String awsAccessKey, final String awsSecretKey) throws S3ServiceException { - // To communicate with S3, create a class that implements an S3Service. We will use the REST/HTTP - // implementation based on HttpClient, as this is the most robust implementation provided with JetS3t. - final AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey); - return new RestS3Service(awsCredentials); - } - - /** - * A runnable that pushes this GATKReport up to s3. - * - * Should be run in a separate thread so we can time it out if something is taking too long - */ - private class S3PutRunnable implements Runnable { - /** Was the upload operation successful? */ - public final AtomicBoolean isSuccess; - /** The name of this report */ - private final String filename; - /** The contents of this report */ - private final byte[] contents; - - /** The s3Object that we created to upload, or null if it failed */ - public S3Object s3Object = null; - - @Requires({"filename != null", "contents != null"}) - public S3PutRunnable(final String filename, final byte[] contents){ - this.isSuccess = new AtomicBoolean(); - this.filename = filename; - this.contents = contents; - } - - public void run() { - try { - switch ( awsMode ) { - case FAIL_WITH_EXCEPTION: - throw new IllegalStateException("We are throwing an exception for testing purposes"); - case TIMEOUT: - try { - Thread.sleep(s3PutTimeOutInMilliseconds * 100); - } catch ( InterruptedException e ) { - // supposed to be empty - } - break; - case NORMAL: - // IAM GATK user credentials -- only right is to PutObject into broad.gsa.gatk.run.reports bucket - final S3Service s3Service = initializeAWSService(getAWSUploadAccessKey(), getAWSUploadSecretKey()); - - // Create an S3Object based on a file, with Content-Length set automatically and - // Content-Type set based on the file's extension (using the Mimetypes utility class) - final S3Object fileObject = new S3Object(filename, contents); - //logger.info("Created S3Object" + fileObject); - //logger.info("Uploading " + localFile + " to AWS bucket"); - s3Object = s3Service.putObject(getS3ReportBucket(), fileObject); - isSuccess.set(true); - break; - default: - throw new IllegalStateException("Unexpected AWS exception"); - } - } catch ( S3ServiceException e ) { - exceptDuringRunReport("S3 exception occurred", e); - } catch ( NoSuchAlgorithmException e ) { - exceptDuringRunReport("Couldn't calculate MD5", e); - } catch ( IOException e ) { - exceptDuringRunReport("Couldn't read report file", e); - } catch ( Exception e ) { - exceptDuringRunReport("An unexpected exception occurred during posting", e); - } - } - } - - /** - * Post this GATK report to the AWS s3 GATK_Run_Report log - * - * @return the s3Object pointing to our pushed report, or null if we failed to push - */ - protected S3Object postReportToAWSS3() { - // modifying example code from http://jets3t.s3.amazonaws.com/toolkit/code-samples.html - this.hostName = Utils.resolveHostname(); // we want to fill in the host name - final String key = getReportFileName(); - logger.debug("Generating GATK report to AWS S3 with key " + key); - - try { - // create an byte output stream so we can capture the output as a byte[] - final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(8096); - final OutputStream outputStream = new GZIPOutputStream(byteStream); - postReportToStream(outputStream); - outputStream.close(); - final byte[] report = byteStream.toByteArray(); - - // stop us from printing the annoying, and meaningless, mime types warning - final Logger mimeTypeLogger = Logger.getLogger(org.jets3t.service.utils.Mimetypes.class); - mimeTypeLogger.setLevel(Level.FATAL); - - // Set the S3 upload on its own thread with timeout: - final S3PutRunnable s3run = new S3PutRunnable(key,report); - final Thread s3thread = new Thread(s3run); - s3thread.setDaemon(true); - s3thread.setName("S3Put-Thread"); - s3thread.start(); - - s3thread.join(s3PutTimeOutInMilliseconds); - - if(s3thread.isAlive()){ - s3thread.interrupt(); - exceptDuringRunReport("Run statistics report upload to AWS S3 timed-out"); - } else if(s3run.isSuccess.get()) { - logger.info("Uploaded run statistics report to AWS S3"); - logger.debug("Uploaded to AWS: " + s3run.s3Object); - return s3run.s3Object; - } else { - // an exception occurred, the thread should have already invoked the exceptDuringRunReport function - } - } catch ( IOException e ) { - exceptDuringRunReport("Couldn't read report file", e); - } catch ( InterruptedException e) { - exceptDuringRunReport("Run statistics report upload interrupted", e); - } - - return null; - } - - // --------------------------------------------------------------------------- - // - // Error handling code - // - // --------------------------------------------------------------------------- - - /** - * Note that an exception occurred during creating or writing this report - * @param msg the message to print - * @param e the exception that occurred - */ - @Ensures("exceptionOccurredDuringPost()") - private void exceptDuringRunReport(final String msg, final Throwable e) { - this.errorMessage = msg; - this.errorThrown = e; - logger.debug("A problem occurred during GATK run reporting [*** everything is fine, but no report could be generated; please do not post this to the support forum ***]. Message is: " + msg + ". Error message is: " + e.getMessage()); - } - - /** - * Note that an exception occurred during creating or writing this report - * @param msg the message to print - */ - @Ensures("exceptionOccurredDuringPost()") - private void exceptDuringRunReport(final String msg) { - this.errorMessage = msg; - logger.debug("A problem occurred during GATK run reporting [*** everything is fine, but no report could be generated; please do not post this to the support forum ***]. Message is " + msg); - } - - /** - * Did an error occur during the posting of this run report? - * @return true if so, false if not - */ - public boolean exceptionOccurredDuringPost() { - return getErrorMessage() != null; - } - - /** - * If an error occurred during posting of this report, retrieve the message of the error that occurred, or null if - * no error occurred - * @return a string describing the error that occurred, or null if none did - */ - public String getErrorMessage() { - return errorMessage; - } - - /** - * Get the throwable that caused the exception during posting of this message, or null if none was available - * - * Note that getting a null valuable from this function doesn't not imply that no error occurred. Some - * errors that occurred many not have generated a throwable. - * - * @return the Throwable that caused the error, or null if no error occurred or was not caused by a throwable - */ - public Throwable getErrorThrown() { - return errorThrown; - } - - /** - * Helper method to format the exception that occurred during posting, or a string saying none occurred - * @return a non-null string - */ - @Ensures("result != null") - protected String formatError() { - return exceptionOccurredDuringPost() - ? String.format("Exception message=%s with cause=%s", getErrorMessage(), getErrorThrown()) - : "No exception occurred"; - } - - // --------------------------------------------------------------------------- - // - // Equals and hashcode -- purely for comparing reports for testing - // - // --------------------------------------------------------------------------- - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - GATKRunReport that = (GATKRunReport) o; - - if (maxMemory != that.maxMemory) return false; - if (nIterations != that.nIterations) return false; - if (numThreads != that.numThreads) return false; - if (runTime != that.runTime) return false; - if (totalMemory != that.totalMemory) return false; - if (endTime != null ? !endTime.equals(that.endTime) : that.endTime != null) return false; - if (hostName != null ? !hostName.equals(that.hostName) : that.hostName != null) return false; - if (id != null ? !id.equals(that.id) : that.id != null) return false; - if (javaVersion != null ? !javaVersion.equals(that.javaVersion) : that.javaVersion != null) return false; - if (mException != null ? !mException.equals(that.mException) : that.mException != null) return false; - if (machine != null ? !machine.equals(that.machine) : that.machine != null) return false; - if (percentTimeBlocking != null ? !percentTimeBlocking.equals(that.percentTimeBlocking) : that.percentTimeBlocking != null) - return false; - if (percentTimeRunning != null ? !percentTimeRunning.equals(that.percentTimeRunning) : that.percentTimeRunning != null) - return false; - if (percentTimeWaiting != null ? !percentTimeWaiting.equals(that.percentTimeWaiting) : that.percentTimeWaiting != null) - return false; - if (percentTimeWaitingForIO != null ? !percentTimeWaitingForIO.equals(that.percentTimeWaitingForIO) : that.percentTimeWaitingForIO != null) - return false; - if (startTime != null ? !startTime.equals(that.startTime) : that.startTime != null) return false; - if (svnVersion != null ? !svnVersion.equals(that.svnVersion) : that.svnVersion != null) return false; - if (tag != null ? !tag.equals(that.tag) : that.tag != null) return false; - if (userName != null ? !userName.equals(that.userName) : that.userName != null) return false; - if (walkerName != null ? !walkerName.equals(that.walkerName) : that.walkerName != null) return false; - - return true; - } - - @Override - public int hashCode() { - int result = id != null ? id.hashCode() : 0; - result = 31 * result + (mException != null ? mException.hashCode() : 0); - result = 31 * result + (startTime != null ? startTime.hashCode() : 0); - result = 31 * result + (endTime != null ? endTime.hashCode() : 0); - result = 31 * result + (int) (runTime ^ (runTime >>> 32)); - result = 31 * result + (walkerName != null ? walkerName.hashCode() : 0); - result = 31 * result + (svnVersion != null ? svnVersion.hashCode() : 0); - result = 31 * result + (int) (totalMemory ^ (totalMemory >>> 32)); - result = 31 * result + (int) (maxMemory ^ (maxMemory >>> 32)); - result = 31 * result + (userName != null ? userName.hashCode() : 0); - result = 31 * result + (hostName != null ? hostName.hashCode() : 0); - result = 31 * result + (javaVersion != null ? javaVersion.hashCode() : 0); - result = 31 * result + (machine != null ? machine.hashCode() : 0); - result = 31 * result + (int) (nIterations ^ (nIterations >>> 32)); - result = 31 * result + (tag != null ? tag.hashCode() : 0); - result = 31 * result + numThreads; - result = 31 * result + (percentTimeRunning != null ? percentTimeRunning.hashCode() : 0); - result = 31 * result + (percentTimeWaiting != null ? percentTimeWaiting.hashCode() : 0); - result = 31 * result + (percentTimeBlocking != null ? percentTimeBlocking.hashCode() : 0); - result = 31 * result + (percentTimeWaitingForIO != null ? percentTimeWaitingForIO.hashCode() : 0); - return result; - } - - // --------------------------------------------------------------------------- - // - // Code specifically for testing the GATKRunReport - // - // --------------------------------------------------------------------------- - - /** - * Enum specifying how the S3 uploader should behave. Must be normal by default. Purely for testing purposes - */ - protected enum AWSMode { - NORMAL, // write normally to AWS - FAIL_WITH_EXCEPTION, // artificially fail during writing - TIMEOUT // sleep, so we time out - } - /** Our AWS mode */ - private AWSMode awsMode = AWSMode.NORMAL; - /** The bucket were we send the GATK report on AWS/s3 */ - private String s3ReportBucket = REPORT_BUCKET_NAME; - /** Did we send the report to AWS? */ - private boolean wentToAWS = false; - - /** - * Send the report to the AWS test bucket -- for testing only - */ - protected void sendAWSToTestBucket() { - s3ReportBucket = TEST_REPORT_BUCKET_NAME; - } - - /** - * Has the report been written to AWS? - * - * Does not imply anything about the success of the send, just that it was attempted - * - * @return true if the report has been sent to AWS, false otherwise - */ - protected boolean wentToAWS() { - return wentToAWS; - } - - /** - * Purely for testing purposes. Tells the AWS uploader whether to actually upload or simulate errors - * @param mode what we want to do - */ - @Requires("mode != null") - protected void setAwsMode(final AWSMode mode) { - this.awsMode = mode; - } -} diff --git a/public/gatk-engine/src/test/java/org/broadinstitute/gatk/engine/crypt/GATKKeyIntegrationTest.java b/public/gatk-engine/src/test/java/org/broadinstitute/gatk/engine/crypt/GATKKeyIntegrationTest.java deleted file mode 100644 index fcf3afa6a..000000000 --- a/public/gatk-engine/src/test/java/org/broadinstitute/gatk/engine/crypt/GATKKeyIntegrationTest.java +++ /dev/null @@ -1,157 +0,0 @@ -/* -* Copyright 2012-2016 Broad Institute, Inc. -* -* Permission is hereby granted, free of charge, to any person -* obtaining a copy of this software and associated documentation -* files (the "Software"), to deal in the Software without -* restriction, including without limitation the rights to use, -* copy, modify, merge, publish, distribute, sublicense, and/or sell -* copies of the Software, and to permit persons to whom the -* Software is furnished to do so, subject to the following -* conditions: -* -* The above copyright notice and this permission notice shall be -* included in all copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR -* THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -package org.broadinstitute.gatk.engine.crypt; - -import org.broadinstitute.gatk.engine.walkers.WalkerTest; -import org.broadinstitute.gatk.engine.phonehome.GATKRunReport; -import org.broadinstitute.gatk.utils.exceptions.UserException; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -import java.util.Arrays; - -public class GATKKeyIntegrationTest extends WalkerTest { - - public static final String BASE_COMMAND = String.format("-T TestPrintReadsWalker -R %s -I %s -o %%s", - publicTestDir + "exampleFASTA.fasta", - publicTestDir + "exampleBAM.bam"); - public static final String MD5_UPON_SUCCESSFUL_RUN = "462656ec9632f8c21ee534d35093c3f8"; - - - private void runGATKKeyTest ( String testName, String etArg, String keyArg, Class expectedException, String md5 ) { - String command = BASE_COMMAND + String.format(" %s %s", etArg, keyArg); - - WalkerTestSpec spec = expectedException != null ? - new WalkerTestSpec(command, 1, expectedException) : - new WalkerTestSpec(command, 1, Arrays.asList(md5)); - - spec.disableImplicitArgs(); // Turn off automatic inclusion of -et/-K args by WalkerTest - executeTest(testName, spec); - } - - @Test - public void testValidKeyNoET() { - runGATKKeyTest("testValidKeyNoET", - "-et " + GATKRunReport.PhoneHomeOption.NO_ET, - "-K " + keysDataLocation + "valid.key", - null, - MD5_UPON_SUCCESSFUL_RUN); - } - - @Test - public void testValidKeyETStdout() { - runGATKKeyTest("testValidKeyETStdout", - "-et " + GATKRunReport.PhoneHomeOption.STDOUT, - "-K " + keysDataLocation + "valid.key", - null, - MD5_UPON_SUCCESSFUL_RUN); - } - - @Test - public void testValidKeyETStandard() { - runGATKKeyTest("testValidKeyETStandard", - "", - "-K " + keysDataLocation + "valid.key", - null, - MD5_UPON_SUCCESSFUL_RUN); - } - - @Test - public void testNoKeyNoET() { - runGATKKeyTest("testNoKeyNoET", - "-et " + GATKRunReport.PhoneHomeOption.NO_ET, - "", - UserException.class, - null); - } - - @Test - public void testNoKeyETStdout() { - runGATKKeyTest("testNoKeyETStdout", - "-et " + GATKRunReport.PhoneHomeOption.STDOUT, - "", - UserException.class, - null); - } - - @Test - public void testNoKeyETStandard() { - runGATKKeyTest("testNoKeyETStandard", - "", - "", - null, - MD5_UPON_SUCCESSFUL_RUN); - } - - @Test - public void testRevokedKey() { - runGATKKeyTest("testRevokedKey", - "-et " + GATKRunReport.PhoneHomeOption.NO_ET, - "-K " + keysDataLocation + "revoked.key", - UserException.KeySignatureVerificationException.class, - null); - } - - @DataProvider(name = "CorruptKeyTestData") - public Object[][] corruptKeyDataProvider() { - return new Object[][] { - { "corrupt_empty.key", UserException.UnreadableKeyException.class }, - { "corrupt_single_byte_file.key", UserException.UnreadableKeyException.class }, - { "corrupt_random_contents.key", UserException.UnreadableKeyException.class }, - { "corrupt_single_byte_deletion.key", UserException.UnreadableKeyException.class }, - { "corrupt_single_byte_insertion.key", UserException.UnreadableKeyException.class }, - { "corrupt_single_byte_change.key", UserException.UnreadableKeyException.class }, - { "corrupt_multi_byte_deletion.key", UserException.UnreadableKeyException.class }, - { "corrupt_multi_byte_insertion.key", UserException.UnreadableKeyException.class }, - { "corrupt_multi_byte_change.key", UserException.UnreadableKeyException.class }, - { "corrupt_bad_isize_field.key", UserException.UnreadableKeyException.class }, - { "corrupt_bad_crc.key", UserException.UnreadableKeyException.class }, - { "corrupt_no_email_address.key", UserException.UnreadableKeyException.class }, - { "corrupt_no_sectional_delimiter.key", UserException.UnreadableKeyException.class }, - { "corrupt_no_signature.key", UserException.UnreadableKeyException.class }, - { "corrupt_bad_signature.key", UserException.KeySignatureVerificationException.class }, - { "corrupt_non_gzipped_valid_key.key", UserException.UnreadableKeyException.class } - }; - } - - @Test(dataProvider = "CorruptKeyTestData") - public void testCorruptKey ( String corruptKeyName, Class expectedException ) { - runGATKKeyTest(String.format("testCorruptKey (%s)", corruptKeyName), - "-et " + GATKRunReport.PhoneHomeOption.NO_ET, - "-K " + keysDataLocation + corruptKeyName, - expectedException, - null); - } - - @Test - public void testCorruptButNonRequiredKey() { - runGATKKeyTest("testCorruptButNonRequiredKey", - "", - "-K " + keysDataLocation + "corrupt_random_contents.key", - null, - MD5_UPON_SUCCESSFUL_RUN); - } -} diff --git a/public/gatk-engine/src/test/java/org/broadinstitute/gatk/engine/phonehome/GATKRunReportUnitTest.java b/public/gatk-engine/src/test/java/org/broadinstitute/gatk/engine/phonehome/GATKRunReportUnitTest.java deleted file mode 100644 index 7a151bbec..000000000 --- a/public/gatk-engine/src/test/java/org/broadinstitute/gatk/engine/phonehome/GATKRunReportUnitTest.java +++ /dev/null @@ -1,358 +0,0 @@ -/* -* Copyright 2012-2016 Broad Institute, Inc. -* -* Permission is hereby granted, free of charge, to any person -* obtaining a copy of this software and associated documentation -* files (the "Software"), to deal in the Software without -* restriction, including without limitation the rights to use, -* copy, modify, merge, publish, distribute, sublicense, and/or sell -* copies of the Software, and to permit persons to whom the -* Software is furnished to do so, subject to the following -* conditions: -* -* The above copyright notice and this permission notice shall be -* included in all copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR -* THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -package org.broadinstitute.gatk.engine.phonehome; - -import org.broadinstitute.gatk.engine.walkers.*; -import org.broadinstitute.gatk.utils.BaseTest; -import org.broadinstitute.gatk.engine.GenomeAnalysisEngine; -import org.broadinstitute.gatk.engine.arguments.GATKArgumentCollection; -import org.broadinstitute.gatk.utils.contexts.AlignmentContext; -import org.broadinstitute.gatk.utils.contexts.ReferenceContext; -import org.broadinstitute.gatk.utils.refdata.RefMetaDataTracker; -import org.broadinstitute.gatk.utils.Utils; -import org.broadinstitute.gatk.utils.activeregion.ActiveRegion; -import org.broadinstitute.gatk.utils.activeregion.ActivityProfileState; -import org.broadinstitute.gatk.utils.exceptions.ReviewedGATKException; -import org.broadinstitute.gatk.utils.exceptions.UserException; -import org.broadinstitute.gatk.utils.sam.GATKSAMRecord; -import org.jets3t.service.S3Service; -import org.jets3t.service.S3ServiceException; -import org.jets3t.service.ServiceException; -import org.jets3t.service.model.S3Object; -import org.testng.Assert; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.FileInputStream; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; - -public class GATKRunReportUnitTest extends BaseTest { - private final static boolean DEBUG = false; - private static final long S3_PUT_TIMEOUT_IN_MILLISECONDS_FOR_TESTING = 30 * 1000; - private static final String AWS_DOWNLOADER_CREDENTIALS_PROPERTIES_FILE = privateTestDir + "phonehome/awsDownloaderCredentials.properties"; - - private Walker walker; - private Exception exception; - private GenomeAnalysisEngine engine; - private String downloaderAccessKey; - private String downloaderSecretKey; - - @BeforeClass - public void setup() throws Exception { - walker = new RunReportDummyReadWalker(); - exception = new IllegalArgumentException("javaException"); - engine = new GenomeAnalysisEngine(); - engine.setArguments(new GATKArgumentCollection()); - - Properties awsProperties = new Properties(); - awsProperties.load(new FileInputStream(AWS_DOWNLOADER_CREDENTIALS_PROPERTIES_FILE)); - downloaderAccessKey = awsProperties.getProperty("accessKey"); - downloaderSecretKey = awsProperties.getProperty("secretKey"); - } - - @Test(enabled = ! DEBUG) - public void testAWSKeysAreValid() { - // throws an exception if they aren't - GATKRunReport.checkAWSAreValid(); - } - - @Test(enabled = ! DEBUG) - public void testAccessKey() throws Exception { - testAWSKey(GATKRunReport.getAWSUploadAccessKey(), GATKRunReport.AWS_ACCESS_KEY_MD5); - } - - @Test(enabled = ! DEBUG) - public void testSecretKey() throws Exception { - testAWSKey(GATKRunReport.getAWSUploadSecretKey(), GATKRunReport.AWS_SECRET_KEY_MD5); - } - - private void testAWSKey(final String accessKey, final String expectedMD5) throws Exception { - Assert.assertNotNull(accessKey, "AccessKey should not be null"); - final String actualmd5 = Utils.calcMD5(accessKey); - Assert.assertEquals(actualmd5, expectedMD5); - } - - @DataProvider(name = "GATKReportCreationTest") - public Object[][] makeGATKReportCreationTest() { - List tests = new ArrayList(); - - final Walker readWalker = new RunReportDummyReadWalker(); - final Walker lociWalker = new RunReportDummyLocusWalker(); - final Walker rodWalker = new RunReportDummyRodWalker(); - final Walker artWalker = new RunReportDummyActiveRegionWalker(); - - final Exception noException = null; - final Exception javaException = new IllegalArgumentException("javaException"); - final Exception stingException = new ReviewedGATKException("GATKException"); - final Exception userException = new UserException("userException"); - - final GenomeAnalysisEngine engine = new GenomeAnalysisEngine(); - engine.setArguments(new GATKArgumentCollection()); - - for ( final Walker walker : Arrays.asList(readWalker, lociWalker, rodWalker, artWalker) ) { - for ( final Exception exception : Arrays.asList(noException, javaException, stingException, userException) ) { - tests.add(new Object[]{walker, exception, engine}); - } - } - - return tests.toArray(new Object[][]{}); - } - - @Test(enabled = !DEBUG, dataProvider = "GATKReportCreationTest") - public void testGATKReportCreationReadingAndWriting(final Walker walker, final Exception exception, final GenomeAnalysisEngine engine) throws Exception { - final GATKRunReport report = new GATKRunReport(walker, exception, engine, GATKRunReport.PhoneHomeOption.STDOUT); - final ByteArrayOutputStream captureStream = new ByteArrayOutputStream(); - final boolean succeeded = report.postReportToStream(captureStream); - Assert.assertTrue(succeeded, "Failed to write report to stream"); - Assert.assertFalse(report.exceptionOccurredDuringPost(), "Post succeeded but report says it failed"); - Assert.assertNull(report.getErrorMessage(), "Post succeeded but there was an error message"); - Assert.assertNull(report.getErrorThrown(), "Post succeeded but there was an error message"); - final InputStream readStream = new ByteArrayInputStream(captureStream.toByteArray()); - - GATKRunReport deserialized = null; - try { - deserialized = GATKRunReport.deserializeReport(readStream); - } catch ( Exception e ) { - final String reportString = new String(captureStream.toByteArray()); - Assert.fail("Failed to deserialize GATK report " + reportString + " with exception " + e); - } - - if ( deserialized != null ) - Assert.assertEquals(report, deserialized); - } - - @DataProvider(name = "GATKAWSReportMode") - public Object[][] makeGATKAWSReportMode() { - List tests = new ArrayList(); - - for ( final GATKRunReport.AWSMode mode : GATKRunReport.AWSMode.values() ) { - tests.add(new Object[]{mode}); - } - - return tests.toArray(new Object[][]{}); - } - - // Will fail with timeout if AWS time out isn't working - // Will fail with exception if AWS doesn't protect itself from errors - @Test(enabled = ! DEBUG, dataProvider = "GATKAWSReportMode", timeOut = S3_PUT_TIMEOUT_IN_MILLISECONDS_FOR_TESTING * 2) - public void testAWS(final GATKRunReport.AWSMode awsMode) { - logger.warn("Starting testAWS mode=" + awsMode); - - // Use a shorter timeout than usual when we're testing GATKRunReport.AWSMode.TIMEOUT - final long thisTestS3Timeout = awsMode == GATKRunReport.AWSMode.TIMEOUT ? 30 * 1000 : S3_PUT_TIMEOUT_IN_MILLISECONDS_FOR_TESTING; - final GATKRunReport report = new GATKRunReport(walker, exception, engine, GATKRunReport.PhoneHomeOption.AWS, thisTestS3Timeout); - report.sendAWSToTestBucket(); - report.setAwsMode(awsMode); - final S3Object s3Object = report.postReportToAWSS3(); - - if ( awsMode == GATKRunReport.AWSMode.NORMAL ) { - Assert.assertNotNull(s3Object, "Upload to AWS failed, s3Object was null. error was " + report.formatError()); - Assert.assertFalse(report.exceptionOccurredDuringPost(), "The upload should have succeeded but the report says it didn't. Error was " + report.formatError()); - Assert.assertNull(report.getErrorMessage(), "Report succeeded but an error message was found"); - Assert.assertNull(report.getErrorThrown(), "Report succeeded but an thrown error was found"); - try { - final GATKRunReport deserialized = GATKRunReport.deserializeReport(downloaderAccessKey, downloaderSecretKey, report.getS3ReportBucket(), s3Object); - Assert.assertEquals(report, deserialized); - deleteFromS3(report); - } catch ( Exception e ) { - Assert.fail("Failed to read, deserialize, or delete GATK report " + s3Object.getName() + " with exception " + e); - } - } else { - Assert.assertNull(s3Object, "AWS upload should have failed for mode " + awsMode + " but got non-null s3 object back " + s3Object + " error was " + report.formatError()); - Assert.assertTrue(report.exceptionOccurredDuringPost(), "S3 object was null but the report says that the upload succeeded"); - Assert.assertNotNull(report.getErrorMessage(), "Report succeeded but an error message wasn't found"); - if ( awsMode == GATKRunReport.AWSMode.FAIL_WITH_EXCEPTION ) - Assert.assertNotNull(report.getErrorThrown()); - } - } - - private void deleteFromS3(final GATKRunReport report) throws Exception { - final S3Service s3Service = GATKRunReport.initializeAWSService(downloaderAccessKey, downloaderSecretKey); - // Retrieve the whole data object we created previously - s3Service.deleteObject(report.getS3ReportBucket(), report.getReportFileName()); - } - - @DataProvider(name = "PostReportByType") - public Object[][] makePostReportByType() { - List tests = new ArrayList(); - - for ( final GATKRunReport.PhoneHomeOption et : GATKRunReport.PhoneHomeOption.values() ) { - tests.add(new Object[]{et}); - } - - return tests.toArray(new Object[][]{}); - } - - @Test(enabled = ! DEBUG, dataProvider = "PostReportByType", timeOut = S3_PUT_TIMEOUT_IN_MILLISECONDS_FOR_TESTING * 2) - public void testPostReportByType(final GATKRunReport.PhoneHomeOption type) { - final GATKRunReport report = new GATKRunReport(walker, exception, engine, GATKRunReport.PhoneHomeOption.AWS, S3_PUT_TIMEOUT_IN_MILLISECONDS_FOR_TESTING); - Assert.assertFalse(report.exceptionOccurredDuringPost(), "An exception occurred during posting the report"); - final boolean succeeded = report.postReport(type); - - if ( type == GATKRunReport.PhoneHomeOption.NO_ET ) - Assert.assertFalse(succeeded, "NO_ET option shouldn't write a report"); - else { - Assert.assertTrue(succeeded, "Any non NO_ET option should succeed in writing a report"); - - if ( type == GATKRunReport.PhoneHomeOption.STDOUT ) { - // nothing to do - } else { - // must have gone to AWS - try { - Assert.assertTrue(report.wentToAWS(), "The report should have gone to AWS but the report says it wasn't"); - deleteFromS3(report); - } catch ( Exception e ) { - Assert.fail("Failed delete GATK report " + report.getReportFileName() + " with exception " + e); - } - } - } - } - - public interface S3Op { - public void apply() throws ServiceException; - } - - // Will fail with timeout if AWS time out isn't working - // Will fail with exception if AWS doesn't protect itself from errors - @Test(timeOut = S3_PUT_TIMEOUT_IN_MILLISECONDS_FOR_TESTING * 2) - public void testAWSPublicKeyHasAccessControls() throws Exception { - final GATKRunReport report = new GATKRunReport(walker, exception, engine, GATKRunReport.PhoneHomeOption.AWS, S3_PUT_TIMEOUT_IN_MILLISECONDS_FOR_TESTING); - report.sendAWSToTestBucket(); - final S3Object s3Object = report.postReportToAWSS3(); - Assert.assertNotNull(s3Object, "Upload to AWS failed, s3Object was null. error was " + report.formatError()); - - // create a service with the public key, and make sure it cannot list or delete - final S3Service s3Service = GATKRunReport.initializeAWSService(GATKRunReport.getAWSUploadAccessKey(), GATKRunReport.getAWSUploadSecretKey()); - assertOperationNotAllowed("listAllBuckets", new S3Op() { - @Override - public void apply() throws S3ServiceException { - s3Service.listAllBuckets(); - } - }); - assertOperationNotAllowed("listBucket", new S3Op() { - @Override - public void apply() throws S3ServiceException { s3Service.listObjects(report.getS3ReportBucket()); } - }); - assertOperationNotAllowed("createBucket", new S3Op() { - @Override - public void apply() throws S3ServiceException { s3Service.createBucket("ShouldNotCreate"); } - }); - assertOperationNotAllowed("deleteObject", new S3Op() { - @Override - public void apply() throws ServiceException { s3Service.deleteObject(report.getS3ReportBucket(), report.getReportFileName()); } - }); - } - - private void assertOperationNotAllowed(final String name, final S3Op op) { - try { - op.apply(); - // only gets here if the operation was successful - Assert.fail("Operation " + name + " ran successfully but we expected to it fail"); - } catch ( ServiceException e ) { - Assert.assertEquals(e.getErrorCode(), "AccessDenied"); - } - } - - class RunReportDummyReadWalker extends ReadWalker { - @Override - public Integer map(ReferenceContext ref, GATKSAMRecord read, RefMetaDataTracker metaDataTracker) { - return 0; - } - - @Override - public Integer reduceInit() { - return 0; - } - - @Override - public Integer reduce(Integer value, Integer sum) { - return 0; - } - } - - class RunReportDummyLocusWalker extends LocusWalker { - @Override - public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) { - return 0; - } - - @Override - public Integer reduceInit() { - return 0; - } - - @Override - public Integer reduce(Integer value, Integer sum) { - return 0; - } - } - - class RunReportDummyRodWalker extends RodWalker { - @Override - public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) { - return 0; - } - - @Override - public Integer reduceInit() { - return 0; - } - - @Override - public Integer reduce(Integer value, Integer sum) { - return 0; - } - } - - class RunReportDummyActiveRegionWalker extends ActiveRegionWalker { - @Override - public ActivityProfileState isActive(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) { - return new ActivityProfileState(ref.getLocus(), 0.0); - } - - @Override - public Integer map(ActiveRegion activeRegion, RefMetaDataTracker metaDataTracker) { - return 0; - } - - @Override - public Integer reduceInit() { - return 0; - } - - @Override - public Integer reduce(Integer value, Integer sum) { - return 0; - } - } -} diff --git a/public/gatk-engine/src/test/java/org/broadinstitute/gatk/engine/walkers/WalkerTest.java b/public/gatk-engine/src/test/java/org/broadinstitute/gatk/engine/walkers/WalkerTest.java index b56618ce2..aad15de7c 100644 --- a/public/gatk-engine/src/test/java/org/broadinstitute/gatk/engine/walkers/WalkerTest.java +++ b/public/gatk-engine/src/test/java/org/broadinstitute/gatk/engine/walkers/WalkerTest.java @@ -34,7 +34,6 @@ import org.apache.commons.lang.StringUtils; import org.broadinstitute.gatk.engine.CommandLineExecutable; import org.broadinstitute.gatk.engine.CommandLineGATK; import org.broadinstitute.gatk.engine.crypt.CryptUtils; -import org.broadinstitute.gatk.engine.phonehome.GATKRunReport; import org.broadinstitute.gatk.utils.BaseTest; import org.broadinstitute.gatk.utils.MD5DB; import org.broadinstitute.gatk.utils.MD5Mismatch; @@ -225,9 +224,6 @@ public class WalkerTest extends BaseTest { public String getArgsWithImplicitArgs() { String args = this.args; if ( includeImplicitArgs ) { - args = args + (ENABLE_PHONE_HOME_FOR_TESTS ? - String.format(" -et %s ", GATKRunReport.PhoneHomeOption.AWS) : - String.format(" -et %s -K %s ", GATKRunReport.PhoneHomeOption.NO_ET, gatkKeyFile)); if ( includeShadowBCF && GENERATE_SHADOW_BCF ) args = args + " --generateShadowBCF "; if ( ! ENABLE_AUTO_INDEX_CREATION_AND_LOCKING_FOR_TESTS )