From 3fd2392090b2964192ffe0bd00880a0d9e9a667f Mon Sep 17 00:00:00 2001 From: depristo Date: Sat, 28 Aug 2010 22:53:32 +0000 Subject: [PATCH] Improved interface to getting command line options. Now fully traverses all objects to get all internal argument collections. Preliminary (but disabled version) of phoning home (see -et argument for more information). Captures correct and erroring out runs and writes out gzipped, xml report with lots of useful information. Needs a bit more information but is approximately working. Reports going to /humgen/gsa-hpprojects/GATK/reports/ in submitted directory that will be collated by some external tool. Only operating if -et STANDARD or -et STDOUT are provided currently and REPORT_DIR contains a file called ENABLE. WalkerTest now adds -et NO_ET to tests to avoid populating the reports with tests. git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@4155 348d0f76-0448-11de-a6fe-93d51630548a --- .../sting/commandline/CommandLineProgram.java | 35 +-- .../sting/commandline/CommandLineUtils.java | 38 ++- .../sting/commandline/ParsingEngine.java | 54 +++- .../sting/gatk/CommandLineExecutable.java | 40 ++- .../sting/gatk/CommandLineGATK.java | 29 +- .../sting/gatk/GATKErrorReport.java | 103 ------- .../sting/gatk/GenomeAnalysisEngine.java | 11 + .../arguments/GATKArgumentCollection.java | 8 + .../sting/gatk/phonehome/GATKRunReport.java | 252 ++++++++++++++++++ .../gatk/traversals/TraversalEngine.java | 2 +- .../walkers/annotator/VariantAnnotator.java | 4 +- .../filters/VariantFiltrationWalker.java | 4 +- .../walkers/genotyper/UnifiedGenotyper.java | 4 +- .../gatk/walkers/indels/IndelRealigner.java | 4 +- .../TableRecalibrationWalker.java | 4 +- .../walkers/variantutils/VariantsToTable.java | 27 +- .../org/broadinstitute/sting/WalkerTest.java | 4 +- 17 files changed, 420 insertions(+), 203 deletions(-) delete mode 100644 java/src/org/broadinstitute/sting/gatk/GATKErrorReport.java create mode 100644 java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java diff --git a/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java b/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java index df3ea95b3..828032a23 100644 --- a/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java +++ b/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java @@ -26,6 +26,7 @@ package org.broadinstitute.sting.commandline; import org.apache.log4j.*; +import org.broadinstitute.sting.gatk.phonehome.GATKRunReport; import org.broadinstitute.sting.utils.help.ApplicationDetails; import org.broadinstitute.sting.utils.help.HelpFormatter; @@ -143,7 +144,7 @@ public abstract class CommandLineProgram { * * @return the return code to exit the program with */ - protected abstract int execute(); + protected abstract int execute() throws Exception; static { // setup a basic log configuration @@ -245,9 +246,6 @@ public abstract class CommandLineProgram { // call the execute CommandLineProgram.result = clp.execute(); - - // return the result - //System.exit(result); // todo -- is this safe -- why exit here? I want to run the GATK like normal } catch (ArgumentException e) { clp.parser.printHelp(clp.getApplicationDetails()); @@ -258,29 +256,10 @@ public abstract class CommandLineProgram { // we catch all exceptions here. if it makes it to this level, we're in trouble. Let's bail! // TODO: what if the logger is the exception? hmm... logger.fatal("\n"); - toErrorLog(clp, e); throw new RuntimeException(e); } } - /** - * generate an error log - * @param clp the command line program - * @param e the exception - */ - private static void toErrorLog(CommandLineProgram clp, Exception e) { - File logFile = new File("GATK_Error.log"); - PrintStream stream; - try { - stream = new PrintStream(logFile); - } catch (Exception e1) { // catch all the exceptions here, if we can't create the file, do the alternate path - if ( e.getCause() != null ) logger.fatal("with cause: " + e.getCause()); - throw new RuntimeException(e); - } - clp.generateErrorLog(stream, e); - stream.close(); - } - /** * Find fields in the object obj that look like command-line arguments, and put command-line * arguments into them. @@ -401,16 +380,6 @@ public abstract class CommandLineProgram { exitSystemWithError(e.getMessage(), e); } - /** - * generate an error log, given the stream to write to and the execption that was generated - * - * @param stream the output stream - * @param e the exception - */ - public void generateErrorLog(PrintStream stream, Exception e) { - e.printStackTrace(stream); - } - /** * A hack to ensure that numbers are always formatted in the US style. */ diff --git a/java/src/org/broadinstitute/sting/commandline/CommandLineUtils.java b/java/src/org/broadinstitute/sting/commandline/CommandLineUtils.java index 28045cd9a..746486b2b 100644 --- a/java/src/org/broadinstitute/sting/commandline/CommandLineUtils.java +++ b/java/src/org/broadinstitute/sting/commandline/CommandLineUtils.java @@ -53,12 +53,12 @@ public class CommandLineUtils { Map commandLineArguments = new LinkedHashMap(); for(Object argumentProvider: argumentProviders) { - List argumentSources = ParsingEngine.extractArgumentSources(argumentProvider.getClass()); - for(ArgumentSource argumentSource: argumentSources) { - Object argumentValue = JVMUtils.getFieldValue(argumentSource.field,argumentProvider); + Map argBindings = ParsingEngine.extractArgumentBindings(argumentProvider); + for(Map.Entry elt: argBindings.entrySet()) { + Object argumentValue = elt.getValue(); String argumentValueString = argumentValue != null ? argumentValue.toString() : null; - for(ArgumentDefinition definition: argumentSource.createArgumentDefinitions()) { + for(ArgumentDefinition definition: elt.getKey().createArgumentDefinitions()) { String argumentName = definition.fullName; commandLineArguments.put(argumentName,argumentValueString); } @@ -68,15 +68,39 @@ public class CommandLineUtils { return commandLineArguments; } - public static String createApproximateCommandLineArgumentString(GenomeAnalysisEngine toolkit, Collection otherArgumentProviders, Class walkerType) { +// public static Map getApproximateCommandLineArguments(Collection argumentProviders) { +// Map commandLineArguments = new LinkedHashMap(); +// +// for(Object argumentProvider: argumentProviders) { +// Map argBings = ParsingEngine.extractArgumentBindings(argumentProvider); +// List argumentSources = ParsingEngine.extractArgumentSources(argumentProvider.getClass()); +// for(ArgumentSource argumentSource: argumentSources) { +// Object argumentValue = JVMUtils.getFieldValue(argumentSource.field,argumentProvider); +// String argumentValueString = argumentValue != null ? argumentValue.toString() : null; +// +// for(ArgumentDefinition definition: argumentSource.createArgumentDefinitions()) { +// String argumentName = definition.fullName; +// commandLineArguments.put(argumentName,argumentValueString); +// } +// } +// } +// +// return commandLineArguments; +// } + public static String createApproximateCommandLineArgumentString(GenomeAnalysisEngine toolkit, Walker walker) { + return createApproximateCommandLineArgumentString(toolkit, null, walker); + } + + public static String createApproximateCommandLineArgumentString(GenomeAnalysisEngine toolkit, Collection otherArgumentProviders, Walker walker) { StringBuffer sb = new StringBuffer(); sb.append("analysis_type="); - sb.append(toolkit.getWalkerName(walkerType)); + sb.append(toolkit.getWalkerName(walker.getClass())); ArrayList allArgumentProviders = new ArrayList(); allArgumentProviders.add(toolkit.getArguments()); - allArgumentProviders.addAll(otherArgumentProviders); + allArgumentProviders.add(walker); + if (otherArgumentProviders != null) allArgumentProviders.addAll(otherArgumentProviders); Map commandLineArgs = getApproximateCommandLineArguments(allArgumentProviders); diff --git a/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java b/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java index 1f4560317..ffb1b766b 100755 --- a/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java +++ b/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java @@ -361,22 +361,64 @@ public class ParsingEngine { } private static List extractArgumentSources(Class sourceClass, Field[] parentFields) { - List argumentSources = new ArrayList(); + // now simply call into the truly general routine extract argument bindings but with a null + // object so bindings aren't computed + Map bindings = extractArgumentBindings(null, sourceClass, parentFields); + return new ArrayList(bindings.keySet()); + } + +// List argumentSources = new ArrayList(); +// while( sourceClass != null ) { +// Field[] fields = sourceClass.getDeclaredFields(); +// for( Field field: fields ) { +// if( ArgumentTypeDescriptor.isArgumentAnnotationPresent(field) ) +// argumentSources.add( new ArgumentSource(parentFields, field) ); +// if( field.isAnnotationPresent(ArgumentCollection.class) ) { +// Field[] newParentFields = Arrays.copyOf(parentFields, parentFields.length + 1); +// newParentFields[parentFields.length] = field; +// argumentSources.addAll( extractArgumentSources(field.getType(), newParentFields) ); +// } +// } +// sourceClass = sourceClass.getSuperclass(); +// } +// return argumentSources; +// } + + public static Map extractArgumentBindings(Object obj) { + if ( obj == null ) throw new IllegalArgumentException("Incoming object cannot be null"); + return extractArgumentBindings(obj, obj.getClass(), new Field[0]); + } + + /** + * Extract all the argument sources from a given object, along with their bindings if obj != null . + * @param obj the object corresponding to the sourceClass + * @param sourceClass class to act as sources for other arguments. + * @param parentFields + * @return A map of sources associated with this object and its aggregated objects and bindings to their bindings values + */ + private static Map extractArgumentBindings(Object obj, Class sourceClass, Field[] parentFields) { + Map bindings = new HashMap(); + while( sourceClass != null ) { Field[] fields = sourceClass.getDeclaredFields(); for( Field field: fields ) { - if( ArgumentTypeDescriptor.isArgumentAnnotationPresent(field) ) - argumentSources.add( new ArgumentSource(parentFields, field) ); + if( ArgumentTypeDescriptor.isArgumentAnnotationPresent(field) ) { + Object val = obj != null ? JVMUtils.getFieldValue(field, obj) : null; + bindings.put( new ArgumentSource(parentFields, field), val ); + } if( field.isAnnotationPresent(ArgumentCollection.class) ) { + Object val = obj != null ? JVMUtils.getFieldValue(field, obj) : null; Field[] newParentFields = Arrays.copyOf(parentFields, parentFields.length + 1); newParentFields[parentFields.length] = field; - argumentSources.addAll( extractArgumentSources(field.getType(), newParentFields) ); + bindings.putAll( extractArgumentBindings(val, field.getType(), newParentFields) ); } } + sourceClass = sourceClass.getSuperclass(); } - return argumentSources; - } + + return bindings; + } /** * Determines whether a token looks like the name of an argument. diff --git a/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java b/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java index 1dd1e532a..b2c9a323c 100644 --- a/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java +++ b/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java @@ -28,6 +28,7 @@ package org.broadinstitute.sting.gatk; import org.broadinstitute.sting.gatk.arguments.GATKArgumentCollection; import org.broadinstitute.sting.commandline.CommandLineProgram; import org.broadinstitute.sting.commandline.ArgumentTypeDescriptor; +import org.broadinstitute.sting.gatk.phonehome.GATKRunReport; import org.broadinstitute.sting.utils.StingException; import org.broadinstitute.sting.utils.Utils; import org.broadinstitute.sting.utils.text.XReadLines; @@ -35,6 +36,7 @@ import org.broadinstitute.sting.gatk.walkers.Walker; import java.io.File; import java.io.FileNotFoundException; +import java.io.PrintStream; import java.util.*; import net.sf.picard.filter.SamRecordFilter; @@ -68,22 +70,44 @@ public abstract class CommandLineExecutable extends CommandLineProgram { * * @return the return code to exit the program with */ - protected int execute() { - GATKResult = executeGATK(); - return 0; - } - - protected Object executeGATK() { + protected int execute() throws Exception { Walker mWalker = GATKEngine.getWalkerByName(getAnalysisName()); Collection filters = GATKEngine.createFiltersForWalker(getArgumentCollection(),mWalker); // load the arguments into the walker / filters. loadArgumentsIntoObject(mWalker); - for(SamRecordFilter filter: filters) + for (SamRecordFilter filter: filters) loadArgumentsIntoObject(filter); // set the analysis name in the argument collection - return GATKEngine.execute(getArgumentCollection(), mWalker, filters); + try { + GATKResult = GATKEngine.execute(getArgumentCollection(), mWalker, filters); + generateGATKRunReport(mWalker); + } catch ( Exception e ) { + generateGATKRunReport(mWalker, e); + throw e; + } + + // always return 0 + return 0; + } + + /** + * generate an error log + * @param e the exception, can be null if no exception occurred + */ + private void generateGATKRunReport(Walker mWalker, Exception e) { + if ( getArgumentCollection().phoneHomeType != GATKRunReport.PhoneHomeOption.NO_ET ) { + GATKRunReport report = new GATKRunReport(mWalker, e, GATKEngine); + if ( getArgumentCollection().phoneHomeType == GATKRunReport.PhoneHomeOption.STDOUT ) + report.postReport(System.out); + else + report.postReport(); + } + } + + private void generateGATKRunReport(Walker mWalker) { + generateGATKRunReport(mWalker, null); } /** diff --git a/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java b/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java index b13c26cb1..834a3e153 100755 --- a/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java +++ b/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java @@ -65,18 +65,6 @@ public class CommandLineGATK extends CommandLineExecutable { getAdditionalHelp()); } - /** - * generate an error log, given the stream to write to and the exception that was thrown - * - * @param stream the output stream - * @param e the exception - */ - @Override - public void generateErrorLog(PrintStream stream, Exception e) { - GATKErrorReport report = new GATKErrorReport(e, this.argCollection); - report.reportToStream(stream); - } - @Override protected String getAnalysisName() { return analysisName; @@ -114,19 +102,24 @@ public class CommandLineGATK extends CommandLineExecutable { * @return The application header. */ public static List createApplicationHeader() { - ResourceBundle headerInfo = TextFormattingUtils.loadResourceBundle("StingText"); - - String versionNumber = headerInfo.containsKey("org.broadinstitute.sting.gatk.version") ? headerInfo.getString("org.broadinstitute.sting.gatk.version") : ""; - String timestamp = headerInfo.containsKey("build.timestamp") ? headerInfo.getString("build.timestamp") : ""; - List header = new ArrayList(); - header.add(String.format("The Genome Analysis Toolkit (GATK) v%s, Compiled %s",versionNumber,timestamp)); + header.add(String.format("The Genome Analysis Toolkit (GATK) v%s, Compiled %s",getVersionNumber(), getBuildTime())); header.add("Copyright (c) 2009 The Broad Institute"); header.add("Please view our documentation at http://www.broadinstitute.org/gsa/wiki"); header.add("For support, please view our support site at http://getsatisfaction.com/gsa"); return header; } + public static String getVersionNumber() { + ResourceBundle headerInfo = TextFormattingUtils.loadResourceBundle("StingText"); + return headerInfo.containsKey("org.broadinstitute.sting.gatk.version") ? headerInfo.getString("org.broadinstitute.sting.gatk.version") : ""; + } + + public static String getBuildTime() { + ResourceBundle headerInfo = TextFormattingUtils.loadResourceBundle("StingText"); + return headerInfo.containsKey("build.timestamp") ? headerInfo.getString("build.timestamp") : ""; + } + /** * Retrieves additional information about GATK walkers. * the code in HelpFormatter and supply it as a helper to this method. diff --git a/java/src/org/broadinstitute/sting/gatk/GATKErrorReport.java b/java/src/org/broadinstitute/sting/gatk/GATKErrorReport.java deleted file mode 100644 index 4ea40a6ee..000000000 --- a/java/src/org/broadinstitute/sting/gatk/GATKErrorReport.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) 2010 The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR - * THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.gatk; - -import org.broadinstitute.sting.gatk.CommandLineGATK; -import org.broadinstitute.sting.gatk.arguments.GATKArgumentCollection; -import org.broadinstitute.sting.utils.StingException; -import org.simpleframework.xml.Element; -import org.simpleframework. xml.ElementList; -import org.simpleframework.xml.Serializer; -import org.simpleframework.xml.core.Persister; -import org.simpleframework.xml.stream.Format; -import org.simpleframework.xml.stream.HyphenStyle; - -import java.io.PrintStream; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.List; - - -/** - * @author aaron - *

- * Class GATKErrorReport - *

- * A basic description of what went wrong during a run of the GATK, - * what the parameters were, etc. - */ -public class GATKErrorReport { - // the listing of the fields is somewhat important; this is the order that the simple XML will output them - @ElementList(required = true, name = "gatk_header_Information") - private static List mGATKHeader; - - @Element(required = false, name = "exception") - private final ExceptionToXML mException; - - @Element(required = true, name = "date_time") - private final String mDateTime; - - @Element(required = true, name = "argument_collection") - private final GATKArgumentCollection mCollection; - - @Element(required = true, name = "working_directory") - private static String currentPath; - - private static final DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH.mm.ss"); - - static { - GATKErrorReport.mGATKHeader = CommandLineGATK.createApplicationHeader(); - currentPath = System.getProperty("user.dir"); - } - - public GATKErrorReport(Exception e, GATKArgumentCollection collection) { - this.mCollection = collection; - this.mException = new ExceptionToXML(e); - java.util.Date date = new java.util.Date(); - mDateTime = dateFormat.format(date); - } - - public void reportToStream(PrintStream stream) { - Serializer serializer = new Persister(new Format(new HyphenStyle())); - try { - serializer.write(this, stream); - } catch (Exception e) { - throw new StingException("Failed to marshal the data to the file " + stream, e); - } - } - - class ExceptionToXML { - @ElementList(required = false) - final List exceptionDetails = new ArrayList(); - - public ExceptionToXML(Exception e) { - for (StackTraceElement element : e.getStackTrace()) { - exceptionDetails.add(element.toString()); - } - } - } -} diff --git a/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java b/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java index bafae67a6..235c13ada 100755 --- a/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java +++ b/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java @@ -123,6 +123,8 @@ public class GenomeAnalysisEngine { */ private final FilterManager filterManager; + private Date startTime = null; // the start time for execution + /** * our constructor, where all the work is done *

@@ -149,6 +151,7 @@ public class GenomeAnalysisEngine { public Object execute(GATKArgumentCollection args, Walker my_walker, Collection filters) { //HeapSizeMonitor monitor = new HeapSizeMonitor(); //monitor.start(); + startTime = new java.util.Date(); // validate our parameters if (args == null) { @@ -188,6 +191,14 @@ public class GenomeAnalysisEngine { return result; } + /** + * @return the start time when the execute() function was last called + */ + public Date getStartTime() { + return startTime; + } + + /** * Setup the intervals to be processed */ diff --git a/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java b/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java index efab5c96e..514844948 100755 --- a/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java +++ b/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java @@ -26,6 +26,7 @@ package org.broadinstitute.sting.gatk.arguments; import net.sf.samtools.SAMFileReader; +import org.broadinstitute.sting.gatk.phonehome.GATKRunReport; import org.broadinstitute.sting.utils.StingException; import org.broadinstitute.sting.utils.interval.IntervalMergingRule; import org.broadinstitute.sting.commandline.Argument; @@ -74,6 +75,10 @@ public class GATKArgumentCollection { @Argument(fullName = "read_buffer_size", shortName = "rbs", doc="Number of reads per SAM file to buffer in memory", required = false) public Integer readBufferSize = null; + @Element(required = false) + @Argument(fullName = "phone_home", shortName = "et", doc="What kind of GATK run report should we generate? Standard is the default, can be verbose or NO_ET so nothing is posted to the run repository", required = false) + public GATKRunReport.PhoneHomeOption phoneHomeType = GATKRunReport.PhoneHomeOption.NO_ET; + @ElementList(required = false) @Argument(fullName = "read_filter", shortName = "rf", doc = "Specify filtration criteria to apply to each read individually.", required = false) public List readFilters = new ArrayList(); @@ -337,6 +342,9 @@ public class GATKArgumentCollection { (other.RODToInterval != null && !other.RODToInterval.equals(RODToInterval))) { return false; } + if (other.phoneHomeType != this.phoneHomeType) { + return false; + } if (BTIMergeRule != other.BTIMergeRule) return false; diff --git a/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java b/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java new file mode 100644 index 000000000..69594018a --- /dev/null +++ b/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2010, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.phonehome; + +import org.apache.log4j.Logger; +import org.broadinstitute.sting.commandline.CommandLineUtils; +import org.broadinstitute.sting.gatk.CommandLineGATK; +import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; +import org.broadinstitute.sting.gatk.arguments.GATKArgumentCollection; +import org.broadinstitute.sting.gatk.walkers.Walker; +import org.broadinstitute.sting.utils.StingException; +import org.broadinstitute.sting.utils.Utils; +import org.simpleframework.xml.Element; +import org.simpleframework.xml.ElementList; +import org.simpleframework.xml.Serializer; +import org.simpleframework.xml.core.Persister; +import org.simpleframework.xml.stream.Format; +import org.simpleframework.xml.stream.HyphenStyle; + +import java.io.*; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.zip.GZIPOutputStream; + + +/** + * @author depristo + *

+ * Class GATKRunReport + *

+ * A detailed description of a GATK run, and error if applicable + */ +public class GATKRunReport { + /** + * The root file system directory where we keep common report data + */ + private static File REPORT_DIR = new File("/humgen/gsa-hpprojects/GATK/reports"); + + /** + * The full path to the direct where submitted (and uncharacterized) report files are written + */ + private static File REPORT_SUBMIT_DIR = new File(REPORT_DIR.getAbsolutePath() + "/submitted"); + + /** + * Full path to the sentinel file that controls whether reports are written out. If this file doesn't + * exist, no long will be written + */ + private static File REPORT_SENTINEL = new File(REPORT_DIR.getAbsolutePath() + "/ENABLE"); + + /** + * our log + */ + protected static Logger logger = Logger.getLogger(GATKRunReport.class); + + + // the listing of the fields is somewhat important; this is the order that the simple XML will output them + @ElementList(required = true, name = "gatk_header_Information") + private static List mGATKHeader; + + @Element(required = false, name = "exception") + private final ExceptionToXML mException; + + @Element(required = true, name = "argument_collection") + private final GATKArgumentCollection mCollection; + + @Element(required = true, name = "working_directory") + private static String currentPath; + + @Element(required = true, name = "start_time") + private static String startTime; + + @Element(required = true, name = "end_time") + private static String endTime; + + @Element(required = true, name = "run_time") + private static long runTime; + + @Element(required = true, name = "command_line") + private static String cmdLine; + + @Element(required = true, name = "walker_name") + private static String walkerName; + + @Element(required = true, name = "svn_version") + private static String svnVersion; + + @Element(required = true, name = "memory") + private static long memory; + + @Element(required = true, name = "java_tmp_directory") + private static String tmpDir; + + @Element(required = true, name = "domain_name") + private static String domainName; + + @Element(required = true, name = "user_name") + private static String userName; + + @Element(required = true, name = "host_name") + private static String hostName; + + @Element(required = true, name = "java") + private static String java; + + @Element(required = true, name = "machine") + private static String machine; + + @Element(required = true, name = "iterations") + private static long nIterations; + + @Element(required = true, name = "reads") + private static long nReads; + + @Element(required = true, name = "read_metrics") + private static String readMetrics; + + // not done + //- walker-specific args + //+ md5 all filenames + //- size of filenames + //- # reads/loci + //- free memory on machine + + public enum PhoneHomeOption { + NO_ET, + STANDARD, + STDOUT + } + + private static final DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH.mm.ss"); + + static { + GATKRunReport.mGATKHeader = CommandLineGATK.createApplicationHeader(); + currentPath = System.getProperty("user.dir"); + } + + /** Create a new RunReport and population all of the fields with values from the walker and engine */ + public GATKRunReport(Walker walker, Exception e, GenomeAnalysisEngine engine) { + this.mCollection = engine.getArguments(); + this.mException = e == null ? null : new ExceptionToXML(e); + + startTime = dateFormat.format(engine.getStartTime()); // fixme + Date end = new java.util.Date(); + endTime = dateFormat.format(end); + runTime = (end.getTime() - engine.getStartTime().getTime()) / 1000L; // difference in seconds + + cmdLine = CommandLineUtils.createApproximateCommandLineArgumentString(engine, walker); + walkerName = engine.getWalkerName(walker.getClass()); + svnVersion = CommandLineGATK.getVersionNumber(); + nIterations = engine.getCumulativeMetrics().getNumIterations(); + nReads = engine.getCumulativeMetrics().getNumReadsSeen(); + readMetrics = engine.getCumulativeMetrics().toString(); + memory = Runtime.getRuntime().totalMemory(); + tmpDir = System.getProperty("java.io.tmpdir"); + domainName = "Need to figure this out"; + hostName = "Need to figure this out"; + userName = System.getProperty("user.name"); + java = Utils.join("-", Arrays.asList(System.getProperty("java.vendor"), System.getProperty("java.version"))); + machine = Utils.join("-", Arrays.asList(System.getProperty("os.name"), System.getProperty("os.arch"))); + } + + public void postReport(OutputStream stream) { + Serializer serializer = new Persister(new Format(new HyphenStyle())); + try { + serializer.write(this, stream); + //throw new StingException("test"); + } catch (Exception e) { + throw new StingException("Failed to marshal the data to the file " + stream, e); + } + } + + public void postReport(File destination) throws FileNotFoundException, IOException { + BufferedOutputStream out = + new BufferedOutputStream( + new GZIPOutputStream( + new FileOutputStream(destination))); + try { + postReport(out); + } finally { + out.close(); + } + } + + public void postReport() { + try { + if ( sentinelExists() ) { + String filename = org.apache.commons.lang.RandomStringUtils.randomAlphanumeric(32) + ".report.xml.gz"; + File file = new File(REPORT_SUBMIT_DIR, filename); + postReport(file); + logger.info("Wrote report to " + file); + } else { + logger.info("Not writing report: sentinel " + REPORT_SENTINEL + " doesn't exist"); + } + } catch ( Exception e ) { + // we catch everything, and no matter what eat the error + logger.warn("Received error while posting report. GATK continuing on but no run report has been generated because: " + e.getMessage()); + } + } + + private boolean sentinelExists() { + return REPORT_SENTINEL.exists(); + } + + class ExceptionToXML { + @Element(required = false, name = "message") + String message = null; + + @ElementList(required = false, name = "stacktrace") + final List stackTrace = new ArrayList(); + + @Element(required = false, name = "cause") + ExceptionToXML cause = null; + + public ExceptionToXML(Throwable e) { + message = e.getMessage(); + for (StackTraceElement element : e.getStackTrace()) { + stackTrace.add(element.toString()); + } + + if ( e.getCause() != null ) { + //message += " because " + e.getCause().getMessage(); + cause = new ExceptionToXML(e.getCause()); + } + } + } +} diff --git a/java/src/org/broadinstitute/sting/gatk/traversals/TraversalEngine.java b/java/src/org/broadinstitute/sting/gatk/traversals/TraversalEngine.java index fc4c03a44..798238b94 100755 --- a/java/src/org/broadinstitute/sting/gatk/traversals/TraversalEngine.java +++ b/java/src/org/broadinstitute/sting/gatk/traversals/TraversalEngine.java @@ -91,7 +91,7 @@ public abstract class TraversalEngine,Provide for ( Map.Entry countsByFilter: cumulativeMetrics.getCountsByFilter().entrySet()) nSkippedReads += countsByFilter.getValue(); - logger.info(String.format("Total runtime %.2f secs, %.2f min, %.2f hours%n", elapsed, elapsed / 60, elapsed / 3600)); + logger.info(String.format("Total runtime %.2f secs, %.2f min, %.2f hours", elapsed, elapsed / 60, elapsed / 3600)); logger.info(String.format("%d reads were filtered out during traversal out of %d total (%.2f%%)", nSkippedReads, cumulativeMetrics.getNumReadsSeen(), diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java b/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java index 9b1e97b94..0356a0cd2 100755 --- a/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java @@ -146,9 +146,7 @@ public class VariantAnnotator extends RodWalker { hInfo.add(line); } if ( !NO_VCF_HEADER_LINE ) { - Set args = new HashSet(); - args.add(this); - hInfo.add(new VCFHeaderLine("VariantAnnotator", "\"" + CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), args, getClass()) + "\"")); + hInfo.add(new VCFHeaderLine("VariantAnnotator", "\"" + CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), this) + "\"")); } VCFHeader vcfHeader = new VCFHeader(hInfo, samples); diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/filters/VariantFiltrationWalker.java b/java/src/org/broadinstitute/sting/gatk/walkers/filters/VariantFiltrationWalker.java index b8fd57615..ccc5ace3c 100755 --- a/java/src/org/broadinstitute/sting/gatk/walkers/filters/VariantFiltrationWalker.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/filters/VariantFiltrationWalker.java @@ -112,9 +112,7 @@ public class VariantFiltrationWalker extends RodWalker { } if ( !NO_VCF_HEADER_LINE ) { - Set args = new HashSet(); - args.add(this); - hInfo.add(new VCFHeaderLine("VariantFiltration", "\"" + CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), args, getClass()) + "\"")); + hInfo.add(new VCFHeaderLine("VariantFiltration", "\"" + CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), this) + "\"")); } writer.writeHeader(new VCFHeader(hInfo, new TreeSet(vc.getSampleNames()))); diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java b/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java index c955c086d..759555f39 100755 --- a/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java @@ -156,9 +156,7 @@ public class UnifiedGenotyper extends LocusWalker args = new HashSet(); - args.add(UAC); - headerInfo.add(new VCFHeaderLine("UnifiedGenotyper", "\"" + CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), args, getClass()) + "\"")); + headerInfo.add(new VCFHeaderLine("UnifiedGenotyper", "\"" + CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), this) + "\"")); } return headerInfo; diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/indels/IndelRealigner.java b/java/src/org/broadinstitute/sting/gatk/walkers/indels/IndelRealigner.java index 08db26acb..daff48970 100755 --- a/java/src/org/broadinstitute/sting/gatk/walkers/indels/IndelRealigner.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/indels/IndelRealigner.java @@ -224,9 +224,7 @@ public class IndelRealigner extends ReadWalker { final SAMProgramRecord programRecord = new SAMProgramRecord(PROGRAM_RECORD_NAME); final ResourceBundle headerInfo = TextFormattingUtils.loadResourceBundle("StingText"); programRecord.setProgramVersion(headerInfo.getString("org.broadinstitute.sting.gatk.version")); - HashSet args = new HashSet(); - args.add(this); - programRecord.setCommandLine(CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), args, getClass())); + programRecord.setCommandLine(CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), this)); List oldRecords = header.getProgramRecords(); List newRecords = new ArrayList(oldRecords.size()+1); diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java b/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java index 22f6f2d18..dde60fc5f 100644 --- a/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java @@ -262,10 +262,8 @@ public class TableRecalibrationWalker extends ReadWalker args = new HashSet(); - args.add(RAC); StringBuffer sb = new StringBuffer(); - sb.append(CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), args, getClass())); + sb.append(CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), this)); sb.append(" Covariates=["); for( Covariate cov : requestedCovariates ) { sb.append(cov.getClass().getSimpleName()); diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java b/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java index 2c9419a96..ddf5b38db 100755 --- a/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java @@ -64,10 +64,13 @@ public class VariantsToTable extends RodWalker { public int MAX_RECORDS = -1; int nRecords = 0; + @Argument(fullName="allowMultiAllelic", shortName="AMA", doc="If provided, we will not require the site to be biallelic", required=false) + public boolean allowMultiAllelic = false; + private List fieldsToTake; public void initialize() { - fieldsToTake = Arrays.asList(FIELDS.toUpperCase().split(",")); + fieldsToTake = Arrays.asList(FIELDS.split(",")); out.println(Utils.join("\t", fieldsToTake)); } @@ -111,21 +114,23 @@ public class VariantsToTable extends RodWalker { if ( ++nRecords < MAX_RECORDS || MAX_RECORDS == -1 ) { Collection vcs = tracker.getAllVariantContexts(ref, context.getLocation()); for ( VariantContext vc : vcs) { - List vals = new ArrayList(); + if ( allowMultiAllelic || vc.isBiallelic() ) { + List vals = new ArrayList(); - for ( String field : fieldsToTake ) { - String val = "UNK"; + for ( String field : fieldsToTake ) { + String val = "UNK"; - if ( getters.containsKey(field) ) { - val = getters.get(field).get(vc); - } else if ( vc.hasAttribute(field) ) { - val = vc.getAttributeAsString(field); + if ( getters.containsKey(field) ) { + val = getters.get(field).get(vc); + } else if ( vc.hasAttribute(field) ) { + val = vc.getAttributeAsString(field); + } + + vals.add(val); } - vals.add(val); + out.println(Utils.join("\t", vals)); } - - out.println(Utils.join("\t", vals)); } return 1; diff --git a/java/test/org/broadinstitute/sting/WalkerTest.java b/java/test/org/broadinstitute/sting/WalkerTest.java index 089630e8b..5eba9813f 100755 --- a/java/test/org/broadinstitute/sting/WalkerTest.java +++ b/java/test/org/broadinstitute/sting/WalkerTest.java @@ -273,9 +273,11 @@ public class WalkerTest extends BaseTest { System.out.println(String.format("Executing test %s with GATK arguments: %s", name, Utils.join(" ",command))); // add the logging level to each of the integration test commands - String[] cmd2 = Arrays.copyOf(command, command.length + 2); + String[] cmd2 = Arrays.copyOf(command, command.length + 4); cmd2[command.length] = "-l"; cmd2[command.length+1] = "WARN"; + cmd2[command.length+2] = "-et"; + cmd2[command.length+3] = "NO_ET"; // run the executable CommandLineExecutable.start(instance, cmd2);