Improved interface to getting command line options. Now fully traverses all objects to get all internal argument collections. Preliminary (but disabled version) of phoning home (see -et argument for more information). Captures correct and erroring out runs and writes out gzipped, xml report with lots of useful information. Needs a bit more information but is approximately working. Reports going to /humgen/gsa-hpprojects/GATK/reports/ in submitted directory that will be collated by some external tool. Only operating if -et STANDARD or -et STDOUT are provided currently and REPORT_DIR contains a file called ENABLE. WalkerTest now adds -et NO_ET to tests to avoid populating the reports with tests.
git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@4155 348d0f76-0448-11de-a6fe-93d51630548a
This commit is contained in:
depristo
parent
576ae30df1
commit
3fd2392090
|
|
@ -26,6 +26,7 @@
|
|||
package org.broadinstitute.sting.commandline;
|
||||
|
||||
import org.apache.log4j.*;
|
||||
import org.broadinstitute.sting.gatk.phonehome.GATKRunReport;
|
||||
import org.broadinstitute.sting.utils.help.ApplicationDetails;
|
||||
import org.broadinstitute.sting.utils.help.HelpFormatter;
|
||||
|
||||
|
|
@ -143,7 +144,7 @@ public abstract class CommandLineProgram {
|
|||
*
|
||||
* @return the return code to exit the program with
|
||||
*/
|
||||
protected abstract int execute();
|
||||
protected abstract int execute() throws Exception;
|
||||
|
||||
static {
|
||||
// setup a basic log configuration
|
||||
|
|
@ -245,9 +246,6 @@ public abstract class CommandLineProgram {
|
|||
|
||||
// call the execute
|
||||
CommandLineProgram.result = clp.execute();
|
||||
|
||||
// return the result
|
||||
//System.exit(result); // todo -- is this safe -- why exit here? I want to run the GATK like normal
|
||||
}
|
||||
catch (ArgumentException e) {
|
||||
clp.parser.printHelp(clp.getApplicationDetails());
|
||||
|
|
@ -258,29 +256,10 @@ public abstract class CommandLineProgram {
|
|||
// we catch all exceptions here. if it makes it to this level, we're in trouble. Let's bail!
|
||||
// TODO: what if the logger is the exception? hmm...
|
||||
logger.fatal("\n");
|
||||
toErrorLog(clp, e);
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* generate an error log
|
||||
* @param clp the command line program
|
||||
* @param e the exception
|
||||
*/
|
||||
private static void toErrorLog(CommandLineProgram clp, Exception e) {
|
||||
File logFile = new File("GATK_Error.log");
|
||||
PrintStream stream;
|
||||
try {
|
||||
stream = new PrintStream(logFile);
|
||||
} catch (Exception e1) { // catch all the exceptions here, if we can't create the file, do the alternate path
|
||||
if ( e.getCause() != null ) logger.fatal("with cause: " + e.getCause());
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
clp.generateErrorLog(stream, e);
|
||||
stream.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* Find fields in the object obj that look like command-line arguments, and put command-line
|
||||
* arguments into them.
|
||||
|
|
@ -401,16 +380,6 @@ public abstract class CommandLineProgram {
|
|||
exitSystemWithError(e.getMessage(), e);
|
||||
}
|
||||
|
||||
/**
|
||||
* generate an error log, given the stream to write to and the execption that was generated
|
||||
*
|
||||
* @param stream the output stream
|
||||
* @param e the exception
|
||||
*/
|
||||
public void generateErrorLog(PrintStream stream, Exception e) {
|
||||
e.printStackTrace(stream);
|
||||
}
|
||||
|
||||
/**
|
||||
* A hack to ensure that numbers are always formatted in the US style.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -53,12 +53,12 @@ public class CommandLineUtils {
|
|||
Map<String,String> commandLineArguments = new LinkedHashMap<String,String>();
|
||||
|
||||
for(Object argumentProvider: argumentProviders) {
|
||||
List<ArgumentSource> argumentSources = ParsingEngine.extractArgumentSources(argumentProvider.getClass());
|
||||
for(ArgumentSource argumentSource: argumentSources) {
|
||||
Object argumentValue = JVMUtils.getFieldValue(argumentSource.field,argumentProvider);
|
||||
Map<ArgumentSource, Object> argBindings = ParsingEngine.extractArgumentBindings(argumentProvider);
|
||||
for(Map.Entry<ArgumentSource, Object> elt: argBindings.entrySet()) {
|
||||
Object argumentValue = elt.getValue();
|
||||
String argumentValueString = argumentValue != null ? argumentValue.toString() : null;
|
||||
|
||||
for(ArgumentDefinition definition: argumentSource.createArgumentDefinitions()) {
|
||||
for(ArgumentDefinition definition: elt.getKey().createArgumentDefinitions()) {
|
||||
String argumentName = definition.fullName;
|
||||
commandLineArguments.put(argumentName,argumentValueString);
|
||||
}
|
||||
|
|
@ -68,15 +68,39 @@ public class CommandLineUtils {
|
|||
return commandLineArguments;
|
||||
}
|
||||
|
||||
public static String createApproximateCommandLineArgumentString(GenomeAnalysisEngine toolkit, Collection<Object> otherArgumentProviders, Class<? extends Walker> walkerType) {
|
||||
// public static Map<String,String> getApproximateCommandLineArguments(Collection<Object> argumentProviders) {
|
||||
// Map<String,String> commandLineArguments = new LinkedHashMap<String,String>();
|
||||
//
|
||||
// for(Object argumentProvider: argumentProviders) {
|
||||
// Map<ArgumentSource, Object> argBings = ParsingEngine.extractArgumentBindings(argumentProvider);
|
||||
// List<ArgumentSource> argumentSources = ParsingEngine.extractArgumentSources(argumentProvider.getClass());
|
||||
// for(ArgumentSource argumentSource: argumentSources) {
|
||||
// Object argumentValue = JVMUtils.getFieldValue(argumentSource.field,argumentProvider);
|
||||
// String argumentValueString = argumentValue != null ? argumentValue.toString() : null;
|
||||
//
|
||||
// for(ArgumentDefinition definition: argumentSource.createArgumentDefinitions()) {
|
||||
// String argumentName = definition.fullName;
|
||||
// commandLineArguments.put(argumentName,argumentValueString);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// return commandLineArguments;
|
||||
// }
|
||||
|
||||
public static String createApproximateCommandLineArgumentString(GenomeAnalysisEngine toolkit, Walker walker) {
|
||||
return createApproximateCommandLineArgumentString(toolkit, null, walker);
|
||||
}
|
||||
|
||||
public static String createApproximateCommandLineArgumentString(GenomeAnalysisEngine toolkit, Collection<Object> otherArgumentProviders, Walker walker) {
|
||||
StringBuffer sb = new StringBuffer();
|
||||
sb.append("analysis_type=");
|
||||
sb.append(toolkit.getWalkerName(walkerType));
|
||||
sb.append(toolkit.getWalkerName(walker.getClass()));
|
||||
|
||||
ArrayList<Object> allArgumentProviders = new ArrayList<Object>();
|
||||
allArgumentProviders.add(toolkit.getArguments());
|
||||
allArgumentProviders.addAll(otherArgumentProviders);
|
||||
allArgumentProviders.add(walker);
|
||||
if (otherArgumentProviders != null) allArgumentProviders.addAll(otherArgumentProviders);
|
||||
|
||||
Map<String,String> commandLineArgs = getApproximateCommandLineArguments(allArgumentProviders);
|
||||
|
||||
|
|
|
|||
|
|
@ -361,22 +361,64 @@ public class ParsingEngine {
|
|||
}
|
||||
|
||||
private static List<ArgumentSource> extractArgumentSources(Class sourceClass, Field[] parentFields) {
|
||||
List<ArgumentSource> argumentSources = new ArrayList<ArgumentSource>();
|
||||
// now simply call into the truly general routine extract argument bindings but with a null
|
||||
// object so bindings aren't computed
|
||||
Map<ArgumentSource, Object> bindings = extractArgumentBindings(null, sourceClass, parentFields);
|
||||
return new ArrayList<ArgumentSource>(bindings.keySet());
|
||||
}
|
||||
|
||||
// List<ArgumentSource> argumentSources = new ArrayList<ArgumentSource>();
|
||||
// while( sourceClass != null ) {
|
||||
// Field[] fields = sourceClass.getDeclaredFields();
|
||||
// for( Field field: fields ) {
|
||||
// if( ArgumentTypeDescriptor.isArgumentAnnotationPresent(field) )
|
||||
// argumentSources.add( new ArgumentSource(parentFields, field) );
|
||||
// if( field.isAnnotationPresent(ArgumentCollection.class) ) {
|
||||
// Field[] newParentFields = Arrays.copyOf(parentFields, parentFields.length + 1);
|
||||
// newParentFields[parentFields.length] = field;
|
||||
// argumentSources.addAll( extractArgumentSources(field.getType(), newParentFields) );
|
||||
// }
|
||||
// }
|
||||
// sourceClass = sourceClass.getSuperclass();
|
||||
// }
|
||||
// return argumentSources;
|
||||
// }
|
||||
|
||||
public static Map<ArgumentSource, Object> extractArgumentBindings(Object obj) {
|
||||
if ( obj == null ) throw new IllegalArgumentException("Incoming object cannot be null");
|
||||
return extractArgumentBindings(obj, obj.getClass(), new Field[0]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract all the argument sources from a given object, along with their bindings if obj != null .
|
||||
* @param obj the object corresponding to the sourceClass
|
||||
* @param sourceClass class to act as sources for other arguments.
|
||||
* @param parentFields
|
||||
* @return A map of sources associated with this object and its aggregated objects and bindings to their bindings values
|
||||
*/
|
||||
private static Map<ArgumentSource, Object> extractArgumentBindings(Object obj, Class sourceClass, Field[] parentFields) {
|
||||
Map<ArgumentSource, Object> bindings = new HashMap<ArgumentSource, Object>();
|
||||
|
||||
while( sourceClass != null ) {
|
||||
Field[] fields = sourceClass.getDeclaredFields();
|
||||
for( Field field: fields ) {
|
||||
if( ArgumentTypeDescriptor.isArgumentAnnotationPresent(field) )
|
||||
argumentSources.add( new ArgumentSource(parentFields, field) );
|
||||
if( ArgumentTypeDescriptor.isArgumentAnnotationPresent(field) ) {
|
||||
Object val = obj != null ? JVMUtils.getFieldValue(field, obj) : null;
|
||||
bindings.put( new ArgumentSource(parentFields, field), val );
|
||||
}
|
||||
if( field.isAnnotationPresent(ArgumentCollection.class) ) {
|
||||
Object val = obj != null ? JVMUtils.getFieldValue(field, obj) : null;
|
||||
Field[] newParentFields = Arrays.copyOf(parentFields, parentFields.length + 1);
|
||||
newParentFields[parentFields.length] = field;
|
||||
argumentSources.addAll( extractArgumentSources(field.getType(), newParentFields) );
|
||||
bindings.putAll( extractArgumentBindings(val, field.getType(), newParentFields) );
|
||||
}
|
||||
}
|
||||
|
||||
sourceClass = sourceClass.getSuperclass();
|
||||
}
|
||||
return argumentSources;
|
||||
}
|
||||
|
||||
return bindings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines whether a token looks like the name of an argument.
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ package org.broadinstitute.sting.gatk;
|
|||
import org.broadinstitute.sting.gatk.arguments.GATKArgumentCollection;
|
||||
import org.broadinstitute.sting.commandline.CommandLineProgram;
|
||||
import org.broadinstitute.sting.commandline.ArgumentTypeDescriptor;
|
||||
import org.broadinstitute.sting.gatk.phonehome.GATKRunReport;
|
||||
import org.broadinstitute.sting.utils.StingException;
|
||||
import org.broadinstitute.sting.utils.Utils;
|
||||
import org.broadinstitute.sting.utils.text.XReadLines;
|
||||
|
|
@ -35,6 +36,7 @@ import org.broadinstitute.sting.gatk.walkers.Walker;
|
|||
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.PrintStream;
|
||||
import java.util.*;
|
||||
|
||||
import net.sf.picard.filter.SamRecordFilter;
|
||||
|
|
@ -68,22 +70,44 @@ public abstract class CommandLineExecutable extends CommandLineProgram {
|
|||
*
|
||||
* @return the return code to exit the program with
|
||||
*/
|
||||
protected int execute() {
|
||||
GATKResult = executeGATK();
|
||||
return 0;
|
||||
}
|
||||
|
||||
protected Object executeGATK() {
|
||||
protected int execute() throws Exception {
|
||||
Walker<?,?> mWalker = GATKEngine.getWalkerByName(getAnalysisName());
|
||||
Collection<SamRecordFilter> filters = GATKEngine.createFiltersForWalker(getArgumentCollection(),mWalker);
|
||||
|
||||
// load the arguments into the walker / filters.
|
||||
loadArgumentsIntoObject(mWalker);
|
||||
for(SamRecordFilter filter: filters)
|
||||
for (SamRecordFilter filter: filters)
|
||||
loadArgumentsIntoObject(filter);
|
||||
|
||||
// set the analysis name in the argument collection
|
||||
return GATKEngine.execute(getArgumentCollection(), mWalker, filters);
|
||||
try {
|
||||
GATKResult = GATKEngine.execute(getArgumentCollection(), mWalker, filters);
|
||||
generateGATKRunReport(mWalker);
|
||||
} catch ( Exception e ) {
|
||||
generateGATKRunReport(mWalker, e);
|
||||
throw e;
|
||||
}
|
||||
|
||||
// always return 0
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* generate an error log
|
||||
* @param e the exception, can be null if no exception occurred
|
||||
*/
|
||||
private void generateGATKRunReport(Walker<?,?> mWalker, Exception e) {
|
||||
if ( getArgumentCollection().phoneHomeType != GATKRunReport.PhoneHomeOption.NO_ET ) {
|
||||
GATKRunReport report = new GATKRunReport(mWalker, e, GATKEngine);
|
||||
if ( getArgumentCollection().phoneHomeType == GATKRunReport.PhoneHomeOption.STDOUT )
|
||||
report.postReport(System.out);
|
||||
else
|
||||
report.postReport();
|
||||
}
|
||||
}
|
||||
|
||||
private void generateGATKRunReport(Walker<?,?> mWalker) {
|
||||
generateGATKRunReport(mWalker, null);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -65,18 +65,6 @@ public class CommandLineGATK extends CommandLineExecutable {
|
|||
getAdditionalHelp());
|
||||
}
|
||||
|
||||
/**
|
||||
* generate an error log, given the stream to write to and the exception that was thrown
|
||||
*
|
||||
* @param stream the output stream
|
||||
* @param e the exception
|
||||
*/
|
||||
@Override
|
||||
public void generateErrorLog(PrintStream stream, Exception e) {
|
||||
GATKErrorReport report = new GATKErrorReport(e, this.argCollection);
|
||||
report.reportToStream(stream);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getAnalysisName() {
|
||||
return analysisName;
|
||||
|
|
@ -114,19 +102,24 @@ public class CommandLineGATK extends CommandLineExecutable {
|
|||
* @return The application header.
|
||||
*/
|
||||
public static List<String> createApplicationHeader() {
|
||||
ResourceBundle headerInfo = TextFormattingUtils.loadResourceBundle("StingText");
|
||||
|
||||
String versionNumber = headerInfo.containsKey("org.broadinstitute.sting.gatk.version") ? headerInfo.getString("org.broadinstitute.sting.gatk.version") : "<unknown>";
|
||||
String timestamp = headerInfo.containsKey("build.timestamp") ? headerInfo.getString("build.timestamp") : "<unknown>";
|
||||
|
||||
List<String> header = new ArrayList<String>();
|
||||
header.add(String.format("The Genome Analysis Toolkit (GATK) v%s, Compiled %s",versionNumber,timestamp));
|
||||
header.add(String.format("The Genome Analysis Toolkit (GATK) v%s, Compiled %s",getVersionNumber(), getBuildTime()));
|
||||
header.add("Copyright (c) 2009 The Broad Institute");
|
||||
header.add("Please view our documentation at http://www.broadinstitute.org/gsa/wiki");
|
||||
header.add("For support, please view our support site at http://getsatisfaction.com/gsa");
|
||||
return header;
|
||||
}
|
||||
|
||||
public static String getVersionNumber() {
|
||||
ResourceBundle headerInfo = TextFormattingUtils.loadResourceBundle("StingText");
|
||||
return headerInfo.containsKey("org.broadinstitute.sting.gatk.version") ? headerInfo.getString("org.broadinstitute.sting.gatk.version") : "<unknown>";
|
||||
}
|
||||
|
||||
public static String getBuildTime() {
|
||||
ResourceBundle headerInfo = TextFormattingUtils.loadResourceBundle("StingText");
|
||||
return headerInfo.containsKey("build.timestamp") ? headerInfo.getString("build.timestamp") : "<unknown>";
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves additional information about GATK walkers.
|
||||
* the code in HelpFormatter and supply it as a helper to this method.
|
||||
|
|
|
|||
|
|
@ -1,103 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2010 The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
|
||||
* THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.gatk;
|
||||
|
||||
import org.broadinstitute.sting.gatk.CommandLineGATK;
|
||||
import org.broadinstitute.sting.gatk.arguments.GATKArgumentCollection;
|
||||
import org.broadinstitute.sting.utils.StingException;
|
||||
import org.simpleframework.xml.Element;
|
||||
import org.simpleframework. xml.ElementList;
|
||||
import org.simpleframework.xml.Serializer;
|
||||
import org.simpleframework.xml.core.Persister;
|
||||
import org.simpleframework.xml.stream.Format;
|
||||
import org.simpleframework.xml.stream.HyphenStyle;
|
||||
|
||||
import java.io.PrintStream;
|
||||
import java.text.DateFormat;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
/**
|
||||
* @author aaron
|
||||
* <p/>
|
||||
* Class GATKErrorReport
|
||||
* <p/>
|
||||
* A basic description of what went wrong during a run of the GATK,
|
||||
* what the parameters were, etc.
|
||||
*/
|
||||
public class GATKErrorReport {
|
||||
// the listing of the fields is somewhat important; this is the order that the simple XML will output them
|
||||
@ElementList(required = true, name = "gatk_header_Information")
|
||||
private static List<String> mGATKHeader;
|
||||
|
||||
@Element(required = false, name = "exception")
|
||||
private final ExceptionToXML mException;
|
||||
|
||||
@Element(required = true, name = "date_time")
|
||||
private final String mDateTime;
|
||||
|
||||
@Element(required = true, name = "argument_collection")
|
||||
private final GATKArgumentCollection mCollection;
|
||||
|
||||
@Element(required = true, name = "working_directory")
|
||||
private static String currentPath;
|
||||
|
||||
private static final DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH.mm.ss");
|
||||
|
||||
static {
|
||||
GATKErrorReport.mGATKHeader = CommandLineGATK.createApplicationHeader();
|
||||
currentPath = System.getProperty("user.dir");
|
||||
}
|
||||
|
||||
public GATKErrorReport(Exception e, GATKArgumentCollection collection) {
|
||||
this.mCollection = collection;
|
||||
this.mException = new ExceptionToXML(e);
|
||||
java.util.Date date = new java.util.Date();
|
||||
mDateTime = dateFormat.format(date);
|
||||
}
|
||||
|
||||
public void reportToStream(PrintStream stream) {
|
||||
Serializer serializer = new Persister(new Format(new HyphenStyle()));
|
||||
try {
|
||||
serializer.write(this, stream);
|
||||
} catch (Exception e) {
|
||||
throw new StingException("Failed to marshal the data to the file " + stream, e);
|
||||
}
|
||||
}
|
||||
|
||||
class ExceptionToXML {
|
||||
@ElementList(required = false)
|
||||
final List<String> exceptionDetails = new ArrayList<String>();
|
||||
|
||||
public ExceptionToXML(Exception e) {
|
||||
for (StackTraceElement element : e.getStackTrace()) {
|
||||
exceptionDetails.add(element.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -123,6 +123,8 @@ public class GenomeAnalysisEngine {
|
|||
*/
|
||||
private final FilterManager filterManager;
|
||||
|
||||
private Date startTime = null; // the start time for execution
|
||||
|
||||
/**
|
||||
* our constructor, where all the work is done
|
||||
* <p/>
|
||||
|
|
@ -149,6 +151,7 @@ public class GenomeAnalysisEngine {
|
|||
public Object execute(GATKArgumentCollection args, Walker<?, ?> my_walker, Collection<SamRecordFilter> filters) {
|
||||
//HeapSizeMonitor monitor = new HeapSizeMonitor();
|
||||
//monitor.start();
|
||||
startTime = new java.util.Date();
|
||||
|
||||
// validate our parameters
|
||||
if (args == null) {
|
||||
|
|
@ -188,6 +191,14 @@ public class GenomeAnalysisEngine {
|
|||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the start time when the execute() function was last called
|
||||
*/
|
||||
public Date getStartTime() {
|
||||
return startTime;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Setup the intervals to be processed
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@
|
|||
package org.broadinstitute.sting.gatk.arguments;
|
||||
|
||||
import net.sf.samtools.SAMFileReader;
|
||||
import org.broadinstitute.sting.gatk.phonehome.GATKRunReport;
|
||||
import org.broadinstitute.sting.utils.StingException;
|
||||
import org.broadinstitute.sting.utils.interval.IntervalMergingRule;
|
||||
import org.broadinstitute.sting.commandline.Argument;
|
||||
|
|
@ -74,6 +75,10 @@ public class GATKArgumentCollection {
|
|||
@Argument(fullName = "read_buffer_size", shortName = "rbs", doc="Number of reads per SAM file to buffer in memory", required = false)
|
||||
public Integer readBufferSize = null;
|
||||
|
||||
@Element(required = false)
|
||||
@Argument(fullName = "phone_home", shortName = "et", doc="What kind of GATK run report should we generate? Standard is the default, can be verbose or NO_ET so nothing is posted to the run repository", required = false)
|
||||
public GATKRunReport.PhoneHomeOption phoneHomeType = GATKRunReport.PhoneHomeOption.NO_ET;
|
||||
|
||||
@ElementList(required = false)
|
||||
@Argument(fullName = "read_filter", shortName = "rf", doc = "Specify filtration criteria to apply to each read individually.", required = false)
|
||||
public List<String> readFilters = new ArrayList<String>();
|
||||
|
|
@ -337,6 +342,9 @@ public class GATKArgumentCollection {
|
|||
(other.RODToInterval != null && !other.RODToInterval.equals(RODToInterval))) {
|
||||
return false;
|
||||
}
|
||||
if (other.phoneHomeType != this.phoneHomeType) {
|
||||
return false;
|
||||
}
|
||||
if (BTIMergeRule != other.BTIMergeRule)
|
||||
return false;
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,252 @@
|
|||
/*
|
||||
* Copyright (c) 2010, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.gatk.phonehome;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.broadinstitute.sting.commandline.CommandLineUtils;
|
||||
import org.broadinstitute.sting.gatk.CommandLineGATK;
|
||||
import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
|
||||
import org.broadinstitute.sting.gatk.arguments.GATKArgumentCollection;
|
||||
import org.broadinstitute.sting.gatk.walkers.Walker;
|
||||
import org.broadinstitute.sting.utils.StingException;
|
||||
import org.broadinstitute.sting.utils.Utils;
|
||||
import org.simpleframework.xml.Element;
|
||||
import org.simpleframework.xml.ElementList;
|
||||
import org.simpleframework.xml.Serializer;
|
||||
import org.simpleframework.xml.core.Persister;
|
||||
import org.simpleframework.xml.stream.Format;
|
||||
import org.simpleframework.xml.stream.HyphenStyle;
|
||||
|
||||
import java.io.*;
|
||||
import java.text.DateFormat;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
|
||||
/**
|
||||
* @author depristo
|
||||
* <p/>
|
||||
* Class GATKRunReport
|
||||
* <p/>
|
||||
* A detailed description of a GATK run, and error if applicable
|
||||
*/
|
||||
public class GATKRunReport {
|
||||
/**
|
||||
* The root file system directory where we keep common report data
|
||||
*/
|
||||
private static File REPORT_DIR = new File("/humgen/gsa-hpprojects/GATK/reports");
|
||||
|
||||
/**
|
||||
* The full path to the direct where submitted (and uncharacterized) report files are written
|
||||
*/
|
||||
private static File REPORT_SUBMIT_DIR = new File(REPORT_DIR.getAbsolutePath() + "/submitted");
|
||||
|
||||
/**
|
||||
* Full path to the sentinel file that controls whether reports are written out. If this file doesn't
|
||||
* exist, no long will be written
|
||||
*/
|
||||
private static File REPORT_SENTINEL = new File(REPORT_DIR.getAbsolutePath() + "/ENABLE");
|
||||
|
||||
/**
|
||||
* our log
|
||||
*/
|
||||
protected static Logger logger = Logger.getLogger(GATKRunReport.class);
|
||||
|
||||
|
||||
// the listing of the fields is somewhat important; this is the order that the simple XML will output them
|
||||
@ElementList(required = true, name = "gatk_header_Information")
|
||||
private static List<String> mGATKHeader;
|
||||
|
||||
@Element(required = false, name = "exception")
|
||||
private final ExceptionToXML mException;
|
||||
|
||||
@Element(required = true, name = "argument_collection")
|
||||
private final GATKArgumentCollection mCollection;
|
||||
|
||||
@Element(required = true, name = "working_directory")
|
||||
private static String currentPath;
|
||||
|
||||
@Element(required = true, name = "start_time")
|
||||
private static String startTime;
|
||||
|
||||
@Element(required = true, name = "end_time")
|
||||
private static String endTime;
|
||||
|
||||
@Element(required = true, name = "run_time")
|
||||
private static long runTime;
|
||||
|
||||
@Element(required = true, name = "command_line")
|
||||
private static String cmdLine;
|
||||
|
||||
@Element(required = true, name = "walker_name")
|
||||
private static String walkerName;
|
||||
|
||||
@Element(required = true, name = "svn_version")
|
||||
private static String svnVersion;
|
||||
|
||||
@Element(required = true, name = "memory")
|
||||
private static long memory;
|
||||
|
||||
@Element(required = true, name = "java_tmp_directory")
|
||||
private static String tmpDir;
|
||||
|
||||
@Element(required = true, name = "domain_name")
|
||||
private static String domainName;
|
||||
|
||||
@Element(required = true, name = "user_name")
|
||||
private static String userName;
|
||||
|
||||
@Element(required = true, name = "host_name")
|
||||
private static String hostName;
|
||||
|
||||
@Element(required = true, name = "java")
|
||||
private static String java;
|
||||
|
||||
@Element(required = true, name = "machine")
|
||||
private static String machine;
|
||||
|
||||
@Element(required = true, name = "iterations")
|
||||
private static long nIterations;
|
||||
|
||||
@Element(required = true, name = "reads")
|
||||
private static long nReads;
|
||||
|
||||
@Element(required = true, name = "read_metrics")
|
||||
private static String readMetrics;
|
||||
|
||||
// not done
|
||||
//- walker-specific args
|
||||
//+ md5 all filenames
|
||||
//- size of filenames
|
||||
//- # reads/loci
|
||||
//- free memory on machine
|
||||
|
||||
public enum PhoneHomeOption {
|
||||
NO_ET,
|
||||
STANDARD,
|
||||
STDOUT
|
||||
}
|
||||
|
||||
private static final DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH.mm.ss");
|
||||
|
||||
static {
|
||||
GATKRunReport.mGATKHeader = CommandLineGATK.createApplicationHeader();
|
||||
currentPath = System.getProperty("user.dir");
|
||||
}
|
||||
|
||||
/** Create a new RunReport and population all of the fields with values from the walker and engine */
|
||||
public GATKRunReport(Walker<?,?> walker, Exception e, GenomeAnalysisEngine engine) {
|
||||
this.mCollection = engine.getArguments();
|
||||
this.mException = e == null ? null : new ExceptionToXML(e);
|
||||
|
||||
startTime = dateFormat.format(engine.getStartTime()); // fixme
|
||||
Date end = new java.util.Date();
|
||||
endTime = dateFormat.format(end);
|
||||
runTime = (end.getTime() - engine.getStartTime().getTime()) / 1000L; // difference in seconds
|
||||
|
||||
cmdLine = CommandLineUtils.createApproximateCommandLineArgumentString(engine, walker);
|
||||
walkerName = engine.getWalkerName(walker.getClass());
|
||||
svnVersion = CommandLineGATK.getVersionNumber();
|
||||
nIterations = engine.getCumulativeMetrics().getNumIterations();
|
||||
nReads = engine.getCumulativeMetrics().getNumReadsSeen();
|
||||
readMetrics = engine.getCumulativeMetrics().toString();
|
||||
memory = Runtime.getRuntime().totalMemory();
|
||||
tmpDir = System.getProperty("java.io.tmpdir");
|
||||
domainName = "Need to figure this out";
|
||||
hostName = "Need to figure this out";
|
||||
userName = System.getProperty("user.name");
|
||||
java = Utils.join("-", Arrays.asList(System.getProperty("java.vendor"), System.getProperty("java.version")));
|
||||
machine = Utils.join("-", Arrays.asList(System.getProperty("os.name"), System.getProperty("os.arch")));
|
||||
}
|
||||
|
||||
public void postReport(OutputStream stream) {
|
||||
Serializer serializer = new Persister(new Format(new HyphenStyle()));
|
||||
try {
|
||||
serializer.write(this, stream);
|
||||
//throw new StingException("test");
|
||||
} catch (Exception e) {
|
||||
throw new StingException("Failed to marshal the data to the file " + stream, e);
|
||||
}
|
||||
}
|
||||
|
||||
public void postReport(File destination) throws FileNotFoundException, IOException {
|
||||
BufferedOutputStream out =
|
||||
new BufferedOutputStream(
|
||||
new GZIPOutputStream(
|
||||
new FileOutputStream(destination)));
|
||||
try {
|
||||
postReport(out);
|
||||
} finally {
|
||||
out.close();
|
||||
}
|
||||
}
|
||||
|
||||
public void postReport() {
|
||||
try {
|
||||
if ( sentinelExists() ) {
|
||||
String filename = org.apache.commons.lang.RandomStringUtils.randomAlphanumeric(32) + ".report.xml.gz";
|
||||
File file = new File(REPORT_SUBMIT_DIR, filename);
|
||||
postReport(file);
|
||||
logger.info("Wrote report to " + file);
|
||||
} else {
|
||||
logger.info("Not writing report: sentinel " + REPORT_SENTINEL + " doesn't exist");
|
||||
}
|
||||
} catch ( Exception e ) {
|
||||
// we catch everything, and no matter what eat the error
|
||||
logger.warn("Received error while posting report. GATK continuing on but no run report has been generated because: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private boolean sentinelExists() {
|
||||
return REPORT_SENTINEL.exists();
|
||||
}
|
||||
|
||||
class ExceptionToXML {
|
||||
@Element(required = false, name = "message")
|
||||
String message = null;
|
||||
|
||||
@ElementList(required = false, name = "stacktrace")
|
||||
final List<String> stackTrace = new ArrayList<String>();
|
||||
|
||||
@Element(required = false, name = "cause")
|
||||
ExceptionToXML cause = null;
|
||||
|
||||
public ExceptionToXML(Throwable e) {
|
||||
message = e.getMessage();
|
||||
for (StackTraceElement element : e.getStackTrace()) {
|
||||
stackTrace.add(element.toString());
|
||||
}
|
||||
|
||||
if ( e.getCause() != null ) {
|
||||
//message += " because " + e.getCause().getMessage();
|
||||
cause = new ExceptionToXML(e.getCause());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -91,7 +91,7 @@ public abstract class TraversalEngine<M,T,WalkerType extends Walker<M,T>,Provide
|
|||
for ( Map.Entry<Class, Long> countsByFilter: cumulativeMetrics.getCountsByFilter().entrySet())
|
||||
nSkippedReads += countsByFilter.getValue();
|
||||
|
||||
logger.info(String.format("Total runtime %.2f secs, %.2f min, %.2f hours%n", elapsed, elapsed / 60, elapsed / 3600));
|
||||
logger.info(String.format("Total runtime %.2f secs, %.2f min, %.2f hours", elapsed, elapsed / 60, elapsed / 3600));
|
||||
logger.info(String.format("%d reads were filtered out during traversal out of %d total (%.2f%%)",
|
||||
nSkippedReads,
|
||||
cumulativeMetrics.getNumReadsSeen(),
|
||||
|
|
|
|||
|
|
@ -146,9 +146,7 @@ public class VariantAnnotator extends RodWalker<Integer, Integer> {
|
|||
hInfo.add(line);
|
||||
}
|
||||
if ( !NO_VCF_HEADER_LINE ) {
|
||||
Set<Object> args = new HashSet<Object>();
|
||||
args.add(this);
|
||||
hInfo.add(new VCFHeaderLine("VariantAnnotator", "\"" + CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), args, getClass()) + "\""));
|
||||
hInfo.add(new VCFHeaderLine("VariantAnnotator", "\"" + CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), this) + "\""));
|
||||
}
|
||||
|
||||
VCFHeader vcfHeader = new VCFHeader(hInfo, samples);
|
||||
|
|
|
|||
|
|
@ -112,9 +112,7 @@ public class VariantFiltrationWalker extends RodWalker<Integer, Integer> {
|
|||
}
|
||||
|
||||
if ( !NO_VCF_HEADER_LINE ) {
|
||||
Set<Object> args = new HashSet<Object>();
|
||||
args.add(this);
|
||||
hInfo.add(new VCFHeaderLine("VariantFiltration", "\"" + CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), args, getClass()) + "\""));
|
||||
hInfo.add(new VCFHeaderLine("VariantFiltration", "\"" + CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), this) + "\""));
|
||||
}
|
||||
|
||||
writer.writeHeader(new VCFHeader(hInfo, new TreeSet<String>(vc.getSampleNames())));
|
||||
|
|
|
|||
|
|
@ -156,9 +156,7 @@ public class UnifiedGenotyper extends LocusWalker<VariantCallContext, UnifiedGen
|
|||
|
||||
// all of the arguments from the argument collection
|
||||
if ( !NO_VCF_HEADER_LINE ) {
|
||||
Set<Object> args = new HashSet<Object>();
|
||||
args.add(UAC);
|
||||
headerInfo.add(new VCFHeaderLine("UnifiedGenotyper", "\"" + CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), args, getClass()) + "\""));
|
||||
headerInfo.add(new VCFHeaderLine("UnifiedGenotyper", "\"" + CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), this) + "\""));
|
||||
}
|
||||
|
||||
return headerInfo;
|
||||
|
|
|
|||
|
|
@ -224,9 +224,7 @@ public class IndelRealigner extends ReadWalker<Integer, Integer> {
|
|||
final SAMProgramRecord programRecord = new SAMProgramRecord(PROGRAM_RECORD_NAME);
|
||||
final ResourceBundle headerInfo = TextFormattingUtils.loadResourceBundle("StingText");
|
||||
programRecord.setProgramVersion(headerInfo.getString("org.broadinstitute.sting.gatk.version"));
|
||||
HashSet<Object> args = new HashSet<Object>();
|
||||
args.add(this);
|
||||
programRecord.setCommandLine(CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), args, getClass()));
|
||||
programRecord.setCommandLine(CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), this));
|
||||
|
||||
List<SAMProgramRecord> oldRecords = header.getProgramRecords();
|
||||
List<SAMProgramRecord> newRecords = new ArrayList<SAMProgramRecord>(oldRecords.size()+1);
|
||||
|
|
|
|||
|
|
@ -262,10 +262,8 @@ public class TableRecalibrationWalker extends ReadWalker<SAMRecord, SAMFileWrite
|
|||
final ResourceBundle headerInfo = TextFormattingUtils.loadResourceBundle("StingText");
|
||||
programRecord.setProgramVersion( headerInfo.getString("org.broadinstitute.sting.gatk.version") );
|
||||
|
||||
HashSet<Object> args = new HashSet<Object>();
|
||||
args.add(RAC);
|
||||
StringBuffer sb = new StringBuffer();
|
||||
sb.append(CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), args, getClass()));
|
||||
sb.append(CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), this));
|
||||
sb.append(" Covariates=[");
|
||||
for( Covariate cov : requestedCovariates ) {
|
||||
sb.append(cov.getClass().getSimpleName());
|
||||
|
|
|
|||
|
|
@ -64,10 +64,13 @@ public class VariantsToTable extends RodWalker<Integer, Integer> {
|
|||
public int MAX_RECORDS = -1;
|
||||
int nRecords = 0;
|
||||
|
||||
@Argument(fullName="allowMultiAllelic", shortName="AMA", doc="If provided, we will not require the site to be biallelic", required=false)
|
||||
public boolean allowMultiAllelic = false;
|
||||
|
||||
private List<String> fieldsToTake;
|
||||
|
||||
public void initialize() {
|
||||
fieldsToTake = Arrays.asList(FIELDS.toUpperCase().split(","));
|
||||
fieldsToTake = Arrays.asList(FIELDS.split(","));
|
||||
|
||||
out.println(Utils.join("\t", fieldsToTake));
|
||||
}
|
||||
|
|
@ -111,21 +114,23 @@ public class VariantsToTable extends RodWalker<Integer, Integer> {
|
|||
if ( ++nRecords < MAX_RECORDS || MAX_RECORDS == -1 ) {
|
||||
Collection<VariantContext> vcs = tracker.getAllVariantContexts(ref, context.getLocation());
|
||||
for ( VariantContext vc : vcs) {
|
||||
List<String> vals = new ArrayList<String>();
|
||||
if ( allowMultiAllelic || vc.isBiallelic() ) {
|
||||
List<String> vals = new ArrayList<String>();
|
||||
|
||||
for ( String field : fieldsToTake ) {
|
||||
String val = "UNK";
|
||||
for ( String field : fieldsToTake ) {
|
||||
String val = "UNK";
|
||||
|
||||
if ( getters.containsKey(field) ) {
|
||||
val = getters.get(field).get(vc);
|
||||
} else if ( vc.hasAttribute(field) ) {
|
||||
val = vc.getAttributeAsString(field);
|
||||
if ( getters.containsKey(field) ) {
|
||||
val = getters.get(field).get(vc);
|
||||
} else if ( vc.hasAttribute(field) ) {
|
||||
val = vc.getAttributeAsString(field);
|
||||
}
|
||||
|
||||
vals.add(val);
|
||||
}
|
||||
|
||||
vals.add(val);
|
||||
out.println(Utils.join("\t", vals));
|
||||
}
|
||||
|
||||
out.println(Utils.join("\t", vals));
|
||||
}
|
||||
|
||||
return 1;
|
||||
|
|
|
|||
|
|
@ -273,9 +273,11 @@ public class WalkerTest extends BaseTest {
|
|||
System.out.println(String.format("Executing test %s with GATK arguments: %s", name, Utils.join(" ",command)));
|
||||
|
||||
// add the logging level to each of the integration test commands
|
||||
String[] cmd2 = Arrays.copyOf(command, command.length + 2);
|
||||
String[] cmd2 = Arrays.copyOf(command, command.length + 4);
|
||||
cmd2[command.length] = "-l";
|
||||
cmd2[command.length+1] = "WARN";
|
||||
cmd2[command.length+2] = "-et";
|
||||
cmd2[command.length+3] = "NO_ET";
|
||||
|
||||
// run the executable
|
||||
CommandLineExecutable.start(instance, cmd2);
|
||||
|
|
|
|||
Loading…
Reference in New Issue