From 4c23b1fe9cc2a89ba4872f5122e449af97fd2447 Mon Sep 17 00:00:00 2001 From: hanna Date: Wed, 27 Oct 2010 19:44:55 +0000 Subject: [PATCH] Get rid of the static cache of ArgumentTypeDescriptors by making them an integral part of the parsing engine. Hugely lowers our memory footprint in integrationtests, but not yet enough to run Mark's new parallelized VariantEvalIntegrationTests. git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@4585 348d0f76-0448-11de-a6fe-93d51630548a --- .../sting/commandline/ArgumentMatches.java | 4 +- .../sting/commandline/ArgumentSource.java | 40 ++------------- .../commandline/ArgumentTypeDescriptor.java | 51 ++++++------------- .../sting/commandline/CommandLineProgram.java | 2 +- .../sting/commandline/CommandLineUtils.java | 8 +-- .../sting/commandline/ParsingEngine.java | 51 ++++++++++++++----- .../gatk/AbstractGenomeAnalysisEngine.java | 17 +++++++ .../sting/gatk/CommandLineExecutable.java | 1 + .../sting/gatk/GenomeAnalysisEngine.java | 7 ++- .../OutputStreamArgumentTypeDescriptor.java | 6 +-- .../SAMFileWriterArgumentTypeDescriptor.java | 4 +- .../VCFWriterArgumentTypeDescriptor.java | 10 ++-- .../sting/gatk/io/stubs/VCFWriterStub.java | 13 +++-- .../sting/gatk/phonehome/GATKRunReport.java | 2 +- .../indels/IndelGenotyperV2Walker.java | 2 +- .../gatk/walkers/indels/IndelRealigner.java | 2 +- .../TableRecalibrationWalker.java | 2 +- .../GenerateVariantClustersWalker.java | 2 +- .../gatk/ArgumentDefinitionField.java | 4 +- .../gatk/GATKExtensionsGenerator.java | 8 +-- .../extensions/gatk/ReadFilterField.java | 5 +- .../commandline/ParsingEngineUnitTest.java | 1 + .../sting/gatk/WalkerManagerUnitTest.java | 2 + .../sting/queue/QCommandLine.scala | 3 ++ .../sting/queue/function/QFunction.scala | 4 +- .../ScalaCompoundArgumentTypeDescriptor.scala | 2 +- 26 files changed, 129 insertions(+), 124 deletions(-) diff --git a/java/src/org/broadinstitute/sting/commandline/ArgumentMatches.java b/java/src/org/broadinstitute/sting/commandline/ArgumentMatches.java index 9be2862fe..52d3b8232 100755 --- a/java/src/org/broadinstitute/sting/commandline/ArgumentMatches.java +++ b/java/src/org/broadinstitute/sting/commandline/ArgumentMatches.java @@ -111,8 +111,8 @@ public class ArgumentMatches implements Iterable { * @return List of all matches. */ - ArgumentMatches findMatches( ArgumentSource argumentSource ) { - List sourceDefinitions = ArgumentTypeDescriptor.create(argumentSource.field.getType()).createArgumentDefinitions(argumentSource); + ArgumentMatches findMatches(ParsingEngine parsingEngine, ArgumentSource argumentSource) { + List sourceDefinitions = parsingEngine.selectBestTypeDescriptor(argumentSource.field.getType()).createArgumentDefinitions(argumentSource); ArgumentMatches matches = new ArgumentMatches(); for( ArgumentMatch argumentMatch: getUniqueMatches() ) { diff --git a/java/src/org/broadinstitute/sting/commandline/ArgumentSource.java b/java/src/org/broadinstitute/sting/commandline/ArgumentSource.java index 937c9549a..6b8653093 100644 --- a/java/src/org/broadinstitute/sting/commandline/ArgumentSource.java +++ b/java/src/org/broadinstitute/sting/commandline/ArgumentSource.java @@ -55,30 +55,13 @@ public class ArgumentSource { */ private final ArgumentTypeDescriptor typeDescriptor; - /** - * Create a new command-line argument target. - * @param field Field containing the argument. Field must be annotated with 'Input' or 'Output'. - */ - public ArgumentSource( Field field ) { - this(new Field[0], field); - } - - /** - * Create a new command-line argument target. - * @param parentFields Parent fields containing the the field. Field must be annotated with 'ArgumentCollection'. - * @param field Field containing the argument. Field must be annotated with 'Input' or 'Output'. - */ - public ArgumentSource( Field[] parentFields, Field field ) { - this(parentFields,field,ArgumentTypeDescriptor.create(field.getType())); - } - /** * Create a new command-line argument target. * @param parentFields Parent fields containing the the field. Field must be annotated with 'ArgumentCollection'. * @param field Field containing the argument. Field must be annotated with 'Input' or 'Output'. * @param typeDescriptor custom type descriptor to use when parsing. */ - private ArgumentSource( Field[] parentFields, Field field, ArgumentTypeDescriptor typeDescriptor) { + protected ArgumentSource( Field[] parentFields, Field field, ArgumentTypeDescriptor typeDescriptor) { this.parentFields = parentFields; this.field = field; this.typeDescriptor = typeDescriptor; @@ -127,22 +110,6 @@ public class ArgumentSource { return typeDescriptor.createArgumentDefinitions( this ); } - /** - * This argument type descriptor wants to override any default value the user might have specified. - * @return True if this descriptor wants to override any default the user specified. False otherwise. - */ - public boolean overridesDefault() { - return typeDescriptor.createsTypeDefault(this,field.getType()); - } - - /** - * Provides the default value for the command-line argument. - * @return Default value to load into the object. - */ - public Object createDefault() { - return typeDescriptor.createTypeDefault(this,field.getType()); - } - /** * Parses the specified value based on the specified type. * @param values String representation of all values passed. @@ -203,13 +170,14 @@ public class ArgumentSource { /** * Builds out a new type descriptor for the given dependent argument as a function * of the containing object. + * @param parsingEngine the parsing engine to use when building out this custom type descriptor. * @param containingObject The containing object. * @return An argument type descriptor for the custom derivative field. */ - public MultiplexArgumentTypeDescriptor createDependentTypeDescriptor(Object containingObject) { + public MultiplexArgumentTypeDescriptor createDependentTypeDescriptor(ParsingEngine parsingEngine,Object containingObject) { if(!isDependent()) throw new ReviewedStingException("Field " + field.getName() + " is independent; no dependent type descriptor can be derived."); - return ((MultiplexArgumentTypeDescriptor)typeDescriptor).createCustomTypeDescriptor(this,containingObject); + return ((MultiplexArgumentTypeDescriptor)typeDescriptor).createCustomTypeDescriptor(parsingEngine,this,containingObject); } /** diff --git a/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java b/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java index c50a20f4a..f0fe0bebf 100644 --- a/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java +++ b/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java @@ -51,34 +51,14 @@ public abstract class ArgumentTypeDescriptor { * our log, which we want to capture anything from org.broadinstitute.sting */ protected static Logger logger = Logger.getLogger(ArgumentTypeDescriptor.class); - - /** - * Class reference to the different types of descriptors that the create method can create. - * The type of set used must be ordered (but not necessarily sorted). - */ - private static Set descriptors = new LinkedHashSet( Arrays.asList(new SimpleArgumentTypeDescriptor(), - new CompoundArgumentTypeDescriptor(), - new MultiplexArgumentTypeDescriptor()) ); - - /** - * Adds new, user defined descriptors to the head of the descriptor list. - * @param argumentTypeDescriptors New descriptors to add. List can be empty, but should not be null. - */ - public static void addDescriptors( Collection argumentTypeDescriptors ) { - // We care about ordering; newly added descriptors should have priority over stock descriptors. - // Enforce this by creating a new *ordered* set, adding the new descriptors, then adding the old descriptors. - Set allDescriptors = new LinkedHashSet(); - allDescriptors.addAll( argumentTypeDescriptors ); - allDescriptors.addAll( descriptors ); - descriptors = allDescriptors; - } /** * Fetch the given descriptor from the descriptor repository. + * @param descriptors the descriptors from which to select a good match. * @param type Class for which to specify a descriptor. * @return descriptor for the given type. */ - public static ArgumentTypeDescriptor create( Class type ) { + public static ArgumentTypeDescriptor selectBest( Collection descriptors, Class type ) { for( ArgumentTypeDescriptor descriptor: descriptors ) { if( descriptor.supports(type) ) return descriptor; @@ -98,14 +78,15 @@ public abstract class ArgumentTypeDescriptor { * @param source Source of the command-line argument. * @return True to throw in a type specific default. False otherwise. */ - public boolean createsTypeDefault(ArgumentSource source,Class type) { return false; } + public boolean createsTypeDefault(ArgumentSource source) { return false; } /** * Generates a default for the given type. + * @param parsingEngine the parsing engine used to validate this argument type descriptor. * @param source Source of the command-line argument. * @return A default value for the given type. */ - public Object createTypeDefault(ArgumentSource source,Class type) { throw new UnsupportedOperationException("Unable to create default for type " + getClass()); } + public Object createTypeDefault(ParsingEngine parsingEngine,ArgumentSource source) { throw new UnsupportedOperationException("Unable to create default for type " + getClass()); } /** * Given the given argument source and attributes, synthesize argument definitions for command-line arguments. @@ -153,7 +134,7 @@ public abstract class ArgumentTypeDescriptor { ArgumentDefinition.getFullName(argumentAnnotation, source.field.getName()), ArgumentDefinition.getShortName(argumentAnnotation), ArgumentDefinition.getDoc(argumentAnnotation), - source.isRequired() && !source.overridesDefault() && !source.isFlag() && !source.isDeprecated(), + source.isRequired() && !createsTypeDefault(source) && !source.isFlag() && !source.isDeprecated(), source.isFlag(), source.isMultiValued(), source.isHidden(), @@ -412,7 +393,7 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor { } componentType = getCollectionComponentType( source.field ); - ArgumentTypeDescriptor componentArgumentParser = ArgumentTypeDescriptor.create( componentType ); + ArgumentTypeDescriptor componentArgumentParser = parsingEngine.selectBestTypeDescriptor(componentType); Collection collection; try { @@ -439,7 +420,7 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor { } else if( type.isArray() ) { componentType = type.getComponentType(); - ArgumentTypeDescriptor componentArgumentParser = ArgumentTypeDescriptor.create( componentType ); + ArgumentTypeDescriptor componentArgumentParser = parsingEngine.selectBestTypeDescriptor(componentType); // Assemble a collection of individual values used in this computation. Collection values = new ArrayList(); @@ -516,24 +497,24 @@ class MultiplexArgumentTypeDescriptor extends ArgumentTypeDescriptor { } @Override - public boolean createsTypeDefault(ArgumentSource source,Class type) { + public boolean createsTypeDefault(ArgumentSource source) { // Multiplexing always creates a type default. return true; } @Override - public Object createTypeDefault(ArgumentSource source,Class type) { + public Object createTypeDefault(ParsingEngine parsingEngine,ArgumentSource source) { if(multiplexer == null || multiplexedIds == null) throw new ReviewedStingException("No multiplexed ids available"); Map multiplexedMapping = new HashMap(); Class componentType = getCollectionComponentType(source.field); - ArgumentTypeDescriptor componentTypeDescriptor = ArgumentTypeDescriptor.create(componentType); + ArgumentTypeDescriptor componentTypeDescriptor = parsingEngine.selectBestTypeDescriptor(componentType); for(Object id: multiplexedIds) { Object value = null; - if(componentTypeDescriptor.createsTypeDefault(source,componentType)) - value = componentTypeDescriptor.createTypeDefault(source,componentType); + if(componentTypeDescriptor.createsTypeDefault(source)) + value = componentTypeDescriptor.createTypeDefault(parsingEngine,source); multiplexedMapping.put(id,value); } return multiplexedMapping; @@ -551,7 +532,7 @@ class MultiplexArgumentTypeDescriptor extends ArgumentTypeDescriptor { for(Object id: multiplexedIds) { - Object value = ArgumentTypeDescriptor.create(componentType).parse(parsingEngine,source,componentType,matches.transform(multiplexer,id)); + Object value = parsingEngine.selectBestTypeDescriptor(componentType).parse(parsingEngine,source,componentType,matches.transform(multiplexer,id)); multiplexedMapping.put(id,value); } @@ -560,10 +541,10 @@ class MultiplexArgumentTypeDescriptor extends ArgumentTypeDescriptor { return multiplexedMapping; } - public MultiplexArgumentTypeDescriptor createCustomTypeDescriptor(ArgumentSource dependentArgument,Object containingObject) { + public MultiplexArgumentTypeDescriptor createCustomTypeDescriptor(ParsingEngine parsingEngine,ArgumentSource dependentArgument,Object containingObject) { String[] sourceFields = dependentArgument.field.getAnnotation(Multiplex.class).arguments(); - List allSources = ParsingEngine.extractArgumentSources(containingObject.getClass()); + List allSources = parsingEngine.extractArgumentSources(containingObject.getClass()); Class[] sourceTypes = new Class[sourceFields.length]; Object[] sourceValues = new Object[sourceFields.length]; int currentField = 0; diff --git a/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java b/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java index a4af97520..a83d173bc 100644 --- a/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java +++ b/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java @@ -37,7 +37,7 @@ import java.util.*; public abstract class CommandLineProgram { /** The command-line program and the arguments it returned. */ - private ParsingEngine parser = null; + protected ParsingEngine parser = null; /** our log, which we want to capture anything from org.broadinstitute.sting */ private static Logger logger = Logger.getRootLogger(); diff --git a/java/src/org/broadinstitute/sting/commandline/CommandLineUtils.java b/java/src/org/broadinstitute/sting/commandline/CommandLineUtils.java index 417c5d0e7..254e549e0 100644 --- a/java/src/org/broadinstitute/sting/commandline/CommandLineUtils.java +++ b/java/src/org/broadinstitute/sting/commandline/CommandLineUtils.java @@ -48,11 +48,11 @@ public class CommandLineUtils { * @return A key-value mapping of argument full names to argument values. Produces best string representation * possible given the information available. */ - public static Map getApproximateCommandLineArguments(Object... argumentProviders) { + public static Map getApproximateCommandLineArguments(ParsingEngine parsingEngine, Object... argumentProviders) { Map commandLineArguments = new LinkedHashMap(); for(Object argumentProvider: argumentProviders) { - Map argBindings = ParsingEngine.extractArgumentBindings(argumentProvider); + Map argBindings = parsingEngine.extractArgumentBindings(argumentProvider); for(Map.Entry elt: argBindings.entrySet()) { Object argumentValue = elt.getValue(); String argumentValueString = argumentValue != null ? argumentValue.toString() : null; @@ -72,8 +72,8 @@ public class CommandLineUtils { * @param argumentProviders Argument providers to inspect. * @return A string representing the given command-line arguments. */ - public static String createApproximateCommandLineArgumentString(Object... argumentProviders) { - Map commandLineArgs = getApproximateCommandLineArguments(argumentProviders); + public static String createApproximateCommandLineArgumentString(ParsingEngine parsingEngine, Object... argumentProviders) { + Map commandLineArgs = getApproximateCommandLineArguments(parsingEngine,argumentProviders); StringBuffer sb = new StringBuffer(); boolean first = true; diff --git a/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java b/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java index 5c48db670..b2c20e511 100755 --- a/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java +++ b/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java @@ -64,6 +64,16 @@ public class ParsingEngine { */ private List parsingMethods = new ArrayList(); + /** + * Class reference to the different types of descriptors that the create method can create. + * The type of set used must be ordered (but not necessarily sorted). + */ + private static final Set STANDARD_ARGUMENT_TYPE_DESCRIPTORS = new LinkedHashSet( Arrays.asList(new SimpleArgumentTypeDescriptor(), + new CompoundArgumentTypeDescriptor(), + new MultiplexArgumentTypeDescriptor()) ); + + private Set argumentTypeDescriptors = new LinkedHashSet(); + /** * our log, which we want to capture anything from org.broadinstitute.sting */ @@ -75,9 +85,10 @@ public class ParsingEngine { parsingMethods.add( ParsingMethod.FullNameParsingMethod ); parsingMethods.add( ParsingMethod.ShortNameParsingMethod ); - // Null check for unit tests. Perhaps we should mock up an empty CLP? - if( clp != null ) - ArgumentTypeDescriptor.addDescriptors( clp.getArgumentTypeDescriptors() ); + // Order matters here! Make sure the clp's new type descriptors go in before the original type descriptors. + if(clp != null) + argumentTypeDescriptors.addAll(clp.getArgumentTypeDescriptors()); + argumentTypeDescriptors.addAll(STANDARD_ARGUMENT_TYPE_DESCRIPTORS); } /** @@ -259,7 +270,7 @@ public class ParsingEngine { List dependentArguments = new ArrayList(); for( ArgumentSource argumentSource: argumentSources ) { - if(argumentSource.isDeprecated() && argumentMatches.findMatches(argumentSource).size() > 0) + if(argumentSource.isDeprecated() && argumentMatches.findMatches(this,argumentSource).size() > 0) notifyDeprecatedCommandLineArgument(argumentSource); // If this argument source depends on other command-line arguments, skip it and make a note to process it later. @@ -267,13 +278,13 @@ public class ParsingEngine { dependentArguments.add(argumentSource); continue; } - loadValueIntoObject( argumentSource, object, argumentMatches.findMatches(argumentSource) ); + loadValueIntoObject( argumentSource, object, argumentMatches.findMatches(this,argumentSource) ); } for(ArgumentSource dependentArgument: dependentArguments) { - MultiplexArgumentTypeDescriptor dependentDescriptor = dependentArgument.createDependentTypeDescriptor(object); + MultiplexArgumentTypeDescriptor dependentDescriptor = dependentArgument.createDependentTypeDescriptor(this,object); ArgumentSource dependentSource = dependentArgument.copyWithCustomTypeDescriptor(dependentDescriptor); - loadValueIntoObject(dependentSource,object,argumentMatches.findMatches(dependentSource)); + loadValueIntoObject(dependentSource,object,argumentMatches.findMatches(this,dependentSource)); } } @@ -306,8 +317,10 @@ public class ParsingEngine { * @param instance Object into which to inject the value. The target might be in a container within the instance. */ private void loadValueIntoObject( ArgumentSource source, Object instance, ArgumentMatches argumentMatches ) { + ArgumentTypeDescriptor typeDescriptor = selectBestTypeDescriptor(source.field.getType()); + // Nothing to load - if( argumentMatches.size() == 0 && !(source.overridesDefault() && source.isRequired())) + if( argumentMatches.size() == 0 && !(typeDescriptor.createsTypeDefault(source) && source.isRequired())) return; // Target instance into which to inject the value. @@ -318,7 +331,7 @@ public class ParsingEngine { throw new ReviewedStingException("Internal command-line parser error: unable to find a home for argument matches " + argumentMatches); for( Object target: targets ) { - Object value = (argumentMatches.size() != 0) ? source.parse(this,argumentMatches) : source.createDefault(); + Object value = (argumentMatches.size() != 0) ? source.parse(this,argumentMatches) : typeDescriptor.createTypeDefault(this,source); JVMUtils.setFieldValue(source.field,target,value); } @@ -357,18 +370,28 @@ public class ParsingEngine { * @param sourceClass class to act as sources for other arguments. * @return A list of sources associated with this object and its aggregated objects. */ - public static List extractArgumentSources(Class sourceClass) { + public List extractArgumentSources(Class sourceClass) { return extractArgumentSources(sourceClass, new Field[0]); } - private static List extractArgumentSources(Class sourceClass, Field[] parentFields) { + /** + * Fetch the best command-line argument descriptor for the given class. + * @param type Class for which to specify a descriptor. + * @return descriptor for the given type. + */ + public ArgumentTypeDescriptor selectBestTypeDescriptor(Class type) { + return ArgumentTypeDescriptor.selectBest(argumentTypeDescriptors,type); + } + + + private List extractArgumentSources(Class sourceClass, Field[] parentFields) { // now simply call into the truly general routine extract argument bindings but with a null // object so bindings aren't computed Map bindings = extractArgumentBindings(null, sourceClass, parentFields); return new ArrayList(bindings.keySet()); } - public static Map extractArgumentBindings(Object obj) { + public Map extractArgumentBindings(Object obj) { if ( obj == null ) throw new IllegalArgumentException("Incoming object cannot be null"); return extractArgumentBindings(obj, obj.getClass(), new Field[0]); } @@ -380,7 +403,7 @@ public class ParsingEngine { * @param parentFields Parent Fields * @return A map of sources associated with this object and its aggregated objects and bindings to their bindings values */ - private static Map extractArgumentBindings(Object obj, Class sourceClass, Field[] parentFields) { + private Map extractArgumentBindings(Object obj, Class sourceClass, Field[] parentFields) { Map bindings = new LinkedHashMap(); while( sourceClass != null ) { @@ -388,7 +411,7 @@ public class ParsingEngine { for( Field field: fields ) { if( ArgumentTypeDescriptor.isArgumentAnnotationPresent(field) ) { Object val = obj != null ? JVMUtils.getFieldValue(field, obj) : null; - bindings.put( new ArgumentSource(parentFields, field), val ); + bindings.put( new ArgumentSource(parentFields, field, selectBestTypeDescriptor(field.getType())), val ); } if( field.isAnnotationPresent(ArgumentCollection.class) ) { Object val = obj != null ? JVMUtils.getFieldValue(field, obj) : null; diff --git a/java/src/org/broadinstitute/sting/gatk/AbstractGenomeAnalysisEngine.java b/java/src/org/broadinstitute/sting/gatk/AbstractGenomeAnalysisEngine.java index 899a5ce2d..53a39116c 100755 --- a/java/src/org/broadinstitute/sting/gatk/AbstractGenomeAnalysisEngine.java +++ b/java/src/org/broadinstitute/sting/gatk/AbstractGenomeAnalysisEngine.java @@ -31,6 +31,8 @@ import net.sf.samtools.*; import org.apache.log4j.Logger; import org.broad.tribble.util.variantcontext.VariantContext; import org.broadinstitute.sting.commandline.ArgumentSource; +import org.broadinstitute.sting.commandline.CommandLineUtils; +import org.broadinstitute.sting.commandline.ParsingEngine; import org.broadinstitute.sting.gatk.arguments.GATKArgumentCollection; import org.broadinstitute.sting.gatk.arguments.ValidationExclusion; import org.broadinstitute.sting.gatk.datasources.sample.Sample; @@ -62,6 +64,11 @@ import java.util.*; */ public abstract class AbstractGenomeAnalysisEngine { + /** + * The GATK command-line argument parsing code. + */ + private ParsingEngine parsingEngine; + /** * Accessor for sharded read data. */ @@ -126,6 +133,10 @@ public abstract class AbstractGenomeAnalysisEngine { private Date startTime = null; // the start time for execution + public void setParser(ParsingEngine parsingEngine) { + this.parsingEngine = parsingEngine; + } + /** * Actually run the engine. * @return the value of this traversal. @@ -863,5 +874,11 @@ public abstract class AbstractGenomeAnalysisEngine { return sampleDataSource.subContextFromSampleProperty(context, key, value); } + public Map getApproximateCommandLineArguments(Object... argumentProviders) { + return CommandLineUtils.getApproximateCommandLineArguments(parsingEngine,argumentProviders); + } + public String createApproximateCommandLineArgumentString(Object... argumentProviders) { + return CommandLineUtils.createApproximateCommandLineArgumentString(parsingEngine,argumentProviders); + } } diff --git a/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java b/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java index 3a26d970a..36e42adbe 100644 --- a/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java +++ b/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java @@ -69,6 +69,7 @@ public abstract class CommandLineExecutable extends CommandLineProgram { * @return the return code to exit the program with */ protected int execute() throws Exception { + engine.setParser(parser); argumentSources.add(this); Walker walker = engine.getWalkerByName(getAnalysisName()); diff --git a/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java b/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java index fd1fdc4ea..76ec2ed35 100755 --- a/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java +++ b/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010 The Broad Institute + * Copyright (c) 2010, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -12,15 +12,14 @@ * * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software. - * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR - * THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. */ package org.broadinstitute.sting.gatk; diff --git a/java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamArgumentTypeDescriptor.java b/java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamArgumentTypeDescriptor.java index 553d655b5..fae37426d 100644 --- a/java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamArgumentTypeDescriptor.java +++ b/java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamArgumentTypeDescriptor.java @@ -63,15 +63,15 @@ public class OutputStreamArgumentTypeDescriptor extends ArgumentTypeDescriptor { } @Override - public boolean createsTypeDefault(ArgumentSource source,Class type) { + public boolean createsTypeDefault(ArgumentSource source) { return true; } @Override - public Object createTypeDefault(ArgumentSource source,Class type) { + public Object createTypeDefault(ParsingEngine parsingEngine,ArgumentSource source) { OutputStreamStub stub = new OutputStreamStub(defaultOutputStream); engine.addOutput(stub); - return createInstanceOfClass(type,stub); + return createInstanceOfClass(source.field.getType(),stub); } @Override diff --git a/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java b/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java index c7b110b96..54ed9e1d2 100644 --- a/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java +++ b/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java @@ -82,12 +82,12 @@ public class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor } @Override - public boolean createsTypeDefault(ArgumentSource source,Class type) { + public boolean createsTypeDefault(ArgumentSource source) { return true; } @Override - public Object createTypeDefault(ArgumentSource source,Class type) { + public Object createTypeDefault(ParsingEngine parsingEngine,ArgumentSource source) { SAMFileWriterStub stub = new SAMFileWriterStub(engine,defaultOutputStream); engine.addOutput(stub); return stub; diff --git a/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java b/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java index f3f00c1d4..4b3a3326e 100644 --- a/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java +++ b/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java @@ -103,13 +103,13 @@ public class VCFWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor { * @return true always. */ @Override - public boolean createsTypeDefault(ArgumentSource source,Class type) { + public boolean createsTypeDefault(ArgumentSource source) { return true; } @Override - public Object createTypeDefault(ArgumentSource source,Class type) { - VCFWriterStub stub = new VCFWriterStub(defaultOutputStream, false, argumentSources, false); + public Object createTypeDefault(ParsingEngine parsingEngine,ArgumentSource source) { + VCFWriterStub stub = new VCFWriterStub(engine, defaultOutputStream, false, argumentSources, false); engine.addOutput(stub); return stub; } @@ -133,8 +133,8 @@ public class VCFWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor { boolean skipWritingHeader = argumentIsPresent(createNoHeaderArgumentDefinition(),matches); // Create a stub for the given object. - VCFWriterStub stub = (writerFile != null) ? new VCFWriterStub(writerFile, compress, argumentSources, skipWritingHeader) - : new VCFWriterStub(defaultOutputStream, compress, argumentSources, skipWritingHeader); + VCFWriterStub stub = (writerFile != null) ? new VCFWriterStub(engine, writerFile, compress, argumentSources, skipWritingHeader) + : new VCFWriterStub(engine, defaultOutputStream, compress, argumentSources, skipWritingHeader); // WARNING: Side effects required by engine! parsingEngine.addTags(stub,getArgumentTags(matches)); diff --git a/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterStub.java b/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterStub.java index 1a1981300..b4be112b9 100755 --- a/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterStub.java +++ b/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterStub.java @@ -38,6 +38,7 @@ import org.broad.tribble.vcf.VCFHeaderLine; import org.broad.tribble.vcf.VCFWriter; import org.broadinstitute.sting.commandline.CommandLineUtils; import org.broadinstitute.sting.gatk.CommandLineExecutable; +import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.gatk.io.OutputTracker; import org.broadinstitute.sting.utils.classloader.JVMUtils; @@ -48,6 +49,10 @@ import org.broadinstitute.sting.utils.classloader.JVMUtils; * @version 0.1 */ public class VCFWriterStub implements Stub, VCFWriter { + /** + * The engine, central to the GATK's processing. + */ + private final GenomeAnalysisEngine engine; /** * The file that this stub should write to. Should be mutually @@ -93,7 +98,8 @@ public class VCFWriterStub implements Stub, VCFWriter { * @param genotypeFile file to (ultimately) create. * @param isCompressed should we compress the output stream? */ - public VCFWriterStub(File genotypeFile, boolean isCompressed, Collection argumentSources, boolean skipWritingHeader) { + public VCFWriterStub(GenomeAnalysisEngine engine, File genotypeFile, boolean isCompressed, Collection argumentSources, boolean skipWritingHeader) { + this.engine = engine; this.genotypeFile = genotypeFile; this.genotypeStream = null; this.isCompressed = isCompressed; @@ -106,7 +112,8 @@ public class VCFWriterStub implements Stub, VCFWriter { * @param genotypeStream stream to (ultimately) write. * @param isCompressed should we compress the output stream? */ - public VCFWriterStub(OutputStream genotypeStream, boolean isCompressed, Collection argumentSources, boolean skipWritingHeader) { + public VCFWriterStub(GenomeAnalysisEngine engine, OutputStream genotypeStream, boolean isCompressed, Collection argumentSources, boolean skipWritingHeader) { + this.engine = engine; this.genotypeFile = null; this.genotypeStream = new PrintStream(genotypeStream); this.isCompressed = isCompressed; @@ -199,6 +206,6 @@ public class VCFWriterStub implements Stub, VCFWriter { */ private VCFHeaderLine getCommandLineArgumentHeaderLine() { CommandLineExecutable executable = JVMUtils.getObjectOfType(argumentSources,CommandLineExecutable.class); - return new VCFHeaderLine(executable.getAnalysisName(), "\"" + CommandLineUtils.createApproximateCommandLineArgumentString(argumentSources.toArray()) + "\""); + return new VCFHeaderLine(executable.getAnalysisName(), "\"" + engine.createApproximateCommandLineArgumentString(argumentSources.toArray()) + "\""); } } \ No newline at end of file diff --git a/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java b/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java index 24f5a75d6..223242966 100644 --- a/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java +++ b/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java @@ -180,7 +180,7 @@ public class GATKRunReport { // what did we run? id = org.apache.commons.lang.RandomStringUtils.randomAlphanumeric(32); try { - cmdLine = CommandLineUtils.createApproximateCommandLineArgumentString(engine, walker); + cmdLine = engine.createApproximateCommandLineArgumentString(engine, walker); } catch (Exception ignore) { } this.mCollection = engine.getArguments(); diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/indels/IndelGenotyperV2Walker.java b/java/src/org/broadinstitute/sting/gatk/walkers/indels/IndelGenotyperV2Walker.java index 29163c3ca..a253cb792 100644 --- a/java/src/org/broadinstitute/sting/gatk/walkers/indels/IndelGenotyperV2Walker.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/indels/IndelGenotyperV2Walker.java @@ -211,7 +211,7 @@ public class IndelGenotyperV2Walker extends ReadWalker { Set args = new HashSet(); args.add(this); args.addAll(getToolkit().getFilters()); - Map commandLineArgs = CommandLineUtils.getApproximateCommandLineArguments(args); + Map commandLineArgs = getToolkit().getApproximateCommandLineArguments(args); for ( Map.Entry commandLineArg : commandLineArgs.entrySet() ) headerInfo.add(new VCFHeaderLine(String.format("IGv2_%s", commandLineArg.getKey()), commandLineArg.getValue())); // also, the list of input bams diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/indels/IndelRealigner.java b/java/src/org/broadinstitute/sting/gatk/walkers/indels/IndelRealigner.java index 143435af3..b4e7a7bdd 100755 --- a/java/src/org/broadinstitute/sting/gatk/walkers/indels/IndelRealigner.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/indels/IndelRealigner.java @@ -228,7 +228,7 @@ public class IndelRealigner extends ReadWalker { final SAMProgramRecord programRecord = new SAMProgramRecord(PROGRAM_RECORD_NAME); final ResourceBundle headerInfo = TextFormattingUtils.loadResourceBundle("StingText"); programRecord.setProgramVersion(headerInfo.getString("org.broadinstitute.sting.gatk.version")); - programRecord.setCommandLine(CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), this)); + programRecord.setCommandLine(getToolkit().createApproximateCommandLineArgumentString(getToolkit(), this)); List oldRecords = header.getProgramRecords(); List newRecords = new ArrayList(oldRecords.size()+1); diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java b/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java index 96ed496f6..2e6bf8543 100644 --- a/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java @@ -263,7 +263,7 @@ public class TableRecalibrationWalker extends ReadWalker getArgumentFields(Class classType) { + public static List getArgumentFields(ParsingEngine parsingEngine,Class classType) { List argumentFields = new ArrayList(); - for (ArgumentSource argumentSource: ParsingEngine.extractArgumentSources(classType)) + for (ArgumentSource argumentSource: parsingEngine.extractArgumentSources(classType)) if (!argumentSource.isDeprecated()) for (ArgumentDefinition argumentDefinition: argumentSource.createArgumentDefinitions()) argumentFields.addAll(getArgumentFields(argumentDefinition)); diff --git a/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java b/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java index 6c77cc7f5..53736586b 100644 --- a/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java +++ b/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java @@ -105,7 +105,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram { String clpClassName = clpManager.getName(clp); writeClass("org.broadinstitute.sting.queue.function.JarCommandLineFunction", COMMANDLINE_PACKAGE_NAME, clpClassName, - false, "", ArgumentDefinitionField.getArgumentFields(clp)); + false, "", ArgumentDefinitionField.getArgumentFields(parser,clp)); if (clp == CommandLineGATK.class) { for (Entry>> walkersByPackage: walkerManager.getWalkerNamesByPackage(false).entrySet()) { @@ -113,9 +113,9 @@ public class GATKExtensionsGenerator extends CommandLineProgram { String walkerName = walkerManager.getName(walkerType); List argumentFields = new ArrayList(); - argumentFields.addAll(ArgumentDefinitionField.getArgumentFields(walkerType)); + argumentFields.addAll(ArgumentDefinitionField.getArgumentFields(parser,walkerType)); argumentFields.addAll(RodBindField.getRodArguments(walkerType, trackBuilder)); - argumentFields.addAll(ReadFilterField.getFilterArguments(walkerType)); + argumentFields.addAll(ReadFilterField.getFilterArguments(parser,walkerType)); String constructor = String.format("analysisName = \"%1$s\"%nanalysis_type = \"%1$s\"%n", walkerName); String scatterClass = getScatterClass(walkerType); @@ -134,7 +134,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram { for (Class filter: filterManager.getValues()) { String filterName = filterManager.getName(filter); - writeFilter(FILTER_PACKAGE_NAME, filterName, ArgumentDefinitionField.getArgumentFields(filter)); + writeFilter(FILTER_PACKAGE_NAME, filterName, ArgumentDefinitionField.getArgumentFields(new ParsingEngine(null),filter)); } return 0; diff --git a/java/src/org/broadinstitute/sting/queue/extensions/gatk/ReadFilterField.java b/java/src/org/broadinstitute/sting/queue/extensions/gatk/ReadFilterField.java index 23eacceae..712e1292c 100644 --- a/java/src/org/broadinstitute/sting/queue/extensions/gatk/ReadFilterField.java +++ b/java/src/org/broadinstitute/sting/queue/extensions/gatk/ReadFilterField.java @@ -25,6 +25,7 @@ package org.broadinstitute.sting.queue.extensions.gatk; import net.sf.picard.filter.SamRecordFilter; +import org.broadinstitute.sting.commandline.ParsingEngine; import org.broadinstitute.sting.gatk.WalkerManager; import org.broadinstitute.sting.gatk.walkers.Walker; @@ -37,10 +38,10 @@ public class ReadFilterField { * @param walkerClass the class of the walker * @return the list of argument fields */ - public static List getFilterArguments(Class walkerClass) { + public static List getFilterArguments(ParsingEngine parser, Class walkerClass) { List argumentFields = new ArrayList(); for(Class filter: WalkerManager.getReadFilterTypes(walkerClass)) - argumentFields.addAll(ArgumentDefinitionField.getArgumentFields(filter)); + argumentFields.addAll(ArgumentDefinitionField.getArgumentFields(parser,filter)); return argumentFields; } } diff --git a/java/test/org/broadinstitute/sting/commandline/ParsingEngineUnitTest.java b/java/test/org/broadinstitute/sting/commandline/ParsingEngineUnitTest.java index 833e6cce4..ad114c63d 100755 --- a/java/test/org/broadinstitute/sting/commandline/ParsingEngineUnitTest.java +++ b/java/test/org/broadinstitute/sting/commandline/ParsingEngineUnitTest.java @@ -31,6 +31,7 @@ import org.junit.Test; import org.junit.Before; import org.junit.Assert; +import java.util.Collections; import java.util.List; import java.util.EnumSet; /** diff --git a/java/test/org/broadinstitute/sting/gatk/WalkerManagerUnitTest.java b/java/test/org/broadinstitute/sting/gatk/WalkerManagerUnitTest.java index 5196d7451..d8d946cde 100644 --- a/java/test/org/broadinstitute/sting/gatk/WalkerManagerUnitTest.java +++ b/java/test/org/broadinstitute/sting/gatk/WalkerManagerUnitTest.java @@ -25,6 +25,7 @@ package org.broadinstitute.sting.gatk; import org.broadinstitute.sting.commandline.Hidden; +import org.broadinstitute.sting.gatk.walkers.Requires; import org.broadinstitute.sting.gatk.walkers.Walker; import org.broadinstitute.sting.gatk.walkers.qc.CountLociWalker; import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException; @@ -62,6 +63,7 @@ public class WalkerManagerUnitTest { } @Hidden +@Requires(value={}) class UninstantiableWalker extends Walker { // Private constructor will generate uninstantiable message private UninstantiableWalker() {} diff --git a/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala b/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala index 35674afa3..50e4890c1 100755 --- a/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala +++ b/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala @@ -1,5 +1,6 @@ package org.broadinstitute.sting.queue +import function.QFunction import java.io.File import java.util.Arrays import org.broadinstitute.sting.commandline._ @@ -17,6 +18,8 @@ class QCommandLine extends CommandLineProgram with Logging { @ArgumentCollection private val settings = new QGraphSettings + QFunction.parsingEngine = new ParsingEngine(this) + /** * Takes the QScripts passed in, runs their script() methods, retrieves their generated * functions, and then builds and runs a QGraph based on the dependencies. diff --git a/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala index 2b15780f1..fbdcc74e4 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala @@ -410,6 +410,8 @@ object QFunction { /** Job index counter for this run of Queue. */ private var jobIndex = 0 + var parsingEngine: ParsingEngine = _ + /** * Returns the next job name using the prefix. * @param prefix Prefix of the job name. @@ -426,7 +428,7 @@ object QFunction { */ private class ClassFields(clazz: Class[_]) { /** The complete list of fields on this CommandLineFunction. */ - val functionFields: List[ArgumentSource] = ParsingEngine.extractArgumentSources(clazz).toList + val functionFields: List[ArgumentSource] = parsingEngine.extractArgumentSources(clazz).toList /** The @Input fields on this CommandLineFunction. */ val inputFields = functionFields.filter(source => ReflectionUtils.hasAnnotation(source.field, classOf[Input])) /** The @Output fields on this CommandLineFunction. */ diff --git a/scala/src/org/broadinstitute/sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala b/scala/src/org/broadinstitute/sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala index 274535297..40a296022 100644 --- a/scala/src/org/broadinstitute/sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala +++ b/scala/src/org/broadinstitute/sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala @@ -44,7 +44,7 @@ class ScalaCompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor { */ def parse(parsingEngine: ParsingEngine, source: ArgumentSource, classType: Class[_], argumentMatches: ArgumentMatches) = { val componentType = ReflectionUtils.getCollectionType(source.field) - val componentArgumentParser = ArgumentTypeDescriptor.create(componentType) + val componentArgumentParser = parsingEngine.selectBestTypeDescriptor(componentType) if (classOf[List[_]].isAssignableFrom(classType)) { var list = List.empty[Any]