zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Get rid of the static cache of ArgumentTypeDescriptors by making them an integral part of the 

parsing engine.  Hugely lowers our memory footprint in integrationtests, but not yet enough to 
run Mark's new parallelized VariantEvalIntegrationTests.


git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@4585 348d0f76-0448-11de-a6fe-93d51630548a
This commit is contained in:
hanna 2010-10-27 19:44:55 +00:00
parent 1d68b28bbd
commit 4c23b1fe9c
26 changed files with 129 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
java/src/org/broadinstitute/sting/commandline/ArgumentMatches.java
View File

@ -111,8 +111,8 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
* @return List of all matches.
*/
ArgumentMatches findMatches( ArgumentSource argumentSource ) {
List<ArgumentDefinition> sourceDefinitions = ArgumentTypeDescriptor.create(argumentSource.field.getType()).createArgumentDefinitions(argumentSource);
ArgumentMatches findMatches(ParsingEngine parsingEngine, ArgumentSource argumentSource) {
List<ArgumentDefinition> sourceDefinitions = parsingEngine.selectBestTypeDescriptor(argumentSource.field.getType()).createArgumentDefinitions(argumentSource);
ArgumentMatches matches = new ArgumentMatches();
for( ArgumentMatch argumentMatch: getUniqueMatches() ) {

40
java/src/org/broadinstitute/sting/commandline/ArgumentSource.java
View File

@ -55,30 +55,13 @@ public class ArgumentSource {
*/
private final ArgumentTypeDescriptor typeDescriptor;
/**
* Create a new command-line argument target.
* @param field Field containing the argument. Field must be annotated with 'Input' or 'Output'.
*/
public ArgumentSource( Field field ) {
this(new Field[0], field);
}
/**
* Create a new command-line argument target.
* @param parentFields Parent fields containing the the field. Field must be annotated with 'ArgumentCollection'.
* @param field Field containing the argument. Field must be annotated with 'Input' or 'Output'.
*/
public ArgumentSource( Field[] parentFields, Field field ) {
this(parentFields,field,ArgumentTypeDescriptor.create(field.getType()));
}
/**
* Create a new command-line argument target.
* @param parentFields Parent fields containing the the field. Field must be annotated with 'ArgumentCollection'.
* @param field Field containing the argument. Field must be annotated with 'Input' or 'Output'.
* @param typeDescriptor custom type descriptor to use when parsing.
*/
private ArgumentSource( Field[] parentFields, Field field, ArgumentTypeDescriptor typeDescriptor) {
protected ArgumentSource( Field[] parentFields, Field field, ArgumentTypeDescriptor typeDescriptor) {
this.parentFields = parentFields;
this.field = field;
this.typeDescriptor = typeDescriptor;
@ -127,22 +110,6 @@ public class ArgumentSource {
return typeDescriptor.createArgumentDefinitions( this );
}
/**
* This argument type descriptor wants to override any default value the user might have specified.
* @return True if this descriptor wants to override any default the user specified. False otherwise.
*/
public boolean overridesDefault() {
return typeDescriptor.createsTypeDefault(this,field.getType());
}
/**
* Provides the default value for the command-line argument.
* @return Default value to load into the object.
*/
public Object createDefault() {
return typeDescriptor.createTypeDefault(this,field.getType());
}
/**
* Parses the specified value based on the specified type.
* @param values String representation of all values passed.
@ -203,13 +170,14 @@ public class ArgumentSource {
/**
* Builds out a new type descriptor for the given dependent argument as a function
* of the containing object.
* @param parsingEngine the parsing engine to use when building out this custom type descriptor.
* @param containingObject The containing object.
* @return An argument type descriptor for the custom derivative field.
*/
public MultiplexArgumentTypeDescriptor createDependentTypeDescriptor(Object containingObject) {
public MultiplexArgumentTypeDescriptor createDependentTypeDescriptor(ParsingEngine parsingEngine,Object containingObject) {
if(!isDependent())
throw new ReviewedStingException("Field " + field.getName() + " is independent; no dependent type descriptor can be derived.");
return ((MultiplexArgumentTypeDescriptor)typeDescriptor).createCustomTypeDescriptor(this,containingObject);
return ((MultiplexArgumentTypeDescriptor)typeDescriptor).createCustomTypeDescriptor(parsingEngine,this,containingObject);
}
/**

51
java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java
View File

@ -51,34 +51,14 @@ public abstract class ArgumentTypeDescriptor {
* our log, which we want to capture anything from org.broadinstitute.sting
*/
protected static Logger logger = Logger.getLogger(ArgumentTypeDescriptor.class);
/**
* Class reference to the different types of descriptors that the create method can create.
* The type of set used must be ordered (but not necessarily sorted).
*/
private static Set<ArgumentTypeDescriptor> descriptors = new LinkedHashSet<ArgumentTypeDescriptor>( Arrays.asList(new SimpleArgumentTypeDescriptor(),
new CompoundArgumentTypeDescriptor(),
new MultiplexArgumentTypeDescriptor()) );
/**
* Adds new, user defined descriptors to the head of the descriptor list.
* @param argumentTypeDescriptors New descriptors to add. List can be empty, but should not be null.
*/
public static void addDescriptors( Collection<ArgumentTypeDescriptor> argumentTypeDescriptors ) {
// We care about ordering; newly added descriptors should have priority over stock descriptors.
// Enforce this by creating a new *ordered* set, adding the new descriptors, then adding the old descriptors.
Set<ArgumentTypeDescriptor> allDescriptors = new LinkedHashSet<ArgumentTypeDescriptor>();
allDescriptors.addAll( argumentTypeDescriptors );
allDescriptors.addAll( descriptors );
descriptors = allDescriptors;
}
/**
* Fetch the given descriptor from the descriptor repository.
* @param descriptors the descriptors from which to select a good match.
* @param type Class for which to specify a descriptor.
* @return descriptor for the given type.
*/
public static ArgumentTypeDescriptor create( Class type ) {
public static ArgumentTypeDescriptor selectBest( Collection<ArgumentTypeDescriptor> descriptors, Class type ) {
for( ArgumentTypeDescriptor descriptor: descriptors ) {
if( descriptor.supports(type) )
return descriptor;
@ -98,14 +78,15 @@ public abstract class ArgumentTypeDescriptor {
* @param source Source of the command-line argument.
* @return True to throw in a type specific default. False otherwise.
*/
public boolean createsTypeDefault(ArgumentSource source,Class type) { return false; }
public boolean createsTypeDefault(ArgumentSource source) { return false; }
/**
* Generates a default for the given type.
* @param parsingEngine the parsing engine used to validate this argument type descriptor.
* @param source Source of the command-line argument.
* @return A default value for the given type.
*/
public Object createTypeDefault(ArgumentSource source,Class type) { throw new UnsupportedOperationException("Unable to create default for type " + getClass()); }
public Object createTypeDefault(ParsingEngine parsingEngine,ArgumentSource source) { throw new UnsupportedOperationException("Unable to create default for type " + getClass()); }
/**
* Given the given argument source and attributes, synthesize argument definitions for command-line arguments.
@ -153,7 +134,7 @@ public abstract class ArgumentTypeDescriptor {
ArgumentDefinition.getFullName(argumentAnnotation, source.field.getName()),
ArgumentDefinition.getShortName(argumentAnnotation),
ArgumentDefinition.getDoc(argumentAnnotation),
source.isRequired() && !source.overridesDefault() && !source.isFlag() && !source.isDeprecated(),
source.isRequired() && !createsTypeDefault(source) && !source.isFlag() && !source.isDeprecated(),
source.isFlag(),
source.isMultiValued(),
source.isHidden(),
@ -412,7 +393,7 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor {
}
componentType = getCollectionComponentType( source.field );
ArgumentTypeDescriptor componentArgumentParser = ArgumentTypeDescriptor.create( componentType );
ArgumentTypeDescriptor componentArgumentParser = parsingEngine.selectBestTypeDescriptor(componentType);
Collection collection;
try {
@ -439,7 +420,7 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor {
}
else if( type.isArray() ) {
componentType = type.getComponentType();
ArgumentTypeDescriptor componentArgumentParser = ArgumentTypeDescriptor.create( componentType );
ArgumentTypeDescriptor componentArgumentParser = parsingEngine.selectBestTypeDescriptor(componentType);
// Assemble a collection of individual values used in this computation.
Collection<ArgumentMatch> values = new ArrayList<ArgumentMatch>();
@ -516,24 +497,24 @@ class MultiplexArgumentTypeDescriptor extends ArgumentTypeDescriptor {
}
@Override
public boolean createsTypeDefault(ArgumentSource source,Class type) {
public boolean createsTypeDefault(ArgumentSource source) {
// Multiplexing always creates a type default.
return true;
}
@Override
public Object createTypeDefault(ArgumentSource source,Class type) {
public Object createTypeDefault(ParsingEngine parsingEngine,ArgumentSource source) {
if(multiplexer == null || multiplexedIds == null)
throw new ReviewedStingException("No multiplexed ids available");
Map<Object,Object> multiplexedMapping = new HashMap<Object,Object>();
Class componentType = getCollectionComponentType(source.field);
ArgumentTypeDescriptor componentTypeDescriptor = ArgumentTypeDescriptor.create(componentType);
ArgumentTypeDescriptor componentTypeDescriptor = parsingEngine.selectBestTypeDescriptor(componentType);
for(Object id: multiplexedIds) {
Object value = null;
if(componentTypeDescriptor.createsTypeDefault(source,componentType))
value = componentTypeDescriptor.createTypeDefault(source,componentType);
if(componentTypeDescriptor.createsTypeDefault(source))
value = componentTypeDescriptor.createTypeDefault(parsingEngine,source);
multiplexedMapping.put(id,value);
}
return multiplexedMapping;
@ -551,7 +532,7 @@ class MultiplexArgumentTypeDescriptor extends ArgumentTypeDescriptor {
for(Object id: multiplexedIds) {
Object value = ArgumentTypeDescriptor.create(componentType).parse(parsingEngine,source,componentType,matches.transform(multiplexer,id));
Object value = parsingEngine.selectBestTypeDescriptor(componentType).parse(parsingEngine,source,componentType,matches.transform(multiplexer,id));
multiplexedMapping.put(id,value);
}
@ -560,10 +541,10 @@ class MultiplexArgumentTypeDescriptor extends ArgumentTypeDescriptor {
return multiplexedMapping;
}
public MultiplexArgumentTypeDescriptor createCustomTypeDescriptor(ArgumentSource dependentArgument,Object containingObject) {
public MultiplexArgumentTypeDescriptor createCustomTypeDescriptor(ParsingEngine parsingEngine,ArgumentSource dependentArgument,Object containingObject) {
String[] sourceFields = dependentArgument.field.getAnnotation(Multiplex.class).arguments();
List<ArgumentSource> allSources = ParsingEngine.extractArgumentSources(containingObject.getClass());
List<ArgumentSource> allSources = parsingEngine.extractArgumentSources(containingObject.getClass());
Class[] sourceTypes = new Class[sourceFields.length];
Object[] sourceValues = new Object[sourceFields.length];
int currentField = 0;

2
java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java
View File

@ -37,7 +37,7 @@ import java.util.*;
public abstract class CommandLineProgram {
/** The command-line program and the arguments it returned. */
private ParsingEngine parser = null;
protected ParsingEngine parser = null;
/** our log, which we want to capture anything from org.broadinstitute.sting */
private static Logger logger = Logger.getRootLogger();

8
java/src/org/broadinstitute/sting/commandline/CommandLineUtils.java
View File

@ -48,11 +48,11 @@ public class CommandLineUtils {
* @return A key-value mapping of argument full names to argument values. Produces best string representation
* possible given the information available.
*/
public static Map<String,String> getApproximateCommandLineArguments(Object... argumentProviders) {
public static Map<String,String> getApproximateCommandLineArguments(ParsingEngine parsingEngine, Object... argumentProviders) {
Map<String,String> commandLineArguments = new LinkedHashMap<String,String>();
for(Object argumentProvider: argumentProviders) {
Map<ArgumentSource, Object> argBindings = ParsingEngine.extractArgumentBindings(argumentProvider);
Map<ArgumentSource, Object> argBindings = parsingEngine.extractArgumentBindings(argumentProvider);
for(Map.Entry<ArgumentSource, Object> elt: argBindings.entrySet()) {
Object argumentValue = elt.getValue();
String argumentValueString = argumentValue != null ? argumentValue.toString() : null;
@ -72,8 +72,8 @@ public class CommandLineUtils {
* @param argumentProviders Argument providers to inspect.
* @return A string representing the given command-line arguments.
*/
public static String createApproximateCommandLineArgumentString(Object... argumentProviders) {
Map<String,String> commandLineArgs = getApproximateCommandLineArguments(argumentProviders);
public static String createApproximateCommandLineArgumentString(ParsingEngine parsingEngine, Object... argumentProviders) {
Map<String,String> commandLineArgs = getApproximateCommandLineArguments(parsingEngine,argumentProviders);
StringBuffer sb = new StringBuffer();
boolean first = true;

51
java/src/org/broadinstitute/sting/commandline/ParsingEngine.java
View File

@ -64,6 +64,16 @@ public class ParsingEngine {
*/
private List<ParsingMethod> parsingMethods = new ArrayList<ParsingMethod>();
/**
* Class reference to the different types of descriptors that the create method can create.
* The type of set used must be ordered (but not necessarily sorted).
*/
private static final Set<ArgumentTypeDescriptor> STANDARD_ARGUMENT_TYPE_DESCRIPTORS = new LinkedHashSet<ArgumentTypeDescriptor>( Arrays.asList(new SimpleArgumentTypeDescriptor(),
new CompoundArgumentTypeDescriptor(),
new MultiplexArgumentTypeDescriptor()) );
private Set<ArgumentTypeDescriptor> argumentTypeDescriptors = new LinkedHashSet<ArgumentTypeDescriptor>();
/**
* our log, which we want to capture anything from org.broadinstitute.sting
*/
@ -75,9 +85,10 @@ public class ParsingEngine {
parsingMethods.add( ParsingMethod.FullNameParsingMethod );
parsingMethods.add( ParsingMethod.ShortNameParsingMethod );
// Null check for unit tests. Perhaps we should mock up an empty CLP?
if( clp != null )
ArgumentTypeDescriptor.addDescriptors( clp.getArgumentTypeDescriptors() );
// Order matters here! Make sure the clp's new type descriptors go in before the original type descriptors.
if(clp != null)
argumentTypeDescriptors.addAll(clp.getArgumentTypeDescriptors());
argumentTypeDescriptors.addAll(STANDARD_ARGUMENT_TYPE_DESCRIPTORS);
}
/**
@ -259,7 +270,7 @@ public class ParsingEngine {
List<ArgumentSource> dependentArguments = new ArrayList<ArgumentSource>();
for( ArgumentSource argumentSource: argumentSources ) {
if(argumentSource.isDeprecated() && argumentMatches.findMatches(argumentSource).size() > 0)
if(argumentSource.isDeprecated() && argumentMatches.findMatches(this,argumentSource).size() > 0)
notifyDeprecatedCommandLineArgument(argumentSource);
// If this argument source depends on other command-line arguments, skip it and make a note to process it later.
@ -267,13 +278,13 @@ public class ParsingEngine {
dependentArguments.add(argumentSource);
continue;
}
loadValueIntoObject( argumentSource, object, argumentMatches.findMatches(argumentSource) );
loadValueIntoObject( argumentSource, object, argumentMatches.findMatches(this,argumentSource) );
}
for(ArgumentSource dependentArgument: dependentArguments) {
MultiplexArgumentTypeDescriptor dependentDescriptor = dependentArgument.createDependentTypeDescriptor(object);
MultiplexArgumentTypeDescriptor dependentDescriptor = dependentArgument.createDependentTypeDescriptor(this,object);
ArgumentSource dependentSource = dependentArgument.copyWithCustomTypeDescriptor(dependentDescriptor);
loadValueIntoObject(dependentSource,object,argumentMatches.findMatches(dependentSource));
loadValueIntoObject(dependentSource,object,argumentMatches.findMatches(this,dependentSource));
}
}
@ -306,8 +317,10 @@ public class ParsingEngine {
* @param instance Object into which to inject the value. The target might be in a container within the instance.
*/
private void loadValueIntoObject( ArgumentSource source, Object instance, ArgumentMatches argumentMatches ) {
ArgumentTypeDescriptor typeDescriptor = selectBestTypeDescriptor(source.field.getType());
// Nothing to load
if( argumentMatches.size() == 0 && !(source.overridesDefault() && source.isRequired()))
if( argumentMatches.size() == 0 && !(typeDescriptor.createsTypeDefault(source) && source.isRequired()))
return;
// Target instance into which to inject the value.
@ -318,7 +331,7 @@ public class ParsingEngine {
throw new ReviewedStingException("Internal command-line parser error: unable to find a home for argument matches " + argumentMatches);
for( Object target: targets ) {
Object value = (argumentMatches.size() != 0) ? source.parse(this,argumentMatches) : source.createDefault();
Object value = (argumentMatches.size() != 0) ? source.parse(this,argumentMatches) : typeDescriptor.createTypeDefault(this,source);
JVMUtils.setFieldValue(source.field,target,value);
}
@ -357,18 +370,28 @@ public class ParsingEngine {
* @param sourceClass class to act as sources for other arguments.
* @return A list of sources associated with this object and its aggregated objects.
*/
public static List<ArgumentSource> extractArgumentSources(Class sourceClass) {
public List<ArgumentSource> extractArgumentSources(Class sourceClass) {
return extractArgumentSources(sourceClass, new Field[0]);
}
private static List<ArgumentSource> extractArgumentSources(Class sourceClass, Field[] parentFields) {
/**
* Fetch the best command-line argument descriptor for the given class.
* @param type Class for which to specify a descriptor.
* @return descriptor for the given type.
*/
public ArgumentTypeDescriptor selectBestTypeDescriptor(Class type) {
return ArgumentTypeDescriptor.selectBest(argumentTypeDescriptors,type);
}
private List<ArgumentSource> extractArgumentSources(Class sourceClass, Field[] parentFields) {
// now simply call into the truly general routine extract argument bindings but with a null
// object so bindings aren't computed
Map<ArgumentSource, Object> bindings = extractArgumentBindings(null, sourceClass, parentFields);
return new ArrayList<ArgumentSource>(bindings.keySet());
}
public static Map<ArgumentSource, Object> extractArgumentBindings(Object obj) {
public Map<ArgumentSource, Object> extractArgumentBindings(Object obj) {
if ( obj == null ) throw new IllegalArgumentException("Incoming object cannot be null");
return extractArgumentBindings(obj, obj.getClass(), new Field[0]);
}
@ -380,7 +403,7 @@ public class ParsingEngine {
* @param parentFields Parent Fields
* @return A map of sources associated with this object and its aggregated objects and bindings to their bindings values
*/
private static Map<ArgumentSource, Object> extractArgumentBindings(Object obj, Class sourceClass, Field[] parentFields) {
private Map<ArgumentSource, Object> extractArgumentBindings(Object obj, Class sourceClass, Field[] parentFields) {
Map<ArgumentSource, Object> bindings = new LinkedHashMap<ArgumentSource, Object>();
while( sourceClass != null ) {
@ -388,7 +411,7 @@ public class ParsingEngine {
for( Field field: fields ) {
if( ArgumentTypeDescriptor.isArgumentAnnotationPresent(field) ) {
Object val = obj != null ? JVMUtils.getFieldValue(field, obj) : null;
bindings.put( new ArgumentSource(parentFields, field), val );
bindings.put( new ArgumentSource(parentFields, field, selectBestTypeDescriptor(field.getType())), val );
}
if( field.isAnnotationPresent(ArgumentCollection.class) ) {
Object val = obj != null ? JVMUtils.getFieldValue(field, obj) : null;

17
java/src/org/broadinstitute/sting/gatk/AbstractGenomeAnalysisEngine.java
View File

@ -31,6 +31,8 @@ import net.sf.samtools.*;
import org.apache.log4j.Logger;
import org.broad.tribble.util.variantcontext.VariantContext;
import org.broadinstitute.sting.commandline.ArgumentSource;
import org.broadinstitute.sting.commandline.CommandLineUtils;
import org.broadinstitute.sting.commandline.ParsingEngine;
import org.broadinstitute.sting.gatk.arguments.GATKArgumentCollection;
import org.broadinstitute.sting.gatk.arguments.ValidationExclusion;
import org.broadinstitute.sting.gatk.datasources.sample.Sample;
@ -62,6 +64,11 @@ import java.util.*;
*/
public abstract class AbstractGenomeAnalysisEngine {
/**
* The GATK command-line argument parsing code.
*/
private ParsingEngine parsingEngine;
/**
* Accessor for sharded read data.
*/
@ -126,6 +133,10 @@ public abstract class AbstractGenomeAnalysisEngine {
private Date startTime = null; // the start time for execution
public void setParser(ParsingEngine parsingEngine) {
this.parsingEngine = parsingEngine;
}
/**
* Actually run the engine.
* @return the value of this traversal.
@ -863,5 +874,11 @@ public abstract class AbstractGenomeAnalysisEngine {
return sampleDataSource.subContextFromSampleProperty(context, key, value);
}
public Map<String,String> getApproximateCommandLineArguments(Object... argumentProviders) {
return CommandLineUtils.getApproximateCommandLineArguments(parsingEngine,argumentProviders);
}
public String createApproximateCommandLineArgumentString(Object... argumentProviders) {
return CommandLineUtils.createApproximateCommandLineArgumentString(parsingEngine,argumentProviders);
}
}

1
java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java
View File

@ -69,6 +69,7 @@ public abstract class CommandLineExecutable extends CommandLineProgram {
* @return the return code to exit the program with
*/
protected int execute() throws Exception {
engine.setParser(parser);
argumentSources.add(this);
Walker<?,?> walker = engine.getWalkerByName(getAnalysisName());

7
java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2010 The Broad Institute
* Copyright (c) 2010, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -12,15 +12,14 @@
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
* THE USE OR OTHER DEALINGS IN THE SOFTWARE.
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk;

6
java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamArgumentTypeDescriptor.java
View File

@ -63,15 +63,15 @@ public class OutputStreamArgumentTypeDescriptor extends ArgumentTypeDescriptor {
}
@Override
public boolean createsTypeDefault(ArgumentSource source,Class type) {
public boolean createsTypeDefault(ArgumentSource source) {
return true;
}
@Override
public Object createTypeDefault(ArgumentSource source,Class type) {
public Object createTypeDefault(ParsingEngine parsingEngine,ArgumentSource source) {
OutputStreamStub stub = new OutputStreamStub(defaultOutputStream);
engine.addOutput(stub);
return createInstanceOfClass(type,stub);
return createInstanceOfClass(source.field.getType(),stub);
}
@Override

4
java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java
View File

@ -82,12 +82,12 @@ public class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor
}
@Override
public boolean createsTypeDefault(ArgumentSource source,Class type) {
public boolean createsTypeDefault(ArgumentSource source) {
return true;
}
@Override
public Object createTypeDefault(ArgumentSource source,Class type) {
public Object createTypeDefault(ParsingEngine parsingEngine,ArgumentSource source) {
SAMFileWriterStub stub = new SAMFileWriterStub(engine,defaultOutputStream);
engine.addOutput(stub);
return stub;

10
java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java
View File

@ -103,13 +103,13 @@ public class VCFWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor {
* @return true always.
*/
@Override
public boolean createsTypeDefault(ArgumentSource source,Class type) {
public boolean createsTypeDefault(ArgumentSource source) {
return true;
}
@Override
public Object createTypeDefault(ArgumentSource source,Class type) {
VCFWriterStub stub = new VCFWriterStub(defaultOutputStream, false, argumentSources, false);
public Object createTypeDefault(ParsingEngine parsingEngine,ArgumentSource source) {
VCFWriterStub stub = new VCFWriterStub(engine, defaultOutputStream, false, argumentSources, false);
engine.addOutput(stub);
return stub;
}
@ -133,8 +133,8 @@ public class VCFWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor {
boolean skipWritingHeader = argumentIsPresent(createNoHeaderArgumentDefinition(),matches);
// Create a stub for the given object.
VCFWriterStub stub = (writerFile != null) ? new VCFWriterStub(writerFile, compress, argumentSources, skipWritingHeader)
: new VCFWriterStub(defaultOutputStream, compress, argumentSources, skipWritingHeader);
VCFWriterStub stub = (writerFile != null) ? new VCFWriterStub(engine, writerFile, compress, argumentSources, skipWritingHeader)
: new VCFWriterStub(engine, defaultOutputStream, compress, argumentSources, skipWritingHeader);
// WARNING: Side effects required by engine!
parsingEngine.addTags(stub,getArgumentTags(matches));

13
java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterStub.java
View File

@ -38,6 +38,7 @@ import org.broad.tribble.vcf.VCFHeaderLine;
import org.broad.tribble.vcf.VCFWriter;
import org.broadinstitute.sting.commandline.CommandLineUtils;
import org.broadinstitute.sting.gatk.CommandLineExecutable;
import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
import org.broadinstitute.sting.gatk.io.OutputTracker;
import org.broadinstitute.sting.utils.classloader.JVMUtils;
@ -48,6 +49,10 @@ import org.broadinstitute.sting.utils.classloader.JVMUtils;
* @version 0.1
*/
public class VCFWriterStub implements Stub<VCFWriter>, VCFWriter {
/**
* The engine, central to the GATK's processing.
*/
private final GenomeAnalysisEngine engine;
/**
* The file that this stub should write to. Should be mutually
@ -93,7 +98,8 @@ public class VCFWriterStub implements Stub<VCFWriter>, VCFWriter {
* @param genotypeFile file to (ultimately) create.
* @param isCompressed should we compress the output stream?
*/
public VCFWriterStub(File genotypeFile, boolean isCompressed, Collection<Object> argumentSources, boolean skipWritingHeader) {
public VCFWriterStub(GenomeAnalysisEngine engine, File genotypeFile, boolean isCompressed, Collection<Object> argumentSources, boolean skipWritingHeader) {
this.engine = engine;
this.genotypeFile = genotypeFile;
this.genotypeStream = null;
this.isCompressed = isCompressed;
@ -106,7 +112,8 @@ public class VCFWriterStub implements Stub<VCFWriter>, VCFWriter {
* @param genotypeStream stream to (ultimately) write.
* @param isCompressed should we compress the output stream?
*/
public VCFWriterStub(OutputStream genotypeStream, boolean isCompressed, Collection<Object> argumentSources, boolean skipWritingHeader) {
public VCFWriterStub(GenomeAnalysisEngine engine, OutputStream genotypeStream, boolean isCompressed, Collection<Object> argumentSources, boolean skipWritingHeader) {
this.engine = engine;
this.genotypeFile = null;
this.genotypeStream = new PrintStream(genotypeStream);
this.isCompressed = isCompressed;
@ -199,6 +206,6 @@ public class VCFWriterStub implements Stub<VCFWriter>, VCFWriter {
*/
private VCFHeaderLine getCommandLineArgumentHeaderLine() {
CommandLineExecutable executable = JVMUtils.getObjectOfType(argumentSources,CommandLineExecutable.class);
return new VCFHeaderLine(executable.getAnalysisName(), "\"" + CommandLineUtils.createApproximateCommandLineArgumentString(argumentSources.toArray()) + "\"");
return new VCFHeaderLine(executable.getAnalysisName(), "\"" + engine.createApproximateCommandLineArgumentString(argumentSources.toArray()) + "\"");
}
}

2
java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java
View File

@ -180,7 +180,7 @@ public class GATKRunReport {
// what did we run?
id = org.apache.commons.lang.RandomStringUtils.randomAlphanumeric(32);
try {
cmdLine = CommandLineUtils.createApproximateCommandLineArgumentString(engine, walker);
cmdLine = engine.createApproximateCommandLineArgumentString(engine, walker);
} catch (Exception ignore) { }
this.mCollection = engine.getArguments();

2
java/src/org/broadinstitute/sting/gatk/walkers/indels/IndelGenotyperV2Walker.java
View File

@ -211,7 +211,7 @@ public class IndelGenotyperV2Walker extends ReadWalker<Integer,Integer> {
Set<Object> args = new HashSet<Object>();
args.add(this);
args.addAll(getToolkit().getFilters());
Map<String,String> commandLineArgs = CommandLineUtils.getApproximateCommandLineArguments(args);
Map<String,String> commandLineArgs = getToolkit().getApproximateCommandLineArguments(args);
for ( Map.Entry<String, String> commandLineArg : commandLineArgs.entrySet() )
headerInfo.add(new VCFHeaderLine(String.format("IGv2_%s", commandLineArg.getKey()), commandLineArg.getValue()));
// also, the list of input bams

2
java/src/org/broadinstitute/sting/gatk/walkers/indels/IndelRealigner.java
View File

@ -228,7 +228,7 @@ public class IndelRealigner extends ReadWalker<Integer, Integer> {
final SAMProgramRecord programRecord = new SAMProgramRecord(PROGRAM_RECORD_NAME);
final ResourceBundle headerInfo = TextFormattingUtils.loadResourceBundle("StingText");
programRecord.setProgramVersion(headerInfo.getString("org.broadinstitute.sting.gatk.version"));
programRecord.setCommandLine(CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), this));
programRecord.setCommandLine(getToolkit().createApproximateCommandLineArgumentString(getToolkit(), this));
List<SAMProgramRecord> oldRecords = header.getProgramRecords();
List<SAMProgramRecord> newRecords = new ArrayList<SAMProgramRecord>(oldRecords.size()+1);

2
java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java
View File

@ -263,7 +263,7 @@ public class TableRecalibrationWalker extends ReadWalker<SAMRecord, SAMFileWrite
programRecord.setProgramVersion( headerInfo.getString("org.broadinstitute.sting.gatk.version") );
StringBuffer sb = new StringBuffer();
sb.append(CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), this));
sb.append(getToolkit().createApproximateCommandLineArgumentString(getToolkit(), this));
sb.append(" Covariates=[");
for( Covariate cov : requestedCovariates ) {
sb.append(cov.getClass().getSimpleName());

2
java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/GenerateVariantClustersWalker.java
View File

@ -129,7 +129,7 @@ public class GenerateVariantClustersWalker extends RodWalker<ExpandingArrayList<
if( !NO_HEADER_LINE ) {
CLUSTER_FILE.print("##GenerateVariantClusters = ");
CLUSTER_FILE.println("\"" + CommandLineUtils.createApproximateCommandLineArgumentString(getToolkit(), this) + "\"");
CLUSTER_FILE.println("\"" + getToolkit().createApproximateCommandLineArgumentString(getToolkit(), this) + "\"");
}
boolean foundDBSNP = false;

4
java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java
View File

@ -108,9 +108,9 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
return OPTIONAL_TEMPLATE;
}
public static List<? extends ArgumentField> getArgumentFields(Class<?> classType) {
public static List<? extends ArgumentField> getArgumentFields(ParsingEngine parsingEngine,Class<?> classType) {
List<ArgumentField> argumentFields = new ArrayList<ArgumentField>();
for (ArgumentSource argumentSource: ParsingEngine.extractArgumentSources(classType))
for (ArgumentSource argumentSource: parsingEngine.extractArgumentSources(classType))
if (!argumentSource.isDeprecated())
for (ArgumentDefinition argumentDefinition: argumentSource.createArgumentDefinitions())
argumentFields.addAll(getArgumentFields(argumentDefinition));

8
java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java
View File

@ -105,7 +105,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
String clpClassName = clpManager.getName(clp);
writeClass("org.broadinstitute.sting.queue.function.JarCommandLineFunction", COMMANDLINE_PACKAGE_NAME, clpClassName,
false, "", ArgumentDefinitionField.getArgumentFields(clp));
false, "", ArgumentDefinitionField.getArgumentFields(parser,clp));
if (clp == CommandLineGATK.class) {
for (Entry<String, Collection<Class<? extends Walker>>> walkersByPackage: walkerManager.getWalkerNamesByPackage(false).entrySet()) {
@ -113,9 +113,9 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
String walkerName = walkerManager.getName(walkerType);
List<ArgumentField> argumentFields = new ArrayList<ArgumentField>();
argumentFields.addAll(ArgumentDefinitionField.getArgumentFields(walkerType));
argumentFields.addAll(ArgumentDefinitionField.getArgumentFields(parser,walkerType));
argumentFields.addAll(RodBindField.getRodArguments(walkerType, trackBuilder));
argumentFields.addAll(ReadFilterField.getFilterArguments(walkerType));
argumentFields.addAll(ReadFilterField.getFilterArguments(parser,walkerType));
String constructor = String.format("analysisName = \"%1$s\"%nanalysis_type = \"%1$s\"%n", walkerName);
String scatterClass = getScatterClass(walkerType);
@ -134,7 +134,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
for (Class<? extends SamRecordFilter> filter: filterManager.getValues()) {
String filterName = filterManager.getName(filter);
writeFilter(FILTER_PACKAGE_NAME, filterName, ArgumentDefinitionField.getArgumentFields(filter));
writeFilter(FILTER_PACKAGE_NAME, filterName, ArgumentDefinitionField.getArgumentFields(new ParsingEngine(null),filter));
}
return 0;

5
java/src/org/broadinstitute/sting/queue/extensions/gatk/ReadFilterField.java
View File

@ -25,6 +25,7 @@
package org.broadinstitute.sting.queue.extensions.gatk;
import net.sf.picard.filter.SamRecordFilter;
import org.broadinstitute.sting.commandline.ParsingEngine;
import org.broadinstitute.sting.gatk.WalkerManager;
import org.broadinstitute.sting.gatk.walkers.Walker;
@ -37,10 +38,10 @@ public class ReadFilterField {
* @param walkerClass the class of the walker
* @return the list of argument fields
*/
public static List<ArgumentField> getFilterArguments(Class<? extends Walker> walkerClass) {
public static List<ArgumentField> getFilterArguments(ParsingEngine parser, Class<? extends Walker> walkerClass) {
List<ArgumentField> argumentFields = new ArrayList<ArgumentField>();
for(Class<? extends SamRecordFilter> filter: WalkerManager.getReadFilterTypes(walkerClass))
argumentFields.addAll(ArgumentDefinitionField.getArgumentFields(filter));
argumentFields.addAll(ArgumentDefinitionField.getArgumentFields(parser,filter));
return argumentFields;
}
}

1
java/test/org/broadinstitute/sting/commandline/ParsingEngineUnitTest.java
View File

@ -31,6 +31,7 @@ import org.junit.Test;
import org.junit.Before;
import org.junit.Assert;
import java.util.Collections;
import java.util.List;
import java.util.EnumSet;
/**

2
java/test/org/broadinstitute/sting/gatk/WalkerManagerUnitTest.java
View File

@ -25,6 +25,7 @@
package org.broadinstitute.sting.gatk;
import org.broadinstitute.sting.commandline.Hidden;
import org.broadinstitute.sting.gatk.walkers.Requires;
import org.broadinstitute.sting.gatk.walkers.Walker;
import org.broadinstitute.sting.gatk.walkers.qc.CountLociWalker;
import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException;
@ -62,6 +63,7 @@ public class WalkerManagerUnitTest {
}
@Hidden
@Requires(value={})
class UninstantiableWalker extends Walker<Integer,Long> {
// Private constructor will generate uninstantiable message
private UninstantiableWalker() {}

3
scala/src/org/broadinstitute/sting/queue/QCommandLine.scala
View File

@ -1,5 +1,6 @@
package org.broadinstitute.sting.queue
import function.QFunction
import java.io.File
import java.util.Arrays
import org.broadinstitute.sting.commandline._
@ -17,6 +18,8 @@ class QCommandLine extends CommandLineProgram with Logging {
@ArgumentCollection
private val settings = new QGraphSettings
QFunction.parsingEngine = new ParsingEngine(this)
/**
* Takes the QScripts passed in, runs their script() methods, retrieves their generated
* functions, and then builds and runs a QGraph based on the dependencies.

4
scala/src/org/broadinstitute/sting/queue/function/QFunction.scala
View File

@ -410,6 +410,8 @@ object QFunction {
/** Job index counter for this run of Queue. */
private var jobIndex = 0
var parsingEngine: ParsingEngine = _
/**
* Returns the next job name using the prefix.
* @param prefix Prefix of the job name.
@ -426,7 +428,7 @@ object QFunction {
*/
private class ClassFields(clazz: Class[_]) {
/** The complete list of fields on this CommandLineFunction. */
val functionFields: List[ArgumentSource] = ParsingEngine.extractArgumentSources(clazz).toList
val functionFields: List[ArgumentSource] = parsingEngine.extractArgumentSources(clazz).toList
/** The @Input fields on this CommandLineFunction. */
val inputFields = functionFields.filter(source => ReflectionUtils.hasAnnotation(source.field, classOf[Input]))
/** The @Output fields on this CommandLineFunction. */

2
scala/src/org/broadinstitute/sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala
View File

@ -44,7 +44,7 @@ class ScalaCompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor {
*/
def parse(parsingEngine: ParsingEngine, source: ArgumentSource, classType: Class[_], argumentMatches: ArgumentMatches) = {
val componentType = ReflectionUtils.getCollectionType(source.field)
val componentArgumentParser = ArgumentTypeDescriptor.create(componentType)
val componentArgumentParser = parsingEngine.selectBestTypeDescriptor(componentType)
if (classOf[List[_]].isAssignableFrom(classType)) {
var list = List.empty[Any]