diff --git a/ivy.xml b/ivy.xml index 0761cb411..1e3346ff5 100644 --- a/ivy.xml +++ b/ivy.xml @@ -78,8 +78,8 @@ - - + + diff --git a/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatch.java b/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatch.java index c0823e5c5..6c8fb1f4d 100755 --- a/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatch.java +++ b/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatch.java @@ -46,7 +46,7 @@ public class ArgumentMatch implements Iterable { /** * Maps indices of command line arguments to values paired with that argument. */ - public final SortedMap> sites = new TreeMap>(); + public final SortedMap> sites = new TreeMap>(); /** * An ordered, freeform collection of tags. @@ -90,11 +90,11 @@ public class ArgumentMatch implements Iterable { * @param value Value for the argument at this position. * @param tags ordered freeform text tags associated with this argument. */ - private ArgumentMatch(final String label, final ArgumentDefinition definition, final ArgumentMatchSite site, final String value, final Tags tags) { + private ArgumentMatch(final String label, final ArgumentDefinition definition, final ArgumentMatchSite site, final ArgumentMatchValue value, final Tags tags) { this.label = label; this.definition = definition; - ArrayList values = new ArrayList(); + ArrayList values = new ArrayList(); if( value != null ) values.add(value); sites.put(site,values ); @@ -131,11 +131,11 @@ public class ArgumentMatch implements Iterable { */ @SuppressWarnings("unchecked") ArgumentMatch transform(Multiplexer multiplexer, Object key) { - SortedMap> newIndices = new TreeMap>(); - for(Map.Entry> site: sites.entrySet()) { - List newEntries = new ArrayList(); - for(String entry: site.getValue()) - newEntries.add(multiplexer.transformArgument(key,entry)); + SortedMap> newIndices = new TreeMap>(); + for(Map.Entry> site: sites.entrySet()) { + List newEntries = new ArrayList(); + for(ArgumentMatchValue entry: site.getValue()) + newEntries.add(new ArgumentMatchStringValue(multiplexer.transformArgument(key,entry.asString()))); newIndices.put(site.getKey(),newEntries); } ArgumentMatch newArgumentMatch = new ArgumentMatch(label,definition); @@ -165,7 +165,7 @@ public class ArgumentMatch implements Iterable { /** * Iterate over each available token. */ - private Iterator tokenIterator = null; + private Iterator tokenIterator = null; /** * The next site to return. Null if none remain. @@ -175,7 +175,7 @@ public class ArgumentMatch implements Iterable { /** * The next token to return. Null if none remain. */ - String nextToken = null; + ArgumentMatchValue nextToken = null; { siteIterator = sites.keySet().iterator(); @@ -254,9 +254,9 @@ public class ArgumentMatch implements Iterable { * @param site site of the command-line argument to which this value is mated. * @param value Text representation of value to add. */ - public void addValue( ArgumentMatchSite site, String value ) { + public void addValue( ArgumentMatchSite site, ArgumentMatchValue value ) { if( !sites.containsKey(site) || sites.get(site) == null ) - sites.put(site, new ArrayList() ); + sites.put(site, new ArrayList() ); sites.get(site).add(value); } @@ -275,8 +275,8 @@ public class ArgumentMatch implements Iterable { * Return the values associated with this argument match. * @return A collection of the string representation of these value. */ - public List values() { - List values = new ArrayList(); + public List values() { + List values = new ArrayList(); for( ArgumentMatchSite site: sites.keySet() ) { if( sites.get(site) != null ) values.addAll(sites.get(site)); diff --git a/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchFileValue.java b/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchFileValue.java new file mode 100644 index 000000000..344b6829a --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchFileValue.java @@ -0,0 +1,27 @@ +package org.broadinstitute.sting.commandline; + +import java.io.File; + +/** + * Holds a reference to a file as an argument match value. + * + * This is useful when the type of the stored file may be a subclass of java.io.File, + * for example a Queue RemoteFile. + */ +public class ArgumentMatchFileValue extends ArgumentMatchValue { + private final File file; + + public ArgumentMatchFileValue(File file) { + this.file = file; + } + + @Override + public String asString() { + return file == null ? null : file.getAbsolutePath(); + } + + @Override + public File asFile() { + return file; + } +} diff --git a/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchSource.java b/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchSource.java index ed2700006..9dfb3afbe 100644 --- a/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchSource.java +++ b/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchSource.java @@ -24,38 +24,36 @@ package org.broadinstitute.sting.commandline; -import java.io.File; - /** - * Where an argument match originated, via the commandline or a file. + * Where an argument match originated, via the commandline or a custom provider. */ public class ArgumentMatchSource implements Comparable { public static final ArgumentMatchSource COMMAND_LINE = new ArgumentMatchSource(ArgumentMatchSourceType.CommandLine, null); private final ArgumentMatchSourceType type; - private final File file; + private final String description; /** * Creates an argument match source from the specified file. - * @param file File specifying the arguments. Must not be null. + * @param description Where the arguments originated. */ - public ArgumentMatchSource(File file) { - this(ArgumentMatchSourceType.File, file); + public ArgumentMatchSource(String description) { + this(ArgumentMatchSourceType.Provider, description); } - private ArgumentMatchSource(ArgumentMatchSourceType type, File file) { - if (type == ArgumentMatchSourceType.File && file == null) - throw new IllegalArgumentException("An argument match source of type File cannot have a null file."); + private ArgumentMatchSource(ArgumentMatchSourceType type, String description) { + if (type == ArgumentMatchSourceType.Provider && description == null) + throw new IllegalArgumentException("An argument match source provider cannot have a null description."); this.type = type; - this.file = file; + this.description = description; } public ArgumentMatchSourceType getType() { return type; } - public File getFile() { - return file; + public String getDescription() { + return description; } @Override @@ -65,13 +63,13 @@ public class ArgumentMatchSource implements Comparable { ArgumentMatchSource that = (ArgumentMatchSource) o; - return (type == that.type) && (file == null ? that.file == null : file.equals(that.file)); + return (type == that.type) && (description == null ? that.description == null : description.equals(that.description)); } @Override public int hashCode() { int result = type != null ? type.hashCode() : 0; - result = 31 * result + (file != null ? file.hashCode() : 0); + result = 31 * result + (description != null ? description.hashCode() : 0); return result; } @@ -84,15 +82,15 @@ public class ArgumentMatchSource implements Comparable { if (comp != 0) return comp; - File f1 = this.file; - File f2 = that.file; + String d1 = this.description; + String d2 = that.description; - if ((f1 == null) ^ (f2 == null)) { - // If one of the files is null and the other is not - // put the null file first - return f1 == null ? -1 : 1; + if ((d1 == null) ^ (d2 == null)) { + // If one of the descriptions is null and the other is not + // put the null description first + return d1 == null ? -1 : 1; } - return f1 == null ? 0 : f1.compareTo(f2); + return d1 == null ? 0 : d1.compareTo(d2); } } diff --git a/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchSourceType.java b/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchSourceType.java index 3ff6e21d4..118316473 100644 --- a/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchSourceType.java +++ b/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchSourceType.java @@ -25,8 +25,8 @@ package org.broadinstitute.sting.commandline; /** - * Type of where an argument match originated, via the commandline or a file. + * Type of where an argument match originated, via the commandline or a some other provider. */ public enum ArgumentMatchSourceType { - CommandLine, File + CommandLine, Provider } diff --git a/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchStringValue.java b/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchStringValue.java new file mode 100644 index 000000000..bb2015c3b --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchStringValue.java @@ -0,0 +1,24 @@ +package org.broadinstitute.sting.commandline; + +import java.io.File; + +/** + * Argument values that originated from a string. + */ +public class ArgumentMatchStringValue extends ArgumentMatchValue { + private final String value; + + public ArgumentMatchStringValue(String value) { + this.value = value; + } + + @Override + public String asString() { + return value; + } + + @Override + public File asFile() { + return value == null ? null : new File(value); + } +} diff --git a/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchValue.java b/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchValue.java new file mode 100644 index 000000000..bed4edfa6 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchValue.java @@ -0,0 +1,18 @@ +package org.broadinstitute.sting.commandline; + +import java.io.File; + +/** + * Returns argument values as either strings or values. + */ +public abstract class ArgumentMatchValue { + /** + * @return the value of this argument as a String object. + */ + public abstract String asString(); + + /** + * @return the value of this argument as a File object. + */ + public abstract File asFile(); +} diff --git a/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java b/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java index dd4a151bf..4b9774806 100644 --- a/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java +++ b/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java @@ -215,8 +215,8 @@ public abstract class ArgumentTypeDescriptor { * @param matches The matches for the given argument. * @return The value of the argument if available, or null if not present. */ - protected String getArgumentValue( ArgumentDefinition definition, ArgumentMatches matches ) { - Collection argumentValues = getArgumentValues( definition, matches ); + protected ArgumentMatchValue getArgumentValue( ArgumentDefinition definition, ArgumentMatches matches ) { + Collection argumentValues = getArgumentValues( definition, matches ); if( argumentValues.size() > 1 ) throw new UserException.CommandLineException("Multiple values associated with given definition, but this argument expects only one: " + definition.fullName); return argumentValues.size() > 0 ? argumentValues.iterator().next() : null; @@ -244,8 +244,8 @@ public abstract class ArgumentTypeDescriptor { * @param matches The matches for the given argument. * @return The value of the argument if available, or an empty collection if not present. */ - protected Collection getArgumentValues( ArgumentDefinition definition, ArgumentMatches matches ) { - Collection values = new ArrayList(); + protected Collection getArgumentValues( ArgumentDefinition definition, ArgumentMatches matches ) { + Collection values = new ArrayList(); for( ArgumentMatch match: matches ) { if( match.definition.equals(definition) ) values.addAll(match.values()); @@ -310,7 +310,7 @@ public abstract class ArgumentTypeDescriptor { */ protected Object parseBinding(ArgumentSource source, Type type, ArgumentMatches matches, Tags tags) { ArgumentDefinition defaultDefinition = createDefaultArgumentDefinition(source); - String value = getArgumentValue(defaultDefinition, matches); + ArgumentMatchValue value = getArgumentValue(defaultDefinition, matches); @SuppressWarnings("unchecked") Class parameterType = JVMUtils.getParameterizedTypeClass(type); String name = defaultDefinition.fullName; @@ -328,7 +328,7 @@ public abstract class ArgumentTypeDescriptor { * @param fieldName The name of the field that was parsed. Used for error reporting. * @return The newly created binding object of type bindingClass. */ - public static Object parseBinding(String value, Class parameterType, Type bindingClass, + public static Object parseBinding(ArgumentMatchValue value, Class parameterType, Type bindingClass, String bindingName, Tags tags, String fieldName) { try { String tribbleType = null; @@ -337,7 +337,7 @@ public abstract class ArgumentTypeDescriptor { throw new UserException.CommandLineException( String.format("Unexpected number of positional tags for argument %s : %s. " + "Rod bindings only support -X:type and -X:name,type argument styles", - value, fieldName)); + value.asString(), fieldName)); } else if ( tags.getPositionalTags().size() == 2 ) { // -X:name,type style bindingName = tags.getPositionalTags().get(0); @@ -366,7 +366,7 @@ public abstract class ArgumentTypeDescriptor { if ( tribbleType == null ) { // try to determine the file type dynamically - File file = new File(value); + File file = value.asFile(); if ( file.canRead() && file.isFile() ) { FeatureManager.FeatureDescriptor featureDescriptor = manager.getByFiletype(file); if ( featureDescriptor != null ) { @@ -379,7 +379,7 @@ public abstract class ArgumentTypeDescriptor { // IntervalBinding can be created from a normal String Class rawType = (makeRawTypeIfNecessary(bindingClass)); try { - return rawType.getConstructor(String.class).newInstance(value); + return rawType.getConstructor(String.class).newInstance(value.asString()); } catch (NoSuchMethodException e) { /* ignore */ } @@ -399,14 +399,14 @@ public abstract class ArgumentTypeDescriptor { } Constructor ctor = (makeRawTypeIfNecessary(bindingClass)).getConstructor(Class.class, String.class, String.class, String.class, Tags.class); - return ctor.newInstance(parameterType, bindingName, value, tribbleType, tags); + return ctor.newInstance(parameterType, bindingName, value.asString(), tribbleType, tags); } catch (Exception e) { if ( e instanceof UserException ) throw ((UserException)e); else throw new UserException.CommandLineException( String.format("Failed to parse value %s for argument %s. Message: %s", - value, fieldName, e.getMessage())); + value.asString(), fieldName, e.getMessage())); } } } @@ -517,7 +517,7 @@ class SimpleArgumentTypeDescriptor extends ArgumentTypeDescriptor { return true; ArgumentDefinition defaultDefinition = createDefaultArgumentDefinition(source); - String value = getArgumentValue( defaultDefinition, matches ); + ArgumentMatchValue value = getArgumentValue(defaultDefinition, matches); Object result; Tags tags = getArgumentTags(matches); @@ -527,12 +527,12 @@ class SimpleArgumentTypeDescriptor extends ArgumentTypeDescriptor { Method valueOf = primitiveToWrapperMap.get(type).getMethod("valueOf",String.class); if(value == null) throw new MissingArgumentValueException(createDefaultArgumentDefinition(source)); - result = valueOf.invoke(null,value.trim()); + result = valueOf.invoke(null,value.asString().trim()); } else if (type.isEnum()) { Object[] vals = type.getEnumConstants(); Object defaultEnumeration = null; // as we look at options, record the default option if it exists for (Object val : vals) { - if (String.valueOf(val).equalsIgnoreCase(value)) return val; + if (String.valueOf(val).equalsIgnoreCase(value.asString())) return val; try { if (type.getField(val.toString()).isAnnotationPresent(EnumerationArgumentDefault.class)) defaultEnumeration = val; } catch (NoSuchFieldException e) { throw new ReviewedStingException("parsing " + type.toString() + "doesn't contain the field " + val.toString()); } } @@ -544,10 +544,12 @@ class SimpleArgumentTypeDescriptor extends ArgumentTypeDescriptor { else if (value == null) throw new MissingArgumentValueException(createDefaultArgumentDefinition(source)); else - throw new UnknownEnumeratedValueException(createDefaultArgumentDefinition(source),value); + throw new UnknownEnumeratedValueException(createDefaultArgumentDefinition(source),value.asString()); + } else if (type.equals(File.class)) { + result = value.asFile(); } else { Constructor ctor = type.getConstructor(String.class); - result = ctor.newInstance(value); + result = ctor.newInstance(value.asString()); } } catch (UserException e) { throw e; diff --git a/public/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java b/public/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java index 15ec9dfe5..d77ae67cf 100644 --- a/public/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java +++ b/public/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java @@ -174,7 +174,7 @@ public abstract class CommandLineProgram { ParsingEngine parser = clp.parser = new ParsingEngine(clp); parser.addArgumentSource(clp.getClass()); - Map> parsedArgs; + Map parsedArgs; // process the args if (clp.canAddArgumentsDynamically()) { diff --git a/public/java/src/org/broadinstitute/sting/commandline/ParsedArgs.java b/public/java/src/org/broadinstitute/sting/commandline/ParsedArgs.java new file mode 100644 index 000000000..9ab315175 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/commandline/ParsedArgs.java @@ -0,0 +1,13 @@ +package org.broadinstitute.sting.commandline; + +/** + * Represents a collection of parsed arguments for an argument source. + * + * Useful for printing out help documents. + */ +public abstract class ParsedArgs { + /** + * @return A compact description of the arguments from an provider/source. + */ + public abstract String getDescription(); +} diff --git a/public/java/src/org/broadinstitute/sting/commandline/ParsedListArgs.java b/public/java/src/org/broadinstitute/sting/commandline/ParsedListArgs.java new file mode 100644 index 000000000..a77e73bcf --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/commandline/ParsedListArgs.java @@ -0,0 +1,30 @@ +package org.broadinstitute.sting.commandline; + +import org.apache.commons.lang.StringUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * A list of string arguments, usually from the command line or an args list file. + */ +public class ParsedListArgs extends ParsedArgs { + private final List args = new ArrayList(); + + public ParsedListArgs() { + } + + public ParsedListArgs(List args) { + this.args.addAll(args); + } + + public void add(String... args) { + this.args.addAll(Arrays.asList(args)); + } + + @Override + public String getDescription() { + return StringUtils.join(this.args, " "); + } +} diff --git a/public/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java b/public/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java index 0fac195e1..a8b729be4 100755 --- a/public/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java +++ b/public/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java @@ -30,6 +30,7 @@ import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import org.broadinstitute.sting.utils.Utils; import org.broadinstitute.sting.utils.classloader.JVMUtils; +import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.collections.Pair; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; @@ -61,7 +62,7 @@ public class ParsingEngine { * Indicates as best as possible where command-line text remains unmatched * to existing arguments. */ - ArgumentMatches argumentMatches = null; + private ArgumentMatches argumentMatches = null; /** * Techniques for parsing and for argument lookup. @@ -88,7 +89,10 @@ public class ParsingEngine { /** * List of tags associated with the given instantiation of the command-line argument. */ - private final Map tags = new IdentityHashMap(); + private final Map tags = new IdentityHashMap(); + + private PluginManager argumentProviderPluginManager = + new PluginManager(ParsingEngineArgumentProvider.class); /** * our log, which we want to capture anything from org.broadinstitute.sting @@ -105,7 +109,10 @@ public class ParsingEngine { argumentTypeDescriptors.addAll(clp.getArgumentTypeDescriptors()); argumentTypeDescriptors.addAll(STANDARD_ARGUMENT_TYPE_DESCRIPTORS); - addArgumentSource(ParsingEngineArgumentFiles.class); + List> providers = argumentProviderPluginManager.getPlugins(); + for (Class provider: providers) { + addArgumentSource(provider); + } } /** @@ -117,6 +124,10 @@ public class ParsingEngine { addArgumentSource(null, source); } + public ArgumentMatches getArgumentMatches() { + return argumentMatches; + } + /** * Add an argument source. Argument sources are expected to have * any number of fields with an @Argument annotation attached. @@ -156,29 +167,30 @@ public class ParsingEngine { * @param tokens Tokens passed on the command line. * @return The parsed arguments by file. */ - public SortedMap> parse( String[] tokens ) { + public SortedMap parse( String[] tokens ) { argumentMatches = new ArgumentMatches(); - SortedMap> parsedArgs = new TreeMap>(); + SortedMap parsedArgs = new TreeMap(); List cmdLineTokens = Arrays.asList(tokens); parse(ArgumentMatchSource.COMMAND_LINE, cmdLineTokens, argumentMatches, parsedArgs); - ParsingEngineArgumentFiles argumentFiles = new ParsingEngineArgumentFiles(); + List providers = argumentProviderPluginManager.createAllTypes(); - // Load the arguments ONLY into the argument files. - // Validation may optionally run on the rest of the arguments. - loadArgumentsIntoObject(argumentFiles); + for (ParsingEngineArgumentProvider provider: providers) { + // Load the arguments ONLY into the provider. + // Validation may optionally run on the rest of the arguments. + loadArgumentsIntoObject(provider); + } - for (File file: argumentFiles.files) { - List fileTokens = getArguments(file); - parse(new ArgumentMatchSource(file), fileTokens, argumentMatches, parsedArgs); + for (ParsingEngineArgumentProvider provider: providers) { + provider.parse(this, parsedArgs); } return parsedArgs; } - private void parse(ArgumentMatchSource matchSource, List tokens, - ArgumentMatches argumentMatches, SortedMap> parsedArgs) { + public void parse(ArgumentMatchSource matchSource, List tokens, + ArgumentMatches argumentMatches, SortedMap parsedArgs) { ArgumentMatchSite lastArgumentMatchSite = new ArgumentMatchSite(matchSource, -1); int i = 0; @@ -195,19 +207,44 @@ public class ParsingEngine { } else { if( argumentMatches.hasMatch(lastArgumentMatchSite) && - !argumentMatches.getMatch(lastArgumentMatchSite).hasValueAtSite(lastArgumentMatchSite)) - argumentMatches.getMatch(lastArgumentMatchSite).addValue( lastArgumentMatchSite, token ); + !argumentMatches.getMatch(lastArgumentMatchSite).hasValueAtSite(lastArgumentMatchSite)) + argumentMatches.getMatch(lastArgumentMatchSite).addValue( lastArgumentMatchSite, new ArgumentMatchStringValue(token) ); else - argumentMatches.MissingArgument.addValue( site, token ); + argumentMatches.MissingArgument.addValue( site, new ArgumentMatchStringValue(token) ); } i++; } - parsedArgs.put(matchSource, tokens); + parsedArgs.put(matchSource, new ParsedListArgs(tokens)); } - private List getArguments(File file) { + public void parsePairs(ArgumentMatchSource matchSource, List> tokens, + ArgumentMatches argumentMatches, ParsedArgs matchSourceArgs, + SortedMap parsedArgs) { + int i = 0; + for (Pair pair: tokens) { + + ArgumentMatchSite site = new ArgumentMatchSite(matchSource, i); + List matchers = Arrays.asList(ArgumentDefinitions.FullNameDefinitionMatcher, ArgumentDefinitions.ShortNameDefinitionMatcher); + ArgumentDefinition definition = null; + for (DefinitionMatcher matcher: matchers) { + definition = argumentDefinitions.findArgumentDefinition( pair.getFirst(), matcher ); + if (definition != null) + break; + } + if (definition == null) + continue; + ArgumentMatch argumentMatch = new ArgumentMatch(pair.getFirst(), definition, site, new Tags()); + argumentMatches.mergeInto(argumentMatch); + argumentMatch.addValue(site, pair.getSecond()); + i++; + } + + parsedArgs.put(matchSource, matchSourceArgs); + } + + protected List getArguments(File file) { try { if (file.getAbsolutePath().endsWith(".list")) { return getListArguments(file); @@ -283,9 +320,9 @@ public class ParsingEngine { // Ensure that the field contents meet the validation criteria specified by the regular expression. for( ArgumentMatch verifiableMatch: verifiableMatches ) { - for( String value: verifiableMatch.values() ) { - if( verifiableArgument.validation != null && !value.matches(verifiableArgument.validation) ) - invalidValues.add( new Pair(verifiableArgument, value) ); + for( ArgumentMatchValue value: verifiableMatch.values() ) { + if( verifiableArgument.validation != null && !value.asString().matches(verifiableArgument.validation) ) + invalidValues.add( new Pair(verifiableArgument, value.asString()) ); } } } @@ -629,21 +666,21 @@ class UnmatchedArgumentException extends ArgumentException { private static String formatArguments( ArgumentMatch invalidValues ) { StringBuilder sb = new StringBuilder(); for( ArgumentMatchSite site: invalidValues.sites.keySet() ) - for( String value: invalidValues.sites.get(site) ) { + for( ArgumentMatchValue value: invalidValues.sites.get(site) ) { switch (site.getSource().getType()) { case CommandLine: sb.append( String.format("%nInvalid argument value '%s' at position %d.", - value, site.getIndex()) ); + value.asString(), site.getIndex()) ); break; - case File: - sb.append( String.format("%nInvalid argument value '%s' in file %s at position %d.", - value, site.getSource().getFile().getAbsolutePath(), site.getIndex()) ); + case Provider: + sb.append( String.format("%nInvalid argument value '%s' in %s at position %d.", + value.asString(), site.getSource().getDescription(), site.getIndex()) ); break; default: throw new RuntimeException( String.format("Unexpected argument match source type: %s", site.getSource().getType())); } - if(value != null && Utils.dupString(' ',value.length()).equals(value)) + if(value.asString() != null && Utils.dupString(' ',value.asString().length()).equals(value.asString())) sb.append(" Please make sure any line continuation backslashes on your command line are not followed by whitespace."); } return sb.toString(); @@ -696,12 +733,3 @@ class UnknownEnumeratedValueException extends ArgumentException { return String.format("Invalid value %s specified for argument %s; valid options are (%s).", argumentPassed, definition.fullName, Utils.join(",",definition.validOptions)); } } - -/** - * Container class to store the list of argument files. - * The files will be parsed after the command line arguments. - */ -class ParsingEngineArgumentFiles { - @Argument(fullName = "arg_file", shortName = "args", doc = "Reads arguments from the specified file", required = false) - public List files = new ArrayList(); -} diff --git a/public/java/src/org/broadinstitute/sting/commandline/ParsingEngineArgumentFiles.java b/public/java/src/org/broadinstitute/sting/commandline/ParsingEngineArgumentFiles.java new file mode 100644 index 000000000..3f3921937 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/commandline/ParsingEngineArgumentFiles.java @@ -0,0 +1,30 @@ +package org.broadinstitute.sting.commandline; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.SortedMap; + +/** + * Container class to store the list of argument files. + * The files will be parsed after the command line arguments. + */ +public class ParsingEngineArgumentFiles extends ParsingEngineArgumentProvider { + @Argument(fullName = "arg_file", shortName = "args", doc = "Reads arguments from the specified file", required = false) + public List files = new ArrayList(); + + @Override + public void parse(ParsingEngine parsingEngine, SortedMap parsedArgs) { + ArgumentMatches argumentMatches = parsingEngine.getArgumentMatches(); + for (File file: this.files) { + List fileTokens = parsingEngine.getArguments(file); + parsingEngine.parse(new ArgumentMatchFileSource(file), fileTokens, argumentMatches, parsedArgs); + } + } +} + +class ArgumentMatchFileSource extends ArgumentMatchSource { + ArgumentMatchFileSource(File file) { + super("file " + file.getAbsolutePath()); + } +} diff --git a/public/java/src/org/broadinstitute/sting/commandline/ParsingEngineArgumentProvider.java b/public/java/src/org/broadinstitute/sting/commandline/ParsingEngineArgumentProvider.java new file mode 100644 index 000000000..a57f8b08a --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/commandline/ParsingEngineArgumentProvider.java @@ -0,0 +1,12 @@ +package org.broadinstitute.sting.commandline; + +import java.util.List; +import java.util.SortedMap; + +/** + * A class that can parse arguments for the engine + */ +public abstract class ParsingEngineArgumentProvider { + public abstract void parse(ParsingEngine parsingEngine, SortedMap parsedArgs); +} + diff --git a/public/java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamArgumentTypeDescriptor.java b/public/java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamArgumentTypeDescriptor.java index da4eb3955..ac01468eb 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamArgumentTypeDescriptor.java +++ b/public/java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamArgumentTypeDescriptor.java @@ -86,7 +86,7 @@ public class OutputStreamArgumentTypeDescriptor extends ArgumentTypeDescriptor { @Override public Object parse( ParsingEngine parsingEngine, ArgumentSource source, Type type, ArgumentMatches matches ) { ArgumentDefinition definition = createDefaultArgumentDefinition(source); - String fileName = getArgumentValue( definition, matches ); + String fileName = getArgumentValue( definition, matches ).asString(); // This parser has been passed a null filename and the GATK is not responsible for creating a type default for the object; // therefore, the user must have failed to specify a type default diff --git a/public/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileReaderArgumentTypeDescriptor.java b/public/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileReaderArgumentTypeDescriptor.java index 83d1b7eb2..f13cb8fa8 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileReaderArgumentTypeDescriptor.java +++ b/public/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileReaderArgumentTypeDescriptor.java @@ -25,15 +25,11 @@ package org.broadinstitute.sting.gatk.io.stubs; import net.sf.samtools.SAMFileReader; -import org.broadinstitute.sting.commandline.ArgumentMatches; -import org.broadinstitute.sting.commandline.ArgumentSource; -import org.broadinstitute.sting.commandline.ArgumentTypeDescriptor; -import org.broadinstitute.sting.commandline.ParsingEngine; +import org.broadinstitute.sting.commandline.*; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.utils.exceptions.UserException; import org.broadinstitute.sting.utils.sam.SAMFileReaderBuilder; -import java.io.File; import java.lang.reflect.Type; /** @@ -47,7 +43,7 @@ public class SAMFileReaderArgumentTypeDescriptor extends ArgumentTypeDescriptor /** * Create a new SAMFileReader argument, notifying the given engine when that argument has been created. - * @param engine + * @param engine engine */ public SAMFileReaderArgumentTypeDescriptor( GenomeAnalysisEngine engine ) { this.engine = engine; @@ -62,12 +58,12 @@ public class SAMFileReaderArgumentTypeDescriptor extends ArgumentTypeDescriptor public Object parse( ParsingEngine parsingEngine, ArgumentSource source, Type type, ArgumentMatches matches ) { SAMFileReaderBuilder builder = new SAMFileReaderBuilder(); - String readerFileName = getArgumentValue( createDefaultArgumentDefinition(source), matches ); + ArgumentMatchValue readerFileName = getArgumentValue( createDefaultArgumentDefinition(source), matches ); if( readerFileName == null ) throw new UserException.CommandLineException("SAM file compression was supplied, but no associated writer was supplied with it."); - builder.setSAMFile(new File(readerFileName)); + builder.setSAMFile(readerFileName.asFile()); // WARNING: Skipping required side-effect because stub is impossible to generate. engine.addInput(source, builder); diff --git a/public/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java b/public/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java index dcf2704f5..2ea4bdfb0 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java +++ b/public/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java @@ -31,7 +31,6 @@ import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.gatk.io.StingSAMFileWriter; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import java.io.File; import java.io.OutputStream; import java.lang.annotation.Annotation; import java.lang.reflect.Type; @@ -111,10 +110,10 @@ public class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor public Object parse( ParsingEngine parsingEngine, ArgumentSource source, Type type, ArgumentMatches matches ) { // Extract all possible parameters that could be passed to a BAM file writer? ArgumentDefinition bamArgumentDefinition = createBAMArgumentDefinition(source); - String writerFileName = getArgumentValue( bamArgumentDefinition, matches ); + ArgumentMatchValue writerFileName = getArgumentValue( bamArgumentDefinition, matches ); - String compressionLevelText = getArgumentValue( createBAMCompressionArgumentDefinition(source), matches ); - Integer compressionLevel = compressionLevelText != null ? Integer.valueOf(compressionLevelText) : null; + ArgumentMatchValue compressionLevelText = getArgumentValue( createBAMCompressionArgumentDefinition(source), matches ); + Integer compressionLevel = compressionLevelText != null ? Integer.valueOf(compressionLevelText.asString()) : null; boolean indexOnTheFly = !argumentIsPresent(disableWriteIndexArgumentDefinition(source),matches); boolean generateMD5 = argumentIsPresent(this.enableMD5GenerationArgumentDefinition(source),matches); @@ -124,14 +123,14 @@ public class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor // This parser has been passed a null filename and the GATK is not responsible for creating a type default for the object; // therefore, the user must have failed to specify a type default - if(writerFileName == null && generateMD5) + if(writerFileName.asFile() == null && generateMD5) throw new ArgumentException("MD5 generation specified, but no output file specified. If md5 generation is desired, please specify a BAM output file and an md5 file will be written alongside."); // Create the stub and set parameters. SAMFileWriterStub stub = null; // stub = new SAMFileWriterStub(engine, defaultOutputStream); - if ( writerFileName != null ) { - stub = new SAMFileWriterStub(engine, new File(writerFileName)); + if ( writerFileName.asFile() != null ) { + stub = new SAMFileWriterStub(engine, writerFileName.asFile()); if ( compressionLevel != null ) stub.setCompressionLevel(compressionLevel); diff --git a/public/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java b/public/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java index 5e1132d45..43350ccc1 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java +++ b/public/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java @@ -138,8 +138,8 @@ public class VCFWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor { public Object parse( ParsingEngine parsingEngine, ArgumentSource source, Type type, ArgumentMatches matches ) { ArgumentDefinition defaultArgumentDefinition = createDefaultArgumentDefinition(source); // Get the filename for the genotype file, if it exists. If not, we'll need to send output to out. - String writerFileName = getArgumentValue(defaultArgumentDefinition,matches); - File writerFile = writerFileName != null ? new File(writerFileName) : null; + ArgumentMatchValue writerFileName = getArgumentValue(defaultArgumentDefinition,matches); + File writerFile = writerFileName != null ? writerFileName.asFile() : null; // This parser has been passed a null filename and the GATK is not responsible for creating a type default for the object; // therefore, the user must have failed to specify a type default @@ -151,7 +151,7 @@ public class VCFWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor { ? new VariantContextWriterStub(engine, writerFile, argumentSources) : new VariantContextWriterStub(engine, defaultOutputStream, argumentSources); - stub.setCompressed(isCompressed(writerFileName)); + stub.setCompressed(isCompressed(writerFileName.asString())); stub.setDoNotWriteGenotypes(argumentIsPresent(createSitesOnlyArgumentDefinition(),matches)); stub.setSkipWritingCommandLineHeader(argumentIsPresent(createNoCommandLineHeaderArgumentDefinition(),matches)); stub.setForceBCF(argumentIsPresent(createBCFArgumentDefinition(),matches)); diff --git a/public/java/src/org/broadinstitute/sting/utils/help/HelpFormatter.java b/public/java/src/org/broadinstitute/sting/utils/help/HelpFormatter.java index 25ef8ccd2..0f6808718 100755 --- a/public/java/src/org/broadinstitute/sting/utils/help/HelpFormatter.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/HelpFormatter.java @@ -26,10 +26,7 @@ package org.broadinstitute.sting.utils.help; import org.apache.log4j.Logger; -import org.broadinstitute.sting.commandline.ArgumentDefinition; -import org.broadinstitute.sting.commandline.ArgumentDefinitionGroup; -import org.broadinstitute.sting.commandline.ArgumentDefinitions; -import org.broadinstitute.sting.commandline.ArgumentMatchSource; +import org.broadinstitute.sting.commandline.*; import org.broadinstitute.sting.utils.Utils; import org.broadinstitute.sting.utils.text.TextFormattingUtils; @@ -273,9 +270,9 @@ public class HelpFormatter { * Generate a standard header for the logger * * @param applicationDetails details of the application to run. - * @param parsedArgs the command line arguments passed in + * @param parsedArgs the arguments passed in */ - public static void generateHeaderInformation(ApplicationDetails applicationDetails, Map> parsedArgs) { + public static void generateHeaderInformation(ApplicationDetails applicationDetails, Map parsedArgs) { DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); java.util.Date date = new java.util.Date(); @@ -286,19 +283,16 @@ public class HelpFormatter { for (String headerLine : applicationDetails.applicationHeader) logger.info(headerLine); logger.debug("Current directory: " + System.getProperty("user.dir")); - for (Map.Entry> entry: parsedArgs.entrySet()) { + for (Map.Entry entry: parsedArgs.entrySet()) { ArgumentMatchSource matchSource = entry.getKey(); final String sourceName; switch (matchSource.getType()) { case CommandLine: sourceName = "Program"; break; - case File: sourceName = matchSource.getFile().getPath(); break; + case Provider: sourceName = matchSource.getDescription(); break; default: throw new RuntimeException("Unexpected argument match source type: " + matchSource.getType()); } - String output = sourceName + " Args:"; - for (String str : entry.getValue()) { - output = output + " " + str; - } + String output = sourceName + " Args: " + entry.getValue().getDescription(); logger.info(output); } logger.info("Date/Time: " + dateFormat.format(date)); diff --git a/public/java/test/org/broadinstitute/sting/commandline/ArgumentMatchSiteUnitTest.java b/public/java/test/org/broadinstitute/sting/commandline/ArgumentMatchSiteUnitTest.java index 99d6b88f3..b1e788dc5 100644 --- a/public/java/test/org/broadinstitute/sting/commandline/ArgumentMatchSiteUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/commandline/ArgumentMatchSiteUnitTest.java @@ -39,7 +39,7 @@ public class ArgumentMatchSiteUnitTest { @Test public void testFile() { - ArgumentMatchSource source = new ArgumentMatchSource(new File("test")); + ArgumentMatchSource source = new ArgumentMatchFileSource(new File("test")); ArgumentMatchSite site = new ArgumentMatchSite(source, 1); Assert.assertEquals(site.getSource(), source); Assert.assertEquals(site.getIndex(), 1); diff --git a/public/java/test/org/broadinstitute/sting/commandline/ArgumentMatchSourceUnitTest.java b/public/java/test/org/broadinstitute/sting/commandline/ArgumentMatchSourceUnitTest.java index 4bc7eb822..a183b2001 100644 --- a/public/java/test/org/broadinstitute/sting/commandline/ArgumentMatchSourceUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/commandline/ArgumentMatchSourceUnitTest.java @@ -35,15 +35,15 @@ public class ArgumentMatchSourceUnitTest extends BaseTest { public void testCommandLine() { ArgumentMatchSource source = ArgumentMatchSource.COMMAND_LINE; Assert.assertEquals(source.getType(), ArgumentMatchSourceType.CommandLine); - Assert.assertNull(source.getFile()); + Assert.assertNull(source.getDescription()); } @Test public void testFile() { File f = new File("test"); - ArgumentMatchSource source = new ArgumentMatchSource(f); - Assert.assertEquals(source.getType(), ArgumentMatchSourceType.File); - Assert.assertEquals(source.getFile(), f); + ArgumentMatchSource source = new ArgumentMatchFileSource(f); + Assert.assertEquals(source.getType(), ArgumentMatchSourceType.Provider); + Assert.assertEquals(source.getDescription(), "file " + f.getAbsolutePath()); } @Test(expectedExceptions = IllegalArgumentException.class) @@ -54,8 +54,8 @@ public class ArgumentMatchSourceUnitTest extends BaseTest { @Test public void testEquals() { ArgumentMatchSource cmdLine = ArgumentMatchSource.COMMAND_LINE; - ArgumentMatchSource fileA = new ArgumentMatchSource(new File("a")); - ArgumentMatchSource fileB = new ArgumentMatchSource(new File("b")); + ArgumentMatchSource fileA = new ArgumentMatchFileSource(new File("a")); + ArgumentMatchSource fileB = new ArgumentMatchFileSource(new File("b")); Assert.assertFalse(cmdLine.equals(null)); @@ -75,8 +75,8 @@ public class ArgumentMatchSourceUnitTest extends BaseTest { @Test public void testCompareTo() { ArgumentMatchSource cmdLine = ArgumentMatchSource.COMMAND_LINE; - ArgumentMatchSource fileA = new ArgumentMatchSource(new File("a")); - ArgumentMatchSource fileB = new ArgumentMatchSource(new File("b")); + ArgumentMatchSource fileA = new ArgumentMatchFileSource(new File("a")); + ArgumentMatchSource fileB = new ArgumentMatchFileSource(new File("b")); Assert.assertTrue(cmdLine.compareTo(cmdLine) == 0); Assert.assertTrue(cmdLine.compareTo(fileA) < 0); diff --git a/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala b/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala index d0379d022..f4c4b613f 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala @@ -28,7 +28,7 @@ import function.QFunction import java.io.File import org.broadinstitute.sting.commandline._ import org.broadinstitute.sting.queue.util._ -import org.broadinstitute.sting.queue.engine.{QGraphSettings, QGraph} +import org.broadinstitute.sting.queue.engine.{QStatusMessenger, QGraphSettings, QGraph} import collection.JavaConversions._ import org.broadinstitute.sting.utils.classloader.PluginManager import org.broadinstitute.sting.utils.exceptions.UserException @@ -90,12 +90,16 @@ class QCommandLine extends CommandLineProgram with Logging { private var qScriptClasses: File = _ private var shuttingDown = false - private lazy val pluginManager = { + private lazy val qScriptPluginManager = { qScriptClasses = IOUtils.tempDir("Q-Classes-", "", settings.qSettings.tempDirectory) qScriptManager.loadScripts(scripts, qScriptClasses) new PluginManager[QScript](classOf[QScript], Seq(qScriptClasses.toURI.toURL)) } + private lazy val qStatusMessengerPluginManager = { + new PluginManager[QStatusMessenger](classOf[QStatusMessenger]) + } + QFunction.parsingEngine = new ParsingEngine(this) /** @@ -103,15 +107,25 @@ class QCommandLine extends CommandLineProgram with Logging { * functions, and then builds and runs a QGraph based on the dependencies. */ def execute = { + val allStatusMessengers = qStatusMessengerPluginManager.createAllTypes() + if (settings.qSettings.runName == null) settings.qSettings.runName = FilenameUtils.removeExtension(scripts.head.getName) if (IOUtils.isDefaultTempDir(settings.qSettings.tempDirectory)) settings.qSettings.tempDirectory = IOUtils.absolute(settings.qSettings.runDirectory, ".queue/tmp") qGraph.initializeWithSettings(settings) - val allQScripts = pluginManager.createAllTypes() + for (statusMessenger <- allStatusMessengers) { + loadArgumentsIntoObject(statusMessenger) + } + + for (statusMessenger <- allStatusMessengers) { + statusMessenger.started() + } + + val allQScripts = qScriptPluginManager.createAllTypes() for (script <- allQScripts) { - logger.info("Scripting " + pluginManager.getName(script.getClass.asSubclass(classOf[QScript]))) + logger.info("Scripting " + qScriptPluginManager.getName(script.getClass.asSubclass(classOf[QScript]))) loadArgumentsIntoObject(script) script.qSettings = settings.qSettings try { @@ -124,6 +138,10 @@ class QCommandLine extends CommandLineProgram with Logging { logger.info("Added " + script.functions.size + " functions") } + if (settings.run) { + allQScripts.foreach(_.pullInputs()) + } + // Execute the job graph qGraph.run() @@ -142,11 +160,18 @@ class QCommandLine extends CommandLineProgram with Logging { logger.info("Writing final jobs report...") qGraph.writeJobsReport() - if (!qGraph.success) { + if (!success) { logger.info("Done with errors") qGraph.logFailed() + for (statusMessenger <- allStatusMessengers) + statusMessenger.exit("Done with errors") 1 } else { + if (settings.run) { + allQScripts.foreach(_.pushOutputs()) + for (statusMessenger <- allStatusMessengers) + statusMessenger.done() + } 0 } } @@ -158,19 +183,30 @@ class QCommandLine extends CommandLineProgram with Logging { override def canAddArgumentsDynamically = true /** - * Returns the list of QScripts passed in via -S so that their - * arguments can be inspected before QScript.script is called. - * @return Array of QScripts passed in. + * Returns the list of QScripts passed in via -S and other plugins + * so that their arguments can be inspected before QScript.script is called. + * @return Array of dynamic sources */ - override def getArgumentSources = - pluginManager.getPlugins.toIterable.toArray.asInstanceOf[Array[Class[_]]] + override def getArgumentSources = { + var plugins = Seq.empty[Class[_]] + plugins ++= qScriptPluginManager.getPlugins + plugins ++= qStatusMessengerPluginManager.getPlugins + plugins.toArray + } /** - * Returns the name of a QScript - * @return The name of a QScript + * Returns the name of a script/plugin + * @return The name of a script/plugin */ - override def getArgumentSourceName(source: Class[_]) = - pluginManager.getName(source.asSubclass(classOf[QScript])) + override def getArgumentSourceName(source: Class[_]) = { + if (classOf[QScript].isAssignableFrom(source)) + qScriptPluginManager.getName(source.asSubclass(classOf[QScript])) + else if (classOf[QStatusMessenger].isAssignableFrom(source)) + qStatusMessengerPluginManager.getName(source.asSubclass(classOf[QStatusMessenger])) + else + null + + } /** * Returns a ScalaCompoundArgumentTypeDescriptor that can parse argument sources into scala collections. diff --git a/public/scala/src/org/broadinstitute/sting/queue/QScript.scala b/public/scala/src/org/broadinstitute/sting/queue/QScript.scala index 6f887ea00..c59220d4b 100755 --- a/public/scala/src/org/broadinstitute/sting/queue/QScript.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/QScript.scala @@ -27,7 +27,9 @@ package org.broadinstitute.sting.queue import engine.JobRunInfo import org.broadinstitute.sting.queue.function.QFunction import annotation.target.field -import util.{StringFileConversions, PrimitiveOptionConversions, Logging} +import util.{ReflectionUtils, StringFileConversions, PrimitiveOptionConversions, Logging} +import org.broadinstitute.sting.utils.classloader.JVMUtils +import java.lang.reflect.Field /** * Defines a Queue pipeline as a collection of CommandLineFunctions. @@ -106,6 +108,33 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon def addAll(functions: Seq[QFunction]) { functions.foreach( f => add(f) ) } + + def pullInputs() { + val inputs = getInputs + inputs.filter(_.isInstanceOf[RemoteFile]).map(_.asInstanceOf[RemoteFile]).foreach(_.pullToLocal()) + } + + def pushOutputs() { + val outputs = getOutputs + outputs.filter(_.isInstanceOf[RemoteFile]).map(_.asInstanceOf[RemoteFile]).foreach(_.pushToRemote()) + } + + private def getInputs: Seq[File] = { + getFieldValues(classOf[Input]) + } + + private def getOutputs: Seq[File] = { + getFieldValues(classOf[Output]) + } + + private def getFieldValues(annotation: Class[_ <: java.lang.annotation.Annotation]): Seq[File] = { + val filtered: Seq[Field] = fields.filter(field => ReflectionUtils.hasAnnotation(field, annotation)) + val files = filtered.filter(field => classOf[File].isAssignableFrom(field.getType)).map(field => ReflectionUtils.getValue(this, field).asInstanceOf[File]) + val seqFiles = filtered.filter(field => classOf[Seq[File]].isAssignableFrom(field.getType)).map(field => ReflectionUtils.getValue(this, field).asInstanceOf[Seq[File]]) + seqFiles.foldLeft(files)(_ ++ _).filter(_ != null) + } + + private lazy val fields = collection.JavaConversions.asScalaBuffer(JVMUtils.getAllFields(this.getClass)).toSeq } object QScript { diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/QStatusMessenger.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/QStatusMessenger.scala new file mode 100644 index 000000000..c61f2ef1f --- /dev/null +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/QStatusMessenger.scala @@ -0,0 +1,10 @@ +package org.broadinstitute.sting.queue.engine + +/** + * Plugin to sends QStatus messages + */ +trait QStatusMessenger { + def started() + def done() + def exit(message: String) +} diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala index e619c0a02..395a34c60 100755 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala @@ -92,6 +92,6 @@ object GATKIntervals { } private def createBinding(interval: String, argumentName: String, tags: Tags): IntervalBinding[Feature] = { - ArgumentTypeDescriptor.parseBinding(interval, classOf[Feature], classOf[IntervalBinding[Feature]], argumentName, tags, argumentName).asInstanceOf[IntervalBinding[Feature]] + ArgumentTypeDescriptor.parseBinding(new ArgumentMatchStringValue(interval), classOf[Feature], classOf[IntervalBinding[Feature]], argumentName, tags, argumentName).asInstanceOf[IntervalBinding[Feature]] } } diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/RemoteFile.scala b/public/scala/src/org/broadinstitute/sting/queue/util/RemoteFile.scala new file mode 100644 index 000000000..cfe848ba8 --- /dev/null +++ b/public/scala/src/org/broadinstitute/sting/queue/util/RemoteFile.scala @@ -0,0 +1,13 @@ +package org.broadinstitute.sting.queue.util + +import java.io.File +import org.broadinstitute.sting.utils.io.FileExtension + +/** + * An extension of java.io.File that can be pulled from or pushed to a remote location. + */ +trait RemoteFile extends File with FileExtension { + def pullToLocal() + def pushToRemote() + def deleteRemote() +}