Refactored argument matching to support other plugins in addition to file lists.

Added plugin support for sending Queue status messages.
Argument parsing can store subclasses of java.io.File, for example RemoteFile.

ivy.xml

@@ -78,8 +78,8 @@
         <dependency org="net.sf.gridscheduler" name="drmaa" rev="latest.integration"/>
 
         <!-- Scala dependancies -->
-        <dependency org="org.scala-lang" name="scala-compiler" rev="2.8.1"/>
+        <dependency org="org.scala-lang" name="scala-compiler" rev="2.9.2"/>
-        <dependency org="org.scala-lang" name="scala-library" rev="2.8.1"/>
+        <dependency org="org.scala-lang" name="scala-library" rev="2.9.2"/>
 
         <!-- testing and evaluation dependencies -->
         <dependency org="org.testng" name="testng" rev="5.14.1" conf="test"/>

public/java/src/org/broadinstitute/sting/commandline/ArgumentMatch.java

@@ -46,7 +46,7 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
     /**
      * Maps indices of command line arguments to values paired with that argument.
      */
-    public final SortedMap<ArgumentMatchSite,List<String>> sites = new TreeMap<ArgumentMatchSite,List<String>>();
+    public final SortedMap<ArgumentMatchSite,List<ArgumentMatchValue>> sites = new TreeMap<ArgumentMatchSite,List<ArgumentMatchValue>>();
 
     /**
      * An ordered, freeform collection of tags.
@@ -90,11 +90,11 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
      * @param value Value for the argument at this position.
      * @param tags ordered freeform text tags associated with this argument.
      */
-    private ArgumentMatch(final String label, final ArgumentDefinition definition, final ArgumentMatchSite site, final String value, final Tags tags) {
+    private ArgumentMatch(final String label, final ArgumentDefinition definition, final ArgumentMatchSite site, final ArgumentMatchValue value, final Tags tags) {
         this.label = label;
         this.definition = definition;
 
-        ArrayList<String> values = new ArrayList<String>();
+        ArrayList<ArgumentMatchValue> values = new ArrayList<ArgumentMatchValue>();
         if( value != null )
             values.add(value);
         sites.put(site,values );
@@ -131,11 +131,11 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
      */
     @SuppressWarnings("unchecked")
     ArgumentMatch transform(Multiplexer multiplexer, Object key) {
-        SortedMap<ArgumentMatchSite,List<String>> newIndices = new TreeMap<ArgumentMatchSite,List<String>>();
+        SortedMap<ArgumentMatchSite,List<ArgumentMatchValue>> newIndices = new TreeMap<ArgumentMatchSite,List<ArgumentMatchValue>>();
-        for(Map.Entry<ArgumentMatchSite,List<String>> site: sites.entrySet()) {
+        for(Map.Entry<ArgumentMatchSite,List<ArgumentMatchValue>> site: sites.entrySet()) {
-            List<String> newEntries = new ArrayList<String>();
+            List<ArgumentMatchValue> newEntries = new ArrayList<ArgumentMatchValue>();
-            for(String entry: site.getValue())
+            for(ArgumentMatchValue entry: site.getValue())
-                newEntries.add(multiplexer.transformArgument(key,entry));
+                newEntries.add(new ArgumentMatchStringValue(multiplexer.transformArgument(key,entry.asString())));
             newIndices.put(site.getKey(),newEntries);
         }
         ArgumentMatch newArgumentMatch = new ArgumentMatch(label,definition);
@@ -165,7 +165,7 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
             /**
              * Iterate over each available token.
              */
-            private Iterator<String> tokenIterator = null;
+            private Iterator<ArgumentMatchValue> tokenIterator = null;
 
             /**
              * The next site to return.  Null if none remain.
@@ -175,7 +175,7 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
             /**
              * The next token to return.  Null if none remain.
              */
-            String nextToken = null;
+            ArgumentMatchValue nextToken = null;
 
             {
                 siteIterator = sites.keySet().iterator();
@@ -254,9 +254,9 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
      * @param site site of the command-line argument to which this value is mated.
      * @param value Text representation of value to add.
      */
-    public void addValue( ArgumentMatchSite site, String value ) {
+    public void addValue( ArgumentMatchSite site, ArgumentMatchValue value ) {
         if( !sites.containsKey(site) || sites.get(site) == null )
-            sites.put(site, new ArrayList<String>() );
+            sites.put(site, new ArrayList<ArgumentMatchValue>() );
         sites.get(site).add(value);
     }
 
@@ -275,8 +275,8 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
      * Return the values associated with this argument match.
      * @return A collection of the string representation of these value.
      */
-    public List<String> values() {
+    public List<ArgumentMatchValue> values() {
-        List<String> values = new ArrayList<String>();
+        List<ArgumentMatchValue> values = new ArrayList<ArgumentMatchValue>();
         for( ArgumentMatchSite site: sites.keySet() ) {
             if( sites.get(site) != null )
                 values.addAll(sites.get(site));

public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchFileValue.java

@@ -0,0 +1,27 @@
+package org.broadinstitute.sting.commandline;
+
+import java.io.File;
+
+/**
+ * Holds a reference to a file as an argument match value.
+ *
+ * This is useful when the type of the stored file may be a subclass of java.io.File,
+ * for example a Queue RemoteFile.
+ */
+public class ArgumentMatchFileValue extends ArgumentMatchValue {
+    private final File file;
+
+    public ArgumentMatchFileValue(File file) {
+        this.file = file;
+    }
+
+    @Override
+    public String asString() {
+        return file == null ? null : file.getAbsolutePath();
+    }
+
+    @Override
+    public File asFile() {
+        return file;
+    }
+}

public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchSource.java

@@ -24,38 +24,36 @@
 
 package org.broadinstitute.sting.commandline;
 
-import java.io.File;
-
 /**
- * Where an argument match originated, via the commandline or a file.
+ * Where an argument match originated, via the commandline or a custom provider.
  */
 public class ArgumentMatchSource implements Comparable<ArgumentMatchSource> {
     public static final ArgumentMatchSource COMMAND_LINE = new ArgumentMatchSource(ArgumentMatchSourceType.CommandLine, null);
 
     private final ArgumentMatchSourceType type;
-    private final File file;
+    private final String description;
 
     /**
      * Creates an argument match source from the specified file.
-     * @param file File specifying the arguments. Must not be null.
+     * @param description Where the arguments originated.
      */
-    public ArgumentMatchSource(File file) {
+    public ArgumentMatchSource(String description) {
-        this(ArgumentMatchSourceType.File, file);
+        this(ArgumentMatchSourceType.Provider, description);
     }
 
-    private ArgumentMatchSource(ArgumentMatchSourceType type, File file) {
+    private ArgumentMatchSource(ArgumentMatchSourceType type, String description) {
-        if (type == ArgumentMatchSourceType.File && file == null)
+        if (type == ArgumentMatchSourceType.Provider && description == null)
-            throw new IllegalArgumentException("An argument match source of type File cannot have a null file.");
+            throw new IllegalArgumentException("An argument match source provider cannot have a null description.");
         this.type = type;
-        this.file = file;
+        this.description = description;
     }
 
     public ArgumentMatchSourceType getType() {
         return type;
     }
 
-    public File getFile() {
+    public String getDescription() {
-        return file;
+        return description;
     }
 
     @Override
@@ -65,13 +63,13 @@ public class ArgumentMatchSource implements Comparable<ArgumentMatchSource> {
 
         ArgumentMatchSource that = (ArgumentMatchSource) o;
 
-        return (type == that.type) && (file == null ? that.file == null : file.equals(that.file));
+        return (type == that.type) && (description == null ? that.description == null : description.equals(that.description));
     }
 
     @Override
     public int hashCode() {
         int result = type != null ? type.hashCode() : 0;
-        result = 31 * result + (file != null ? file.hashCode() : 0);
+        result = 31 * result + (description != null ? description.hashCode() : 0);
         return result;
     }
 
@@ -84,15 +82,15 @@ public class ArgumentMatchSource implements Comparable<ArgumentMatchSource> {
         if (comp != 0)
             return comp;
 
-        File f1 = this.file;
+        String d1 = this.description;
-        File f2 = that.file;
+        String d2 = that.description;
 
-        if ((f1 == null) ^ (f2 == null)) {
+        if ((d1 == null) ^ (d2 == null)) {
-            // If one of the files is null and the other is not
+            // If one of the descriptions is null and the other is not
-            // put the null file first
+            // put the null description first
-            return f1 == null ? -1 : 1;
+            return d1 == null ? -1 : 1;
         }
 
-        return f1 == null ? 0 : f1.compareTo(f2);
+        return d1 == null ? 0 : d1.compareTo(d2);
     }
 }

public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchSourceType.java

@@ -25,8 +25,8 @@
 package org.broadinstitute.sting.commandline;
 
 /**
- * Type of where an argument match originated, via the commandline or a file.
+ * Type of where an argument match originated, via the commandline or a some other provider.
  */
 public enum ArgumentMatchSourceType {
-    CommandLine, File
+    CommandLine, Provider
 }

public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchStringValue.java

@@ -0,0 +1,24 @@
+package org.broadinstitute.sting.commandline;
+
+import java.io.File;
+
+/**
+ * Argument values that originated from a string.
+ */
+public class ArgumentMatchStringValue extends ArgumentMatchValue {
+    private final String value;
+
+    public ArgumentMatchStringValue(String value) {
+        this.value = value;
+    }
+
+    @Override
+    public String asString() {
+        return value;
+    }
+
+    @Override
+    public File asFile() {
+        return value == null ? null : new File(value);
+    }
+}

public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchValue.java

@@ -0,0 +1,18 @@
+package org.broadinstitute.sting.commandline;
+
+import java.io.File;
+
+/**
+ * Returns argument values as either strings or values.
+ */
+public abstract class ArgumentMatchValue {
+    /**
+     * @return the value of this argument as a String object.
+     */
+    public abstract String asString();
+
+    /**
+     * @return the value of this argument as a File object.
+     */
+    public abstract File asFile();
+}

public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java

@@ -215,8 +215,8 @@ public abstract class ArgumentTypeDescriptor {
      * @param matches The matches for the given argument.
      * @return The value of the argument if available, or null if not present.
      */
-    protected String getArgumentValue( ArgumentDefinition definition, ArgumentMatches matches ) {
+    protected ArgumentMatchValue getArgumentValue( ArgumentDefinition definition, ArgumentMatches matches ) {
-        Collection<String> argumentValues = getArgumentValues( definition, matches );
+        Collection<ArgumentMatchValue> argumentValues = getArgumentValues( definition, matches );
         if( argumentValues.size() > 1 )
             throw new UserException.CommandLineException("Multiple values associated with given definition, but this argument expects only one: " + definition.fullName);
         return argumentValues.size() > 0 ? argumentValues.iterator().next() : null;
@@ -244,8 +244,8 @@ public abstract class ArgumentTypeDescriptor {
      * @param matches The matches for the given argument.
      * @return The value of the argument if available, or an empty collection if not present.
      */
-    protected Collection<String> getArgumentValues( ArgumentDefinition definition, ArgumentMatches matches ) {
+    protected Collection<ArgumentMatchValue> getArgumentValues( ArgumentDefinition definition, ArgumentMatches matches ) {
-        Collection<String> values = new ArrayList<String>();
+        Collection<ArgumentMatchValue> values = new ArrayList<ArgumentMatchValue>();
         for( ArgumentMatch match: matches ) {
             if( match.definition.equals(definition) )
                 values.addAll(match.values());
@@ -310,7 +310,7 @@ public abstract class ArgumentTypeDescriptor {
      */
     protected Object parseBinding(ArgumentSource source, Type type, ArgumentMatches matches, Tags tags) {
         ArgumentDefinition defaultDefinition = createDefaultArgumentDefinition(source);
-        String value = getArgumentValue(defaultDefinition, matches);
+        ArgumentMatchValue value = getArgumentValue(defaultDefinition, matches);
         @SuppressWarnings("unchecked")
         Class<? extends Feature> parameterType = JVMUtils.getParameterizedTypeClass(type);
         String name = defaultDefinition.fullName;
@@ -328,7 +328,7 @@ public abstract class ArgumentTypeDescriptor {
      * @param fieldName The name of the field that was parsed. Used for error reporting.
      * @return The newly created binding object of type bindingClass.
      */
-    public static Object parseBinding(String value, Class<? extends Feature> parameterType, Type bindingClass,
+    public static Object parseBinding(ArgumentMatchValue value, Class<? extends Feature> parameterType, Type bindingClass,
                                       String bindingName, Tags tags, String fieldName) {
         try {
             String tribbleType = null;
@@ -337,7 +337,7 @@ public abstract class ArgumentTypeDescriptor {
                 throw new UserException.CommandLineException(
                         String.format("Unexpected number of positional tags for argument %s : %s. " +
                                 "Rod bindings only support -X:type and -X:name,type argument styles",
-                                value, fieldName));
+                                value.asString(), fieldName));
             } else if ( tags.getPositionalTags().size() == 2 ) {
                 // -X:name,type style
                 bindingName = tags.getPositionalTags().get(0);
@@ -366,7 +366,7 @@ public abstract class ArgumentTypeDescriptor {
 
                 if ( tribbleType == null ) {
                     // try to determine the file type dynamically
-                    File file = new File(value);
+                    File file = value.asFile();
                     if ( file.canRead() && file.isFile() ) {
                         FeatureManager.FeatureDescriptor featureDescriptor = manager.getByFiletype(file);
                         if ( featureDescriptor != null ) {
@@ -379,7 +379,7 @@ public abstract class ArgumentTypeDescriptor {
                         // IntervalBinding can be created from a normal String
                         Class rawType = (makeRawTypeIfNecessary(bindingClass));
                         try {
-                            return rawType.getConstructor(String.class).newInstance(value);
+                            return rawType.getConstructor(String.class).newInstance(value.asString());
                         } catch (NoSuchMethodException e) {
                             /* ignore */
                         }
@@ -399,14 +399,14 @@ public abstract class ArgumentTypeDescriptor {
             }
 
             Constructor ctor = (makeRawTypeIfNecessary(bindingClass)).getConstructor(Class.class, String.class, String.class, String.class, Tags.class);
-            return ctor.newInstance(parameterType, bindingName, value, tribbleType, tags);
+            return ctor.newInstance(parameterType, bindingName, value.asString(), tribbleType, tags);
         } catch (Exception e) {
             if ( e instanceof UserException )
                 throw ((UserException)e);
             else
                 throw new UserException.CommandLineException(
                         String.format("Failed to parse value %s for argument %s. Message: %s",
-                                value, fieldName, e.getMessage()));
+                                value.asString(), fieldName, e.getMessage()));
         }
     }
 }
@@ -517,7 +517,7 @@ class SimpleArgumentTypeDescriptor extends ArgumentTypeDescriptor {
             return true;
 
         ArgumentDefinition defaultDefinition = createDefaultArgumentDefinition(source);
-        String value = getArgumentValue( defaultDefinition, matches );
+        ArgumentMatchValue value = getArgumentValue(defaultDefinition, matches);
         Object result;
         Tags tags = getArgumentTags(matches);
 
@@ -527,12 +527,12 @@ class SimpleArgumentTypeDescriptor extends ArgumentTypeDescriptor {
                 Method valueOf = primitiveToWrapperMap.get(type).getMethod("valueOf",String.class);
                 if(value == null)
                     throw new MissingArgumentValueException(createDefaultArgumentDefinition(source));
-                result = valueOf.invoke(null,value.trim());
+                result = valueOf.invoke(null,value.asString().trim());
             } else if (type.isEnum()) {
                 Object[] vals = type.getEnumConstants();
                 Object defaultEnumeration = null;  // as we look at options, record the default option if it exists
                 for (Object val : vals) {
-                    if (String.valueOf(val).equalsIgnoreCase(value)) return val;
+                    if (String.valueOf(val).equalsIgnoreCase(value.asString())) return val;
                     try { if (type.getField(val.toString()).isAnnotationPresent(EnumerationArgumentDefault.class)) defaultEnumeration = val; }
                     catch (NoSuchFieldException e) { throw new ReviewedStingException("parsing " + type.toString() + "doesn't contain the field " + val.toString()); }
                 }
@@ -544,10 +544,12 @@ class SimpleArgumentTypeDescriptor extends ArgumentTypeDescriptor {
                 else if (value == null)
                     throw new MissingArgumentValueException(createDefaultArgumentDefinition(source));
                 else
-                    throw new UnknownEnumeratedValueException(createDefaultArgumentDefinition(source),value);
+                    throw new UnknownEnumeratedValueException(createDefaultArgumentDefinition(source),value.asString());
+            } else if (type.equals(File.class)) {
+                result = value.asFile();
             } else {
                 Constructor ctor = type.getConstructor(String.class);
-                result = ctor.newInstance(value);
+                result = ctor.newInstance(value.asString());
             }
         } catch (UserException e) {
             throw e;

public/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java

@@ -174,7 +174,7 @@ public abstract class CommandLineProgram {
             ParsingEngine parser = clp.parser = new ParsingEngine(clp);
             parser.addArgumentSource(clp.getClass());
 
-            Map<ArgumentMatchSource, List<String>> parsedArgs;
+            Map<ArgumentMatchSource, ParsedArgs> parsedArgs;
 
             // process the args
             if (clp.canAddArgumentsDynamically()) {

public/java/src/org/broadinstitute/sting/commandline/ParsedArgs.java

@@ -0,0 +1,13 @@
+package org.broadinstitute.sting.commandline;
+
+/**
+ * Represents a collection of parsed arguments for an argument source.
+ *
+ * Useful for printing out help documents.
+ */
+public abstract class ParsedArgs {
+    /**
+     * @return A compact description of the arguments from an provider/source.
+     */
+    public abstract String getDescription();
+}

public/java/src/org/broadinstitute/sting/commandline/ParsedListArgs.java

@@ -0,0 +1,30 @@
+package org.broadinstitute.sting.commandline;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * A list of string arguments, usually from the command line or an args list file.
+ */
+public class ParsedListArgs extends ParsedArgs {
+    private final List<String> args = new ArrayList<String>();
+
+    public ParsedListArgs() {
+    }
+
+    public ParsedListArgs(List<String> args) {
+        this.args.addAll(args);
+    }
+
+    public void add(String... args) {
+        this.args.addAll(Arrays.asList(args));
+    }
+
+    @Override
+    public String getDescription() {
+        return StringUtils.join(this.args, " ");
+    }
+}

public/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java

@@ -30,6 +30,7 @@ import org.apache.commons.io.FileUtils;
 import org.apache.log4j.Logger;
 import org.broadinstitute.sting.utils.Utils;
 import org.broadinstitute.sting.utils.classloader.JVMUtils;
+import org.broadinstitute.sting.utils.classloader.PluginManager;
 import org.broadinstitute.sting.utils.collections.Pair;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.broadinstitute.sting.utils.exceptions.UserException;
@@ -61,7 +62,7 @@ public class ParsingEngine {
      * Indicates as best as possible where command-line text remains unmatched
      * to existing arguments.
      */
-    ArgumentMatches argumentMatches = null;
+    private ArgumentMatches argumentMatches = null;
 
     /**
      * Techniques for parsing and for argument lookup.
@@ -88,7 +89,10 @@ public class ParsingEngine {
     /**
      * List of tags associated with the given instantiation of the command-line argument.
      */
-    private final Map<Object,Tags> tags = new IdentityHashMap<Object,Tags>();    
+    private final Map<Object,Tags> tags = new IdentityHashMap<Object,Tags>();
+
+    private PluginManager<ParsingEngineArgumentProvider> argumentProviderPluginManager =
+            new PluginManager<ParsingEngineArgumentProvider>(ParsingEngineArgumentProvider.class);
 
     /**
      * our log, which we want to capture anything from org.broadinstitute.sting
@@ -105,7 +109,10 @@ public class ParsingEngine {
             argumentTypeDescriptors.addAll(clp.getArgumentTypeDescriptors());
         argumentTypeDescriptors.addAll(STANDARD_ARGUMENT_TYPE_DESCRIPTORS);
 
-        addArgumentSource(ParsingEngineArgumentFiles.class);
+        List<Class<? extends ParsingEngineArgumentProvider>> providers = argumentProviderPluginManager.getPlugins();
+        for (Class<? extends ParsingEngineArgumentProvider> provider: providers) {
+            addArgumentSource(provider);
+        }
     }
 
     /**
@@ -117,6 +124,10 @@ public class ParsingEngine {
         addArgumentSource(null, source);
     }
 
+    public ArgumentMatches getArgumentMatches() {
+        return argumentMatches;
+    }
+
     /**
      * Add an argument source.  Argument sources are expected to have
      * any number of fields with an @Argument annotation attached.
@@ -156,29 +167,30 @@ public class ParsingEngine {
      * @param tokens Tokens passed on the command line.
      * @return The parsed arguments by file.
      */
-    public SortedMap<ArgumentMatchSource, List<String>> parse( String[] tokens ) {
+    public SortedMap<ArgumentMatchSource, ParsedArgs> parse( String[] tokens ) {
         argumentMatches = new ArgumentMatches();
-        SortedMap<ArgumentMatchSource, List<String>> parsedArgs = new TreeMap<ArgumentMatchSource, List<String>>();
+        SortedMap<ArgumentMatchSource, ParsedArgs> parsedArgs = new TreeMap<ArgumentMatchSource, ParsedArgs>();
 
         List<String> cmdLineTokens = Arrays.asList(tokens);
         parse(ArgumentMatchSource.COMMAND_LINE, cmdLineTokens, argumentMatches, parsedArgs);
 
-        ParsingEngineArgumentFiles argumentFiles = new ParsingEngineArgumentFiles();
+        List<ParsingEngineArgumentProvider> providers = argumentProviderPluginManager.createAllTypes();
 
-        // Load the arguments ONLY into the argument files.
+        for (ParsingEngineArgumentProvider provider: providers) {
-        // Validation may optionally run on the rest of the arguments.
+            // Load the arguments ONLY into the provider.
-        loadArgumentsIntoObject(argumentFiles);
+            // Validation may optionally run on the rest of the arguments.
+            loadArgumentsIntoObject(provider);
+        }
 
-        for (File file: argumentFiles.files) {
+        for (ParsingEngineArgumentProvider provider: providers) {
-            List<String> fileTokens = getArguments(file);
+            provider.parse(this, parsedArgs);
-            parse(new ArgumentMatchSource(file), fileTokens, argumentMatches, parsedArgs);
         }
 
         return parsedArgs;
     }
 
-    private void parse(ArgumentMatchSource matchSource, List<String> tokens,
+    public void parse(ArgumentMatchSource matchSource, List<String> tokens,
-                       ArgumentMatches argumentMatches, SortedMap<ArgumentMatchSource, List<String>> parsedArgs) {
+                         ArgumentMatches argumentMatches, SortedMap<ArgumentMatchSource, ParsedArgs> parsedArgs) {
         ArgumentMatchSite lastArgumentMatchSite = new ArgumentMatchSite(matchSource, -1);
 
         int i = 0;
@@ -195,19 +207,44 @@ public class ParsingEngine {
             }
             else {
                 if( argumentMatches.hasMatch(lastArgumentMatchSite) &&
-                    !argumentMatches.getMatch(lastArgumentMatchSite).hasValueAtSite(lastArgumentMatchSite))
+                        !argumentMatches.getMatch(lastArgumentMatchSite).hasValueAtSite(lastArgumentMatchSite))
-                    argumentMatches.getMatch(lastArgumentMatchSite).addValue( lastArgumentMatchSite, token );
+                    argumentMatches.getMatch(lastArgumentMatchSite).addValue( lastArgumentMatchSite, new ArgumentMatchStringValue(token) );
                 else
-                    argumentMatches.MissingArgument.addValue( site, token );
+                    argumentMatches.MissingArgument.addValue( site, new ArgumentMatchStringValue(token) );
 
             }
             i++;
         }
 
-        parsedArgs.put(matchSource, tokens);
+        parsedArgs.put(matchSource, new ParsedListArgs(tokens));
     }
 
-    private List<String> getArguments(File file) {
+    public void parsePairs(ArgumentMatchSource matchSource, List<Pair<String, ArgumentMatchValue>> tokens,
+                         ArgumentMatches argumentMatches, ParsedArgs matchSourceArgs,
+                         SortedMap<ArgumentMatchSource, ParsedArgs> parsedArgs) {
+        int i = 0;
+        for (Pair<String, ArgumentMatchValue> pair: tokens) {
+
+            ArgumentMatchSite site = new ArgumentMatchSite(matchSource, i);
+            List<DefinitionMatcher> matchers = Arrays.asList(ArgumentDefinitions.FullNameDefinitionMatcher, ArgumentDefinitions.ShortNameDefinitionMatcher);
+            ArgumentDefinition definition = null;
+            for (DefinitionMatcher matcher: matchers) {
+                definition = argumentDefinitions.findArgumentDefinition( pair.getFirst(), matcher );
+                if (definition != null)
+                    break;
+            }
+            if (definition == null)
+                continue;
+            ArgumentMatch argumentMatch = new ArgumentMatch(pair.getFirst(), definition, site, new Tags());
+            argumentMatches.mergeInto(argumentMatch);
+            argumentMatch.addValue(site, pair.getSecond());
+            i++;
+        }
+
+        parsedArgs.put(matchSource, matchSourceArgs);
+    }
+
+    protected List<String> getArguments(File file) {
         try {
             if (file.getAbsolutePath().endsWith(".list")) {
                 return getListArguments(file);
@@ -283,9 +320,9 @@ public class ParsingEngine {
 
                 // Ensure that the field contents meet the validation criteria specified by the regular expression.
                 for( ArgumentMatch verifiableMatch: verifiableMatches ) {
-                    for( String value: verifiableMatch.values() ) {
+                    for( ArgumentMatchValue value: verifiableMatch.values() ) {
-                        if( verifiableArgument.validation != null && !value.matches(verifiableArgument.validation) )
+                        if( verifiableArgument.validation != null && !value.asString().matches(verifiableArgument.validation) )
-                            invalidValues.add( new Pair<ArgumentDefinition,String>(verifiableArgument, value) );
+                            invalidValues.add( new Pair<ArgumentDefinition,String>(verifiableArgument, value.asString()) );
                     }
                 }
             }
@@ -629,21 +666,21 @@ class UnmatchedArgumentException extends ArgumentException {
     private static String formatArguments( ArgumentMatch invalidValues ) {
         StringBuilder sb = new StringBuilder();
         for( ArgumentMatchSite site: invalidValues.sites.keySet() )
-            for( String value: invalidValues.sites.get(site) ) {
+            for( ArgumentMatchValue value: invalidValues.sites.get(site) ) {
                 switch (site.getSource().getType()) {
                     case CommandLine:
                         sb.append( String.format("%nInvalid argument value '%s' at position %d.",
-                                value, site.getIndex()) );
+                                value.asString(), site.getIndex()) );
                         break;
-                    case File:
+                    case Provider:
-                        sb.append( String.format("%nInvalid argument value '%s' in file %s at position %d.",
+                        sb.append( String.format("%nInvalid argument value '%s' in %s at position %d.",
-                                value, site.getSource().getFile().getAbsolutePath(), site.getIndex()) );
+                                value.asString(), site.getSource().getDescription(), site.getIndex()) );
                         break;
                     default:
                         throw new RuntimeException( String.format("Unexpected argument match source type: %s",
                                 site.getSource().getType()));
                 }
-                if(value != null && Utils.dupString(' ',value.length()).equals(value))
+                if(value.asString() != null && Utils.dupString(' ',value.asString().length()).equals(value.asString()))
                     sb.append("  Please make sure any line continuation backslashes on your command line are not followed by whitespace.");
             }
         return sb.toString();
@@ -696,12 +733,3 @@ class UnknownEnumeratedValueException extends ArgumentException {
         return String.format("Invalid value %s specified for argument %s; valid options are (%s).", argumentPassed, definition.fullName, Utils.join(",",definition.validOptions));
     }
 }
-
-/**
- * Container class to store the list of argument files.
- * The files will be parsed after the command line arguments.
- */
-class ParsingEngineArgumentFiles {
-    @Argument(fullName = "arg_file", shortName = "args", doc = "Reads arguments from the specified file", required = false)
-    public List<File> files = new ArrayList<File>();
-}

public/java/src/org/broadinstitute/sting/commandline/ParsingEngineArgumentFiles.java

@@ -0,0 +1,30 @@
+package org.broadinstitute.sting.commandline;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.SortedMap;
+
+/**
+ * Container class to store the list of argument files.
+ * The files will be parsed after the command line arguments.
+ */
+public class ParsingEngineArgumentFiles extends ParsingEngineArgumentProvider {
+    @Argument(fullName = "arg_file", shortName = "args", doc = "Reads arguments from the specified file", required = false)
+    public List<File> files = new ArrayList<File>();
+
+    @Override
+    public void parse(ParsingEngine parsingEngine, SortedMap<ArgumentMatchSource, ParsedArgs> parsedArgs) {
+        ArgumentMatches argumentMatches = parsingEngine.getArgumentMatches();
+        for (File file: this.files) {
+            List<String> fileTokens = parsingEngine.getArguments(file);
+            parsingEngine.parse(new ArgumentMatchFileSource(file), fileTokens, argumentMatches, parsedArgs);
+        }
+    }
+}
+
+class ArgumentMatchFileSource extends ArgumentMatchSource {
+    ArgumentMatchFileSource(File file) {
+        super("file " + file.getAbsolutePath());
+    }
+}

public/java/src/org/broadinstitute/sting/commandline/ParsingEngineArgumentProvider.java

@@ -0,0 +1,12 @@
+package org.broadinstitute.sting.commandline;
+
+import java.util.List;
+import java.util.SortedMap;
+
+/**
+ * A class that can parse arguments for the engine
+ */
+public abstract class ParsingEngineArgumentProvider {
+    public abstract void parse(ParsingEngine parsingEngine, SortedMap<ArgumentMatchSource, ParsedArgs> parsedArgs);
+}
+

public/java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamArgumentTypeDescriptor.java

@@ -86,7 +86,7 @@ public class OutputStreamArgumentTypeDescriptor extends ArgumentTypeDescriptor {
     @Override
     public Object parse( ParsingEngine parsingEngine, ArgumentSource source, Type type, ArgumentMatches matches )  {
         ArgumentDefinition definition = createDefaultArgumentDefinition(source);
-        String fileName = getArgumentValue( definition, matches );
+        String fileName = getArgumentValue( definition, matches ).asString();
 
         // This parser has been passed a null filename and the GATK is not responsible for creating a type default for the object;
         // therefore, the user must have failed to specify a type default

public/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileReaderArgumentTypeDescriptor.java

@@ -25,15 +25,11 @@
 package org.broadinstitute.sting.gatk.io.stubs;
 
 import net.sf.samtools.SAMFileReader;
-import org.broadinstitute.sting.commandline.ArgumentMatches;
+import org.broadinstitute.sting.commandline.*;
-import org.broadinstitute.sting.commandline.ArgumentSource;
-import org.broadinstitute.sting.commandline.ArgumentTypeDescriptor;
-import org.broadinstitute.sting.commandline.ParsingEngine;
 import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
 import org.broadinstitute.sting.utils.exceptions.UserException;
 import org.broadinstitute.sting.utils.sam.SAMFileReaderBuilder;
 
-import java.io.File;
 import java.lang.reflect.Type;
 
 /**
@@ -47,7 +43,7 @@ public class SAMFileReaderArgumentTypeDescriptor extends ArgumentTypeDescriptor
 
     /**
      * Create a new SAMFileReader argument, notifying the given engine when that argument has been created.
-     * @param engine
+     * @param engine engine
      */
     public SAMFileReaderArgumentTypeDescriptor( GenomeAnalysisEngine engine ) {
         this.engine = engine;
@@ -62,12 +58,12 @@ public class SAMFileReaderArgumentTypeDescriptor extends ArgumentTypeDescriptor
     public Object parse( ParsingEngine parsingEngine, ArgumentSource source, Type type, ArgumentMatches matches ) {
         SAMFileReaderBuilder builder = new SAMFileReaderBuilder();
 
-        String readerFileName = getArgumentValue( createDefaultArgumentDefinition(source), matches );
+        ArgumentMatchValue readerFileName = getArgumentValue( createDefaultArgumentDefinition(source), matches );
 
         if( readerFileName == null )
             throw new UserException.CommandLineException("SAM file compression was supplied, but no associated writer was supplied with it.");
 
-        builder.setSAMFile(new File(readerFileName));
+        builder.setSAMFile(readerFileName.asFile());
 
         // WARNING: Skipping required side-effect because stub is impossible to generate.
         engine.addInput(source, builder);

public/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java

@@ -31,7 +31,6 @@ import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
 import org.broadinstitute.sting.gatk.io.StingSAMFileWriter;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 
-import java.io.File;
 import java.io.OutputStream;
 import java.lang.annotation.Annotation;
 import java.lang.reflect.Type;
@@ -111,10 +110,10 @@ public class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor
     public Object parse( ParsingEngine parsingEngine, ArgumentSource source, Type type, ArgumentMatches matches )  {
         // Extract all possible parameters that could be passed to a BAM file writer?
         ArgumentDefinition bamArgumentDefinition = createBAMArgumentDefinition(source);
-        String writerFileName = getArgumentValue( bamArgumentDefinition, matches );
+        ArgumentMatchValue writerFileName = getArgumentValue( bamArgumentDefinition, matches );
 
-        String compressionLevelText = getArgumentValue( createBAMCompressionArgumentDefinition(source), matches );
+        ArgumentMatchValue compressionLevelText = getArgumentValue( createBAMCompressionArgumentDefinition(source), matches );
-        Integer compressionLevel = compressionLevelText != null ? Integer.valueOf(compressionLevelText) : null;
+        Integer compressionLevel = compressionLevelText != null ? Integer.valueOf(compressionLevelText.asString()) : null;
 
         boolean indexOnTheFly = !argumentIsPresent(disableWriteIndexArgumentDefinition(source),matches);
         boolean generateMD5 = argumentIsPresent(this.enableMD5GenerationArgumentDefinition(source),matches);
@@ -124,14 +123,14 @@ public class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor
 
         // This parser has been passed a null filename and the GATK is not responsible for creating a type default for the object;
         // therefore, the user must have failed to specify a type default
-        if(writerFileName == null && generateMD5)
+        if(writerFileName.asFile() == null && generateMD5)
                 throw new ArgumentException("MD5 generation specified, but no output file specified.  If md5 generation is desired, please specify a BAM output file and an md5 file will be written alongside.");
 
         // Create the stub and set parameters.
         SAMFileWriterStub stub = null;      // stub = new SAMFileWriterStub(engine, defaultOutputStream);
 
-        if ( writerFileName != null ) {
+        if ( writerFileName.asFile() != null ) {
-            stub = new SAMFileWriterStub(engine, new File(writerFileName));
+            stub = new SAMFileWriterStub(engine, writerFileName.asFile());
 
             if ( compressionLevel != null )
                 stub.setCompressionLevel(compressionLevel);

public/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java

@@ -138,8 +138,8 @@ public class VCFWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor {
     public Object parse( ParsingEngine parsingEngine, ArgumentSource source, Type type, ArgumentMatches matches )  {
         ArgumentDefinition defaultArgumentDefinition = createDefaultArgumentDefinition(source);
         // Get the filename for the genotype file, if it exists.  If not, we'll need to send output to out.
-        String writerFileName = getArgumentValue(defaultArgumentDefinition,matches);
+        ArgumentMatchValue writerFileName = getArgumentValue(defaultArgumentDefinition,matches);
-        File writerFile = writerFileName != null ? new File(writerFileName) : null;
+        File writerFile = writerFileName != null ? writerFileName.asFile() : null;
 
         // This parser has been passed a null filename and the GATK is not responsible for creating a type default for the object;
         // therefore, the user must have failed to specify a type default
@@ -151,7 +151,7 @@ public class VCFWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor {
                 ? new VariantContextWriterStub(engine, writerFile, argumentSources)
                 : new VariantContextWriterStub(engine, defaultOutputStream, argumentSources);
 
-        stub.setCompressed(isCompressed(writerFileName));
+        stub.setCompressed(isCompressed(writerFileName.asString()));
         stub.setDoNotWriteGenotypes(argumentIsPresent(createSitesOnlyArgumentDefinition(),matches));
         stub.setSkipWritingCommandLineHeader(argumentIsPresent(createNoCommandLineHeaderArgumentDefinition(),matches));
         stub.setForceBCF(argumentIsPresent(createBCFArgumentDefinition(),matches));

public/java/src/org/broadinstitute/sting/utils/help/HelpFormatter.java

@@ -26,10 +26,7 @@
 package org.broadinstitute.sting.utils.help;
 
 import org.apache.log4j.Logger;
-import org.broadinstitute.sting.commandline.ArgumentDefinition;
+import org.broadinstitute.sting.commandline.*;
-import org.broadinstitute.sting.commandline.ArgumentDefinitionGroup;
-import org.broadinstitute.sting.commandline.ArgumentDefinitions;
-import org.broadinstitute.sting.commandline.ArgumentMatchSource;
 import org.broadinstitute.sting.utils.Utils;
 import org.broadinstitute.sting.utils.text.TextFormattingUtils;
 
@@ -273,9 +270,9 @@ public class HelpFormatter {
      * Generate a standard header for the logger
      *
      * @param applicationDetails details of the application to run.
-     * @param parsedArgs the command line arguments passed in
+     * @param parsedArgs the arguments passed in
      */
-    public static void generateHeaderInformation(ApplicationDetails applicationDetails, Map<ArgumentMatchSource, List<String>> parsedArgs) {
+    public static void generateHeaderInformation(ApplicationDetails applicationDetails, Map<ArgumentMatchSource, ParsedArgs> parsedArgs) {
 
         DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
         java.util.Date date = new java.util.Date();
@@ -286,19 +283,16 @@ public class HelpFormatter {
         for (String headerLine : applicationDetails.applicationHeader)
             logger.info(headerLine);
         logger.debug("Current directory: " + System.getProperty("user.dir"));
-        for (Map.Entry<ArgumentMatchSource, List<String>> entry: parsedArgs.entrySet()) {
+        for (Map.Entry<ArgumentMatchSource, ParsedArgs> entry: parsedArgs.entrySet()) {
             ArgumentMatchSource matchSource = entry.getKey();
             final String sourceName;
             switch (matchSource.getType()) {
                 case CommandLine: sourceName = "Program"; break;
-                case File: sourceName = matchSource.getFile().getPath(); break;
+                case Provider: sourceName = matchSource.getDescription(); break;
                 default: throw new RuntimeException("Unexpected argument match source type: " + matchSource.getType());
             }
 
-            String output = sourceName + " Args:";
+            String output = sourceName + " Args: " + entry.getValue().getDescription();
-            for (String str : entry.getValue()) {
-                output = output + " " + str;
-            }
             logger.info(output);
         }
         logger.info("Date/Time: " + dateFormat.format(date));

public/java/test/org/broadinstitute/sting/commandline/ArgumentMatchSiteUnitTest.java

@@ -39,7 +39,7 @@ public class ArgumentMatchSiteUnitTest {
 
     @Test
     public void testFile() {
-        ArgumentMatchSource source = new ArgumentMatchSource(new File("test"));
+        ArgumentMatchSource source = new ArgumentMatchFileSource(new File("test"));
         ArgumentMatchSite site = new ArgumentMatchSite(source, 1);
         Assert.assertEquals(site.getSource(), source);
         Assert.assertEquals(site.getIndex(), 1);

public/java/test/org/broadinstitute/sting/commandline/ArgumentMatchSourceUnitTest.java

@@ -35,15 +35,15 @@ public class ArgumentMatchSourceUnitTest extends BaseTest {
     public void testCommandLine() {
         ArgumentMatchSource source = ArgumentMatchSource.COMMAND_LINE;
         Assert.assertEquals(source.getType(), ArgumentMatchSourceType.CommandLine);
-        Assert.assertNull(source.getFile());
+        Assert.assertNull(source.getDescription());
     }
 
     @Test
     public void testFile() {
         File f = new File("test");
-        ArgumentMatchSource source = new ArgumentMatchSource(f);
+        ArgumentMatchSource source = new ArgumentMatchFileSource(f);
-        Assert.assertEquals(source.getType(), ArgumentMatchSourceType.File);
+        Assert.assertEquals(source.getType(), ArgumentMatchSourceType.Provider);
-        Assert.assertEquals(source.getFile(), f);
+        Assert.assertEquals(source.getDescription(), "file " + f.getAbsolutePath());
     }
 
     @Test(expectedExceptions = IllegalArgumentException.class)
@@ -54,8 +54,8 @@ public class ArgumentMatchSourceUnitTest extends BaseTest {
     @Test
     public void testEquals() {
         ArgumentMatchSource cmdLine = ArgumentMatchSource.COMMAND_LINE;
-        ArgumentMatchSource fileA = new ArgumentMatchSource(new File("a"));
+        ArgumentMatchSource fileA = new ArgumentMatchFileSource(new File("a"));
-        ArgumentMatchSource fileB = new ArgumentMatchSource(new File("b"));
+        ArgumentMatchSource fileB = new ArgumentMatchFileSource(new File("b"));
 
         Assert.assertFalse(cmdLine.equals(null));
 
@@ -75,8 +75,8 @@ public class ArgumentMatchSourceUnitTest extends BaseTest {
     @Test
     public void testCompareTo() {
         ArgumentMatchSource cmdLine = ArgumentMatchSource.COMMAND_LINE;
-        ArgumentMatchSource fileA = new ArgumentMatchSource(new File("a"));
+        ArgumentMatchSource fileA = new ArgumentMatchFileSource(new File("a"));
-        ArgumentMatchSource fileB = new ArgumentMatchSource(new File("b"));
+        ArgumentMatchSource fileB = new ArgumentMatchFileSource(new File("b"));
 
         Assert.assertTrue(cmdLine.compareTo(cmdLine) == 0);
         Assert.assertTrue(cmdLine.compareTo(fileA) < 0);

public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala

@@ -28,7 +28,7 @@ import function.QFunction
 import java.io.File
 import org.broadinstitute.sting.commandline._
 import org.broadinstitute.sting.queue.util._
-import org.broadinstitute.sting.queue.engine.{QGraphSettings, QGraph}
+import org.broadinstitute.sting.queue.engine.{QStatusMessenger, QGraphSettings, QGraph}
 import collection.JavaConversions._
 import org.broadinstitute.sting.utils.classloader.PluginManager
 import org.broadinstitute.sting.utils.exceptions.UserException
@@ -90,12 +90,16 @@ class QCommandLine extends CommandLineProgram with Logging {
   private var qScriptClasses: File = _
   private var shuttingDown = false
 
-  private lazy val pluginManager = {
+  private lazy val qScriptPluginManager = {
     qScriptClasses = IOUtils.tempDir("Q-Classes-", "", settings.qSettings.tempDirectory)
     qScriptManager.loadScripts(scripts, qScriptClasses)
     new PluginManager[QScript](classOf[QScript], Seq(qScriptClasses.toURI.toURL))
   }
 
+  private lazy val qStatusMessengerPluginManager = {
+    new PluginManager[QStatusMessenger](classOf[QStatusMessenger])
+  }
+
   QFunction.parsingEngine = new ParsingEngine(this)
 
   /**
@@ -103,15 +107,25 @@ class QCommandLine extends CommandLineProgram with Logging {
    * functions, and then builds and runs a QGraph based on the dependencies.
    */
   def execute = {
+    val allStatusMessengers = qStatusMessengerPluginManager.createAllTypes()
+
     if (settings.qSettings.runName == null)
       settings.qSettings.runName = FilenameUtils.removeExtension(scripts.head.getName)
     if (IOUtils.isDefaultTempDir(settings.qSettings.tempDirectory))
       settings.qSettings.tempDirectory = IOUtils.absolute(settings.qSettings.runDirectory, ".queue/tmp")
     qGraph.initializeWithSettings(settings)
 
-    val allQScripts = pluginManager.createAllTypes()
+    for (statusMessenger <- allStatusMessengers) {
+      loadArgumentsIntoObject(statusMessenger)
+    }
+
+    for (statusMessenger <- allStatusMessengers) {
+      statusMessenger.started()
+    }
+
+    val allQScripts = qScriptPluginManager.createAllTypes()
     for (script <- allQScripts) {
-      logger.info("Scripting " + pluginManager.getName(script.getClass.asSubclass(classOf[QScript])))
+      logger.info("Scripting " + qScriptPluginManager.getName(script.getClass.asSubclass(classOf[QScript])))
       loadArgumentsIntoObject(script)
       script.qSettings = settings.qSettings
       try {
@@ -124,6 +138,10 @@ class QCommandLine extends CommandLineProgram with Logging {
       logger.info("Added " + script.functions.size + " functions")
     }
 
+    if (settings.run) {
+      allQScripts.foreach(_.pullInputs())
+    }
+
     // Execute the job graph
     qGraph.run()
 
@@ -142,11 +160,18 @@ class QCommandLine extends CommandLineProgram with Logging {
     logger.info("Writing final jobs report...")
     qGraph.writeJobsReport()
 
-    if (!qGraph.success) {
+    if (!success) {
       logger.info("Done with errors")
       qGraph.logFailed()
+      for (statusMessenger <- allStatusMessengers)
+        statusMessenger.exit("Done with errors")
       1
     } else {
+      if (settings.run) {
+        allQScripts.foreach(_.pushOutputs())
+        for (statusMessenger <- allStatusMessengers)
+          statusMessenger.done()
+      }
       0
     }
   }
@@ -158,19 +183,30 @@ class QCommandLine extends CommandLineProgram with Logging {
   override def canAddArgumentsDynamically = true
 
   /**
-   * Returns the list of QScripts passed in via -S so that their
+   * Returns the list of QScripts passed in via -S and other plugins
-   * arguments can be inspected before QScript.script is called.
+   * so that their arguments can be inspected before QScript.script is called.
-   * @return Array of QScripts passed in.
+   * @return Array of dynamic sources
    */
-  override def getArgumentSources =
+  override def getArgumentSources = {
-    pluginManager.getPlugins.toIterable.toArray.asInstanceOf[Array[Class[_]]]
+    var plugins = Seq.empty[Class[_]]
+    plugins ++= qScriptPluginManager.getPlugins
+    plugins ++= qStatusMessengerPluginManager.getPlugins
+    plugins.toArray
+  }
 
   /**
-   * Returns the name of a QScript
+   * Returns the name of a script/plugin
-   * @return The name of a QScript
+   * @return The name of a script/plugin
    */
-  override def getArgumentSourceName(source: Class[_]) =
+  override def getArgumentSourceName(source: Class[_]) = {
-    pluginManager.getName(source.asSubclass(classOf[QScript]))
+    if (classOf[QScript].isAssignableFrom(source))
+      qScriptPluginManager.getName(source.asSubclass(classOf[QScript]))
+    else if (classOf[QStatusMessenger].isAssignableFrom(source))
+      qStatusMessengerPluginManager.getName(source.asSubclass(classOf[QStatusMessenger]))
+    else
+      null
+
+  }
 
   /**
    * Returns a ScalaCompoundArgumentTypeDescriptor that can parse argument sources into scala collections.

public/scala/src/org/broadinstitute/sting/queue/QScript.scala

@@ -27,7 +27,9 @@ package org.broadinstitute.sting.queue
 import engine.JobRunInfo
 import org.broadinstitute.sting.queue.function.QFunction
 import annotation.target.field
-import util.{StringFileConversions, PrimitiveOptionConversions, Logging}
+import util.{ReflectionUtils, StringFileConversions, PrimitiveOptionConversions, Logging}
+import org.broadinstitute.sting.utils.classloader.JVMUtils
+import java.lang.reflect.Field
 
 /**
  * Defines a Queue pipeline as a collection of CommandLineFunctions.
@@ -106,6 +108,33 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon
   def addAll(functions: Seq[QFunction]) {
     functions.foreach( f => add(f) )
   }
+
+  def pullInputs() {
+    val inputs = getInputs
+    inputs.filter(_.isInstanceOf[RemoteFile]).map(_.asInstanceOf[RemoteFile]).foreach(_.pullToLocal())
+  }
+
+  def pushOutputs() {
+    val outputs = getOutputs
+    outputs.filter(_.isInstanceOf[RemoteFile]).map(_.asInstanceOf[RemoteFile]).foreach(_.pushToRemote())
+  }
+
+  private def getInputs: Seq[File] = {
+    getFieldValues(classOf[Input])
+  }
+
+  private def getOutputs: Seq[File] = {
+    getFieldValues(classOf[Output])
+  }
+
+  private def getFieldValues(annotation: Class[_ <: java.lang.annotation.Annotation]): Seq[File] = {
+    val filtered: Seq[Field] = fields.filter(field => ReflectionUtils.hasAnnotation(field, annotation))
+    val files = filtered.filter(field => classOf[File].isAssignableFrom(field.getType)).map(field => ReflectionUtils.getValue(this, field).asInstanceOf[File])
+    val seqFiles = filtered.filter(field => classOf[Seq[File]].isAssignableFrom(field.getType)).map(field => ReflectionUtils.getValue(this, field).asInstanceOf[Seq[File]])
+    seqFiles.foldLeft(files)(_ ++ _).filter(_ != null)
+  }
+
+  private lazy val fields = collection.JavaConversions.asScalaBuffer(JVMUtils.getAllFields(this.getClass)).toSeq
 }
 
 object QScript {

public/scala/src/org/broadinstitute/sting/queue/engine/QStatusMessenger.scala

@@ -0,0 +1,10 @@
+package org.broadinstitute.sting.queue.engine
+
+/**
+ * Plugin to sends QStatus messages
+ */
+trait QStatusMessenger {
+  def started()
+  def done()
+  def exit(message: String)
+}

public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala

@@ -92,6 +92,6 @@ object GATKIntervals {
   }
 
   private def createBinding(interval: String, argumentName: String, tags: Tags): IntervalBinding[Feature] = {
-    ArgumentTypeDescriptor.parseBinding(interval, classOf[Feature], classOf[IntervalBinding[Feature]], argumentName, tags, argumentName).asInstanceOf[IntervalBinding[Feature]]
+    ArgumentTypeDescriptor.parseBinding(new ArgumentMatchStringValue(interval), classOf[Feature], classOf[IntervalBinding[Feature]], argumentName, tags, argumentName).asInstanceOf[IntervalBinding[Feature]]
   }
 }

public/scala/src/org/broadinstitute/sting/queue/util/RemoteFile.scala

@@ -0,0 +1,13 @@
+package org.broadinstitute.sting.queue.util
+
+import java.io.File
+import org.broadinstitute.sting.utils.io.FileExtension
+
+/**
+ * An extension of java.io.File that can be pulled from or pushed to a remote location.
+ */
+trait RemoteFile extends File with FileExtension {
+  def pullToLocal()
+  def pushToRemote()
+  def deleteRemote()
+}