Started path of deprecation of Sting's @Argument by splitting the annotation into @Output and @Input. Anything that's not an @Output should be an @Input.

Checked in example qscripts that are basically todo integration tests.
Replaced use of queue @Input/@Output with Sting's new @Input/@Output.  This means you'll now have to doc-ument the annotations.
More work on dependency resolution cycles being created in the graph during scatter/gather.
Filtering nulls to avoid NPE exceptions in scala's 'Collection'.hashCode.


git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@3643 348d0f76-0448-11de-a6fe-93d51630548a

java/src/org/broadinstitute/sting/commandline/ArgumentDefinition.java

@@ -33,6 +33,11 @@ import java.util.List;
  * A specific argument definition.  Maps one-to-one with a field in some class.
  */
 public class ArgumentDefinition {
+    /**
+     * Whether an argument is an input or an output.
+     */
+    public final ArgumentIOType ioType;
+
     /**
      * Full name of the argument.  Must have a value.
      */
@@ -80,6 +85,7 @@ public class ArgumentDefinition {
 
     /**
      * Creates a new argument definition.
+     * @param ioType Whether the argument is an input or an output.
      * @param fullName Full name for this argument definition.
      * @param shortName Short name for this argument definition.
      * @param doc Doc string for this argument.
@@ -90,7 +96,8 @@ public class ArgumentDefinition {
      * @param validation A regular expression for command-line argument validation.
      * @param validOptions is there a particular list of options that's valid for this argument definition?  List them if so, otherwise set this to null. 
      */
-    public ArgumentDefinition( String fullName,
+    public ArgumentDefinition( ArgumentIOType ioType,
+                               String fullName,
                                String shortName,
                                String doc,
                                boolean required,
@@ -99,6 +106,7 @@ public class ArgumentDefinition {
                                String exclusiveOf,
                                String validation,
                                List<String> validOptions) {
+        this.ioType = ioType;
         this.fullName = fullName;
         this.shortName = shortName;
         this.doc = doc;

java/src/org/broadinstitute/sting/commandline/ArgumentDescription.java

@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2010, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.commandline;
+
+import org.broadinstitute.sting.utils.StingException;
+
+import java.lang.annotation.Annotation;
+
+/**
+ * Should be an interface, but java doesn't like inheritance in Annotations.
+ */
+public class ArgumentDescription {
+    private Annotation annotation;
+
+    public ArgumentDescription(Annotation annotation) {
+        this.annotation = annotation;
+    }
+
+    /**
+     * A hack to get around the fact that Java doesn't like inheritance in Annotations.
+     * @param method the method to invoke
+     * @return the return value of the method
+     */
+    private Object invoke(String method) {
+        try {
+            return this.annotation.getClass().getMethod(method).invoke(this.annotation);
+        } catch (Exception e) {
+            throw new StingException("Unable to access method " + method + " on annotation " + annotation.getClass(), e);
+        }
+    }
+
+    /**
+     * The directionality of the command-line argument.
+     * @return INPUT, OUTPUT, or UNKNOWN
+     */
+    public ArgumentIOType getIOType() {
+        if (annotation instanceof Input) return ArgumentIOType.INPUT;
+        if (annotation instanceof Output) return ArgumentIOType.OUTPUT;
+        return ArgumentIOType.UNKNOWN;
+    }
+
+    /**
+     * The full name of the command-line argument.  Full names should be
+     * prefixed on the command-line with a double dash (--).
+     * @return Selected full name, or "" to use the default.
+     */
+    public String fullName() { return (String)invoke("fullName"); }
+
+    /**
+     * Specified short name of the command.  Short names should be prefixed
+     * with a single dash.  Argument values can directly abut single-char
+     * short names or be separated from them by a space.
+     * @return Selected short name, or "" for none.
+     */
+    public String shortName() { return (String)invoke("shortName"); }
+
+    /**
+     * Documentation for the command-line argument.  Should appear when the
+     * --help argument is specified.
+     * @return Doc string associated with this command-line argument.
+     */
+    public String doc() { return (String)invoke("doc"); }
+
+    /**
+     * Is this command-line argument required.  The application should exit
+     * printing help if this command-line argument is not specified.
+     * @return True if the argument is required.  False otherwise.
+     */
+    public boolean required() { return (Boolean)invoke("required"); }
+
+    /**
+     * Should this command-line argument be exclusive of others.  Should be
+     * a comma-separated list of names of arguments of which this should be
+     * independent.
+     * @return A comma-separated string listing other arguments of which this
+     *         argument should be independent.
+     */
+    public String exclusiveOf() { return (String)invoke("exclusiveOf"); }
+
+    /**
+     * Provide a regexp-based validation string.
+     * @return Non-empty regexp for validation, blank otherwise.
+     */
+    public String validation() { return (String)invoke("validation"); }
+}

java/src/org/broadinstitute/sting/commandline/ArgumentIOType.java

@@ -22,14 +22,8 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package org.broadinstitute.sting.queue.util;
+package org.broadinstitute.sting.commandline;
 
-import java.lang.annotation.*;
+public enum ArgumentIOType {
-
+    INPUT, OUTPUT, UNKNOWN
-@Documented
-@Inherited
-@Retention(RetentionPolicy.RUNTIME)
-@Target({ElementType.FIELD})
-public @interface Scatter {
-    Class value();
 }

java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java

@@ -39,6 +39,7 @@ import java.util.*;
  * @version 0.1
  */
 public abstract class ArgumentTypeDescriptor {
+    private static Class[] ARGUMENT_ANNOTATIONS = {Input.class, Output.class, Argument.class};
 
     /**
      * our log, which we want to capture anything from org.broadinstitute.sting
@@ -121,7 +122,8 @@ public abstract class ArgumentTypeDescriptor {
      * @return The default definition for this argument source.
      */
     protected ArgumentDefinition createDefaultArgumentDefinition( ArgumentSource source ) {
-        return new ArgumentDefinition( getFullName(source),
+        return new ArgumentDefinition( getIOType(source),
+                                       getFullName(source),
                                        getShortName(source),
                                        getDoc(source),
                                        isRequired(source),
@@ -132,7 +134,17 @@ public abstract class ArgumentTypeDescriptor {
                                        getValidOptions(source) );
     }
 
-    protected abstract Object parse( ArgumentSource source, Class type, ArgumentMatches matches );
+    public abstract Object parse( ArgumentSource source, Class type, ArgumentMatches matches );
+
+    /**
+     * Specifies other arguments which cannot be used in conjunction with tihs argument.  Comma-separated list.
+     * @param source Original field specifying command-line arguments.
+     * @return A comma-separated list of exclusive arguments, or null if none are present.
+     */
+    protected ArgumentIOType getIOType( ArgumentSource source ) {
+        ArgumentDescription description = getArgumentDescription(source);
+        return description.getIOType();
+    }
 
     /**
      * Retrieves the full name of the argument, specifiable with the '--' prefix.  The full name can be
@@ -141,7 +153,7 @@ public abstract class ArgumentTypeDescriptor {
      * @return full name of the argument.  Never null.
      */
     protected String getFullName( ArgumentSource source ) {
-        Argument description = getArgumentDescription(source);
+        ArgumentDescription description = getArgumentDescription(source);
         return description.fullName().trim().length() > 0 ? description.fullName().trim() : source.field.getName().toLowerCase();
     }
 
@@ -152,7 +164,7 @@ public abstract class ArgumentTypeDescriptor {
      * @return short name of the argument.  Null if no short name exists.
      */
     protected String getShortName( ArgumentSource source ) {
-        Argument description = getArgumentDescription(source);
+        ArgumentDescription description = getArgumentDescription(source);
         return description.shortName().trim().length() > 0 ? description.shortName().trim() : null;
     }
 
@@ -162,7 +174,7 @@ public abstract class ArgumentTypeDescriptor {
      * @return Documentation for this argument.
      */
     protected String getDoc( ArgumentSource source ) {
-        Argument description = getArgumentDescription(source);
+        ArgumentDescription description = getArgumentDescription(source);
         return description.doc();
     }
 
@@ -172,7 +184,7 @@ public abstract class ArgumentTypeDescriptor {
      * @return True if the field is mandatory and not a boolean flag.  False otherwise.
      */
     protected boolean isRequired( ArgumentSource source ) {
-        Argument description = getArgumentDescription(source);
+        ArgumentDescription description = getArgumentDescription(source);
         return description.required() && !source.isFlag();
     }
 
@@ -182,7 +194,7 @@ public abstract class ArgumentTypeDescriptor {
      * @return A comma-separated list of exclusive arguments, or null if none are present.
      */
     protected String getExclusiveOf( ArgumentSource source ) {
-        Argument description = getArgumentDescription(source);
+        ArgumentDescription description = getArgumentDescription(source);
         return description.exclusiveOf().trim().length() > 0 ? description.exclusiveOf().trim() : null;
     }
 
@@ -192,15 +204,15 @@ public abstract class ArgumentTypeDescriptor {
      * @return a JVM regex-compatible regular expression, or null to permit any possible value.
      */
     protected String getValidationRegex( ArgumentSource source ) {
-        Argument description = getArgumentDescription(source);
+        ArgumentDescription description = getArgumentDescription(source);
         return description.validation().trim().length() > 0 ? description.validation().trim() : null;
     }
 
     /**
      * If the argument source only accepts a small set of options, populate the returned list with
      * those options.  Otherwise, leave the list empty.
-     * @param source 
+     * @param source Original field specifying command-line arguments.
-     * @return
+     * @return A list of valid options.
      */
     protected List<String> getValidOptions( ArgumentSource source ) {
         if(!source.field.getType().isEnum())
@@ -246,10 +258,20 @@ public abstract class ArgumentTypeDescriptor {
      * @param source source of the argument.
      * @return Argument description annotation associated with the given field.
      */
-    protected Argument getArgumentDescription( ArgumentSource source ) {
+    @SuppressWarnings("unchecked")
-        if( !source.field.isAnnotationPresent(Argument.class) )
+    protected ArgumentDescription getArgumentDescription( ArgumentSource source ) {
-            throw new StingException("ArgumentAnnotation is not present for the argument field: " + source.field.getName());
+        for (Class annotation: ARGUMENT_ANNOTATIONS)
-        return source.field.getAnnotation(Argument.class);
+            if (source.field.isAnnotationPresent(annotation))
+                return new ArgumentDescription(source.field.getAnnotation(annotation));
+        throw new StingException("ArgumentAnnotation is not present for the argument field: " + source.field.getName());
+    }
+
+    @SuppressWarnings("unchecked")
+    public static boolean isArgumentDescriptionPresent(Field field) {
+        for (Class annotation: ARGUMENT_ANNOTATIONS)
+            if (field.isAnnotationPresent(annotation))
+                return true;
+        return false;
     }
 }
 
@@ -276,7 +298,7 @@ class SimpleArgumentTypeDescriptor extends ArgumentTypeDescriptor {
     }
 
     @Override
-    protected Object parse( ArgumentSource source, Class type, ArgumentMatches matches ) {
+    public Object parse( ArgumentSource source, Class type, ArgumentMatches matches ) {
         String value = getArgumentValue( createDefaultArgumentDefinition(source), matches );
 
         // lets go through the types we support
@@ -345,9 +367,10 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor {
     }
     
     @Override
+    @SuppressWarnings("unchecked")
     public Object parse( ArgumentSource source, Class type, ArgumentMatches matches )
     {
-        Class componentType = null;
+        Class componentType;
 
         if( Collection.class.isAssignableFrom(type) ) {
 
@@ -372,7 +395,7 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor {
 
             ArgumentTypeDescriptor componentArgumentParser = ArgumentTypeDescriptor.create( componentType );
 
-            Collection collection = null;
+            Collection collection;
             try {
                 collection = (Collection)type.newInstance();
             }

java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java

@@ -46,7 +46,7 @@ public abstract class CommandLineProgram {
     private static Logger logger = Logger.getRootLogger();
 
     /** the default log level */
-    @Argument(fullName = "logging_level",
+    @Input(fullName = "logging_level",
               shortName = "l",
               doc = "Set the minimum level of logging, i.e. setting INFO get's you INFO up to FATAL, setting ERROR gets you ERROR and FATAL level logging.",
               required = false)
@@ -54,28 +54,28 @@ public abstract class CommandLineProgram {
 
 
     /** where to send the output of our logger */
-    @Argument(fullName = "log_to_file",
+    @Output(fullName = "log_to_file",
               shortName = "log",
               doc = "Set the logging location",
               required = false)
     protected String toFile = null;
 
     /** do we want to silence the command line output */
-    @Argument(fullName = "quiet_output_mode",
+    @Input(fullName = "quiet_output_mode",
               shortName = "quiet",
               doc = "Set the logging to quiet mode, no output to stdout",
               required = false)
     protected Boolean quietMode = false;
 
     /** do we want to generate debugging information with the logs */
-    @Argument(fullName = "debug_mode",
+    @Input(fullName = "debug_mode",
               shortName = "debug",
               doc = "Set the logging file string to include a lot of debugging information (SLOW!)",
               required = false)
     protected Boolean debugMode = false;
 
     /** this is used to indicate if they've asked for help */
-    @Argument(fullName = "help", shortName = "h", doc = "Generate this help message", required = false)
+    @Input(fullName = "help", shortName = "h", doc = "Generate this help message", required = false)
     public Boolean help = false;
 
     /** our logging output patterns */

java/src/org/broadinstitute/sting/commandline/Input.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2010, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.commandline;
+
+import java.lang.annotation.*;
+
+/**
+ * Annotates fields in objects that should be used as command-line arguments.
+ * Any field annotated with @Input can appear as a command-line parameter.
+ */
+@Documented
+@Inherited
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.FIELD})
+public @interface Input {
+    /**
+     * The full name of the command-line argument.  Full names should be
+     * prefixed on the command-line with a double dash (--).
+     * @return Selected full name, or "" to use the default.
+     */
+    String fullName() default "";
+
+    /**
+     * Specified short name of the command.  Short names should be prefixed
+     * with a single dash.  Argument values can directly abut single-char
+     * short names or be separated from them by a space.
+     * @return Selected short name, or "" for none.
+     */
+    String shortName() default "";
+
+    /**
+     * Documentation for the command-line argument.  Should appear when the
+     * --help argument is specified.
+     * @return Doc string associated with this command-line argument.
+     */
+    String doc();
+
+    /**
+     * Is this command-line argument required.  The application should exit
+     * printing help if this command-line argument is not specified.
+     * @return True if the argument is required.  False otherwise.
+     */
+    boolean required() default true;
+
+    /**
+     * Should this command-line argument be exclusive of others.  Should be
+     * a comma-separated list of names of arguments of which this should be
+     * independent.
+     * @return A comma-separated string listing other arguments of which this
+     *         argument should be independent.
+     */
+    String exclusiveOf() default "";
+
+    /**
+     * Provide a regexp-based validation string.
+     * @return Non-empty regexp for validation, blank otherwise.
+     */
+    String validation() default "";
+}

java/src/org/broadinstitute/sting/commandline/Output.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2010, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.commandline;
+
+import java.lang.annotation.*;
+
+/**
+ * Annotates fields in objects that should be used as command-line arguments.
+ * Any field annotated with @Argument can appear as a command-line parameter.
+ */
+@Documented
+@Inherited
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.FIELD})
+public @interface Output {
+    /**
+     * The full name of the command-line argument.  Full names should be
+     * prefixed on the command-line with a double dash (--).
+     * @return Selected full name, or "" to use the default.
+     */
+    String fullName() default "";
+
+    /**
+     * Specified short name of the command.  Short names should be prefixed
+     * with a single dash.  Argument values can directly abut single-char
+     * short names or be separated from them by a space.
+     * @return Selected short name, or "" for none.
+     */
+    String shortName() default "";
+
+    /**
+     * Documentation for the command-line argument.  Should appear when the
+     * --help argument is specified.
+     * @return Doc string associated with this command-line argument.
+     */
+    String doc();
+
+    /**
+     * Is this command-line argument required.  The application should exit
+     * printing help if this command-line argument is not specified.
+     * @return True if the argument is required.  False otherwise.
+     */
+    boolean required() default true;
+
+    /**
+     * Should this command-line argument be exclusive of others.  Should be
+     * a comma-separated list of names of arguments of which this should be
+     * independent.
+     * @return A comma-separated string listing other arguments of which this
+     *         argument should be independent.
+     */
+    String exclusiveOf() default "";
+
+    /**
+     * Provide a regexp-based validation string.
+     * @return Non-empty regexp for validation, blank otherwise.
+     */
+    String validation() default "";
+}

java/src/org/broadinstitute/sting/commandline/ParsingEngine.java

@@ -45,11 +45,6 @@ public class ParsingEngine {
      */
     CommandLineProgram clp = null;
 
-    /**
-     * A collection of all the source fields which define command-line arguments.
-     */
-    List<ArgumentSource> argumentSources = new ArrayList<ArgumentSource>();
-
     /**
      * A list of defined arguments against which command lines are matched.
      * Package protected for testing access.
@@ -125,8 +120,6 @@ public class ParsingEngine {
      * command-line arguments to the arguments that are actually
      * required.
      * @param tokens Tokens passed on the command line.
-     * @return A object indicating which matches are best.  Might return
-     *         an empty object, but will never return null.
      */
     public void parse( String[] tokens ) {
         argumentMatches = new ArgumentMatches();
@@ -315,7 +308,7 @@ public class ParsingEngine {
         while( sourceClass != null ) {
             Field[] fields = sourceClass.getDeclaredFields();
             for( Field field: fields ) {
-                if( field.isAnnotationPresent(Argument.class) )
+                if( ArgumentTypeDescriptor.isArgumentDescriptionPresent(field) )
                     argumentSources.add( new ArgumentSource(sourceClass,field) );
                 if( field.isAnnotationPresent(ArgumentCollection.class) )
                     argumentSources.addAll( extractArgumentSources(field.getType()) );

java/src/org/broadinstitute/sting/gatk/io/stubs/GenotypeWriterArgumentTypeDescriptor.java

@@ -25,11 +25,7 @@
 
 package org.broadinstitute.sting.gatk.io.stubs;
 
-import org.broadinstitute.sting.commandline.ArgumentTypeDescriptor;
+import org.broadinstitute.sting.commandline.*;
-import org.broadinstitute.sting.commandline.ArgumentSource;
-import org.broadinstitute.sting.commandline.ArgumentMatches;
-import org.broadinstitute.sting.commandline.ArgumentDefinition;
-import org.broadinstitute.sting.commandline.Argument;
 import org.broadinstitute.sting.utils.StingException;
 import org.broadinstitute.sting.utils.genotype.GenotypeWriter;
 import org.broadinstitute.sting.utils.genotype.GenotypeWriterFactory;
@@ -53,7 +49,7 @@ public class GenotypeWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor
 
     /**
      * Create a new GenotypeWriter argument, notifying the given engine when that argument has been created.
-     * @param engine
+     * @param engine the engine to be notified.
      */
     public GenotypeWriterArgumentTypeDescriptor(GenomeAnalysisEngine engine) {
         this.engine = engine;
@@ -104,7 +100,7 @@ public class GenotypeWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor
     /**
      * Convert the given argument matches into a single object suitable for feeding into the ArgumentSource.
      * @param source Source for this argument.
-     * @param type
+     * @param type not used
      * @param matches Matches that match with this argument.
      * @return Transform from the matches into the associated argument.
      */
@@ -127,7 +123,7 @@ public class GenotypeWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor
         }
 
         // Create a stub for the given object.
-        GenotypeWriterStub stub = null;
+        GenotypeWriterStub stub;
         switch(genotypeFormat) {
             case GELI:
                 stub = (writerFile != null) ? new GeliTextGenotypeWriterStub(engine, writerFile) : new GeliTextGenotypeWriterStub(engine,System.out);
@@ -158,7 +154,7 @@ public class GenotypeWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor
      * @return Argument definition for the BAM file itself.  Will not be null.
      */
     private ArgumentDefinition createGenotypeFileArgumentDefinition(ArgumentSource source) {
-        Argument description = this.getArgumentDescription(source);
+        ArgumentDescription description = this.getArgumentDescription(source);
 
         boolean isFullNameProvided = description.fullName().trim().length() > 0;
         boolean isShortNameProvided = description.shortName().trim().length() > 0;
@@ -175,7 +171,8 @@ public class GenotypeWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor
         else
             shortName = null;
 
-        return new ArgumentDefinition( fullName,
+        return new ArgumentDefinition( getIOType(source),
+                                       fullName,
                                        shortName,
                                        getDoc(source),
                                        isRequired(source),
@@ -192,7 +189,8 @@ public class GenotypeWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor
      * @return Argument definition for the BAM file itself.  Will not be null.
      */
     private ArgumentDefinition createGenotypeFormatArgumentDefinition(ArgumentSource source) {
-        return new ArgumentDefinition( "variant_output_format",
+        return new ArgumentDefinition( getIOType(source),
+                                       "variant_output_format",
                                        "vf",
                                        "Format to be used to represent variants; default is VCF",
                                        false,

java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java

@@ -94,7 +94,7 @@ public class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor
      * @return Argument definition for the BAM file itself.  Will not be null.
      */
     private ArgumentDefinition createBAMArgumentDefinition(ArgumentSource source) {
-        Argument description = this.getArgumentDescription(source);
+        ArgumentDescription description = this.getArgumentDescription(source);
 
         boolean isFullNameProvided = description.fullName().trim().length() > 0;
         boolean isShortNameProvided = description.shortName().trim().length() > 0;
@@ -111,7 +111,8 @@ public class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor
         else
             shortName = null;
 
-        return new ArgumentDefinition( fullName,
+        return new ArgumentDefinition( getIOType(source),
+                                       fullName,
                                        shortName,
                                        getDoc(source),
                                        isRequired(source),
@@ -128,7 +129,8 @@ public class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor
      * @return Argument definition for the BAM file itself.  Will not be null.
      */
     private ArgumentDefinition createBAMCompressionArgumentDefinition(ArgumentSource source) {
-        return new ArgumentDefinition( COMPRESSION_FULLNAME,
+        return new ArgumentDefinition( getIOType(source),
+                                       COMPRESSION_FULLNAME,
                                        COMPRESSION_SHORTNAME,
                                        "Compression level to use for writing BAM files",
                                        false,

java/src/org/broadinstitute/sting/queue/function/scattergather/Gather.java

@@ -22,18 +22,18 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package org.broadinstitute.sting.queue.util;
+package org.broadinstitute.sting.queue.function.scattergather;
 
 import java.lang.annotation.*;
 
 /**
- * Specifies the type of an input or output field.
+ * Specifies the class type of the CommandLineFunction to gather an @Output
  * Written in java because scala doesn't support RetentionPolicy.RUNTIME
  */
 @Documented
 @Inherited
 @Retention(RetentionPolicy.RUNTIME)
 @Target({ElementType.FIELD})
-public @interface ClassType {
+public @interface Gather {
     Class value();
 }

java/src/org/broadinstitute/sting/queue/function/scattergather/Scatter.java

@@ -22,17 +22,18 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package org.broadinstitute.sting.queue.util;
+package org.broadinstitute.sting.queue.function.scattergather;
 
 import java.lang.annotation.*;
 
 /**
- * Specifies an input or output to a QFunction is optional
+ * Specifies the class type of the CommandLineFunction to scatter an @Input
  * Written in java because scala doesn't support RetentionPolicy.RUNTIME
  */
 @Documented
 @Inherited
 @Retention(RetentionPolicy.RUNTIME)
 @Target({ElementType.FIELD})
-public @interface Optional {
+public @interface Scatter {
+    Class value();
 }

java/src/org/broadinstitute/sting/queue/util/Gather.java

@@ -1,43 +0,0 @@
-/*
- * Copyright (c) 2010, The Broad Institute
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-package org.broadinstitute.sting.queue.util;
-
-import java.lang.annotation.*;
-
-/**
- * Specifies the class to gather an output of a QFunction.
- * Not an input or output but should be copied with a function.
- * Internals should have default values that should be handled, i.e. they are always @Optional
- * A common use for @Internal is to specify WHERE a function runs: farm queue, directory, etc.
- * or to name part of a function: farm job name
- * Written in java because scala doesn't support RetentionPolicy.RUNTIME
- */
-@Documented
-@Inherited
-@Retention(RetentionPolicy.RUNTIME)
-@Target({ElementType.FIELD})
-public @interface Gather {
-    Class value();
-}

java/src/org/broadinstitute/sting/queue/util/Input.java

@@ -1,38 +0,0 @@
-/*
- * Copyright (c) 2010, The Broad Institute
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-package org.broadinstitute.sting.queue.util;
-
-import java.lang.annotation.*;
-
-/**
- * Specifies an input to a QFunction
- * Written in java because scala doesn't support RetentionPolicy.RUNTIME
- */
-@Documented
-@Inherited
-@Retention(RetentionPolicy.RUNTIME)
-@Target({ElementType.FIELD})
-public @interface Input {
-}

java/src/org/broadinstitute/sting/queue/util/Internal.java

@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2010, The Broad Institute
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-package org.broadinstitute.sting.queue.util;
-
-import java.lang.annotation.*;
-
-/**
- * Specifies an internal setting for a QFunction.
- * Not an input or output but should be copied with a function.
- * Internals should have default values that should be handled, i.e. they are always @Optional
- * A common use for @Internal is to specify WHERE a function runs: farm queue, directory, etc.
- * or to name part of a function: farm job name
- * Written in java because scala doesn't support RetentionPolicy.RUNTIME
- */
-@Documented
-@Inherited
-@Retention(RetentionPolicy.RUNTIME)
-@Target({ElementType.FIELD})
-public @interface Internal {
-}

java/src/org/broadinstitute/sting/queue/util/Output.java

@@ -1,38 +0,0 @@
-/*
- * Copyright (c) 2010, The Broad Institute
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-package org.broadinstitute.sting.queue.util;
-
-import java.lang.annotation.*;
-
-/**
- * Specifies an output to a QFunction
- * Written in java because scala doesn't support RetentionPolicy.RUNTIME
- */
-@Documented
-@Inherited
-@Retention(RetentionPolicy.RUNTIME)
-@Target({ElementType.FIELD})
-public @interface Output {
-}

scala/qscript/unifiedgenotyper_example.properties

@@ -0,0 +1,5 @@
+gatkJar = /humgen/gsa-hpprojects/GATK/bin/current/GenomeAnalysisTK.jar
+referenceFile = /path/to/reference.fasta
+dbsnp = /path/to/dbsnp
+intervals = /path/to/my.interval_list
+jobNamePrefix = Q

scala/qscript/unifiedgenotyper_example.scala

@@ -0,0 +1,54 @@
+import org.broadinstitute.sting.queue.QScript._
+
+setArgs(args)
+
+for (bam <- inputs("bam")) {
+  val ug = new UnifiedGenotyper
+  val vf = new VariantFiltration
+  val ve = new GatkFunction {
+    @Input(doc="vcf") var vcfFile: File = _
+    @Output(doc="eval") var evalFile: File = _
+    def commandLine = gatkCommandLine("VariantEval") + "-B eval,VCF,%s -o %s".format(vcfFile, evalFile)
+  }
+
+  // Make sure the Sting/scripts folder is in your path to use mergeText.sh and splitIntervals.sh.
+  ug.scatterCount = 3
+  ug.bamFiles :+= bam
+  ug.vcfFile = swapExt(bam, "bam", "unfiltered.vcf")
+
+  vf.vcfInput = ug.vcfFile
+  vf.vcfOutput = swapExt(bam, "bam", "filtered.vcf")
+
+  ve.vcfFile = vf.vcfOutput
+  ve.evalFile = swapExt(bam, "bam", "eval")
+
+  add(ug, vf, ve)
+}
+
+setParams
+run
+
+
+class UnifiedGenotyper extends GatkFunction {
+  @Output(doc="vcf")
+  @Gather(classOf[SimpleTextGatherFunction])
+  var vcfFile: File = _
+  def commandLine = gatkCommandLine("UnifiedGenotyper") + "-varout %s".format(vcfFile)
+}
+
+class VariantFiltration extends GatkFunction {
+  @Input(doc="input vcf")
+  var vcfInput: File = _
+
+  @Input(doc="filter names")
+  var filterNames: List[String] = Nil
+
+  @Input(doc="filter expressions")
+  var filterExpressions: List[String] = Nil
+
+  @Output(doc="output vcf")
+  var vcfOutput: File = _
+
+  def commandLine = gatkCommandLine("VariantFiltration") + "%s%s -B variant,VCF,%s -o %s"
+      .format(repeat(" -filterName ", filterNames), repeat(" -filterExpression ", filterExpressions), vcfInput, vcfOutput)
+}

scala/src/org/broadinstitute/sting/queue/QException.scala

@@ -1,4 +1,6 @@
 package org.broadinstitute.sting.queue
 
+import org.broadinstitute.sting.utils.StingException
+
 class QException(private val message: String, private val throwable: Throwable = null)
-  extends RuntimeException(message, throwable)
+  extends StingException(message, throwable)

scala/src/org/broadinstitute/sting/queue/QScript.scala

@@ -9,15 +9,13 @@ import org.broadinstitute.sting.queue.engine.QGraph
 object QScript {
   // Type aliases so users don't have to import
   type File = java.io.File
-  type Input = org.broadinstitute.sting.queue.util.Input
+  type Input = org.broadinstitute.sting.commandline.Input
-  type Output = org.broadinstitute.sting.queue.util.Output
+  type Output = org.broadinstitute.sting.commandline.Output
-  type Optional = org.broadinstitute.sting.queue.util.Optional
-  type ClassType = org.broadinstitute.sting.queue.util.ClassType
   type CommandLineFunction = org.broadinstitute.sting.queue.function.CommandLineFunction
   type GatkFunction = org.broadinstitute.sting.queue.function.gatk.GatkFunction
   type ScatterGatherableFunction = org.broadinstitute.sting.queue.function.scattergather.ScatterGatherableFunction
-  type Scatter = org.broadinstitute.sting.queue.util.Scatter
+  type Scatter = org.broadinstitute.sting.queue.function.scattergather.Scatter
-  type Gather = org.broadinstitute.sting.queue.util.Gather
+  type Gather = org.broadinstitute.sting.queue.function.scattergather.Gather
   type BamGatherFunction = org.broadinstitute.sting.queue.function.scattergather.BamGatherFunction
   type SimpleTextGatherFunction = org.broadinstitute.sting.queue.function.scattergather.SimpleTextGatherFunction
 

scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala

@@ -3,7 +3,6 @@ package org.broadinstitute.sting.queue.engine
 import org.jgrapht.graph.SimpleDirectedGraph
 import scala.collection.JavaConversions
 import scala.collection.JavaConversions._
-import scala.collection.immutable.ListMap
 import org.broadinstitute.sting.queue.function.{MappingFunction, CommandLineFunction, QFunction}
 import org.broadinstitute.sting.queue.function.scattergather.ScatterGatherableFunction
 import org.broadinstitute.sting.queue.util.{CollectionUtils, Logging}
@@ -18,7 +17,7 @@ class QGraph extends Logging {
   def numJobs = JavaConversions.asSet(jobGraph.edgeSet).filter(_.isInstanceOf[CommandLineFunction]).size
 
   def add(command: CommandLineFunction) {
-    add(command, true)
+    addFunction(command)
   }
 
   /**
@@ -27,20 +26,8 @@ class QGraph extends Logging {
   def fillIn = {
     // clone since edgeSet is backed by the graph
     for (function <- JavaConversions.asSet(jobGraph.edgeSet).clone) {
-      val inputs = function.inputs
+      addCollectionOutputs(function.outputs)
-      val outputs = function.outputs
+      addCollectionInputs(function.inputs)
-
-      for ((name, input) <- inputs) {
-        addCollectionInputs(name, input)
-        if (inputs.size > 1)
-          addMappingEdge(ListMap(name -> input), inputs)
-      }
-
-      for ((name, output) <- outputs) {
-        addCollectionOutputs(name, output)
-        if (outputs.size > 1)
-          addMappingEdge(outputs, ListMap(name -> output))
-      }
     }
 
     var pruning = true
@@ -85,9 +72,9 @@ class QGraph extends Logging {
   }
 
   private def newGraph = new SimpleDirectedGraph[QNode, QFunction](new EdgeFactory[QNode, QFunction] {
-    def createEdge(input: QNode, output: QNode) = new MappingFunction(input.valueMap, output.valueMap)})
+    def createEdge(input: QNode, output: QNode) = new MappingFunction(input.items, output.items)})
 
-  private def add(f: QFunction, replace: Boolean): Unit = {
+  private def addFunction(f: QFunction): Unit = {
     try {
       f.freeze
 
@@ -96,13 +83,13 @@ class QGraph extends Logging {
           val functions = scatterGather.generateFunctions()
           if (logger.isTraceEnabled)
             logger.trace("Scattered into %d parts: %s".format(functions.size, functions))
-          functions.foreach(add(_))
+          functions.foreach(addFunction(_))
         case _ =>
-          val inputs = QNode(f.inputs.values.filter(_ != null).toSet)
+          val inputs = QNode(f.inputs)
-          val outputs = QNode(f.outputs.values.filter(_ != null).toSet)
+          val outputs = QNode(f.outputs)
           val newSource = jobGraph.addVertex(inputs)
           val newTarget = jobGraph.addVertex(outputs)
-          val removedEdges = if (replace) jobGraph.removeAllEdges(inputs, outputs) else Nil
+          val removedEdges = jobGraph.removeAllEdges(inputs, outputs)
           val added = jobGraph.addEdge(inputs, outputs, f)
           if (logger.isTraceEnabled) {
             logger.trace("Mapped from:   " + inputs)
@@ -120,45 +107,41 @@ class QGraph extends Logging {
     }
   }
 
-  private def addCollectionInputs(name: String, value: Any): Unit = {
+  private def addCollectionInputs(value: Any): Unit = {
     CollectionUtils.foreach(value, (item, collection) =>
-      addMappingEdge(ListMap(name -> item), ListMap(name -> collection)))
+      addMappingEdge(item, collection))
   }
 
-  private def addCollectionOutputs(name: String, value: Any): Unit = {
+  private def addCollectionOutputs(value: Any): Unit = {
     CollectionUtils.foreach(value, (item, collection) =>
-      addMappingEdge(ListMap(name -> collection), ListMap(name -> item)))
+      addMappingEdge(collection, item))
   }
 
-  private def addMappingEdge(input: ListMap[String, Any], output: ListMap[String, Any]) =
+  private def addMappingEdge(input: Any, output: Any) = {
-    add(new MappingFunction(input, output), false)
+    val inputSet = asSet(input)
+    val outputSet = asSet(output)
+    val hasEdge = inputSet == outputSet ||
+            jobGraph.getEdge(QNode(inputSet), QNode(outputSet)) != null ||
+            jobGraph.getEdge(QNode(outputSet), QNode(inputSet)) != null
+    if (!hasEdge)
+      addFunction(new MappingFunction(inputSet, outputSet))
+  }
+
+  private def asSet(value: Any): Set[Any] = if (value.isInstanceOf[Set[_]]) value.asInstanceOf[Set[Any]] else Set(value)
 
   private def isMappingEdge(edge: QFunction) =
     edge.isInstanceOf[MappingFunction]
 
   private def isFiller(edge: QFunction) = {
     if (isMappingEdge(edge)) {
-      val source = jobGraph.getEdgeSource(edge)
+      if (jobGraph.outgoingEdgesOf(jobGraph.getEdgeTarget(edge)).size == 0)
-      val target = jobGraph.getEdgeTarget(edge)
-      if (jobGraph.outgoingEdgesOf(target).size == 0 || jobGraph.incomingEdgesOf(source).size == 0)
         true
-      else if (isLoopback(source) || isLoopback(target))
+      else if (jobGraph.incomingEdgesOf(jobGraph.getEdgeSource(edge)).size == 0)
         true
       else false
     } else false
   }
 
-  private def isLoopback(node: QNode) = {
-    var loopback = false
-    val incoming = jobGraph.incomingEdgesOf(node)
-    val outgoing = jobGraph.outgoingEdgesOf(node)
-    if (incoming.size == 1 && outgoing.size == 1)
-      if (isMappingEdge(incoming.head) && isMappingEdge(outgoing.head))
-        if (jobGraph.getEdgeSource(incoming.head) == jobGraph.getEdgeTarget(outgoing.head))
-          loopback = true
-    loopback
-  }
-
   private def isOrphan(node: QNode) =
     (jobGraph.incomingEdgesOf(node).size + jobGraph.outgoingEdgesOf(node).size) == 0
 }

scala/src/org/broadinstitute/sting/queue/engine/QNode.scala

@@ -1,20 +1,6 @@
 package org.broadinstitute.sting.queue.engine
 
-import scala.collection.immutable.ListMap
-
 /**
  * Represents a state between QFunctions the directed acyclic QGraph
  */
-case class QNode (private val items: Set[Any]) {
+case class QNode (val items: Set[Any])
-  /**
-   * Used during QGraph error reporting.
-   * The EdgeFactory uses the valueMap to create new edges for the CycleDetector.
-   */
-  def valueMap = {
-    var map = ListMap.empty[String, Any]
-    for (item <- items)
-      if (item != null)
-        map += item.toString -> item
-    map
-  }
-}

scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala

@@ -1,11 +1,14 @@
 package org.broadinstitute.sting.queue.function
 
-import java.io.File
 import org.broadinstitute.sting.queue.util._
-import org.broadinstitute.sting.queue.engine.{CommandLineRunner, QGraph}
 import java.lang.reflect.Field
+import java.lang.annotation.Annotation
+import org.broadinstitute.sting.commandline.{Input, Output, ArgumentDescription}
 
 trait CommandLineFunction extends InputOutputFunction with DispatchFunction {
+  def inputFieldsWithValues = inputFields.filter(hasFieldValue(_))
+  def outputFieldsWithValues = outputFields.filter(hasFieldValue(_))
+
   /**
    * Repeats parameters with a prefix/suffix if they are set otherwise returns "".
    * Skips null, Nil, None.  Unwraps Some(x) to x.  Everything else is called with x.toString.
@@ -21,22 +24,24 @@ trait CommandLineFunction extends InputOutputFunction with DispatchFunction {
     if (hasValue(param)) prefix + toValue(param) + suffix else ""
 
   def missingValues = {
-    val missingInputs = missingFields(inputFields)
+    val missingInputs = missingFields(inputFields, classOf[Input])
-    val missingOutputs = missingFields(outputFields)
+    val missingOutputs = missingFields(outputFields, classOf[Output])
     missingInputs | missingOutputs
   }
 
-  private def missingFields(fields: List[Field]) = {
+  private def missingFields(fields: List[Field], annotation: Class[_ <: Annotation]) = {
     var missing = Set.empty[String]
     for (field <- fields) {
-      val isOptional = ReflectionUtils.hasAnnotation(field, classOf[Optional])
+      if (isRequired(field, annotation))
-      if (!isOptional)
         if (!hasValue(ReflectionUtils.getValue(this, field)))
           missing += field.getName
     }
     missing
   }
 
+  private def isRequired(field: Field, annotation: Class[_ <: Annotation]) =
+    new ArgumentDescription(field.getAnnotation(annotation)).required
+
   protected def hasFieldValue(field: Field) = hasValue(this.getFieldValue(field))
 
   private def hasValue(param: Any) = param match {

scala/src/org/broadinstitute/sting/queue/function/DispatchFunction.scala

@@ -2,41 +2,40 @@ package org.broadinstitute.sting.queue.function
 
 import java.io.File
 import java.lang.management.ManagementFactory
-import org.broadinstitute.sting.queue.function.scattergather.SimpleTextGatherFunction
+import org.broadinstitute.sting.commandline.{Output, Input}
-import org.broadinstitute.sting.queue.util._
+import org.broadinstitute.sting.queue.function.scattergather.{Gather, SimpleTextGatherFunction}
+import org.broadinstitute.sting.queue.util.IOUtils
 
 trait DispatchFunction extends InputOutputFunction {
   def commandLine: String
 
-  @Input
+  @Input(doc="Upper memory limit", required=false)
-  @Optional
-  @ClassType(classOf[Int])
   var memoryLimit: Option[Int] = None
 
   /**
    * The directory where the command should run.
    */
-  @Internal
+  @Input(doc="Directory to write any files", required=false)
   var commandDirectory: File = IOUtils.CURRENT_DIR
 
-  @Internal
+  @Input(doc="Prefix for automatic job name creation", required=false)
   var jobNamePrefix: String = _
 
-  @Internal
+  @Input(doc="Job name to run on the farm", required=false)
   var jobName: String = _
 
-  @Output
+  @Output(doc="File to redirect any output", required=false)
   @Gather(classOf[SimpleTextGatherFunction])
   var jobOutputFile: File = _
 
-  @Output
+  @Output(doc="File to redirect any errors", required=false)
   @Gather(classOf[SimpleTextGatherFunction])
   var jobErrorFile: File = _
 
-  @Internal
+  @Input(doc="Job project to run the command", required=false)
   var jobProject = "Queue"
 
-  @Internal
+  @Input(doc="Job queue to run the command", required=false)
   var jobQueue = "broad"
 
   override def freeze = {

scala/src/org/broadinstitute/sting/queue/function/InputOutputFunction.scala

@@ -2,6 +2,7 @@ package org.broadinstitute.sting.queue.function
 
 import java.lang.reflect.Field
 import org.broadinstitute.sting.queue.util._
+import org.broadinstitute.sting.commandline.{Input, Output}
 
 /**
  * A function with @Inputs and @Outputs tagging fields that can be set by the user in a QScript
@@ -10,31 +11,30 @@ trait InputOutputFunction extends QFunction with Cloneable {
   def getFieldValue(field: Field) = ReflectionUtils.getValue(this, field)
   def setFieldValue(field: Field, value: Any) = ReflectionUtils.setValue(this, field, value)
 
-  def functionFields: List[Field] = inputFields ::: outputFields ::: internalFields
+  def functionFields: List[Field] = inputFields ::: outputFields
   def inputFields = ReflectionUtils.filterFields(fields, classOf[Input])
   def outputFields = ReflectionUtils.filterFields(fields, classOf[Output])
-  def internalFields = ReflectionUtils.filterFields(fields, classOf[Internal])
 
   private lazy val fields = ReflectionUtils.getAllFields(this.getClass)
-  def inputs = ReflectionUtils.getFieldNamesValues(this, inputFields)
+  // TODO: Need to handle argument collections where field is not on THIS
-  def outputs = ReflectionUtils.getFieldNamesValues(this, outputFields)
+  def inputs = CollectionUtils.removeNullOrEmpty(ReflectionUtils.getFieldValues(this, inputFields)).toSet
-  def internals = ReflectionUtils.getFieldNamesValues(this, internalFields)
+  def outputs = CollectionUtils.removeNullOrEmpty(ReflectionUtils.getFieldValues(this, outputFields)).toSet
 
   /**
    * Sets a field value using the name of the field.
    * Field must be annotated with @Input, @Output, or @Internal
-   * @returns true if the value was found and set
+   * @return true if the value was found and set
    */
   def addOrUpdateWithStringValue(name: String, value: String) = {
     fields.find(_.getName == name) match {
       case Some(field) =>
         val isInput = ReflectionUtils.hasAnnotation(field, classOf[Input])
         val isOutput = ReflectionUtils.hasAnnotation(field, classOf[Output])
-        val isInternal = ReflectionUtils.hasAnnotation(field, classOf[Internal])
+        if (isInput || isOutput) {
-        if (isInput || isOutput || isInternal) {
           ReflectionUtils.addOrUpdateWithStringValue(this, field, value)
         }
         true
+      // TODO: Need to handle argument collections where field is not on THIS
       case None => false
     }
   }

scala/src/org/broadinstitute/sting/queue/function/MappingFunction.scala

@@ -1,14 +1,9 @@
 package org.broadinstitute.sting.queue.function
 
-import org.broadinstitute.sting.queue.engine.QGraph
-import scala.collection.immutable.ListMap
-
 /**
  * Utility class to map a set of inputs to set of outputs.
- * The QGraph uses this function internally to return
+ * The QGraph uses this function internally to map between user defined functions.
  */
-class MappingFunction(private val in: ListMap[String, Any], private val out: ListMap[String, Any]) extends QFunction {
+class MappingFunction(val inputs: Set[Any], val outputs: Set[Any]) extends QFunction {
-  def inputs = in
+  override def toString = "<map>" // For debugging
-  def outputs = out
-  override def toString = "<map>"
 }

scala/src/org/broadinstitute/sting/queue/function/QFunction.scala

@@ -1,11 +1,8 @@
 package org.broadinstitute.sting.queue.function
 
-import scala.collection.immutable.ListMap
-
 /**
  * The base interface for all functions in Queue.
- * Inputs and outputs are specified as ListMaps of name -> value.
+ * Inputs and outputs are specified as Sets of values.
- * The names are used for debugging.
  * Inputs are matched to other outputs by using .equals()
  */
 trait QFunction {
@@ -17,12 +14,12 @@ trait QFunction {
   def freeze = {}
 
   /**
-   * ListMap of name -> value inputs for this function.
+   * Set of inputs for this function.
    */
-  def inputs: ListMap[String, Any]
+  def inputs: Set[Any]
 
   /**
-   * ListMap of name -> value outputs for this function.
+   * Set of outputs for this function.
    */
-  def outputs: ListMap[String, Any]
+  def outputs: Set[Any]
 }

scala/src/org/broadinstitute/sting/queue/function/gatk/GatkFunction.scala

@@ -2,31 +2,27 @@ package org.broadinstitute.sting.queue.function.gatk
 
 import java.io.File
 import org.broadinstitute.sting.queue.function.IntervalFunction
-import org.broadinstitute.sting.queue.util.{Scatter, Internal, Input, Optional}
+import org.broadinstitute.sting.queue.function.scattergather.{Scatter, ScatterGatherableFunction, IntervalScatterFunction}
-import org.broadinstitute.sting.queue.function.scattergather.{ScatterGatherableFunction, IntervalScatterFunction}
+import org.broadinstitute.sting.commandline.Input
 
 trait GatkFunction extends ScatterGatherableFunction with IntervalFunction {
-  @Internal
+  @Input(doc="Temporary directory to write any files", required=false)
-  @Optional
   var javaTmpDir: String = _
 
-  @Input
+  @Input(doc="GATK jar")
   var gatkJar: String = _
 
-  @Input
+  @Input(doc="Reference fasta")
   var referenceFile: File = _
 
-  @Input
+  @Input(doc="Bam files", required=false)
-  @Optional
   var bamFiles: List[File] = Nil
 
-  @Input
+  @Input(doc="Intervals", required=false)
-  @Optional
   @Scatter(classOf[IntervalScatterFunction])
   var intervals: File = _
 
-  @Input
+  @Input(doc="DBSNP", required=false)
-  @Optional
   var dbsnp: File = _
 
   protected def gatkCommandLine(walker: String) =

scala/src/org/broadinstitute/sting/queue/function/scattergather/CleanupTempDirsFunction.scala

@@ -1,14 +1,14 @@
 package org.broadinstitute.sting.queue.function.scattergather
 
 import org.broadinstitute.sting.queue.function.CommandLineFunction
-import org.broadinstitute.sting.queue.util.Input
+import org.broadinstitute.sting.commandline.Input
 import java.io.File
 
 class CleanupTempDirsFunction extends CommandLineFunction {
-  @Input
+  @Input(doc="Original outputs of the gather functions")
-  var originalOutputs: List[Any] = Nil
+  var originalOutputs: Set[Any] = Set.empty[Any]
 
-  @Input
+  @Input(doc="Temporary directories to be deleted")
   var tempDirectories: List[File] = Nil
 
   def commandLine = "rm -rf%s".format(repeat(" '", tempDirectories, "'"))

scala/src/org/broadinstitute/sting/queue/function/scattergather/CreateTempDirsFunction.scala

@@ -2,13 +2,13 @@ package org.broadinstitute.sting.queue.function.scattergather
 
 import java.io.File
 import org.broadinstitute.sting.queue.function.CommandLineFunction
-import org.broadinstitute.sting.queue.util.{Output, Input}
+import org.broadinstitute.sting.commandline.{Output, Input}
 
 class CreateTempDirsFunction extends CommandLineFunction {
-  @Input
+  @Input(doc="Original inputs to the scattered function")
-  var originalInputs: List[Any] = Nil
+  var originalInputs: Set[Any] = Set.empty[Any]
 
-  @Output
+  @Output(doc="Temporary directories to create")
   var tempDirectories: List[File] = Nil
 
   def commandLine = "mkdir%s".format(repeat(" '", tempDirectories, "'"))

scala/src/org/broadinstitute/sting/queue/function/scattergather/GatherFunction.scala

@@ -1,15 +1,15 @@
 package org.broadinstitute.sting.queue.function.scattergather
 
 import org.broadinstitute.sting.queue.function.{CommandLineFunction}
-import org.broadinstitute.sting.queue.util.{Input, Output}
+import org.broadinstitute.sting.commandline.{Input, Output}
 
 trait GatherFunction extends CommandLineFunction {
   type GatherType
 
-  @Input
+  @Input(doc="Parts to gather back into the original output")
   var gatherParts: List[GatherType] = Nil
 
-  @Output
+  @Output(doc="The original output of the scattered function")
   var originalOutput: GatherType = _
 
   def setOriginalFunction(originalFunction: ScatterGatherableFunction) = {}

scala/src/org/broadinstitute/sting/queue/function/scattergather/IntervalScatterFunction.scala

@@ -1,13 +1,13 @@
 package org.broadinstitute.sting.queue.function.scattergather
 
 import java.io.File
-import org.broadinstitute.sting.queue.util.Input
+import org.broadinstitute.sting.commandline.Input
 import org.broadinstitute.sting.queue.function.IntervalFunction
 
 class IntervalScatterFunction extends ScatterFunction {
   type ScatterType = File
 
-  @Input
+  @Input(doc="Reference file to scatter")
   var referenceFile: File = _
 
   override def setOriginalFunction(originalFunction: ScatterGatherableFunction) = {

scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterFunction.scala

@@ -1,19 +1,19 @@
 package org.broadinstitute.sting.queue.function.scattergather
 
 import org.broadinstitute.sting.queue.function.CommandLineFunction
-import org.broadinstitute.sting.queue.util.{Input, Output}
+import org.broadinstitute.sting.commandline.{Input, Output}
 import java.io.File
 
 trait ScatterFunction extends CommandLineFunction {
   type ScatterType
 
-  @Input
+  @Input(doc="Original input to scatter")
   var originalInput: ScatterType = _
 
-  @Input
+  @Input(doc="Temporary directories for each scatter part")
   var tempDirectories: List[File] = Nil
 
-  @Output
+  @Output(doc="Scattered parts of the original input, one per temp directory")
   var scatterParts: List[ScatterType] = Nil
 
   def setOriginalFunction(originalFunction: ScatterGatherableFunction) = {}

scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala

@@ -4,9 +4,11 @@ import org.broadinstitute.sting.queue.function.CommandLineFunction
 import java.lang.reflect.Field
 import java.io.File
 import org.broadinstitute.sting.queue.util._
+import org.broadinstitute.sting.commandline.Input
 
 trait ScatterGatherableFunction extends CommandLineFunction {
-  @Internal
+
+  @Input(doc="Number of parts to scatter the function into")
   var scatterCount: Int = 1
 
   def scatterField = this.inputFields.find(field => ReflectionUtils.hasAnnotation(field, classOf[Scatter])).get
@@ -48,7 +50,7 @@ object ScatterGatherableFunction {
 
     // Create the gather functions for each output field
     var gatherFunctions = Map.empty[Field, GatherFunction]
-    for (outputField <- originalFunction.outputFields) {
+    for (outputField <- originalFunction.outputFieldsWithValues) {
 
       // Create the gather function based on @Gather
       val gatherFunction = getGatherFunction(outputField)
@@ -60,7 +62,7 @@ object ScatterGatherableFunction {
       gatherFunction.originalOutput = gatheredValue
 
       tempDirectories :+= gatherFunction.commandDirectory
-      cleanupFunction.originalOutputs :+= gatheredValue
+      cleanupFunction.originalOutputs += gatheredValue
 
       functions :+= gatherFunction
 
@@ -97,8 +99,8 @@ object ScatterGatherableFunction {
     initializeFunction.jobNamePrefix = originalFunction.jobNamePrefix
     initializeFunction.commandDirectory = originalFunction.commandDirectory
 
-    for (inputField <- originalFunction.inputFields)
+    for (inputField <- originalFunction.inputFieldsWithValues)
-      initializeFunction.originalInputs :+= originalFunction.getFieldValue(inputField)
+      initializeFunction.originalInputs += originalFunction.getFieldValue(inputField)
 
     initializeFunction.tempDirectories = tempDirectories
     scatterFunction.tempDirectories = tempDirectories

scala/src/org/broadinstitute/sting/queue/util/CollectionUtils.scala

@@ -13,10 +13,9 @@ object CollectionUtils {
     result
   }
 
-  def updated(value: Any, f: (Any) => Any): Any = {
+  def updated(value: Any, f: Any => Any): Any = {
     value match {
-      case seq: Seq[_] => seq.map(updated(_, f))
+      case traversable: Traversable[_] => traversable.map(updated(_, f))
-      case array: Array[_] => array.map(updated(_, f))
       case option: Option[_] => option.map(updated(_, f))
       case x => f(x)
     }
@@ -24,22 +23,40 @@ object CollectionUtils {
 
   def foreach(value: Any, f: (Any, Any) => Unit): Unit = {
     value match {
-      case seq: Seq[_] =>
+      case traversable: Traversable[_] =>
-        for (item <- seq) {
+        for (item <- traversable) {
-          f(item, seq)
+          f(item, traversable)
           foreach(item, f)
         }
-      case product: Product =>
+      case option: Option[_] =>
-        for (item <- product.productIterator) {
+        for (item <- option) {
-          f(item, product)
+          f(item, option)
-          foreach(item, f)
-        }
-      case array: Array[_] =>
-        for (item <- array) {
-          f(item, array)
           foreach(item, f)
         }
       case _ =>
     }
   }
+
+  // Because scala allows but throws NPE when trying to hash a collection with a null in it.
+  // http://thread.gmane.org/gmane.comp.lang.scala.internals/3267
+  // https://lampsvn.epfl.ch/trac/scala/ticket/2935
+  def removeNullOrEmpty[T](value: T): T = filterNotNullOrNotEmpty(value)
+
+  private def filterNotNullOrNotEmpty[T](value: T): T = {
+    val newValue = value match {
+      case traversable: Traversable[_] => traversable.map(filterNotNullOrNotEmpty(_)).filter(isNotNullOrNotEmpty(_)).asInstanceOf[T]
+      case option: Option[_] => option.map(filterNotNullOrNotEmpty(_)).filter(isNotNullOrNotEmpty(_)).asInstanceOf[T]
+      case x => x
+    }
+    newValue
+  }
+
+  private def isNotNullOrNotEmpty(value: Any): Boolean = {
+    val result = value match {
+      case traversable: Traversable[_] => !filterNotNullOrNotEmpty(traversable).isEmpty
+      case option: Option[_] => !filterNotNullOrNotEmpty(option).isEmpty
+      case x => x != null
+    }
+    result
+  }
 }

scala/src/org/broadinstitute/sting/queue/util/ReflectionUtils.scala

@@ -2,9 +2,7 @@ package org.broadinstitute.sting.queue.util
 
 import org.broadinstitute.sting.queue.QException
 import java.lang.annotation.Annotation
-import scala.concurrent.JavaConversions
 import scala.concurrent.JavaConversions._
-import scala.collection.immutable.ListMap
 import java.lang.reflect.{ParameterizedType, Field}
 
 object ReflectionUtils {
@@ -20,8 +18,7 @@ object ReflectionUtils {
 
   def filterFields(fields: List[Field], annotation: Class[_ <: Annotation]) = fields.filter(field => hasAnnotation(field, annotation))
 
-  def getFieldNamesValues(obj: AnyRef, fields: List[Field]) =
+  def getFieldValues(obj: AnyRef, fields: List[Field]) = fields.map(field => fieldGetter(field).invoke(obj))
-    ListMap(fields.map(field => (field.getName -> fieldGetter(field).invoke(obj))) :_*)
 
   def getAllTypes(clazz: Class[_]) = {
     var types = List.empty[Class[_]]
@@ -65,16 +62,14 @@ object ReflectionUtils {
     }
   }
 
-  private def getCollectionType(field: Field) = {
+  def getCollectionType(field: Field) = {
     getGenericTypes(field) match {
       case Some(classes) =>
         if (classes.length > 1)
           throw new IllegalArgumentException("Field contains more than one generic type: " + field)
         classes(0)
       case None =>
-        if (!field.isAnnotationPresent(classOf[ClassType]))
+        throw new QException("Generic type not set for collection: " + field)
-          throw new QException("@ClassType must be specified for unparameterized field: " + field)
-        field.getAnnotation(classOf[ClassType]).asInstanceOf[ClassType].value
     }
   }