diff --git a/java/src/org/broadinstitute/sting/commandline/ArgumentDefinition.java b/java/src/org/broadinstitute/sting/commandline/ArgumentDefinition.java index 72cf400b6..985645a8d 100644 --- a/java/src/org/broadinstitute/sting/commandline/ArgumentDefinition.java +++ b/java/src/org/broadinstitute/sting/commandline/ArgumentDefinition.java @@ -33,6 +33,11 @@ import java.util.List; * A specific argument definition. Maps one-to-one with a field in some class. */ public class ArgumentDefinition { + /** + * Whether an argument is an input or an output. + */ + public final ArgumentIOType ioType; + /** * Full name of the argument. Must have a value. */ @@ -80,6 +85,7 @@ public class ArgumentDefinition { /** * Creates a new argument definition. + * @param ioType Whether the argument is an input or an output. * @param fullName Full name for this argument definition. * @param shortName Short name for this argument definition. * @param doc Doc string for this argument. @@ -90,7 +96,8 @@ public class ArgumentDefinition { * @param validation A regular expression for command-line argument validation. * @param validOptions is there a particular list of options that's valid for this argument definition? List them if so, otherwise set this to null. */ - public ArgumentDefinition( String fullName, + public ArgumentDefinition( ArgumentIOType ioType, + String fullName, String shortName, String doc, boolean required, @@ -99,6 +106,7 @@ public class ArgumentDefinition { String exclusiveOf, String validation, List validOptions) { + this.ioType = ioType; this.fullName = fullName; this.shortName = shortName; this.doc = doc; diff --git a/java/src/org/broadinstitute/sting/commandline/ArgumentDescription.java b/java/src/org/broadinstitute/sting/commandline/ArgumentDescription.java new file mode 100644 index 000000000..e0e1b80c7 --- /dev/null +++ b/java/src/org/broadinstitute/sting/commandline/ArgumentDescription.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2010, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.commandline; + +import org.broadinstitute.sting.utils.StingException; + +import java.lang.annotation.Annotation; + +/** + * Should be an interface, but java doesn't like inheritance in Annotations. + */ +public class ArgumentDescription { + private Annotation annotation; + + public ArgumentDescription(Annotation annotation) { + this.annotation = annotation; + } + + /** + * A hack to get around the fact that Java doesn't like inheritance in Annotations. + * @param method the method to invoke + * @return the return value of the method + */ + private Object invoke(String method) { + try { + return this.annotation.getClass().getMethod(method).invoke(this.annotation); + } catch (Exception e) { + throw new StingException("Unable to access method " + method + " on annotation " + annotation.getClass(), e); + } + } + + /** + * The directionality of the command-line argument. + * @return INPUT, OUTPUT, or UNKNOWN + */ + public ArgumentIOType getIOType() { + if (annotation instanceof Input) return ArgumentIOType.INPUT; + if (annotation instanceof Output) return ArgumentIOType.OUTPUT; + return ArgumentIOType.UNKNOWN; + } + + /** + * The full name of the command-line argument. Full names should be + * prefixed on the command-line with a double dash (--). + * @return Selected full name, or "" to use the default. + */ + public String fullName() { return (String)invoke("fullName"); } + + /** + * Specified short name of the command. Short names should be prefixed + * with a single dash. Argument values can directly abut single-char + * short names or be separated from them by a space. + * @return Selected short name, or "" for none. + */ + public String shortName() { return (String)invoke("shortName"); } + + /** + * Documentation for the command-line argument. Should appear when the + * --help argument is specified. + * @return Doc string associated with this command-line argument. + */ + public String doc() { return (String)invoke("doc"); } + + /** + * Is this command-line argument required. The application should exit + * printing help if this command-line argument is not specified. + * @return True if the argument is required. False otherwise. + */ + public boolean required() { return (Boolean)invoke("required"); } + + /** + * Should this command-line argument be exclusive of others. Should be + * a comma-separated list of names of arguments of which this should be + * independent. + * @return A comma-separated string listing other arguments of which this + * argument should be independent. + */ + public String exclusiveOf() { return (String)invoke("exclusiveOf"); } + + /** + * Provide a regexp-based validation string. + * @return Non-empty regexp for validation, blank otherwise. + */ + public String validation() { return (String)invoke("validation"); } +} diff --git a/java/src/org/broadinstitute/sting/queue/util/Scatter.java b/java/src/org/broadinstitute/sting/commandline/ArgumentIOType.java similarity index 84% rename from java/src/org/broadinstitute/sting/queue/util/Scatter.java rename to java/src/org/broadinstitute/sting/commandline/ArgumentIOType.java index 8314bea9c..af516004a 100644 --- a/java/src/org/broadinstitute/sting/queue/util/Scatter.java +++ b/java/src/org/broadinstitute/sting/commandline/ArgumentIOType.java @@ -22,14 +22,8 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -package org.broadinstitute.sting.queue.util; +package org.broadinstitute.sting.commandline; -import java.lang.annotation.*; - -@Documented -@Inherited -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.FIELD}) -public @interface Scatter { - Class value(); +public enum ArgumentIOType { + INPUT, OUTPUT, UNKNOWN } diff --git a/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java b/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java index b4005c80a..ffcca6b68 100644 --- a/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java +++ b/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java @@ -39,6 +39,7 @@ import java.util.*; * @version 0.1 */ public abstract class ArgumentTypeDescriptor { + private static Class[] ARGUMENT_ANNOTATIONS = {Input.class, Output.class, Argument.class}; /** * our log, which we want to capture anything from org.broadinstitute.sting @@ -121,7 +122,8 @@ public abstract class ArgumentTypeDescriptor { * @return The default definition for this argument source. */ protected ArgumentDefinition createDefaultArgumentDefinition( ArgumentSource source ) { - return new ArgumentDefinition( getFullName(source), + return new ArgumentDefinition( getIOType(source), + getFullName(source), getShortName(source), getDoc(source), isRequired(source), @@ -132,7 +134,17 @@ public abstract class ArgumentTypeDescriptor { getValidOptions(source) ); } - protected abstract Object parse( ArgumentSource source, Class type, ArgumentMatches matches ); + public abstract Object parse( ArgumentSource source, Class type, ArgumentMatches matches ); + + /** + * Specifies other arguments which cannot be used in conjunction with tihs argument. Comma-separated list. + * @param source Original field specifying command-line arguments. + * @return A comma-separated list of exclusive arguments, or null if none are present. + */ + protected ArgumentIOType getIOType( ArgumentSource source ) { + ArgumentDescription description = getArgumentDescription(source); + return description.getIOType(); + } /** * Retrieves the full name of the argument, specifiable with the '--' prefix. The full name can be @@ -141,7 +153,7 @@ public abstract class ArgumentTypeDescriptor { * @return full name of the argument. Never null. */ protected String getFullName( ArgumentSource source ) { - Argument description = getArgumentDescription(source); + ArgumentDescription description = getArgumentDescription(source); return description.fullName().trim().length() > 0 ? description.fullName().trim() : source.field.getName().toLowerCase(); } @@ -152,7 +164,7 @@ public abstract class ArgumentTypeDescriptor { * @return short name of the argument. Null if no short name exists. */ protected String getShortName( ArgumentSource source ) { - Argument description = getArgumentDescription(source); + ArgumentDescription description = getArgumentDescription(source); return description.shortName().trim().length() > 0 ? description.shortName().trim() : null; } @@ -162,7 +174,7 @@ public abstract class ArgumentTypeDescriptor { * @return Documentation for this argument. */ protected String getDoc( ArgumentSource source ) { - Argument description = getArgumentDescription(source); + ArgumentDescription description = getArgumentDescription(source); return description.doc(); } @@ -172,7 +184,7 @@ public abstract class ArgumentTypeDescriptor { * @return True if the field is mandatory and not a boolean flag. False otherwise. */ protected boolean isRequired( ArgumentSource source ) { - Argument description = getArgumentDescription(source); + ArgumentDescription description = getArgumentDescription(source); return description.required() && !source.isFlag(); } @@ -182,7 +194,7 @@ public abstract class ArgumentTypeDescriptor { * @return A comma-separated list of exclusive arguments, or null if none are present. */ protected String getExclusiveOf( ArgumentSource source ) { - Argument description = getArgumentDescription(source); + ArgumentDescription description = getArgumentDescription(source); return description.exclusiveOf().trim().length() > 0 ? description.exclusiveOf().trim() : null; } @@ -192,15 +204,15 @@ public abstract class ArgumentTypeDescriptor { * @return a JVM regex-compatible regular expression, or null to permit any possible value. */ protected String getValidationRegex( ArgumentSource source ) { - Argument description = getArgumentDescription(source); + ArgumentDescription description = getArgumentDescription(source); return description.validation().trim().length() > 0 ? description.validation().trim() : null; } /** * If the argument source only accepts a small set of options, populate the returned list with * those options. Otherwise, leave the list empty. - * @param source - * @return + * @param source Original field specifying command-line arguments. + * @return A list of valid options. */ protected List getValidOptions( ArgumentSource source ) { if(!source.field.getType().isEnum()) @@ -246,11 +258,21 @@ public abstract class ArgumentTypeDescriptor { * @param source source of the argument. * @return Argument description annotation associated with the given field. */ - protected Argument getArgumentDescription( ArgumentSource source ) { - if( !source.field.isAnnotationPresent(Argument.class) ) - throw new StingException("ArgumentAnnotation is not present for the argument field: " + source.field.getName()); - return source.field.getAnnotation(Argument.class); - } + @SuppressWarnings("unchecked") + protected ArgumentDescription getArgumentDescription( ArgumentSource source ) { + for (Class annotation: ARGUMENT_ANNOTATIONS) + if (source.field.isAnnotationPresent(annotation)) + return new ArgumentDescription(source.field.getAnnotation(annotation)); + throw new StingException("ArgumentAnnotation is not present for the argument field: " + source.field.getName()); + } + + @SuppressWarnings("unchecked") + public static boolean isArgumentDescriptionPresent(Field field) { + for (Class annotation: ARGUMENT_ANNOTATIONS) + if (field.isAnnotationPresent(annotation)) + return true; + return false; + } } /** @@ -276,7 +298,7 @@ class SimpleArgumentTypeDescriptor extends ArgumentTypeDescriptor { } @Override - protected Object parse( ArgumentSource source, Class type, ArgumentMatches matches ) { + public Object parse( ArgumentSource source, Class type, ArgumentMatches matches ) { String value = getArgumentValue( createDefaultArgumentDefinition(source), matches ); // lets go through the types we support @@ -345,9 +367,10 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor { } @Override + @SuppressWarnings("unchecked") public Object parse( ArgumentSource source, Class type, ArgumentMatches matches ) { - Class componentType = null; + Class componentType; if( Collection.class.isAssignableFrom(type) ) { @@ -372,7 +395,7 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor { ArgumentTypeDescriptor componentArgumentParser = ArgumentTypeDescriptor.create( componentType ); - Collection collection = null; + Collection collection; try { collection = (Collection)type.newInstance(); } diff --git a/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java b/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java index 8f63146e0..ef25f5993 100644 --- a/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java +++ b/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java @@ -46,7 +46,7 @@ public abstract class CommandLineProgram { private static Logger logger = Logger.getRootLogger(); /** the default log level */ - @Argument(fullName = "logging_level", + @Input(fullName = "logging_level", shortName = "l", doc = "Set the minimum level of logging, i.e. setting INFO get's you INFO up to FATAL, setting ERROR gets you ERROR and FATAL level logging.", required = false) @@ -54,28 +54,28 @@ public abstract class CommandLineProgram { /** where to send the output of our logger */ - @Argument(fullName = "log_to_file", + @Output(fullName = "log_to_file", shortName = "log", doc = "Set the logging location", required = false) protected String toFile = null; /** do we want to silence the command line output */ - @Argument(fullName = "quiet_output_mode", + @Input(fullName = "quiet_output_mode", shortName = "quiet", doc = "Set the logging to quiet mode, no output to stdout", required = false) protected Boolean quietMode = false; /** do we want to generate debugging information with the logs */ - @Argument(fullName = "debug_mode", + @Input(fullName = "debug_mode", shortName = "debug", doc = "Set the logging file string to include a lot of debugging information (SLOW!)", required = false) protected Boolean debugMode = false; /** this is used to indicate if they've asked for help */ - @Argument(fullName = "help", shortName = "h", doc = "Generate this help message", required = false) + @Input(fullName = "help", shortName = "h", doc = "Generate this help message", required = false) public Boolean help = false; /** our logging output patterns */ diff --git a/java/src/org/broadinstitute/sting/commandline/Input.java b/java/src/org/broadinstitute/sting/commandline/Input.java new file mode 100644 index 000000000..87fb741ea --- /dev/null +++ b/java/src/org/broadinstitute/sting/commandline/Input.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2010, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.commandline; + +import java.lang.annotation.*; + +/** + * Annotates fields in objects that should be used as command-line arguments. + * Any field annotated with @Input can appear as a command-line parameter. + */ +@Documented +@Inherited +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD}) +public @interface Input { + /** + * The full name of the command-line argument. Full names should be + * prefixed on the command-line with a double dash (--). + * @return Selected full name, or "" to use the default. + */ + String fullName() default ""; + + /** + * Specified short name of the command. Short names should be prefixed + * with a single dash. Argument values can directly abut single-char + * short names or be separated from them by a space. + * @return Selected short name, or "" for none. + */ + String shortName() default ""; + + /** + * Documentation for the command-line argument. Should appear when the + * --help argument is specified. + * @return Doc string associated with this command-line argument. + */ + String doc(); + + /** + * Is this command-line argument required. The application should exit + * printing help if this command-line argument is not specified. + * @return True if the argument is required. False otherwise. + */ + boolean required() default true; + + /** + * Should this command-line argument be exclusive of others. Should be + * a comma-separated list of names of arguments of which this should be + * independent. + * @return A comma-separated string listing other arguments of which this + * argument should be independent. + */ + String exclusiveOf() default ""; + + /** + * Provide a regexp-based validation string. + * @return Non-empty regexp for validation, blank otherwise. + */ + String validation() default ""; +} \ No newline at end of file diff --git a/java/src/org/broadinstitute/sting/commandline/Output.java b/java/src/org/broadinstitute/sting/commandline/Output.java new file mode 100644 index 000000000..23f1a4cd2 --- /dev/null +++ b/java/src/org/broadinstitute/sting/commandline/Output.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2010, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.commandline; + +import java.lang.annotation.*; + +/** + * Annotates fields in objects that should be used as command-line arguments. + * Any field annotated with @Argument can appear as a command-line parameter. + */ +@Documented +@Inherited +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD}) +public @interface Output { + /** + * The full name of the command-line argument. Full names should be + * prefixed on the command-line with a double dash (--). + * @return Selected full name, or "" to use the default. + */ + String fullName() default ""; + + /** + * Specified short name of the command. Short names should be prefixed + * with a single dash. Argument values can directly abut single-char + * short names or be separated from them by a space. + * @return Selected short name, or "" for none. + */ + String shortName() default ""; + + /** + * Documentation for the command-line argument. Should appear when the + * --help argument is specified. + * @return Doc string associated with this command-line argument. + */ + String doc(); + + /** + * Is this command-line argument required. The application should exit + * printing help if this command-line argument is not specified. + * @return True if the argument is required. False otherwise. + */ + boolean required() default true; + + /** + * Should this command-line argument be exclusive of others. Should be + * a comma-separated list of names of arguments of which this should be + * independent. + * @return A comma-separated string listing other arguments of which this + * argument should be independent. + */ + String exclusiveOf() default ""; + + /** + * Provide a regexp-based validation string. + * @return Non-empty regexp for validation, blank otherwise. + */ + String validation() default ""; +} \ No newline at end of file diff --git a/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java b/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java index c8b7cf419..52d8b1a59 100755 --- a/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java +++ b/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java @@ -45,11 +45,6 @@ public class ParsingEngine { */ CommandLineProgram clp = null; - /** - * A collection of all the source fields which define command-line arguments. - */ - List argumentSources = new ArrayList(); - /** * A list of defined arguments against which command lines are matched. * Package protected for testing access. @@ -125,8 +120,6 @@ public class ParsingEngine { * command-line arguments to the arguments that are actually * required. * @param tokens Tokens passed on the command line. - * @return A object indicating which matches are best. Might return - * an empty object, but will never return null. */ public void parse( String[] tokens ) { argumentMatches = new ArgumentMatches(); @@ -315,7 +308,7 @@ public class ParsingEngine { while( sourceClass != null ) { Field[] fields = sourceClass.getDeclaredFields(); for( Field field: fields ) { - if( field.isAnnotationPresent(Argument.class) ) + if( ArgumentTypeDescriptor.isArgumentDescriptionPresent(field) ) argumentSources.add( new ArgumentSource(sourceClass,field) ); if( field.isAnnotationPresent(ArgumentCollection.class) ) argumentSources.addAll( extractArgumentSources(field.getType()) ); diff --git a/java/src/org/broadinstitute/sting/gatk/io/stubs/GenotypeWriterArgumentTypeDescriptor.java b/java/src/org/broadinstitute/sting/gatk/io/stubs/GenotypeWriterArgumentTypeDescriptor.java index 73628b2a7..b207655c5 100644 --- a/java/src/org/broadinstitute/sting/gatk/io/stubs/GenotypeWriterArgumentTypeDescriptor.java +++ b/java/src/org/broadinstitute/sting/gatk/io/stubs/GenotypeWriterArgumentTypeDescriptor.java @@ -25,11 +25,7 @@ package org.broadinstitute.sting.gatk.io.stubs; -import org.broadinstitute.sting.commandline.ArgumentTypeDescriptor; -import org.broadinstitute.sting.commandline.ArgumentSource; -import org.broadinstitute.sting.commandline.ArgumentMatches; -import org.broadinstitute.sting.commandline.ArgumentDefinition; -import org.broadinstitute.sting.commandline.Argument; +import org.broadinstitute.sting.commandline.*; import org.broadinstitute.sting.utils.StingException; import org.broadinstitute.sting.utils.genotype.GenotypeWriter; import org.broadinstitute.sting.utils.genotype.GenotypeWriterFactory; @@ -53,7 +49,7 @@ public class GenotypeWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor /** * Create a new GenotypeWriter argument, notifying the given engine when that argument has been created. - * @param engine + * @param engine the engine to be notified. */ public GenotypeWriterArgumentTypeDescriptor(GenomeAnalysisEngine engine) { this.engine = engine; @@ -104,7 +100,7 @@ public class GenotypeWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor /** * Convert the given argument matches into a single object suitable for feeding into the ArgumentSource. * @param source Source for this argument. - * @param type + * @param type not used * @param matches Matches that match with this argument. * @return Transform from the matches into the associated argument. */ @@ -127,7 +123,7 @@ public class GenotypeWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor } // Create a stub for the given object. - GenotypeWriterStub stub = null; + GenotypeWriterStub stub; switch(genotypeFormat) { case GELI: stub = (writerFile != null) ? new GeliTextGenotypeWriterStub(engine, writerFile) : new GeliTextGenotypeWriterStub(engine,System.out); @@ -158,7 +154,7 @@ public class GenotypeWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor * @return Argument definition for the BAM file itself. Will not be null. */ private ArgumentDefinition createGenotypeFileArgumentDefinition(ArgumentSource source) { - Argument description = this.getArgumentDescription(source); + ArgumentDescription description = this.getArgumentDescription(source); boolean isFullNameProvided = description.fullName().trim().length() > 0; boolean isShortNameProvided = description.shortName().trim().length() > 0; @@ -175,7 +171,8 @@ public class GenotypeWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor else shortName = null; - return new ArgumentDefinition( fullName, + return new ArgumentDefinition( getIOType(source), + fullName, shortName, getDoc(source), isRequired(source), @@ -192,7 +189,8 @@ public class GenotypeWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor * @return Argument definition for the BAM file itself. Will not be null. */ private ArgumentDefinition createGenotypeFormatArgumentDefinition(ArgumentSource source) { - return new ArgumentDefinition( "variant_output_format", + return new ArgumentDefinition( getIOType(source), + "variant_output_format", "vf", "Format to be used to represent variants; default is VCF", false, diff --git a/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java b/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java index 682c14914..f6fd39107 100644 --- a/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java +++ b/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java @@ -94,7 +94,7 @@ public class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor * @return Argument definition for the BAM file itself. Will not be null. */ private ArgumentDefinition createBAMArgumentDefinition(ArgumentSource source) { - Argument description = this.getArgumentDescription(source); + ArgumentDescription description = this.getArgumentDescription(source); boolean isFullNameProvided = description.fullName().trim().length() > 0; boolean isShortNameProvided = description.shortName().trim().length() > 0; @@ -111,7 +111,8 @@ public class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor else shortName = null; - return new ArgumentDefinition( fullName, + return new ArgumentDefinition( getIOType(source), + fullName, shortName, getDoc(source), isRequired(source), @@ -128,7 +129,8 @@ public class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor * @return Argument definition for the BAM file itself. Will not be null. */ private ArgumentDefinition createBAMCompressionArgumentDefinition(ArgumentSource source) { - return new ArgumentDefinition( COMPRESSION_FULLNAME, + return new ArgumentDefinition( getIOType(source), + COMPRESSION_FULLNAME, COMPRESSION_SHORTNAME, "Compression level to use for writing BAM files", false, diff --git a/java/src/org/broadinstitute/sting/queue/util/ClassType.java b/java/src/org/broadinstitute/sting/queue/function/scattergather/Gather.java similarity index 89% rename from java/src/org/broadinstitute/sting/queue/util/ClassType.java rename to java/src/org/broadinstitute/sting/queue/function/scattergather/Gather.java index 50fbf58e7..45e0afaba 100644 --- a/java/src/org/broadinstitute/sting/queue/util/ClassType.java +++ b/java/src/org/broadinstitute/sting/queue/function/scattergather/Gather.java @@ -22,18 +22,18 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -package org.broadinstitute.sting.queue.util; +package org.broadinstitute.sting.queue.function.scattergather; import java.lang.annotation.*; /** - * Specifies the type of an input or output field. + * Specifies the class type of the CommandLineFunction to gather an @Output * Written in java because scala doesn't support RetentionPolicy.RUNTIME */ @Documented @Inherited @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.FIELD}) -public @interface ClassType { +public @interface Gather { Class value(); } diff --git a/java/src/org/broadinstitute/sting/queue/util/Optional.java b/java/src/org/broadinstitute/sting/queue/function/scattergather/Scatter.java similarity index 87% rename from java/src/org/broadinstitute/sting/queue/util/Optional.java rename to java/src/org/broadinstitute/sting/queue/function/scattergather/Scatter.java index 07ee36031..471f7fe29 100644 --- a/java/src/org/broadinstitute/sting/queue/util/Optional.java +++ b/java/src/org/broadinstitute/sting/queue/function/scattergather/Scatter.java @@ -22,17 +22,18 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -package org.broadinstitute.sting.queue.util; +package org.broadinstitute.sting.queue.function.scattergather; import java.lang.annotation.*; /** - * Specifies an input or output to a QFunction is optional + * Specifies the class type of the CommandLineFunction to scatter an @Input * Written in java because scala doesn't support RetentionPolicy.RUNTIME */ @Documented @Inherited @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.FIELD}) -public @interface Optional { +public @interface Scatter { + Class value(); } diff --git a/java/src/org/broadinstitute/sting/queue/util/Gather.java b/java/src/org/broadinstitute/sting/queue/util/Gather.java deleted file mode 100644 index 9928b3ad3..000000000 --- a/java/src/org/broadinstitute/sting/queue/util/Gather.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2010, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.queue.util; - -import java.lang.annotation.*; - -/** - * Specifies the class to gather an output of a QFunction. - * Not an input or output but should be copied with a function. - * Internals should have default values that should be handled, i.e. they are always @Optional - * A common use for @Internal is to specify WHERE a function runs: farm queue, directory, etc. - * or to name part of a function: farm job name - * Written in java because scala doesn't support RetentionPolicy.RUNTIME - */ -@Documented -@Inherited -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.FIELD}) -public @interface Gather { - Class value(); -} diff --git a/java/src/org/broadinstitute/sting/queue/util/Input.java b/java/src/org/broadinstitute/sting/queue/util/Input.java deleted file mode 100644 index 941945fc2..000000000 --- a/java/src/org/broadinstitute/sting/queue/util/Input.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2010, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.queue.util; - -import java.lang.annotation.*; - -/** - * Specifies an input to a QFunction - * Written in java because scala doesn't support RetentionPolicy.RUNTIME - */ -@Documented -@Inherited -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.FIELD}) -public @interface Input { -} diff --git a/java/src/org/broadinstitute/sting/queue/util/Internal.java b/java/src/org/broadinstitute/sting/queue/util/Internal.java deleted file mode 100644 index 758f28b6d..000000000 --- a/java/src/org/broadinstitute/sting/queue/util/Internal.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2010, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.queue.util; - -import java.lang.annotation.*; - -/** - * Specifies an internal setting for a QFunction. - * Not an input or output but should be copied with a function. - * Internals should have default values that should be handled, i.e. they are always @Optional - * A common use for @Internal is to specify WHERE a function runs: farm queue, directory, etc. - * or to name part of a function: farm job name - * Written in java because scala doesn't support RetentionPolicy.RUNTIME - */ -@Documented -@Inherited -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.FIELD}) -public @interface Internal { -} diff --git a/java/src/org/broadinstitute/sting/queue/util/Output.java b/java/src/org/broadinstitute/sting/queue/util/Output.java deleted file mode 100644 index ca0ba8dac..000000000 --- a/java/src/org/broadinstitute/sting/queue/util/Output.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2010, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.queue.util; - -import java.lang.annotation.*; - -/** - * Specifies an output to a QFunction - * Written in java because scala doesn't support RetentionPolicy.RUNTIME - */ -@Documented -@Inherited -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.FIELD}) -public @interface Output { -} diff --git a/scala/qscript/unifiedgenotyper_example.properties b/scala/qscript/unifiedgenotyper_example.properties new file mode 100644 index 000000000..6730f6b22 --- /dev/null +++ b/scala/qscript/unifiedgenotyper_example.properties @@ -0,0 +1,5 @@ +gatkJar = /humgen/gsa-hpprojects/GATK/bin/current/GenomeAnalysisTK.jar +referenceFile = /path/to/reference.fasta +dbsnp = /path/to/dbsnp +intervals = /path/to/my.interval_list +jobNamePrefix = Q diff --git a/scala/qscript/unifiedgenotyper_example.scala b/scala/qscript/unifiedgenotyper_example.scala new file mode 100644 index 000000000..ecd21b00f --- /dev/null +++ b/scala/qscript/unifiedgenotyper_example.scala @@ -0,0 +1,54 @@ +import org.broadinstitute.sting.queue.QScript._ + +setArgs(args) + +for (bam <- inputs("bam")) { + val ug = new UnifiedGenotyper + val vf = new VariantFiltration + val ve = new GatkFunction { + @Input(doc="vcf") var vcfFile: File = _ + @Output(doc="eval") var evalFile: File = _ + def commandLine = gatkCommandLine("VariantEval") + "-B eval,VCF,%s -o %s".format(vcfFile, evalFile) + } + + // Make sure the Sting/scripts folder is in your path to use mergeText.sh and splitIntervals.sh. + ug.scatterCount = 3 + ug.bamFiles :+= bam + ug.vcfFile = swapExt(bam, "bam", "unfiltered.vcf") + + vf.vcfInput = ug.vcfFile + vf.vcfOutput = swapExt(bam, "bam", "filtered.vcf") + + ve.vcfFile = vf.vcfOutput + ve.evalFile = swapExt(bam, "bam", "eval") + + add(ug, vf, ve) +} + +setParams +run + + +class UnifiedGenotyper extends GatkFunction { + @Output(doc="vcf") + @Gather(classOf[SimpleTextGatherFunction]) + var vcfFile: File = _ + def commandLine = gatkCommandLine("UnifiedGenotyper") + "-varout %s".format(vcfFile) +} + +class VariantFiltration extends GatkFunction { + @Input(doc="input vcf") + var vcfInput: File = _ + + @Input(doc="filter names") + var filterNames: List[String] = Nil + + @Input(doc="filter expressions") + var filterExpressions: List[String] = Nil + + @Output(doc="output vcf") + var vcfOutput: File = _ + + def commandLine = gatkCommandLine("VariantFiltration") + "%s%s -B variant,VCF,%s -o %s" + .format(repeat(" -filterName ", filterNames), repeat(" -filterExpression ", filterExpressions), vcfInput, vcfOutput) +} diff --git a/scala/src/org/broadinstitute/sting/queue/QException.scala b/scala/src/org/broadinstitute/sting/queue/QException.scala index 5c439edeb..241d11b75 100755 --- a/scala/src/org/broadinstitute/sting/queue/QException.scala +++ b/scala/src/org/broadinstitute/sting/queue/QException.scala @@ -1,4 +1,6 @@ package org.broadinstitute.sting.queue +import org.broadinstitute.sting.utils.StingException + class QException(private val message: String, private val throwable: Throwable = null) - extends RuntimeException(message, throwable) + extends StingException(message, throwable) diff --git a/scala/src/org/broadinstitute/sting/queue/QScript.scala b/scala/src/org/broadinstitute/sting/queue/QScript.scala index 4151c7802..68cf734b5 100755 --- a/scala/src/org/broadinstitute/sting/queue/QScript.scala +++ b/scala/src/org/broadinstitute/sting/queue/QScript.scala @@ -9,15 +9,13 @@ import org.broadinstitute.sting.queue.engine.QGraph object QScript { // Type aliases so users don't have to import type File = java.io.File - type Input = org.broadinstitute.sting.queue.util.Input - type Output = org.broadinstitute.sting.queue.util.Output - type Optional = org.broadinstitute.sting.queue.util.Optional - type ClassType = org.broadinstitute.sting.queue.util.ClassType + type Input = org.broadinstitute.sting.commandline.Input + type Output = org.broadinstitute.sting.commandline.Output type CommandLineFunction = org.broadinstitute.sting.queue.function.CommandLineFunction type GatkFunction = org.broadinstitute.sting.queue.function.gatk.GatkFunction type ScatterGatherableFunction = org.broadinstitute.sting.queue.function.scattergather.ScatterGatherableFunction - type Scatter = org.broadinstitute.sting.queue.util.Scatter - type Gather = org.broadinstitute.sting.queue.util.Gather + type Scatter = org.broadinstitute.sting.queue.function.scattergather.Scatter + type Gather = org.broadinstitute.sting.queue.function.scattergather.Gather type BamGatherFunction = org.broadinstitute.sting.queue.function.scattergather.BamGatherFunction type SimpleTextGatherFunction = org.broadinstitute.sting.queue.function.scattergather.SimpleTextGatherFunction diff --git a/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala b/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala index 3c27cfa58..b952e13cf 100755 --- a/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala +++ b/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala @@ -3,7 +3,6 @@ package org.broadinstitute.sting.queue.engine import org.jgrapht.graph.SimpleDirectedGraph import scala.collection.JavaConversions import scala.collection.JavaConversions._ -import scala.collection.immutable.ListMap import org.broadinstitute.sting.queue.function.{MappingFunction, CommandLineFunction, QFunction} import org.broadinstitute.sting.queue.function.scattergather.ScatterGatherableFunction import org.broadinstitute.sting.queue.util.{CollectionUtils, Logging} @@ -18,7 +17,7 @@ class QGraph extends Logging { def numJobs = JavaConversions.asSet(jobGraph.edgeSet).filter(_.isInstanceOf[CommandLineFunction]).size def add(command: CommandLineFunction) { - add(command, true) + addFunction(command) } /** @@ -27,20 +26,8 @@ class QGraph extends Logging { def fillIn = { // clone since edgeSet is backed by the graph for (function <- JavaConversions.asSet(jobGraph.edgeSet).clone) { - val inputs = function.inputs - val outputs = function.outputs - - for ((name, input) <- inputs) { - addCollectionInputs(name, input) - if (inputs.size > 1) - addMappingEdge(ListMap(name -> input), inputs) - } - - for ((name, output) <- outputs) { - addCollectionOutputs(name, output) - if (outputs.size > 1) - addMappingEdge(outputs, ListMap(name -> output)) - } + addCollectionOutputs(function.outputs) + addCollectionInputs(function.inputs) } var pruning = true @@ -85,9 +72,9 @@ class QGraph extends Logging { } private def newGraph = new SimpleDirectedGraph[QNode, QFunction](new EdgeFactory[QNode, QFunction] { - def createEdge(input: QNode, output: QNode) = new MappingFunction(input.valueMap, output.valueMap)}) + def createEdge(input: QNode, output: QNode) = new MappingFunction(input.items, output.items)}) - private def add(f: QFunction, replace: Boolean): Unit = { + private def addFunction(f: QFunction): Unit = { try { f.freeze @@ -96,13 +83,13 @@ class QGraph extends Logging { val functions = scatterGather.generateFunctions() if (logger.isTraceEnabled) logger.trace("Scattered into %d parts: %s".format(functions.size, functions)) - functions.foreach(add(_)) + functions.foreach(addFunction(_)) case _ => - val inputs = QNode(f.inputs.values.filter(_ != null).toSet) - val outputs = QNode(f.outputs.values.filter(_ != null).toSet) + val inputs = QNode(f.inputs) + val outputs = QNode(f.outputs) val newSource = jobGraph.addVertex(inputs) val newTarget = jobGraph.addVertex(outputs) - val removedEdges = if (replace) jobGraph.removeAllEdges(inputs, outputs) else Nil + val removedEdges = jobGraph.removeAllEdges(inputs, outputs) val added = jobGraph.addEdge(inputs, outputs, f) if (logger.isTraceEnabled) { logger.trace("Mapped from: " + inputs) @@ -120,45 +107,41 @@ class QGraph extends Logging { } } - private def addCollectionInputs(name: String, value: Any): Unit = { + private def addCollectionInputs(value: Any): Unit = { CollectionUtils.foreach(value, (item, collection) => - addMappingEdge(ListMap(name -> item), ListMap(name -> collection))) + addMappingEdge(item, collection)) } - private def addCollectionOutputs(name: String, value: Any): Unit = { + private def addCollectionOutputs(value: Any): Unit = { CollectionUtils.foreach(value, (item, collection) => - addMappingEdge(ListMap(name -> collection), ListMap(name -> item))) + addMappingEdge(collection, item)) } - private def addMappingEdge(input: ListMap[String, Any], output: ListMap[String, Any]) = - add(new MappingFunction(input, output), false) + private def addMappingEdge(input: Any, output: Any) = { + val inputSet = asSet(input) + val outputSet = asSet(output) + val hasEdge = inputSet == outputSet || + jobGraph.getEdge(QNode(inputSet), QNode(outputSet)) != null || + jobGraph.getEdge(QNode(outputSet), QNode(inputSet)) != null + if (!hasEdge) + addFunction(new MappingFunction(inputSet, outputSet)) + } + + private def asSet(value: Any): Set[Any] = if (value.isInstanceOf[Set[_]]) value.asInstanceOf[Set[Any]] else Set(value) private def isMappingEdge(edge: QFunction) = edge.isInstanceOf[MappingFunction] private def isFiller(edge: QFunction) = { if (isMappingEdge(edge)) { - val source = jobGraph.getEdgeSource(edge) - val target = jobGraph.getEdgeTarget(edge) - if (jobGraph.outgoingEdgesOf(target).size == 0 || jobGraph.incomingEdgesOf(source).size == 0) + if (jobGraph.outgoingEdgesOf(jobGraph.getEdgeTarget(edge)).size == 0) true - else if (isLoopback(source) || isLoopback(target)) + else if (jobGraph.incomingEdgesOf(jobGraph.getEdgeSource(edge)).size == 0) true else false } else false } - private def isLoopback(node: QNode) = { - var loopback = false - val incoming = jobGraph.incomingEdgesOf(node) - val outgoing = jobGraph.outgoingEdgesOf(node) - if (incoming.size == 1 && outgoing.size == 1) - if (isMappingEdge(incoming.head) && isMappingEdge(outgoing.head)) - if (jobGraph.getEdgeSource(incoming.head) == jobGraph.getEdgeTarget(outgoing.head)) - loopback = true - loopback - } - private def isOrphan(node: QNode) = (jobGraph.incomingEdgesOf(node).size + jobGraph.outgoingEdgesOf(node).size) == 0 } diff --git a/scala/src/org/broadinstitute/sting/queue/engine/QNode.scala b/scala/src/org/broadinstitute/sting/queue/engine/QNode.scala index 48d74dae1..01d3b814c 100644 --- a/scala/src/org/broadinstitute/sting/queue/engine/QNode.scala +++ b/scala/src/org/broadinstitute/sting/queue/engine/QNode.scala @@ -1,20 +1,6 @@ package org.broadinstitute.sting.queue.engine -import scala.collection.immutable.ListMap - /** * Represents a state between QFunctions the directed acyclic QGraph */ -case class QNode (private val items: Set[Any]) { - /** - * Used during QGraph error reporting. - * The EdgeFactory uses the valueMap to create new edges for the CycleDetector. - */ - def valueMap = { - var map = ListMap.empty[String, Any] - for (item <- items) - if (item != null) - map += item.toString -> item - map - } -} +case class QNode (val items: Set[Any]) diff --git a/scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala index 5b0700fc6..2aee31327 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala @@ -1,11 +1,14 @@ package org.broadinstitute.sting.queue.function -import java.io.File import org.broadinstitute.sting.queue.util._ -import org.broadinstitute.sting.queue.engine.{CommandLineRunner, QGraph} import java.lang.reflect.Field +import java.lang.annotation.Annotation +import org.broadinstitute.sting.commandline.{Input, Output, ArgumentDescription} trait CommandLineFunction extends InputOutputFunction with DispatchFunction { + def inputFieldsWithValues = inputFields.filter(hasFieldValue(_)) + def outputFieldsWithValues = outputFields.filter(hasFieldValue(_)) + /** * Repeats parameters with a prefix/suffix if they are set otherwise returns "". * Skips null, Nil, None. Unwraps Some(x) to x. Everything else is called with x.toString. @@ -21,22 +24,24 @@ trait CommandLineFunction extends InputOutputFunction with DispatchFunction { if (hasValue(param)) prefix + toValue(param) + suffix else "" def missingValues = { - val missingInputs = missingFields(inputFields) - val missingOutputs = missingFields(outputFields) + val missingInputs = missingFields(inputFields, classOf[Input]) + val missingOutputs = missingFields(outputFields, classOf[Output]) missingInputs | missingOutputs } - private def missingFields(fields: List[Field]) = { + private def missingFields(fields: List[Field], annotation: Class[_ <: Annotation]) = { var missing = Set.empty[String] for (field <- fields) { - val isOptional = ReflectionUtils.hasAnnotation(field, classOf[Optional]) - if (!isOptional) + if (isRequired(field, annotation)) if (!hasValue(ReflectionUtils.getValue(this, field))) missing += field.getName } missing } + private def isRequired(field: Field, annotation: Class[_ <: Annotation]) = + new ArgumentDescription(field.getAnnotation(annotation)).required + protected def hasFieldValue(field: Field) = hasValue(this.getFieldValue(field)) private def hasValue(param: Any) = param match { diff --git a/scala/src/org/broadinstitute/sting/queue/function/DispatchFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/DispatchFunction.scala index 0a685d545..cea5e862e 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/DispatchFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/DispatchFunction.scala @@ -2,41 +2,40 @@ package org.broadinstitute.sting.queue.function import java.io.File import java.lang.management.ManagementFactory -import org.broadinstitute.sting.queue.function.scattergather.SimpleTextGatherFunction -import org.broadinstitute.sting.queue.util._ +import org.broadinstitute.sting.commandline.{Output, Input} +import org.broadinstitute.sting.queue.function.scattergather.{Gather, SimpleTextGatherFunction} +import org.broadinstitute.sting.queue.util.IOUtils trait DispatchFunction extends InputOutputFunction { def commandLine: String - @Input - @Optional - @ClassType(classOf[Int]) + @Input(doc="Upper memory limit", required=false) var memoryLimit: Option[Int] = None /** * The directory where the command should run. */ - @Internal + @Input(doc="Directory to write any files", required=false) var commandDirectory: File = IOUtils.CURRENT_DIR - @Internal + @Input(doc="Prefix for automatic job name creation", required=false) var jobNamePrefix: String = _ - @Internal + @Input(doc="Job name to run on the farm", required=false) var jobName: String = _ - @Output + @Output(doc="File to redirect any output", required=false) @Gather(classOf[SimpleTextGatherFunction]) var jobOutputFile: File = _ - @Output + @Output(doc="File to redirect any errors", required=false) @Gather(classOf[SimpleTextGatherFunction]) var jobErrorFile: File = _ - @Internal + @Input(doc="Job project to run the command", required=false) var jobProject = "Queue" - @Internal + @Input(doc="Job queue to run the command", required=false) var jobQueue = "broad" override def freeze = { diff --git a/scala/src/org/broadinstitute/sting/queue/function/InputOutputFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/InputOutputFunction.scala index 61b298f16..ebf1ed768 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/InputOutputFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/InputOutputFunction.scala @@ -2,6 +2,7 @@ package org.broadinstitute.sting.queue.function import java.lang.reflect.Field import org.broadinstitute.sting.queue.util._ +import org.broadinstitute.sting.commandline.{Input, Output} /** * A function with @Inputs and @Outputs tagging fields that can be set by the user in a QScript @@ -10,31 +11,30 @@ trait InputOutputFunction extends QFunction with Cloneable { def getFieldValue(field: Field) = ReflectionUtils.getValue(this, field) def setFieldValue(field: Field, value: Any) = ReflectionUtils.setValue(this, field, value) - def functionFields: List[Field] = inputFields ::: outputFields ::: internalFields + def functionFields: List[Field] = inputFields ::: outputFields def inputFields = ReflectionUtils.filterFields(fields, classOf[Input]) def outputFields = ReflectionUtils.filterFields(fields, classOf[Output]) - def internalFields = ReflectionUtils.filterFields(fields, classOf[Internal]) private lazy val fields = ReflectionUtils.getAllFields(this.getClass) - def inputs = ReflectionUtils.getFieldNamesValues(this, inputFields) - def outputs = ReflectionUtils.getFieldNamesValues(this, outputFields) - def internals = ReflectionUtils.getFieldNamesValues(this, internalFields) + // TODO: Need to handle argument collections where field is not on THIS + def inputs = CollectionUtils.removeNullOrEmpty(ReflectionUtils.getFieldValues(this, inputFields)).toSet + def outputs = CollectionUtils.removeNullOrEmpty(ReflectionUtils.getFieldValues(this, outputFields)).toSet /** * Sets a field value using the name of the field. * Field must be annotated with @Input, @Output, or @Internal - * @returns true if the value was found and set + * @return true if the value was found and set */ def addOrUpdateWithStringValue(name: String, value: String) = { fields.find(_.getName == name) match { case Some(field) => val isInput = ReflectionUtils.hasAnnotation(field, classOf[Input]) val isOutput = ReflectionUtils.hasAnnotation(field, classOf[Output]) - val isInternal = ReflectionUtils.hasAnnotation(field, classOf[Internal]) - if (isInput || isOutput || isInternal) { + if (isInput || isOutput) { ReflectionUtils.addOrUpdateWithStringValue(this, field, value) } true + // TODO: Need to handle argument collections where field is not on THIS case None => false } } diff --git a/scala/src/org/broadinstitute/sting/queue/function/MappingFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/MappingFunction.scala index affeb7cc4..cd4b14246 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/MappingFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/MappingFunction.scala @@ -1,14 +1,9 @@ package org.broadinstitute.sting.queue.function -import org.broadinstitute.sting.queue.engine.QGraph -import scala.collection.immutable.ListMap - /** * Utility class to map a set of inputs to set of outputs. - * The QGraph uses this function internally to return + * The QGraph uses this function internally to map between user defined functions. */ -class MappingFunction(private val in: ListMap[String, Any], private val out: ListMap[String, Any]) extends QFunction { - def inputs = in - def outputs = out - override def toString = "" +class MappingFunction(val inputs: Set[Any], val outputs: Set[Any]) extends QFunction { + override def toString = "" // For debugging } diff --git a/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala index 4d639d09f..29dfabd57 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala @@ -1,11 +1,8 @@ package org.broadinstitute.sting.queue.function -import scala.collection.immutable.ListMap - /** * The base interface for all functions in Queue. - * Inputs and outputs are specified as ListMaps of name -> value. - * The names are used for debugging. + * Inputs and outputs are specified as Sets of values. * Inputs are matched to other outputs by using .equals() */ trait QFunction { @@ -17,12 +14,12 @@ trait QFunction { def freeze = {} /** - * ListMap of name -> value inputs for this function. + * Set of inputs for this function. */ - def inputs: ListMap[String, Any] + def inputs: Set[Any] /** - * ListMap of name -> value outputs for this function. + * Set of outputs for this function. */ - def outputs: ListMap[String, Any] + def outputs: Set[Any] } diff --git a/scala/src/org/broadinstitute/sting/queue/function/gatk/GatkFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/gatk/GatkFunction.scala index 84a81ec16..e985f39d2 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/gatk/GatkFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/gatk/GatkFunction.scala @@ -2,31 +2,27 @@ package org.broadinstitute.sting.queue.function.gatk import java.io.File import org.broadinstitute.sting.queue.function.IntervalFunction -import org.broadinstitute.sting.queue.util.{Scatter, Internal, Input, Optional} -import org.broadinstitute.sting.queue.function.scattergather.{ScatterGatherableFunction, IntervalScatterFunction} +import org.broadinstitute.sting.queue.function.scattergather.{Scatter, ScatterGatherableFunction, IntervalScatterFunction} +import org.broadinstitute.sting.commandline.Input trait GatkFunction extends ScatterGatherableFunction with IntervalFunction { - @Internal - @Optional + @Input(doc="Temporary directory to write any files", required=false) var javaTmpDir: String = _ - @Input + @Input(doc="GATK jar") var gatkJar: String = _ - @Input + @Input(doc="Reference fasta") var referenceFile: File = _ - @Input - @Optional + @Input(doc="Bam files", required=false) var bamFiles: List[File] = Nil - @Input - @Optional + @Input(doc="Intervals", required=false) @Scatter(classOf[IntervalScatterFunction]) var intervals: File = _ - @Input - @Optional + @Input(doc="DBSNP", required=false) var dbsnp: File = _ protected def gatkCommandLine(walker: String) = diff --git a/scala/src/org/broadinstitute/sting/queue/function/scattergather/CleanupTempDirsFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/scattergather/CleanupTempDirsFunction.scala index 5de2ed33c..a3ebc953b 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/scattergather/CleanupTempDirsFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/scattergather/CleanupTempDirsFunction.scala @@ -1,14 +1,14 @@ package org.broadinstitute.sting.queue.function.scattergather import org.broadinstitute.sting.queue.function.CommandLineFunction -import org.broadinstitute.sting.queue.util.Input +import org.broadinstitute.sting.commandline.Input import java.io.File class CleanupTempDirsFunction extends CommandLineFunction { - @Input - var originalOutputs: List[Any] = Nil + @Input(doc="Original outputs of the gather functions") + var originalOutputs: Set[Any] = Set.empty[Any] - @Input + @Input(doc="Temporary directories to be deleted") var tempDirectories: List[File] = Nil def commandLine = "rm -rf%s".format(repeat(" '", tempDirectories, "'")) diff --git a/scala/src/org/broadinstitute/sting/queue/function/scattergather/CreateTempDirsFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/scattergather/CreateTempDirsFunction.scala index b10090348..8e31bf091 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/scattergather/CreateTempDirsFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/scattergather/CreateTempDirsFunction.scala @@ -2,13 +2,13 @@ package org.broadinstitute.sting.queue.function.scattergather import java.io.File import org.broadinstitute.sting.queue.function.CommandLineFunction -import org.broadinstitute.sting.queue.util.{Output, Input} +import org.broadinstitute.sting.commandline.{Output, Input} class CreateTempDirsFunction extends CommandLineFunction { - @Input - var originalInputs: List[Any] = Nil + @Input(doc="Original inputs to the scattered function") + var originalInputs: Set[Any] = Set.empty[Any] - @Output + @Output(doc="Temporary directories to create") var tempDirectories: List[File] = Nil def commandLine = "mkdir%s".format(repeat(" '", tempDirectories, "'")) diff --git a/scala/src/org/broadinstitute/sting/queue/function/scattergather/GatherFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/scattergather/GatherFunction.scala index 8bdce674b..72941b18f 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/scattergather/GatherFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/scattergather/GatherFunction.scala @@ -1,15 +1,15 @@ package org.broadinstitute.sting.queue.function.scattergather import org.broadinstitute.sting.queue.function.{CommandLineFunction} -import org.broadinstitute.sting.queue.util.{Input, Output} +import org.broadinstitute.sting.commandline.{Input, Output} trait GatherFunction extends CommandLineFunction { type GatherType - @Input + @Input(doc="Parts to gather back into the original output") var gatherParts: List[GatherType] = Nil - @Output + @Output(doc="The original output of the scattered function") var originalOutput: GatherType = _ def setOriginalFunction(originalFunction: ScatterGatherableFunction) = {} diff --git a/scala/src/org/broadinstitute/sting/queue/function/scattergather/IntervalScatterFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/scattergather/IntervalScatterFunction.scala index 979acebd8..8408622c0 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/scattergather/IntervalScatterFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/scattergather/IntervalScatterFunction.scala @@ -1,13 +1,13 @@ package org.broadinstitute.sting.queue.function.scattergather import java.io.File -import org.broadinstitute.sting.queue.util.Input +import org.broadinstitute.sting.commandline.Input import org.broadinstitute.sting.queue.function.IntervalFunction class IntervalScatterFunction extends ScatterFunction { type ScatterType = File - @Input + @Input(doc="Reference file to scatter") var referenceFile: File = _ override def setOriginalFunction(originalFunction: ScatterGatherableFunction) = { diff --git a/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterFunction.scala index f93bc857a..c17beb9c4 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterFunction.scala @@ -1,19 +1,19 @@ package org.broadinstitute.sting.queue.function.scattergather import org.broadinstitute.sting.queue.function.CommandLineFunction -import org.broadinstitute.sting.queue.util.{Input, Output} +import org.broadinstitute.sting.commandline.{Input, Output} import java.io.File trait ScatterFunction extends CommandLineFunction { type ScatterType - @Input + @Input(doc="Original input to scatter") var originalInput: ScatterType = _ - @Input + @Input(doc="Temporary directories for each scatter part") var tempDirectories: List[File] = Nil - @Output + @Output(doc="Scattered parts of the original input, one per temp directory") var scatterParts: List[ScatterType] = Nil def setOriginalFunction(originalFunction: ScatterGatherableFunction) = {} diff --git a/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala index 13da1b459..6f5e73727 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala @@ -4,9 +4,11 @@ import org.broadinstitute.sting.queue.function.CommandLineFunction import java.lang.reflect.Field import java.io.File import org.broadinstitute.sting.queue.util._ +import org.broadinstitute.sting.commandline.Input trait ScatterGatherableFunction extends CommandLineFunction { - @Internal + + @Input(doc="Number of parts to scatter the function into") var scatterCount: Int = 1 def scatterField = this.inputFields.find(field => ReflectionUtils.hasAnnotation(field, classOf[Scatter])).get @@ -48,7 +50,7 @@ object ScatterGatherableFunction { // Create the gather functions for each output field var gatherFunctions = Map.empty[Field, GatherFunction] - for (outputField <- originalFunction.outputFields) { + for (outputField <- originalFunction.outputFieldsWithValues) { // Create the gather function based on @Gather val gatherFunction = getGatherFunction(outputField) @@ -60,7 +62,7 @@ object ScatterGatherableFunction { gatherFunction.originalOutput = gatheredValue tempDirectories :+= gatherFunction.commandDirectory - cleanupFunction.originalOutputs :+= gatheredValue + cleanupFunction.originalOutputs += gatheredValue functions :+= gatherFunction @@ -97,8 +99,8 @@ object ScatterGatherableFunction { initializeFunction.jobNamePrefix = originalFunction.jobNamePrefix initializeFunction.commandDirectory = originalFunction.commandDirectory - for (inputField <- originalFunction.inputFields) - initializeFunction.originalInputs :+= originalFunction.getFieldValue(inputField) + for (inputField <- originalFunction.inputFieldsWithValues) + initializeFunction.originalInputs += originalFunction.getFieldValue(inputField) initializeFunction.tempDirectories = tempDirectories scatterFunction.tempDirectories = tempDirectories diff --git a/scala/src/org/broadinstitute/sting/queue/util/CollectionUtils.scala b/scala/src/org/broadinstitute/sting/queue/util/CollectionUtils.scala index 169de8677..b384c8dfa 100644 --- a/scala/src/org/broadinstitute/sting/queue/util/CollectionUtils.scala +++ b/scala/src/org/broadinstitute/sting/queue/util/CollectionUtils.scala @@ -13,10 +13,9 @@ object CollectionUtils { result } - def updated(value: Any, f: (Any) => Any): Any = { + def updated(value: Any, f: Any => Any): Any = { value match { - case seq: Seq[_] => seq.map(updated(_, f)) - case array: Array[_] => array.map(updated(_, f)) + case traversable: Traversable[_] => traversable.map(updated(_, f)) case option: Option[_] => option.map(updated(_, f)) case x => f(x) } @@ -24,22 +23,40 @@ object CollectionUtils { def foreach(value: Any, f: (Any, Any) => Unit): Unit = { value match { - case seq: Seq[_] => - for (item <- seq) { - f(item, seq) + case traversable: Traversable[_] => + for (item <- traversable) { + f(item, traversable) foreach(item, f) } - case product: Product => - for (item <- product.productIterator) { - f(item, product) - foreach(item, f) - } - case array: Array[_] => - for (item <- array) { - f(item, array) + case option: Option[_] => + for (item <- option) { + f(item, option) foreach(item, f) } case _ => } } + + // Because scala allows but throws NPE when trying to hash a collection with a null in it. + // http://thread.gmane.org/gmane.comp.lang.scala.internals/3267 + // https://lampsvn.epfl.ch/trac/scala/ticket/2935 + def removeNullOrEmpty[T](value: T): T = filterNotNullOrNotEmpty(value) + + private def filterNotNullOrNotEmpty[T](value: T): T = { + val newValue = value match { + case traversable: Traversable[_] => traversable.map(filterNotNullOrNotEmpty(_)).filter(isNotNullOrNotEmpty(_)).asInstanceOf[T] + case option: Option[_] => option.map(filterNotNullOrNotEmpty(_)).filter(isNotNullOrNotEmpty(_)).asInstanceOf[T] + case x => x + } + newValue + } + + private def isNotNullOrNotEmpty(value: Any): Boolean = { + val result = value match { + case traversable: Traversable[_] => !filterNotNullOrNotEmpty(traversable).isEmpty + case option: Option[_] => !filterNotNullOrNotEmpty(option).isEmpty + case x => x != null + } + result + } } diff --git a/scala/src/org/broadinstitute/sting/queue/util/ReflectionUtils.scala b/scala/src/org/broadinstitute/sting/queue/util/ReflectionUtils.scala index 75e4e0a89..c865f35c5 100644 --- a/scala/src/org/broadinstitute/sting/queue/util/ReflectionUtils.scala +++ b/scala/src/org/broadinstitute/sting/queue/util/ReflectionUtils.scala @@ -2,9 +2,7 @@ package org.broadinstitute.sting.queue.util import org.broadinstitute.sting.queue.QException import java.lang.annotation.Annotation -import scala.concurrent.JavaConversions import scala.concurrent.JavaConversions._ -import scala.collection.immutable.ListMap import java.lang.reflect.{ParameterizedType, Field} object ReflectionUtils { @@ -20,8 +18,7 @@ object ReflectionUtils { def filterFields(fields: List[Field], annotation: Class[_ <: Annotation]) = fields.filter(field => hasAnnotation(field, annotation)) - def getFieldNamesValues(obj: AnyRef, fields: List[Field]) = - ListMap(fields.map(field => (field.getName -> fieldGetter(field).invoke(obj))) :_*) + def getFieldValues(obj: AnyRef, fields: List[Field]) = fields.map(field => fieldGetter(field).invoke(obj)) def getAllTypes(clazz: Class[_]) = { var types = List.empty[Class[_]] @@ -65,16 +62,14 @@ object ReflectionUtils { } } - private def getCollectionType(field: Field) = { + def getCollectionType(field: Field) = { getGenericTypes(field) match { case Some(classes) => if (classes.length > 1) throw new IllegalArgumentException("Field contains more than one generic type: " + field) classes(0) case None => - if (!field.isAnnotationPresent(classOf[ClassType])) - throw new QException("@ClassType must be specified for unparameterized field: " + field) - field.getAnnotation(classOf[ClassType]).asInstanceOf[ClassType].value + throw new QException("Generic type not set for collection: " + field) } }