diff --git a/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java b/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java index e5a6e86c2..056c366a0 100644 --- a/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java +++ b/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java @@ -48,15 +48,6 @@ public abstract class ArgumentField { return imports; } - /** - * Returns true if a class is built in and doesn't need to be imported. - * @param argType The class to check. - * @return true if the class is built in and doesn't need to be imported - */ - private static boolean isBuiltIn(Class argType) { - return argType.isPrimitive() || argType == String.class || Number.class.isAssignableFrom(argType); - } - /** @return Scala code defining the argument and it's annotation. */ public final String getArgumentAddition() { return String.format("%n" + @@ -136,6 +127,16 @@ public abstract class ArgumentField { return importClasses; } + /** @return Classes that should be imported by BCEL during packaging. */ + protected Collection> getDependentClasses() { + ArrayList> dependentClasses = new ArrayList>(); + Class innerType = this.getInnerType(); + if (innerType != null) + if (innerType.isEnum()) // Enums are not being implicitly picked up... + dependentClasses.add(innerType); + return dependentClasses; + } + /** @return True if this field uses @Gather. */ public boolean isGather() { return false; } @@ -146,6 +147,15 @@ public abstract class ArgumentField { return getFieldName(this.getRawFieldName()); } + /** + * Returns true if a class is built in and doesn't need to be imported. + * @param argType The class to check. + * @return true if the class is built in and doesn't need to be imported + */ + public static boolean isBuiltIn(Class argType) { + return argType.isPrimitive() || argType == String.class || Number.class.isAssignableFrom(argType); + } + /** * @param rawFieldName The raw field name * @return The field name checked against reserved words. diff --git a/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java b/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java index 039cf955f..144aad026 100644 --- a/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java +++ b/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java @@ -41,11 +41,16 @@ import org.broadinstitute.sting.gatk.io.stubs.SAMFileWriterArgumentTypeDescripto import org.broadinstitute.sting.gatk.refdata.tracks.builders.RMDTrackBuilder; import org.broadinstitute.sting.gatk.walkers.ReadWalker; import org.broadinstitute.sting.gatk.walkers.Walker; +import org.broadinstitute.sting.utils.classloader.JVMUtils; import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import java.io.File; import java.io.IOException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.lang.reflect.Type; +import java.lang.reflect.TypeVariable; import java.util.*; import java.util.Map.Entry; @@ -56,10 +61,29 @@ import java.util.Map.Entry; */ public class GATKExtensionsGenerator extends CommandLineProgram { private static final Logger logger = Logger.getLogger(GATKExtensionsGenerator.class); - public static final String GATK_EXTENSIONS_PACKAGE_NAME = "org.broadinstitute.sting.queue.extensions.gatk"; - private static final String COMMANDLINE_PACKAGE_NAME = GATK_EXTENSIONS_PACKAGE_NAME; - private static final String FILTER_PACKAGE_NAME = GATK_EXTENSIONS_PACKAGE_NAME; - private static final String WALKER_PACKAGE_NAME = GATK_EXTENSIONS_PACKAGE_NAME; + public static final String GATK_EXTENSIONS_PACKAGE_NAME = GATKExtensionsGenerator.class.getPackage().getName(); + private static final String NEWLINE = String.format("%n"); + + private static final String CLASS_TEMPLATE = "package %s%n"+ + "%s%n" + + "class %s extends %s {%n" + + "%s%s%n" + + "%soverride def commandLine = super.commandLine%s%n" + + "}%n"; + + private static final String TRAIT_TEMPLATE = "package %s%n"+ + "%s%n" + + "trait %s extends %s {%n" + + "%s%s%n" + + "%sabstract override def commandLine = super.commandLine%s%n" + + "}%n"; + + private static final String GATK_DEPENDENCIES_TEMPLATE = "package %s%n" + + "%n" + + "/** A dynamicly generated list of classes that the GATK Extensions depend on, but are not be detected by default by BCEL. */%n" + + "class %s {%n" + + "val types = List(%n%s)%n" + + "}%n"; @Output(fullName="output_directory", shortName="outDir", doc="Directory to output the generated scala", required=true) public File outputDirectory; @@ -96,12 +120,18 @@ public class GATKExtensionsGenerator extends CommandLineProgram { return typeDescriptors; } + /** + * Loops over all the walkers and filters and generates a scala class for each. + * @return zero if the run was successful, non-zero if there was an error. + */ @Override protected int execute() { try { if (!outputDirectory.isDirectory() && !outputDirectory.mkdirs()) throw new ReviewedStingException("Unable to create output directory: " + outputDirectory); + SortedSet> dependents = new TreeSet>(classComparator); + for (Class clp: clpManager.getPlugins()) { if (!isGatkProgram(clp)) @@ -109,8 +139,8 @@ public class GATKExtensionsGenerator extends CommandLineProgram { String clpClassName = clpManager.getName(clp); - writeClass("org.broadinstitute.sting.queue.function.JarCommandLineFunction", COMMANDLINE_PACKAGE_NAME, clpClassName, - false, "", ArgumentDefinitionField.getArgumentFields(parser,clp)); + writeClass("org.broadinstitute.sting.queue.function.JarCommandLineFunction", clpClassName, + false, "", ArgumentDefinitionField.getArgumentFields(parser,clp), dependents); if (clp == CommandLineGATK.class) { for (Entry>> walkersByPackage: walkerManager.getWalkerNamesByPackage(false).entrySet()) { @@ -130,8 +160,8 @@ public class GATKExtensionsGenerator extends CommandLineProgram { constructor += String.format("scatterClass = classOf[%s]%n", scatterClass); } - writeClass(COMMANDLINE_PACKAGE_NAME + "." + clpClassName, WALKER_PACKAGE_NAME, walkerName, - isScatter, constructor, argumentFields); + writeClass(GATK_EXTENSIONS_PACKAGE_NAME + "." + clpClassName, walkerName, + isScatter, constructor, argumentFields, dependents); } } } @@ -139,9 +169,11 @@ public class GATKExtensionsGenerator extends CommandLineProgram { for (Class filter: filterManager.getValues()) { String filterName = filterManager.getName(filter); - writeFilter(FILTER_PACKAGE_NAME, filterName, ArgumentDefinitionField.getArgumentFields(new ParsingEngine(null),filter)); + writeFilter(filterName, ArgumentDefinitionField.getArgumentFields(new ParsingEngine(null),filter), dependents); } + writeDependencies(dependents); + return 0; } catch (IOException exception) { logger.error("Error generating queue output.", exception); @@ -149,9 +181,18 @@ public class GATKExtensionsGenerator extends CommandLineProgram { } } + /** + * The list of packages to search through. + */ private static final List gatkPackages = Arrays.asList( "org.broadinstitute.sting.gatk", "org.broadinstitute.sting.analyzecovariates"); + + /** + * Returns true if the class is part of the GATK. + * @param clazz Class to check. + * @return True if the class is part of the GATK. + */ private boolean isGatkProgram(Class clazz) { if (clazz.getPackage() == null) return false; @@ -162,6 +203,11 @@ public class GATKExtensionsGenerator extends CommandLineProgram { return false; } + /** + * Returns the scatter type for a walker. + * @param walkerType The walker to check. + * @return The scatter type for the walker. + */ private String getScatterClass(Class walkerType) { if (ReadWalker.class.isAssignableFrom(walkerType)) return "ContigScatterFunction"; @@ -169,18 +215,102 @@ public class GATKExtensionsGenerator extends CommandLineProgram { return "IntervalScatterFunction"; } - private void writeClass(String baseClass, String packageName, String className, boolean isScatter, - String constructor, List argumentFields) throws IOException { - String content = getContent(CLASS_TEMPLATE, baseClass, packageName, className, constructor, isScatter, "", argumentFields); - writeFile(packageName + "." + className, content); + /** + * Writes a dynamically generated scala wrapper for a class. + * @param baseClass The class to extend from. + * @param className The class name to generate. + * @param isScatter True if the class is scatter/gatherable. + * @param constructor Additional logic for the constructor, or an empty string. + * @param argumentFields The list of argument fields for the generated class. + * @param dependents A set that should be updated with explicit dependencies that need to be packaged. + * @throws IOException If the file cannot be written. + */ + private void writeClass(String baseClass, String className, boolean isScatter, + String constructor, List argumentFields, Set> dependents) throws IOException { + String content = getContent(CLASS_TEMPLATE, baseClass, className, constructor, isScatter, "", argumentFields, dependents); + writeFile(GATK_EXTENSIONS_PACKAGE_NAME + "." + className, content); } - private void writeFilter(String packageName, String className, List argumentFields) throws IOException { + /** + * Writes a dynamically generated scala wrapper for a GATK filter. + * The filter is defined as a trait in scala, and can be mixed into a GATK command via "val myMixin = new PrintReads with FilterName" + * @param className The class name to generate. + * @param argumentFields The list of argument fields for the generated class. + * @param dependents A set that should be updated with explicit dependencies that need to be packaged. + * @throws IOException If the file cannot be written. + */ + private void writeFilter(String className, List argumentFields, Set> dependents) throws IOException { String content = getContent(TRAIT_TEMPLATE, "org.broadinstitute.sting.queue.function.CommandLineFunction", - packageName, className, "", false, String.format(" + \" -read_filter %s\"", className), argumentFields); - writeFile(packageName + "." + className, content); + className, "", false, String.format(" + \" -read_filter %s\"", className), argumentFields, dependents); + writeFile(GATK_EXTENSIONS_PACKAGE_NAME + "." + className, content); } + /** + * Writes the dependents to a scala wrapper that will compile and get picked up by BCEL. + * BCEL was missing some classes, such as Enums, when they were defined in the other generated classes. + * This generated wrapper makes sure they are explicitly seen by BCEL. + * @param dependents Explicit dependencies that need to be packaged. + * @throws IOException If the file cannot be written. + */ + private void writeDependencies(SortedSet> dependents) throws IOException { + // Include the enclosing classes too. Scala will be looking for them. + SortedSet> enclosings = new TreeSet>(classComparator); + for (Class dependent: dependents) + for (Class enclosing = dependent; enclosing != null; enclosing = enclosing.getEnclosingClass()) + enclosings.add(enclosing); + dependents = enclosings; + + // Oh, and include the classes defined on methods too! + enclosings = new TreeSet>(classComparator); + for (Class dependent: dependents) { + for (Method method: dependent.getDeclaredMethods()) { + JVMUtils.addGenericTypes(enclosings, method.getGenericReturnType()); + for (Type parameterType: method.getGenericParameterTypes()) + JVMUtils.addGenericTypes(enclosings, parameterType); + for (Type exceptionType: method.getGenericExceptionTypes()) + JVMUtils.addGenericTypes(enclosings, exceptionType); + } + } + dependents = enclosings; + + // Generate the dependents. + String className = "GATKClassDependencies"; + StringBuilder classes = new StringBuilder(); + + for (Class dependent: dependents) { + if (dependent.isArray()) + continue; + if (ArgumentField.isBuiltIn(dependent)) + continue; + if (!Modifier.isPublic(dependent.getModifiers())) + continue; + if (classes.length() > 0) + classes.append(",").append(NEWLINE); + String typeParams = getScalaTypeParams(dependent); + classes.append("classOf[").append(dependent.getName().replace("$", ".")).append(typeParams).append("]"); + } + String content = String.format(GATK_DEPENDENCIES_TEMPLATE, GATK_EXTENSIONS_PACKAGE_NAME, className, classes); + writeFile(GATK_EXTENSIONS_PACKAGE_NAME + "." + className, content); + } + + /** + * Returns a string representing the type parameters for a class, or an empty string if there are no type parameters. + * @param clazz The class to look for type parameters. + * @return The type parameters or an empty string. + */ + private String getScalaTypeParams(Class clazz) { + TypeVariable[] typeParams = clazz.getTypeParameters(); + if (typeParams.length == 0) + return ""; + return "[" + StringUtils.repeat("_", ",", typeParams.length) + "]"; + } + + /** + * Writes the generated scala file with this content. + * @param fullClassName Generated class name. + * @param content scala content. + * @throws IOException If the file cannot be written. + */ private void writeFile(String fullClassName, String content) throws IOException { File outputFile = new File(outputDirectory, fullClassName.replace(".", "/") + ".scala"); if (outputFile.exists()) { @@ -191,9 +321,21 @@ public class GATKExtensionsGenerator extends CommandLineProgram { FileUtils.writeStringToFile(outputFile, content); } - private static String getContent(String scalaTemplate, String baseClass, String packageName, String className, - String constructor, boolean isScatter, - String commandLinePrefix, List argumentFields) { + /** + * Generates scala content using CLASS_TEMPLATE or TRAIT_TEMPLATE. + * @param scalaTemplate CLASS_TEMPLATE or TRAIT_TEMPLATE + * @param baseClass The class to extend from. + * @param className The class name to generate. + * @param constructor Additional logic for the constructor, or an empty string. + * @param isScatter True if the class is scatter/gatherable. + * @param commandLinePrefix Additional logic to prefix to the QCommandLine.commandLine, or an empty string. + * @param argumentFields The list of argument fields for the generated class. + * @param dependents A set that should be updated with explicit dependencies that need to be packaged. + * @return The populated template. + */ + private static String getContent(String scalaTemplate, String baseClass, String className, + String constructor, boolean isScatter, String commandLinePrefix, + List argumentFields, Set> dependents) { StringBuilder arguments = new StringBuilder(); StringBuilder commandLine = new StringBuilder(commandLinePrefix); @@ -205,6 +347,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram { commandLine.append(argumentField.getCommandLineAddition()); importSet.addAll(argumentField.getImportStatements()); freezeFields.add(argumentField.getFreezeFields()); + dependents.addAll(argumentField.getDependentClasses()); isGather |= argumentField.isGather(); } @@ -231,23 +374,14 @@ public class GATKExtensionsGenerator extends CommandLineProgram { String importText = sortedImports.size() == 0 ? "" : NEWLINE + StringUtils.join(sortedImports, NEWLINE) + NEWLINE; // see CLASS_TEMPLATE and TRAIT_TEMPLATE below - return String.format(scalaTemplate, packageName, importText, + return String.format(scalaTemplate, GATK_EXTENSIONS_PACKAGE_NAME, importText, className, baseClass, constructor, arguments, freezeFieldOverride, commandLine); } - - private static final String NEWLINE = String.format("%n"); - private static final String CLASS_TEMPLATE = "package %s%n"+ - "%s%n" + - "class %s extends %s {%n" + - "%s%s%n" + - "%soverride def commandLine = super.commandLine%s%n" + - "}%n"; - - private static final String TRAIT_TEMPLATE = "package %s%n"+ - "%s%n" + - "trait %s extends %s {%n" + - "%s%s%n" + - "%sabstract override def commandLine = super.commandLine%s%n" + - "}%n"; + private static final Comparator> classComparator = new Comparator>() { + @Override + public int compare(Class a, Class b) { + return (a == null ? "" : a.getName()).compareTo(b == null ? "" : b.getName()); + } + }; } diff --git a/java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java b/java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java index 1aac9b565..4b2873108 100755 --- a/java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java +++ b/java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java @@ -26,10 +26,10 @@ package org.broadinstitute.sting.utils.classloader; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import org.broadinstitute.sting.utils.exceptions.StingException; import org.reflections.util.ClasspathHelper; -import java.lang.reflect.Modifier; -import java.lang.reflect.Field; +import java.lang.reflect.*; import java.io.File; import java.io.IOException; import java.net.URL; @@ -52,8 +52,9 @@ public class JVMUtils { /** * Determines which location contains the specified class. - * + * @param clazz The specified class. * @return Location (either jar file or directory) of path containing class. + * @throws IOException when the URI cannot be found. */ public static File getLocationFor( Class clazz ) throws IOException { try { @@ -148,6 +149,7 @@ public class JVMUtils { /** * Gets a single object in the list matching or type-compatible with the given type. Exceptions out if multiple objects match. + * @param objectsToFilter objects to filter. * @param type The desired type. * @param The selected type. * @return A collection of the given arguments with the specified type. @@ -163,11 +165,13 @@ public class JVMUtils { } /** - * Gets a collection of all objects in the list matching or type-compatible with the given type. + * Gets a collection of all objects in the list matching or type-compatible with the given type. + * @param objectsToFilter objects to filter. * @param type The desired type. * @param Again, the desired type. Used so that clients can ignore type safety. * @return A collection of the given arguments with the specified type. */ + @SuppressWarnings("unchecked") public static Collection getObjectsOfType(Collection objectsToFilter, Class type) { Collection selectedObjects = new ArrayList(); for(Object object: objectsToFilter) { @@ -184,4 +188,30 @@ public class JVMUtils { public static Set getClasspathURLs() { return ClasspathHelper.getUrlsForManifestsCurrentClasspath(); } + + /** + * Adds all the generic types from a class definition to the collection. + * Does not inspect the methods or fields, only the class. + * @param classes Set to collect the classes. + * @param type Type to inspect. + */ + public static void addGenericTypes(Set> classes, Type type) { + if (type instanceof ParameterizedType) { + ParameterizedType parameterizedType = (ParameterizedType)type; + for (Type actualType: parameterizedType.getActualTypeArguments()) + addGenericTypes(classes, actualType); + } else if (type instanceof GenericArrayType) { + addGenericTypes(classes, ((GenericArrayType)type).getGenericComponentType()); + } else if (type instanceof WildcardType) { + WildcardType wildcardType = (WildcardType)type; + for (Type upperType: wildcardType.getUpperBounds()) + addGenericTypes(classes, upperType); + for (Type lowerType: wildcardType.getLowerBounds()) + addGenericTypes(classes, lowerType); + } else if (type instanceof Class) { + classes.add((Class) type); + } else { + throw new StingException("Unknown type: " + type + " (" + type.getClass().getName() + ")"); + } + } } diff --git a/packages/Queue.xml b/packages/Queue.xml index 1d520811d..a5b0f5a88 100644 --- a/packages/Queue.xml +++ b/packages/Queue.xml @@ -14,6 +14,10 @@ + + + + diff --git a/scala/test/org/broadinstitute/sting/queue/pipeline/FullCallingPipelineTest.scala b/scala/test/org/broadinstitute/sting/queue/pipeline/FullCallingPipelineTest.scala index 4829e7a02..8df180367 100644 --- a/scala/test/org/broadinstitute/sting/queue/pipeline/FullCallingPipelineTest.scala +++ b/scala/test/org/broadinstitute/sting/queue/pipeline/FullCallingPipelineTest.scala @@ -5,7 +5,6 @@ import collection.JavaConversions._ import java.io.File import org.broadinstitute.sting.datasources.pipeline.{PipelineSample, PipelineProject, Pipeline} import org.broadinstitute.sting.utils.yaml.YamlUtils -import org.broadinstitute.sting.queue.PipelineTest import org.broadinstitute.sting.{WalkerTest, BaseTest} import java.text.SimpleDateFormat import java.util.Date @@ -67,6 +66,7 @@ class FullCallingPipelineTest extends BaseTest { dataset.pipeline = pipeline dataset.refseq = BaseTest.hg19Refseq dataset.targetTiTv = "3.0" + dataset.bigMemQueue = "gsa" dataset } @@ -90,6 +90,9 @@ class FullCallingPipelineTest extends BaseTest { if (dataset.jobQueue != null) pipelineCommand += " -jobQueue " + dataset.jobQueue + if (dataset.bigMemQueue != null) + pipelineCommand += " -bigMemQueue " + dataset.bigMemQueue + // Run the test, at least checking if the command compiles PipelineTest.executeTest(testName, pipelineCommand, null) @@ -131,7 +134,8 @@ class FullCallingPipelineTest extends BaseTest { var refseq: String = null, var targetTiTv: String = null, var validations: List[PipelineValidation] = Nil, - var jobQueue: String = null) { + var jobQueue: String = null, + var bigMemQueue: String = null) { override def toString = pipeline.getProject.getName } diff --git a/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala b/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala index d7e285d89..610e832a5 100644 --- a/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala +++ b/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala @@ -1,9 +1,10 @@ -package org.broadinstitute.sting.queue +package org.broadinstitute.sting.queue.pipeline import org.broadinstitute.sting.utils.Utils import org.testng.Assert import org.broadinstitute.sting.commandline.CommandLineProgram import org.broadinstitute.sting.queue.util.ProcessController +import org.broadinstitute.sting.queue.QCommandLine object PipelineTest { private var runningCommandLines = Set.empty[QCommandLine]