From e21a66d8766fc5cf484f25de172fa37cf8c484e6 Mon Sep 17 00:00:00 2001 From: kshakir Date: Tue, 30 Nov 2010 15:29:40 +0000 Subject: [PATCH] Updated the Queue GATK generator and packaging to include more dependencies for fullCallingPipeline.q. Set the -bigMemQueue in the FullCallingPipelineTest to GSA to avoid waiting for the week queue when it is busy. Fixed the package definition of PipelineTest so that scalac won't recompile it every time. git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@4755 348d0f76-0448-11de-a6fe-93d51630548a --- .../queue/extensions/gatk/ArgumentField.java | 28 ++- .../gatk/GATKExtensionsGenerator.java | 204 +++++++++++++++--- .../sting/utils/classloader/JVMUtils.java | 38 +++- packages/Queue.xml | 4 + .../pipeline/FullCallingPipelineTest.scala | 8 +- .../sting/queue/pipeline/PipelineTest.scala | 3 +- 6 files changed, 234 insertions(+), 51 deletions(-) diff --git a/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java b/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java index e5a6e86c2..056c366a0 100644 --- a/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java +++ b/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java @@ -48,15 +48,6 @@ public abstract class ArgumentField { return imports; } - /** - * Returns true if a class is built in and doesn't need to be imported. - * @param argType The class to check. - * @return true if the class is built in and doesn't need to be imported - */ - private static boolean isBuiltIn(Class argType) { - return argType.isPrimitive() || argType == String.class || Number.class.isAssignableFrom(argType); - } - /** @return Scala code defining the argument and it's annotation. */ public final String getArgumentAddition() { return String.format("%n" + @@ -136,6 +127,16 @@ public abstract class ArgumentField { return importClasses; } + /** @return Classes that should be imported by BCEL during packaging. */ + protected Collection> getDependentClasses() { + ArrayList> dependentClasses = new ArrayList>(); + Class innerType = this.getInnerType(); + if (innerType != null) + if (innerType.isEnum()) // Enums are not being implicitly picked up... + dependentClasses.add(innerType); + return dependentClasses; + } + /** @return True if this field uses @Gather. */ public boolean isGather() { return false; } @@ -146,6 +147,15 @@ public abstract class ArgumentField { return getFieldName(this.getRawFieldName()); } + /** + * Returns true if a class is built in and doesn't need to be imported. + * @param argType The class to check. + * @return true if the class is built in and doesn't need to be imported + */ + public static boolean isBuiltIn(Class argType) { + return argType.isPrimitive() || argType == String.class || Number.class.isAssignableFrom(argType); + } + /** * @param rawFieldName The raw field name * @return The field name checked against reserved words. diff --git a/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java b/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java index 039cf955f..144aad026 100644 --- a/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java +++ b/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java @@ -41,11 +41,16 @@ import org.broadinstitute.sting.gatk.io.stubs.SAMFileWriterArgumentTypeDescripto import org.broadinstitute.sting.gatk.refdata.tracks.builders.RMDTrackBuilder; import org.broadinstitute.sting.gatk.walkers.ReadWalker; import org.broadinstitute.sting.gatk.walkers.Walker; +import org.broadinstitute.sting.utils.classloader.JVMUtils; import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import java.io.File; import java.io.IOException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.lang.reflect.Type; +import java.lang.reflect.TypeVariable; import java.util.*; import java.util.Map.Entry; @@ -56,10 +61,29 @@ import java.util.Map.Entry; */ public class GATKExtensionsGenerator extends CommandLineProgram { private static final Logger logger = Logger.getLogger(GATKExtensionsGenerator.class); - public static final String GATK_EXTENSIONS_PACKAGE_NAME = "org.broadinstitute.sting.queue.extensions.gatk"; - private static final String COMMANDLINE_PACKAGE_NAME = GATK_EXTENSIONS_PACKAGE_NAME; - private static final String FILTER_PACKAGE_NAME = GATK_EXTENSIONS_PACKAGE_NAME; - private static final String WALKER_PACKAGE_NAME = GATK_EXTENSIONS_PACKAGE_NAME; + public static final String GATK_EXTENSIONS_PACKAGE_NAME = GATKExtensionsGenerator.class.getPackage().getName(); + private static final String NEWLINE = String.format("%n"); + + private static final String CLASS_TEMPLATE = "package %s%n"+ + "%s%n" + + "class %s extends %s {%n" + + "%s%s%n" + + "%soverride def commandLine = super.commandLine%s%n" + + "}%n"; + + private static final String TRAIT_TEMPLATE = "package %s%n"+ + "%s%n" + + "trait %s extends %s {%n" + + "%s%s%n" + + "%sabstract override def commandLine = super.commandLine%s%n" + + "}%n"; + + private static final String GATK_DEPENDENCIES_TEMPLATE = "package %s%n" + + "%n" + + "/** A dynamicly generated list of classes that the GATK Extensions depend on, but are not be detected by default by BCEL. */%n" + + "class %s {%n" + + "val types = List(%n%s)%n" + + "}%n"; @Output(fullName="output_directory", shortName="outDir", doc="Directory to output the generated scala", required=true) public File outputDirectory; @@ -96,12 +120,18 @@ public class GATKExtensionsGenerator extends CommandLineProgram { return typeDescriptors; } + /** + * Loops over all the walkers and filters and generates a scala class for each. + * @return zero if the run was successful, non-zero if there was an error. + */ @Override protected int execute() { try { if (!outputDirectory.isDirectory() && !outputDirectory.mkdirs()) throw new ReviewedStingException("Unable to create output directory: " + outputDirectory); + SortedSet> dependents = new TreeSet>(classComparator); + for (Class clp: clpManager.getPlugins()) { if (!isGatkProgram(clp)) @@ -109,8 +139,8 @@ public class GATKExtensionsGenerator extends CommandLineProgram { String clpClassName = clpManager.getName(clp); - writeClass("org.broadinstitute.sting.queue.function.JarCommandLineFunction", COMMANDLINE_PACKAGE_NAME, clpClassName, - false, "", ArgumentDefinitionField.getArgumentFields(parser,clp)); + writeClass("org.broadinstitute.sting.queue.function.JarCommandLineFunction", clpClassName, + false, "", ArgumentDefinitionField.getArgumentFields(parser,clp), dependents); if (clp == CommandLineGATK.class) { for (Entry>> walkersByPackage: walkerManager.getWalkerNamesByPackage(false).entrySet()) { @@ -130,8 +160,8 @@ public class GATKExtensionsGenerator extends CommandLineProgram { constructor += String.format("scatterClass = classOf[%s]%n", scatterClass); } - writeClass(COMMANDLINE_PACKAGE_NAME + "." + clpClassName, WALKER_PACKAGE_NAME, walkerName, - isScatter, constructor, argumentFields); + writeClass(GATK_EXTENSIONS_PACKAGE_NAME + "." + clpClassName, walkerName, + isScatter, constructor, argumentFields, dependents); } } } @@ -139,9 +169,11 @@ public class GATKExtensionsGenerator extends CommandLineProgram { for (Class filter: filterManager.getValues()) { String filterName = filterManager.getName(filter); - writeFilter(FILTER_PACKAGE_NAME, filterName, ArgumentDefinitionField.getArgumentFields(new ParsingEngine(null),filter)); + writeFilter(filterName, ArgumentDefinitionField.getArgumentFields(new ParsingEngine(null),filter), dependents); } + writeDependencies(dependents); + return 0; } catch (IOException exception) { logger.error("Error generating queue output.", exception); @@ -149,9 +181,18 @@ public class GATKExtensionsGenerator extends CommandLineProgram { } } + /** + * The list of packages to search through. + */ private static final List gatkPackages = Arrays.asList( "org.broadinstitute.sting.gatk", "org.broadinstitute.sting.analyzecovariates"); + + /** + * Returns true if the class is part of the GATK. + * @param clazz Class to check. + * @return True if the class is part of the GATK. + */ private boolean isGatkProgram(Class clazz) { if (clazz.getPackage() == null) return false; @@ -162,6 +203,11 @@ public class GATKExtensionsGenerator extends CommandLineProgram { return false; } + /** + * Returns the scatter type for a walker. + * @param walkerType The walker to check. + * @return The scatter type for the walker. + */ private String getScatterClass(Class walkerType) { if (ReadWalker.class.isAssignableFrom(walkerType)) return "ContigScatterFunction"; @@ -169,18 +215,102 @@ public class GATKExtensionsGenerator extends CommandLineProgram { return "IntervalScatterFunction"; } - private void writeClass(String baseClass, String packageName, String className, boolean isScatter, - String constructor, List argumentFields) throws IOException { - String content = getContent(CLASS_TEMPLATE, baseClass, packageName, className, constructor, isScatter, "", argumentFields); - writeFile(packageName + "." + className, content); + /** + * Writes a dynamically generated scala wrapper for a class. + * @param baseClass The class to extend from. + * @param className The class name to generate. + * @param isScatter True if the class is scatter/gatherable. + * @param constructor Additional logic for the constructor, or an empty string. + * @param argumentFields The list of argument fields for the generated class. + * @param dependents A set that should be updated with explicit dependencies that need to be packaged. + * @throws IOException If the file cannot be written. + */ + private void writeClass(String baseClass, String className, boolean isScatter, + String constructor, List argumentFields, Set> dependents) throws IOException { + String content = getContent(CLASS_TEMPLATE, baseClass, className, constructor, isScatter, "", argumentFields, dependents); + writeFile(GATK_EXTENSIONS_PACKAGE_NAME + "." + className, content); } - private void writeFilter(String packageName, String className, List argumentFields) throws IOException { + /** + * Writes a dynamically generated scala wrapper for a GATK filter. + * The filter is defined as a trait in scala, and can be mixed into a GATK command via "val myMixin = new PrintReads with FilterName" + * @param className The class name to generate. + * @param argumentFields The list of argument fields for the generated class. + * @param dependents A set that should be updated with explicit dependencies that need to be packaged. + * @throws IOException If the file cannot be written. + */ + private void writeFilter(String className, List argumentFields, Set> dependents) throws IOException { String content = getContent(TRAIT_TEMPLATE, "org.broadinstitute.sting.queue.function.CommandLineFunction", - packageName, className, "", false, String.format(" + \" -read_filter %s\"", className), argumentFields); - writeFile(packageName + "." + className, content); + className, "", false, String.format(" + \" -read_filter %s\"", className), argumentFields, dependents); + writeFile(GATK_EXTENSIONS_PACKAGE_NAME + "." + className, content); } + /** + * Writes the dependents to a scala wrapper that will compile and get picked up by BCEL. + * BCEL was missing some classes, such as Enums, when they were defined in the other generated classes. + * This generated wrapper makes sure they are explicitly seen by BCEL. + * @param dependents Explicit dependencies that need to be packaged. + * @throws IOException If the file cannot be written. + */ + private void writeDependencies(SortedSet> dependents) throws IOException { + // Include the enclosing classes too. Scala will be looking for them. + SortedSet> enclosings = new TreeSet>(classComparator); + for (Class dependent: dependents) + for (Class enclosing = dependent; enclosing != null; enclosing = enclosing.getEnclosingClass()) + enclosings.add(enclosing); + dependents = enclosings; + + // Oh, and include the classes defined on methods too! + enclosings = new TreeSet>(classComparator); + for (Class dependent: dependents) { + for (Method method: dependent.getDeclaredMethods()) { + JVMUtils.addGenericTypes(enclosings, method.getGenericReturnType()); + for (Type parameterType: method.getGenericParameterTypes()) + JVMUtils.addGenericTypes(enclosings, parameterType); + for (Type exceptionType: method.getGenericExceptionTypes()) + JVMUtils.addGenericTypes(enclosings, exceptionType); + } + } + dependents = enclosings; + + // Generate the dependents. + String className = "GATKClassDependencies"; + StringBuilder classes = new StringBuilder(); + + for (Class dependent: dependents) { + if (dependent.isArray()) + continue; + if (ArgumentField.isBuiltIn(dependent)) + continue; + if (!Modifier.isPublic(dependent.getModifiers())) + continue; + if (classes.length() > 0) + classes.append(",").append(NEWLINE); + String typeParams = getScalaTypeParams(dependent); + classes.append("classOf[").append(dependent.getName().replace("$", ".")).append(typeParams).append("]"); + } + String content = String.format(GATK_DEPENDENCIES_TEMPLATE, GATK_EXTENSIONS_PACKAGE_NAME, className, classes); + writeFile(GATK_EXTENSIONS_PACKAGE_NAME + "." + className, content); + } + + /** + * Returns a string representing the type parameters for a class, or an empty string if there are no type parameters. + * @param clazz The class to look for type parameters. + * @return The type parameters or an empty string. + */ + private String getScalaTypeParams(Class clazz) { + TypeVariable[] typeParams = clazz.getTypeParameters(); + if (typeParams.length == 0) + return ""; + return "[" + StringUtils.repeat("_", ",", typeParams.length) + "]"; + } + + /** + * Writes the generated scala file with this content. + * @param fullClassName Generated class name. + * @param content scala content. + * @throws IOException If the file cannot be written. + */ private void writeFile(String fullClassName, String content) throws IOException { File outputFile = new File(outputDirectory, fullClassName.replace(".", "/") + ".scala"); if (outputFile.exists()) { @@ -191,9 +321,21 @@ public class GATKExtensionsGenerator extends CommandLineProgram { FileUtils.writeStringToFile(outputFile, content); } - private static String getContent(String scalaTemplate, String baseClass, String packageName, String className, - String constructor, boolean isScatter, - String commandLinePrefix, List argumentFields) { + /** + * Generates scala content using CLASS_TEMPLATE or TRAIT_TEMPLATE. + * @param scalaTemplate CLASS_TEMPLATE or TRAIT_TEMPLATE + * @param baseClass The class to extend from. + * @param className The class name to generate. + * @param constructor Additional logic for the constructor, or an empty string. + * @param isScatter True if the class is scatter/gatherable. + * @param commandLinePrefix Additional logic to prefix to the QCommandLine.commandLine, or an empty string. + * @param argumentFields The list of argument fields for the generated class. + * @param dependents A set that should be updated with explicit dependencies that need to be packaged. + * @return The populated template. + */ + private static String getContent(String scalaTemplate, String baseClass, String className, + String constructor, boolean isScatter, String commandLinePrefix, + List argumentFields, Set> dependents) { StringBuilder arguments = new StringBuilder(); StringBuilder commandLine = new StringBuilder(commandLinePrefix); @@ -205,6 +347,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram { commandLine.append(argumentField.getCommandLineAddition()); importSet.addAll(argumentField.getImportStatements()); freezeFields.add(argumentField.getFreezeFields()); + dependents.addAll(argumentField.getDependentClasses()); isGather |= argumentField.isGather(); } @@ -231,23 +374,14 @@ public class GATKExtensionsGenerator extends CommandLineProgram { String importText = sortedImports.size() == 0 ? "" : NEWLINE + StringUtils.join(sortedImports, NEWLINE) + NEWLINE; // see CLASS_TEMPLATE and TRAIT_TEMPLATE below - return String.format(scalaTemplate, packageName, importText, + return String.format(scalaTemplate, GATK_EXTENSIONS_PACKAGE_NAME, importText, className, baseClass, constructor, arguments, freezeFieldOverride, commandLine); } - - private static final String NEWLINE = String.format("%n"); - private static final String CLASS_TEMPLATE = "package %s%n"+ - "%s%n" + - "class %s extends %s {%n" + - "%s%s%n" + - "%soverride def commandLine = super.commandLine%s%n" + - "}%n"; - - private static final String TRAIT_TEMPLATE = "package %s%n"+ - "%s%n" + - "trait %s extends %s {%n" + - "%s%s%n" + - "%sabstract override def commandLine = super.commandLine%s%n" + - "}%n"; + private static final Comparator> classComparator = new Comparator>() { + @Override + public int compare(Class a, Class b) { + return (a == null ? "" : a.getName()).compareTo(b == null ? "" : b.getName()); + } + }; } diff --git a/java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java b/java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java index 1aac9b565..4b2873108 100755 --- a/java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java +++ b/java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java @@ -26,10 +26,10 @@ package org.broadinstitute.sting.utils.classloader; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import org.broadinstitute.sting.utils.exceptions.StingException; import org.reflections.util.ClasspathHelper; -import java.lang.reflect.Modifier; -import java.lang.reflect.Field; +import java.lang.reflect.*; import java.io.File; import java.io.IOException; import java.net.URL; @@ -52,8 +52,9 @@ public class JVMUtils { /** * Determines which location contains the specified class. - * + * @param clazz The specified class. * @return Location (either jar file or directory) of path containing class. + * @throws IOException when the URI cannot be found. */ public static File getLocationFor( Class clazz ) throws IOException { try { @@ -148,6 +149,7 @@ public class JVMUtils { /** * Gets a single object in the list matching or type-compatible with the given type. Exceptions out if multiple objects match. + * @param objectsToFilter objects to filter. * @param type The desired type. * @param The selected type. * @return A collection of the given arguments with the specified type. @@ -163,11 +165,13 @@ public class JVMUtils { } /** - * Gets a collection of all objects in the list matching or type-compatible with the given type. + * Gets a collection of all objects in the list matching or type-compatible with the given type. + * @param objectsToFilter objects to filter. * @param type The desired type. * @param Again, the desired type. Used so that clients can ignore type safety. * @return A collection of the given arguments with the specified type. */ + @SuppressWarnings("unchecked") public static Collection getObjectsOfType(Collection objectsToFilter, Class type) { Collection selectedObjects = new ArrayList(); for(Object object: objectsToFilter) { @@ -184,4 +188,30 @@ public class JVMUtils { public static Set getClasspathURLs() { return ClasspathHelper.getUrlsForManifestsCurrentClasspath(); } + + /** + * Adds all the generic types from a class definition to the collection. + * Does not inspect the methods or fields, only the class. + * @param classes Set to collect the classes. + * @param type Type to inspect. + */ + public static void addGenericTypes(Set> classes, Type type) { + if (type instanceof ParameterizedType) { + ParameterizedType parameterizedType = (ParameterizedType)type; + for (Type actualType: parameterizedType.getActualTypeArguments()) + addGenericTypes(classes, actualType); + } else if (type instanceof GenericArrayType) { + addGenericTypes(classes, ((GenericArrayType)type).getGenericComponentType()); + } else if (type instanceof WildcardType) { + WildcardType wildcardType = (WildcardType)type; + for (Type upperType: wildcardType.getUpperBounds()) + addGenericTypes(classes, upperType); + for (Type lowerType: wildcardType.getLowerBounds()) + addGenericTypes(classes, lowerType); + } else if (type instanceof Class) { + classes.add((Class) type); + } else { + throw new StingException("Unknown type: " + type + " (" + type.getClass().getName() + ")"); + } + } } diff --git a/packages/Queue.xml b/packages/Queue.xml index 1d520811d..a5b0f5a88 100644 --- a/packages/Queue.xml +++ b/packages/Queue.xml @@ -14,6 +14,10 @@ + + + + diff --git a/scala/test/org/broadinstitute/sting/queue/pipeline/FullCallingPipelineTest.scala b/scala/test/org/broadinstitute/sting/queue/pipeline/FullCallingPipelineTest.scala index 4829e7a02..8df180367 100644 --- a/scala/test/org/broadinstitute/sting/queue/pipeline/FullCallingPipelineTest.scala +++ b/scala/test/org/broadinstitute/sting/queue/pipeline/FullCallingPipelineTest.scala @@ -5,7 +5,6 @@ import collection.JavaConversions._ import java.io.File import org.broadinstitute.sting.datasources.pipeline.{PipelineSample, PipelineProject, Pipeline} import org.broadinstitute.sting.utils.yaml.YamlUtils -import org.broadinstitute.sting.queue.PipelineTest import org.broadinstitute.sting.{WalkerTest, BaseTest} import java.text.SimpleDateFormat import java.util.Date @@ -67,6 +66,7 @@ class FullCallingPipelineTest extends BaseTest { dataset.pipeline = pipeline dataset.refseq = BaseTest.hg19Refseq dataset.targetTiTv = "3.0" + dataset.bigMemQueue = "gsa" dataset } @@ -90,6 +90,9 @@ class FullCallingPipelineTest extends BaseTest { if (dataset.jobQueue != null) pipelineCommand += " -jobQueue " + dataset.jobQueue + if (dataset.bigMemQueue != null) + pipelineCommand += " -bigMemQueue " + dataset.bigMemQueue + // Run the test, at least checking if the command compiles PipelineTest.executeTest(testName, pipelineCommand, null) @@ -131,7 +134,8 @@ class FullCallingPipelineTest extends BaseTest { var refseq: String = null, var targetTiTv: String = null, var validations: List[PipelineValidation] = Nil, - var jobQueue: String = null) { + var jobQueue: String = null, + var bigMemQueue: String = null) { override def toString = pipeline.getProject.getName } diff --git a/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala b/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala index d7e285d89..610e832a5 100644 --- a/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala +++ b/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala @@ -1,9 +1,10 @@ -package org.broadinstitute.sting.queue +package org.broadinstitute.sting.queue.pipeline import org.broadinstitute.sting.utils.Utils import org.testng.Assert import org.broadinstitute.sting.commandline.CommandLineProgram import org.broadinstitute.sting.queue.util.ProcessController +import org.broadinstitute.sting.queue.QCommandLine object PipelineTest { private var runningCommandLines = Set.empty[QCommandLine]