zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated the Queue GATK generator and packaging to include more dependencies for fullCallingPipeline.q. 

Set the -bigMemQueue in the FullCallingPipelineTest to GSA to avoid waiting for the week queue when it is busy.
Fixed the package definition of PipelineTest so that scalac won't recompile it every time.



git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@4755 348d0f76-0448-11de-a6fe-93d51630548a
This commit is contained in:
kshakir 2010-11-30 15:29:40 +00:00
parent 7f2ded0706
commit e21a66d876
6 changed files with 234 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java
View File

@ -48,15 +48,6 @@ public abstract class ArgumentField {
return imports;
}
/**
* Returns true if a class is built in and doesn't need to be imported.
* @param argType The class to check.
* @return true if the class is built in and doesn't need to be imported
*/
private static boolean isBuiltIn(Class<?> argType) {
return argType.isPrimitive() || argType == String.class || Number.class.isAssignableFrom(argType);
}
/** @return Scala code defining the argument and it's annotation. */
public final String getArgumentAddition() {
return String.format("%n" +
@ -136,6 +127,16 @@ public abstract class ArgumentField {
return importClasses;
}
/** @return Classes that should be imported by BCEL during packaging. */
protected Collection<Class<?>> getDependentClasses() {
ArrayList<Class<?>> dependentClasses = new ArrayList<Class<?>>();
Class<?> innerType = this.getInnerType();
if (innerType != null)
if (innerType.isEnum()) // Enums are not being implicitly picked up...
dependentClasses.add(innerType);
return dependentClasses;
}
/** @return True if this field uses @Gather. */
public boolean isGather() { return false; }
@ -146,6 +147,15 @@ public abstract class ArgumentField {
return getFieldName(this.getRawFieldName());
}
/**
* Returns true if a class is built in and doesn't need to be imported.
* @param argType The class to check.
* @return true if the class is built in and doesn't need to be imported
*/
public static boolean isBuiltIn(Class<?> argType) {
return argType.isPrimitive() || argType == String.class || Number.class.isAssignableFrom(argType);
}
/**
* @param rawFieldName The raw field name
* @return The field name checked against reserved words.

204
java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java
View File

@ -41,11 +41,16 @@ import org.broadinstitute.sting.gatk.io.stubs.SAMFileWriterArgumentTypeDescripto
import org.broadinstitute.sting.gatk.refdata.tracks.builders.RMDTrackBuilder;
import org.broadinstitute.sting.gatk.walkers.ReadWalker;
import org.broadinstitute.sting.gatk.walkers.Walker;
import org.broadinstitute.sting.utils.classloader.JVMUtils;
import org.broadinstitute.sting.utils.classloader.PluginManager;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.lang.reflect.Type;
import java.lang.reflect.TypeVariable;
import java.util.*;
import java.util.Map.Entry;
@ -56,10 +61,29 @@ import java.util.Map.Entry;
*/
public class GATKExtensionsGenerator extends CommandLineProgram {
private static final Logger logger = Logger.getLogger(GATKExtensionsGenerator.class);
public static final String GATK_EXTENSIONS_PACKAGE_NAME = "org.broadinstitute.sting.queue.extensions.gatk";
private static final String COMMANDLINE_PACKAGE_NAME = GATK_EXTENSIONS_PACKAGE_NAME;
private static final String FILTER_PACKAGE_NAME = GATK_EXTENSIONS_PACKAGE_NAME;
private static final String WALKER_PACKAGE_NAME = GATK_EXTENSIONS_PACKAGE_NAME;
public static final String GATK_EXTENSIONS_PACKAGE_NAME = GATKExtensionsGenerator.class.getPackage().getName();
private static final String NEWLINE = String.format("%n");
private static final String CLASS_TEMPLATE = "package %s%n"+
"%s%n" +
"class %s extends %s {%n" +
"%s%s%n" +
"%soverride def commandLine = super.commandLine%s%n" +
"}%n";
private static final String TRAIT_TEMPLATE = "package %s%n"+
"%s%n" +
"trait %s extends %s {%n" +
"%s%s%n" +
"%sabstract override def commandLine = super.commandLine%s%n" +
"}%n";
private static final String GATK_DEPENDENCIES_TEMPLATE = "package %s%n" +
"%n" +
"/** A dynamicly generated list of classes that the GATK Extensions depend on, but are not be detected by default by BCEL. */%n" +
"class %s {%n" +
"val types = List(%n%s)%n" +
"}%n";
@Output(fullName="output_directory", shortName="outDir", doc="Directory to output the generated scala", required=true)
public File outputDirectory;
@ -96,12 +120,18 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
return typeDescriptors;
}
/**
* Loops over all the walkers and filters and generates a scala class for each.
* @return zero if the run was successful, non-zero if there was an error.
*/
@Override
protected int execute() {
try {
if (!outputDirectory.isDirectory() && !outputDirectory.mkdirs())
throw new ReviewedStingException("Unable to create output directory: " + outputDirectory);
SortedSet<Class<?>> dependents = new TreeSet<Class<?>>(classComparator);
for (Class<? extends CommandLineProgram> clp: clpManager.getPlugins()) {
if (!isGatkProgram(clp))
@ -109,8 +139,8 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
String clpClassName = clpManager.getName(clp);
writeClass("org.broadinstitute.sting.queue.function.JarCommandLineFunction", COMMANDLINE_PACKAGE_NAME, clpClassName,
false, "", ArgumentDefinitionField.getArgumentFields(parser,clp));
writeClass("org.broadinstitute.sting.queue.function.JarCommandLineFunction", clpClassName,
false, "", ArgumentDefinitionField.getArgumentFields(parser,clp), dependents);
if (clp == CommandLineGATK.class) {
for (Entry<String, Collection<Class<? extends Walker>>> walkersByPackage: walkerManager.getWalkerNamesByPackage(false).entrySet()) {
@ -130,8 +160,8 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
constructor += String.format("scatterClass = classOf[%s]%n", scatterClass);
}
writeClass(COMMANDLINE_PACKAGE_NAME + "." + clpClassName, WALKER_PACKAGE_NAME, walkerName,
isScatter, constructor, argumentFields);
writeClass(GATK_EXTENSIONS_PACKAGE_NAME + "." + clpClassName, walkerName,
isScatter, constructor, argumentFields, dependents);
}
}
}
@ -139,9 +169,11 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
for (Class<? extends SamRecordFilter> filter: filterManager.getValues()) {
String filterName = filterManager.getName(filter);
writeFilter(FILTER_PACKAGE_NAME, filterName, ArgumentDefinitionField.getArgumentFields(new ParsingEngine(null),filter));
writeFilter(filterName, ArgumentDefinitionField.getArgumentFields(new ParsingEngine(null),filter), dependents);
}
writeDependencies(dependents);
return 0;
} catch (IOException exception) {
logger.error("Error generating queue output.", exception);
@ -149,9 +181,18 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
}
}
/**
* The list of packages to search through.
*/
private static final List<String> gatkPackages = Arrays.asList(
"org.broadinstitute.sting.gatk",
"org.broadinstitute.sting.analyzecovariates");
/**
* Returns true if the class is part of the GATK.
* @param clazz Class to check.
* @return True if the class is part of the GATK.
*/
private boolean isGatkProgram(Class<?> clazz) {
if (clazz.getPackage() == null)
return false;
@ -162,6 +203,11 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
return false;
}
/**
* Returns the scatter type for a walker.
* @param walkerType The walker to check.
* @return The scatter type for the walker.
*/
private String getScatterClass(Class<? extends Walker> walkerType) {
if (ReadWalker.class.isAssignableFrom(walkerType))
return "ContigScatterFunction";
@ -169,18 +215,102 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
return "IntervalScatterFunction";
}
private void writeClass(String baseClass, String packageName, String className, boolean isScatter,
String constructor, List<? extends ArgumentField> argumentFields) throws IOException {
String content = getContent(CLASS_TEMPLATE, baseClass, packageName, className, constructor, isScatter, "", argumentFields);
writeFile(packageName + "." + className, content);
/**
* Writes a dynamically generated scala wrapper for a class.
* @param baseClass The class to extend from.
* @param className The class name to generate.
* @param isScatter True if the class is scatter/gatherable.
* @param constructor Additional logic for the constructor, or an empty string.
* @param argumentFields The list of argument fields for the generated class.
* @param dependents A set that should be updated with explicit dependencies that need to be packaged.
* @throws IOException If the file cannot be written.
*/
private void writeClass(String baseClass, String className, boolean isScatter,
String constructor, List<? extends ArgumentField> argumentFields, Set<Class<?>> dependents) throws IOException {
String content = getContent(CLASS_TEMPLATE, baseClass, className, constructor, isScatter, "", argumentFields, dependents);
writeFile(GATK_EXTENSIONS_PACKAGE_NAME + "." + className, content);
}
private void writeFilter(String packageName, String className, List<? extends ArgumentField> argumentFields) throws IOException {
/**
* Writes a dynamically generated scala wrapper for a GATK filter.
* The filter is defined as a trait in scala, and can be mixed into a GATK command via "val myMixin = new PrintReads with FilterName"
* @param className The class name to generate.
* @param argumentFields The list of argument fields for the generated class.
* @param dependents A set that should be updated with explicit dependencies that need to be packaged.
* @throws IOException If the file cannot be written.
*/
private void writeFilter(String className, List<? extends ArgumentField> argumentFields, Set<Class<?>> dependents) throws IOException {
String content = getContent(TRAIT_TEMPLATE, "org.broadinstitute.sting.queue.function.CommandLineFunction",
packageName, className, "", false, String.format(" + \" -read_filter %s\"", className), argumentFields);
writeFile(packageName + "." + className, content);
className, "", false, String.format(" + \" -read_filter %s\"", className), argumentFields, dependents);
writeFile(GATK_EXTENSIONS_PACKAGE_NAME + "." + className, content);
}
/**
* Writes the dependents to a scala wrapper that will compile and get picked up by BCEL.
* BCEL was missing some classes, such as Enums, when they were defined in the other generated classes.
* This generated wrapper makes sure they are explicitly seen by BCEL.
* @param dependents Explicit dependencies that need to be packaged.
* @throws IOException If the file cannot be written.
*/
private void writeDependencies(SortedSet<Class<?>> dependents) throws IOException {
// Include the enclosing classes too. Scala will be looking for them.
SortedSet<Class<?>> enclosings = new TreeSet<Class<?>>(classComparator);
for (Class<?> dependent: dependents)
for (Class<?> enclosing = dependent; enclosing != null; enclosing = enclosing.getEnclosingClass())
enclosings.add(enclosing);
dependents = enclosings;
// Oh, and include the classes defined on methods too!
enclosings = new TreeSet<Class<?>>(classComparator);
for (Class<?> dependent: dependents) {
for (Method method: dependent.getDeclaredMethods()) {
JVMUtils.addGenericTypes(enclosings, method.getGenericReturnType());
for (Type parameterType: method.getGenericParameterTypes())
JVMUtils.addGenericTypes(enclosings, parameterType);
for (Type exceptionType: method.getGenericExceptionTypes())
JVMUtils.addGenericTypes(enclosings, exceptionType);
}
}
dependents = enclosings;
// Generate the dependents.
String className = "GATKClassDependencies";
StringBuilder classes = new StringBuilder();
for (Class<?> dependent: dependents) {
if (dependent.isArray())
continue;
if (ArgumentField.isBuiltIn(dependent))
continue;
if (!Modifier.isPublic(dependent.getModifiers()))
continue;
if (classes.length() > 0)
classes.append(",").append(NEWLINE);
String typeParams = getScalaTypeParams(dependent);
classes.append("classOf[").append(dependent.getName().replace("$", ".")).append(typeParams).append("]");
}
String content = String.format(GATK_DEPENDENCIES_TEMPLATE, GATK_EXTENSIONS_PACKAGE_NAME, className, classes);
writeFile(GATK_EXTENSIONS_PACKAGE_NAME + "." + className, content);
}
/**
* Returns a string representing the type parameters for a class, or an empty string if there are no type parameters.
* @param clazz The class to look for type parameters.
* @return The type parameters or an empty string.
*/
private String getScalaTypeParams(Class<?> clazz) {
TypeVariable[] typeParams = clazz.getTypeParameters();
if (typeParams.length == 0)
return "";
return "[" + StringUtils.repeat("_", ",", typeParams.length) + "]";
}
/**
* Writes the generated scala file with this content.
* @param fullClassName Generated class name.
* @param content scala content.
* @throws IOException If the file cannot be written.
*/
private void writeFile(String fullClassName, String content) throws IOException {
File outputFile = new File(outputDirectory, fullClassName.replace(".", "/") + ".scala");
if (outputFile.exists()) {
@ -191,9 +321,21 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
FileUtils.writeStringToFile(outputFile, content);
}
private static String getContent(String scalaTemplate, String baseClass, String packageName, String className,
String constructor, boolean isScatter,
String commandLinePrefix, List<? extends ArgumentField> argumentFields) {
/**
* Generates scala content using CLASS_TEMPLATE or TRAIT_TEMPLATE.
* @param scalaTemplate CLASS_TEMPLATE or TRAIT_TEMPLATE
* @param baseClass The class to extend from.
* @param className The class name to generate.
* @param constructor Additional logic for the constructor, or an empty string.
* @param isScatter True if the class is scatter/gatherable.
* @param commandLinePrefix Additional logic to prefix to the QCommandLine.commandLine, or an empty string.
* @param argumentFields The list of argument fields for the generated class.
* @param dependents A set that should be updated with explicit dependencies that need to be packaged.
* @return The populated template.
*/
private static String getContent(String scalaTemplate, String baseClass, String className,
String constructor, boolean isScatter, String commandLinePrefix,
List<? extends ArgumentField> argumentFields, Set<Class<?>> dependents) {
StringBuilder arguments = new StringBuilder();
StringBuilder commandLine = new StringBuilder(commandLinePrefix);
@ -205,6 +347,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
commandLine.append(argumentField.getCommandLineAddition());
importSet.addAll(argumentField.getImportStatements());
freezeFields.add(argumentField.getFreezeFields());
dependents.addAll(argumentField.getDependentClasses());
isGather |= argumentField.isGather();
}
@ -231,23 +374,14 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
String importText = sortedImports.size() == 0 ? "" : NEWLINE + StringUtils.join(sortedImports, NEWLINE) + NEWLINE;
// see CLASS_TEMPLATE and TRAIT_TEMPLATE below
return String.format(scalaTemplate, packageName, importText,
return String.format(scalaTemplate, GATK_EXTENSIONS_PACKAGE_NAME, importText,
className, baseClass, constructor, arguments, freezeFieldOverride, commandLine);
}
private static final String NEWLINE = String.format("%n");
private static final String CLASS_TEMPLATE = "package %s%n"+
"%s%n" +
"class %s extends %s {%n" +
"%s%s%n" +
"%soverride def commandLine = super.commandLine%s%n" +
"}%n";
private static final String TRAIT_TEMPLATE = "package %s%n"+
"%s%n" +
"trait %s extends %s {%n" +
"%s%s%n" +
"%sabstract override def commandLine = super.commandLine%s%n" +
"}%n";
private static final Comparator<Class<?>> classComparator = new Comparator<Class<?>>() {
@Override
public int compare(Class<?> a, Class<?> b) {
return (a == null ? "" : a.getName()).compareTo(b == null ? "" : b.getName());
}
};
}

38
java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java
View File

@ -26,10 +26,10 @@
package org.broadinstitute.sting.utils.classloader;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import org.broadinstitute.sting.utils.exceptions.StingException;
import org.reflections.util.ClasspathHelper;
import java.lang.reflect.Modifier;
import java.lang.reflect.Field;
import java.lang.reflect.*;
import java.io.File;
import java.io.IOException;
import java.net.URL;
@ -52,8 +52,9 @@ public class JVMUtils {
/**
* Determines which location contains the specified class.
*
* @param clazz The specified class.
* @return Location (either jar file or directory) of path containing class.
* @throws IOException when the URI cannot be found.
*/
public static File getLocationFor( Class clazz ) throws IOException {
try {
@ -148,6 +149,7 @@ public class JVMUtils {
/**
* Gets a single object in the list matching or type-compatible with the given type. Exceptions out if multiple objects match.
* @param objectsToFilter objects to filter.
* @param type The desired type.
* @param <T> The selected type.
* @return A collection of the given arguments with the specified type.
@ -163,11 +165,13 @@ public class JVMUtils {
}
/**
* Gets a collection of all objects in the list matching or type-compatible with the given type.
* Gets a collection of all objects in the list matching or type-compatible with the given type.
* @param objectsToFilter objects to filter.
* @param type The desired type.
* @param <T> Again, the desired type. Used so that clients can ignore type safety.
* @return A collection of the given arguments with the specified type.
*/
@SuppressWarnings("unchecked")
public static <T> Collection<T> getObjectsOfType(Collection<Object> objectsToFilter, Class<T> type) {
Collection<T> selectedObjects = new ArrayList<T>();
for(Object object: objectsToFilter) {
@ -184,4 +188,30 @@ public class JVMUtils {
public static Set<URL> getClasspathURLs() {
return ClasspathHelper.getUrlsForManifestsCurrentClasspath();
}
/**
* Adds all the generic types from a class definition to the collection.
* Does not inspect the methods or fields, only the class.
* @param classes Set to collect the classes.
* @param type Type to inspect.
*/
public static void addGenericTypes(Set<Class<?>> classes, Type type) {
if (type instanceof ParameterizedType) {
ParameterizedType parameterizedType = (ParameterizedType)type;
for (Type actualType: parameterizedType.getActualTypeArguments())
addGenericTypes(classes, actualType);
} else if (type instanceof GenericArrayType) {
addGenericTypes(classes, ((GenericArrayType)type).getGenericComponentType());
} else if (type instanceof WildcardType) {
WildcardType wildcardType = (WildcardType)type;
for (Type upperType: wildcardType.getUpperBounds())
addGenericTypes(classes, upperType);
for (Type lowerType: wildcardType.getLowerBounds())
addGenericTypes(classes, lowerType);
} else if (type instanceof Class<?>) {
classes.add((Class<?>) type);
} else {
throw new StingException("Unknown type: " + type + " (" + type.getClass().getName() + ")");
}
}
}

4
packages/Queue.xml
View File

@ -14,6 +14,10 @@
<!-- Queue Extensions -->
<package name="org.broadinstitute.sting.queue.extensions.*" />
<!-- Pipeline + Utils -->
<package name="org.broadinstitute.sting.datasources.pipeline.**" />
<package name="org.broadinstitute.sting.utils.yaml.**" />
<!-- Scala -->
<package name="scala.**" />
</dependencies>

8
scala/test/org/broadinstitute/sting/queue/pipeline/FullCallingPipelineTest.scala
View File

@ -5,7 +5,6 @@ import collection.JavaConversions._
import java.io.File
import org.broadinstitute.sting.datasources.pipeline.{PipelineSample, PipelineProject, Pipeline}
import org.broadinstitute.sting.utils.yaml.YamlUtils
import org.broadinstitute.sting.queue.PipelineTest
import org.broadinstitute.sting.{WalkerTest, BaseTest}
import java.text.SimpleDateFormat
import java.util.Date
@ -67,6 +66,7 @@ class FullCallingPipelineTest extends BaseTest {
dataset.pipeline = pipeline
dataset.refseq = BaseTest.hg19Refseq
dataset.targetTiTv = "3.0"
dataset.bigMemQueue = "gsa"
dataset
}
@ -90,6 +90,9 @@ class FullCallingPipelineTest extends BaseTest {
if (dataset.jobQueue != null)
pipelineCommand += " -jobQueue " + dataset.jobQueue
if (dataset.bigMemQueue != null)
pipelineCommand += " -bigMemQueue " + dataset.bigMemQueue
// Run the test, at least checking if the command compiles
PipelineTest.executeTest(testName, pipelineCommand, null)
@ -131,7 +134,8 @@ class FullCallingPipelineTest extends BaseTest {
var refseq: String = null,
var targetTiTv: String = null,
var validations: List[PipelineValidation] = Nil,
var jobQueue: String = null) {
var jobQueue: String = null,
var bigMemQueue: String = null) {
override def toString = pipeline.getProject.getName
}

3
scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala
View File

@ -1,9 +1,10 @@
package org.broadinstitute.sting.queue
package org.broadinstitute.sting.queue.pipeline
import org.broadinstitute.sting.utils.Utils
import org.testng.Assert
import org.broadinstitute.sting.commandline.CommandLineProgram
import org.broadinstitute.sting.queue.util.ProcessController
import org.broadinstitute.sting.queue.QCommandLine
object PipelineTest {
private var runningCommandLines = Set.empty[QCommandLine]