zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated PluginManager so that during testing Queue can dynamically compile and load separately multiple class directories into the same class loader. 

Removed obsolete usages of PackageUtils with updated PluginManager.
Ported Queue interval utilities written in scala over to Sting's java IntervalUtils.
Added a very basic intergration test to ensure that the fullCallingPipeline.q compiles.
Added options to specify the temporary directories without having to use -Djava.io.tmpdir (useful during the above integration test).
While adding tempDir added options to specify the run directory from the command line, for example "-runDir v1".
Upgraded to scala 2.8.1 and updated calls to deprecated functions.


git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@4661 348d0f76-0448-11de-a6fe-93d51630548a
This commit is contained in:
kshakir 2010-11-12 20:14:28 +00:00
parent 42acc968b1
commit 673fa841a4
37 changed files with 1116 additions and 1048 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ivy.xml
View File

@ -41,8 +41,8 @@
<dependency org="commons-lang" name="commons-lang" rev="2.5"/>
<!-- Scala dependancies -->
<dependency org="org.scala-lang" name="scala-compiler" rev="2.8.0" conf="scala->default"/>
<dependency org="org.scala-lang" name="scala-library" rev="2.8.0" conf="scala->default"/>
<dependency org="org.scala-lang" name="scala-compiler" rev="2.8.1" conf="scala->default"/>
<dependency org="org.scala-lang" name="scala-library" rev="2.8.1" conf="scala->default"/>
<dependency org="org.apache.commons" name="commons-email" rev="1.2"/>
<!-- findbug dependencies -->

5
java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java
View File

@ -27,13 +27,14 @@ package org.broadinstitute.sting.analyzecovariates;
import org.broadinstitute.sting.commandline.Input;
import org.broadinstitute.sting.gatk.walkers.recalibration.*;
import org.broadinstitute.sting.utils.classloader.PackageUtils;
import org.broadinstitute.sting.utils.classloader.PluginManager;
import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException;
import org.broadinstitute.sting.utils.text.XReadLines;
import org.broadinstitute.sting.commandline.CommandLineProgram;
import org.broadinstitute.sting.commandline.Argument;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
@ -114,7 +115,7 @@ class AnalyzeCovariatesCLP extends CommandLineProgram {
private void initializeData() {
// Get a list of all available covariates
List<Class<? extends Covariate>> classes = PackageUtils.getClassesImplementingInterface(Covariate.class);
Collection<Class<? extends Covariate>> classes = new PluginManager<Covariate>(Covariate.class).getPlugins();
int lineNumber = 0;
boolean foundAllCovariates = false;

4
java/src/org/broadinstitute/sting/gatk/WalkerManager.java
View File

@ -62,7 +62,7 @@ public class WalkerManager extends PluginManager<Walker> {
*/
public Map<String,Collection<Class<? extends Walker>>> getWalkerNamesByPackage(boolean visibleWalkersOnly) {
Map<String,Collection<Class<? extends Walker>>> walkersByPackage = new HashMap<String,Collection<Class<? extends Walker>>>();
for(Class<? extends Walker> walker: pluginsByName.values()) {
for(Class<? extends Walker> walker: getPlugins()) {
if(visibleWalkersOnly && isHidden(walker))
continue;
@ -157,7 +157,7 @@ public class WalkerManager extends PluginManager<Walker> {
* @return Class representing the walker.
*/
public Class<? extends Walker> getWalkerClassByName(String walkerName) {
return pluginsByName.get(walkerName);
return getPluginsByName().get(walkerName);
}
/**

2
java/src/org/broadinstitute/sting/gatk/filters/FilterManager.java
View File

@ -55,6 +55,6 @@ public class FilterManager extends PluginManager<SamRecordFilter> {
}
public Collection<Class<? extends SamRecordFilter>> getValues() {
return this.pluginsByName.values();
return this.getPlugins();
}
}

9
java/src/org/broadinstitute/sting/gatk/refdata/tracks/builders/RMDTrackBuilder.java
View File

@ -32,7 +32,6 @@ import org.broad.tribble.*;
import org.broad.tribble.index.Index;
import org.broad.tribble.index.IndexFactory;
import org.broad.tribble.source.BasicFeatureSource;
import org.broad.tribble.source.CachingFeatureSource;
import org.broad.tribble.util.LittleEndianOutputStream;
import org.broadinstitute.sting.gatk.arguments.GATKArgumentCollection;
import org.broadinstitute.sting.gatk.refdata.ReferenceDependentFeatureCodec;
@ -42,7 +41,6 @@ import org.broadinstitute.sting.gatk.refdata.utils.RMDTriplet;
import org.broadinstitute.sting.gatk.AbstractGenomeAnalysisEngine;
import org.broadinstitute.sting.gatk.refdata.utils.helpers.DbSNPHelper;
import org.broadinstitute.sting.utils.GenomeLocParser;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.collections.Pair;
import org.broadinstitute.sting.utils.classloader.PluginManager;
import org.broadinstitute.sting.utils.exceptions.UserException;
@ -114,8 +112,8 @@ public class RMDTrackBuilder extends PluginManager<FeatureCodec> {
/** @return a list of all available track types we currently have access to create */
public Map<String, Class> getAvailableTrackNamesAndTypes() {
classes = new HashMap<String, Class>();
for (String name: this.pluginsByName.keySet()) {
classes.put(name.toUpperCase(), pluginsByName.get(name));
for (String name: this.getPluginsByName().keySet()) {
classes.put(name.toUpperCase(), getPluginsByName().get(name));
}
return classes;
}
@ -123,7 +121,7 @@ public class RMDTrackBuilder extends PluginManager<FeatureCodec> {
/** @return a list of all available track record types we currently have access to create */
public Map<String, Class> getAvailableTrackNamesAndRecordTypes() {
HashMap classToRecord = new HashMap<String, Class>();
for (String name: this.pluginsByName.keySet()) {
for (String name: this.getPluginsByName().keySet()) {
FeatureCodec codec = this.createByName(name);
classToRecord.put(name, codec.getFeatureType());
}
@ -133,7 +131,6 @@ public class RMDTrackBuilder extends PluginManager<FeatureCodec> {
/**
* create a RMDTrack of the specified type
*
* @param genomeLocParser GenomeLocParser to use, if case track needs additional reference context.
* @param targetClass the target class of track
* @param name what to call the track
* @param inputFile the input file

8
java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java
View File

@ -38,7 +38,7 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotationType
import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.GenotypeAnnotation;
import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
import org.broadinstitute.sting.utils.BaseUtils;
import org.broadinstitute.sting.utils.classloader.PackageUtils;
import org.broadinstitute.sting.utils.classloader.PluginManager;
import org.broadinstitute.sting.utils.SampleUtils;
import org.broadinstitute.sting.commandline.Argument;
import org.broadinstitute.sting.commandline.Output;
@ -93,18 +93,18 @@ public class VariantAnnotator extends RodWalker<Integer, Integer> {
private void listAnnotationsAndExit() {
List<Class<? extends InfoFieldAnnotation>> infoAnnotationClasses = PackageUtils.getClassesImplementingInterface(InfoFieldAnnotation.class);
List<Class<? extends InfoFieldAnnotation>> infoAnnotationClasses = new PluginManager<InfoFieldAnnotation>(InfoFieldAnnotation.class).getPlugins();
System.out.println("\nAvailable annotations for the VCF INFO field:");
for (int i = 0; i < infoAnnotationClasses.size(); i++)
System.out.println("\t" + infoAnnotationClasses.get(i).getSimpleName());
System.out.println();
List<Class<? extends GenotypeAnnotation>> genotypeAnnotationClasses = PackageUtils.getClassesImplementingInterface(GenotypeAnnotation.class);
List<Class<? extends GenotypeAnnotation>> genotypeAnnotationClasses = new PluginManager<GenotypeAnnotation>(GenotypeAnnotation.class).getPlugins();
System.out.println("\nAvailable annotations for the VCF FORMAT field:");
for (int i = 0; i < genotypeAnnotationClasses.size(); i++)
System.out.println("\t" + genotypeAnnotationClasses.get(i).getSimpleName());
System.out.println();
System.out.println("\nAvailable classes/groups of annotations:");
for ( Class c : PackageUtils.getInterfacesExtendingInterface(AnnotationType.class) )
for ( Class c : new PluginManager<AnnotationType>(AnnotationType.class).getInterfaces() )
System.out.println("\t" + c.getSimpleName());
System.out.println();
System.exit(0);

51
java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java
View File

@ -50,7 +50,6 @@ import org.broadinstitute.sting.gatk.refdata.utils.helpers.DbSNPHelper;
import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.*;
import org.broadinstitute.sting.gatk.walkers.annotator.genomicannotator.*;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import org.broadinstitute.sting.utils.classloader.PackageUtils;
import org.broadinstitute.sting.utils.exceptions.UserException;
@ -94,8 +93,8 @@ public class VariantAnnotatorEngine {
// use this constructor if you want all possible annotations
public VariantAnnotatorEngine(GenomeAnalysisEngine engine) {
requestedInfoAnnotations = PackageUtils.getInstancesOfClassesImplementingInterface(InfoFieldAnnotation.class);
requestedGenotypeAnnotations = PackageUtils.getInstancesOfClassesImplementingInterface(GenotypeAnnotation.class);
requestedInfoAnnotations = AnnotationInterfaceManager.createAllInfoFieldAnnotations();
requestedGenotypeAnnotations = AnnotationInterfaceManager.createAllGenotypeAnnotations();
initializeDBs(engine);
}
@ -116,49 +115,9 @@ public class VariantAnnotatorEngine {
}
private void initializeAnnotations(List<String> annotationGroupsToUse, List<String> annotationsToUse) {
// create a map for all annotation classes which implement our top-level interfaces
HashMap<String, Class> classMap = new HashMap<String, Class>();
for ( Class c : PackageUtils.getClassesImplementingInterface(InfoFieldAnnotation.class) )
classMap.put(c.getSimpleName(), c);
for ( Class c : PackageUtils.getClassesImplementingInterface(GenotypeAnnotation.class) )
classMap.put(c.getSimpleName(), c);
for ( Class c : PackageUtils.getInterfacesExtendingInterface(AnnotationType.class) )
classMap.put(c.getSimpleName(), c);
HashSet<Class> classes = new HashSet<Class>();
// get the classes from the provided groups (interfaces)
if ( annotationGroupsToUse.size() != 1 || ! annotationGroupsToUse.get(0).toLowerCase().equals("none") ) {
for ( String group : annotationGroupsToUse ) {
Class interfaceClass = classMap.get(group);
if ( interfaceClass == null )
interfaceClass = classMap.get(group + "Annotation");
if ( interfaceClass == null )
throw new UserException.BadArgumentValue("group", "Class " + group + " is not found; please check that you have specified the class name correctly");
classes.addAll(PackageUtils.getClassesImplementingInterface(interfaceClass));
}
}
// get the specific classes provided
for ( String annotation : annotationsToUse ) {
Class annotationClass = classMap.get(annotation);
if ( annotationClass == null )
annotationClass = classMap.get(annotation + "Annotation");
if ( annotationClass == null )
throw new UserException.BadArgumentValue("annotation", "Class " + annotation + " is not found; please check that you have specified the class name correctly");
classes.add(annotationClass);
}
// get the instances
requestedInfoAnnotations = new ArrayList<InfoFieldAnnotation>();
requestedGenotypeAnnotations = new ArrayList<GenotypeAnnotation>();
for ( Class c : classes ) {
// note that technically an annotation can work on both the INFO and FORMAT fields
if ( InfoFieldAnnotation.class.isAssignableFrom(c) )
requestedInfoAnnotations.add((InfoFieldAnnotation)PackageUtils.getSimpleInstance(c));
if ( GenotypeAnnotation.class.isAssignableFrom(c) )
requestedGenotypeAnnotations.add((GenotypeAnnotation)PackageUtils.getSimpleInstance(c));
}
AnnotationInterfaceManager.validateAnnotations(annotationGroupsToUse, annotationsToUse);
requestedInfoAnnotations = AnnotationInterfaceManager.createInfoFieldAnnotations(annotationGroupsToUse, annotationsToUse);
requestedGenotypeAnnotations = AnnotationInterfaceManager.createGenotypeAnnotations(annotationGroupsToUse, annotationsToUse);
}
private void initializeDBs(GenomeAnalysisEngine engine) {

100
java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/AnnotationInterfaceManager.java 100644
View File

@ -0,0 +1,100 @@
package org.broadinstitute.sting.gatk.walkers.annotator.interfaces;
import org.broadinstitute.sting.utils.classloader.PluginManager;
import org.broadinstitute.sting.utils.exceptions.UserException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
public class AnnotationInterfaceManager {
private static PluginManager<InfoFieldAnnotation> infoFieldAnnotationPluginManager = new PluginManager<InfoFieldAnnotation>(InfoFieldAnnotation.class);
private static PluginManager<GenotypeAnnotation> genotypeAnnotationPluginManager = new PluginManager<GenotypeAnnotation>(GenotypeAnnotation.class);
private static PluginManager<AnnotationType> annotationTypePluginManager = new PluginManager<AnnotationType>(AnnotationType.class);
public static List<InfoFieldAnnotation> createAllInfoFieldAnnotations() {
return infoFieldAnnotationPluginManager.createAllTypes();
}
public static List<GenotypeAnnotation> createAllGenotypeAnnotations() {
return genotypeAnnotationPluginManager.createAllTypes();
}
public static void validateAnnotations(List<String> annotationGroupsToUse, List<String> annotationsToUse) {
HashMap<String, Class> classMap = new HashMap<String, Class>();
for ( Class c : infoFieldAnnotationPluginManager.getPlugins() )
classMap.put(c.getSimpleName(), c);
for ( Class c : genotypeAnnotationPluginManager.getPlugins() )
classMap.put(c.getSimpleName(), c);
for ( Class c : annotationTypePluginManager.getInterfaces() )
classMap.put(c.getSimpleName(), c);
if ( annotationGroupsToUse.size() != 1 || !"none".equals(annotationGroupsToUse.get(0)) ) {
for ( String group : annotationGroupsToUse ) {
Class interfaceClass = classMap.get(group);
if ( interfaceClass == null )
interfaceClass = classMap.get(group + "Annotation");
if ( interfaceClass == null )
throw new UserException.BadArgumentValue("group", "Class " + group + " is not found; please check that you have specified the class name correctly");
}
}
// validate the specific classes provided
for ( String annotation : annotationsToUse ) {
Class annotationClass = classMap.get(annotation);
if ( annotationClass == null )
annotationClass = classMap.get(annotation + "Annotation");
if ( annotationClass == null )
throw new UserException.BadArgumentValue("annotation", "Class " + annotation + " is not found; please check that you have specified the class name correctly");
}
}
public static List<InfoFieldAnnotation> createInfoFieldAnnotations(List<String> annotationGroupsToUse, List<String> annotationsToUse) {
return createAnnotations(infoFieldAnnotationPluginManager, annotationGroupsToUse, annotationsToUse);
}
public static List<GenotypeAnnotation> createGenotypeAnnotations(List<String> annotationGroupsToUse, List<String> annotationsToUse) {
return createAnnotations(genotypeAnnotationPluginManager, annotationGroupsToUse, annotationsToUse);
}
private static <T> List<T> createAnnotations(PluginManager<T> pluginManager, List<String> annotationGroupsToUse, List<String> annotationsToUse) {
// get the instances
List<T> annotations = new ArrayList<T>();
// get the classes from the provided groups (interfaces)
// create a map for all annotation classes which implement our top-level interfaces
HashMap<String, Class> classMap = new HashMap<String, Class>();
for ( Class c : pluginManager.getPlugins() )
classMap.put(c.getSimpleName(), c);
for ( Class c : annotationTypePluginManager.getInterfaces() )
classMap.put(c.getSimpleName(), c);
HashSet<Class> classes = new HashSet<Class>();
if ( annotationGroupsToUse.size() != 1 || !"none".equals(annotationGroupsToUse.get(0)) ) {
for ( String group : annotationGroupsToUse ) {
Class interfaceClass = classMap.get(group);
if ( interfaceClass == null )
interfaceClass = classMap.get(group + "Annotation");
if ( interfaceClass != null )
classes.addAll(pluginManager.getPluginsImplementing(interfaceClass));
}
}
// get the specific classes provided
for ( String annotation : annotationsToUse ) {
Class annotationClass = classMap.get(annotation);
if ( annotationClass == null )
annotationClass = classMap.get(annotation + "Annotation");
if ( annotationClass != null )
classes.add(annotationClass);
}
// note that technically an annotation can work on both the INFO and FORMAT fields
for ( Class c : classes )
annotations.add(pluginManager.createByType(c));
return annotations;
}
}

8
java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CovariateCounterWalker.java
View File

@ -37,7 +37,7 @@ import org.broadinstitute.sting.gatk.filters.ZeroMappingQualityReadFilter;
import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
import org.broadinstitute.sting.gatk.walkers.*;
import org.broadinstitute.sting.utils.*;
import org.broadinstitute.sting.utils.classloader.PackageUtils;
import org.broadinstitute.sting.utils.classloader.PluginManager;
import org.broadinstitute.sting.utils.collections.NestedHashMap;
import org.broadinstitute.sting.commandline.Argument;
import org.broadinstitute.sting.commandline.ArgumentCollection;
@ -167,9 +167,9 @@ public class CovariateCounterWalker extends LocusWalker<CovariateCounterWalker.C
if( RAC.FORCE_PLATFORM != null ) { RAC.DEFAULT_PLATFORM = RAC.FORCE_PLATFORM; }
// Get a list of all available covariates
final List<Class<? extends Covariate>> covariateClasses = PackageUtils.getClassesImplementingInterface( Covariate.class );
final List<Class<? extends RequiredCovariate>> requiredClasses = PackageUtils.getClassesImplementingInterface( RequiredCovariate.class );
final List<Class<? extends StandardCovariate>> standardClasses = PackageUtils.getClassesImplementingInterface( StandardCovariate.class );
final List<Class<? extends Covariate>> covariateClasses = new PluginManager<Covariate>( Covariate.class ).getPlugins();
final List<Class<? extends RequiredCovariate>> requiredClasses = new PluginManager<RequiredCovariate>( RequiredCovariate.class ).getPlugins();
final List<Class<? extends StandardCovariate>> standardClasses = new PluginManager<StandardCovariate>( StandardCovariate.class ).getPlugins();
// Print and exit if that's what was requested
if ( LIST_ONLY ) {

4
java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java
View File

@ -42,7 +42,7 @@ import org.broadinstitute.sting.gatk.walkers.ReadWalker;
import org.broadinstitute.sting.gatk.walkers.Requires;
import org.broadinstitute.sting.gatk.walkers.WalkerName;
import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
import org.broadinstitute.sting.utils.classloader.PackageUtils;
import org.broadinstitute.sting.utils.classloader.PluginManager;
import org.broadinstitute.sting.utils.collections.NestedHashMap;
import org.broadinstitute.sting.utils.QualityUtils;
import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException;
@ -149,7 +149,7 @@ public class TableRecalibrationWalker extends ReadWalker<SAMRecord, SAMFileWrite
if( RAC.FORCE_PLATFORM != null ) { RAC.DEFAULT_PLATFORM = RAC.FORCE_PLATFORM; }
// Get a list of all available covariates
final List<Class<? extends Covariate>> classes = PackageUtils.getClassesImplementingInterface(Covariate.class);
final List<Class<? extends Covariate>> classes = new PluginManager<Covariate>(Covariate.class).getPlugins();
int lineNumber = 0;
boolean foundAllCovariates = false;

8
java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java
View File

@ -49,7 +49,7 @@ import org.broadinstitute.sting.utils.report.VE2ReportFactory;
import org.broadinstitute.sting.utils.report.templates.ReportFormat;
import org.broadinstitute.sting.utils.report.utils.Node;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import org.broadinstitute.sting.utils.classloader.PackageUtils;
import org.broadinstitute.sting.utils.classloader.PluginManager;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.commandline.Argument;
import org.broadinstitute.sting.commandline.Output;
@ -351,7 +351,7 @@ public class VariantEvalWalker extends RodWalker<Integer, Integer> implements Tr
}
private void listModulesAndExit() {
List<Class<? extends VariantEvaluator>> veClasses = PackageUtils.getClassesImplementingInterface(VariantEvaluator.class);
List<Class<? extends VariantEvaluator>> veClasses = new PluginManager<VariantEvaluator>( VariantEvaluator.class ).getPlugins();
out.println("\nAvailable eval modules:");
out.println("(Standard modules are starred)");
for (Class<? extends VariantEvaluator> veClass : veClasses)
@ -399,14 +399,14 @@ public class VariantEvalWalker extends RodWalker<Integer, Integer> implements Tr
private void determineEvalations() {
// create a map for all eval modules for easy lookup
HashMap<String, Class<? extends VariantEvaluator>> classMap = new HashMap<String, Class<? extends VariantEvaluator>>();
for ( Class<? extends VariantEvaluator> c : PackageUtils.getClassesImplementingInterface(VariantEvaluator.class) )
for ( Class<? extends VariantEvaluator> c : new PluginManager<VariantEvaluator>( VariantEvaluator.class ).getPlugins() )
classMap.put(c.getSimpleName(), c);
evaluationClasses = new HashSet<Class<? extends VariantEvaluator>>();
// by default, use standard eval modules
if ( !NO_STANDARD ) {
for ( Class<? extends StandardEval> myClass : PackageUtils.getClassesImplementingInterface(StandardEval.class) ) {
for ( Class<? extends StandardEval> myClass : new PluginManager<StandardEval>( StandardEval.class ).getPlugins() ) {
if ( classMap.containsKey(myClass.getSimpleName()) )
evaluationClasses.add(classMap.get(myClass.getSimpleName()));
}

56
java/src/org/broadinstitute/sting/oneoffprojects/walkers/AnnotationByAlleleFrequencyWalker.java
View File

@ -31,13 +31,11 @@ import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
import org.broadinstitute.sting.gatk.walkers.RodWalker;
import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotationType;
import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotationInterfaceManager;
import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.GenotypeAnnotation;
import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
import org.broadinstitute.sting.utils.GenomeLoc;
import org.broadinstitute.sting.utils.classloader.PackageUtils;
import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException;
import org.broadinstitute.sting.utils.exceptions.UserException;
import java.util.*;
@ -63,8 +61,8 @@ public class AnnotationByAlleleFrequencyWalker extends RodWalker<Integer, Integ
@Argument(fullName="group", shortName="G", doc="One or more classes/groups of annotations to apply to variant calls", required=false)
protected String[] annotationClassesToUse = { };
private ArrayList<InfoFieldAnnotation> requestedInfoAnnotations;
private ArrayList<GenotypeAnnotation> requestedGenotypeAnnotations;
private List<InfoFieldAnnotation> requestedInfoAnnotations;
private List<GenotypeAnnotation> requestedGenotypeAnnotations;
//---------------------------------------------------------------------------------------------------------------
@ -74,49 +72,11 @@ public class AnnotationByAlleleFrequencyWalker extends RodWalker<Integer, Integ
//---------------------------------------------------------------------------------------------------------------
public void initialize() {
// create a map for all annotation classes which implement our top-level interfaces
HashMap<String, Class> classMap = new HashMap<String, Class>();
for ( Class c : PackageUtils.getClassesImplementingInterface(InfoFieldAnnotation.class) )
classMap.put(c.getSimpleName(), c);
for ( Class c : PackageUtils.getClassesImplementingInterface(GenotypeAnnotation.class) )
classMap.put(c.getSimpleName(), c);
for ( Class c : PackageUtils.getInterfacesExtendingInterface(AnnotationType.class) )
classMap.put(c.getSimpleName(), c);
HashSet<Class> classes = new HashSet<Class>();
// get the classes from the provided groups (interfaces)
for ( String group : annotationClassesToUse ) {
Class interfaceClass = classMap.get(group);
if ( interfaceClass == null )
interfaceClass = classMap.get(group + "Annotation");
if ( interfaceClass == null )
throw new UserException.BadArgumentValue("group", "Class " + group + " is not found; please check that you have specified the class name correctly");
classes.addAll(PackageUtils.getClassesImplementingInterface(interfaceClass));
}
// get the specific classes provided
for ( String annotation : annotationsToUse ) {
Class annotationClass = classMap.get(annotation);
if ( annotationClass == null )
annotationClass = classMap.get(annotation + "Annotation");
if ( annotationClass == null )
throw new UserException.BadArgumentValue("annotation", "Class " + annotation + " is not found; please check that you have specified the class name correctly");
classes.add(annotationClass);
}
// get the instances
requestedInfoAnnotations = new ArrayList<InfoFieldAnnotation>();
requestedGenotypeAnnotations = new ArrayList<GenotypeAnnotation>();
for ( Class c : classes ) {
// note that technically an annotation can work on both the INFO and FORMAT fields
if ( InfoFieldAnnotation.class.isAssignableFrom(c) )
requestedInfoAnnotations.add((InfoFieldAnnotation)getInstance(c));
if ( GenotypeAnnotation.class.isAssignableFrom(c) )
requestedGenotypeAnnotations.add((GenotypeAnnotation)getInstance(c));
}
List<String> annotationClasses = Arrays.asList(annotationClassesToUse);
List<String> annotations = Arrays.asList(annotationsToUse);
AnnotationInterfaceManager.validateAnnotations(annotationClasses, annotations);
requestedInfoAnnotations = AnnotationInterfaceManager.createInfoFieldAnnotations(annotationClasses, annotations);
requestedGenotypeAnnotations = AnnotationInterfaceManager.createGenotypeAnnotations(annotationClasses, annotations);
}
private static <T> ArrayList<T> getInstances(List<Class<? extends T>> classes) {

43
java/src/org/broadinstitute/sting/queue/extensions/gatk/CommandLineProgramManager.java
View File

@ -1,43 +0,0 @@
/*
* Copyright (c) 2010, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.extensions.gatk;
import org.broadinstitute.sting.commandline.CommandLineProgram;
import org.broadinstitute.sting.utils.classloader.PluginManager;
import java.util.Collection;
/**
* Finds all command line programs.
*/
public class CommandLineProgramManager extends PluginManager<CommandLineProgram> {
public CommandLineProgramManager() {
super(CommandLineProgram.class, "CommandLineProgram", "CLP");
}
public Collection<Class<? extends CommandLineProgram>> getValues() {
return this.pluginsByName.values();
}
}

5
java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java
View File

@ -40,6 +40,7 @@ import org.broadinstitute.sting.gatk.io.stubs.SAMFileWriterArgumentTypeDescripto
import org.broadinstitute.sting.gatk.refdata.tracks.builders.RMDTrackBuilder;
import org.broadinstitute.sting.gatk.walkers.ReadWalker;
import org.broadinstitute.sting.gatk.walkers.Walker;
import org.broadinstitute.sting.utils.classloader.PluginManager;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import java.io.File;
@ -62,7 +63,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
@Output(fullName="output_directory", shortName="outDir", doc="Directory to output the generated scala", required=true)
public File outputDirectory;
CommandLineProgramManager clpManager = new CommandLineProgramManager();
PluginManager<CommandLineProgram> clpManager = new PluginManager<CommandLineProgram>(CommandLineProgram.class, "CommandLineProgram", "CLP");
GenomeAnalysisEngine GATKEngine = new GenomeAnalysisEngine();
WalkerManager walkerManager = new WalkerManager();
FilterManager filterManager = new FilterManager();
@ -97,7 +98,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
if (!outputDirectory.isDirectory() && !outputDirectory.mkdirs())
throw new ReviewedStingException("Unable to create output directory: " + outputDirectory);
for (Class<? extends CommandLineProgram> clp: clpManager.getValues()) {
for (Class<? extends CommandLineProgram> clp: clpManager.getPlugins()) {
if (!isGatkProgram(clp))
continue;

66
java/src/org/broadinstitute/sting/utils/Utils.java
View File

@ -312,6 +312,44 @@ public class Utils {
return new Pair<Double, Integer>(max, index);
}
/**
* Splits expressions in command args by spaces and returns the array of expressions.
* Expressions may use single or double quotes to group any individual expression, but not both.
* @param args Arguments to parse.
* @return Parsed expressions.
*/
public static String[] escapeExpressions(String args) {
// special case for ' and " so we can allow expressions
if (args.indexOf('\'') != -1)
return escapeExpressions(args, "'");
else if (args.indexOf('\"') != -1)
return escapeExpressions(args, "\"");
else
return args.split(" ");
}
/**
* Splits expressions in command args by spaces and the supplied delimiter and returns the array of expressions.
* @param args Arguments to parse.
* @param delimiter Delimiter for grouping expressions.
* @return Parsed expressions.
*/
private static String[] escapeExpressions(String args, String delimiter) {
String[] command = {};
String[] split = args.split(delimiter);
for (int i = 0; i < split.length - 1; i += 2) {
command = Utils.concatArrays(command, split[i].trim().split(" "));
command = Utils.concatArrays(command, new String[]{split[i + 1]});
}
return Utils.concatArrays(command, split[split.length - 1].trim().split(" "));
}
/**
* Concatenates two String arrays.
* @param A First array.
* @param B Second array.
* @return Concatenation of A then B.
*/
public static String[] concatArrays(String[] A, String[] B) {
String[] C = new String[A.length + B.length];
System.arraycopy(A, 0, C, 0, A.length);
@ -319,8 +357,22 @@ public class Utils {
return C;
}
/**
* Appends String(s) B to array A.
* @param A First array.
* @param B Strings to append.
* @return A with B(s) appended.
*/
public static String[] appendArray(String[] A, String... B) {
return concatArrays(A, B);
}
/** Returns indices of all occurrences of the specified symbol in the string */
/**
* Returns indices of all occurrences of the specified symbol in the string
* @param s Search string
* @param ch Character to search for
* @return Indices of all occurrences of the specified symbol
*/
public static int[] indexOfAll(String s, int ch) {
int[] pos = new int[64];
int z = 0;
@ -339,10 +391,10 @@ public class Utils {
* with zeros up to the new size. Finally, if new size is the same as original size, no memory reallocation
* will be performed and the original array will be returned instead.
*
* @param orig
* @param newSize
* @param orig Original size.
* @param newSize New Size.
*
* @return
* @return New array with length equal to newSize.
*/
public static int[] reallocate(int[] orig, int newSize) {
if (orig.length == newSize) return orig;
@ -360,7 +412,8 @@ public class Utils {
* array a.
* @param a original array
* @param n number of (v-filled) elements to append to a on the right (n>0) or on the left (n<0)
* @return
* @param v element value
* @return the extended copy of array a with additional n elements
*/
public static byte [] extend(final byte[] a, int n, byte v) {
@ -396,7 +449,8 @@ public class Utils {
* array a.
* @param a original array
* @param n number of (v-filled) elements to append to a on the right (n>0) or on the left (n<0)
* @return
* @param v element value
* @return the extended copy of array a with additional n elements
*/
public static short [] extend(final short[] a, int n, short v) {

14
java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java
View File

@ -26,15 +26,14 @@
package org.broadinstitute.sting.utils.classloader;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import org.reflections.util.ClasspathHelper;
import java.lang.reflect.Modifier;
import java.lang.reflect.Field;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.ArrayList;
import java.util.Arrays;
import java.net.URL;
import java.util.*;
/**
* Created by IntelliJ IDEA.
@ -178,4 +177,11 @@ public class JVMUtils {
return selectedObjects;
}
/**
* Returns the list of class path urls.
* @return the list of class path urls.
*/
public static Set<URL> getClasspathURLs() {
return ClasspathHelper.getUrlsForManifestsCurrentClasspath();
}
}

159
java/src/org/broadinstitute/sting/utils/classloader/PackageUtils.java
View File

@ -1,159 +0,0 @@
/*
* Copyright (c) 2010 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
* THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.utils.classloader;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.Logger;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException;
import org.reflections.Reflections;
import org.reflections.scanners.SubTypesScanner;
import org.reflections.util.ClasspathHelper;
import org.reflections.util.ConfigurationBuilder;
import org.slf4j.LoggerFactory;
import java.lang.reflect.Method;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.Set;
import java.util.ArrayList;
import java.util.List;
/**
* PackageUtils contains some useful methods for package introspection.
*/
public class PackageUtils {
/**
* A reference into our introspection utility.
*/
private static Reflections reflections = null;
static {
// turn off logging in the reflections library - they talk too much (to the wrong logger factory as well, logback)
Logger logger = (ch.qos.logback.classic.Logger)LoggerFactory.getLogger(Reflections.class);
logger.setLevel(Level.OFF);
}
/**
* Private constructor. No instantiating this class!
*/
private PackageUtils() {}
/**
* Return the classes that implement the specified interface.
*
* @param iface the interface which returned classes should implement.
* @return the list of classes that implement the interface.
*/
public static <T> List<Class<? extends T>> getClassesImplementingInterface(Class<T> iface) {
// Load all classes implementing the given interface, then filter out any class that isn't concrete.
initReflections();
Set<Class<? extends T>> allTypes = reflections.getSubTypesOf(iface);
List<Class<? extends T>> concreteTypes = new ArrayList<Class<? extends T>>();
for( Class<? extends T> type: allTypes ) {
if( JVMUtils.isConcrete(type) )
concreteTypes.add(type);
}
return concreteTypes;
}
public static <T> List<T> getInstancesOfClassesImplementingInterface(Class<T> iface) {
List<Class<? extends T>> classes = PackageUtils.getClassesImplementingInterface(iface);
List<T> instances = new ArrayList<T>();
for ( Class<? extends T> c : classes )
instances.add(getSimpleInstance(c));
return instances;
}
public static <T> T getSimpleInstance(Class<T> c) {
try {
return c.newInstance();
} catch (Exception e) {
throw new DynamicClassResolutionException(c, e);
}
}
/**
* Return the interface classes that extend the specified interface.
*
* @param iface the interface which returned classes should extend.
* @return the list of interface classes that implement the interface.
*/
public static <T> List<Class<? extends T>> getInterfacesExtendingInterface(Class<T> iface) {
// Load all classes extending the given interface, then filter out any class that is concrete.
initReflections();
Set<Class<? extends T>> allTypes = reflections.getSubTypesOf(iface);
List<Class<? extends T>> nonConcreteTypes = new ArrayList<Class<? extends T>>();
for( Class<? extends T> type: allTypes ) {
if( !JVMUtils.isConcrete(type) )
nonConcreteTypes.add(type);
}
return nonConcreteTypes;
}
public static Set<URL> getClassPathURLs() {
return ClasspathHelper.getUrlsForManifestsCurrentClasspath();
}
/**
* Adds the URL to the system class loader classpath using reflection.
* HACK: Uses reflection to modify the class path, and assumes loader is a URLClassLoader.
* @param url URL to add to the system class loader classpath.
*/
public static void addClasspath(URL url) {
try {
Method method = URLClassLoader.class.getDeclaredMethod("addURL", URL.class);
if (!method.isAccessible())
method.setAccessible(true);
method.invoke(ClassLoader.getSystemClassLoader(), url);
resetReflections();
} catch (Exception e) {
throw new ReviewedStingException("Error adding url to the current classloader.", e);
}
}
/**
* Create new reflections object if it does not currently exists.
*/
private static void initReflections() {
if (reflections == null) {
// Initialize general-purpose source tree reflector.
reflections = new Reflections( new ConfigurationBuilder()
.setUrls(getClassPathURLs())
.setScanners(new SubTypesScanner()));
}
}
/**
* Resets the reflections object after a class has been dynamically added to the classpath.
*/
private static void resetReflections() {
reflections = null;
}
}

194
java/src/org/broadinstitute/sting/utils/classloader/PluginManager.java
View File

@ -25,20 +25,42 @@
package org.broadinstitute.sting.utils.classloader;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.Logger;
import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import org.broadinstitute.sting.utils.exceptions.UserException;
import org.reflections.Reflections;
import org.reflections.scanners.SubTypesScanner;
import org.reflections.util.ConfigurationBuilder;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.lang.reflect.Method;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.*;
/**
* Manage plugins and plugin configuration.
* @author mhanna
* @version 0.1
*/
public abstract class PluginManager<PluginType> {
public class PluginManager<PluginType> {
/**
* A reference into our introspection utility.
*/
private static final Reflections defaultReflections;
static {
// turn off logging in the reflections library - they talk too much (to the wrong logger factory as well, logback)
Logger logger = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Reflections.class);
logger.setLevel(Level.OFF);
defaultReflections = new Reflections( new ConfigurationBuilder()
.setUrls(JVMUtils.getClasspathURLs())
.setScanners(new SubTypesScanner()));
}
/**
* Defines the category of plugin defined by the subclass.
*/
@ -48,11 +70,31 @@ public abstract class PluginManager<PluginType> {
* Define common strings to trim off the end of the name.
*/
protected final String pluginSuffix;
/**
* Plugins stored based on their name.
*/
protected final Map<String, Class<? extends PluginType>> pluginsByName;
private SortedMap<String, Class<? extends PluginType>> pluginsByName = null;
private List<Class<? extends PluginType>> plugins;
private List<Class<? extends PluginType>> interfaces;
/**
* Create a new plugin manager.
* @param pluginType Core type for a plugin.
*/
public PluginManager(Class<PluginType> pluginType) {
this(pluginType, pluginType.getSimpleName().toLowerCase(), pluginType.getSimpleName(), null);
}
/**
* Create a new plugin manager.
* @param pluginType Core type for a plugin.
* @param classpath Custom class path to search for classes.
*/
public PluginManager(Class<PluginType> pluginType, List<URL> classpath) {
this(pluginType, pluginType.getSimpleName().toLowerCase(), pluginType.getSimpleName(), classpath);
}
/**
* Create a new plugin manager.
@ -60,11 +102,75 @@ public abstract class PluginManager<PluginType> {
* @param pluginCategory Provides a category name to the plugin. Must not be null.
* @param pluginSuffix Provides a suffix that will be trimmed off when converting to a plugin name. Can be null.
*/
protected PluginManager(Class<PluginType> pluginType, String pluginCategory, String pluginSuffix) {
public PluginManager(Class<PluginType> pluginType, String pluginCategory, String pluginSuffix) {
this(pluginType, pluginCategory, pluginSuffix, null);
}
/**
* Create a new plugin manager.
* @param pluginType Core type for a plugin.
* @param pluginCategory Provides a category name to the plugin. Must not be null.
* @param pluginSuffix Provides a suffix that will be trimmed off when converting to a plugin name. Can be null.
* @param classpath Custom class path to search for classes.
*/
public PluginManager(Class<PluginType> pluginType, String pluginCategory, String pluginSuffix, List<URL> classpath) {
this.pluginCategory = pluginCategory;
this.pluginSuffix = pluginSuffix;
List<Class<? extends PluginType>> plugins = PackageUtils.getClassesImplementingInterface(pluginType);
pluginsByName = createPluginDatabase(plugins);
this.plugins = new ArrayList<Class<? extends PluginType>>();
this.interfaces = new ArrayList<Class<? extends PluginType>>();
Reflections reflections;
if (classpath == null) {
reflections = defaultReflections;
} else {
addClasspath(classpath);
reflections = new Reflections( new ConfigurationBuilder()
.setUrls(classpath)
.setScanners(new SubTypesScanner()));
}
// Load all classes types filtering them by concrete.
Set<Class<? extends PluginType>> allTypes = reflections.getSubTypesOf(pluginType);
for( Class<? extends PluginType> type: allTypes ) {
if( JVMUtils.isConcrete(type) )
plugins.add(type);
else
interfaces.add(type);
}
}
/**
* Adds the URL to the system class loader classpath using reflection.
* HACK: Uses reflection to modify the class path, and assumes loader is a URLClassLoader.
* @param urls URLs to add to the system class loader classpath.
*/
private static void addClasspath(List<URL> urls) {
Set<URL> existing = JVMUtils.getClasspathURLs();
for (URL url : urls) {
if (existing.contains(url))
continue;
try {
Method method = URLClassLoader.class.getDeclaredMethod("addURL", URL.class);
if (!method.isAccessible())
method.setAccessible(true);
method.invoke(ClassLoader.getSystemClassLoader(), url);
} catch (Exception e) {
throw new ReviewedStingException("Error adding url to the current classloader.", e);
}
}
}
protected SortedMap<String, Class<? extends PluginType>> getPluginsByName() {
if (pluginsByName == null) {
SortedMap<String, Class<? extends PluginType>> newPlugins = new TreeMap<String, Class<? extends PluginType>>();
for (Class<? extends PluginType> pluginClass : plugins) {
String pluginName = getName(pluginClass);
newPlugins.put(pluginName, pluginClass);
}
pluginsByName = newPlugins;
}
return pluginsByName;
}
/**
@ -74,9 +180,49 @@ public abstract class PluginManager<PluginType> {
* @return True if the plugin exists, false otherwise.
*/
public boolean exists(String pluginName) {
return pluginsByName.containsKey(pluginName);
return getPluginsByName().containsKey(pluginName);
}
/**
* Does a plugin with the given name exist?
*
* @param plugin Name of the plugin for which to search.
* @return True if the plugin exists, false otherwise.
*/
public boolean exists(Class<?> plugin) {
return getPluginsByName().containsValue(plugin);
}
/**
* Returns the plugin classes
* @return the plugin classes
*/
public List<Class<? extends PluginType>> getPlugins() {
return plugins;
}
/**
* Returns the interface classes
* @return the interface classes
*/
public List<Class<? extends PluginType>> getInterfaces() {
return interfaces;
}
/**
* Returns the plugin classes implementing interface or base clase
* @param type type of interface or base class
* @return the plugin classes implementing interface or base class
*/
public List<Class<? extends PluginType>> getPluginsImplementing(Class<?> type) {
List<Class<? extends PluginType>> implementing = new ArrayList<Class<? extends PluginType>>();
for (Class<? extends PluginType> plugin: getPlugins())
if (type.isAssignableFrom(plugin))
implementing.add(plugin);
return implementing;
}
/**
* Gets a plugin with the given name
@ -85,7 +231,7 @@ public abstract class PluginManager<PluginType> {
* @return The plugin object if found; null otherwise.
*/
public PluginType createByName(String pluginName) {
Class<? extends PluginType> plugin = pluginsByName.get(pluginName);
Class<? extends PluginType> plugin = getPluginsByName().get(pluginName);
if( plugin == null )
throw new UserException(String.format("Could not find %s with name: %s", pluginCategory,pluginName));
try {
@ -101,6 +247,7 @@ public abstract class PluginManager<PluginType> {
* @param pluginType type of the plugin to create.
* @return The plugin object if created; null otherwise.
*/
@SuppressWarnings("unchecked")
public PluginType createByType(Class pluginType) {
try {
return ((Class<? extends PluginType>) pluginType).newInstance();
@ -110,20 +257,19 @@ public abstract class PluginManager<PluginType> {
}
/**
* Create the list of available plugins and add them to the database.
*
* @param pluginClasses Classes to record.
* @return map of plugin name -> plugin.
* Returns concrete instances of the plugins
* @return concrete instances of the plugins
*/
private Map<String, Class<? extends PluginType>> createPluginDatabase(List<Class<? extends PluginType>> pluginClasses) {
Map<String, Class<? extends PluginType>> plugins = new HashMap<String, Class<? extends PluginType>>();
for (Class<? extends PluginType> pluginClass : pluginClasses) {
String pluginName = getName(pluginClass);
plugins.put(pluginName, pluginClass);
public List<PluginType> createAllTypes() {
List<PluginType> instances = new ArrayList<PluginType>();
for ( Class<? extends PluginType> c : getPlugins() ) {
try {
instances.add(c.newInstance());
} catch (Exception e) {
throw new DynamicClassResolutionException(c, e);
}
}
return plugins;
return instances;
}
/**

137
java/src/org/broadinstitute/sting/utils/interval/IntervalUtils.java
View File

@ -1,8 +1,12 @@
package org.broadinstitute.sting.utils.interval;
import net.sf.picard.util.IntervalList;
import net.sf.samtools.SAMFileHeader;
import org.broadinstitute.sting.gatk.datasources.simpleDataSources.ReferenceDataSource;
import org.broadinstitute.sting.utils.GenomeLocSortedSet;
import org.broadinstitute.sting.utils.GenomeLoc;
import org.broadinstitute.sting.utils.GenomeLocParser;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.exceptions.UserException;
import java.util.LinkedList;
@ -25,6 +29,7 @@ public class IntervalUtils {
* 'all' can be supplied to indicate all possible intervals, but 'all' must be exclusive of all other interval
* specifications.
*
* @param parser Genome loc parser.
* @param argList A list of strings containing interval data.
* @param allowEmptyIntervalList If false instead of an empty interval list will return null.
* @return an unsorted, unmerged representation of the given intervals. Null is used to indicate that all intervals should be used.
@ -107,7 +112,7 @@ public class IntervalUtils {
if (setOne.get(iOne).getStop() < setTwo.get(iTwo).getStop()) iOne++;
else iTwo++;
}
// we don't need to add the rest of remaining locations, since we know they don't overlap. return what we have
return retList;
}
@ -117,6 +122,7 @@ public class IntervalUtils {
* all overlapping and abutting intervals into an interval that spans the union of all covered bases, and
* OVERLAPPING_ONLY, which unions overlapping intervals but keeps abutting intervals separate.
*
* @param parser Genome loc parser for the intervals.
* @param intervals A collection of intervals to merge.
* @param mergingRule A descriptor for the type of merging to perform.
* @return A sorted, merged version of the intervals passed in.
@ -158,6 +164,131 @@ public class IntervalUtils {
else return false;
}
/**
* Returns the list of GenomeLocs from the list of intervals.
* @param referenceSource The reference for the intervals.
* @param intervals The interval as strings or file paths.
* @return The list of GenomeLocs.
*/
private static List<GenomeLoc> parseIntervalArguments(ReferenceDataSource referenceSource, List<String> intervals) {
GenomeLocParser parser = new GenomeLocParser(referenceSource.getReference());
GenomeLocSortedSet locs;
// TODO: Abstract genome analysis engine has richer logic for parsing. We need to use it!
if (intervals.size() == 0) {
locs = GenomeLocSortedSet.createSetFromSequenceDictionary(referenceSource.getReference().getSequenceDictionary());
} else {
locs = new GenomeLocSortedSet(parser, IntervalUtils.parseIntervalArguments(parser, intervals, false));
}
if (locs == null || locs.size() == 0)
throw new UserException.MalformedFile("Intervals are empty: " + Utils.join(", ", intervals));
return locs.toList();
}
/**
* Returns the list of contigs from the list of intervals.
* @param reference The reference for the intervals.
* @return The list of contig names.
*/
public static List<String> distinctContigs(File reference) {
return distinctContigs(reference, Collections.<String>emptyList());
}
/**
* Returns the list of contigs from the list of intervals.
* @param reference The reference for the intervals.
* @param intervals The interval as strings or file paths.
* @return The list of contig names.
*/
public static List<String> distinctContigs(File reference, List<String> intervals) {
ReferenceDataSource referenceSource = new ReferenceDataSource(reference);
List<GenomeLoc> locs = parseIntervalArguments(referenceSource, intervals);
String contig = null;
List<String> contigs = new ArrayList<String>();
for (GenomeLoc loc: locs) {
if (contig == null || !contig.equals(loc.getContig())) {
contig = loc.getContig();
contigs.add(contig);
}
}
return contigs;
}
/**
* Splits an interval list into multiple files.
* @param reference The reference for the intervals.
* @param intervals The interval as strings or file paths.
* @param scatterParts The output interval lists to write to.
* @param splitByContig If true then one contig will not be written to multiple files.
*/
public static void scatterIntervalArguments(File reference, List<String> intervals, List<File> scatterParts, boolean splitByContig) {
ReferenceDataSource referenceSource = new ReferenceDataSource(reference);
List<GenomeLoc> locs = parseIntervalArguments(referenceSource, intervals);
SAMFileHeader fileHeader = new SAMFileHeader();
fileHeader.setSequenceDictionary(referenceSource.getReference().getSequenceDictionary());
IntervalList intervalList = null;
int fileIndex = -1;
int locIndex = 0;
if (splitByContig) {
String contig = null;
for (GenomeLoc loc: locs) {
// If there are still more files to write and the contig doesn't match...
if ((fileIndex+1 < scatterParts.size()) && (contig == null || !contig.equals(loc.getContig()))) {
// Then close the current file and start a new one.
if (intervalList != null) {
intervalList.write(scatterParts.get(fileIndex));
intervalList = null;
}
fileIndex++;
contig = loc.getContig();
}
if (intervalList == null)
intervalList = new IntervalList(fileHeader);
intervalList.add(toInterval(loc, ++locIndex));
}
if (intervalList != null)
intervalList.write(scatterParts.get(fileIndex));
} else {
int locsPerFile = locs.size() / scatterParts.size();
int locRemainder = locs.size() % scatterParts.size();
// At the start, put an extra loc per file
locsPerFile++;
int locsLeftFile = 0;
for (GenomeLoc loc: locs) {
if (locsLeftFile == 0) {
if (intervalList != null)
intervalList.write(scatterParts.get(fileIndex));
fileIndex++;
intervalList = new IntervalList(fileHeader);
// When we have put enough locs into each file,
// reduce the number of locs per file back
// to the original calculated value.
if (fileIndex == locRemainder)
locsPerFile -= 1;
locsLeftFile = locsPerFile;
}
locsLeftFile -= 1;
intervalList.add(toInterval(loc, ++locIndex));
}
if (intervalList != null)
intervalList.write(scatterParts.get(fileIndex));
}
if ((fileIndex + 1) != scatterParts.size())
throw new UserException.BadArgumentValue("scatterParts", String.format("Only able to write contigs into %d of %d files.", fileIndex + 1, scatterParts.size()));
}
/**
* Converts a GenomeLoc to a picard interval.
* @param loc The GenomeLoc.
* @param locIndex The loc index for use in the file.
* @return The picard interval.
*/
private static net.sf.picard.util.Interval toInterval(GenomeLoc loc, int locIndex) {
return new net.sf.picard.util.Interval(loc.getContig(), loc.getStart(), loc.getStop(), false, "interval_" + locIndex);
}
}

42
java/test/org/broadinstitute/sting/WalkerTest.java
View File

@ -311,35 +311,19 @@ public class WalkerTest extends BaseTest {
*/
private Pair<List<File>, List<String>> executeTest(String name, List<String> md5s, List<File> tmpFiles, String args, Class expectedException) {
CommandLineGATK instance = new CommandLineGATK();
String[] command;
String[] command = Utils.escapeExpressions(args);
// special case for ' and " so we can allow expressions
if (args.indexOf('\'') != -1)
command = escapeExpressions(args, "'");
else if (args.indexOf('\"') != -1)
command = escapeExpressions(args, "\"");
else
command = args.split(" ");
if (outputFileLocation != null) {
String[] cmd2 = Arrays.copyOf(command, command.length + 2);
cmd2[command.length] = "-o";
cmd2[command.length + 1] = this.outputFileLocation.getAbsolutePath();
command = cmd2;
}
if (outputFileLocation != null)
command = Utils.appendArray(command, "-o", this.outputFileLocation.getAbsolutePath());
// add the logging level to each of the integration test commands
String[] cmd2 = Arrays.copyOf(command, command.length + 4);
cmd2[command.length] = "-l";
cmd2[command.length+1] = "WARN";
cmd2[command.length+2] = "-et";
cmd2[command.length+3] = ENABLE_REPORTING ? "STANDARD" : "NO_ET";
command = Utils.appendArray(command, "-l", "WARN", "-et", ENABLE_REPORTING ? "STANDARD" : "NO_ET");
// run the executable
boolean gotAnException = false;
try {
System.out.println(String.format("Executing test %s with GATK arguments: %s", name, Utils.join(" ",cmd2)));
CommandLineExecutable.start(instance, cmd2);
System.out.println(String.format("Executing test %s with GATK arguments: %s", name, Utils.join(" ",command)));
CommandLineExecutable.start(instance, command);
} catch (Exception e) {
gotAnException = true;
if ( expectedException != null ) {
@ -370,25 +354,11 @@ public class WalkerTest extends BaseTest {
throw new RuntimeException("Error running the GATK with arguments: " + args);
}
// clean up some memory
instance = null;
cmd2 = null;
// we need to check MD5s
return new Pair<List<File>, List<String>>(tmpFiles, assertMatchingMD5s(name, tmpFiles, md5s));
}
}
private static String[] escapeExpressions(String args, String delimiter) {
String[] command = {};
String[] split = args.split(delimiter);
for (int i = 0; i < split.length - 1; i += 2) {
command = Utils.concatArrays(command, split[i].trim().split(" "));
command = Utils.concatArrays(command, new String[]{split[i + 1]});
}
return Utils.concatArrays(command, split[split.length - 1].trim().split(" "));
}
@Test
public void testWalkerUnitTest() {
//System.out.println("WalkerTest is just a framework");

91
java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsTest.java
View File

@ -1,91 +0,0 @@
package org.broadinstitute.sting.utils.interval;
import net.sf.picard.reference.IndexedFastaSequenceFile;
import net.sf.picard.reference.ReferenceSequenceFile;
import org.broadinstitute.sting.BaseTest;
import org.testng.Assert;
import org.broadinstitute.sting.utils.GenomeLoc;
import org.broadinstitute.sting.utils.GenomeLocParser;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.io.File;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.List;
/**
* test out the interval utility methods
*/
public class IntervalUtilsTest extends BaseTest {
// used to seed the genome loc parser with a sequence dictionary
private static ReferenceSequenceFile seq;
private GenomeLocParser genomeLocParser;
@BeforeClass
public void init() throws FileNotFoundException {
seq = new IndexedFastaSequenceFile(new File(hg18Reference));
genomeLocParser = new GenomeLocParser(seq);
}
@Test
public void testMergeListsBySetOperatorNoOverlap() {
// a couple of lists we'll use for the testing
List<GenomeLoc> listEveryTwoFromOne = new ArrayList<GenomeLoc>();
List<GenomeLoc> listEveryTwoFromTwo = new ArrayList<GenomeLoc>();
// create the two lists we'll use
for (int x = 1; x < 101; x++) {
if (x % 2 == 0)
listEveryTwoFromTwo.add(genomeLocParser.createGenomeLoc("chr1",x,x));
else
listEveryTwoFromOne.add(genomeLocParser.createGenomeLoc("chr1",x,x));
}
List<GenomeLoc> ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, listEveryTwoFromOne, IntervalSetRule.UNION);
Assert.assertEquals(ret.size(), 100);
ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, listEveryTwoFromOne, IntervalSetRule.INTERSECTION);
Assert.assertEquals(ret.size(), 0);
}
@Test
public void testMergeListsBySetOperatorAllOverlap() {
// a couple of lists we'll use for the testing
List<GenomeLoc> allSites = new ArrayList<GenomeLoc>();
List<GenomeLoc> listEveryTwoFromTwo = new ArrayList<GenomeLoc>();
// create the two lists we'll use
for (int x = 1; x < 101; x++) {
if (x % 2 == 0)
listEveryTwoFromTwo.add(genomeLocParser.createGenomeLoc("chr1",x,x));
allSites.add(genomeLocParser.createGenomeLoc("chr1",x,x));
}
List<GenomeLoc> ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.UNION);
Assert.assertEquals(ret.size(), 150);
ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.INTERSECTION);
Assert.assertEquals(ret.size(), 50);
}
@Test
public void testMergeListsBySetOperator() {
// a couple of lists we'll use for the testing
List<GenomeLoc> allSites = new ArrayList<GenomeLoc>();
List<GenomeLoc> listEveryTwoFromTwo = new ArrayList<GenomeLoc>();
// create the two lists we'll use
for (int x = 1; x < 101; x++) {
if (x % 5 == 0) {
listEveryTwoFromTwo.add(genomeLocParser.createGenomeLoc("chr1",x,x));
allSites.add(genomeLocParser.createGenomeLoc("chr1",x,x));
}
}
List<GenomeLoc> ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.UNION);
Assert.assertEquals(ret.size(), 40);
ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.INTERSECTION);
Assert.assertEquals(ret.size(), 20);
}
}

356
java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsUnitTest.java 100644
View File

@ -0,0 +1,356 @@
package org.broadinstitute.sting.utils.interval;
import net.sf.picard.reference.IndexedFastaSequenceFile;
import net.sf.picard.reference.ReferenceSequenceFile;
import org.broadinstitute.sting.BaseTest;
import org.testng.Assert;
import org.broadinstitute.sting.utils.exceptions.UserException;
import org.broadinstitute.sting.utils.GenomeLoc;
import org.broadinstitute.sting.utils.GenomeLocParser;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* test out the interval utility methods
*/
public class IntervalUtilsUnitTest extends BaseTest {
// used to seed the genome loc parser with a sequence dictionary
private static File reference = new File(BaseTest.hg18Reference);
private GenomeLocParser genomeLocParser;
@BeforeClass
public void init() {
ReferenceSequenceFile seq = new IndexedFastaSequenceFile(reference);
genomeLocParser = new GenomeLocParser(seq);
}
@Test
public void testMergeListsBySetOperatorNoOverlap() {
// a couple of lists we'll use for the testing
List<GenomeLoc> listEveryTwoFromOne = new ArrayList<GenomeLoc>();
List<GenomeLoc> listEveryTwoFromTwo = new ArrayList<GenomeLoc>();
// create the two lists we'll use
for (int x = 1; x < 101; x++) {
if (x % 2 == 0)
listEveryTwoFromTwo.add(genomeLocParser.createGenomeLoc("chr1",x,x));
else
listEveryTwoFromOne.add(genomeLocParser.createGenomeLoc("chr1",x,x));
}
List<GenomeLoc> ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, listEveryTwoFromOne, IntervalSetRule.UNION);
Assert.assertEquals(ret.size(), 100);
ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, listEveryTwoFromOne, IntervalSetRule.INTERSECTION);
Assert.assertEquals(ret.size(), 0);
}
@Test
public void testMergeListsBySetOperatorAllOverlap() {
// a couple of lists we'll use for the testing
List<GenomeLoc> allSites = new ArrayList<GenomeLoc>();
List<GenomeLoc> listEveryTwoFromTwo = new ArrayList<GenomeLoc>();
// create the two lists we'll use
for (int x = 1; x < 101; x++) {
if (x % 2 == 0)
listEveryTwoFromTwo.add(genomeLocParser.createGenomeLoc("chr1",x,x));
allSites.add(genomeLocParser.createGenomeLoc("chr1",x,x));
}
List<GenomeLoc> ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.UNION);
Assert.assertEquals(ret.size(), 150);
ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.INTERSECTION);
Assert.assertEquals(ret.size(), 50);
}
@Test
public void testMergeListsBySetOperator() {
// a couple of lists we'll use for the testing
List<GenomeLoc> allSites = new ArrayList<GenomeLoc>();
List<GenomeLoc> listEveryTwoFromTwo = new ArrayList<GenomeLoc>();
// create the two lists we'll use
for (int x = 1; x < 101; x++) {
if (x % 5 == 0) {
listEveryTwoFromTwo.add(genomeLocParser.createGenomeLoc("chr1",x,x));
allSites.add(genomeLocParser.createGenomeLoc("chr1",x,x));
}
}
List<GenomeLoc> ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.UNION);
Assert.assertEquals(ret.size(), 40);
ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.INTERSECTION);
Assert.assertEquals(ret.size(), 20);
}
@Test
public void testCountContigs() {
List<String> chrs = new ArrayList<String>();
for (int i = 1; i <= 22; i++)
chrs.add("chr" + i);
chrs.add("chrX");
chrs.add("chrY");
List<String> chrsNoRandom = Arrays.asList("chr12", "chr14", "chr20", "chrY");
List<String> chrsWithRandom = new ArrayList<String>();
chrsWithRandom.add("chrM");
chrsWithRandom.addAll(chrs);
for (String chr: chrs)
if(!chrsNoRandom.contains(chr))
chrsWithRandom.add(chr + "_random");
Assert.assertEquals(IntervalUtils.distinctContigs(reference), chrsWithRandom);
Assert.assertEquals(IntervalUtils.distinctContigs(reference, Arrays.asList(BaseTest.validationDataLocation + "TCGA-06-0188.interval_list")), chrs);
Assert.assertEquals(IntervalUtils.distinctContigs(reference, Arrays.asList("chr1:1-1", "chr2:1-1", "chr3:2-2")), Arrays.asList("chr1","chr2","chr3"));
Assert.assertEquals(IntervalUtils.distinctContigs(reference, Arrays.asList("chr2:1-1", "chr1:1-1", "chr3:2-2")), Arrays.asList("chr1","chr2","chr3"));
}
@Test
public void testBasicScatter() {
GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1");
GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2");
GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3");
List<File> files = testFiles("basic.", 3, ".intervals");
IntervalUtils.scatterIntervalArguments(reference, Arrays.asList("chr1", "chr2", "chr3"), files, false);
List<GenomeLoc> locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false);
List<GenomeLoc> locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false);
List<GenomeLoc> locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false);
Assert.assertEquals(locs1.size(), 1);
Assert.assertEquals(locs2.size(), 1);
Assert.assertEquals(locs3.size(), 1);
Assert.assertEquals(locs1.get(0), chr1);
Assert.assertEquals(locs2.get(0), chr2);
Assert.assertEquals(locs3.get(0), chr3);
}
@Test
public void testScatterLessFiles() {
GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1");
GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2");
GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3");
GenomeLoc chr4 = genomeLocParser.parseGenomeInterval("chr4");
List<File> files = testFiles("less.", 3, ".intervals");
IntervalUtils.scatterIntervalArguments(reference, Arrays.asList("chr1", "chr2", "chr3", "chr4"), files, false);
List<GenomeLoc> locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false);
List<GenomeLoc> locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false);
List<GenomeLoc> locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false);
Assert.assertEquals(locs1.size(), 2);
Assert.assertEquals(locs2.size(), 1);
Assert.assertEquals(locs3.size(), 1);
Assert.assertEquals(locs1.get(0), chr1);
Assert.assertEquals(locs1.get(1), chr2);
Assert.assertEquals(locs2.get(0), chr3);
Assert.assertEquals(locs3.get(0), chr4);
}
@Test(expectedExceptions=UserException.BadArgumentValue.class)
public void testScatterMoreFiles() {
List<File> files = testFiles("more.", 3, ".intervals");
IntervalUtils.scatterIntervalArguments(reference, Arrays.asList("chr1", "chr2"), files, false);
}
@Test
public void testScatterIntervals() {
List<String> intervals = Arrays.asList("chr1:1-2", "chr1:4-5", "chr2:1-1", "chr3:2-2");
GenomeLoc chr1a = genomeLocParser.parseGenomeInterval("chr1:1-2");
GenomeLoc chr1b = genomeLocParser.parseGenomeInterval("chr1:4-5");
GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2:1-1");
GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3:2-2");
List<File> files = testFiles("split.", 3, ".intervals");
IntervalUtils.scatterIntervalArguments(reference, intervals, files, true);
List<GenomeLoc> locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false);
List<GenomeLoc> locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false);
List<GenomeLoc> locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false);
Assert.assertEquals(locs1.size(), 2);
Assert.assertEquals(locs2.size(), 1);
Assert.assertEquals(locs3.size(), 1);
Assert.assertEquals(locs1.get(0), chr1a);
Assert.assertEquals(locs1.get(1), chr1b);
Assert.assertEquals(locs2.get(0), chr2);
Assert.assertEquals(locs3.get(0), chr3);
}
@Test
public void testScatterOrder() {
List<String> intervals = Arrays.asList("chr2:1-1", "chr1:1-1", "chr3:2-2");
GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1:1-1");
GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2:1-1");
GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3:2-2");
List<File> files = testFiles("split.", 3, ".intervals");
IntervalUtils.scatterIntervalArguments(reference, intervals, files, true);
List<GenomeLoc> locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false);
List<GenomeLoc> locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false);
List<GenomeLoc> locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false);
Assert.assertEquals(locs1.size(), 1);
Assert.assertEquals(locs2.size(), 1);
Assert.assertEquals(locs3.size(), 1);
Assert.assertEquals(locs1.get(0), chr1);
Assert.assertEquals(locs2.get(0), chr2);
Assert.assertEquals(locs3.get(0), chr3);
}
@Test
public void testBasicScatterByContig() {
GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1");
GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2");
GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3");
List<File> files = testFiles("contig_basic.", 3, ".intervals");
IntervalUtils.scatterIntervalArguments(reference, Arrays.asList("chr1", "chr2", "chr3"), files, true);
List<GenomeLoc> locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false);
List<GenomeLoc> locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false);
List<GenomeLoc> locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false);
Assert.assertEquals(locs1.size(), 1);
Assert.assertEquals(locs2.size(), 1);
Assert.assertEquals(locs3.size(), 1);
Assert.assertEquals(locs1.get(0), chr1);
Assert.assertEquals(locs2.get(0), chr2);
Assert.assertEquals(locs3.get(0), chr3);
}
@Test
public void testScatterByContigLessFiles() {
GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1");
GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2");
GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3");
GenomeLoc chr4 = genomeLocParser.parseGenomeInterval("chr4");
List<File> files = testFiles("contig_less.", 3, ".intervals");
IntervalUtils.scatterIntervalArguments(reference, Arrays.asList("chr1", "chr2", "chr3", "chr4"), files, true);
List<GenomeLoc> locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false);
List<GenomeLoc> locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false);
List<GenomeLoc> locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false);
Assert.assertEquals(locs1.size(), 1);
Assert.assertEquals(locs2.size(), 1);
Assert.assertEquals(locs3.size(), 2);
Assert.assertEquals(locs1.get(0), chr1);
Assert.assertEquals(locs2.get(0), chr2);
Assert.assertEquals(locs3.get(0), chr3);
Assert.assertEquals(locs3.get(1), chr4);
}
@Test(expectedExceptions=UserException.BadArgumentValue.class)
public void testScatterByContigMoreFiles() {
List<File> files = testFiles("contig_more.", 3, ".intervals");
IntervalUtils.scatterIntervalArguments(reference, Arrays.asList("chr1", "chr2"), files, true);
}
@Test
public void testScatterByContigIntervalsStart() {
List<String> intervals = Arrays.asList("chr1:1-2", "chr1:4-5", "chr2:1-1", "chr3:2-2");
GenomeLoc chr1a = genomeLocParser.parseGenomeInterval("chr1:1-2");
GenomeLoc chr1b = genomeLocParser.parseGenomeInterval("chr1:4-5");
GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2:1-1");
GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3:2-2");
List<File> files = testFiles("contig_split_start.", 3, ".intervals");
IntervalUtils.scatterIntervalArguments(reference, intervals, files, true);
List<GenomeLoc> locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false);
List<GenomeLoc> locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false);
List<GenomeLoc> locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false);
Assert.assertEquals(locs1.size(), 2);
Assert.assertEquals(locs2.size(), 1);
Assert.assertEquals(locs3.size(), 1);
Assert.assertEquals(locs1.get(0), chr1a);
Assert.assertEquals(locs1.get(1), chr1b);
Assert.assertEquals(locs2.get(0), chr2);
Assert.assertEquals(locs3.get(0), chr3);
}
@Test
public void testScatterByContigIntervalsMiddle() {
List<String> intervals = Arrays.asList("chr1:1-1", "chr2:1-2", "chr2:4-5", "chr3:2-2");
GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1:1-1");
GenomeLoc chr2a = genomeLocParser.parseGenomeInterval("chr2:1-2");
GenomeLoc chr2b = genomeLocParser.parseGenomeInterval("chr2:4-5");
GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3:2-2");
List<File> files = testFiles("contig_split_middle.", 3, ".intervals");
IntervalUtils.scatterIntervalArguments(reference, intervals, files, true);
List<GenomeLoc> locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false);
List<GenomeLoc> locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false);
List<GenomeLoc> locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false);
Assert.assertEquals(locs1.size(), 1);
Assert.assertEquals(locs2.size(), 2);
Assert.assertEquals(locs3.size(), 1);
Assert.assertEquals(locs1.get(0), chr1);
Assert.assertEquals(locs2.get(0), chr2a);
Assert.assertEquals(locs2.get(1), chr2b);
Assert.assertEquals(locs3.get(0), chr3);
}
@Test
public void testScatterByContigIntervalsEnd() {
List<String> intervals = Arrays.asList("chr1:1-1", "chr2:2-2", "chr3:1-2", "chr3:4-5");
GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1:1-1");
GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2:2-2");
GenomeLoc chr3a = genomeLocParser.parseGenomeInterval("chr3:1-2");
GenomeLoc chr3b = genomeLocParser.parseGenomeInterval("chr3:4-5");
List<File> files = testFiles("contig_split_end.", 3 ,".intervals");
IntervalUtils.scatterIntervalArguments(reference, intervals, files, true);
List<GenomeLoc> locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false);
List<GenomeLoc> locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false);
List<GenomeLoc> locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false);
Assert.assertEquals(locs1.size(), 1);
Assert.assertEquals(locs2.size(), 1);
Assert.assertEquals(locs3.size(), 2);
Assert.assertEquals(locs1.get(0), chr1);
Assert.assertEquals(locs2.get(0), chr2);
Assert.assertEquals(locs3.get(0), chr3a);
Assert.assertEquals(locs3.get(1), chr3b);
}
private List<File> testFiles(String prefix, int count, String suffix) {
ArrayList<File> files = new ArrayList<File>();
for (int i = 1; i <= count; i++)
files.add(new File(testDir + prefix + i + suffix));
return files;
}
}

19
scala/qscript/fullCallingPipeline.q
View File

@ -9,6 +9,7 @@ import org.broadinstitute.sting.queue.function.scattergather.{GatherFunction, Cl
import org.broadinstitute.sting.queue.util.IOUtils
import org.broadinstitute.sting.queue.QScript
import collection.JavaConversions._
import org.broadinstitute.sting.utils.interval.IntervalUtils
import org.broadinstitute.sting.utils.yaml.YamlUtils
import org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType
@ -91,9 +92,9 @@ class fullCallingPipeline extends QScript {
//val expKind = qscript.protocol
// get contigs (needed for indel cleaning parallelism)
val contigs = IntervalScatterFunction.distinctContigs(
val contigs = IntervalUtils.distinctContigs(
qscript.pipeline.getProject.getReferenceFile,
List(qscript.pipeline.getProject.getIntervalList.getAbsolutePath))
List(qscript.pipeline.getProject.getIntervalList.getAbsolutePath)).toList
for ( sample <- recalibratedSamples ) {
val sampleId = sample.getId
@ -135,24 +136,24 @@ class fullCallingPipeline extends QScript {
realigner.setupScatterFunction = {
case scatter: ScatterFunction =>
scatter.commandDirectory = new File("CleanedBams/IntermediateFiles/%s/ScatterGather".format(sampleId))
scatter.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/Cleaning/%s/Scatter.out".format(sampleId))
scatter.jobOutputFile = new File(".queue/logs/Cleaning/%s/Scatter.out".format(sampleId))
}
realigner.setupCloneFunction = {
case (clone: CloneFunction, index: Int) =>
clone.commandDirectory = new File("CleanedBams/IntermediateFiles/%s/ScatterGather/Scatter_%s".format(sampleId, index))
clone.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/Cleaning/%s/Scatter_%s.out".format(sampleId, index))
clone.jobOutputFile = new File(".queue/logs/Cleaning/%s/Scatter_%s.out".format(sampleId, index))
}
realigner.setupGatherFunction = {
case (gather: BamGatherFunction, source: ArgumentSource) =>
gather.commandDirectory = new File("CleanedBams/IntermediateFiles/%s/ScatterGather/Gather_%s".format(sampleId, source.field.getName))
gather.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/Cleaning/%s/FixMates.out".format(sampleId))
gather.jobOutputFile = new File(".queue/logs/Cleaning/%s/FixMates.out".format(sampleId))
gather.memoryLimit = Some(6)
gather.jarFile = qscript.picardFixMatesJar
// Don't pass this AS=true to fix mates!
gather.assumeSorted = None
case (gather: GatherFunction, source: ArgumentSource) =>
gather.commandDirectory = new File("CleanedBams/IntermediateFiles/%s/ScatterGather/Gather_%s".format(sampleId, source.field.getName))
gather.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/Cleaning/%s/Gather_%s.out".format(sampleId, source.field.getName))
gather.jobOutputFile = new File(".queue/logs/Cleaning/%s/Gather_%s.out".format(sampleId, source.field.getName))
}
add(targetCreator,realigner)
@ -229,17 +230,17 @@ class fullCallingPipeline extends QScript {
snps.setupScatterFunction = {
case scatter: ScatterFunction =>
scatter.commandDirectory = new File("SnpCalls/ScatterGather")
scatter.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/SNPCalling/ScatterGather/Scatter.out")
scatter.jobOutputFile = new File(".queue/logs/SNPCalling/ScatterGather/Scatter.out")
}
snps.setupCloneFunction = {
case (clone: CloneFunction, index: Int) =>
clone.commandDirectory = new File("SnpCalls/ScatterGather/Scatter_%s".format(index))
clone.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/SNPCalling/ScatterGather/Scatter_%s.out".format(index))
clone.jobOutputFile = new File(".queue/logs/SNPCalling/ScatterGather/Scatter_%s.out".format(index))
}
snps.setupGatherFunction = {
case (gather: GatherFunction, source: ArgumentSource) =>
gather.commandDirectory = new File("SnpCalls/ScatterGather/Gather_%s".format(source.field.getName))
gather.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/SNPCalling/ScatterGather/Gather_%s.out".format(source.field.getName))
gather.jobOutputFile = new File(".queue/logs/SNPCalling/ScatterGather/Gather_%s.out".format(source.field.getName))
}
// indel genotyper does one sample at a time

46
scala/src/org/broadinstitute/sting/queue/QCommandLine.scala
View File

@ -6,6 +6,8 @@ import java.util.Arrays
import org.broadinstitute.sting.commandline._
import org.broadinstitute.sting.queue.util._
import org.broadinstitute.sting.queue.engine.{QGraphSettings, QGraph}
import collection.JavaConversions._
import org.broadinstitute.sting.utils.classloader.PluginManager
/**
* Entry point of Queue. Compiles and runs QScripts passed in to the command line.
@ -18,21 +20,28 @@ class QCommandLine extends CommandLineProgram with Logging {
@ArgumentCollection
private val settings = new QGraphSettings
QFunction.parsingEngine = new ParsingEngine(this)
private val qScriptManager = new QScriptManager
private val qGraph = new QGraph
private var qScriptClasses: File = _
private lazy val pluginManager = {
qScriptClasses = IOUtils.tempDir("Q-Classes", "", settings.qSettings.tempDirectory)
qScriptManager.loadScripts(scripts, qScriptClasses)
new PluginManager[QScript](classOf[QScript], List(qScriptClasses.toURI.toURL))
}
QFunction.parsingEngine = new ParsingEngine(this)
/**
* Takes the QScripts passed in, runs their script() methods, retrieves their generated
* functions, and then builds and runs a QGraph based on the dependencies.
*/
def execute = {
val qGraph = QCommandLine.qGraph
qGraph.settings = settings
qGraph.debugMode = debugMode == true
val scripts = qScriptManager.createScripts()
for (script <- scripts) {
logger.info("Scripting " + qScriptManager.getName(script.getClass.asSubclass(classOf[QScript])))
for (script <- pluginManager.createAllTypes()) {
logger.info("Scripting " + pluginManager.getName(script.getClass.asSubclass(classOf[QScript])))
loadArgumentsIntoObject(script)
script.script
script.functions.foreach(qGraph.add(_))
@ -63,14 +72,14 @@ class QCommandLine extends CommandLineProgram with Logging {
* @return Array of QScripts passed in.
*/
override def getArgumentSources =
qScriptManager.getValues.asInstanceOf[Array[Class[_]]]
pluginManager.getPlugins.toIterable.toArray.asInstanceOf[Array[Class[_]]]
/**
* Returns the name of a QScript
* @return The name of a QScript
*/
override def getArgumentSourceName(source: Class[_]) =
qScriptManager.getName(source.asSubclass(classOf[QScript]))
pluginManager.getName(source.asSubclass(classOf[QScript]))
/**
* Returns a ScalaCompoundArgumentTypeDescriptor that can parse argument sources into scala collections.
@ -79,12 +88,9 @@ class QCommandLine extends CommandLineProgram with Logging {
override def getArgumentTypeDescriptors =
Arrays.asList(new ScalaCompoundArgumentTypeDescriptor)
/**
* Loads the QScripts passed in and returns a new QScriptManager than can be used to create them.
*/
private lazy val qScriptManager = {
QScriptManager.loadScripts(scripts)
new QScriptManager
def shutdown() = {
qGraph.shutdown()
if (qScriptClasses != null) IOUtils.tryDelete(qScriptClasses)
}
}
@ -92,29 +98,29 @@ class QCommandLine extends CommandLineProgram with Logging {
* Entry point of Queue. Compiles and runs QScripts passed in to the command line.
*/
object QCommandLine {
private val qGraph = new QGraph
/**
* Main.
* @param argv Arguments.
*/
def main(argv: Array[String]) {
val qCommandLine = new QCommandLine
Runtime.getRuntime.addShutdownHook(new Thread {
/** Cleanup as the JVM shuts down. */
override def run = {
qGraph.shutdown()
ProcessController.shutdown()
QScriptManager.deleteOutdir()
qCommandLine.shutdown()
}
})
try {
CommandLineProgram.start(new QCommandLine, argv);
CommandLineProgram.start(qCommandLine, argv);
if (CommandLineProgram.result != 0)
System.exit(CommandLineProgram.result);
} catch {
case e: Exception => CommandLineProgram.exitSystemWithError(e)
} finally {
}
}
}

58
scala/src/org/broadinstitute/sting/queue/QScriptManager.scala
View File

@ -3,62 +3,33 @@ package org.broadinstitute.sting.queue
import scala.tools.nsc.{Global, Settings}
import scala.tools.nsc.io.PlainFile
import org.broadinstitute.sting.queue.util.{Logging, IOUtils}
import collection.JavaConversions
import collection.JavaConversions._
import java.io.File
import scala.tools.nsc.reporters.AbstractReporter
import java.lang.String
import org.apache.log4j.Level
import scala.tools.nsc.util.{FakePos, NoPosition, Position}
import org.broadinstitute.sting.utils.classloader.{PackageUtils, PluginManager}
import org.broadinstitute.sting.queue.util.TextFormatUtils._
import org.apache.commons.io.FileUtils
import org.broadinstitute.sting.utils.classloader.JVMUtils
/**
* Plugin manager for QScripts which loads QScripts into the current class loader.
*/
class QScriptManager extends PluginManager[QScript](classOf[QScript], "QScript", "Script") with Logging {
/**
* Returns the list of QScripts classes found in the classpath.
* @return QScripts classes found in the classpath.
*/
def getValues = {
if (logger.isDebugEnabled) {
JavaConversions.asMap(this.pluginsByName)
.foreach{case (name, clazz) => logger.debug("Found QScript %s: %s".format(name, clazz))}
}
JavaConversions.asIterable(this.pluginsByName.values).toArray
}
/**
* Creates the QScripts for all values found in the classpath.
* @return QScripts found in the classpath.
*/
def createScripts() = getValues.map(_.newInstance.asInstanceOf[QScript])
}
/**
* Plugin manager for QScripts which loads QScripts into the current classloader.
*/
object QScriptManager extends Logging {
private val outdir = IOUtils.tempDir("Q-classes")
class QScriptManager() extends Logging {
/**
* Compiles and loads the scripts in the files into the current classloader.
* Heavily based on scala/src/compiler/scala/tools/ant/Scalac.scala
* @param scripts Scala classes to compile.
*/
def loadScripts(scripts: List[File]) {
def loadScripts(scripts: List[File], tempDir: File) = {
if (scripts.size > 0) {
val settings = new Settings((error: String) => logger.error(error))
settings.deprecation.value = true
settings.outdir.value = outdir.getPath
settings.outdir.value = tempDir.getPath
// Set the classpath to the current class path.
JavaConversions.asSet(PackageUtils.getClassPathURLs).foreach(url => settings.classpath.append(url.getPath))
JVMUtils.getClasspathURLs.foreach(url => settings.classpath.append(url.getPath))
val reporter = new Log4JReporter(settings)
val reporter = new QScriptManager.Log4JReporter(settings)
val compiler = new Global(settings, reporter)
val run = new compiler.Run
@ -78,19 +49,14 @@ object QScriptManager extends Logging {
reporter.WARNING.count, plural(reporter.WARNING.count)))
else
logger.info("Compilation complete")
// Add the new compilation output directory to the classpath.
PackageUtils.addClasspath(outdir.toURI.toURL)
}
}
}
/**
* Removes the outdir cleaning up the temporary classes.
*/
def deleteOutdir() = {
if (FileUtils.deleteQuietly(outdir))
logger.debug("Deleted " + outdir)
}
/**
* Plugin manager for QScripts which loads QScripts into the current classloader.
*/
object QScriptManager extends Logging {
/**
* NSC (New Scala Compiler) reporter which logs to Log4J.

6
scala/src/org/broadinstitute/sting/queue/QSettings.scala
View File

@ -23,6 +23,12 @@ class QSettings {
@Argument(fullName="default_memory_limit", shortName="memLimit", doc="Default memory limit for jobs, in gigabytes.", required=false)
var memoryLimit: Option[Int] = None
@Argument(fullName="run_directory", shortName="runDir", doc="Root directory to run functions from.", required=false)
var runDirectory = new File(".")
@Argument(fullName="temp_directory", shortName="tempDir", doc="Temp directory to pass to functions.", required=false)
var tempDirectory = new File(System.getProperty("java.io.tmpdir"))
@ArgumentCollection
val emailSettings = new EmailSettings
}

8
scala/src/org/broadinstitute/sting/queue/engine/LsfJobRunner.scala
View File

@ -45,7 +45,7 @@ class LsfJobRunner(val function: CommandLineFunction) extends DispatchJobRunner
job.project = function.jobProject
job.queue = function.jobQueue
if (!IOUtils.CURRENT_DIR_ABS.equals(function.commandDirectory))
if (function.commandDirectory != new File(".").getAbsoluteFile)
job.workingDir = function.commandDirectory
job.extraBsubArgs ++= function.extraArgs
@ -166,7 +166,7 @@ class LsfJobRunner(val function: CommandLineFunction) extends DispatchJobRunner
* @return the file path to the pre-exec.
*/
private def writeExec() = {
IOUtils.writeTempFile(function.commandLine, ".exec", "")
IOUtils.writeTempFile(function.commandLine, ".exec", "", function.jobTempDir)
}
/**
@ -185,7 +185,7 @@ class LsfJobRunner(val function: CommandLineFunction) extends DispatchJobRunner
mountCommand(function).foreach(command =>
preExec.append("%s%n".format(command)))
IOUtils.writeTempFile(preExec.toString, ".preExec", "")
IOUtils.writeTempFile(preExec.toString, ".preExec", "", function.jobTempDir)
}
/**
@ -211,6 +211,6 @@ class LsfJobRunner(val function: CommandLineFunction) extends DispatchJobRunner
|fi
|""".stripMargin.format(function.commandDirectory, touchDone, touchFail))
IOUtils.writeTempFile(postExec.toString, ".postExec", "")
IOUtils.writeTempFile(postExec.toString, ".postExec", "", function.jobTempDir)
}
}

7
scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala
View File

@ -2,7 +2,6 @@ package org.broadinstitute.sting.queue.engine
import org.jgrapht.traverse.TopologicalOrderIterator
import org.jgrapht.graph.SimpleDirectedGraph
import scala.collection.JavaConversions
import scala.collection.JavaConversions._
import org.jgrapht.alg.CycleDetector
import org.jgrapht.EdgeFactory
@ -46,7 +45,7 @@ class QGraph extends Logging {
* Checks the functions for missing values and the graph for cyclic dependencies and then runs the functions in the graph.
*/
def run = {
IOUtils.checkTempDir
IOUtils.checkTempDir(settings.qSettings.tempDirectory)
val numMissingValues = fillGraph
val isReady = numMissingValues == 0
@ -158,7 +157,7 @@ class QGraph extends Logging {
*/
private def fillIn = {
// clone since edgeSet is backed by the graph
JavaConversions.asSet(jobGraph.edgeSet).clone.foreach {
asScalaSet(jobGraph.edgeSet).clone.foreach {
case cmd: FunctionEdge => {
addCollectionOutputs(cmd.outputs)
addCollectionInputs(cmd.inputs)
@ -205,7 +204,7 @@ class QGraph extends Logging {
*/
private def validate = {
var numMissingValues = 0
JavaConversions.asSet(jobGraph.edgeSet).foreach {
asScalaSet(jobGraph.edgeSet).foreach {
case cmd: FunctionEdge =>
val missingFieldValues = cmd.function.missingFields
if (missingFieldValues.size > 0) {

97
scala/src/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunction.scala
View File

@ -48,105 +48,12 @@ class IntervalScatterFunction extends ScatterFunction with InProcessFunction {
def bindCloneInputs(cloneFunction: CloneFunction, index: Int) = {
val scatterPart = cloneFunction.getFieldValue(this.intervalsField)
.asInstanceOf[List[File]]
.map(file => IOUtils.subDir(cloneFunction.commandDirectory, file))
.map(file => IOUtils.absolute(cloneFunction.commandDirectory, file))
cloneFunction.setFieldValue(this.intervalsField, scatterPart)
this.scatterParts ++= scatterPart
}
def run() = {
IntervalScatterFunction.scatter(this.referenceSequence, this.intervals, this.scatterParts, this.splitByContig)
IntervalUtils.scatterIntervalArguments(this.referenceSequence, this.intervals, this.scatterParts, this.splitByContig)
}
}
object IntervalScatterFunction {
private def parseLocs(referenceSource: ReferenceDataSource, intervals: List[String]) = {
var genomeLocParser: GenomeLocParser = new GenomeLocParser(referenceSource.getReference)
val locs = {
// TODO: Abstract genome analysis engine has richer logic for parsing. We need to use it!
if (intervals.size == 0) {
GenomeLocSortedSet.createSetFromSequenceDictionary(referenceSource.getReference.getSequenceDictionary)
} else {
new GenomeLocSortedSet(genomeLocParser,IntervalUtils.parseIntervalArguments(genomeLocParser, intervals, false))
}
}
if (locs == null || locs.size == 0)
throw new QException("Intervals are empty: " + intervals.mkString(", "))
locs.toList
}
def distinctContigs(reference: File, intervals: List[String] = Nil) = {
val referenceSource = new ReferenceDataSource(reference)
val locs = parseLocs(referenceSource, intervals)
var contig: String = null
var contigs = List.empty[String]
for (loc <- locs) {
if (contig != loc.getContig) {
contig = loc.getContig
contigs :+= contig
}
}
contigs
}
def scatter(reference: File, intervals: List[String], scatterParts: List[File], splitByContig: Boolean) = {
val referenceSource = new ReferenceDataSource(reference)
val locs = parseLocs(referenceSource, intervals)
val fileHeader = new SAMFileHeader
fileHeader.setSequenceDictionary(referenceSource.getReference.getSequenceDictionary)
var intervalList: IntervalList = null
var fileIndex = -1
var locIndex = 0
if (splitByContig) {
var contig: String = null
for (loc <- locs) {
if (contig != loc.getContig && (fileIndex + 1) < scatterParts.size) {
if (fileIndex >= 0)
intervalList.write(scatterParts(fileIndex))
fileIndex += 1
contig = loc.getContig
intervalList = new IntervalList(fileHeader)
}
locIndex += 1
intervalList.add(toInterval(loc, locIndex))
}
intervalList.write(scatterParts(fileIndex))
if ((fileIndex + 1) != scatterParts.size)
throw new QException("Only able to write contigs into %d of %d files.".format(fileIndex + 1, scatterParts.size))
} else {
var locsPerFile = locs.size / scatterParts.size
val locRemainder = locs.size % scatterParts.size
// At the start, put an extra loc per file
locsPerFile += 1
var locsLeftFile = 0
for (loc <- locs) {
if (locsLeftFile == 0) {
if (fileIndex >= 0)
intervalList.write(scatterParts(fileIndex))
fileIndex += 1
intervalList = new IntervalList(fileHeader)
// When we have put enough locs into each file,
// reduce the number of locs per file back
// to the original calculated value.
if (fileIndex == locRemainder)
locsPerFile -= 1
locsLeftFile = locsPerFile
}
locsLeftFile -= 1
locIndex += 1
intervalList.add(toInterval(loc, locIndex))
}
intervalList.write(scatterParts(fileIndex))
if ((fileIndex + 1) != scatterParts.size)
throw new QException("Only able to write intervals into %d of %d files.".format(fileIndex + 1, scatterParts.size))
}
}
private def toInterval(loc: GenomeLoc, locIndex: Int) =
new net.sf.picard.util.Interval(loc.getContig, loc.getStart.toInt, loc.getStop.toInt, false, "interval_" + locIndex)
}

14
scala/src/org/broadinstitute/sting/queue/function/QFunction.scala
View File

@ -29,10 +29,10 @@ trait QFunction extends Logging {
var qSettings: QSettings = _
/** Directory to run the command in. */
var commandDirectory: File = IOUtils.CURRENT_DIR
var commandDirectory: File = new File(".")
/** Temporary directory to write any files */
var jobTempDir: File = IOUtils.javaTempDir
var jobTempDir: File = null
/** Order the function was added to the graph. */
var addOrder: List[Int] = Nil
@ -279,7 +279,11 @@ trait QFunction extends Logging {
if (jobOutputFile == null)
jobOutputFile = new File(jobName + ".out")
commandDirectory = IOUtils.subDir(IOUtils.CURRENT_DIR, commandDirectory)
if (jobTempDir == null)
jobTempDir = qSettings.tempDirectory
// If the command directory is relative, insert the run directory ahead of it.
commandDirectory = new File(qSettings.runDirectory, commandDirectory.getPath)
}
/**
@ -315,11 +319,11 @@ trait QFunction extends Logging {
}
/**
* Returns the absolute path to the file relative to the job command directory.
* Returns the absolute path to the file relative to the run directory and the job command directory.
* @param file File to root relative to the command directory if it is not already absolute.
* @return The absolute path to file.
*/
private def absolute(file: File) = IOUtils.subDir(commandDirectory, file)
private def absolute(file: File) = IOUtils.absolute(commandDirectory, file)
/**

4
scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala
View File

@ -124,7 +124,7 @@ trait ScatterGatherableFunction extends CommandLineFunction {
// Get absolute paths to the files and bind the sg functions to the clone function via the absolute paths.
scatterFunction.bindCloneInputs(cloneFunction, i)
for (gatherField <- outputFieldsWithValues) {
val gatherPart = IOUtils.subDir(cloneFunction.commandDirectory, cloneFunction.getFieldFile(gatherField))
val gatherPart = IOUtils.absolute(cloneFunction.commandDirectory, cloneFunction.getFieldFile(gatherField))
cloneFunction.setFieldValue(gatherField, gatherPart)
gatherFunctions(gatherField).gatherParts :+= gatherPart
}
@ -244,5 +244,5 @@ trait ScatterGatherableFunction extends CommandLineFunction {
* @param Sub directory under the scatter gather directory.
* @return temporary directory under this scatter gather directory.
*/
private def scatterGatherTempDir(subDir: String) = IOUtils.subDir(this.scatterGatherDirectory, this.jobName + "-sg/" + subDir)
private def scatterGatherTempDir(subDir: String) = IOUtils.absolute(this.scatterGatherDirectory, this.jobName + "-sg/" + subDir)
}

4
scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala
View File

@ -3,7 +3,7 @@ package org.broadinstitute.sting.queue.util
import org.apache.commons.mail.{MultiPartEmail, EmailAttachment}
import java.io.{FileReader, File}
import javax.mail.internet.InternetAddress
import scala.collection.JavaConversions
import scala.collection.JavaConversions._
/**
* Encapsulates a message to be sent over email.
@ -91,7 +91,7 @@ class EmailMessage extends Logging {
* @return java.util.List of InternetAddress'es
*/
private def convert(addresses: List[String]) = {
JavaConversions.asList(addresses.map(address => new InternetAddress(address, false)))
asJavaList(addresses.map(address => new InternetAddress(address, false)))
}
override def toString = {

88
scala/src/org/broadinstitute/sting/queue/util/IOUtils.scala
View File

@ -8,78 +8,32 @@ import org.broadinstitute.sting.utils.exceptions.UserException
* A collection of utilities for modifying java.io.
*/
object IOUtils extends Logging {
/** The current directory "." */
val CURRENT_DIR = new File(".")
val CURRENT_DIR_ABS = absolute(CURRENT_DIR)
/**
* Returns the sub path rooted at the parent.
* If the sub path is already absolute, returns the sub path.
* If the parent is the current directory, returns the sub path.
* If the sub bath is the current directory, returns the parent.
* Else returns new File(parent, subPath)
* @param parent The parent directory
* @param path The sub path to append to the parent, if the path is not absolute.
* @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path.
* Checks if the temp directory has been setup and throws an exception if they user hasn't set it correctly.
* @param tempDir Temporary directory.
*/
def subDir(parent: File, path: String): File =
subDir(parent, new File(path))
/**
* Returns the sub path rooted at the parent.
* If the sub path is already absolute, returns the sub path.
* If the parent is the current directory, returns the sub path.
* If the sub path is the current directory, returns the parent.
* Else returns new File(parent, subPath)
* @param parent The parent directory
* @param file The sub path to append to the parent, if the path is not absolute.
* @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path.
*/
def subDir(parent: File, file: File): File = {
val parentAbs = absolute(parent)
val fileAbs = absolute(file)
if (parentAbs == CURRENT_DIR_ABS && fileAbs == CURRENT_DIR_ABS)
CURRENT_DIR_ABS
else if (parentAbs == CURRENT_DIR_ABS || file.isAbsolute)
fileAbs
else if (fileAbs == CURRENT_DIR_ABS)
parentAbs
else
absolute(new File(parentAbs, file.getPath))
}
def checkTempDir = {
val javaTemp = System.getProperty("java.io.tmpdir")
def checkTempDir(tempDir: File) = {
val tempDirPath = tempDir.getAbsolutePath
// Keeps the user from leaving the temp directory as the default, and on Macs from having pluses
// in the path which can cause problems with the Google Reflections library.
// see also: http://benjchristensen.com/2009/09/22/mac-osx-10-6-java-java-io-tmpdir/
if (javaTemp.startsWith("/var/folders/") || (javaTemp == "/tmp") || (javaTemp == "/tmp/"))
if (tempDirPath.startsWith("/var/folders/") || (tempDirPath == "/tmp") || (tempDirPath == "/tmp/"))
throw new UserException.BadTmpDir("java.io.tmpdir must be explicitly set")
}
/**
* Returns the temp directory as defined by java.
* @return the temp directory as defined by java.
*/
def javaTempDir() = {
val tempDir = new File(System.getProperty("java.io.tmpdir"))
if (!tempDir.exists && !tempDir.mkdirs)
throw new UserException.BadTmpDir("Could not create directory: " + tempDir.getAbsolutePath())
absolute(tempDir)
}
/**
* Creates a temp directory with the prefix and optional suffix.
* @param prefix Prefix for the directory name.
* @param suffix Optional suffix for the directory name. Defaults to "".
* @param suffix Optional suffix for the directory name.
* @param tempDirParent Parent directory for the temp directory.
* @return The created temporary directory.
*/
def tempDir(prefix: String, suffix: String = "") = {
val tempDirParent = javaTempDir()
def tempDir(prefix: String, suffix: String = "", tempDirParent: File) = {
if (!tempDirParent.exists && !tempDirParent.mkdirs)
throw new UserException.BadTmpDir("Could not create temp directory: " + tempDirParent)
val temp = File.createTempFile(prefix + "-", suffix)
val temp = File.createTempFile(prefix + "-", suffix, tempDirParent)
if (!temp.delete)
throw new UserException.BadTmpDir("Could not delete sub file: " + temp.getAbsolutePath())
if (!temp.mkdir)
@ -89,7 +43,7 @@ object IOUtils extends Logging {
def writeContents(file: File, content: String) = FileUtils.writeStringToFile(file, content)
def writeTempFile(content: String, prefix: String, suffix: String = "", directory: File = null) = {
def writeTempFile(content: String, prefix: String, suffix: String = "", directory: File) = {
val tempFile = absolute(File.createTempFile(prefix, suffix, directory))
writeContents(tempFile, content)
tempFile
@ -115,6 +69,28 @@ object IOUtils extends Logging {
directories(level)
}
/**
* Returns the sub path rooted at the parent.
* @param parent The parent directory.
* @param path The sub path to append to the parent, if the path is not absolute.
* @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path.
*/
def absolute(parent: File, path: String): File =
absolute(parent, new File(path))
/**
* Returns the sub path rooted at the parent.
* @param parent The parent directory.
* @param file The sub path to append to the parent, if the path is not absolute.
* @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path.
*/
def absolute(parent: File, file: File): File = {
if (file.isAbsolute)
absolute(file)
else
absolute(new File(parent, file.getPath))
}
/**
* A mix of getCanonicalFile and getAbsoluteFile that returns the
* absolute path to the file without deferencing symbolic links.

11
scala/src/org/broadinstitute/sting/queue/util/ProcessController.scala
View File

@ -2,7 +2,6 @@ package org.broadinstitute.sting.queue.util
import java.io._
import scala.collection.mutable.{HashSet, ListMap}
import scala.collection.JavaConversions
/**
* Facade to Runtime.exec() and java.lang.Process. Handles
@ -258,7 +257,11 @@ object ProcessController extends Logging {
def shutdown() = {
for (process <- running.clone) {
logger.warn("Killing: " + process)
process.destroy
try {
process.destroy
} catch {
case _ => /* ignore */
}
}
}
@ -349,8 +352,10 @@ object ProcessController extends Logging {
* @param len Number of characters in the buffer.
*/
private def writeFile(chars: Array[Char], len: Int) = {
if (fileWriter != null)
if (fileWriter != null) {
fileWriter.write(chars, 0, len)
fileWriter.flush()
}
}
/** Closes the fileWriter if it is not null. */

88
scala/test/org/broadinstitute/sting/queue/QScriptTest.scala 100644
View File

@ -0,0 +1,88 @@
package org.broadinstitute.sting.queue
import org.broadinstitute.sting.utils.Utils
import org.testng.Assert
import org.broadinstitute.sting.commandline.CommandLineProgram
import org.broadinstitute.sting.BaseTest
import org.broadinstitute.sting.queue.util.ProcessController
class QScriptTest extends BaseTest {
protected val stingDir = "./"
/**
* execute the test
* @param name the name of the test
* @param args the argument list
* @param expectedException the expected exception or null if no exception is expected.
*/
def executeTest(name: String, args: String, expectedException: Class[_]) = {
var command = Utils.escapeExpressions(args)
// add the logging level to each of the integration test commands
command = Utils.appendArray(command, "-l", "WARN", "-startFromScratch", "-tempDir", "integrationtests")
// run the executable
var gotAnException = false
val instance = new QCommandLine
QScriptTest.runningCommandLines += instance
try {
println("Executing test %s with Queue arguments: %s".format(name, Utils.join(" ",command)))
CommandLineProgram.start(instance, command)
} catch {
case e =>
gotAnException = true
if (expectedException != null) {
// we expect an exception
println("Wanted exception %s, saw %s".format(expectedException, e.getClass))
if (expectedException.isInstance(e)) {
// it's the type we expected
println(String.format(" => %s PASSED", name))
} else {
e.printStackTrace()
Assert.fail("Test %s expected exception %s but got %s instead".format(
name, expectedException, e.getClass))
}
} else {
// we didn't expect an exception but we got one :-(
throw new RuntimeException(e)
}
} finally {
instance.shutdown()
QScriptTest.runningCommandLines -= instance
}
// catch failures from the integration test
if (expectedException != null) {
if (!gotAnException)
// we expected an exception but didn't see it
Assert.fail("Test %s expected exception %s but none was thrown".format(name, expectedException.toString))
} else {
if (CommandLineProgram.result != 0)
throw new RuntimeException("Error running the GATK with arguments: " + args)
}
}
}
object QScriptTest {
private var runningCommandLines = Set.empty[QCommandLine]
Runtime.getRuntime.addShutdownHook(new Thread {
/** Cleanup as the JVM shuts down. */
override def run = {
try {
ProcessController.shutdown()
} catch {
case _ => /*ignore */
}
runningCommandLines.foreach(commandLine =>
try {
commandLine.shutdown()
} catch {
case _ => /* ignore */
})
}
})
}

266
scala/test/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunctionUnitTest.scala
View File

@ -1,266 +0,0 @@
package org.broadinstitute.sting.queue.extensions.gatk
import collection.JavaConversions._
import java.io.File
import org.testng.Assert
import org.broadinstitute.sting.BaseTest
import org.broadinstitute.sting.utils.interval.IntervalUtils
import org.broadinstitute.sting.queue.QException
import net.sf.picard.reference.IndexedFastaSequenceFile
import org.testng.annotations.{Test, BeforeMethod}
import org.broadinstitute.sting.utils.GenomeLocParser
class IntervalScatterFunctionUnitTest extends BaseTest {
private def reference = new File(BaseTest.b36KGReference)
private var header: IndexedFastaSequenceFile = _
private var genomeLocParser: GenomeLocParser = _
@BeforeMethod
def setup() {
header = new IndexedFastaSequenceFile(reference)
genomeLocParser = new GenomeLocParser(header.getSequenceDictionary())
}
@Test
def testCountContigs = {
Assert.assertEquals(List("1"), IntervalScatterFunction.distinctContigs(reference, List(BaseTest.validationDataLocation + "chr1_b36_pilot3.interval_list")))
Assert.assertEquals(List("1","2","3"), IntervalScatterFunction.distinctContigs(reference, List("1:1-1", "2:1-1", "3:2-2")))
Assert.assertEquals(List("1","2","3"), IntervalScatterFunction.distinctContigs(reference, List("2:1-1", "1:1-1", "3:2-2")))
}
@Test
def testBasicScatter = {
val chr1 = genomeLocParser.parseGenomeInterval("1")
val chr2 = genomeLocParser.parseGenomeInterval("2")
val chr3 = genomeLocParser.parseGenomeInterval("3")
val files = (1 to 3).toList.map(index => new File(testDir + "basic." + index + ".intervals"))
IntervalScatterFunction.scatter(reference, List("1", "2", "3"), files, false)
val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false)
val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false)
val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false)
Assert.assertEquals(1, locs1.size)
Assert.assertEquals(1, locs2.size)
Assert.assertEquals(1, locs3.size)
Assert.assertEquals(chr1, locs1.get(0))
Assert.assertEquals(chr2, locs2.get(0))
Assert.assertEquals(chr3, locs3.get(0))
}
@Test
def testScatterLessFiles = {
val chr1 = genomeLocParser.parseGenomeInterval("1")
val chr2 = genomeLocParser.parseGenomeInterval("2")
val chr3 = genomeLocParser.parseGenomeInterval("3")
val chr4 = genomeLocParser.parseGenomeInterval("4")
val files = (1 to 3).toList.map(index => new File(testDir + "less." + index + ".intervals"))
IntervalScatterFunction.scatter(reference, List("1", "2", "3", "4"), files, false)
val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false)
val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false)
val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false)
Assert.assertEquals(2, locs1.size)
Assert.assertEquals(1, locs2.size)
Assert.assertEquals(1, locs3.size)
Assert.assertEquals(chr1, locs1.get(0))
Assert.assertEquals(chr2, locs1.get(1))
Assert.assertEquals(chr3, locs2.get(0))
Assert.assertEquals(chr4, locs3.get(0))
}
@Test(expectedExceptions=Array(classOf[QException]))
def testScatterMoreFiles = {
val files = (1 to 3).toList.map(index => new File(testDir + "more." + index + ".intervals"))
IntervalScatterFunction.scatter(reference, List("1", "2"), files, false)
}
@Test
def testScatterIntervals = {
val intervals = List("1:1-2", "1:4-5", "2:1-1", "3:2-2")
val chr1a = genomeLocParser.parseGenomeInterval("1:1-2")
val chr1b = genomeLocParser.parseGenomeInterval("1:4-5")
val chr2 = genomeLocParser.parseGenomeInterval("2:1-1")
val chr3 = genomeLocParser.parseGenomeInterval("3:2-2")
val files = (1 to 3).toList.map(index => new File(testDir + "split." + index + ".intervals"))
IntervalScatterFunction.scatter(reference, intervals, files, true)
val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false)
val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false)
val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false)
Assert.assertEquals(2, locs1.size)
Assert.assertEquals(1, locs2.size)
Assert.assertEquals(1, locs3.size)
Assert.assertEquals(chr1a, locs1.get(0))
Assert.assertEquals(chr1b, locs1.get(1))
Assert.assertEquals(chr2, locs2.get(0))
Assert.assertEquals(chr3, locs3.get(0))
}
@Test
def testScatterOrder = {
val intervals = List("2:1-1", "1:1-1", "3:2-2")
val chr1 = genomeLocParser.parseGenomeInterval("1:1-1")
val chr2 = genomeLocParser.parseGenomeInterval("2:1-1")
val chr3 = genomeLocParser.parseGenomeInterval("3:2-2")
val files = (1 to 3).toList.map(index => new File(testDir + "split." + index + ".intervals"))
IntervalScatterFunction.scatter(reference, intervals, files, true)
val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false)
val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false)
val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false)
Assert.assertEquals(1, locs1.size)
Assert.assertEquals(1, locs2.size)
Assert.assertEquals(1, locs3.size)
Assert.assertEquals(chr1, locs1.get(0))
Assert.assertEquals(chr2, locs2.get(0))
Assert.assertEquals(chr3, locs3.get(0))
}
@Test
def testBasicScatterByContig = {
val chr1 = genomeLocParser.parseGenomeInterval("1")
val chr2 = genomeLocParser.parseGenomeInterval("2")
val chr3 = genomeLocParser.parseGenomeInterval("3")
val files = (1 to 3).toList.map(index => new File(testDir + "contig_basic." + index + ".intervals"))
IntervalScatterFunction.scatter(reference, List("1", "2", "3"), files, true)
val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false)
val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false)
val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false)
Assert.assertEquals(1, locs1.size)
Assert.assertEquals(1, locs2.size)
Assert.assertEquals(1, locs3.size)
Assert.assertEquals(chr1, locs1.get(0))
Assert.assertEquals(chr2, locs2.get(0))
Assert.assertEquals(chr3, locs3.get(0))
}
@Test
def testScatterByContigLessFiles = {
val chr1 = genomeLocParser.parseGenomeInterval("1")
val chr2 = genomeLocParser.parseGenomeInterval("2")
val chr3 = genomeLocParser.parseGenomeInterval("3")
val chr4 = genomeLocParser.parseGenomeInterval("4")
val files = (1 to 3).toList.map(index => new File(testDir + "contig_less." + index + ".intervals"))
IntervalScatterFunction.scatter(reference, List("1", "2", "3", "4"), files, true)
val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false)
val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false)
val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false)
Assert.assertEquals(1, locs1.size)
Assert.assertEquals(1, locs2.size)
Assert.assertEquals(2, locs3.size)
Assert.assertEquals(chr1, locs1.get(0))
Assert.assertEquals(chr2, locs2.get(0))
Assert.assertEquals(chr3, locs3.get(0))
Assert.assertEquals(chr4, locs3.get(1))
}
@Test(expectedExceptions=Array(classOf[QException]))
def testScatterByContigMoreFiles = {
val files = (1 to 3).toList.map(index => new File(testDir + "contig_more." + index + ".intervals"))
IntervalScatterFunction.scatter(reference, List("1", "2"), files, true)
}
@Test
def testScatterByContigIntervalsStart = {
val intervals = List("1:1-2", "1:4-5", "2:1-1", "3:2-2")
val chr1a = genomeLocParser.parseGenomeInterval("1:1-2")
val chr1b = genomeLocParser.parseGenomeInterval("1:4-5")
val chr2 = genomeLocParser.parseGenomeInterval("2:1-1")
val chr3 = genomeLocParser.parseGenomeInterval("3:2-2")
val files = (1 to 3).toList.map(index => new File(testDir + "contig_split_start." + index + ".intervals"))
IntervalScatterFunction.scatter(reference, intervals, files, true)
val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false)
val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false)
val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false)
Assert.assertEquals(2, locs1.size)
Assert.assertEquals(1, locs2.size)
Assert.assertEquals(1, locs3.size)
Assert.assertEquals(chr1a, locs1.get(0))
Assert.assertEquals(chr1b, locs1.get(1))
Assert.assertEquals(chr2, locs2.get(0))
Assert.assertEquals(chr3, locs3.get(0))
}
@Test
def testScatterByContigIntervalsMiddle = {
val intervals = List("1:1-1", "2:1-2", "2:4-5", "3:2-2")
val chr1 = genomeLocParser.parseGenomeInterval("1:1-1")
val chr2a = genomeLocParser.parseGenomeInterval("2:1-2")
val chr2b = genomeLocParser.parseGenomeInterval("2:4-5")
val chr3 = genomeLocParser.parseGenomeInterval("3:2-2")
val files = (1 to 3).toList.map(index => new File(testDir + "contig_split_middle." + index + ".intervals"))
IntervalScatterFunction.scatter(reference, intervals, files, true)
val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false)
val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false)
val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false)
Assert.assertEquals(1, locs1.size)
Assert.assertEquals(2, locs2.size)
Assert.assertEquals(1, locs3.size)
Assert.assertEquals(chr1, locs1.get(0))
Assert.assertEquals(chr2a, locs2.get(0))
Assert.assertEquals(chr2b, locs2.get(1))
Assert.assertEquals(chr3, locs3.get(0))
}
@Test
def testScatterByContigIntervalsEnd = {
val intervals = List("1:1-1", "2:2-2", "3:1-2", "3:4-5")
val chr1 = genomeLocParser.parseGenomeInterval("1:1-1")
val chr2 = genomeLocParser.parseGenomeInterval("2:2-2")
val chr3a = genomeLocParser.parseGenomeInterval("3:1-2")
val chr3b = genomeLocParser.parseGenomeInterval("3:4-5")
val files = (1 to 3).toList.map(index => new File(testDir + "contig_split_end." + index + ".intervals"))
IntervalScatterFunction.scatter(reference, intervals, files, true)
val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false)
val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false)
val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false)
Assert.assertEquals(1, locs1.size)
Assert.assertEquals(1, locs2.size)
Assert.assertEquals(2, locs3.size)
Assert.assertEquals(chr1, locs1.get(0))
Assert.assertEquals(chr2, locs2.get(0))
Assert.assertEquals(chr3a, locs3.get(0))
Assert.assertEquals(chr3b, locs3.get(1))
}
}

82
scala/test/org/broadinstitute/sting/queue/util/IOUtilsUnitTest.scala
View File

@ -9,65 +9,53 @@ import org.testng.annotations.Test
class IOUtilsUnitTest extends BaseTest {
@Test
def testGoodTempDir = {
val tmpDir = System.getProperty("java.io.tmpdir")
try {
System.setProperty("java.io.tmpdir", "/tmp/queue")
IOUtils.checkTempDir
} finally {
System.setProperty("java.io.tmpdir", tmpDir)
}
IOUtils.checkTempDir(new File("/tmp/queue"))
}
@Test(expectedExceptions=Array(classOf[UserException.BadTmpDir]))
def testBadTempDir = {
val tmpDir = System.getProperty("java.io.tmpdir")
try {
System.setProperty("java.io.tmpdir", "/tmp")
IOUtils.checkTempDir
} finally {
System.setProperty("java.io.tmpdir", tmpDir)
}
IOUtils.checkTempDir(new File("/tmp"))
}
@Test
def testAbsoluteSubDir = {
var subDir = IOUtils.subDir(IOUtils.CURRENT_DIR, new File("/path/to/file"))
Assert.assertEquals(new File("/path/to/file"), subDir)
var subDir = IOUtils.absolute(new File("."), new File("/path/to/file"))
Assert.assertEquals(subDir, new File("/path/to/file"))
subDir = IOUtils.subDir(new File("/different/path"), new File("/path/to/file"))
Assert.assertEquals(new File("/path/to/file"), subDir)
subDir = IOUtils.absolute(new File("/different/path"), new File("/path/to/file"))
Assert.assertEquals(subDir, new File("/path/to/file"))
subDir = IOUtils.subDir(new File("/different/path"), IOUtils.CURRENT_DIR)
Assert.assertEquals(new File("/different/path"), subDir)
subDir = IOUtils.absolute(new File("/different/path"), new File("."))
Assert.assertEquals(subDir, new File("/different/path"))
}
@Test
def testRelativeSubDir = {
var subDir = IOUtils.subDir(IOUtils.CURRENT_DIR, new File("path/to/file"))
Assert.assertEquals(new File("path/to/file").getCanonicalFile, subDir.getCanonicalFile)
var subDir = IOUtils.absolute(new File("."), new File("path/to/file"))
Assert.assertEquals(subDir.getCanonicalFile, new File("path/to/file").getCanonicalFile)
subDir = IOUtils.subDir(new File("/different/path"), new File("path/to/file"))
Assert.assertEquals(new File("/different/path/path/to/file"), subDir)
subDir = IOUtils.absolute(new File("/different/path"), new File("path/to/file"))
Assert.assertEquals(subDir, new File("/different/path/path/to/file"))
}
@Test
def testDottedSubDir = {
var subDir = IOUtils.subDir(IOUtils.CURRENT_DIR, new File("path/../to/file"))
Assert.assertEquals(new File("path/../to/./file").getCanonicalFile, subDir.getCanonicalFile)
var subDir = IOUtils.absolute(new File("."), new File("path/../to/file"))
Assert.assertEquals(subDir.getCanonicalFile, new File("path/../to/./file").getCanonicalFile)
subDir = IOUtils.subDir(IOUtils.CURRENT_DIR, new File("/path/../to/file"))
Assert.assertEquals(new File("/path/../to/file"), subDir)
subDir = IOUtils.absolute(new File("."), new File("/path/../to/file"))
Assert.assertEquals(subDir, new File("/path/../to/file"))
subDir = IOUtils.subDir(new File("/different/../path"), new File("path/to/file"))
Assert.assertEquals(new File("/different/../path/path/to/file"), subDir)
subDir = IOUtils.absolute(new File("/different/../path"), new File("path/to/file"))
Assert.assertEquals(subDir, new File("/different/../path/path/to/file"))
subDir = IOUtils.subDir(new File("/different/./path"), new File("/path/../to/file"))
Assert.assertEquals(new File("/path/../to/file"), subDir)
subDir = IOUtils.absolute(new File("/different/./path"), new File("/path/../to/file"))
Assert.assertEquals(subDir, new File("/path/../to/file"))
}
@Test
def testTempDir = {
val tempDir = IOUtils.tempDir("Q-Unit-Test")
val tempDir = IOUtils.tempDir("Q-Unit-Test", "", new File("queueTempDirToDelete"))
Assert.assertTrue(tempDir.exists)
Assert.assertFalse(tempDir.isFile)
Assert.assertTrue(tempDir.isDirectory)
@ -79,43 +67,43 @@ class IOUtilsUnitTest extends BaseTest {
@Test
def testDirLevel = {
var dir = IOUtils.dirLevel(new File("/path/to/directory"), 1)
Assert.assertEquals(new File("/path"), dir)
Assert.assertEquals(dir, new File("/path"))
dir = IOUtils.dirLevel(new File("/path/to/directory"), 2)
Assert.assertEquals(new File("/path/to"), dir)
Assert.assertEquals(dir, new File("/path/to"))
dir = IOUtils.dirLevel(new File("/path/to/directory"), 3)
Assert.assertEquals(new File("/path/to/directory"), dir)
Assert.assertEquals(dir, new File("/path/to/directory"))
dir = IOUtils.dirLevel(new File("/path/to/directory"), 4)
Assert.assertEquals(new File("/path/to/directory"), dir)
Assert.assertEquals(dir, new File("/path/to/directory"))
}
@Test
def testAbsolute = {
var dir = IOUtils.absolute(new File("/path/./to/./directory/."))
Assert.assertEquals(new File("/path/to/directory"), dir)
Assert.assertEquals(dir, new File("/path/to/directory"))
dir = IOUtils.absolute(new File("/"))
Assert.assertEquals(new File("/"), dir)
Assert.assertEquals(dir, new File("/"))
dir = IOUtils.absolute(new File("/."))
Assert.assertEquals(new File("/"), dir)
Assert.assertEquals(dir, new File("/"))
dir = IOUtils.absolute(new File("/././."))
Assert.assertEquals(new File("/"), dir)
Assert.assertEquals(dir, new File("/"))
dir = IOUtils.absolute(new File("/./directory/."))
Assert.assertEquals(new File("/directory"), dir)
Assert.assertEquals(dir, new File("/directory"))
dir = IOUtils.absolute(new File("/./directory/./"))
Assert.assertEquals(new File("/directory"), dir)
Assert.assertEquals(dir, new File("/directory"))
dir = IOUtils.absolute(new File("/./directory./"))
Assert.assertEquals(new File("/directory."), dir)
Assert.assertEquals(dir, new File("/directory."))
dir = IOUtils.absolute(new File("/./.directory/"))
Assert.assertEquals(new File("/.directory"), dir)
Assert.assertEquals(dir, new File("/.directory"))
}
@Test
@ -127,8 +115,8 @@ class IOUtilsUnitTest extends BaseTest {
"chr22_random 257318 3156435963 50 51",
"chrX_random 1719168 3156698441 50 51")
val tail = IOUtils.tail(new File(BaseTest.hg18Reference + ".fai"), 5)
Assert.assertEquals(5, tail.size)
Assert.assertEquals(tail.size, 5)
for (i <- 0 until 5)
Assert.assertEquals(lines(i), tail(i))
Assert.assertEquals(tail(i), lines(i))
}
}