diff --git a/ivy.xml b/ivy.xml index f47949227..f7893cb32 100644 --- a/ivy.xml +++ b/ivy.xml @@ -41,8 +41,8 @@ - - + + diff --git a/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java b/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java index 41abd092c..e9b0035cd 100755 --- a/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java +++ b/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java @@ -27,13 +27,14 @@ package org.broadinstitute.sting.analyzecovariates; import org.broadinstitute.sting.commandline.Input; import org.broadinstitute.sting.gatk.walkers.recalibration.*; -import org.broadinstitute.sting.utils.classloader.PackageUtils; +import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException; import org.broadinstitute.sting.utils.text.XReadLines; import org.broadinstitute.sting.commandline.CommandLineProgram; import org.broadinstitute.sting.commandline.Argument; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.regex.Pattern; @@ -114,7 +115,7 @@ class AnalyzeCovariatesCLP extends CommandLineProgram { private void initializeData() { // Get a list of all available covariates - List> classes = PackageUtils.getClassesImplementingInterface(Covariate.class); + Collection> classes = new PluginManager(Covariate.class).getPlugins(); int lineNumber = 0; boolean foundAllCovariates = false; diff --git a/java/src/org/broadinstitute/sting/gatk/WalkerManager.java b/java/src/org/broadinstitute/sting/gatk/WalkerManager.java index 45675092c..cb1da2b99 100755 --- a/java/src/org/broadinstitute/sting/gatk/WalkerManager.java +++ b/java/src/org/broadinstitute/sting/gatk/WalkerManager.java @@ -62,7 +62,7 @@ public class WalkerManager extends PluginManager { */ public Map>> getWalkerNamesByPackage(boolean visibleWalkersOnly) { Map>> walkersByPackage = new HashMap>>(); - for(Class walker: pluginsByName.values()) { + for(Class walker: getPlugins()) { if(visibleWalkersOnly && isHidden(walker)) continue; @@ -157,7 +157,7 @@ public class WalkerManager extends PluginManager { * @return Class representing the walker. */ public Class getWalkerClassByName(String walkerName) { - return pluginsByName.get(walkerName); + return getPluginsByName().get(walkerName); } /** diff --git a/java/src/org/broadinstitute/sting/gatk/filters/FilterManager.java b/java/src/org/broadinstitute/sting/gatk/filters/FilterManager.java index bd899b80c..3bf11c0ac 100644 --- a/java/src/org/broadinstitute/sting/gatk/filters/FilterManager.java +++ b/java/src/org/broadinstitute/sting/gatk/filters/FilterManager.java @@ -55,6 +55,6 @@ public class FilterManager extends PluginManager { } public Collection> getValues() { - return this.pluginsByName.values(); + return this.getPlugins(); } } diff --git a/java/src/org/broadinstitute/sting/gatk/refdata/tracks/builders/RMDTrackBuilder.java b/java/src/org/broadinstitute/sting/gatk/refdata/tracks/builders/RMDTrackBuilder.java index b3bd6ef86..fb68a8e37 100644 --- a/java/src/org/broadinstitute/sting/gatk/refdata/tracks/builders/RMDTrackBuilder.java +++ b/java/src/org/broadinstitute/sting/gatk/refdata/tracks/builders/RMDTrackBuilder.java @@ -32,7 +32,6 @@ import org.broad.tribble.*; import org.broad.tribble.index.Index; import org.broad.tribble.index.IndexFactory; import org.broad.tribble.source.BasicFeatureSource; -import org.broad.tribble.source.CachingFeatureSource; import org.broad.tribble.util.LittleEndianOutputStream; import org.broadinstitute.sting.gatk.arguments.GATKArgumentCollection; import org.broadinstitute.sting.gatk.refdata.ReferenceDependentFeatureCodec; @@ -42,7 +41,6 @@ import org.broadinstitute.sting.gatk.refdata.utils.RMDTriplet; import org.broadinstitute.sting.gatk.AbstractGenomeAnalysisEngine; import org.broadinstitute.sting.gatk.refdata.utils.helpers.DbSNPHelper; import org.broadinstitute.sting.utils.GenomeLocParser; -import org.broadinstitute.sting.utils.Utils; import org.broadinstitute.sting.utils.collections.Pair; import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.exceptions.UserException; @@ -114,8 +112,8 @@ public class RMDTrackBuilder extends PluginManager { /** @return a list of all available track types we currently have access to create */ public Map getAvailableTrackNamesAndTypes() { classes = new HashMap(); - for (String name: this.pluginsByName.keySet()) { - classes.put(name.toUpperCase(), pluginsByName.get(name)); + for (String name: this.getPluginsByName().keySet()) { + classes.put(name.toUpperCase(), getPluginsByName().get(name)); } return classes; } @@ -123,7 +121,7 @@ public class RMDTrackBuilder extends PluginManager { /** @return a list of all available track record types we currently have access to create */ public Map getAvailableTrackNamesAndRecordTypes() { HashMap classToRecord = new HashMap(); - for (String name: this.pluginsByName.keySet()) { + for (String name: this.getPluginsByName().keySet()) { FeatureCodec codec = this.createByName(name); classToRecord.put(name, codec.getFeatureType()); } @@ -133,7 +131,6 @@ public class RMDTrackBuilder extends PluginManager { /** * create a RMDTrack of the specified type * - * @param genomeLocParser GenomeLocParser to use, if case track needs additional reference context. * @param targetClass the target class of track * @param name what to call the track * @param inputFile the input file diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java b/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java index ab4bf068b..be80ede2f 100755 --- a/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java @@ -38,7 +38,7 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotationType import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.GenotypeAnnotation; import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation; import org.broadinstitute.sting.utils.BaseUtils; -import org.broadinstitute.sting.utils.classloader.PackageUtils; +import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.SampleUtils; import org.broadinstitute.sting.commandline.Argument; import org.broadinstitute.sting.commandline.Output; @@ -93,18 +93,18 @@ public class VariantAnnotator extends RodWalker { private void listAnnotationsAndExit() { - List> infoAnnotationClasses = PackageUtils.getClassesImplementingInterface(InfoFieldAnnotation.class); + List> infoAnnotationClasses = new PluginManager(InfoFieldAnnotation.class).getPlugins(); System.out.println("\nAvailable annotations for the VCF INFO field:"); for (int i = 0; i < infoAnnotationClasses.size(); i++) System.out.println("\t" + infoAnnotationClasses.get(i).getSimpleName()); System.out.println(); - List> genotypeAnnotationClasses = PackageUtils.getClassesImplementingInterface(GenotypeAnnotation.class); + List> genotypeAnnotationClasses = new PluginManager(GenotypeAnnotation.class).getPlugins(); System.out.println("\nAvailable annotations for the VCF FORMAT field:"); for (int i = 0; i < genotypeAnnotationClasses.size(); i++) System.out.println("\t" + genotypeAnnotationClasses.get(i).getSimpleName()); System.out.println(); System.out.println("\nAvailable classes/groups of annotations:"); - for ( Class c : PackageUtils.getInterfacesExtendingInterface(AnnotationType.class) ) + for ( Class c : new PluginManager(AnnotationType.class).getInterfaces() ) System.out.println("\t" + c.getSimpleName()); System.out.println(); System.exit(0); diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java b/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java index 65aca754b..6347f2e4e 100755 --- a/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java @@ -50,7 +50,6 @@ import org.broadinstitute.sting.gatk.refdata.utils.helpers.DbSNPHelper; import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.*; import org.broadinstitute.sting.gatk.walkers.annotator.genomicannotator.*; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import org.broadinstitute.sting.utils.classloader.PackageUtils; import org.broadinstitute.sting.utils.exceptions.UserException; @@ -94,8 +93,8 @@ public class VariantAnnotatorEngine { // use this constructor if you want all possible annotations public VariantAnnotatorEngine(GenomeAnalysisEngine engine) { - requestedInfoAnnotations = PackageUtils.getInstancesOfClassesImplementingInterface(InfoFieldAnnotation.class); - requestedGenotypeAnnotations = PackageUtils.getInstancesOfClassesImplementingInterface(GenotypeAnnotation.class); + requestedInfoAnnotations = AnnotationInterfaceManager.createAllInfoFieldAnnotations(); + requestedGenotypeAnnotations = AnnotationInterfaceManager.createAllGenotypeAnnotations(); initializeDBs(engine); } @@ -116,49 +115,9 @@ public class VariantAnnotatorEngine { } private void initializeAnnotations(List annotationGroupsToUse, List annotationsToUse) { - // create a map for all annotation classes which implement our top-level interfaces - HashMap classMap = new HashMap(); - for ( Class c : PackageUtils.getClassesImplementingInterface(InfoFieldAnnotation.class) ) - classMap.put(c.getSimpleName(), c); - for ( Class c : PackageUtils.getClassesImplementingInterface(GenotypeAnnotation.class) ) - classMap.put(c.getSimpleName(), c); - for ( Class c : PackageUtils.getInterfacesExtendingInterface(AnnotationType.class) ) - classMap.put(c.getSimpleName(), c); - - HashSet classes = new HashSet(); - // get the classes from the provided groups (interfaces) - if ( annotationGroupsToUse.size() != 1 || ! annotationGroupsToUse.get(0).toLowerCase().equals("none") ) { - for ( String group : annotationGroupsToUse ) { - Class interfaceClass = classMap.get(group); - if ( interfaceClass == null ) - interfaceClass = classMap.get(group + "Annotation"); - if ( interfaceClass == null ) - throw new UserException.BadArgumentValue("group", "Class " + group + " is not found; please check that you have specified the class name correctly"); - classes.addAll(PackageUtils.getClassesImplementingInterface(interfaceClass)); - } - } - - // get the specific classes provided - for ( String annotation : annotationsToUse ) { - Class annotationClass = classMap.get(annotation); - if ( annotationClass == null ) - annotationClass = classMap.get(annotation + "Annotation"); - if ( annotationClass == null ) - throw new UserException.BadArgumentValue("annotation", "Class " + annotation + " is not found; please check that you have specified the class name correctly"); - classes.add(annotationClass); - } - - // get the instances - requestedInfoAnnotations = new ArrayList(); - requestedGenotypeAnnotations = new ArrayList(); - - for ( Class c : classes ) { - // note that technically an annotation can work on both the INFO and FORMAT fields - if ( InfoFieldAnnotation.class.isAssignableFrom(c) ) - requestedInfoAnnotations.add((InfoFieldAnnotation)PackageUtils.getSimpleInstance(c)); - if ( GenotypeAnnotation.class.isAssignableFrom(c) ) - requestedGenotypeAnnotations.add((GenotypeAnnotation)PackageUtils.getSimpleInstance(c)); - } + AnnotationInterfaceManager.validateAnnotations(annotationGroupsToUse, annotationsToUse); + requestedInfoAnnotations = AnnotationInterfaceManager.createInfoFieldAnnotations(annotationGroupsToUse, annotationsToUse); + requestedGenotypeAnnotations = AnnotationInterfaceManager.createGenotypeAnnotations(annotationGroupsToUse, annotationsToUse); } private void initializeDBs(GenomeAnalysisEngine engine) { diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/AnnotationInterfaceManager.java b/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/AnnotationInterfaceManager.java new file mode 100644 index 000000000..89106a360 --- /dev/null +++ b/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/AnnotationInterfaceManager.java @@ -0,0 +1,100 @@ +package org.broadinstitute.sting.gatk.walkers.annotator.interfaces; + +import org.broadinstitute.sting.utils.classloader.PluginManager; +import org.broadinstitute.sting.utils.exceptions.UserException; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +public class AnnotationInterfaceManager { + private static PluginManager infoFieldAnnotationPluginManager = new PluginManager(InfoFieldAnnotation.class); + private static PluginManager genotypeAnnotationPluginManager = new PluginManager(GenotypeAnnotation.class); + private static PluginManager annotationTypePluginManager = new PluginManager(AnnotationType.class); + + public static List createAllInfoFieldAnnotations() { + return infoFieldAnnotationPluginManager.createAllTypes(); + } + + public static List createAllGenotypeAnnotations() { + return genotypeAnnotationPluginManager.createAllTypes(); + } + + public static void validateAnnotations(List annotationGroupsToUse, List annotationsToUse) { + HashMap classMap = new HashMap(); + for ( Class c : infoFieldAnnotationPluginManager.getPlugins() ) + classMap.put(c.getSimpleName(), c); + for ( Class c : genotypeAnnotationPluginManager.getPlugins() ) + classMap.put(c.getSimpleName(), c); + for ( Class c : annotationTypePluginManager.getInterfaces() ) + classMap.put(c.getSimpleName(), c); + + if ( annotationGroupsToUse.size() != 1 || !"none".equals(annotationGroupsToUse.get(0)) ) { + for ( String group : annotationGroupsToUse ) { + Class interfaceClass = classMap.get(group); + if ( interfaceClass == null ) + interfaceClass = classMap.get(group + "Annotation"); + if ( interfaceClass == null ) + throw new UserException.BadArgumentValue("group", "Class " + group + " is not found; please check that you have specified the class name correctly"); + } + } + + // validate the specific classes provided + for ( String annotation : annotationsToUse ) { + Class annotationClass = classMap.get(annotation); + if ( annotationClass == null ) + annotationClass = classMap.get(annotation + "Annotation"); + if ( annotationClass == null ) + throw new UserException.BadArgumentValue("annotation", "Class " + annotation + " is not found; please check that you have specified the class name correctly"); + } + } + + public static List createInfoFieldAnnotations(List annotationGroupsToUse, List annotationsToUse) { + return createAnnotations(infoFieldAnnotationPluginManager, annotationGroupsToUse, annotationsToUse); + } + + public static List createGenotypeAnnotations(List annotationGroupsToUse, List annotationsToUse) { + return createAnnotations(genotypeAnnotationPluginManager, annotationGroupsToUse, annotationsToUse); + } + + private static List createAnnotations(PluginManager pluginManager, List annotationGroupsToUse, List annotationsToUse) { + // get the instances + List annotations = new ArrayList(); + + // get the classes from the provided groups (interfaces) + // create a map for all annotation classes which implement our top-level interfaces + HashMap classMap = new HashMap(); + for ( Class c : pluginManager.getPlugins() ) + classMap.put(c.getSimpleName(), c); + for ( Class c : annotationTypePluginManager.getInterfaces() ) + classMap.put(c.getSimpleName(), c); + + HashSet classes = new HashSet(); + + if ( annotationGroupsToUse.size() != 1 || !"none".equals(annotationGroupsToUse.get(0)) ) { + for ( String group : annotationGroupsToUse ) { + Class interfaceClass = classMap.get(group); + if ( interfaceClass == null ) + interfaceClass = classMap.get(group + "Annotation"); + if ( interfaceClass != null ) + classes.addAll(pluginManager.getPluginsImplementing(interfaceClass)); + } + } + + // get the specific classes provided + for ( String annotation : annotationsToUse ) { + Class annotationClass = classMap.get(annotation); + if ( annotationClass == null ) + annotationClass = classMap.get(annotation + "Annotation"); + if ( annotationClass != null ) + classes.add(annotationClass); + } + + // note that technically an annotation can work on both the INFO and FORMAT fields + for ( Class c : classes ) + annotations.add(pluginManager.createByType(c)); + + return annotations; + } +} diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CovariateCounterWalker.java b/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CovariateCounterWalker.java index 0df273eab..211ebb95c 100755 --- a/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CovariateCounterWalker.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CovariateCounterWalker.java @@ -37,7 +37,7 @@ import org.broadinstitute.sting.gatk.filters.ZeroMappingQualityReadFilter; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; import org.broadinstitute.sting.gatk.walkers.*; import org.broadinstitute.sting.utils.*; -import org.broadinstitute.sting.utils.classloader.PackageUtils; +import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.collections.NestedHashMap; import org.broadinstitute.sting.commandline.Argument; import org.broadinstitute.sting.commandline.ArgumentCollection; @@ -167,9 +167,9 @@ public class CovariateCounterWalker extends LocusWalker> covariateClasses = PackageUtils.getClassesImplementingInterface( Covariate.class ); - final List> requiredClasses = PackageUtils.getClassesImplementingInterface( RequiredCovariate.class ); - final List> standardClasses = PackageUtils.getClassesImplementingInterface( StandardCovariate.class ); + final List> covariateClasses = new PluginManager( Covariate.class ).getPlugins(); + final List> requiredClasses = new PluginManager( RequiredCovariate.class ).getPlugins(); + final List> standardClasses = new PluginManager( StandardCovariate.class ).getPlugins(); // Print and exit if that's what was requested if ( LIST_ONLY ) { diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java b/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java index 2e6bf8543..a498dd501 100644 --- a/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java @@ -42,7 +42,7 @@ import org.broadinstitute.sting.gatk.walkers.ReadWalker; import org.broadinstitute.sting.gatk.walkers.Requires; import org.broadinstitute.sting.gatk.walkers.WalkerName; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; -import org.broadinstitute.sting.utils.classloader.PackageUtils; +import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.collections.NestedHashMap; import org.broadinstitute.sting.utils.QualityUtils; import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException; @@ -149,7 +149,7 @@ public class TableRecalibrationWalker extends ReadWalker> classes = PackageUtils.getClassesImplementingInterface(Covariate.class); + final List> classes = new PluginManager(Covariate.class).getPlugins(); int lineNumber = 0; boolean foundAllCovariates = false; diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java b/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java index 53a336464..7cfdbcafa 100755 --- a/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java @@ -49,7 +49,7 @@ import org.broadinstitute.sting.utils.report.VE2ReportFactory; import org.broadinstitute.sting.utils.report.templates.ReportFormat; import org.broadinstitute.sting.utils.report.utils.Node; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import org.broadinstitute.sting.utils.classloader.PackageUtils; +import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.Utils; import org.broadinstitute.sting.commandline.Argument; import org.broadinstitute.sting.commandline.Output; @@ -351,7 +351,7 @@ public class VariantEvalWalker extends RodWalker implements Tr } private void listModulesAndExit() { - List> veClasses = PackageUtils.getClassesImplementingInterface(VariantEvaluator.class); + List> veClasses = new PluginManager( VariantEvaluator.class ).getPlugins(); out.println("\nAvailable eval modules:"); out.println("(Standard modules are starred)"); for (Class veClass : veClasses) @@ -399,14 +399,14 @@ public class VariantEvalWalker extends RodWalker implements Tr private void determineEvalations() { // create a map for all eval modules for easy lookup HashMap> classMap = new HashMap>(); - for ( Class c : PackageUtils.getClassesImplementingInterface(VariantEvaluator.class) ) + for ( Class c : new PluginManager( VariantEvaluator.class ).getPlugins() ) classMap.put(c.getSimpleName(), c); evaluationClasses = new HashSet>(); // by default, use standard eval modules if ( !NO_STANDARD ) { - for ( Class myClass : PackageUtils.getClassesImplementingInterface(StandardEval.class) ) { + for ( Class myClass : new PluginManager( StandardEval.class ).getPlugins() ) { if ( classMap.containsKey(myClass.getSimpleName()) ) evaluationClasses.add(classMap.get(myClass.getSimpleName())); } diff --git a/java/src/org/broadinstitute/sting/oneoffprojects/walkers/AnnotationByAlleleFrequencyWalker.java b/java/src/org/broadinstitute/sting/oneoffprojects/walkers/AnnotationByAlleleFrequencyWalker.java index 0e10b2c6c..926842b7f 100755 --- a/java/src/org/broadinstitute/sting/oneoffprojects/walkers/AnnotationByAlleleFrequencyWalker.java +++ b/java/src/org/broadinstitute/sting/oneoffprojects/walkers/AnnotationByAlleleFrequencyWalker.java @@ -31,13 +31,11 @@ import org.broadinstitute.sting.gatk.contexts.AlignmentContext; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; import org.broadinstitute.sting.gatk.walkers.RodWalker; -import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotationType; +import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotationInterfaceManager; import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.GenotypeAnnotation; import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation; import org.broadinstitute.sting.utils.GenomeLoc; -import org.broadinstitute.sting.utils.classloader.PackageUtils; import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException; -import org.broadinstitute.sting.utils.exceptions.UserException; import java.util.*; @@ -63,8 +61,8 @@ public class AnnotationByAlleleFrequencyWalker extends RodWalker requestedInfoAnnotations; - private ArrayList requestedGenotypeAnnotations; + private List requestedInfoAnnotations; + private List requestedGenotypeAnnotations; //--------------------------------------------------------------------------------------------------------------- @@ -74,49 +72,11 @@ public class AnnotationByAlleleFrequencyWalker extends RodWalker classMap = new HashMap(); - for ( Class c : PackageUtils.getClassesImplementingInterface(InfoFieldAnnotation.class) ) - classMap.put(c.getSimpleName(), c); - for ( Class c : PackageUtils.getClassesImplementingInterface(GenotypeAnnotation.class) ) - classMap.put(c.getSimpleName(), c); - for ( Class c : PackageUtils.getInterfacesExtendingInterface(AnnotationType.class) ) - classMap.put(c.getSimpleName(), c); - - HashSet classes = new HashSet(); - // get the classes from the provided groups (interfaces) - for ( String group : annotationClassesToUse ) { - Class interfaceClass = classMap.get(group); - if ( interfaceClass == null ) - interfaceClass = classMap.get(group + "Annotation"); - if ( interfaceClass == null ) - throw new UserException.BadArgumentValue("group", "Class " + group + " is not found; please check that you have specified the class name correctly"); - classes.addAll(PackageUtils.getClassesImplementingInterface(interfaceClass)); - } - - // get the specific classes provided - for ( String annotation : annotationsToUse ) { - Class annotationClass = classMap.get(annotation); - if ( annotationClass == null ) - annotationClass = classMap.get(annotation + "Annotation"); - if ( annotationClass == null ) - throw new UserException.BadArgumentValue("annotation", "Class " + annotation + " is not found; please check that you have specified the class name correctly"); - classes.add(annotationClass); - } - - // get the instances - requestedInfoAnnotations = new ArrayList(); - requestedGenotypeAnnotations = new ArrayList(); - - for ( Class c : classes ) { - // note that technically an annotation can work on both the INFO and FORMAT fields - if ( InfoFieldAnnotation.class.isAssignableFrom(c) ) - requestedInfoAnnotations.add((InfoFieldAnnotation)getInstance(c)); - if ( GenotypeAnnotation.class.isAssignableFrom(c) ) - requestedGenotypeAnnotations.add((GenotypeAnnotation)getInstance(c)); - } - + List annotationClasses = Arrays.asList(annotationClassesToUse); + List annotations = Arrays.asList(annotationsToUse); + AnnotationInterfaceManager.validateAnnotations(annotationClasses, annotations); + requestedInfoAnnotations = AnnotationInterfaceManager.createInfoFieldAnnotations(annotationClasses, annotations); + requestedGenotypeAnnotations = AnnotationInterfaceManager.createGenotypeAnnotations(annotationClasses, annotations); } private static ArrayList getInstances(List> classes) { diff --git a/java/src/org/broadinstitute/sting/queue/extensions/gatk/CommandLineProgramManager.java b/java/src/org/broadinstitute/sting/queue/extensions/gatk/CommandLineProgramManager.java deleted file mode 100644 index cefca44da..000000000 --- a/java/src/org/broadinstitute/sting/queue/extensions/gatk/CommandLineProgramManager.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2010, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.queue.extensions.gatk; - -import org.broadinstitute.sting.commandline.CommandLineProgram; -import org.broadinstitute.sting.utils.classloader.PluginManager; - -import java.util.Collection; - -/** - * Finds all command line programs. - */ -public class CommandLineProgramManager extends PluginManager { - public CommandLineProgramManager() { - super(CommandLineProgram.class, "CommandLineProgram", "CLP"); - } - - public Collection> getValues() { - return this.pluginsByName.values(); - } -} diff --git a/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java b/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java index 53736586b..06d7cadef 100644 --- a/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java +++ b/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java @@ -40,6 +40,7 @@ import org.broadinstitute.sting.gatk.io.stubs.SAMFileWriterArgumentTypeDescripto import org.broadinstitute.sting.gatk.refdata.tracks.builders.RMDTrackBuilder; import org.broadinstitute.sting.gatk.walkers.ReadWalker; import org.broadinstitute.sting.gatk.walkers.Walker; +import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import java.io.File; @@ -62,7 +63,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram { @Output(fullName="output_directory", shortName="outDir", doc="Directory to output the generated scala", required=true) public File outputDirectory; - CommandLineProgramManager clpManager = new CommandLineProgramManager(); + PluginManager clpManager = new PluginManager(CommandLineProgram.class, "CommandLineProgram", "CLP"); GenomeAnalysisEngine GATKEngine = new GenomeAnalysisEngine(); WalkerManager walkerManager = new WalkerManager(); FilterManager filterManager = new FilterManager(); @@ -97,7 +98,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram { if (!outputDirectory.isDirectory() && !outputDirectory.mkdirs()) throw new ReviewedStingException("Unable to create output directory: " + outputDirectory); - for (Class clp: clpManager.getValues()) { + for (Class clp: clpManager.getPlugins()) { if (!isGatkProgram(clp)) continue; diff --git a/java/src/org/broadinstitute/sting/utils/Utils.java b/java/src/org/broadinstitute/sting/utils/Utils.java index f902100ef..31fe6252d 100755 --- a/java/src/org/broadinstitute/sting/utils/Utils.java +++ b/java/src/org/broadinstitute/sting/utils/Utils.java @@ -312,6 +312,44 @@ public class Utils { return new Pair(max, index); } + /** + * Splits expressions in command args by spaces and returns the array of expressions. + * Expressions may use single or double quotes to group any individual expression, but not both. + * @param args Arguments to parse. + * @return Parsed expressions. + */ + public static String[] escapeExpressions(String args) { + // special case for ' and " so we can allow expressions + if (args.indexOf('\'') != -1) + return escapeExpressions(args, "'"); + else if (args.indexOf('\"') != -1) + return escapeExpressions(args, "\""); + else + return args.split(" "); + } + + /** + * Splits expressions in command args by spaces and the supplied delimiter and returns the array of expressions. + * @param args Arguments to parse. + * @param delimiter Delimiter for grouping expressions. + * @return Parsed expressions. + */ + private static String[] escapeExpressions(String args, String delimiter) { + String[] command = {}; + String[] split = args.split(delimiter); + for (int i = 0; i < split.length - 1; i += 2) { + command = Utils.concatArrays(command, split[i].trim().split(" ")); + command = Utils.concatArrays(command, new String[]{split[i + 1]}); + } + return Utils.concatArrays(command, split[split.length - 1].trim().split(" ")); + } + + /** + * Concatenates two String arrays. + * @param A First array. + * @param B Second array. + * @return Concatenation of A then B. + */ public static String[] concatArrays(String[] A, String[] B) { String[] C = new String[A.length + B.length]; System.arraycopy(A, 0, C, 0, A.length); @@ -319,8 +357,22 @@ public class Utils { return C; } + /** + * Appends String(s) B to array A. + * @param A First array. + * @param B Strings to append. + * @return A with B(s) appended. + */ + public static String[] appendArray(String[] A, String... B) { + return concatArrays(A, B); + } - /** Returns indices of all occurrences of the specified symbol in the string */ + /** + * Returns indices of all occurrences of the specified symbol in the string + * @param s Search string + * @param ch Character to search for + * @return Indices of all occurrences of the specified symbol + */ public static int[] indexOfAll(String s, int ch) { int[] pos = new int[64]; int z = 0; @@ -339,10 +391,10 @@ public class Utils { * with zeros up to the new size. Finally, if new size is the same as original size, no memory reallocation * will be performed and the original array will be returned instead. * - * @param orig - * @param newSize + * @param orig Original size. + * @param newSize New Size. * - * @return + * @return New array with length equal to newSize. */ public static int[] reallocate(int[] orig, int newSize) { if (orig.length == newSize) return orig; @@ -360,7 +412,8 @@ public class Utils { * array a. * @param a original array * @param n number of (v-filled) elements to append to a on the right (n>0) or on the left (n<0) - * @return + * @param v element value + * @return the extended copy of array a with additional n elements */ public static byte [] extend(final byte[] a, int n, byte v) { @@ -396,7 +449,8 @@ public class Utils { * array a. * @param a original array * @param n number of (v-filled) elements to append to a on the right (n>0) or on the left (n<0) - * @return + * @param v element value + * @return the extended copy of array a with additional n elements */ public static short [] extend(final short[] a, int n, short v) { diff --git a/java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java b/java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java index 83258aaf9..1aac9b565 100755 --- a/java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java +++ b/java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java @@ -26,15 +26,14 @@ package org.broadinstitute.sting.utils.classloader; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import org.reflections.util.ClasspathHelper; import java.lang.reflect.Modifier; import java.lang.reflect.Field; import java.io.File; import java.io.IOException; -import java.util.Collection; -import java.util.List; -import java.util.ArrayList; -import java.util.Arrays; +import java.net.URL; +import java.util.*; /** * Created by IntelliJ IDEA. @@ -178,4 +177,11 @@ public class JVMUtils { return selectedObjects; } + /** + * Returns the list of class path urls. + * @return the list of class path urls. + */ + public static Set getClasspathURLs() { + return ClasspathHelper.getUrlsForManifestsCurrentClasspath(); + } } diff --git a/java/src/org/broadinstitute/sting/utils/classloader/PackageUtils.java b/java/src/org/broadinstitute/sting/utils/classloader/PackageUtils.java deleted file mode 100755 index 7d49de573..000000000 --- a/java/src/org/broadinstitute/sting/utils/classloader/PackageUtils.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright (c) 2010 The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR - * THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.utils.classloader; - -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.Logger; -import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException; -import org.reflections.Reflections; -import org.reflections.scanners.SubTypesScanner; -import org.reflections.util.ClasspathHelper; -import org.reflections.util.ConfigurationBuilder; -import org.slf4j.LoggerFactory; - -import java.lang.reflect.Method; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.Set; -import java.util.ArrayList; -import java.util.List; - -/** - * PackageUtils contains some useful methods for package introspection. - */ -public class PackageUtils { - - /** - * A reference into our introspection utility. - */ - private static Reflections reflections = null; - - static { - // turn off logging in the reflections library - they talk too much (to the wrong logger factory as well, logback) - Logger logger = (ch.qos.logback.classic.Logger)LoggerFactory.getLogger(Reflections.class); - logger.setLevel(Level.OFF); - } - - /** - * Private constructor. No instantiating this class! - */ - private PackageUtils() {} - - /** - * Return the classes that implement the specified interface. - * - * @param iface the interface which returned classes should implement. - * @return the list of classes that implement the interface. - */ - public static List> getClassesImplementingInterface(Class iface) { - // Load all classes implementing the given interface, then filter out any class that isn't concrete. - initReflections(); - Set> allTypes = reflections.getSubTypesOf(iface); - List> concreteTypes = new ArrayList>(); - for( Class type: allTypes ) { - if( JVMUtils.isConcrete(type) ) - concreteTypes.add(type); - } - - return concreteTypes; - } - - public static List getInstancesOfClassesImplementingInterface(Class iface) { - List> classes = PackageUtils.getClassesImplementingInterface(iface); - List instances = new ArrayList(); - for ( Class c : classes ) - instances.add(getSimpleInstance(c)); - return instances; - } - - public static T getSimpleInstance(Class c) { - try { - return c.newInstance(); - } catch (Exception e) { - throw new DynamicClassResolutionException(c, e); - } - } - - /** - * Return the interface classes that extend the specified interface. - * - * @param iface the interface which returned classes should extend. - * @return the list of interface classes that implement the interface. - */ - public static List> getInterfacesExtendingInterface(Class iface) { - // Load all classes extending the given interface, then filter out any class that is concrete. - initReflections(); - Set> allTypes = reflections.getSubTypesOf(iface); - List> nonConcreteTypes = new ArrayList>(); - for( Class type: allTypes ) { - if( !JVMUtils.isConcrete(type) ) - nonConcreteTypes.add(type); - } - - return nonConcreteTypes; - } - - public static Set getClassPathURLs() { - return ClasspathHelper.getUrlsForManifestsCurrentClasspath(); - } - - /** - * Adds the URL to the system class loader classpath using reflection. - * HACK: Uses reflection to modify the class path, and assumes loader is a URLClassLoader. - * @param url URL to add to the system class loader classpath. - */ - public static void addClasspath(URL url) { - try { - Method method = URLClassLoader.class.getDeclaredMethod("addURL", URL.class); - if (!method.isAccessible()) - method.setAccessible(true); - method.invoke(ClassLoader.getSystemClassLoader(), url); - resetReflections(); - } catch (Exception e) { - throw new ReviewedStingException("Error adding url to the current classloader.", e); - } - } - - /** - * Create new reflections object if it does not currently exists. - */ - private static void initReflections() { - if (reflections == null) { - // Initialize general-purpose source tree reflector. - reflections = new Reflections( new ConfigurationBuilder() - .setUrls(getClassPathURLs()) - .setScanners(new SubTypesScanner())); - } - } - - /** - * Resets the reflections object after a class has been dynamically added to the classpath. - */ - private static void resetReflections() { - reflections = null; - } -} diff --git a/java/src/org/broadinstitute/sting/utils/classloader/PluginManager.java b/java/src/org/broadinstitute/sting/utils/classloader/PluginManager.java index 463e25cd3..c34f3c75d 100644 --- a/java/src/org/broadinstitute/sting/utils/classloader/PluginManager.java +++ b/java/src/org/broadinstitute/sting/utils/classloader/PluginManager.java @@ -25,20 +25,42 @@ package org.broadinstitute.sting.utils.classloader; -import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; +import org.reflections.Reflections; +import org.reflections.scanners.SubTypesScanner; +import org.reflections.util.ConfigurationBuilder; +import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.*; /** * Manage plugins and plugin configuration. * @author mhanna * @version 0.1 */ -public abstract class PluginManager { +public class PluginManager { + /** + * A reference into our introspection utility. + */ + private static final Reflections defaultReflections; + + static { + // turn off logging in the reflections library - they talk too much (to the wrong logger factory as well, logback) + Logger logger = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Reflections.class); + logger.setLevel(Level.OFF); + + defaultReflections = new Reflections( new ConfigurationBuilder() + .setUrls(JVMUtils.getClasspathURLs()) + .setScanners(new SubTypesScanner())); + } + /** * Defines the category of plugin defined by the subclass. */ @@ -48,11 +70,31 @@ public abstract class PluginManager { * Define common strings to trim off the end of the name. */ protected final String pluginSuffix; - + /** * Plugins stored based on their name. */ - protected final Map> pluginsByName; + private SortedMap> pluginsByName = null; + + private List> plugins; + private List> interfaces; + + /** + * Create a new plugin manager. + * @param pluginType Core type for a plugin. + */ + public PluginManager(Class pluginType) { + this(pluginType, pluginType.getSimpleName().toLowerCase(), pluginType.getSimpleName(), null); + } + + /** + * Create a new plugin manager. + * @param pluginType Core type for a plugin. + * @param classpath Custom class path to search for classes. + */ + public PluginManager(Class pluginType, List classpath) { + this(pluginType, pluginType.getSimpleName().toLowerCase(), pluginType.getSimpleName(), classpath); + } /** * Create a new plugin manager. @@ -60,11 +102,75 @@ public abstract class PluginManager { * @param pluginCategory Provides a category name to the plugin. Must not be null. * @param pluginSuffix Provides a suffix that will be trimmed off when converting to a plugin name. Can be null. */ - protected PluginManager(Class pluginType, String pluginCategory, String pluginSuffix) { + public PluginManager(Class pluginType, String pluginCategory, String pluginSuffix) { + this(pluginType, pluginCategory, pluginSuffix, null); + } + + /** + * Create a new plugin manager. + * @param pluginType Core type for a plugin. + * @param pluginCategory Provides a category name to the plugin. Must not be null. + * @param pluginSuffix Provides a suffix that will be trimmed off when converting to a plugin name. Can be null. + * @param classpath Custom class path to search for classes. + */ + public PluginManager(Class pluginType, String pluginCategory, String pluginSuffix, List classpath) { this.pluginCategory = pluginCategory; this.pluginSuffix = pluginSuffix; - List> plugins = PackageUtils.getClassesImplementingInterface(pluginType); - pluginsByName = createPluginDatabase(plugins); + + this.plugins = new ArrayList>(); + this.interfaces = new ArrayList>(); + + Reflections reflections; + if (classpath == null) { + reflections = defaultReflections; + } else { + addClasspath(classpath); + reflections = new Reflections( new ConfigurationBuilder() + .setUrls(classpath) + .setScanners(new SubTypesScanner())); + } + + // Load all classes types filtering them by concrete. + Set> allTypes = reflections.getSubTypesOf(pluginType); + for( Class type: allTypes ) { + if( JVMUtils.isConcrete(type) ) + plugins.add(type); + else + interfaces.add(type); + } + } + + /** + * Adds the URL to the system class loader classpath using reflection. + * HACK: Uses reflection to modify the class path, and assumes loader is a URLClassLoader. + * @param urls URLs to add to the system class loader classpath. + */ + private static void addClasspath(List urls) { + Set existing = JVMUtils.getClasspathURLs(); + for (URL url : urls) { + if (existing.contains(url)) + continue; + try { + Method method = URLClassLoader.class.getDeclaredMethod("addURL", URL.class); + if (!method.isAccessible()) + method.setAccessible(true); + method.invoke(ClassLoader.getSystemClassLoader(), url); + } catch (Exception e) { + throw new ReviewedStingException("Error adding url to the current classloader.", e); + } + } + } + + protected SortedMap> getPluginsByName() { + if (pluginsByName == null) { + SortedMap> newPlugins = new TreeMap>(); + for (Class pluginClass : plugins) { + String pluginName = getName(pluginClass); + newPlugins.put(pluginName, pluginClass); + } + pluginsByName = newPlugins; + } + return pluginsByName; } /** @@ -74,9 +180,49 @@ public abstract class PluginManager { * @return True if the plugin exists, false otherwise. */ public boolean exists(String pluginName) { - return pluginsByName.containsKey(pluginName); + return getPluginsByName().containsKey(pluginName); } + /** + * Does a plugin with the given name exist? + * + * @param plugin Name of the plugin for which to search. + * @return True if the plugin exists, false otherwise. + */ + public boolean exists(Class plugin) { + return getPluginsByName().containsValue(plugin); + } + + /** + * Returns the plugin classes + * @return the plugin classes + */ + public List> getPlugins() { + return plugins; + } + + /** + * Returns the interface classes + * @return the interface classes + */ + public List> getInterfaces() { + return interfaces; + } + + /** + * Returns the plugin classes implementing interface or base clase + * @param type type of interface or base class + * @return the plugin classes implementing interface or base class + */ + public List> getPluginsImplementing(Class type) { + List> implementing = new ArrayList>(); + for (Class plugin: getPlugins()) + if (type.isAssignableFrom(plugin)) + implementing.add(plugin); + return implementing; + } + + /** * Gets a plugin with the given name @@ -85,7 +231,7 @@ public abstract class PluginManager { * @return The plugin object if found; null otherwise. */ public PluginType createByName(String pluginName) { - Class plugin = pluginsByName.get(pluginName); + Class plugin = getPluginsByName().get(pluginName); if( plugin == null ) throw new UserException(String.format("Could not find %s with name: %s", pluginCategory,pluginName)); try { @@ -101,6 +247,7 @@ public abstract class PluginManager { * @param pluginType type of the plugin to create. * @return The plugin object if created; null otherwise. */ + @SuppressWarnings("unchecked") public PluginType createByType(Class pluginType) { try { return ((Class) pluginType).newInstance(); @@ -110,20 +257,19 @@ public abstract class PluginManager { } /** - * Create the list of available plugins and add them to the database. - * - * @param pluginClasses Classes to record. - * @return map of plugin name -> plugin. + * Returns concrete instances of the plugins + * @return concrete instances of the plugins */ - private Map> createPluginDatabase(List> pluginClasses) { - Map> plugins = new HashMap>(); - - for (Class pluginClass : pluginClasses) { - String pluginName = getName(pluginClass); - plugins.put(pluginName, pluginClass); + public List createAllTypes() { + List instances = new ArrayList(); + for ( Class c : getPlugins() ) { + try { + instances.add(c.newInstance()); + } catch (Exception e) { + throw new DynamicClassResolutionException(c, e); + } } - - return plugins; + return instances; } /** diff --git a/java/src/org/broadinstitute/sting/utils/interval/IntervalUtils.java b/java/src/org/broadinstitute/sting/utils/interval/IntervalUtils.java index a2967d455..7b10ccf86 100644 --- a/java/src/org/broadinstitute/sting/utils/interval/IntervalUtils.java +++ b/java/src/org/broadinstitute/sting/utils/interval/IntervalUtils.java @@ -1,8 +1,12 @@ package org.broadinstitute.sting.utils.interval; +import net.sf.picard.util.IntervalList; +import net.sf.samtools.SAMFileHeader; +import org.broadinstitute.sting.gatk.datasources.simpleDataSources.ReferenceDataSource; import org.broadinstitute.sting.utils.GenomeLocSortedSet; import org.broadinstitute.sting.utils.GenomeLoc; import org.broadinstitute.sting.utils.GenomeLocParser; +import org.broadinstitute.sting.utils.Utils; import org.broadinstitute.sting.utils.exceptions.UserException; import java.util.LinkedList; @@ -25,6 +29,7 @@ public class IntervalUtils { * 'all' can be supplied to indicate all possible intervals, but 'all' must be exclusive of all other interval * specifications. * + * @param parser Genome loc parser. * @param argList A list of strings containing interval data. * @param allowEmptyIntervalList If false instead of an empty interval list will return null. * @return an unsorted, unmerged representation of the given intervals. Null is used to indicate that all intervals should be used. @@ -107,7 +112,7 @@ public class IntervalUtils { if (setOne.get(iOne).getStop() < setTwo.get(iTwo).getStop()) iOne++; else iTwo++; } - + // we don't need to add the rest of remaining locations, since we know they don't overlap. return what we have return retList; } @@ -117,6 +122,7 @@ public class IntervalUtils { * all overlapping and abutting intervals into an interval that spans the union of all covered bases, and * OVERLAPPING_ONLY, which unions overlapping intervals but keeps abutting intervals separate. * + * @param parser Genome loc parser for the intervals. * @param intervals A collection of intervals to merge. * @param mergingRule A descriptor for the type of merging to perform. * @return A sorted, merged version of the intervals passed in. @@ -158,6 +164,131 @@ public class IntervalUtils { else return false; } + /** + * Returns the list of GenomeLocs from the list of intervals. + * @param referenceSource The reference for the intervals. + * @param intervals The interval as strings or file paths. + * @return The list of GenomeLocs. + */ + private static List parseIntervalArguments(ReferenceDataSource referenceSource, List intervals) { + GenomeLocParser parser = new GenomeLocParser(referenceSource.getReference()); + GenomeLocSortedSet locs; + // TODO: Abstract genome analysis engine has richer logic for parsing. We need to use it! + if (intervals.size() == 0) { + locs = GenomeLocSortedSet.createSetFromSequenceDictionary(referenceSource.getReference().getSequenceDictionary()); + } else { + locs = new GenomeLocSortedSet(parser, IntervalUtils.parseIntervalArguments(parser, intervals, false)); + } + if (locs == null || locs.size() == 0) + throw new UserException.MalformedFile("Intervals are empty: " + Utils.join(", ", intervals)); + return locs.toList(); + } + + /** + * Returns the list of contigs from the list of intervals. + * @param reference The reference for the intervals. + * @return The list of contig names. + */ + public static List distinctContigs(File reference) { + return distinctContigs(reference, Collections.emptyList()); + } + + /** + * Returns the list of contigs from the list of intervals. + * @param reference The reference for the intervals. + * @param intervals The interval as strings or file paths. + * @return The list of contig names. + */ + public static List distinctContigs(File reference, List intervals) { + ReferenceDataSource referenceSource = new ReferenceDataSource(reference); + List locs = parseIntervalArguments(referenceSource, intervals); + String contig = null; + List contigs = new ArrayList(); + for (GenomeLoc loc: locs) { + if (contig == null || !contig.equals(loc.getContig())) { + contig = loc.getContig(); + contigs.add(contig); + } + } + return contigs; + } + + /** + * Splits an interval list into multiple files. + * @param reference The reference for the intervals. + * @param intervals The interval as strings or file paths. + * @param scatterParts The output interval lists to write to. + * @param splitByContig If true then one contig will not be written to multiple files. + */ + public static void scatterIntervalArguments(File reference, List intervals, List scatterParts, boolean splitByContig) { + ReferenceDataSource referenceSource = new ReferenceDataSource(reference); + List locs = parseIntervalArguments(referenceSource, intervals); + SAMFileHeader fileHeader = new SAMFileHeader(); + fileHeader.setSequenceDictionary(referenceSource.getReference().getSequenceDictionary()); + + IntervalList intervalList = null; + int fileIndex = -1; + int locIndex = 0; + + if (splitByContig) { + String contig = null; + for (GenomeLoc loc: locs) { + // If there are still more files to write and the contig doesn't match... + if ((fileIndex+1 < scatterParts.size()) && (contig == null || !contig.equals(loc.getContig()))) { + // Then close the current file and start a new one. + if (intervalList != null) { + intervalList.write(scatterParts.get(fileIndex)); + intervalList = null; + } + fileIndex++; + contig = loc.getContig(); + } + if (intervalList == null) + intervalList = new IntervalList(fileHeader); + intervalList.add(toInterval(loc, ++locIndex)); + } + if (intervalList != null) + intervalList.write(scatterParts.get(fileIndex)); + } else { + int locsPerFile = locs.size() / scatterParts.size(); + int locRemainder = locs.size() % scatterParts.size(); + + // At the start, put an extra loc per file + locsPerFile++; + int locsLeftFile = 0; + + for (GenomeLoc loc: locs) { + if (locsLeftFile == 0) { + if (intervalList != null) + intervalList.write(scatterParts.get(fileIndex)); + + fileIndex++; + intervalList = new IntervalList(fileHeader); + + // When we have put enough locs into each file, + // reduce the number of locs per file back + // to the original calculated value. + if (fileIndex == locRemainder) + locsPerFile -= 1; + locsLeftFile = locsPerFile; + } + locsLeftFile -= 1; + intervalList.add(toInterval(loc, ++locIndex)); + } + if (intervalList != null) + intervalList.write(scatterParts.get(fileIndex)); + } + if ((fileIndex + 1) != scatterParts.size()) + throw new UserException.BadArgumentValue("scatterParts", String.format("Only able to write contigs into %d of %d files.", fileIndex + 1, scatterParts.size())); + } + + /** + * Converts a GenomeLoc to a picard interval. + * @param loc The GenomeLoc. + * @param locIndex The loc index for use in the file. + * @return The picard interval. + */ + private static net.sf.picard.util.Interval toInterval(GenomeLoc loc, int locIndex) { + return new net.sf.picard.util.Interval(loc.getContig(), loc.getStart(), loc.getStop(), false, "interval_" + locIndex); + } } - - diff --git a/java/test/org/broadinstitute/sting/WalkerTest.java b/java/test/org/broadinstitute/sting/WalkerTest.java index e8712fcb0..22a5180d5 100755 --- a/java/test/org/broadinstitute/sting/WalkerTest.java +++ b/java/test/org/broadinstitute/sting/WalkerTest.java @@ -311,35 +311,19 @@ public class WalkerTest extends BaseTest { */ private Pair, List> executeTest(String name, List md5s, List tmpFiles, String args, Class expectedException) { CommandLineGATK instance = new CommandLineGATK(); - String[] command; + String[] command = Utils.escapeExpressions(args); - // special case for ' and " so we can allow expressions - if (args.indexOf('\'') != -1) - command = escapeExpressions(args, "'"); - else if (args.indexOf('\"') != -1) - command = escapeExpressions(args, "\""); - else - command = args.split(" "); - - if (outputFileLocation != null) { - String[] cmd2 = Arrays.copyOf(command, command.length + 2); - cmd2[command.length] = "-o"; - cmd2[command.length + 1] = this.outputFileLocation.getAbsolutePath(); - command = cmd2; - } + if (outputFileLocation != null) + command = Utils.appendArray(command, "-o", this.outputFileLocation.getAbsolutePath()); // add the logging level to each of the integration test commands - String[] cmd2 = Arrays.copyOf(command, command.length + 4); - cmd2[command.length] = "-l"; - cmd2[command.length+1] = "WARN"; - cmd2[command.length+2] = "-et"; - cmd2[command.length+3] = ENABLE_REPORTING ? "STANDARD" : "NO_ET"; + command = Utils.appendArray(command, "-l", "WARN", "-et", ENABLE_REPORTING ? "STANDARD" : "NO_ET"); // run the executable boolean gotAnException = false; try { - System.out.println(String.format("Executing test %s with GATK arguments: %s", name, Utils.join(" ",cmd2))); - CommandLineExecutable.start(instance, cmd2); + System.out.println(String.format("Executing test %s with GATK arguments: %s", name, Utils.join(" ",command))); + CommandLineExecutable.start(instance, command); } catch (Exception e) { gotAnException = true; if ( expectedException != null ) { @@ -370,25 +354,11 @@ public class WalkerTest extends BaseTest { throw new RuntimeException("Error running the GATK with arguments: " + args); } - // clean up some memory - instance = null; - cmd2 = null; - // we need to check MD5s return new Pair, List>(tmpFiles, assertMatchingMD5s(name, tmpFiles, md5s)); } } - private static String[] escapeExpressions(String args, String delimiter) { - String[] command = {}; - String[] split = args.split(delimiter); - for (int i = 0; i < split.length - 1; i += 2) { - command = Utils.concatArrays(command, split[i].trim().split(" ")); - command = Utils.concatArrays(command, new String[]{split[i + 1]}); - } - return Utils.concatArrays(command, split[split.length - 1].trim().split(" ")); - } - @Test public void testWalkerUnitTest() { //System.out.println("WalkerTest is just a framework"); diff --git a/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsTest.java b/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsTest.java deleted file mode 100644 index d4fcb8b9f..000000000 --- a/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsTest.java +++ /dev/null @@ -1,91 +0,0 @@ -package org.broadinstitute.sting.utils.interval; - -import net.sf.picard.reference.IndexedFastaSequenceFile; -import net.sf.picard.reference.ReferenceSequenceFile; -import org.broadinstitute.sting.BaseTest; -import org.testng.Assert; -import org.broadinstitute.sting.utils.GenomeLoc; -import org.broadinstitute.sting.utils.GenomeLocParser; - -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -import java.io.File; -import java.io.FileNotFoundException; -import java.util.ArrayList; -import java.util.List; - -/** - * test out the interval utility methods - */ -public class IntervalUtilsTest extends BaseTest { - // used to seed the genome loc parser with a sequence dictionary - private static ReferenceSequenceFile seq; - private GenomeLocParser genomeLocParser; - - @BeforeClass - public void init() throws FileNotFoundException { - seq = new IndexedFastaSequenceFile(new File(hg18Reference)); - genomeLocParser = new GenomeLocParser(seq); - - } - - @Test - public void testMergeListsBySetOperatorNoOverlap() { - // a couple of lists we'll use for the testing - List listEveryTwoFromOne = new ArrayList(); - List listEveryTwoFromTwo = new ArrayList(); - - // create the two lists we'll use - for (int x = 1; x < 101; x++) { - if (x % 2 == 0) - listEveryTwoFromTwo.add(genomeLocParser.createGenomeLoc("chr1",x,x)); - else - listEveryTwoFromOne.add(genomeLocParser.createGenomeLoc("chr1",x,x)); - } - - List ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, listEveryTwoFromOne, IntervalSetRule.UNION); - Assert.assertEquals(ret.size(), 100); - ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, listEveryTwoFromOne, IntervalSetRule.INTERSECTION); - Assert.assertEquals(ret.size(), 0); - } - - @Test - public void testMergeListsBySetOperatorAllOverlap() { - // a couple of lists we'll use for the testing - List allSites = new ArrayList(); - List listEveryTwoFromTwo = new ArrayList(); - - // create the two lists we'll use - for (int x = 1; x < 101; x++) { - if (x % 2 == 0) - listEveryTwoFromTwo.add(genomeLocParser.createGenomeLoc("chr1",x,x)); - allSites.add(genomeLocParser.createGenomeLoc("chr1",x,x)); - } - - List ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.UNION); - Assert.assertEquals(ret.size(), 150); - ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.INTERSECTION); - Assert.assertEquals(ret.size(), 50); - } - - @Test - public void testMergeListsBySetOperator() { - // a couple of lists we'll use for the testing - List allSites = new ArrayList(); - List listEveryTwoFromTwo = new ArrayList(); - - // create the two lists we'll use - for (int x = 1; x < 101; x++) { - if (x % 5 == 0) { - listEveryTwoFromTwo.add(genomeLocParser.createGenomeLoc("chr1",x,x)); - allSites.add(genomeLocParser.createGenomeLoc("chr1",x,x)); - } - } - - List ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.UNION); - Assert.assertEquals(ret.size(), 40); - ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.INTERSECTION); - Assert.assertEquals(ret.size(), 20); - } -} diff --git a/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsUnitTest.java b/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsUnitTest.java new file mode 100644 index 000000000..9bd541866 --- /dev/null +++ b/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsUnitTest.java @@ -0,0 +1,356 @@ +package org.broadinstitute.sting.utils.interval; + +import net.sf.picard.reference.IndexedFastaSequenceFile; +import net.sf.picard.reference.ReferenceSequenceFile; +import org.broadinstitute.sting.BaseTest; +import org.testng.Assert; +import org.broadinstitute.sting.utils.exceptions.UserException; +import org.broadinstitute.sting.utils.GenomeLoc; +import org.broadinstitute.sting.utils.GenomeLocParser; + +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * test out the interval utility methods + */ +public class IntervalUtilsUnitTest extends BaseTest { + // used to seed the genome loc parser with a sequence dictionary + private static File reference = new File(BaseTest.hg18Reference); + private GenomeLocParser genomeLocParser; + + @BeforeClass + public void init() { + ReferenceSequenceFile seq = new IndexedFastaSequenceFile(reference); + genomeLocParser = new GenomeLocParser(seq); + } + + @Test + public void testMergeListsBySetOperatorNoOverlap() { + // a couple of lists we'll use for the testing + List listEveryTwoFromOne = new ArrayList(); + List listEveryTwoFromTwo = new ArrayList(); + + // create the two lists we'll use + for (int x = 1; x < 101; x++) { + if (x % 2 == 0) + listEveryTwoFromTwo.add(genomeLocParser.createGenomeLoc("chr1",x,x)); + else + listEveryTwoFromOne.add(genomeLocParser.createGenomeLoc("chr1",x,x)); + } + + List ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, listEveryTwoFromOne, IntervalSetRule.UNION); + Assert.assertEquals(ret.size(), 100); + ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, listEveryTwoFromOne, IntervalSetRule.INTERSECTION); + Assert.assertEquals(ret.size(), 0); + } + + @Test + public void testMergeListsBySetOperatorAllOverlap() { + // a couple of lists we'll use for the testing + List allSites = new ArrayList(); + List listEveryTwoFromTwo = new ArrayList(); + + // create the two lists we'll use + for (int x = 1; x < 101; x++) { + if (x % 2 == 0) + listEveryTwoFromTwo.add(genomeLocParser.createGenomeLoc("chr1",x,x)); + allSites.add(genomeLocParser.createGenomeLoc("chr1",x,x)); + } + + List ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.UNION); + Assert.assertEquals(ret.size(), 150); + ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.INTERSECTION); + Assert.assertEquals(ret.size(), 50); + } + + @Test + public void testMergeListsBySetOperator() { + // a couple of lists we'll use for the testing + List allSites = new ArrayList(); + List listEveryTwoFromTwo = new ArrayList(); + + // create the two lists we'll use + for (int x = 1; x < 101; x++) { + if (x % 5 == 0) { + listEveryTwoFromTwo.add(genomeLocParser.createGenomeLoc("chr1",x,x)); + allSites.add(genomeLocParser.createGenomeLoc("chr1",x,x)); + } + } + + List ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.UNION); + Assert.assertEquals(ret.size(), 40); + ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.INTERSECTION); + Assert.assertEquals(ret.size(), 20); + } + + @Test + public void testCountContigs() { + List chrs = new ArrayList(); + for (int i = 1; i <= 22; i++) + chrs.add("chr" + i); + chrs.add("chrX"); + chrs.add("chrY"); + + List chrsNoRandom = Arrays.asList("chr12", "chr14", "chr20", "chrY"); + List chrsWithRandom = new ArrayList(); + chrsWithRandom.add("chrM"); + chrsWithRandom.addAll(chrs); + for (String chr: chrs) + if(!chrsNoRandom.contains(chr)) + chrsWithRandom.add(chr + "_random"); + + Assert.assertEquals(IntervalUtils.distinctContigs(reference), chrsWithRandom); + Assert.assertEquals(IntervalUtils.distinctContigs(reference, Arrays.asList(BaseTest.validationDataLocation + "TCGA-06-0188.interval_list")), chrs); + Assert.assertEquals(IntervalUtils.distinctContigs(reference, Arrays.asList("chr1:1-1", "chr2:1-1", "chr3:2-2")), Arrays.asList("chr1","chr2","chr3")); + Assert.assertEquals(IntervalUtils.distinctContigs(reference, Arrays.asList("chr2:1-1", "chr1:1-1", "chr3:2-2")), Arrays.asList("chr1","chr2","chr3")); + } + + @Test + public void testBasicScatter() { + GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1"); + GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2"); + GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3"); + + List files = testFiles("basic.", 3, ".intervals"); + + IntervalUtils.scatterIntervalArguments(reference, Arrays.asList("chr1", "chr2", "chr3"), files, false); + + List locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false); + List locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false); + List locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false); + + Assert.assertEquals(locs1.size(), 1); + Assert.assertEquals(locs2.size(), 1); + Assert.assertEquals(locs3.size(), 1); + + Assert.assertEquals(locs1.get(0), chr1); + Assert.assertEquals(locs2.get(0), chr2); + Assert.assertEquals(locs3.get(0), chr3); + } + + @Test + public void testScatterLessFiles() { + GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1"); + GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2"); + GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3"); + GenomeLoc chr4 = genomeLocParser.parseGenomeInterval("chr4"); + + List files = testFiles("less.", 3, ".intervals"); + + IntervalUtils.scatterIntervalArguments(reference, Arrays.asList("chr1", "chr2", "chr3", "chr4"), files, false); + + List locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false); + List locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false); + List locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false); + + Assert.assertEquals(locs1.size(), 2); + Assert.assertEquals(locs2.size(), 1); + Assert.assertEquals(locs3.size(), 1); + + Assert.assertEquals(locs1.get(0), chr1); + Assert.assertEquals(locs1.get(1), chr2); + Assert.assertEquals(locs2.get(0), chr3); + Assert.assertEquals(locs3.get(0), chr4); + } + + @Test(expectedExceptions=UserException.BadArgumentValue.class) + public void testScatterMoreFiles() { + List files = testFiles("more.", 3, ".intervals"); + IntervalUtils.scatterIntervalArguments(reference, Arrays.asList("chr1", "chr2"), files, false); + } + + @Test + public void testScatterIntervals() { + List intervals = Arrays.asList("chr1:1-2", "chr1:4-5", "chr2:1-1", "chr3:2-2"); + GenomeLoc chr1a = genomeLocParser.parseGenomeInterval("chr1:1-2"); + GenomeLoc chr1b = genomeLocParser.parseGenomeInterval("chr1:4-5"); + GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2:1-1"); + GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3:2-2"); + + List files = testFiles("split.", 3, ".intervals"); + + IntervalUtils.scatterIntervalArguments(reference, intervals, files, true); + + List locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false); + List locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false); + List locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false); + + Assert.assertEquals(locs1.size(), 2); + Assert.assertEquals(locs2.size(), 1); + Assert.assertEquals(locs3.size(), 1); + + Assert.assertEquals(locs1.get(0), chr1a); + Assert.assertEquals(locs1.get(1), chr1b); + Assert.assertEquals(locs2.get(0), chr2); + Assert.assertEquals(locs3.get(0), chr3); + } + + @Test + public void testScatterOrder() { + List intervals = Arrays.asList("chr2:1-1", "chr1:1-1", "chr3:2-2"); + GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1:1-1"); + GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2:1-1"); + GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3:2-2"); + + List files = testFiles("split.", 3, ".intervals"); + + IntervalUtils.scatterIntervalArguments(reference, intervals, files, true); + + List locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false); + List locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false); + List locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false); + + Assert.assertEquals(locs1.size(), 1); + Assert.assertEquals(locs2.size(), 1); + Assert.assertEquals(locs3.size(), 1); + + Assert.assertEquals(locs1.get(0), chr1); + Assert.assertEquals(locs2.get(0), chr2); + Assert.assertEquals(locs3.get(0), chr3); + } + + @Test + public void testBasicScatterByContig() { + GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1"); + GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2"); + GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3"); + + List files = testFiles("contig_basic.", 3, ".intervals"); + + IntervalUtils.scatterIntervalArguments(reference, Arrays.asList("chr1", "chr2", "chr3"), files, true); + + List locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false); + List locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false); + List locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false); + + Assert.assertEquals(locs1.size(), 1); + Assert.assertEquals(locs2.size(), 1); + Assert.assertEquals(locs3.size(), 1); + + Assert.assertEquals(locs1.get(0), chr1); + Assert.assertEquals(locs2.get(0), chr2); + Assert.assertEquals(locs3.get(0), chr3); + } + + @Test + public void testScatterByContigLessFiles() { + GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1"); + GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2"); + GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3"); + GenomeLoc chr4 = genomeLocParser.parseGenomeInterval("chr4"); + + List files = testFiles("contig_less.", 3, ".intervals"); + + IntervalUtils.scatterIntervalArguments(reference, Arrays.asList("chr1", "chr2", "chr3", "chr4"), files, true); + + List locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false); + List locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false); + List locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false); + + Assert.assertEquals(locs1.size(), 1); + Assert.assertEquals(locs2.size(), 1); + Assert.assertEquals(locs3.size(), 2); + + Assert.assertEquals(locs1.get(0), chr1); + Assert.assertEquals(locs2.get(0), chr2); + Assert.assertEquals(locs3.get(0), chr3); + Assert.assertEquals(locs3.get(1), chr4); + } + + @Test(expectedExceptions=UserException.BadArgumentValue.class) + public void testScatterByContigMoreFiles() { + List files = testFiles("contig_more.", 3, ".intervals"); + IntervalUtils.scatterIntervalArguments(reference, Arrays.asList("chr1", "chr2"), files, true); + } + + @Test + public void testScatterByContigIntervalsStart() { + List intervals = Arrays.asList("chr1:1-2", "chr1:4-5", "chr2:1-1", "chr3:2-2"); + GenomeLoc chr1a = genomeLocParser.parseGenomeInterval("chr1:1-2"); + GenomeLoc chr1b = genomeLocParser.parseGenomeInterval("chr1:4-5"); + GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2:1-1"); + GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3:2-2"); + + List files = testFiles("contig_split_start.", 3, ".intervals"); + + IntervalUtils.scatterIntervalArguments(reference, intervals, files, true); + + List locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false); + List locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false); + List locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false); + + Assert.assertEquals(locs1.size(), 2); + Assert.assertEquals(locs2.size(), 1); + Assert.assertEquals(locs3.size(), 1); + + Assert.assertEquals(locs1.get(0), chr1a); + Assert.assertEquals(locs1.get(1), chr1b); + Assert.assertEquals(locs2.get(0), chr2); + Assert.assertEquals(locs3.get(0), chr3); + } + + @Test + public void testScatterByContigIntervalsMiddle() { + List intervals = Arrays.asList("chr1:1-1", "chr2:1-2", "chr2:4-5", "chr3:2-2"); + GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1:1-1"); + GenomeLoc chr2a = genomeLocParser.parseGenomeInterval("chr2:1-2"); + GenomeLoc chr2b = genomeLocParser.parseGenomeInterval("chr2:4-5"); + GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3:2-2"); + + List files = testFiles("contig_split_middle.", 3, ".intervals"); + + IntervalUtils.scatterIntervalArguments(reference, intervals, files, true); + + List locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false); + List locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false); + List locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false); + + Assert.assertEquals(locs1.size(), 1); + Assert.assertEquals(locs2.size(), 2); + Assert.assertEquals(locs3.size(), 1); + + Assert.assertEquals(locs1.get(0), chr1); + Assert.assertEquals(locs2.get(0), chr2a); + Assert.assertEquals(locs2.get(1), chr2b); + Assert.assertEquals(locs3.get(0), chr3); + } + + @Test + public void testScatterByContigIntervalsEnd() { + List intervals = Arrays.asList("chr1:1-1", "chr2:2-2", "chr3:1-2", "chr3:4-5"); + GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1:1-1"); + GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2:2-2"); + GenomeLoc chr3a = genomeLocParser.parseGenomeInterval("chr3:1-2"); + GenomeLoc chr3b = genomeLocParser.parseGenomeInterval("chr3:4-5"); + + List files = testFiles("contig_split_end.", 3 ,".intervals"); + + IntervalUtils.scatterIntervalArguments(reference, intervals, files, true); + + List locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false); + List locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false); + List locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false); + + Assert.assertEquals(locs1.size(), 1); + Assert.assertEquals(locs2.size(), 1); + Assert.assertEquals(locs3.size(), 2); + + Assert.assertEquals(locs1.get(0), chr1); + Assert.assertEquals(locs2.get(0), chr2); + Assert.assertEquals(locs3.get(0), chr3a); + Assert.assertEquals(locs3.get(1), chr3b); + } + + private List testFiles(String prefix, int count, String suffix) { + ArrayList files = new ArrayList(); + for (int i = 1; i <= count; i++) + files.add(new File(testDir + prefix + i + suffix)); + return files; + } +} diff --git a/scala/qscript/fullCallingPipeline.q b/scala/qscript/fullCallingPipeline.q index 5aa9b89e9..ca907f997 100755 --- a/scala/qscript/fullCallingPipeline.q +++ b/scala/qscript/fullCallingPipeline.q @@ -9,6 +9,7 @@ import org.broadinstitute.sting.queue.function.scattergather.{GatherFunction, Cl import org.broadinstitute.sting.queue.util.IOUtils import org.broadinstitute.sting.queue.QScript import collection.JavaConversions._ +import org.broadinstitute.sting.utils.interval.IntervalUtils import org.broadinstitute.sting.utils.yaml.YamlUtils import org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType @@ -91,9 +92,9 @@ class fullCallingPipeline extends QScript { //val expKind = qscript.protocol // get contigs (needed for indel cleaning parallelism) - val contigs = IntervalScatterFunction.distinctContigs( + val contigs = IntervalUtils.distinctContigs( qscript.pipeline.getProject.getReferenceFile, - List(qscript.pipeline.getProject.getIntervalList.getAbsolutePath)) + List(qscript.pipeline.getProject.getIntervalList.getAbsolutePath)).toList for ( sample <- recalibratedSamples ) { val sampleId = sample.getId @@ -135,24 +136,24 @@ class fullCallingPipeline extends QScript { realigner.setupScatterFunction = { case scatter: ScatterFunction => scatter.commandDirectory = new File("CleanedBams/IntermediateFiles/%s/ScatterGather".format(sampleId)) - scatter.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/Cleaning/%s/Scatter.out".format(sampleId)) + scatter.jobOutputFile = new File(".queue/logs/Cleaning/%s/Scatter.out".format(sampleId)) } realigner.setupCloneFunction = { case (clone: CloneFunction, index: Int) => clone.commandDirectory = new File("CleanedBams/IntermediateFiles/%s/ScatterGather/Scatter_%s".format(sampleId, index)) - clone.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/Cleaning/%s/Scatter_%s.out".format(sampleId, index)) + clone.jobOutputFile = new File(".queue/logs/Cleaning/%s/Scatter_%s.out".format(sampleId, index)) } realigner.setupGatherFunction = { case (gather: BamGatherFunction, source: ArgumentSource) => gather.commandDirectory = new File("CleanedBams/IntermediateFiles/%s/ScatterGather/Gather_%s".format(sampleId, source.field.getName)) - gather.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/Cleaning/%s/FixMates.out".format(sampleId)) + gather.jobOutputFile = new File(".queue/logs/Cleaning/%s/FixMates.out".format(sampleId)) gather.memoryLimit = Some(6) gather.jarFile = qscript.picardFixMatesJar // Don't pass this AS=true to fix mates! gather.assumeSorted = None case (gather: GatherFunction, source: ArgumentSource) => gather.commandDirectory = new File("CleanedBams/IntermediateFiles/%s/ScatterGather/Gather_%s".format(sampleId, source.field.getName)) - gather.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/Cleaning/%s/Gather_%s.out".format(sampleId, source.field.getName)) + gather.jobOutputFile = new File(".queue/logs/Cleaning/%s/Gather_%s.out".format(sampleId, source.field.getName)) } add(targetCreator,realigner) @@ -229,17 +230,17 @@ class fullCallingPipeline extends QScript { snps.setupScatterFunction = { case scatter: ScatterFunction => scatter.commandDirectory = new File("SnpCalls/ScatterGather") - scatter.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/SNPCalling/ScatterGather/Scatter.out") + scatter.jobOutputFile = new File(".queue/logs/SNPCalling/ScatterGather/Scatter.out") } snps.setupCloneFunction = { case (clone: CloneFunction, index: Int) => clone.commandDirectory = new File("SnpCalls/ScatterGather/Scatter_%s".format(index)) - clone.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/SNPCalling/ScatterGather/Scatter_%s.out".format(index)) + clone.jobOutputFile = new File(".queue/logs/SNPCalling/ScatterGather/Scatter_%s.out".format(index)) } snps.setupGatherFunction = { case (gather: GatherFunction, source: ArgumentSource) => gather.commandDirectory = new File("SnpCalls/ScatterGather/Gather_%s".format(source.field.getName)) - gather.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/SNPCalling/ScatterGather/Gather_%s.out".format(source.field.getName)) + gather.jobOutputFile = new File(".queue/logs/SNPCalling/ScatterGather/Gather_%s.out".format(source.field.getName)) } // indel genotyper does one sample at a time diff --git a/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala b/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala index 50e4890c1..0a2a53a32 100755 --- a/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala +++ b/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala @@ -6,6 +6,8 @@ import java.util.Arrays import org.broadinstitute.sting.commandline._ import org.broadinstitute.sting.queue.util._ import org.broadinstitute.sting.queue.engine.{QGraphSettings, QGraph} +import collection.JavaConversions._ +import org.broadinstitute.sting.utils.classloader.PluginManager /** * Entry point of Queue. Compiles and runs QScripts passed in to the command line. @@ -18,21 +20,28 @@ class QCommandLine extends CommandLineProgram with Logging { @ArgumentCollection private val settings = new QGraphSettings - QFunction.parsingEngine = new ParsingEngine(this) + private val qScriptManager = new QScriptManager + private val qGraph = new QGraph + private var qScriptClasses: File = _ + + private lazy val pluginManager = { + qScriptClasses = IOUtils.tempDir("Q-Classes", "", settings.qSettings.tempDirectory) + qScriptManager.loadScripts(scripts, qScriptClasses) + new PluginManager[QScript](classOf[QScript], List(qScriptClasses.toURI.toURL)) + } + + QFunction.parsingEngine = new ParsingEngine(this) /** * Takes the QScripts passed in, runs their script() methods, retrieves their generated * functions, and then builds and runs a QGraph based on the dependencies. */ def execute = { - - val qGraph = QCommandLine.qGraph qGraph.settings = settings qGraph.debugMode = debugMode == true - val scripts = qScriptManager.createScripts() - for (script <- scripts) { - logger.info("Scripting " + qScriptManager.getName(script.getClass.asSubclass(classOf[QScript]))) + for (script <- pluginManager.createAllTypes()) { + logger.info("Scripting " + pluginManager.getName(script.getClass.asSubclass(classOf[QScript]))) loadArgumentsIntoObject(script) script.script script.functions.foreach(qGraph.add(_)) @@ -63,14 +72,14 @@ class QCommandLine extends CommandLineProgram with Logging { * @return Array of QScripts passed in. */ override def getArgumentSources = - qScriptManager.getValues.asInstanceOf[Array[Class[_]]] + pluginManager.getPlugins.toIterable.toArray.asInstanceOf[Array[Class[_]]] /** * Returns the name of a QScript * @return The name of a QScript */ override def getArgumentSourceName(source: Class[_]) = - qScriptManager.getName(source.asSubclass(classOf[QScript])) + pluginManager.getName(source.asSubclass(classOf[QScript])) /** * Returns a ScalaCompoundArgumentTypeDescriptor that can parse argument sources into scala collections. @@ -79,12 +88,9 @@ class QCommandLine extends CommandLineProgram with Logging { override def getArgumentTypeDescriptors = Arrays.asList(new ScalaCompoundArgumentTypeDescriptor) - /** - * Loads the QScripts passed in and returns a new QScriptManager than can be used to create them. - */ - private lazy val qScriptManager = { - QScriptManager.loadScripts(scripts) - new QScriptManager + def shutdown() = { + qGraph.shutdown() + if (qScriptClasses != null) IOUtils.tryDelete(qScriptClasses) } } @@ -92,29 +98,29 @@ class QCommandLine extends CommandLineProgram with Logging { * Entry point of Queue. Compiles and runs QScripts passed in to the command line. */ object QCommandLine { - private val qGraph = new QGraph - - /** * Main. * @param argv Arguments. */ def main(argv: Array[String]) { + val qCommandLine = new QCommandLine + Runtime.getRuntime.addShutdownHook(new Thread { /** Cleanup as the JVM shuts down. */ override def run = { - qGraph.shutdown() ProcessController.shutdown() - QScriptManager.deleteOutdir() + qCommandLine.shutdown() } }) try { - CommandLineProgram.start(new QCommandLine, argv); + CommandLineProgram.start(qCommandLine, argv); if (CommandLineProgram.result != 0) System.exit(CommandLineProgram.result); } catch { case e: Exception => CommandLineProgram.exitSystemWithError(e) + } finally { + } } } diff --git a/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala b/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala index 9dfbb481c..6af376810 100644 --- a/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala +++ b/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala @@ -3,62 +3,33 @@ package org.broadinstitute.sting.queue import scala.tools.nsc.{Global, Settings} import scala.tools.nsc.io.PlainFile import org.broadinstitute.sting.queue.util.{Logging, IOUtils} -import collection.JavaConversions +import collection.JavaConversions._ import java.io.File import scala.tools.nsc.reporters.AbstractReporter import java.lang.String import org.apache.log4j.Level import scala.tools.nsc.util.{FakePos, NoPosition, Position} -import org.broadinstitute.sting.utils.classloader.{PackageUtils, PluginManager} import org.broadinstitute.sting.queue.util.TextFormatUtils._ -import org.apache.commons.io.FileUtils +import org.broadinstitute.sting.utils.classloader.JVMUtils /** * Plugin manager for QScripts which loads QScripts into the current class loader. */ -class QScriptManager extends PluginManager[QScript](classOf[QScript], "QScript", "Script") with Logging { - - /** - * Returns the list of QScripts classes found in the classpath. - * @return QScripts classes found in the classpath. - */ - def getValues = { - if (logger.isDebugEnabled) { - JavaConversions.asMap(this.pluginsByName) - .foreach{case (name, clazz) => logger.debug("Found QScript %s: %s".format(name, clazz))} - } - JavaConversions.asIterable(this.pluginsByName.values).toArray - } - - /** - * Creates the QScripts for all values found in the classpath. - * @return QScripts found in the classpath. - */ - def createScripts() = getValues.map(_.newInstance.asInstanceOf[QScript]) -} - -/** - * Plugin manager for QScripts which loads QScripts into the current classloader. - */ -object QScriptManager extends Logging { - private val outdir = IOUtils.tempDir("Q-classes") - +class QScriptManager() extends Logging { /** * Compiles and loads the scripts in the files into the current classloader. * Heavily based on scala/src/compiler/scala/tools/ant/Scalac.scala - * @param scripts Scala classes to compile. */ - def loadScripts(scripts: List[File]) { + def loadScripts(scripts: List[File], tempDir: File) = { if (scripts.size > 0) { - val settings = new Settings((error: String) => logger.error(error)) settings.deprecation.value = true - settings.outdir.value = outdir.getPath + settings.outdir.value = tempDir.getPath // Set the classpath to the current class path. - JavaConversions.asSet(PackageUtils.getClassPathURLs).foreach(url => settings.classpath.append(url.getPath)) + JVMUtils.getClasspathURLs.foreach(url => settings.classpath.append(url.getPath)) - val reporter = new Log4JReporter(settings) + val reporter = new QScriptManager.Log4JReporter(settings) val compiler = new Global(settings, reporter) val run = new compiler.Run @@ -78,19 +49,14 @@ object QScriptManager extends Logging { reporter.WARNING.count, plural(reporter.WARNING.count))) else logger.info("Compilation complete") - - // Add the new compilation output directory to the classpath. - PackageUtils.addClasspath(outdir.toURI.toURL) } } +} - /** - * Removes the outdir cleaning up the temporary classes. - */ - def deleteOutdir() = { - if (FileUtils.deleteQuietly(outdir)) - logger.debug("Deleted " + outdir) - } +/** + * Plugin manager for QScripts which loads QScripts into the current classloader. + */ +object QScriptManager extends Logging { /** * NSC (New Scala Compiler) reporter which logs to Log4J. diff --git a/scala/src/org/broadinstitute/sting/queue/QSettings.scala b/scala/src/org/broadinstitute/sting/queue/QSettings.scala index 5ae9a7c77..b3dd6fb3c 100644 --- a/scala/src/org/broadinstitute/sting/queue/QSettings.scala +++ b/scala/src/org/broadinstitute/sting/queue/QSettings.scala @@ -23,6 +23,12 @@ class QSettings { @Argument(fullName="default_memory_limit", shortName="memLimit", doc="Default memory limit for jobs, in gigabytes.", required=false) var memoryLimit: Option[Int] = None + @Argument(fullName="run_directory", shortName="runDir", doc="Root directory to run functions from.", required=false) + var runDirectory = new File(".") + + @Argument(fullName="temp_directory", shortName="tempDir", doc="Temp directory to pass to functions.", required=false) + var tempDirectory = new File(System.getProperty("java.io.tmpdir")) + @ArgumentCollection val emailSettings = new EmailSettings } diff --git a/scala/src/org/broadinstitute/sting/queue/engine/LsfJobRunner.scala b/scala/src/org/broadinstitute/sting/queue/engine/LsfJobRunner.scala index 883b08477..085c66365 100644 --- a/scala/src/org/broadinstitute/sting/queue/engine/LsfJobRunner.scala +++ b/scala/src/org/broadinstitute/sting/queue/engine/LsfJobRunner.scala @@ -45,7 +45,7 @@ class LsfJobRunner(val function: CommandLineFunction) extends DispatchJobRunner job.project = function.jobProject job.queue = function.jobQueue - if (!IOUtils.CURRENT_DIR_ABS.equals(function.commandDirectory)) + if (function.commandDirectory != new File(".").getAbsoluteFile) job.workingDir = function.commandDirectory job.extraBsubArgs ++= function.extraArgs @@ -166,7 +166,7 @@ class LsfJobRunner(val function: CommandLineFunction) extends DispatchJobRunner * @return the file path to the pre-exec. */ private def writeExec() = { - IOUtils.writeTempFile(function.commandLine, ".exec", "") + IOUtils.writeTempFile(function.commandLine, ".exec", "", function.jobTempDir) } /** @@ -185,7 +185,7 @@ class LsfJobRunner(val function: CommandLineFunction) extends DispatchJobRunner mountCommand(function).foreach(command => preExec.append("%s%n".format(command))) - IOUtils.writeTempFile(preExec.toString, ".preExec", "") + IOUtils.writeTempFile(preExec.toString, ".preExec", "", function.jobTempDir) } /** @@ -211,6 +211,6 @@ class LsfJobRunner(val function: CommandLineFunction) extends DispatchJobRunner |fi |""".stripMargin.format(function.commandDirectory, touchDone, touchFail)) - IOUtils.writeTempFile(postExec.toString, ".postExec", "") + IOUtils.writeTempFile(postExec.toString, ".postExec", "", function.jobTempDir) } } diff --git a/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala b/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala index 5479ef535..c5b814e26 100755 --- a/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala +++ b/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala @@ -2,7 +2,6 @@ package org.broadinstitute.sting.queue.engine import org.jgrapht.traverse.TopologicalOrderIterator import org.jgrapht.graph.SimpleDirectedGraph -import scala.collection.JavaConversions import scala.collection.JavaConversions._ import org.jgrapht.alg.CycleDetector import org.jgrapht.EdgeFactory @@ -46,7 +45,7 @@ class QGraph extends Logging { * Checks the functions for missing values and the graph for cyclic dependencies and then runs the functions in the graph. */ def run = { - IOUtils.checkTempDir + IOUtils.checkTempDir(settings.qSettings.tempDirectory) val numMissingValues = fillGraph val isReady = numMissingValues == 0 @@ -158,7 +157,7 @@ class QGraph extends Logging { */ private def fillIn = { // clone since edgeSet is backed by the graph - JavaConversions.asSet(jobGraph.edgeSet).clone.foreach { + asScalaSet(jobGraph.edgeSet).clone.foreach { case cmd: FunctionEdge => { addCollectionOutputs(cmd.outputs) addCollectionInputs(cmd.inputs) @@ -205,7 +204,7 @@ class QGraph extends Logging { */ private def validate = { var numMissingValues = 0 - JavaConversions.asSet(jobGraph.edgeSet).foreach { + asScalaSet(jobGraph.edgeSet).foreach { case cmd: FunctionEdge => val missingFieldValues = cmd.function.missingFields if (missingFieldValues.size > 0) { diff --git a/scala/src/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunction.scala b/scala/src/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunction.scala index f444044bf..78206816a 100644 --- a/scala/src/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunction.scala @@ -48,105 +48,12 @@ class IntervalScatterFunction extends ScatterFunction with InProcessFunction { def bindCloneInputs(cloneFunction: CloneFunction, index: Int) = { val scatterPart = cloneFunction.getFieldValue(this.intervalsField) .asInstanceOf[List[File]] - .map(file => IOUtils.subDir(cloneFunction.commandDirectory, file)) + .map(file => IOUtils.absolute(cloneFunction.commandDirectory, file)) cloneFunction.setFieldValue(this.intervalsField, scatterPart) this.scatterParts ++= scatterPart } def run() = { - IntervalScatterFunction.scatter(this.referenceSequence, this.intervals, this.scatterParts, this.splitByContig) + IntervalUtils.scatterIntervalArguments(this.referenceSequence, this.intervals, this.scatterParts, this.splitByContig) } } - -object IntervalScatterFunction { - private def parseLocs(referenceSource: ReferenceDataSource, intervals: List[String]) = { - var genomeLocParser: GenomeLocParser = new GenomeLocParser(referenceSource.getReference) - val locs = { - // TODO: Abstract genome analysis engine has richer logic for parsing. We need to use it! - if (intervals.size == 0) { - GenomeLocSortedSet.createSetFromSequenceDictionary(referenceSource.getReference.getSequenceDictionary) - } else { - new GenomeLocSortedSet(genomeLocParser,IntervalUtils.parseIntervalArguments(genomeLocParser, intervals, false)) - } - } - if (locs == null || locs.size == 0) - throw new QException("Intervals are empty: " + intervals.mkString(", ")) - locs.toList - } - - def distinctContigs(reference: File, intervals: List[String] = Nil) = { - val referenceSource = new ReferenceDataSource(reference) - val locs = parseLocs(referenceSource, intervals) - var contig: String = null - var contigs = List.empty[String] - for (loc <- locs) { - if (contig != loc.getContig) { - contig = loc.getContig - contigs :+= contig - } - } - contigs - } - - def scatter(reference: File, intervals: List[String], scatterParts: List[File], splitByContig: Boolean) = { - val referenceSource = new ReferenceDataSource(reference) - val locs = parseLocs(referenceSource, intervals) - val fileHeader = new SAMFileHeader - fileHeader.setSequenceDictionary(referenceSource.getReference.getSequenceDictionary) - - var intervalList: IntervalList = null - var fileIndex = -1 - var locIndex = 0 - - if (splitByContig) { - var contig: String = null - for (loc <- locs) { - if (contig != loc.getContig && (fileIndex + 1) < scatterParts.size) { - if (fileIndex >= 0) - intervalList.write(scatterParts(fileIndex)) - fileIndex += 1 - contig = loc.getContig - intervalList = new IntervalList(fileHeader) - } - locIndex += 1 - intervalList.add(toInterval(loc, locIndex)) - } - intervalList.write(scatterParts(fileIndex)) - if ((fileIndex + 1) != scatterParts.size) - throw new QException("Only able to write contigs into %d of %d files.".format(fileIndex + 1, scatterParts.size)) - } else { - var locsPerFile = locs.size / scatterParts.size - val locRemainder = locs.size % scatterParts.size - - // At the start, put an extra loc per file - locsPerFile += 1 - var locsLeftFile = 0 - - for (loc <- locs) { - if (locsLeftFile == 0) { - if (fileIndex >= 0) - intervalList.write(scatterParts(fileIndex)) - - fileIndex += 1 - intervalList = new IntervalList(fileHeader) - - // When we have put enough locs into each file, - // reduce the number of locs per file back - // to the original calculated value. - if (fileIndex == locRemainder) - locsPerFile -= 1 - locsLeftFile = locsPerFile - } - locsLeftFile -= 1 - locIndex += 1 - intervalList.add(toInterval(loc, locIndex)) - } - intervalList.write(scatterParts(fileIndex)) - if ((fileIndex + 1) != scatterParts.size) - throw new QException("Only able to write intervals into %d of %d files.".format(fileIndex + 1, scatterParts.size)) - } - } - - private def toInterval(loc: GenomeLoc, locIndex: Int) = - new net.sf.picard.util.Interval(loc.getContig, loc.getStart.toInt, loc.getStop.toInt, false, "interval_" + locIndex) -} diff --git a/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala index fda03c496..afe8e5a1c 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala @@ -29,10 +29,10 @@ trait QFunction extends Logging { var qSettings: QSettings = _ /** Directory to run the command in. */ - var commandDirectory: File = IOUtils.CURRENT_DIR + var commandDirectory: File = new File(".") /** Temporary directory to write any files */ - var jobTempDir: File = IOUtils.javaTempDir + var jobTempDir: File = null /** Order the function was added to the graph. */ var addOrder: List[Int] = Nil @@ -279,7 +279,11 @@ trait QFunction extends Logging { if (jobOutputFile == null) jobOutputFile = new File(jobName + ".out") - commandDirectory = IOUtils.subDir(IOUtils.CURRENT_DIR, commandDirectory) + if (jobTempDir == null) + jobTempDir = qSettings.tempDirectory + + // If the command directory is relative, insert the run directory ahead of it. + commandDirectory = new File(qSettings.runDirectory, commandDirectory.getPath) } /** @@ -315,11 +319,11 @@ trait QFunction extends Logging { } /** - * Returns the absolute path to the file relative to the job command directory. + * Returns the absolute path to the file relative to the run directory and the job command directory. * @param file File to root relative to the command directory if it is not already absolute. * @return The absolute path to file. */ - private def absolute(file: File) = IOUtils.subDir(commandDirectory, file) + private def absolute(file: File) = IOUtils.absolute(commandDirectory, file) /** diff --git a/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala index 4bc270436..c8dd4d694 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala @@ -124,7 +124,7 @@ trait ScatterGatherableFunction extends CommandLineFunction { // Get absolute paths to the files and bind the sg functions to the clone function via the absolute paths. scatterFunction.bindCloneInputs(cloneFunction, i) for (gatherField <- outputFieldsWithValues) { - val gatherPart = IOUtils.subDir(cloneFunction.commandDirectory, cloneFunction.getFieldFile(gatherField)) + val gatherPart = IOUtils.absolute(cloneFunction.commandDirectory, cloneFunction.getFieldFile(gatherField)) cloneFunction.setFieldValue(gatherField, gatherPart) gatherFunctions(gatherField).gatherParts :+= gatherPart } @@ -244,5 +244,5 @@ trait ScatterGatherableFunction extends CommandLineFunction { * @param Sub directory under the scatter gather directory. * @return temporary directory under this scatter gather directory. */ - private def scatterGatherTempDir(subDir: String) = IOUtils.subDir(this.scatterGatherDirectory, this.jobName + "-sg/" + subDir) + private def scatterGatherTempDir(subDir: String) = IOUtils.absolute(this.scatterGatherDirectory, this.jobName + "-sg/" + subDir) } diff --git a/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala b/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala index 3ed8deb0e..96430b824 100644 --- a/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala +++ b/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala @@ -3,7 +3,7 @@ package org.broadinstitute.sting.queue.util import org.apache.commons.mail.{MultiPartEmail, EmailAttachment} import java.io.{FileReader, File} import javax.mail.internet.InternetAddress -import scala.collection.JavaConversions +import scala.collection.JavaConversions._ /** * Encapsulates a message to be sent over email. @@ -91,7 +91,7 @@ class EmailMessage extends Logging { * @return java.util.List of InternetAddress'es */ private def convert(addresses: List[String]) = { - JavaConversions.asList(addresses.map(address => new InternetAddress(address, false))) + asJavaList(addresses.map(address => new InternetAddress(address, false))) } override def toString = { diff --git a/scala/src/org/broadinstitute/sting/queue/util/IOUtils.scala b/scala/src/org/broadinstitute/sting/queue/util/IOUtils.scala index 56c568d83..1bb08a5cf 100644 --- a/scala/src/org/broadinstitute/sting/queue/util/IOUtils.scala +++ b/scala/src/org/broadinstitute/sting/queue/util/IOUtils.scala @@ -8,78 +8,32 @@ import org.broadinstitute.sting.utils.exceptions.UserException * A collection of utilities for modifying java.io. */ object IOUtils extends Logging { - /** The current directory "." */ - val CURRENT_DIR = new File(".") - - val CURRENT_DIR_ABS = absolute(CURRENT_DIR) - /** - * Returns the sub path rooted at the parent. - * If the sub path is already absolute, returns the sub path. - * If the parent is the current directory, returns the sub path. - * If the sub bath is the current directory, returns the parent. - * Else returns new File(parent, subPath) - * @param parent The parent directory - * @param path The sub path to append to the parent, if the path is not absolute. - * @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path. + * Checks if the temp directory has been setup and throws an exception if they user hasn't set it correctly. + * @param tempDir Temporary directory. */ - def subDir(parent: File, path: String): File = - subDir(parent, new File(path)) - - /** - * Returns the sub path rooted at the parent. - * If the sub path is already absolute, returns the sub path. - * If the parent is the current directory, returns the sub path. - * If the sub path is the current directory, returns the parent. - * Else returns new File(parent, subPath) - * @param parent The parent directory - * @param file The sub path to append to the parent, if the path is not absolute. - * @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path. - */ - def subDir(parent: File, file: File): File = { - val parentAbs = absolute(parent) - val fileAbs = absolute(file) - if (parentAbs == CURRENT_DIR_ABS && fileAbs == CURRENT_DIR_ABS) - CURRENT_DIR_ABS - else if (parentAbs == CURRENT_DIR_ABS || file.isAbsolute) - fileAbs - else if (fileAbs == CURRENT_DIR_ABS) - parentAbs - else - absolute(new File(parentAbs, file.getPath)) - } - - def checkTempDir = { - val javaTemp = System.getProperty("java.io.tmpdir") + def checkTempDir(tempDir: File) = { + val tempDirPath = tempDir.getAbsolutePath // Keeps the user from leaving the temp directory as the default, and on Macs from having pluses // in the path which can cause problems with the Google Reflections library. // see also: http://benjchristensen.com/2009/09/22/mac-osx-10-6-java-java-io-tmpdir/ - if (javaTemp.startsWith("/var/folders/") || (javaTemp == "/tmp") || (javaTemp == "/tmp/")) + if (tempDirPath.startsWith("/var/folders/") || (tempDirPath == "/tmp") || (tempDirPath == "/tmp/")) throw new UserException.BadTmpDir("java.io.tmpdir must be explicitly set") - } - - /** - * Returns the temp directory as defined by java. - * @return the temp directory as defined by java. - */ - def javaTempDir() = { - val tempDir = new File(System.getProperty("java.io.tmpdir")) if (!tempDir.exists && !tempDir.mkdirs) throw new UserException.BadTmpDir("Could not create directory: " + tempDir.getAbsolutePath()) - absolute(tempDir) } /** * Creates a temp directory with the prefix and optional suffix. * @param prefix Prefix for the directory name. - * @param suffix Optional suffix for the directory name. Defaults to "". + * @param suffix Optional suffix for the directory name. + * @param tempDirParent Parent directory for the temp directory. * @return The created temporary directory. */ - def tempDir(prefix: String, suffix: String = "") = { - val tempDirParent = javaTempDir() + def tempDir(prefix: String, suffix: String = "", tempDirParent: File) = { if (!tempDirParent.exists && !tempDirParent.mkdirs) throw new UserException.BadTmpDir("Could not create temp directory: " + tempDirParent) - val temp = File.createTempFile(prefix + "-", suffix) + val temp = File.createTempFile(prefix + "-", suffix, tempDirParent) if (!temp.delete) throw new UserException.BadTmpDir("Could not delete sub file: " + temp.getAbsolutePath()) if (!temp.mkdir) @@ -89,7 +43,7 @@ object IOUtils extends Logging { def writeContents(file: File, content: String) = FileUtils.writeStringToFile(file, content) - def writeTempFile(content: String, prefix: String, suffix: String = "", directory: File = null) = { + def writeTempFile(content: String, prefix: String, suffix: String = "", directory: File) = { val tempFile = absolute(File.createTempFile(prefix, suffix, directory)) writeContents(tempFile, content) tempFile @@ -115,6 +69,28 @@ object IOUtils extends Logging { directories(level) } + /** + * Returns the sub path rooted at the parent. + * @param parent The parent directory. + * @param path The sub path to append to the parent, if the path is not absolute. + * @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path. + */ + def absolute(parent: File, path: String): File = + absolute(parent, new File(path)) + + /** + * Returns the sub path rooted at the parent. + * @param parent The parent directory. + * @param file The sub path to append to the parent, if the path is not absolute. + * @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path. + */ + def absolute(parent: File, file: File): File = { + if (file.isAbsolute) + absolute(file) + else + absolute(new File(parent, file.getPath)) + } + /** * A mix of getCanonicalFile and getAbsoluteFile that returns the * absolute path to the file without deferencing symbolic links. diff --git a/scala/src/org/broadinstitute/sting/queue/util/ProcessController.scala b/scala/src/org/broadinstitute/sting/queue/util/ProcessController.scala index 92b63c628..e5cb50202 100644 --- a/scala/src/org/broadinstitute/sting/queue/util/ProcessController.scala +++ b/scala/src/org/broadinstitute/sting/queue/util/ProcessController.scala @@ -2,7 +2,6 @@ package org.broadinstitute.sting.queue.util import java.io._ import scala.collection.mutable.{HashSet, ListMap} -import scala.collection.JavaConversions /** * Facade to Runtime.exec() and java.lang.Process. Handles @@ -258,7 +257,11 @@ object ProcessController extends Logging { def shutdown() = { for (process <- running.clone) { logger.warn("Killing: " + process) - process.destroy + try { + process.destroy + } catch { + case _ => /* ignore */ + } } } @@ -349,8 +352,10 @@ object ProcessController extends Logging { * @param len Number of characters in the buffer. */ private def writeFile(chars: Array[Char], len: Int) = { - if (fileWriter != null) + if (fileWriter != null) { fileWriter.write(chars, 0, len) + fileWriter.flush() + } } /** Closes the fileWriter if it is not null. */ diff --git a/scala/test/org/broadinstitute/sting/queue/QScriptTest.scala b/scala/test/org/broadinstitute/sting/queue/QScriptTest.scala new file mode 100644 index 000000000..3014f10b9 --- /dev/null +++ b/scala/test/org/broadinstitute/sting/queue/QScriptTest.scala @@ -0,0 +1,88 @@ +package org.broadinstitute.sting.queue + +import org.broadinstitute.sting.utils.Utils +import org.testng.Assert +import org.broadinstitute.sting.commandline.CommandLineProgram +import org.broadinstitute.sting.BaseTest +import org.broadinstitute.sting.queue.util.ProcessController + +class QScriptTest extends BaseTest { + + protected val stingDir = "./" + + /** + * execute the test + * @param name the name of the test + * @param args the argument list + * @param expectedException the expected exception or null if no exception is expected. + */ + def executeTest(name: String, args: String, expectedException: Class[_]) = { + var command = Utils.escapeExpressions(args) + + // add the logging level to each of the integration test commands + + command = Utils.appendArray(command, "-l", "WARN", "-startFromScratch", "-tempDir", "integrationtests") + + // run the executable + var gotAnException = false + + val instance = new QCommandLine + QScriptTest.runningCommandLines += instance + try { + println("Executing test %s with Queue arguments: %s".format(name, Utils.join(" ",command))) + CommandLineProgram.start(instance, command) + } catch { + case e => + gotAnException = true + if (expectedException != null) { + // we expect an exception + println("Wanted exception %s, saw %s".format(expectedException, e.getClass)) + if (expectedException.isInstance(e)) { + // it's the type we expected + println(String.format(" => %s PASSED", name)) + } else { + e.printStackTrace() + Assert.fail("Test %s expected exception %s but got %s instead".format( + name, expectedException, e.getClass)) + } + } else { + // we didn't expect an exception but we got one :-( + throw new RuntimeException(e) + } + } finally { + instance.shutdown() + QScriptTest.runningCommandLines -= instance + } + + // catch failures from the integration test + if (expectedException != null) { + if (!gotAnException) + // we expected an exception but didn't see it + Assert.fail("Test %s expected exception %s but none was thrown".format(name, expectedException.toString)) + } else { + if (CommandLineProgram.result != 0) + throw new RuntimeException("Error running the GATK with arguments: " + args) + } + } +} + +object QScriptTest { + private var runningCommandLines = Set.empty[QCommandLine] + + Runtime.getRuntime.addShutdownHook(new Thread { + /** Cleanup as the JVM shuts down. */ + override def run = { + try { + ProcessController.shutdown() + } catch { + case _ => /*ignore */ + } + runningCommandLines.foreach(commandLine => + try { + commandLine.shutdown() + } catch { + case _ => /* ignore */ + }) + } + }) +} diff --git a/scala/test/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunctionUnitTest.scala b/scala/test/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunctionUnitTest.scala deleted file mode 100644 index bcd2e254e..000000000 --- a/scala/test/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunctionUnitTest.scala +++ /dev/null @@ -1,266 +0,0 @@ -package org.broadinstitute.sting.queue.extensions.gatk - -import collection.JavaConversions._ -import java.io.File -import org.testng.Assert -import org.broadinstitute.sting.BaseTest -import org.broadinstitute.sting.utils.interval.IntervalUtils -import org.broadinstitute.sting.queue.QException -import net.sf.picard.reference.IndexedFastaSequenceFile -import org.testng.annotations.{Test, BeforeMethod} -import org.broadinstitute.sting.utils.GenomeLocParser - -class IntervalScatterFunctionUnitTest extends BaseTest { - private def reference = new File(BaseTest.b36KGReference) - private var header: IndexedFastaSequenceFile = _ - private var genomeLocParser: GenomeLocParser = _ - - @BeforeMethod - def setup() { - header = new IndexedFastaSequenceFile(reference) - genomeLocParser = new GenomeLocParser(header.getSequenceDictionary()) - } - - @Test - def testCountContigs = { - Assert.assertEquals(List("1"), IntervalScatterFunction.distinctContigs(reference, List(BaseTest.validationDataLocation + "chr1_b36_pilot3.interval_list"))) - Assert.assertEquals(List("1","2","3"), IntervalScatterFunction.distinctContigs(reference, List("1:1-1", "2:1-1", "3:2-2"))) - Assert.assertEquals(List("1","2","3"), IntervalScatterFunction.distinctContigs(reference, List("2:1-1", "1:1-1", "3:2-2"))) - } - - @Test - def testBasicScatter = { - val chr1 = genomeLocParser.parseGenomeInterval("1") - val chr2 = genomeLocParser.parseGenomeInterval("2") - val chr3 = genomeLocParser.parseGenomeInterval("3") - - val files = (1 to 3).toList.map(index => new File(testDir + "basic." + index + ".intervals")) - - IntervalScatterFunction.scatter(reference, List("1", "2", "3"), files, false) - - val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false) - val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false) - val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false) - - Assert.assertEquals(1, locs1.size) - Assert.assertEquals(1, locs2.size) - Assert.assertEquals(1, locs3.size) - - Assert.assertEquals(chr1, locs1.get(0)) - Assert.assertEquals(chr2, locs2.get(0)) - Assert.assertEquals(chr3, locs3.get(0)) - } - - @Test - def testScatterLessFiles = { - val chr1 = genomeLocParser.parseGenomeInterval("1") - val chr2 = genomeLocParser.parseGenomeInterval("2") - val chr3 = genomeLocParser.parseGenomeInterval("3") - val chr4 = genomeLocParser.parseGenomeInterval("4") - - val files = (1 to 3).toList.map(index => new File(testDir + "less." + index + ".intervals")) - - IntervalScatterFunction.scatter(reference, List("1", "2", "3", "4"), files, false) - - val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false) - val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false) - val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false) - - Assert.assertEquals(2, locs1.size) - Assert.assertEquals(1, locs2.size) - Assert.assertEquals(1, locs3.size) - - Assert.assertEquals(chr1, locs1.get(0)) - Assert.assertEquals(chr2, locs1.get(1)) - Assert.assertEquals(chr3, locs2.get(0)) - Assert.assertEquals(chr4, locs3.get(0)) - } - - @Test(expectedExceptions=Array(classOf[QException])) - def testScatterMoreFiles = { - val files = (1 to 3).toList.map(index => new File(testDir + "more." + index + ".intervals")) - IntervalScatterFunction.scatter(reference, List("1", "2"), files, false) - } - - @Test - def testScatterIntervals = { - val intervals = List("1:1-2", "1:4-5", "2:1-1", "3:2-2") - val chr1a = genomeLocParser.parseGenomeInterval("1:1-2") - val chr1b = genomeLocParser.parseGenomeInterval("1:4-5") - val chr2 = genomeLocParser.parseGenomeInterval("2:1-1") - val chr3 = genomeLocParser.parseGenomeInterval("3:2-2") - - val files = (1 to 3).toList.map(index => new File(testDir + "split." + index + ".intervals")) - - IntervalScatterFunction.scatter(reference, intervals, files, true) - - val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false) - val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false) - val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false) - - Assert.assertEquals(2, locs1.size) - Assert.assertEquals(1, locs2.size) - Assert.assertEquals(1, locs3.size) - - Assert.assertEquals(chr1a, locs1.get(0)) - Assert.assertEquals(chr1b, locs1.get(1)) - Assert.assertEquals(chr2, locs2.get(0)) - Assert.assertEquals(chr3, locs3.get(0)) - } - - @Test - def testScatterOrder = { - val intervals = List("2:1-1", "1:1-1", "3:2-2") - val chr1 = genomeLocParser.parseGenomeInterval("1:1-1") - val chr2 = genomeLocParser.parseGenomeInterval("2:1-1") - val chr3 = genomeLocParser.parseGenomeInterval("3:2-2") - - val files = (1 to 3).toList.map(index => new File(testDir + "split." + index + ".intervals")) - - IntervalScatterFunction.scatter(reference, intervals, files, true) - - val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false) - val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false) - val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false) - - Assert.assertEquals(1, locs1.size) - Assert.assertEquals(1, locs2.size) - Assert.assertEquals(1, locs3.size) - - Assert.assertEquals(chr1, locs1.get(0)) - Assert.assertEquals(chr2, locs2.get(0)) - Assert.assertEquals(chr3, locs3.get(0)) - } - - @Test - def testBasicScatterByContig = { - val chr1 = genomeLocParser.parseGenomeInterval("1") - val chr2 = genomeLocParser.parseGenomeInterval("2") - val chr3 = genomeLocParser.parseGenomeInterval("3") - - val files = (1 to 3).toList.map(index => new File(testDir + "contig_basic." + index + ".intervals")) - - IntervalScatterFunction.scatter(reference, List("1", "2", "3"), files, true) - - val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false) - val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false) - val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false) - - Assert.assertEquals(1, locs1.size) - Assert.assertEquals(1, locs2.size) - Assert.assertEquals(1, locs3.size) - - Assert.assertEquals(chr1, locs1.get(0)) - Assert.assertEquals(chr2, locs2.get(0)) - Assert.assertEquals(chr3, locs3.get(0)) - } - - @Test - def testScatterByContigLessFiles = { - val chr1 = genomeLocParser.parseGenomeInterval("1") - val chr2 = genomeLocParser.parseGenomeInterval("2") - val chr3 = genomeLocParser.parseGenomeInterval("3") - val chr4 = genomeLocParser.parseGenomeInterval("4") - - val files = (1 to 3).toList.map(index => new File(testDir + "contig_less." + index + ".intervals")) - - IntervalScatterFunction.scatter(reference, List("1", "2", "3", "4"), files, true) - - val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false) - val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false) - val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false) - - Assert.assertEquals(1, locs1.size) - Assert.assertEquals(1, locs2.size) - Assert.assertEquals(2, locs3.size) - - Assert.assertEquals(chr1, locs1.get(0)) - Assert.assertEquals(chr2, locs2.get(0)) - Assert.assertEquals(chr3, locs3.get(0)) - Assert.assertEquals(chr4, locs3.get(1)) - } - - @Test(expectedExceptions=Array(classOf[QException])) - def testScatterByContigMoreFiles = { - val files = (1 to 3).toList.map(index => new File(testDir + "contig_more." + index + ".intervals")) - IntervalScatterFunction.scatter(reference, List("1", "2"), files, true) - } - - @Test - def testScatterByContigIntervalsStart = { - val intervals = List("1:1-2", "1:4-5", "2:1-1", "3:2-2") - val chr1a = genomeLocParser.parseGenomeInterval("1:1-2") - val chr1b = genomeLocParser.parseGenomeInterval("1:4-5") - val chr2 = genomeLocParser.parseGenomeInterval("2:1-1") - val chr3 = genomeLocParser.parseGenomeInterval("3:2-2") - - val files = (1 to 3).toList.map(index => new File(testDir + "contig_split_start." + index + ".intervals")) - - IntervalScatterFunction.scatter(reference, intervals, files, true) - - val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false) - val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false) - val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false) - - Assert.assertEquals(2, locs1.size) - Assert.assertEquals(1, locs2.size) - Assert.assertEquals(1, locs3.size) - - Assert.assertEquals(chr1a, locs1.get(0)) - Assert.assertEquals(chr1b, locs1.get(1)) - Assert.assertEquals(chr2, locs2.get(0)) - Assert.assertEquals(chr3, locs3.get(0)) - } - - @Test - def testScatterByContigIntervalsMiddle = { - val intervals = List("1:1-1", "2:1-2", "2:4-5", "3:2-2") - val chr1 = genomeLocParser.parseGenomeInterval("1:1-1") - val chr2a = genomeLocParser.parseGenomeInterval("2:1-2") - val chr2b = genomeLocParser.parseGenomeInterval("2:4-5") - val chr3 = genomeLocParser.parseGenomeInterval("3:2-2") - - val files = (1 to 3).toList.map(index => new File(testDir + "contig_split_middle." + index + ".intervals")) - - IntervalScatterFunction.scatter(reference, intervals, files, true) - - val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false) - val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false) - val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false) - - Assert.assertEquals(1, locs1.size) - Assert.assertEquals(2, locs2.size) - Assert.assertEquals(1, locs3.size) - - Assert.assertEquals(chr1, locs1.get(0)) - Assert.assertEquals(chr2a, locs2.get(0)) - Assert.assertEquals(chr2b, locs2.get(1)) - Assert.assertEquals(chr3, locs3.get(0)) - } - - @Test - def testScatterByContigIntervalsEnd = { - val intervals = List("1:1-1", "2:2-2", "3:1-2", "3:4-5") - val chr1 = genomeLocParser.parseGenomeInterval("1:1-1") - val chr2 = genomeLocParser.parseGenomeInterval("2:2-2") - val chr3a = genomeLocParser.parseGenomeInterval("3:1-2") - val chr3b = genomeLocParser.parseGenomeInterval("3:4-5") - - val files = (1 to 3).toList.map(index => new File(testDir + "contig_split_end." + index + ".intervals")) - - IntervalScatterFunction.scatter(reference, intervals, files, true) - - val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false) - val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false) - val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false) - - Assert.assertEquals(1, locs1.size) - Assert.assertEquals(1, locs2.size) - Assert.assertEquals(2, locs3.size) - - Assert.assertEquals(chr1, locs1.get(0)) - Assert.assertEquals(chr2, locs2.get(0)) - Assert.assertEquals(chr3a, locs3.get(0)) - Assert.assertEquals(chr3b, locs3.get(1)) - } -} diff --git a/scala/test/org/broadinstitute/sting/queue/util/IOUtilsUnitTest.scala b/scala/test/org/broadinstitute/sting/queue/util/IOUtilsUnitTest.scala index 06e5f4122..f0e5153a4 100644 --- a/scala/test/org/broadinstitute/sting/queue/util/IOUtilsUnitTest.scala +++ b/scala/test/org/broadinstitute/sting/queue/util/IOUtilsUnitTest.scala @@ -9,65 +9,53 @@ import org.testng.annotations.Test class IOUtilsUnitTest extends BaseTest { @Test def testGoodTempDir = { - val tmpDir = System.getProperty("java.io.tmpdir") - try { - System.setProperty("java.io.tmpdir", "/tmp/queue") - IOUtils.checkTempDir - } finally { - System.setProperty("java.io.tmpdir", tmpDir) - } + IOUtils.checkTempDir(new File("/tmp/queue")) } @Test(expectedExceptions=Array(classOf[UserException.BadTmpDir])) def testBadTempDir = { - val tmpDir = System.getProperty("java.io.tmpdir") - try { - System.setProperty("java.io.tmpdir", "/tmp") - IOUtils.checkTempDir - } finally { - System.setProperty("java.io.tmpdir", tmpDir) - } + IOUtils.checkTempDir(new File("/tmp")) } @Test def testAbsoluteSubDir = { - var subDir = IOUtils.subDir(IOUtils.CURRENT_DIR, new File("/path/to/file")) - Assert.assertEquals(new File("/path/to/file"), subDir) + var subDir = IOUtils.absolute(new File("."), new File("/path/to/file")) + Assert.assertEquals(subDir, new File("/path/to/file")) - subDir = IOUtils.subDir(new File("/different/path"), new File("/path/to/file")) - Assert.assertEquals(new File("/path/to/file"), subDir) + subDir = IOUtils.absolute(new File("/different/path"), new File("/path/to/file")) + Assert.assertEquals(subDir, new File("/path/to/file")) - subDir = IOUtils.subDir(new File("/different/path"), IOUtils.CURRENT_DIR) - Assert.assertEquals(new File("/different/path"), subDir) + subDir = IOUtils.absolute(new File("/different/path"), new File(".")) + Assert.assertEquals(subDir, new File("/different/path")) } @Test def testRelativeSubDir = { - var subDir = IOUtils.subDir(IOUtils.CURRENT_DIR, new File("path/to/file")) - Assert.assertEquals(new File("path/to/file").getCanonicalFile, subDir.getCanonicalFile) + var subDir = IOUtils.absolute(new File("."), new File("path/to/file")) + Assert.assertEquals(subDir.getCanonicalFile, new File("path/to/file").getCanonicalFile) - subDir = IOUtils.subDir(new File("/different/path"), new File("path/to/file")) - Assert.assertEquals(new File("/different/path/path/to/file"), subDir) + subDir = IOUtils.absolute(new File("/different/path"), new File("path/to/file")) + Assert.assertEquals(subDir, new File("/different/path/path/to/file")) } @Test def testDottedSubDir = { - var subDir = IOUtils.subDir(IOUtils.CURRENT_DIR, new File("path/../to/file")) - Assert.assertEquals(new File("path/../to/./file").getCanonicalFile, subDir.getCanonicalFile) + var subDir = IOUtils.absolute(new File("."), new File("path/../to/file")) + Assert.assertEquals(subDir.getCanonicalFile, new File("path/../to/./file").getCanonicalFile) - subDir = IOUtils.subDir(IOUtils.CURRENT_DIR, new File("/path/../to/file")) - Assert.assertEquals(new File("/path/../to/file"), subDir) + subDir = IOUtils.absolute(new File("."), new File("/path/../to/file")) + Assert.assertEquals(subDir, new File("/path/../to/file")) - subDir = IOUtils.subDir(new File("/different/../path"), new File("path/to/file")) - Assert.assertEquals(new File("/different/../path/path/to/file"), subDir) + subDir = IOUtils.absolute(new File("/different/../path"), new File("path/to/file")) + Assert.assertEquals(subDir, new File("/different/../path/path/to/file")) - subDir = IOUtils.subDir(new File("/different/./path"), new File("/path/../to/file")) - Assert.assertEquals(new File("/path/../to/file"), subDir) + subDir = IOUtils.absolute(new File("/different/./path"), new File("/path/../to/file")) + Assert.assertEquals(subDir, new File("/path/../to/file")) } @Test def testTempDir = { - val tempDir = IOUtils.tempDir("Q-Unit-Test") + val tempDir = IOUtils.tempDir("Q-Unit-Test", "", new File("queueTempDirToDelete")) Assert.assertTrue(tempDir.exists) Assert.assertFalse(tempDir.isFile) Assert.assertTrue(tempDir.isDirectory) @@ -79,43 +67,43 @@ class IOUtilsUnitTest extends BaseTest { @Test def testDirLevel = { var dir = IOUtils.dirLevel(new File("/path/to/directory"), 1) - Assert.assertEquals(new File("/path"), dir) + Assert.assertEquals(dir, new File("/path")) dir = IOUtils.dirLevel(new File("/path/to/directory"), 2) - Assert.assertEquals(new File("/path/to"), dir) + Assert.assertEquals(dir, new File("/path/to")) dir = IOUtils.dirLevel(new File("/path/to/directory"), 3) - Assert.assertEquals(new File("/path/to/directory"), dir) + Assert.assertEquals(dir, new File("/path/to/directory")) dir = IOUtils.dirLevel(new File("/path/to/directory"), 4) - Assert.assertEquals(new File("/path/to/directory"), dir) + Assert.assertEquals(dir, new File("/path/to/directory")) } @Test def testAbsolute = { var dir = IOUtils.absolute(new File("/path/./to/./directory/.")) - Assert.assertEquals(new File("/path/to/directory"), dir) + Assert.assertEquals(dir, new File("/path/to/directory")) dir = IOUtils.absolute(new File("/")) - Assert.assertEquals(new File("/"), dir) + Assert.assertEquals(dir, new File("/")) dir = IOUtils.absolute(new File("/.")) - Assert.assertEquals(new File("/"), dir) + Assert.assertEquals(dir, new File("/")) dir = IOUtils.absolute(new File("/././.")) - Assert.assertEquals(new File("/"), dir) + Assert.assertEquals(dir, new File("/")) dir = IOUtils.absolute(new File("/./directory/.")) - Assert.assertEquals(new File("/directory"), dir) + Assert.assertEquals(dir, new File("/directory")) dir = IOUtils.absolute(new File("/./directory/./")) - Assert.assertEquals(new File("/directory"), dir) + Assert.assertEquals(dir, new File("/directory")) dir = IOUtils.absolute(new File("/./directory./")) - Assert.assertEquals(new File("/directory."), dir) + Assert.assertEquals(dir, new File("/directory.")) dir = IOUtils.absolute(new File("/./.directory/")) - Assert.assertEquals(new File("/.directory"), dir) + Assert.assertEquals(dir, new File("/.directory")) } @Test @@ -127,8 +115,8 @@ class IOUtilsUnitTest extends BaseTest { "chr22_random 257318 3156435963 50 51", "chrX_random 1719168 3156698441 50 51") val tail = IOUtils.tail(new File(BaseTest.hg18Reference + ".fai"), 5) - Assert.assertEquals(5, tail.size) + Assert.assertEquals(tail.size, 5) for (i <- 0 until 5) - Assert.assertEquals(lines(i), tail(i)) + Assert.assertEquals(tail(i), lines(i)) } }