From 673fa841a420808e4fe5bf83a1c76ad8ce540094 Mon Sep 17 00:00:00 2001 From: kshakir Date: Fri, 12 Nov 2010 20:14:28 +0000 Subject: [PATCH] Updated PluginManager so that during testing Queue can dynamically compile and load separately multiple class directories into the same class loader. Removed obsolete usages of PackageUtils with updated PluginManager. Ported Queue interval utilities written in scala over to Sting's java IntervalUtils. Added a very basic intergration test to ensure that the fullCallingPipeline.q compiles. Added options to specify the temporary directories without having to use -Djava.io.tmpdir (useful during the above integration test). While adding tempDir added options to specify the run directory from the command line, for example "-runDir v1". Upgraded to scala 2.8.1 and updated calls to deprecated functions. git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@4661 348d0f76-0448-11de-a6fe-93d51630548a --- ivy.xml | 4 +- .../analyzecovariates/AnalyzeCovariates.java | 5 +- .../sting/gatk/WalkerManager.java | 4 +- .../sting/gatk/filters/FilterManager.java | 2 +- .../tracks/builders/RMDTrackBuilder.java | 9 +- .../walkers/annotator/VariantAnnotator.java | 8 +- .../annotator/VariantAnnotatorEngine.java | 51 +-- .../AnnotationInterfaceManager.java | 100 +++++ .../recalibration/CovariateCounterWalker.java | 8 +- .../TableRecalibrationWalker.java | 4 +- .../varianteval/VariantEvalWalker.java | 8 +- .../AnnotationByAlleleFrequencyWalker.java | 56 +-- .../gatk/CommandLineProgramManager.java | 43 --- .../gatk/GATKExtensionsGenerator.java | 5 +- .../org/broadinstitute/sting/utils/Utils.java | 66 +++- .../sting/utils/classloader/JVMUtils.java | 14 +- .../sting/utils/classloader/PackageUtils.java | 159 -------- .../utils/classloader/PluginManager.java | 194 ++++++++-- .../sting/utils/interval/IntervalUtils.java | 137 ++++++- .../org/broadinstitute/sting/WalkerTest.java | 42 +-- .../utils/interval/IntervalUtilsTest.java | 91 ----- .../utils/interval/IntervalUtilsUnitTest.java | 356 ++++++++++++++++++ scala/qscript/fullCallingPipeline.q | 19 +- .../sting/queue/QCommandLine.scala | 46 ++- .../sting/queue/QScriptManager.scala | 58 +-- .../sting/queue/QSettings.scala | 6 + .../sting/queue/engine/LsfJobRunner.scala | 8 +- .../sting/queue/engine/QGraph.scala | 7 +- .../gatk/IntervalScatterFunction.scala | 97 +---- .../sting/queue/function/QFunction.scala | 14 +- .../ScatterGatherableFunction.scala | 4 +- .../sting/queue/util/EmailMessage.scala | 4 +- .../sting/queue/util/IOUtils.scala | 88 ++--- .../sting/queue/util/ProcessController.scala | 11 +- .../sting/queue/QScriptTest.scala | 88 +++++ .../IntervalScatterFunctionUnitTest.scala | 266 ------------- .../sting/queue/util/IOUtilsUnitTest.scala | 82 ++-- 37 files changed, 1116 insertions(+), 1048 deletions(-) create mode 100644 java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/AnnotationInterfaceManager.java delete mode 100644 java/src/org/broadinstitute/sting/queue/extensions/gatk/CommandLineProgramManager.java delete mode 100755 java/src/org/broadinstitute/sting/utils/classloader/PackageUtils.java delete mode 100644 java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsTest.java create mode 100644 java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsUnitTest.java create mode 100644 scala/test/org/broadinstitute/sting/queue/QScriptTest.scala delete mode 100644 scala/test/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunctionUnitTest.scala diff --git a/ivy.xml b/ivy.xml index f47949227..f7893cb32 100644 --- a/ivy.xml +++ b/ivy.xml @@ -41,8 +41,8 @@ - - + + diff --git a/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java b/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java index 41abd092c..e9b0035cd 100755 --- a/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java +++ b/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java @@ -27,13 +27,14 @@ package org.broadinstitute.sting.analyzecovariates; import org.broadinstitute.sting.commandline.Input; import org.broadinstitute.sting.gatk.walkers.recalibration.*; -import org.broadinstitute.sting.utils.classloader.PackageUtils; +import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException; import org.broadinstitute.sting.utils.text.XReadLines; import org.broadinstitute.sting.commandline.CommandLineProgram; import org.broadinstitute.sting.commandline.Argument; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.regex.Pattern; @@ -114,7 +115,7 @@ class AnalyzeCovariatesCLP extends CommandLineProgram { private void initializeData() { // Get a list of all available covariates - List> classes = PackageUtils.getClassesImplementingInterface(Covariate.class); + Collection> classes = new PluginManager(Covariate.class).getPlugins(); int lineNumber = 0; boolean foundAllCovariates = false; diff --git a/java/src/org/broadinstitute/sting/gatk/WalkerManager.java b/java/src/org/broadinstitute/sting/gatk/WalkerManager.java index 45675092c..cb1da2b99 100755 --- a/java/src/org/broadinstitute/sting/gatk/WalkerManager.java +++ b/java/src/org/broadinstitute/sting/gatk/WalkerManager.java @@ -62,7 +62,7 @@ public class WalkerManager extends PluginManager { */ public Map>> getWalkerNamesByPackage(boolean visibleWalkersOnly) { Map>> walkersByPackage = new HashMap>>(); - for(Class walker: pluginsByName.values()) { + for(Class walker: getPlugins()) { if(visibleWalkersOnly && isHidden(walker)) continue; @@ -157,7 +157,7 @@ public class WalkerManager extends PluginManager { * @return Class representing the walker. */ public Class getWalkerClassByName(String walkerName) { - return pluginsByName.get(walkerName); + return getPluginsByName().get(walkerName); } /** diff --git a/java/src/org/broadinstitute/sting/gatk/filters/FilterManager.java b/java/src/org/broadinstitute/sting/gatk/filters/FilterManager.java index bd899b80c..3bf11c0ac 100644 --- a/java/src/org/broadinstitute/sting/gatk/filters/FilterManager.java +++ b/java/src/org/broadinstitute/sting/gatk/filters/FilterManager.java @@ -55,6 +55,6 @@ public class FilterManager extends PluginManager { } public Collection> getValues() { - return this.pluginsByName.values(); + return this.getPlugins(); } } diff --git a/java/src/org/broadinstitute/sting/gatk/refdata/tracks/builders/RMDTrackBuilder.java b/java/src/org/broadinstitute/sting/gatk/refdata/tracks/builders/RMDTrackBuilder.java index b3bd6ef86..fb68a8e37 100644 --- a/java/src/org/broadinstitute/sting/gatk/refdata/tracks/builders/RMDTrackBuilder.java +++ b/java/src/org/broadinstitute/sting/gatk/refdata/tracks/builders/RMDTrackBuilder.java @@ -32,7 +32,6 @@ import org.broad.tribble.*; import org.broad.tribble.index.Index; import org.broad.tribble.index.IndexFactory; import org.broad.tribble.source.BasicFeatureSource; -import org.broad.tribble.source.CachingFeatureSource; import org.broad.tribble.util.LittleEndianOutputStream; import org.broadinstitute.sting.gatk.arguments.GATKArgumentCollection; import org.broadinstitute.sting.gatk.refdata.ReferenceDependentFeatureCodec; @@ -42,7 +41,6 @@ import org.broadinstitute.sting.gatk.refdata.utils.RMDTriplet; import org.broadinstitute.sting.gatk.AbstractGenomeAnalysisEngine; import org.broadinstitute.sting.gatk.refdata.utils.helpers.DbSNPHelper; import org.broadinstitute.sting.utils.GenomeLocParser; -import org.broadinstitute.sting.utils.Utils; import org.broadinstitute.sting.utils.collections.Pair; import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.exceptions.UserException; @@ -114,8 +112,8 @@ public class RMDTrackBuilder extends PluginManager { /** @return a list of all available track types we currently have access to create */ public Map getAvailableTrackNamesAndTypes() { classes = new HashMap(); - for (String name: this.pluginsByName.keySet()) { - classes.put(name.toUpperCase(), pluginsByName.get(name)); + for (String name: this.getPluginsByName().keySet()) { + classes.put(name.toUpperCase(), getPluginsByName().get(name)); } return classes; } @@ -123,7 +121,7 @@ public class RMDTrackBuilder extends PluginManager { /** @return a list of all available track record types we currently have access to create */ public Map getAvailableTrackNamesAndRecordTypes() { HashMap classToRecord = new HashMap(); - for (String name: this.pluginsByName.keySet()) { + for (String name: this.getPluginsByName().keySet()) { FeatureCodec codec = this.createByName(name); classToRecord.put(name, codec.getFeatureType()); } @@ -133,7 +131,6 @@ public class RMDTrackBuilder extends PluginManager { /** * create a RMDTrack of the specified type * - * @param genomeLocParser GenomeLocParser to use, if case track needs additional reference context. * @param targetClass the target class of track * @param name what to call the track * @param inputFile the input file diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java b/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java index ab4bf068b..be80ede2f 100755 --- a/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java @@ -38,7 +38,7 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotationType import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.GenotypeAnnotation; import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation; import org.broadinstitute.sting.utils.BaseUtils; -import org.broadinstitute.sting.utils.classloader.PackageUtils; +import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.SampleUtils; import org.broadinstitute.sting.commandline.Argument; import org.broadinstitute.sting.commandline.Output; @@ -93,18 +93,18 @@ public class VariantAnnotator extends RodWalker { private void listAnnotationsAndExit() { - List> infoAnnotationClasses = PackageUtils.getClassesImplementingInterface(InfoFieldAnnotation.class); + List> infoAnnotationClasses = new PluginManager(InfoFieldAnnotation.class).getPlugins(); System.out.println("\nAvailable annotations for the VCF INFO field:"); for (int i = 0; i < infoAnnotationClasses.size(); i++) System.out.println("\t" + infoAnnotationClasses.get(i).getSimpleName()); System.out.println(); - List> genotypeAnnotationClasses = PackageUtils.getClassesImplementingInterface(GenotypeAnnotation.class); + List> genotypeAnnotationClasses = new PluginManager(GenotypeAnnotation.class).getPlugins(); System.out.println("\nAvailable annotations for the VCF FORMAT field:"); for (int i = 0; i < genotypeAnnotationClasses.size(); i++) System.out.println("\t" + genotypeAnnotationClasses.get(i).getSimpleName()); System.out.println(); System.out.println("\nAvailable classes/groups of annotations:"); - for ( Class c : PackageUtils.getInterfacesExtendingInterface(AnnotationType.class) ) + for ( Class c : new PluginManager(AnnotationType.class).getInterfaces() ) System.out.println("\t" + c.getSimpleName()); System.out.println(); System.exit(0); diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java b/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java index 65aca754b..6347f2e4e 100755 --- a/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java @@ -50,7 +50,6 @@ import org.broadinstitute.sting.gatk.refdata.utils.helpers.DbSNPHelper; import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.*; import org.broadinstitute.sting.gatk.walkers.annotator.genomicannotator.*; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import org.broadinstitute.sting.utils.classloader.PackageUtils; import org.broadinstitute.sting.utils.exceptions.UserException; @@ -94,8 +93,8 @@ public class VariantAnnotatorEngine { // use this constructor if you want all possible annotations public VariantAnnotatorEngine(GenomeAnalysisEngine engine) { - requestedInfoAnnotations = PackageUtils.getInstancesOfClassesImplementingInterface(InfoFieldAnnotation.class); - requestedGenotypeAnnotations = PackageUtils.getInstancesOfClassesImplementingInterface(GenotypeAnnotation.class); + requestedInfoAnnotations = AnnotationInterfaceManager.createAllInfoFieldAnnotations(); + requestedGenotypeAnnotations = AnnotationInterfaceManager.createAllGenotypeAnnotations(); initializeDBs(engine); } @@ -116,49 +115,9 @@ public class VariantAnnotatorEngine { } private void initializeAnnotations(List annotationGroupsToUse, List annotationsToUse) { - // create a map for all annotation classes which implement our top-level interfaces - HashMap classMap = new HashMap(); - for ( Class c : PackageUtils.getClassesImplementingInterface(InfoFieldAnnotation.class) ) - classMap.put(c.getSimpleName(), c); - for ( Class c : PackageUtils.getClassesImplementingInterface(GenotypeAnnotation.class) ) - classMap.put(c.getSimpleName(), c); - for ( Class c : PackageUtils.getInterfacesExtendingInterface(AnnotationType.class) ) - classMap.put(c.getSimpleName(), c); - - HashSet classes = new HashSet(); - // get the classes from the provided groups (interfaces) - if ( annotationGroupsToUse.size() != 1 || ! annotationGroupsToUse.get(0).toLowerCase().equals("none") ) { - for ( String group : annotationGroupsToUse ) { - Class interfaceClass = classMap.get(group); - if ( interfaceClass == null ) - interfaceClass = classMap.get(group + "Annotation"); - if ( interfaceClass == null ) - throw new UserException.BadArgumentValue("group", "Class " + group + " is not found; please check that you have specified the class name correctly"); - classes.addAll(PackageUtils.getClassesImplementingInterface(interfaceClass)); - } - } - - // get the specific classes provided - for ( String annotation : annotationsToUse ) { - Class annotationClass = classMap.get(annotation); - if ( annotationClass == null ) - annotationClass = classMap.get(annotation + "Annotation"); - if ( annotationClass == null ) - throw new UserException.BadArgumentValue("annotation", "Class " + annotation + " is not found; please check that you have specified the class name correctly"); - classes.add(annotationClass); - } - - // get the instances - requestedInfoAnnotations = new ArrayList(); - requestedGenotypeAnnotations = new ArrayList(); - - for ( Class c : classes ) { - // note that technically an annotation can work on both the INFO and FORMAT fields - if ( InfoFieldAnnotation.class.isAssignableFrom(c) ) - requestedInfoAnnotations.add((InfoFieldAnnotation)PackageUtils.getSimpleInstance(c)); - if ( GenotypeAnnotation.class.isAssignableFrom(c) ) - requestedGenotypeAnnotations.add((GenotypeAnnotation)PackageUtils.getSimpleInstance(c)); - } + AnnotationInterfaceManager.validateAnnotations(annotationGroupsToUse, annotationsToUse); + requestedInfoAnnotations = AnnotationInterfaceManager.createInfoFieldAnnotations(annotationGroupsToUse, annotationsToUse); + requestedGenotypeAnnotations = AnnotationInterfaceManager.createGenotypeAnnotations(annotationGroupsToUse, annotationsToUse); } private void initializeDBs(GenomeAnalysisEngine engine) { diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/AnnotationInterfaceManager.java b/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/AnnotationInterfaceManager.java new file mode 100644 index 000000000..89106a360 --- /dev/null +++ b/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/AnnotationInterfaceManager.java @@ -0,0 +1,100 @@ +package org.broadinstitute.sting.gatk.walkers.annotator.interfaces; + +import org.broadinstitute.sting.utils.classloader.PluginManager; +import org.broadinstitute.sting.utils.exceptions.UserException; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +public class AnnotationInterfaceManager { + private static PluginManager infoFieldAnnotationPluginManager = new PluginManager(InfoFieldAnnotation.class); + private static PluginManager genotypeAnnotationPluginManager = new PluginManager(GenotypeAnnotation.class); + private static PluginManager annotationTypePluginManager = new PluginManager(AnnotationType.class); + + public static List createAllInfoFieldAnnotations() { + return infoFieldAnnotationPluginManager.createAllTypes(); + } + + public static List createAllGenotypeAnnotations() { + return genotypeAnnotationPluginManager.createAllTypes(); + } + + public static void validateAnnotations(List annotationGroupsToUse, List annotationsToUse) { + HashMap classMap = new HashMap(); + for ( Class c : infoFieldAnnotationPluginManager.getPlugins() ) + classMap.put(c.getSimpleName(), c); + for ( Class c : genotypeAnnotationPluginManager.getPlugins() ) + classMap.put(c.getSimpleName(), c); + for ( Class c : annotationTypePluginManager.getInterfaces() ) + classMap.put(c.getSimpleName(), c); + + if ( annotationGroupsToUse.size() != 1 || !"none".equals(annotationGroupsToUse.get(0)) ) { + for ( String group : annotationGroupsToUse ) { + Class interfaceClass = classMap.get(group); + if ( interfaceClass == null ) + interfaceClass = classMap.get(group + "Annotation"); + if ( interfaceClass == null ) + throw new UserException.BadArgumentValue("group", "Class " + group + " is not found; please check that you have specified the class name correctly"); + } + } + + // validate the specific classes provided + for ( String annotation : annotationsToUse ) { + Class annotationClass = classMap.get(annotation); + if ( annotationClass == null ) + annotationClass = classMap.get(annotation + "Annotation"); + if ( annotationClass == null ) + throw new UserException.BadArgumentValue("annotation", "Class " + annotation + " is not found; please check that you have specified the class name correctly"); + } + } + + public static List createInfoFieldAnnotations(List annotationGroupsToUse, List annotationsToUse) { + return createAnnotations(infoFieldAnnotationPluginManager, annotationGroupsToUse, annotationsToUse); + } + + public static List createGenotypeAnnotations(List annotationGroupsToUse, List annotationsToUse) { + return createAnnotations(genotypeAnnotationPluginManager, annotationGroupsToUse, annotationsToUse); + } + + private static List createAnnotations(PluginManager pluginManager, List annotationGroupsToUse, List annotationsToUse) { + // get the instances + List annotations = new ArrayList(); + + // get the classes from the provided groups (interfaces) + // create a map for all annotation classes which implement our top-level interfaces + HashMap classMap = new HashMap(); + for ( Class c : pluginManager.getPlugins() ) + classMap.put(c.getSimpleName(), c); + for ( Class c : annotationTypePluginManager.getInterfaces() ) + classMap.put(c.getSimpleName(), c); + + HashSet classes = new HashSet(); + + if ( annotationGroupsToUse.size() != 1 || !"none".equals(annotationGroupsToUse.get(0)) ) { + for ( String group : annotationGroupsToUse ) { + Class interfaceClass = classMap.get(group); + if ( interfaceClass == null ) + interfaceClass = classMap.get(group + "Annotation"); + if ( interfaceClass != null ) + classes.addAll(pluginManager.getPluginsImplementing(interfaceClass)); + } + } + + // get the specific classes provided + for ( String annotation : annotationsToUse ) { + Class annotationClass = classMap.get(annotation); + if ( annotationClass == null ) + annotationClass = classMap.get(annotation + "Annotation"); + if ( annotationClass != null ) + classes.add(annotationClass); + } + + // note that technically an annotation can work on both the INFO and FORMAT fields + for ( Class c : classes ) + annotations.add(pluginManager.createByType(c)); + + return annotations; + } +} diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CovariateCounterWalker.java b/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CovariateCounterWalker.java index 0df273eab..211ebb95c 100755 --- a/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CovariateCounterWalker.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CovariateCounterWalker.java @@ -37,7 +37,7 @@ import org.broadinstitute.sting.gatk.filters.ZeroMappingQualityReadFilter; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; import org.broadinstitute.sting.gatk.walkers.*; import org.broadinstitute.sting.utils.*; -import org.broadinstitute.sting.utils.classloader.PackageUtils; +import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.collections.NestedHashMap; import org.broadinstitute.sting.commandline.Argument; import org.broadinstitute.sting.commandline.ArgumentCollection; @@ -167,9 +167,9 @@ public class CovariateCounterWalker extends LocusWalker> covariateClasses = PackageUtils.getClassesImplementingInterface( Covariate.class ); - final List> requiredClasses = PackageUtils.getClassesImplementingInterface( RequiredCovariate.class ); - final List> standardClasses = PackageUtils.getClassesImplementingInterface( StandardCovariate.class ); + final List> covariateClasses = new PluginManager( Covariate.class ).getPlugins(); + final List> requiredClasses = new PluginManager( RequiredCovariate.class ).getPlugins(); + final List> standardClasses = new PluginManager( StandardCovariate.class ).getPlugins(); // Print and exit if that's what was requested if ( LIST_ONLY ) { diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java b/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java index 2e6bf8543..a498dd501 100644 --- a/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java @@ -42,7 +42,7 @@ import org.broadinstitute.sting.gatk.walkers.ReadWalker; import org.broadinstitute.sting.gatk.walkers.Requires; import org.broadinstitute.sting.gatk.walkers.WalkerName; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; -import org.broadinstitute.sting.utils.classloader.PackageUtils; +import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.collections.NestedHashMap; import org.broadinstitute.sting.utils.QualityUtils; import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException; @@ -149,7 +149,7 @@ public class TableRecalibrationWalker extends ReadWalker> classes = PackageUtils.getClassesImplementingInterface(Covariate.class); + final List> classes = new PluginManager(Covariate.class).getPlugins(); int lineNumber = 0; boolean foundAllCovariates = false; diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java b/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java index 53a336464..7cfdbcafa 100755 --- a/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java @@ -49,7 +49,7 @@ import org.broadinstitute.sting.utils.report.VE2ReportFactory; import org.broadinstitute.sting.utils.report.templates.ReportFormat; import org.broadinstitute.sting.utils.report.utils.Node; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import org.broadinstitute.sting.utils.classloader.PackageUtils; +import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.Utils; import org.broadinstitute.sting.commandline.Argument; import org.broadinstitute.sting.commandline.Output; @@ -351,7 +351,7 @@ public class VariantEvalWalker extends RodWalker implements Tr } private void listModulesAndExit() { - List> veClasses = PackageUtils.getClassesImplementingInterface(VariantEvaluator.class); + List> veClasses = new PluginManager( VariantEvaluator.class ).getPlugins(); out.println("\nAvailable eval modules:"); out.println("(Standard modules are starred)"); for (Class veClass : veClasses) @@ -399,14 +399,14 @@ public class VariantEvalWalker extends RodWalker implements Tr private void determineEvalations() { // create a map for all eval modules for easy lookup HashMap> classMap = new HashMap>(); - for ( Class c : PackageUtils.getClassesImplementingInterface(VariantEvaluator.class) ) + for ( Class c : new PluginManager( VariantEvaluator.class ).getPlugins() ) classMap.put(c.getSimpleName(), c); evaluationClasses = new HashSet>(); // by default, use standard eval modules if ( !NO_STANDARD ) { - for ( Class myClass : PackageUtils.getClassesImplementingInterface(StandardEval.class) ) { + for ( Class myClass : new PluginManager( StandardEval.class ).getPlugins() ) { if ( classMap.containsKey(myClass.getSimpleName()) ) evaluationClasses.add(classMap.get(myClass.getSimpleName())); } diff --git a/java/src/org/broadinstitute/sting/oneoffprojects/walkers/AnnotationByAlleleFrequencyWalker.java b/java/src/org/broadinstitute/sting/oneoffprojects/walkers/AnnotationByAlleleFrequencyWalker.java index 0e10b2c6c..926842b7f 100755 --- a/java/src/org/broadinstitute/sting/oneoffprojects/walkers/AnnotationByAlleleFrequencyWalker.java +++ b/java/src/org/broadinstitute/sting/oneoffprojects/walkers/AnnotationByAlleleFrequencyWalker.java @@ -31,13 +31,11 @@ import org.broadinstitute.sting.gatk.contexts.AlignmentContext; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; import org.broadinstitute.sting.gatk.walkers.RodWalker; -import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotationType; +import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotationInterfaceManager; import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.GenotypeAnnotation; import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation; import org.broadinstitute.sting.utils.GenomeLoc; -import org.broadinstitute.sting.utils.classloader.PackageUtils; import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException; -import org.broadinstitute.sting.utils.exceptions.UserException; import java.util.*; @@ -63,8 +61,8 @@ public class AnnotationByAlleleFrequencyWalker extends RodWalker requestedInfoAnnotations; - private ArrayList requestedGenotypeAnnotations; + private List requestedInfoAnnotations; + private List requestedGenotypeAnnotations; //--------------------------------------------------------------------------------------------------------------- @@ -74,49 +72,11 @@ public class AnnotationByAlleleFrequencyWalker extends RodWalker classMap = new HashMap(); - for ( Class c : PackageUtils.getClassesImplementingInterface(InfoFieldAnnotation.class) ) - classMap.put(c.getSimpleName(), c); - for ( Class c : PackageUtils.getClassesImplementingInterface(GenotypeAnnotation.class) ) - classMap.put(c.getSimpleName(), c); - for ( Class c : PackageUtils.getInterfacesExtendingInterface(AnnotationType.class) ) - classMap.put(c.getSimpleName(), c); - - HashSet classes = new HashSet(); - // get the classes from the provided groups (interfaces) - for ( String group : annotationClassesToUse ) { - Class interfaceClass = classMap.get(group); - if ( interfaceClass == null ) - interfaceClass = classMap.get(group + "Annotation"); - if ( interfaceClass == null ) - throw new UserException.BadArgumentValue("group", "Class " + group + " is not found; please check that you have specified the class name correctly"); - classes.addAll(PackageUtils.getClassesImplementingInterface(interfaceClass)); - } - - // get the specific classes provided - for ( String annotation : annotationsToUse ) { - Class annotationClass = classMap.get(annotation); - if ( annotationClass == null ) - annotationClass = classMap.get(annotation + "Annotation"); - if ( annotationClass == null ) - throw new UserException.BadArgumentValue("annotation", "Class " + annotation + " is not found; please check that you have specified the class name correctly"); - classes.add(annotationClass); - } - - // get the instances - requestedInfoAnnotations = new ArrayList(); - requestedGenotypeAnnotations = new ArrayList(); - - for ( Class c : classes ) { - // note that technically an annotation can work on both the INFO and FORMAT fields - if ( InfoFieldAnnotation.class.isAssignableFrom(c) ) - requestedInfoAnnotations.add((InfoFieldAnnotation)getInstance(c)); - if ( GenotypeAnnotation.class.isAssignableFrom(c) ) - requestedGenotypeAnnotations.add((GenotypeAnnotation)getInstance(c)); - } - + List annotationClasses = Arrays.asList(annotationClassesToUse); + List annotations = Arrays.asList(annotationsToUse); + AnnotationInterfaceManager.validateAnnotations(annotationClasses, annotations); + requestedInfoAnnotations = AnnotationInterfaceManager.createInfoFieldAnnotations(annotationClasses, annotations); + requestedGenotypeAnnotations = AnnotationInterfaceManager.createGenotypeAnnotations(annotationClasses, annotations); } private static ArrayList getInstances(List> classes) { diff --git a/java/src/org/broadinstitute/sting/queue/extensions/gatk/CommandLineProgramManager.java b/java/src/org/broadinstitute/sting/queue/extensions/gatk/CommandLineProgramManager.java deleted file mode 100644 index cefca44da..000000000 --- a/java/src/org/broadinstitute/sting/queue/extensions/gatk/CommandLineProgramManager.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2010, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.queue.extensions.gatk; - -import org.broadinstitute.sting.commandline.CommandLineProgram; -import org.broadinstitute.sting.utils.classloader.PluginManager; - -import java.util.Collection; - -/** - * Finds all command line programs. - */ -public class CommandLineProgramManager extends PluginManager { - public CommandLineProgramManager() { - super(CommandLineProgram.class, "CommandLineProgram", "CLP"); - } - - public Collection> getValues() { - return this.pluginsByName.values(); - } -} diff --git a/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java b/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java index 53736586b..06d7cadef 100644 --- a/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java +++ b/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java @@ -40,6 +40,7 @@ import org.broadinstitute.sting.gatk.io.stubs.SAMFileWriterArgumentTypeDescripto import org.broadinstitute.sting.gatk.refdata.tracks.builders.RMDTrackBuilder; import org.broadinstitute.sting.gatk.walkers.ReadWalker; import org.broadinstitute.sting.gatk.walkers.Walker; +import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import java.io.File; @@ -62,7 +63,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram { @Output(fullName="output_directory", shortName="outDir", doc="Directory to output the generated scala", required=true) public File outputDirectory; - CommandLineProgramManager clpManager = new CommandLineProgramManager(); + PluginManager clpManager = new PluginManager(CommandLineProgram.class, "CommandLineProgram", "CLP"); GenomeAnalysisEngine GATKEngine = new GenomeAnalysisEngine(); WalkerManager walkerManager = new WalkerManager(); FilterManager filterManager = new FilterManager(); @@ -97,7 +98,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram { if (!outputDirectory.isDirectory() && !outputDirectory.mkdirs()) throw new ReviewedStingException("Unable to create output directory: " + outputDirectory); - for (Class clp: clpManager.getValues()) { + for (Class clp: clpManager.getPlugins()) { if (!isGatkProgram(clp)) continue; diff --git a/java/src/org/broadinstitute/sting/utils/Utils.java b/java/src/org/broadinstitute/sting/utils/Utils.java index f902100ef..31fe6252d 100755 --- a/java/src/org/broadinstitute/sting/utils/Utils.java +++ b/java/src/org/broadinstitute/sting/utils/Utils.java @@ -312,6 +312,44 @@ public class Utils { return new Pair(max, index); } + /** + * Splits expressions in command args by spaces and returns the array of expressions. + * Expressions may use single or double quotes to group any individual expression, but not both. + * @param args Arguments to parse. + * @return Parsed expressions. + */ + public static String[] escapeExpressions(String args) { + // special case for ' and " so we can allow expressions + if (args.indexOf('\'') != -1) + return escapeExpressions(args, "'"); + else if (args.indexOf('\"') != -1) + return escapeExpressions(args, "\""); + else + return args.split(" "); + } + + /** + * Splits expressions in command args by spaces and the supplied delimiter and returns the array of expressions. + * @param args Arguments to parse. + * @param delimiter Delimiter for grouping expressions. + * @return Parsed expressions. + */ + private static String[] escapeExpressions(String args, String delimiter) { + String[] command = {}; + String[] split = args.split(delimiter); + for (int i = 0; i < split.length - 1; i += 2) { + command = Utils.concatArrays(command, split[i].trim().split(" ")); + command = Utils.concatArrays(command, new String[]{split[i + 1]}); + } + return Utils.concatArrays(command, split[split.length - 1].trim().split(" ")); + } + + /** + * Concatenates two String arrays. + * @param A First array. + * @param B Second array. + * @return Concatenation of A then B. + */ public static String[] concatArrays(String[] A, String[] B) { String[] C = new String[A.length + B.length]; System.arraycopy(A, 0, C, 0, A.length); @@ -319,8 +357,22 @@ public class Utils { return C; } + /** + * Appends String(s) B to array A. + * @param A First array. + * @param B Strings to append. + * @return A with B(s) appended. + */ + public static String[] appendArray(String[] A, String... B) { + return concatArrays(A, B); + } - /** Returns indices of all occurrences of the specified symbol in the string */ + /** + * Returns indices of all occurrences of the specified symbol in the string + * @param s Search string + * @param ch Character to search for + * @return Indices of all occurrences of the specified symbol + */ public static int[] indexOfAll(String s, int ch) { int[] pos = new int[64]; int z = 0; @@ -339,10 +391,10 @@ public class Utils { * with zeros up to the new size. Finally, if new size is the same as original size, no memory reallocation * will be performed and the original array will be returned instead. * - * @param orig - * @param newSize + * @param orig Original size. + * @param newSize New Size. * - * @return + * @return New array with length equal to newSize. */ public static int[] reallocate(int[] orig, int newSize) { if (orig.length == newSize) return orig; @@ -360,7 +412,8 @@ public class Utils { * array a. * @param a original array * @param n number of (v-filled) elements to append to a on the right (n>0) or on the left (n<0) - * @return + * @param v element value + * @return the extended copy of array a with additional n elements */ public static byte [] extend(final byte[] a, int n, byte v) { @@ -396,7 +449,8 @@ public class Utils { * array a. * @param a original array * @param n number of (v-filled) elements to append to a on the right (n>0) or on the left (n<0) - * @return + * @param v element value + * @return the extended copy of array a with additional n elements */ public static short [] extend(final short[] a, int n, short v) { diff --git a/java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java b/java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java index 83258aaf9..1aac9b565 100755 --- a/java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java +++ b/java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java @@ -26,15 +26,14 @@ package org.broadinstitute.sting.utils.classloader; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import org.reflections.util.ClasspathHelper; import java.lang.reflect.Modifier; import java.lang.reflect.Field; import java.io.File; import java.io.IOException; -import java.util.Collection; -import java.util.List; -import java.util.ArrayList; -import java.util.Arrays; +import java.net.URL; +import java.util.*; /** * Created by IntelliJ IDEA. @@ -178,4 +177,11 @@ public class JVMUtils { return selectedObjects; } + /** + * Returns the list of class path urls. + * @return the list of class path urls. + */ + public static Set getClasspathURLs() { + return ClasspathHelper.getUrlsForManifestsCurrentClasspath(); + } } diff --git a/java/src/org/broadinstitute/sting/utils/classloader/PackageUtils.java b/java/src/org/broadinstitute/sting/utils/classloader/PackageUtils.java deleted file mode 100755 index 7d49de573..000000000 --- a/java/src/org/broadinstitute/sting/utils/classloader/PackageUtils.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright (c) 2010 The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR - * THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.utils.classloader; - -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.Logger; -import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException; -import org.reflections.Reflections; -import org.reflections.scanners.SubTypesScanner; -import org.reflections.util.ClasspathHelper; -import org.reflections.util.ConfigurationBuilder; -import org.slf4j.LoggerFactory; - -import java.lang.reflect.Method; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.Set; -import java.util.ArrayList; -import java.util.List; - -/** - * PackageUtils contains some useful methods for package introspection. - */ -public class PackageUtils { - - /** - * A reference into our introspection utility. - */ - private static Reflections reflections = null; - - static { - // turn off logging in the reflections library - they talk too much (to the wrong logger factory as well, logback) - Logger logger = (ch.qos.logback.classic.Logger)LoggerFactory.getLogger(Reflections.class); - logger.setLevel(Level.OFF); - } - - /** - * Private constructor. No instantiating this class! - */ - private PackageUtils() {} - - /** - * Return the classes that implement the specified interface. - * - * @param iface the interface which returned classes should implement. - * @return the list of classes that implement the interface. - */ - public static List> getClassesImplementingInterface(Class iface) { - // Load all classes implementing the given interface, then filter out any class that isn't concrete. - initReflections(); - Set> allTypes = reflections.getSubTypesOf(iface); - List> concreteTypes = new ArrayList>(); - for( Class type: allTypes ) { - if( JVMUtils.isConcrete(type) ) - concreteTypes.add(type); - } - - return concreteTypes; - } - - public static List getInstancesOfClassesImplementingInterface(Class iface) { - List> classes = PackageUtils.getClassesImplementingInterface(iface); - List instances = new ArrayList(); - for ( Class c : classes ) - instances.add(getSimpleInstance(c)); - return instances; - } - - public static T getSimpleInstance(Class c) { - try { - return c.newInstance(); - } catch (Exception e) { - throw new DynamicClassResolutionException(c, e); - } - } - - /** - * Return the interface classes that extend the specified interface. - * - * @param iface the interface which returned classes should extend. - * @return the list of interface classes that implement the interface. - */ - public static List> getInterfacesExtendingInterface(Class iface) { - // Load all classes extending the given interface, then filter out any class that is concrete. - initReflections(); - Set> allTypes = reflections.getSubTypesOf(iface); - List> nonConcreteTypes = new ArrayList>(); - for( Class type: allTypes ) { - if( !JVMUtils.isConcrete(type) ) - nonConcreteTypes.add(type); - } - - return nonConcreteTypes; - } - - public static Set getClassPathURLs() { - return ClasspathHelper.getUrlsForManifestsCurrentClasspath(); - } - - /** - * Adds the URL to the system class loader classpath using reflection. - * HACK: Uses reflection to modify the class path, and assumes loader is a URLClassLoader. - * @param url URL to add to the system class loader classpath. - */ - public static void addClasspath(URL url) { - try { - Method method = URLClassLoader.class.getDeclaredMethod("addURL", URL.class); - if (!method.isAccessible()) - method.setAccessible(true); - method.invoke(ClassLoader.getSystemClassLoader(), url); - resetReflections(); - } catch (Exception e) { - throw new ReviewedStingException("Error adding url to the current classloader.", e); - } - } - - /** - * Create new reflections object if it does not currently exists. - */ - private static void initReflections() { - if (reflections == null) { - // Initialize general-purpose source tree reflector. - reflections = new Reflections( new ConfigurationBuilder() - .setUrls(getClassPathURLs()) - .setScanners(new SubTypesScanner())); - } - } - - /** - * Resets the reflections object after a class has been dynamically added to the classpath. - */ - private static void resetReflections() { - reflections = null; - } -} diff --git a/java/src/org/broadinstitute/sting/utils/classloader/PluginManager.java b/java/src/org/broadinstitute/sting/utils/classloader/PluginManager.java index 463e25cd3..c34f3c75d 100644 --- a/java/src/org/broadinstitute/sting/utils/classloader/PluginManager.java +++ b/java/src/org/broadinstitute/sting/utils/classloader/PluginManager.java @@ -25,20 +25,42 @@ package org.broadinstitute.sting.utils.classloader; -import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; +import org.reflections.Reflections; +import org.reflections.scanners.SubTypesScanner; +import org.reflections.util.ConfigurationBuilder; +import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.*; /** * Manage plugins and plugin configuration. * @author mhanna * @version 0.1 */ -public abstract class PluginManager { +public class PluginManager { + /** + * A reference into our introspection utility. + */ + private static final Reflections defaultReflections; + + static { + // turn off logging in the reflections library - they talk too much (to the wrong logger factory as well, logback) + Logger logger = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Reflections.class); + logger.setLevel(Level.OFF); + + defaultReflections = new Reflections( new ConfigurationBuilder() + .setUrls(JVMUtils.getClasspathURLs()) + .setScanners(new SubTypesScanner())); + } + /** * Defines the category of plugin defined by the subclass. */ @@ -48,11 +70,31 @@ public abstract class PluginManager { * Define common strings to trim off the end of the name. */ protected final String pluginSuffix; - + /** * Plugins stored based on their name. */ - protected final Map> pluginsByName; + private SortedMap> pluginsByName = null; + + private List> plugins; + private List> interfaces; + + /** + * Create a new plugin manager. + * @param pluginType Core type for a plugin. + */ + public PluginManager(Class pluginType) { + this(pluginType, pluginType.getSimpleName().toLowerCase(), pluginType.getSimpleName(), null); + } + + /** + * Create a new plugin manager. + * @param pluginType Core type for a plugin. + * @param classpath Custom class path to search for classes. + */ + public PluginManager(Class pluginType, List classpath) { + this(pluginType, pluginType.getSimpleName().toLowerCase(), pluginType.getSimpleName(), classpath); + } /** * Create a new plugin manager. @@ -60,11 +102,75 @@ public abstract class PluginManager { * @param pluginCategory Provides a category name to the plugin. Must not be null. * @param pluginSuffix Provides a suffix that will be trimmed off when converting to a plugin name. Can be null. */ - protected PluginManager(Class pluginType, String pluginCategory, String pluginSuffix) { + public PluginManager(Class pluginType, String pluginCategory, String pluginSuffix) { + this(pluginType, pluginCategory, pluginSuffix, null); + } + + /** + * Create a new plugin manager. + * @param pluginType Core type for a plugin. + * @param pluginCategory Provides a category name to the plugin. Must not be null. + * @param pluginSuffix Provides a suffix that will be trimmed off when converting to a plugin name. Can be null. + * @param classpath Custom class path to search for classes. + */ + public PluginManager(Class pluginType, String pluginCategory, String pluginSuffix, List classpath) { this.pluginCategory = pluginCategory; this.pluginSuffix = pluginSuffix; - List> plugins = PackageUtils.getClassesImplementingInterface(pluginType); - pluginsByName = createPluginDatabase(plugins); + + this.plugins = new ArrayList>(); + this.interfaces = new ArrayList>(); + + Reflections reflections; + if (classpath == null) { + reflections = defaultReflections; + } else { + addClasspath(classpath); + reflections = new Reflections( new ConfigurationBuilder() + .setUrls(classpath) + .setScanners(new SubTypesScanner())); + } + + // Load all classes types filtering them by concrete. + Set> allTypes = reflections.getSubTypesOf(pluginType); + for( Class type: allTypes ) { + if( JVMUtils.isConcrete(type) ) + plugins.add(type); + else + interfaces.add(type); + } + } + + /** + * Adds the URL to the system class loader classpath using reflection. + * HACK: Uses reflection to modify the class path, and assumes loader is a URLClassLoader. + * @param urls URLs to add to the system class loader classpath. + */ + private static void addClasspath(List urls) { + Set existing = JVMUtils.getClasspathURLs(); + for (URL url : urls) { + if (existing.contains(url)) + continue; + try { + Method method = URLClassLoader.class.getDeclaredMethod("addURL", URL.class); + if (!method.isAccessible()) + method.setAccessible(true); + method.invoke(ClassLoader.getSystemClassLoader(), url); + } catch (Exception e) { + throw new ReviewedStingException("Error adding url to the current classloader.", e); + } + } + } + + protected SortedMap> getPluginsByName() { + if (pluginsByName == null) { + SortedMap> newPlugins = new TreeMap>(); + for (Class pluginClass : plugins) { + String pluginName = getName(pluginClass); + newPlugins.put(pluginName, pluginClass); + } + pluginsByName = newPlugins; + } + return pluginsByName; } /** @@ -74,9 +180,49 @@ public abstract class PluginManager { * @return True if the plugin exists, false otherwise. */ public boolean exists(String pluginName) { - return pluginsByName.containsKey(pluginName); + return getPluginsByName().containsKey(pluginName); } + /** + * Does a plugin with the given name exist? + * + * @param plugin Name of the plugin for which to search. + * @return True if the plugin exists, false otherwise. + */ + public boolean exists(Class plugin) { + return getPluginsByName().containsValue(plugin); + } + + /** + * Returns the plugin classes + * @return the plugin classes + */ + public List> getPlugins() { + return plugins; + } + + /** + * Returns the interface classes + * @return the interface classes + */ + public List> getInterfaces() { + return interfaces; + } + + /** + * Returns the plugin classes implementing interface or base clase + * @param type type of interface or base class + * @return the plugin classes implementing interface or base class + */ + public List> getPluginsImplementing(Class type) { + List> implementing = new ArrayList>(); + for (Class plugin: getPlugins()) + if (type.isAssignableFrom(plugin)) + implementing.add(plugin); + return implementing; + } + + /** * Gets a plugin with the given name @@ -85,7 +231,7 @@ public abstract class PluginManager { * @return The plugin object if found; null otherwise. */ public PluginType createByName(String pluginName) { - Class plugin = pluginsByName.get(pluginName); + Class plugin = getPluginsByName().get(pluginName); if( plugin == null ) throw new UserException(String.format("Could not find %s with name: %s", pluginCategory,pluginName)); try { @@ -101,6 +247,7 @@ public abstract class PluginManager { * @param pluginType type of the plugin to create. * @return The plugin object if created; null otherwise. */ + @SuppressWarnings("unchecked") public PluginType createByType(Class pluginType) { try { return ((Class) pluginType).newInstance(); @@ -110,20 +257,19 @@ public abstract class PluginManager { } /** - * Create the list of available plugins and add them to the database. - * - * @param pluginClasses Classes to record. - * @return map of plugin name -> plugin. + * Returns concrete instances of the plugins + * @return concrete instances of the plugins */ - private Map> createPluginDatabase(List> pluginClasses) { - Map> plugins = new HashMap>(); - - for (Class pluginClass : pluginClasses) { - String pluginName = getName(pluginClass); - plugins.put(pluginName, pluginClass); + public List createAllTypes() { + List instances = new ArrayList(); + for ( Class c : getPlugins() ) { + try { + instances.add(c.newInstance()); + } catch (Exception e) { + throw new DynamicClassResolutionException(c, e); + } } - - return plugins; + return instances; } /** diff --git a/java/src/org/broadinstitute/sting/utils/interval/IntervalUtils.java b/java/src/org/broadinstitute/sting/utils/interval/IntervalUtils.java index a2967d455..7b10ccf86 100644 --- a/java/src/org/broadinstitute/sting/utils/interval/IntervalUtils.java +++ b/java/src/org/broadinstitute/sting/utils/interval/IntervalUtils.java @@ -1,8 +1,12 @@ package org.broadinstitute.sting.utils.interval; +import net.sf.picard.util.IntervalList; +import net.sf.samtools.SAMFileHeader; +import org.broadinstitute.sting.gatk.datasources.simpleDataSources.ReferenceDataSource; import org.broadinstitute.sting.utils.GenomeLocSortedSet; import org.broadinstitute.sting.utils.GenomeLoc; import org.broadinstitute.sting.utils.GenomeLocParser; +import org.broadinstitute.sting.utils.Utils; import org.broadinstitute.sting.utils.exceptions.UserException; import java.util.LinkedList; @@ -25,6 +29,7 @@ public class IntervalUtils { * 'all' can be supplied to indicate all possible intervals, but 'all' must be exclusive of all other interval * specifications. * + * @param parser Genome loc parser. * @param argList A list of strings containing interval data. * @param allowEmptyIntervalList If false instead of an empty interval list will return null. * @return an unsorted, unmerged representation of the given intervals. Null is used to indicate that all intervals should be used. @@ -107,7 +112,7 @@ public class IntervalUtils { if (setOne.get(iOne).getStop() < setTwo.get(iTwo).getStop()) iOne++; else iTwo++; } - + // we don't need to add the rest of remaining locations, since we know they don't overlap. return what we have return retList; } @@ -117,6 +122,7 @@ public class IntervalUtils { * all overlapping and abutting intervals into an interval that spans the union of all covered bases, and * OVERLAPPING_ONLY, which unions overlapping intervals but keeps abutting intervals separate. * + * @param parser Genome loc parser for the intervals. * @param intervals A collection of intervals to merge. * @param mergingRule A descriptor for the type of merging to perform. * @return A sorted, merged version of the intervals passed in. @@ -158,6 +164,131 @@ public class IntervalUtils { else return false; } + /** + * Returns the list of GenomeLocs from the list of intervals. + * @param referenceSource The reference for the intervals. + * @param intervals The interval as strings or file paths. + * @return The list of GenomeLocs. + */ + private static List parseIntervalArguments(ReferenceDataSource referenceSource, List intervals) { + GenomeLocParser parser = new GenomeLocParser(referenceSource.getReference()); + GenomeLocSortedSet locs; + // TODO: Abstract genome analysis engine has richer logic for parsing. We need to use it! + if (intervals.size() == 0) { + locs = GenomeLocSortedSet.createSetFromSequenceDictionary(referenceSource.getReference().getSequenceDictionary()); + } else { + locs = new GenomeLocSortedSet(parser, IntervalUtils.parseIntervalArguments(parser, intervals, false)); + } + if (locs == null || locs.size() == 0) + throw new UserException.MalformedFile("Intervals are empty: " + Utils.join(", ", intervals)); + return locs.toList(); + } + + /** + * Returns the list of contigs from the list of intervals. + * @param reference The reference for the intervals. + * @return The list of contig names. + */ + public static List distinctContigs(File reference) { + return distinctContigs(reference, Collections.emptyList()); + } + + /** + * Returns the list of contigs from the list of intervals. + * @param reference The reference for the intervals. + * @param intervals The interval as strings or file paths. + * @return The list of contig names. + */ + public static List distinctContigs(File reference, List intervals) { + ReferenceDataSource referenceSource = new ReferenceDataSource(reference); + List locs = parseIntervalArguments(referenceSource, intervals); + String contig = null; + List contigs = new ArrayList(); + for (GenomeLoc loc: locs) { + if (contig == null || !contig.equals(loc.getContig())) { + contig = loc.getContig(); + contigs.add(contig); + } + } + return contigs; + } + + /** + * Splits an interval list into multiple files. + * @param reference The reference for the intervals. + * @param intervals The interval as strings or file paths. + * @param scatterParts The output interval lists to write to. + * @param splitByContig If true then one contig will not be written to multiple files. + */ + public static void scatterIntervalArguments(File reference, List intervals, List scatterParts, boolean splitByContig) { + ReferenceDataSource referenceSource = new ReferenceDataSource(reference); + List locs = parseIntervalArguments(referenceSource, intervals); + SAMFileHeader fileHeader = new SAMFileHeader(); + fileHeader.setSequenceDictionary(referenceSource.getReference().getSequenceDictionary()); + + IntervalList intervalList = null; + int fileIndex = -1; + int locIndex = 0; + + if (splitByContig) { + String contig = null; + for (GenomeLoc loc: locs) { + // If there are still more files to write and the contig doesn't match... + if ((fileIndex+1 < scatterParts.size()) && (contig == null || !contig.equals(loc.getContig()))) { + // Then close the current file and start a new one. + if (intervalList != null) { + intervalList.write(scatterParts.get(fileIndex)); + intervalList = null; + } + fileIndex++; + contig = loc.getContig(); + } + if (intervalList == null) + intervalList = new IntervalList(fileHeader); + intervalList.add(toInterval(loc, ++locIndex)); + } + if (intervalList != null) + intervalList.write(scatterParts.get(fileIndex)); + } else { + int locsPerFile = locs.size() / scatterParts.size(); + int locRemainder = locs.size() % scatterParts.size(); + + // At the start, put an extra loc per file + locsPerFile++; + int locsLeftFile = 0; + + for (GenomeLoc loc: locs) { + if (locsLeftFile == 0) { + if (intervalList != null) + intervalList.write(scatterParts.get(fileIndex)); + + fileIndex++; + intervalList = new IntervalList(fileHeader); + + // When we have put enough locs into each file, + // reduce the number of locs per file back + // to the original calculated value. + if (fileIndex == locRemainder) + locsPerFile -= 1; + locsLeftFile = locsPerFile; + } + locsLeftFile -= 1; + intervalList.add(toInterval(loc, ++locIndex)); + } + if (intervalList != null) + intervalList.write(scatterParts.get(fileIndex)); + } + if ((fileIndex + 1) != scatterParts.size()) + throw new UserException.BadArgumentValue("scatterParts", String.format("Only able to write contigs into %d of %d files.", fileIndex + 1, scatterParts.size())); + } + + /** + * Converts a GenomeLoc to a picard interval. + * @param loc The GenomeLoc. + * @param locIndex The loc index for use in the file. + * @return The picard interval. + */ + private static net.sf.picard.util.Interval toInterval(GenomeLoc loc, int locIndex) { + return new net.sf.picard.util.Interval(loc.getContig(), loc.getStart(), loc.getStop(), false, "interval_" + locIndex); + } } - - diff --git a/java/test/org/broadinstitute/sting/WalkerTest.java b/java/test/org/broadinstitute/sting/WalkerTest.java index e8712fcb0..22a5180d5 100755 --- a/java/test/org/broadinstitute/sting/WalkerTest.java +++ b/java/test/org/broadinstitute/sting/WalkerTest.java @@ -311,35 +311,19 @@ public class WalkerTest extends BaseTest { */ private Pair, List> executeTest(String name, List md5s, List tmpFiles, String args, Class expectedException) { CommandLineGATK instance = new CommandLineGATK(); - String[] command; + String[] command = Utils.escapeExpressions(args); - // special case for ' and " so we can allow expressions - if (args.indexOf('\'') != -1) - command = escapeExpressions(args, "'"); - else if (args.indexOf('\"') != -1) - command = escapeExpressions(args, "\""); - else - command = args.split(" "); - - if (outputFileLocation != null) { - String[] cmd2 = Arrays.copyOf(command, command.length + 2); - cmd2[command.length] = "-o"; - cmd2[command.length + 1] = this.outputFileLocation.getAbsolutePath(); - command = cmd2; - } + if (outputFileLocation != null) + command = Utils.appendArray(command, "-o", this.outputFileLocation.getAbsolutePath()); // add the logging level to each of the integration test commands - String[] cmd2 = Arrays.copyOf(command, command.length + 4); - cmd2[command.length] = "-l"; - cmd2[command.length+1] = "WARN"; - cmd2[command.length+2] = "-et"; - cmd2[command.length+3] = ENABLE_REPORTING ? "STANDARD" : "NO_ET"; + command = Utils.appendArray(command, "-l", "WARN", "-et", ENABLE_REPORTING ? "STANDARD" : "NO_ET"); // run the executable boolean gotAnException = false; try { - System.out.println(String.format("Executing test %s with GATK arguments: %s", name, Utils.join(" ",cmd2))); - CommandLineExecutable.start(instance, cmd2); + System.out.println(String.format("Executing test %s with GATK arguments: %s", name, Utils.join(" ",command))); + CommandLineExecutable.start(instance, command); } catch (Exception e) { gotAnException = true; if ( expectedException != null ) { @@ -370,25 +354,11 @@ public class WalkerTest extends BaseTest { throw new RuntimeException("Error running the GATK with arguments: " + args); } - // clean up some memory - instance = null; - cmd2 = null; - // we need to check MD5s return new Pair, List>(tmpFiles, assertMatchingMD5s(name, tmpFiles, md5s)); } } - private static String[] escapeExpressions(String args, String delimiter) { - String[] command = {}; - String[] split = args.split(delimiter); - for (int i = 0; i < split.length - 1; i += 2) { - command = Utils.concatArrays(command, split[i].trim().split(" ")); - command = Utils.concatArrays(command, new String[]{split[i + 1]}); - } - return Utils.concatArrays(command, split[split.length - 1].trim().split(" ")); - } - @Test public void testWalkerUnitTest() { //System.out.println("WalkerTest is just a framework"); diff --git a/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsTest.java b/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsTest.java deleted file mode 100644 index d4fcb8b9f..000000000 --- a/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsTest.java +++ /dev/null @@ -1,91 +0,0 @@ -package org.broadinstitute.sting.utils.interval; - -import net.sf.picard.reference.IndexedFastaSequenceFile; -import net.sf.picard.reference.ReferenceSequenceFile; -import org.broadinstitute.sting.BaseTest; -import org.testng.Assert; -import org.broadinstitute.sting.utils.GenomeLoc; -import org.broadinstitute.sting.utils.GenomeLocParser; - -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -import java.io.File; -import java.io.FileNotFoundException; -import java.util.ArrayList; -import java.util.List; - -/** - * test out the interval utility methods - */ -public class IntervalUtilsTest extends BaseTest { - // used to seed the genome loc parser with a sequence dictionary - private static ReferenceSequenceFile seq; - private GenomeLocParser genomeLocParser; - - @BeforeClass - public void init() throws FileNotFoundException { - seq = new IndexedFastaSequenceFile(new File(hg18Reference)); - genomeLocParser = new GenomeLocParser(seq); - - } - - @Test - public void testMergeListsBySetOperatorNoOverlap() { - // a couple of lists we'll use for the testing - List listEveryTwoFromOne = new ArrayList(); - List listEveryTwoFromTwo = new ArrayList(); - - // create the two lists we'll use - for (int x = 1; x < 101; x++) { - if (x % 2 == 0) - listEveryTwoFromTwo.add(genomeLocParser.createGenomeLoc("chr1",x,x)); - else - listEveryTwoFromOne.add(genomeLocParser.createGenomeLoc("chr1",x,x)); - } - - List ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, listEveryTwoFromOne, IntervalSetRule.UNION); - Assert.assertEquals(ret.size(), 100); - ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, listEveryTwoFromOne, IntervalSetRule.INTERSECTION); - Assert.assertEquals(ret.size(), 0); - } - - @Test - public void testMergeListsBySetOperatorAllOverlap() { - // a couple of lists we'll use for the testing - List allSites = new ArrayList(); - List listEveryTwoFromTwo = new ArrayList(); - - // create the two lists we'll use - for (int x = 1; x < 101; x++) { - if (x % 2 == 0) - listEveryTwoFromTwo.add(genomeLocParser.createGenomeLoc("chr1",x,x)); - allSites.add(genomeLocParser.createGenomeLoc("chr1",x,x)); - } - - List ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.UNION); - Assert.assertEquals(ret.size(), 150); - ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.INTERSECTION); - Assert.assertEquals(ret.size(), 50); - } - - @Test - public void testMergeListsBySetOperator() { - // a couple of lists we'll use for the testing - List allSites = new ArrayList(); - List listEveryTwoFromTwo = new ArrayList(); - - // create the two lists we'll use - for (int x = 1; x < 101; x++) { - if (x % 5 == 0) { - listEveryTwoFromTwo.add(genomeLocParser.createGenomeLoc("chr1",x,x)); - allSites.add(genomeLocParser.createGenomeLoc("chr1",x,x)); - } - } - - List ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.UNION); - Assert.assertEquals(ret.size(), 40); - ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.INTERSECTION); - Assert.assertEquals(ret.size(), 20); - } -} diff --git a/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsUnitTest.java b/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsUnitTest.java new file mode 100644 index 000000000..9bd541866 --- /dev/null +++ b/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsUnitTest.java @@ -0,0 +1,356 @@ +package org.broadinstitute.sting.utils.interval; + +import net.sf.picard.reference.IndexedFastaSequenceFile; +import net.sf.picard.reference.ReferenceSequenceFile; +import org.broadinstitute.sting.BaseTest; +import org.testng.Assert; +import org.broadinstitute.sting.utils.exceptions.UserException; +import org.broadinstitute.sting.utils.GenomeLoc; +import org.broadinstitute.sting.utils.GenomeLocParser; + +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * test out the interval utility methods + */ +public class IntervalUtilsUnitTest extends BaseTest { + // used to seed the genome loc parser with a sequence dictionary + private static File reference = new File(BaseTest.hg18Reference); + private GenomeLocParser genomeLocParser; + + @BeforeClass + public void init() { + ReferenceSequenceFile seq = new IndexedFastaSequenceFile(reference); + genomeLocParser = new GenomeLocParser(seq); + } + + @Test + public void testMergeListsBySetOperatorNoOverlap() { + // a couple of lists we'll use for the testing + List listEveryTwoFromOne = new ArrayList(); + List listEveryTwoFromTwo = new ArrayList(); + + // create the two lists we'll use + for (int x = 1; x < 101; x++) { + if (x % 2 == 0) + listEveryTwoFromTwo.add(genomeLocParser.createGenomeLoc("chr1",x,x)); + else + listEveryTwoFromOne.add(genomeLocParser.createGenomeLoc("chr1",x,x)); + } + + List ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, listEveryTwoFromOne, IntervalSetRule.UNION); + Assert.assertEquals(ret.size(), 100); + ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, listEveryTwoFromOne, IntervalSetRule.INTERSECTION); + Assert.assertEquals(ret.size(), 0); + } + + @Test + public void testMergeListsBySetOperatorAllOverlap() { + // a couple of lists we'll use for the testing + List allSites = new ArrayList(); + List listEveryTwoFromTwo = new ArrayList(); + + // create the two lists we'll use + for (int x = 1; x < 101; x++) { + if (x % 2 == 0) + listEveryTwoFromTwo.add(genomeLocParser.createGenomeLoc("chr1",x,x)); + allSites.add(genomeLocParser.createGenomeLoc("chr1",x,x)); + } + + List ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.UNION); + Assert.assertEquals(ret.size(), 150); + ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.INTERSECTION); + Assert.assertEquals(ret.size(), 50); + } + + @Test + public void testMergeListsBySetOperator() { + // a couple of lists we'll use for the testing + List allSites = new ArrayList(); + List listEveryTwoFromTwo = new ArrayList(); + + // create the two lists we'll use + for (int x = 1; x < 101; x++) { + if (x % 5 == 0) { + listEveryTwoFromTwo.add(genomeLocParser.createGenomeLoc("chr1",x,x)); + allSites.add(genomeLocParser.createGenomeLoc("chr1",x,x)); + } + } + + List ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.UNION); + Assert.assertEquals(ret.size(), 40); + ret = IntervalUtils.mergeListsBySetOperator(listEveryTwoFromTwo, allSites, IntervalSetRule.INTERSECTION); + Assert.assertEquals(ret.size(), 20); + } + + @Test + public void testCountContigs() { + List chrs = new ArrayList(); + for (int i = 1; i <= 22; i++) + chrs.add("chr" + i); + chrs.add("chrX"); + chrs.add("chrY"); + + List chrsNoRandom = Arrays.asList("chr12", "chr14", "chr20", "chrY"); + List chrsWithRandom = new ArrayList(); + chrsWithRandom.add("chrM"); + chrsWithRandom.addAll(chrs); + for (String chr: chrs) + if(!chrsNoRandom.contains(chr)) + chrsWithRandom.add(chr + "_random"); + + Assert.assertEquals(IntervalUtils.distinctContigs(reference), chrsWithRandom); + Assert.assertEquals(IntervalUtils.distinctContigs(reference, Arrays.asList(BaseTest.validationDataLocation + "TCGA-06-0188.interval_list")), chrs); + Assert.assertEquals(IntervalUtils.distinctContigs(reference, Arrays.asList("chr1:1-1", "chr2:1-1", "chr3:2-2")), Arrays.asList("chr1","chr2","chr3")); + Assert.assertEquals(IntervalUtils.distinctContigs(reference, Arrays.asList("chr2:1-1", "chr1:1-1", "chr3:2-2")), Arrays.asList("chr1","chr2","chr3")); + } + + @Test + public void testBasicScatter() { + GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1"); + GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2"); + GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3"); + + List files = testFiles("basic.", 3, ".intervals"); + + IntervalUtils.scatterIntervalArguments(reference, Arrays.asList("chr1", "chr2", "chr3"), files, false); + + List locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false); + List locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false); + List locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false); + + Assert.assertEquals(locs1.size(), 1); + Assert.assertEquals(locs2.size(), 1); + Assert.assertEquals(locs3.size(), 1); + + Assert.assertEquals(locs1.get(0), chr1); + Assert.assertEquals(locs2.get(0), chr2); + Assert.assertEquals(locs3.get(0), chr3); + } + + @Test + public void testScatterLessFiles() { + GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1"); + GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2"); + GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3"); + GenomeLoc chr4 = genomeLocParser.parseGenomeInterval("chr4"); + + List files = testFiles("less.", 3, ".intervals"); + + IntervalUtils.scatterIntervalArguments(reference, Arrays.asList("chr1", "chr2", "chr3", "chr4"), files, false); + + List locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false); + List locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false); + List locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false); + + Assert.assertEquals(locs1.size(), 2); + Assert.assertEquals(locs2.size(), 1); + Assert.assertEquals(locs3.size(), 1); + + Assert.assertEquals(locs1.get(0), chr1); + Assert.assertEquals(locs1.get(1), chr2); + Assert.assertEquals(locs2.get(0), chr3); + Assert.assertEquals(locs3.get(0), chr4); + } + + @Test(expectedExceptions=UserException.BadArgumentValue.class) + public void testScatterMoreFiles() { + List files = testFiles("more.", 3, ".intervals"); + IntervalUtils.scatterIntervalArguments(reference, Arrays.asList("chr1", "chr2"), files, false); + } + + @Test + public void testScatterIntervals() { + List intervals = Arrays.asList("chr1:1-2", "chr1:4-5", "chr2:1-1", "chr3:2-2"); + GenomeLoc chr1a = genomeLocParser.parseGenomeInterval("chr1:1-2"); + GenomeLoc chr1b = genomeLocParser.parseGenomeInterval("chr1:4-5"); + GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2:1-1"); + GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3:2-2"); + + List files = testFiles("split.", 3, ".intervals"); + + IntervalUtils.scatterIntervalArguments(reference, intervals, files, true); + + List locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false); + List locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false); + List locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false); + + Assert.assertEquals(locs1.size(), 2); + Assert.assertEquals(locs2.size(), 1); + Assert.assertEquals(locs3.size(), 1); + + Assert.assertEquals(locs1.get(0), chr1a); + Assert.assertEquals(locs1.get(1), chr1b); + Assert.assertEquals(locs2.get(0), chr2); + Assert.assertEquals(locs3.get(0), chr3); + } + + @Test + public void testScatterOrder() { + List intervals = Arrays.asList("chr2:1-1", "chr1:1-1", "chr3:2-2"); + GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1:1-1"); + GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2:1-1"); + GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3:2-2"); + + List files = testFiles("split.", 3, ".intervals"); + + IntervalUtils.scatterIntervalArguments(reference, intervals, files, true); + + List locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false); + List locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false); + List locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false); + + Assert.assertEquals(locs1.size(), 1); + Assert.assertEquals(locs2.size(), 1); + Assert.assertEquals(locs3.size(), 1); + + Assert.assertEquals(locs1.get(0), chr1); + Assert.assertEquals(locs2.get(0), chr2); + Assert.assertEquals(locs3.get(0), chr3); + } + + @Test + public void testBasicScatterByContig() { + GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1"); + GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2"); + GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3"); + + List files = testFiles("contig_basic.", 3, ".intervals"); + + IntervalUtils.scatterIntervalArguments(reference, Arrays.asList("chr1", "chr2", "chr3"), files, true); + + List locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false); + List locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false); + List locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false); + + Assert.assertEquals(locs1.size(), 1); + Assert.assertEquals(locs2.size(), 1); + Assert.assertEquals(locs3.size(), 1); + + Assert.assertEquals(locs1.get(0), chr1); + Assert.assertEquals(locs2.get(0), chr2); + Assert.assertEquals(locs3.get(0), chr3); + } + + @Test + public void testScatterByContigLessFiles() { + GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1"); + GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2"); + GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3"); + GenomeLoc chr4 = genomeLocParser.parseGenomeInterval("chr4"); + + List files = testFiles("contig_less.", 3, ".intervals"); + + IntervalUtils.scatterIntervalArguments(reference, Arrays.asList("chr1", "chr2", "chr3", "chr4"), files, true); + + List locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false); + List locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false); + List locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false); + + Assert.assertEquals(locs1.size(), 1); + Assert.assertEquals(locs2.size(), 1); + Assert.assertEquals(locs3.size(), 2); + + Assert.assertEquals(locs1.get(0), chr1); + Assert.assertEquals(locs2.get(0), chr2); + Assert.assertEquals(locs3.get(0), chr3); + Assert.assertEquals(locs3.get(1), chr4); + } + + @Test(expectedExceptions=UserException.BadArgumentValue.class) + public void testScatterByContigMoreFiles() { + List files = testFiles("contig_more.", 3, ".intervals"); + IntervalUtils.scatterIntervalArguments(reference, Arrays.asList("chr1", "chr2"), files, true); + } + + @Test + public void testScatterByContigIntervalsStart() { + List intervals = Arrays.asList("chr1:1-2", "chr1:4-5", "chr2:1-1", "chr3:2-2"); + GenomeLoc chr1a = genomeLocParser.parseGenomeInterval("chr1:1-2"); + GenomeLoc chr1b = genomeLocParser.parseGenomeInterval("chr1:4-5"); + GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2:1-1"); + GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3:2-2"); + + List files = testFiles("contig_split_start.", 3, ".intervals"); + + IntervalUtils.scatterIntervalArguments(reference, intervals, files, true); + + List locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false); + List locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false); + List locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false); + + Assert.assertEquals(locs1.size(), 2); + Assert.assertEquals(locs2.size(), 1); + Assert.assertEquals(locs3.size(), 1); + + Assert.assertEquals(locs1.get(0), chr1a); + Assert.assertEquals(locs1.get(1), chr1b); + Assert.assertEquals(locs2.get(0), chr2); + Assert.assertEquals(locs3.get(0), chr3); + } + + @Test + public void testScatterByContigIntervalsMiddle() { + List intervals = Arrays.asList("chr1:1-1", "chr2:1-2", "chr2:4-5", "chr3:2-2"); + GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1:1-1"); + GenomeLoc chr2a = genomeLocParser.parseGenomeInterval("chr2:1-2"); + GenomeLoc chr2b = genomeLocParser.parseGenomeInterval("chr2:4-5"); + GenomeLoc chr3 = genomeLocParser.parseGenomeInterval("chr3:2-2"); + + List files = testFiles("contig_split_middle.", 3, ".intervals"); + + IntervalUtils.scatterIntervalArguments(reference, intervals, files, true); + + List locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false); + List locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false); + List locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false); + + Assert.assertEquals(locs1.size(), 1); + Assert.assertEquals(locs2.size(), 2); + Assert.assertEquals(locs3.size(), 1); + + Assert.assertEquals(locs1.get(0), chr1); + Assert.assertEquals(locs2.get(0), chr2a); + Assert.assertEquals(locs2.get(1), chr2b); + Assert.assertEquals(locs3.get(0), chr3); + } + + @Test + public void testScatterByContigIntervalsEnd() { + List intervals = Arrays.asList("chr1:1-1", "chr2:2-2", "chr3:1-2", "chr3:4-5"); + GenomeLoc chr1 = genomeLocParser.parseGenomeInterval("chr1:1-1"); + GenomeLoc chr2 = genomeLocParser.parseGenomeInterval("chr2:2-2"); + GenomeLoc chr3a = genomeLocParser.parseGenomeInterval("chr3:1-2"); + GenomeLoc chr3b = genomeLocParser.parseGenomeInterval("chr3:4-5"); + + List files = testFiles("contig_split_end.", 3 ,".intervals"); + + IntervalUtils.scatterIntervalArguments(reference, intervals, files, true); + + List locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(0).toString()), false); + List locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(1).toString()), false); + List locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser, Arrays.asList(files.get(2).toString()), false); + + Assert.assertEquals(locs1.size(), 1); + Assert.assertEquals(locs2.size(), 1); + Assert.assertEquals(locs3.size(), 2); + + Assert.assertEquals(locs1.get(0), chr1); + Assert.assertEquals(locs2.get(0), chr2); + Assert.assertEquals(locs3.get(0), chr3a); + Assert.assertEquals(locs3.get(1), chr3b); + } + + private List testFiles(String prefix, int count, String suffix) { + ArrayList files = new ArrayList(); + for (int i = 1; i <= count; i++) + files.add(new File(testDir + prefix + i + suffix)); + return files; + } +} diff --git a/scala/qscript/fullCallingPipeline.q b/scala/qscript/fullCallingPipeline.q index 5aa9b89e9..ca907f997 100755 --- a/scala/qscript/fullCallingPipeline.q +++ b/scala/qscript/fullCallingPipeline.q @@ -9,6 +9,7 @@ import org.broadinstitute.sting.queue.function.scattergather.{GatherFunction, Cl import org.broadinstitute.sting.queue.util.IOUtils import org.broadinstitute.sting.queue.QScript import collection.JavaConversions._ +import org.broadinstitute.sting.utils.interval.IntervalUtils import org.broadinstitute.sting.utils.yaml.YamlUtils import org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType @@ -91,9 +92,9 @@ class fullCallingPipeline extends QScript { //val expKind = qscript.protocol // get contigs (needed for indel cleaning parallelism) - val contigs = IntervalScatterFunction.distinctContigs( + val contigs = IntervalUtils.distinctContigs( qscript.pipeline.getProject.getReferenceFile, - List(qscript.pipeline.getProject.getIntervalList.getAbsolutePath)) + List(qscript.pipeline.getProject.getIntervalList.getAbsolutePath)).toList for ( sample <- recalibratedSamples ) { val sampleId = sample.getId @@ -135,24 +136,24 @@ class fullCallingPipeline extends QScript { realigner.setupScatterFunction = { case scatter: ScatterFunction => scatter.commandDirectory = new File("CleanedBams/IntermediateFiles/%s/ScatterGather".format(sampleId)) - scatter.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/Cleaning/%s/Scatter.out".format(sampleId)) + scatter.jobOutputFile = new File(".queue/logs/Cleaning/%s/Scatter.out".format(sampleId)) } realigner.setupCloneFunction = { case (clone: CloneFunction, index: Int) => clone.commandDirectory = new File("CleanedBams/IntermediateFiles/%s/ScatterGather/Scatter_%s".format(sampleId, index)) - clone.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/Cleaning/%s/Scatter_%s.out".format(sampleId, index)) + clone.jobOutputFile = new File(".queue/logs/Cleaning/%s/Scatter_%s.out".format(sampleId, index)) } realigner.setupGatherFunction = { case (gather: BamGatherFunction, source: ArgumentSource) => gather.commandDirectory = new File("CleanedBams/IntermediateFiles/%s/ScatterGather/Gather_%s".format(sampleId, source.field.getName)) - gather.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/Cleaning/%s/FixMates.out".format(sampleId)) + gather.jobOutputFile = new File(".queue/logs/Cleaning/%s/FixMates.out".format(sampleId)) gather.memoryLimit = Some(6) gather.jarFile = qscript.picardFixMatesJar // Don't pass this AS=true to fix mates! gather.assumeSorted = None case (gather: GatherFunction, source: ArgumentSource) => gather.commandDirectory = new File("CleanedBams/IntermediateFiles/%s/ScatterGather/Gather_%s".format(sampleId, source.field.getName)) - gather.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/Cleaning/%s/Gather_%s.out".format(sampleId, source.field.getName)) + gather.jobOutputFile = new File(".queue/logs/Cleaning/%s/Gather_%s.out".format(sampleId, source.field.getName)) } add(targetCreator,realigner) @@ -229,17 +230,17 @@ class fullCallingPipeline extends QScript { snps.setupScatterFunction = { case scatter: ScatterFunction => scatter.commandDirectory = new File("SnpCalls/ScatterGather") - scatter.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/SNPCalling/ScatterGather/Scatter.out") + scatter.jobOutputFile = new File(".queue/logs/SNPCalling/ScatterGather/Scatter.out") } snps.setupCloneFunction = { case (clone: CloneFunction, index: Int) => clone.commandDirectory = new File("SnpCalls/ScatterGather/Scatter_%s".format(index)) - clone.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/SNPCalling/ScatterGather/Scatter_%s.out".format(index)) + clone.jobOutputFile = new File(".queue/logs/SNPCalling/ScatterGather/Scatter_%s.out".format(index)) } snps.setupGatherFunction = { case (gather: GatherFunction, source: ArgumentSource) => gather.commandDirectory = new File("SnpCalls/ScatterGather/Gather_%s".format(source.field.getName)) - gather.jobOutputFile = new File(IOUtils.CURRENT_DIR_ABS, ".queue/logs/SNPCalling/ScatterGather/Gather_%s.out".format(source.field.getName)) + gather.jobOutputFile = new File(".queue/logs/SNPCalling/ScatterGather/Gather_%s.out".format(source.field.getName)) } // indel genotyper does one sample at a time diff --git a/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala b/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala index 50e4890c1..0a2a53a32 100755 --- a/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala +++ b/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala @@ -6,6 +6,8 @@ import java.util.Arrays import org.broadinstitute.sting.commandline._ import org.broadinstitute.sting.queue.util._ import org.broadinstitute.sting.queue.engine.{QGraphSettings, QGraph} +import collection.JavaConversions._ +import org.broadinstitute.sting.utils.classloader.PluginManager /** * Entry point of Queue. Compiles and runs QScripts passed in to the command line. @@ -18,21 +20,28 @@ class QCommandLine extends CommandLineProgram with Logging { @ArgumentCollection private val settings = new QGraphSettings - QFunction.parsingEngine = new ParsingEngine(this) + private val qScriptManager = new QScriptManager + private val qGraph = new QGraph + private var qScriptClasses: File = _ + + private lazy val pluginManager = { + qScriptClasses = IOUtils.tempDir("Q-Classes", "", settings.qSettings.tempDirectory) + qScriptManager.loadScripts(scripts, qScriptClasses) + new PluginManager[QScript](classOf[QScript], List(qScriptClasses.toURI.toURL)) + } + + QFunction.parsingEngine = new ParsingEngine(this) /** * Takes the QScripts passed in, runs their script() methods, retrieves their generated * functions, and then builds and runs a QGraph based on the dependencies. */ def execute = { - - val qGraph = QCommandLine.qGraph qGraph.settings = settings qGraph.debugMode = debugMode == true - val scripts = qScriptManager.createScripts() - for (script <- scripts) { - logger.info("Scripting " + qScriptManager.getName(script.getClass.asSubclass(classOf[QScript]))) + for (script <- pluginManager.createAllTypes()) { + logger.info("Scripting " + pluginManager.getName(script.getClass.asSubclass(classOf[QScript]))) loadArgumentsIntoObject(script) script.script script.functions.foreach(qGraph.add(_)) @@ -63,14 +72,14 @@ class QCommandLine extends CommandLineProgram with Logging { * @return Array of QScripts passed in. */ override def getArgumentSources = - qScriptManager.getValues.asInstanceOf[Array[Class[_]]] + pluginManager.getPlugins.toIterable.toArray.asInstanceOf[Array[Class[_]]] /** * Returns the name of a QScript * @return The name of a QScript */ override def getArgumentSourceName(source: Class[_]) = - qScriptManager.getName(source.asSubclass(classOf[QScript])) + pluginManager.getName(source.asSubclass(classOf[QScript])) /** * Returns a ScalaCompoundArgumentTypeDescriptor that can parse argument sources into scala collections. @@ -79,12 +88,9 @@ class QCommandLine extends CommandLineProgram with Logging { override def getArgumentTypeDescriptors = Arrays.asList(new ScalaCompoundArgumentTypeDescriptor) - /** - * Loads the QScripts passed in and returns a new QScriptManager than can be used to create them. - */ - private lazy val qScriptManager = { - QScriptManager.loadScripts(scripts) - new QScriptManager + def shutdown() = { + qGraph.shutdown() + if (qScriptClasses != null) IOUtils.tryDelete(qScriptClasses) } } @@ -92,29 +98,29 @@ class QCommandLine extends CommandLineProgram with Logging { * Entry point of Queue. Compiles and runs QScripts passed in to the command line. */ object QCommandLine { - private val qGraph = new QGraph - - /** * Main. * @param argv Arguments. */ def main(argv: Array[String]) { + val qCommandLine = new QCommandLine + Runtime.getRuntime.addShutdownHook(new Thread { /** Cleanup as the JVM shuts down. */ override def run = { - qGraph.shutdown() ProcessController.shutdown() - QScriptManager.deleteOutdir() + qCommandLine.shutdown() } }) try { - CommandLineProgram.start(new QCommandLine, argv); + CommandLineProgram.start(qCommandLine, argv); if (CommandLineProgram.result != 0) System.exit(CommandLineProgram.result); } catch { case e: Exception => CommandLineProgram.exitSystemWithError(e) + } finally { + } } } diff --git a/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala b/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala index 9dfbb481c..6af376810 100644 --- a/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala +++ b/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala @@ -3,62 +3,33 @@ package org.broadinstitute.sting.queue import scala.tools.nsc.{Global, Settings} import scala.tools.nsc.io.PlainFile import org.broadinstitute.sting.queue.util.{Logging, IOUtils} -import collection.JavaConversions +import collection.JavaConversions._ import java.io.File import scala.tools.nsc.reporters.AbstractReporter import java.lang.String import org.apache.log4j.Level import scala.tools.nsc.util.{FakePos, NoPosition, Position} -import org.broadinstitute.sting.utils.classloader.{PackageUtils, PluginManager} import org.broadinstitute.sting.queue.util.TextFormatUtils._ -import org.apache.commons.io.FileUtils +import org.broadinstitute.sting.utils.classloader.JVMUtils /** * Plugin manager for QScripts which loads QScripts into the current class loader. */ -class QScriptManager extends PluginManager[QScript](classOf[QScript], "QScript", "Script") with Logging { - - /** - * Returns the list of QScripts classes found in the classpath. - * @return QScripts classes found in the classpath. - */ - def getValues = { - if (logger.isDebugEnabled) { - JavaConversions.asMap(this.pluginsByName) - .foreach{case (name, clazz) => logger.debug("Found QScript %s: %s".format(name, clazz))} - } - JavaConversions.asIterable(this.pluginsByName.values).toArray - } - - /** - * Creates the QScripts for all values found in the classpath. - * @return QScripts found in the classpath. - */ - def createScripts() = getValues.map(_.newInstance.asInstanceOf[QScript]) -} - -/** - * Plugin manager for QScripts which loads QScripts into the current classloader. - */ -object QScriptManager extends Logging { - private val outdir = IOUtils.tempDir("Q-classes") - +class QScriptManager() extends Logging { /** * Compiles and loads the scripts in the files into the current classloader. * Heavily based on scala/src/compiler/scala/tools/ant/Scalac.scala - * @param scripts Scala classes to compile. */ - def loadScripts(scripts: List[File]) { + def loadScripts(scripts: List[File], tempDir: File) = { if (scripts.size > 0) { - val settings = new Settings((error: String) => logger.error(error)) settings.deprecation.value = true - settings.outdir.value = outdir.getPath + settings.outdir.value = tempDir.getPath // Set the classpath to the current class path. - JavaConversions.asSet(PackageUtils.getClassPathURLs).foreach(url => settings.classpath.append(url.getPath)) + JVMUtils.getClasspathURLs.foreach(url => settings.classpath.append(url.getPath)) - val reporter = new Log4JReporter(settings) + val reporter = new QScriptManager.Log4JReporter(settings) val compiler = new Global(settings, reporter) val run = new compiler.Run @@ -78,19 +49,14 @@ object QScriptManager extends Logging { reporter.WARNING.count, plural(reporter.WARNING.count))) else logger.info("Compilation complete") - - // Add the new compilation output directory to the classpath. - PackageUtils.addClasspath(outdir.toURI.toURL) } } +} - /** - * Removes the outdir cleaning up the temporary classes. - */ - def deleteOutdir() = { - if (FileUtils.deleteQuietly(outdir)) - logger.debug("Deleted " + outdir) - } +/** + * Plugin manager for QScripts which loads QScripts into the current classloader. + */ +object QScriptManager extends Logging { /** * NSC (New Scala Compiler) reporter which logs to Log4J. diff --git a/scala/src/org/broadinstitute/sting/queue/QSettings.scala b/scala/src/org/broadinstitute/sting/queue/QSettings.scala index 5ae9a7c77..b3dd6fb3c 100644 --- a/scala/src/org/broadinstitute/sting/queue/QSettings.scala +++ b/scala/src/org/broadinstitute/sting/queue/QSettings.scala @@ -23,6 +23,12 @@ class QSettings { @Argument(fullName="default_memory_limit", shortName="memLimit", doc="Default memory limit for jobs, in gigabytes.", required=false) var memoryLimit: Option[Int] = None + @Argument(fullName="run_directory", shortName="runDir", doc="Root directory to run functions from.", required=false) + var runDirectory = new File(".") + + @Argument(fullName="temp_directory", shortName="tempDir", doc="Temp directory to pass to functions.", required=false) + var tempDirectory = new File(System.getProperty("java.io.tmpdir")) + @ArgumentCollection val emailSettings = new EmailSettings } diff --git a/scala/src/org/broadinstitute/sting/queue/engine/LsfJobRunner.scala b/scala/src/org/broadinstitute/sting/queue/engine/LsfJobRunner.scala index 883b08477..085c66365 100644 --- a/scala/src/org/broadinstitute/sting/queue/engine/LsfJobRunner.scala +++ b/scala/src/org/broadinstitute/sting/queue/engine/LsfJobRunner.scala @@ -45,7 +45,7 @@ class LsfJobRunner(val function: CommandLineFunction) extends DispatchJobRunner job.project = function.jobProject job.queue = function.jobQueue - if (!IOUtils.CURRENT_DIR_ABS.equals(function.commandDirectory)) + if (function.commandDirectory != new File(".").getAbsoluteFile) job.workingDir = function.commandDirectory job.extraBsubArgs ++= function.extraArgs @@ -166,7 +166,7 @@ class LsfJobRunner(val function: CommandLineFunction) extends DispatchJobRunner * @return the file path to the pre-exec. */ private def writeExec() = { - IOUtils.writeTempFile(function.commandLine, ".exec", "") + IOUtils.writeTempFile(function.commandLine, ".exec", "", function.jobTempDir) } /** @@ -185,7 +185,7 @@ class LsfJobRunner(val function: CommandLineFunction) extends DispatchJobRunner mountCommand(function).foreach(command => preExec.append("%s%n".format(command))) - IOUtils.writeTempFile(preExec.toString, ".preExec", "") + IOUtils.writeTempFile(preExec.toString, ".preExec", "", function.jobTempDir) } /** @@ -211,6 +211,6 @@ class LsfJobRunner(val function: CommandLineFunction) extends DispatchJobRunner |fi |""".stripMargin.format(function.commandDirectory, touchDone, touchFail)) - IOUtils.writeTempFile(postExec.toString, ".postExec", "") + IOUtils.writeTempFile(postExec.toString, ".postExec", "", function.jobTempDir) } } diff --git a/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala b/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala index 5479ef535..c5b814e26 100755 --- a/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala +++ b/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala @@ -2,7 +2,6 @@ package org.broadinstitute.sting.queue.engine import org.jgrapht.traverse.TopologicalOrderIterator import org.jgrapht.graph.SimpleDirectedGraph -import scala.collection.JavaConversions import scala.collection.JavaConversions._ import org.jgrapht.alg.CycleDetector import org.jgrapht.EdgeFactory @@ -46,7 +45,7 @@ class QGraph extends Logging { * Checks the functions for missing values and the graph for cyclic dependencies and then runs the functions in the graph. */ def run = { - IOUtils.checkTempDir + IOUtils.checkTempDir(settings.qSettings.tempDirectory) val numMissingValues = fillGraph val isReady = numMissingValues == 0 @@ -158,7 +157,7 @@ class QGraph extends Logging { */ private def fillIn = { // clone since edgeSet is backed by the graph - JavaConversions.asSet(jobGraph.edgeSet).clone.foreach { + asScalaSet(jobGraph.edgeSet).clone.foreach { case cmd: FunctionEdge => { addCollectionOutputs(cmd.outputs) addCollectionInputs(cmd.inputs) @@ -205,7 +204,7 @@ class QGraph extends Logging { */ private def validate = { var numMissingValues = 0 - JavaConversions.asSet(jobGraph.edgeSet).foreach { + asScalaSet(jobGraph.edgeSet).foreach { case cmd: FunctionEdge => val missingFieldValues = cmd.function.missingFields if (missingFieldValues.size > 0) { diff --git a/scala/src/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunction.scala b/scala/src/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunction.scala index f444044bf..78206816a 100644 --- a/scala/src/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunction.scala @@ -48,105 +48,12 @@ class IntervalScatterFunction extends ScatterFunction with InProcessFunction { def bindCloneInputs(cloneFunction: CloneFunction, index: Int) = { val scatterPart = cloneFunction.getFieldValue(this.intervalsField) .asInstanceOf[List[File]] - .map(file => IOUtils.subDir(cloneFunction.commandDirectory, file)) + .map(file => IOUtils.absolute(cloneFunction.commandDirectory, file)) cloneFunction.setFieldValue(this.intervalsField, scatterPart) this.scatterParts ++= scatterPart } def run() = { - IntervalScatterFunction.scatter(this.referenceSequence, this.intervals, this.scatterParts, this.splitByContig) + IntervalUtils.scatterIntervalArguments(this.referenceSequence, this.intervals, this.scatterParts, this.splitByContig) } } - -object IntervalScatterFunction { - private def parseLocs(referenceSource: ReferenceDataSource, intervals: List[String]) = { - var genomeLocParser: GenomeLocParser = new GenomeLocParser(referenceSource.getReference) - val locs = { - // TODO: Abstract genome analysis engine has richer logic for parsing. We need to use it! - if (intervals.size == 0) { - GenomeLocSortedSet.createSetFromSequenceDictionary(referenceSource.getReference.getSequenceDictionary) - } else { - new GenomeLocSortedSet(genomeLocParser,IntervalUtils.parseIntervalArguments(genomeLocParser, intervals, false)) - } - } - if (locs == null || locs.size == 0) - throw new QException("Intervals are empty: " + intervals.mkString(", ")) - locs.toList - } - - def distinctContigs(reference: File, intervals: List[String] = Nil) = { - val referenceSource = new ReferenceDataSource(reference) - val locs = parseLocs(referenceSource, intervals) - var contig: String = null - var contigs = List.empty[String] - for (loc <- locs) { - if (contig != loc.getContig) { - contig = loc.getContig - contigs :+= contig - } - } - contigs - } - - def scatter(reference: File, intervals: List[String], scatterParts: List[File], splitByContig: Boolean) = { - val referenceSource = new ReferenceDataSource(reference) - val locs = parseLocs(referenceSource, intervals) - val fileHeader = new SAMFileHeader - fileHeader.setSequenceDictionary(referenceSource.getReference.getSequenceDictionary) - - var intervalList: IntervalList = null - var fileIndex = -1 - var locIndex = 0 - - if (splitByContig) { - var contig: String = null - for (loc <- locs) { - if (contig != loc.getContig && (fileIndex + 1) < scatterParts.size) { - if (fileIndex >= 0) - intervalList.write(scatterParts(fileIndex)) - fileIndex += 1 - contig = loc.getContig - intervalList = new IntervalList(fileHeader) - } - locIndex += 1 - intervalList.add(toInterval(loc, locIndex)) - } - intervalList.write(scatterParts(fileIndex)) - if ((fileIndex + 1) != scatterParts.size) - throw new QException("Only able to write contigs into %d of %d files.".format(fileIndex + 1, scatterParts.size)) - } else { - var locsPerFile = locs.size / scatterParts.size - val locRemainder = locs.size % scatterParts.size - - // At the start, put an extra loc per file - locsPerFile += 1 - var locsLeftFile = 0 - - for (loc <- locs) { - if (locsLeftFile == 0) { - if (fileIndex >= 0) - intervalList.write(scatterParts(fileIndex)) - - fileIndex += 1 - intervalList = new IntervalList(fileHeader) - - // When we have put enough locs into each file, - // reduce the number of locs per file back - // to the original calculated value. - if (fileIndex == locRemainder) - locsPerFile -= 1 - locsLeftFile = locsPerFile - } - locsLeftFile -= 1 - locIndex += 1 - intervalList.add(toInterval(loc, locIndex)) - } - intervalList.write(scatterParts(fileIndex)) - if ((fileIndex + 1) != scatterParts.size) - throw new QException("Only able to write intervals into %d of %d files.".format(fileIndex + 1, scatterParts.size)) - } - } - - private def toInterval(loc: GenomeLoc, locIndex: Int) = - new net.sf.picard.util.Interval(loc.getContig, loc.getStart.toInt, loc.getStop.toInt, false, "interval_" + locIndex) -} diff --git a/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala index fda03c496..afe8e5a1c 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala @@ -29,10 +29,10 @@ trait QFunction extends Logging { var qSettings: QSettings = _ /** Directory to run the command in. */ - var commandDirectory: File = IOUtils.CURRENT_DIR + var commandDirectory: File = new File(".") /** Temporary directory to write any files */ - var jobTempDir: File = IOUtils.javaTempDir + var jobTempDir: File = null /** Order the function was added to the graph. */ var addOrder: List[Int] = Nil @@ -279,7 +279,11 @@ trait QFunction extends Logging { if (jobOutputFile == null) jobOutputFile = new File(jobName + ".out") - commandDirectory = IOUtils.subDir(IOUtils.CURRENT_DIR, commandDirectory) + if (jobTempDir == null) + jobTempDir = qSettings.tempDirectory + + // If the command directory is relative, insert the run directory ahead of it. + commandDirectory = new File(qSettings.runDirectory, commandDirectory.getPath) } /** @@ -315,11 +319,11 @@ trait QFunction extends Logging { } /** - * Returns the absolute path to the file relative to the job command directory. + * Returns the absolute path to the file relative to the run directory and the job command directory. * @param file File to root relative to the command directory if it is not already absolute. * @return The absolute path to file. */ - private def absolute(file: File) = IOUtils.subDir(commandDirectory, file) + private def absolute(file: File) = IOUtils.absolute(commandDirectory, file) /** diff --git a/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala index 4bc270436..c8dd4d694 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala @@ -124,7 +124,7 @@ trait ScatterGatherableFunction extends CommandLineFunction { // Get absolute paths to the files and bind the sg functions to the clone function via the absolute paths. scatterFunction.bindCloneInputs(cloneFunction, i) for (gatherField <- outputFieldsWithValues) { - val gatherPart = IOUtils.subDir(cloneFunction.commandDirectory, cloneFunction.getFieldFile(gatherField)) + val gatherPart = IOUtils.absolute(cloneFunction.commandDirectory, cloneFunction.getFieldFile(gatherField)) cloneFunction.setFieldValue(gatherField, gatherPart) gatherFunctions(gatherField).gatherParts :+= gatherPart } @@ -244,5 +244,5 @@ trait ScatterGatherableFunction extends CommandLineFunction { * @param Sub directory under the scatter gather directory. * @return temporary directory under this scatter gather directory. */ - private def scatterGatherTempDir(subDir: String) = IOUtils.subDir(this.scatterGatherDirectory, this.jobName + "-sg/" + subDir) + private def scatterGatherTempDir(subDir: String) = IOUtils.absolute(this.scatterGatherDirectory, this.jobName + "-sg/" + subDir) } diff --git a/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala b/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala index 3ed8deb0e..96430b824 100644 --- a/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala +++ b/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala @@ -3,7 +3,7 @@ package org.broadinstitute.sting.queue.util import org.apache.commons.mail.{MultiPartEmail, EmailAttachment} import java.io.{FileReader, File} import javax.mail.internet.InternetAddress -import scala.collection.JavaConversions +import scala.collection.JavaConversions._ /** * Encapsulates a message to be sent over email. @@ -91,7 +91,7 @@ class EmailMessage extends Logging { * @return java.util.List of InternetAddress'es */ private def convert(addresses: List[String]) = { - JavaConversions.asList(addresses.map(address => new InternetAddress(address, false))) + asJavaList(addresses.map(address => new InternetAddress(address, false))) } override def toString = { diff --git a/scala/src/org/broadinstitute/sting/queue/util/IOUtils.scala b/scala/src/org/broadinstitute/sting/queue/util/IOUtils.scala index 56c568d83..1bb08a5cf 100644 --- a/scala/src/org/broadinstitute/sting/queue/util/IOUtils.scala +++ b/scala/src/org/broadinstitute/sting/queue/util/IOUtils.scala @@ -8,78 +8,32 @@ import org.broadinstitute.sting.utils.exceptions.UserException * A collection of utilities for modifying java.io. */ object IOUtils extends Logging { - /** The current directory "." */ - val CURRENT_DIR = new File(".") - - val CURRENT_DIR_ABS = absolute(CURRENT_DIR) - /** - * Returns the sub path rooted at the parent. - * If the sub path is already absolute, returns the sub path. - * If the parent is the current directory, returns the sub path. - * If the sub bath is the current directory, returns the parent. - * Else returns new File(parent, subPath) - * @param parent The parent directory - * @param path The sub path to append to the parent, if the path is not absolute. - * @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path. + * Checks if the temp directory has been setup and throws an exception if they user hasn't set it correctly. + * @param tempDir Temporary directory. */ - def subDir(parent: File, path: String): File = - subDir(parent, new File(path)) - - /** - * Returns the sub path rooted at the parent. - * If the sub path is already absolute, returns the sub path. - * If the parent is the current directory, returns the sub path. - * If the sub path is the current directory, returns the parent. - * Else returns new File(parent, subPath) - * @param parent The parent directory - * @param file The sub path to append to the parent, if the path is not absolute. - * @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path. - */ - def subDir(parent: File, file: File): File = { - val parentAbs = absolute(parent) - val fileAbs = absolute(file) - if (parentAbs == CURRENT_DIR_ABS && fileAbs == CURRENT_DIR_ABS) - CURRENT_DIR_ABS - else if (parentAbs == CURRENT_DIR_ABS || file.isAbsolute) - fileAbs - else if (fileAbs == CURRENT_DIR_ABS) - parentAbs - else - absolute(new File(parentAbs, file.getPath)) - } - - def checkTempDir = { - val javaTemp = System.getProperty("java.io.tmpdir") + def checkTempDir(tempDir: File) = { + val tempDirPath = tempDir.getAbsolutePath // Keeps the user from leaving the temp directory as the default, and on Macs from having pluses // in the path which can cause problems with the Google Reflections library. // see also: http://benjchristensen.com/2009/09/22/mac-osx-10-6-java-java-io-tmpdir/ - if (javaTemp.startsWith("/var/folders/") || (javaTemp == "/tmp") || (javaTemp == "/tmp/")) + if (tempDirPath.startsWith("/var/folders/") || (tempDirPath == "/tmp") || (tempDirPath == "/tmp/")) throw new UserException.BadTmpDir("java.io.tmpdir must be explicitly set") - } - - /** - * Returns the temp directory as defined by java. - * @return the temp directory as defined by java. - */ - def javaTempDir() = { - val tempDir = new File(System.getProperty("java.io.tmpdir")) if (!tempDir.exists && !tempDir.mkdirs) throw new UserException.BadTmpDir("Could not create directory: " + tempDir.getAbsolutePath()) - absolute(tempDir) } /** * Creates a temp directory with the prefix and optional suffix. * @param prefix Prefix for the directory name. - * @param suffix Optional suffix for the directory name. Defaults to "". + * @param suffix Optional suffix for the directory name. + * @param tempDirParent Parent directory for the temp directory. * @return The created temporary directory. */ - def tempDir(prefix: String, suffix: String = "") = { - val tempDirParent = javaTempDir() + def tempDir(prefix: String, suffix: String = "", tempDirParent: File) = { if (!tempDirParent.exists && !tempDirParent.mkdirs) throw new UserException.BadTmpDir("Could not create temp directory: " + tempDirParent) - val temp = File.createTempFile(prefix + "-", suffix) + val temp = File.createTempFile(prefix + "-", suffix, tempDirParent) if (!temp.delete) throw new UserException.BadTmpDir("Could not delete sub file: " + temp.getAbsolutePath()) if (!temp.mkdir) @@ -89,7 +43,7 @@ object IOUtils extends Logging { def writeContents(file: File, content: String) = FileUtils.writeStringToFile(file, content) - def writeTempFile(content: String, prefix: String, suffix: String = "", directory: File = null) = { + def writeTempFile(content: String, prefix: String, suffix: String = "", directory: File) = { val tempFile = absolute(File.createTempFile(prefix, suffix, directory)) writeContents(tempFile, content) tempFile @@ -115,6 +69,28 @@ object IOUtils extends Logging { directories(level) } + /** + * Returns the sub path rooted at the parent. + * @param parent The parent directory. + * @param path The sub path to append to the parent, if the path is not absolute. + * @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path. + */ + def absolute(parent: File, path: String): File = + absolute(parent, new File(path)) + + /** + * Returns the sub path rooted at the parent. + * @param parent The parent directory. + * @param file The sub path to append to the parent, if the path is not absolute. + * @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path. + */ + def absolute(parent: File, file: File): File = { + if (file.isAbsolute) + absolute(file) + else + absolute(new File(parent, file.getPath)) + } + /** * A mix of getCanonicalFile and getAbsoluteFile that returns the * absolute path to the file without deferencing symbolic links. diff --git a/scala/src/org/broadinstitute/sting/queue/util/ProcessController.scala b/scala/src/org/broadinstitute/sting/queue/util/ProcessController.scala index 92b63c628..e5cb50202 100644 --- a/scala/src/org/broadinstitute/sting/queue/util/ProcessController.scala +++ b/scala/src/org/broadinstitute/sting/queue/util/ProcessController.scala @@ -2,7 +2,6 @@ package org.broadinstitute.sting.queue.util import java.io._ import scala.collection.mutable.{HashSet, ListMap} -import scala.collection.JavaConversions /** * Facade to Runtime.exec() and java.lang.Process. Handles @@ -258,7 +257,11 @@ object ProcessController extends Logging { def shutdown() = { for (process <- running.clone) { logger.warn("Killing: " + process) - process.destroy + try { + process.destroy + } catch { + case _ => /* ignore */ + } } } @@ -349,8 +352,10 @@ object ProcessController extends Logging { * @param len Number of characters in the buffer. */ private def writeFile(chars: Array[Char], len: Int) = { - if (fileWriter != null) + if (fileWriter != null) { fileWriter.write(chars, 0, len) + fileWriter.flush() + } } /** Closes the fileWriter if it is not null. */ diff --git a/scala/test/org/broadinstitute/sting/queue/QScriptTest.scala b/scala/test/org/broadinstitute/sting/queue/QScriptTest.scala new file mode 100644 index 000000000..3014f10b9 --- /dev/null +++ b/scala/test/org/broadinstitute/sting/queue/QScriptTest.scala @@ -0,0 +1,88 @@ +package org.broadinstitute.sting.queue + +import org.broadinstitute.sting.utils.Utils +import org.testng.Assert +import org.broadinstitute.sting.commandline.CommandLineProgram +import org.broadinstitute.sting.BaseTest +import org.broadinstitute.sting.queue.util.ProcessController + +class QScriptTest extends BaseTest { + + protected val stingDir = "./" + + /** + * execute the test + * @param name the name of the test + * @param args the argument list + * @param expectedException the expected exception or null if no exception is expected. + */ + def executeTest(name: String, args: String, expectedException: Class[_]) = { + var command = Utils.escapeExpressions(args) + + // add the logging level to each of the integration test commands + + command = Utils.appendArray(command, "-l", "WARN", "-startFromScratch", "-tempDir", "integrationtests") + + // run the executable + var gotAnException = false + + val instance = new QCommandLine + QScriptTest.runningCommandLines += instance + try { + println("Executing test %s with Queue arguments: %s".format(name, Utils.join(" ",command))) + CommandLineProgram.start(instance, command) + } catch { + case e => + gotAnException = true + if (expectedException != null) { + // we expect an exception + println("Wanted exception %s, saw %s".format(expectedException, e.getClass)) + if (expectedException.isInstance(e)) { + // it's the type we expected + println(String.format(" => %s PASSED", name)) + } else { + e.printStackTrace() + Assert.fail("Test %s expected exception %s but got %s instead".format( + name, expectedException, e.getClass)) + } + } else { + // we didn't expect an exception but we got one :-( + throw new RuntimeException(e) + } + } finally { + instance.shutdown() + QScriptTest.runningCommandLines -= instance + } + + // catch failures from the integration test + if (expectedException != null) { + if (!gotAnException) + // we expected an exception but didn't see it + Assert.fail("Test %s expected exception %s but none was thrown".format(name, expectedException.toString)) + } else { + if (CommandLineProgram.result != 0) + throw new RuntimeException("Error running the GATK with arguments: " + args) + } + } +} + +object QScriptTest { + private var runningCommandLines = Set.empty[QCommandLine] + + Runtime.getRuntime.addShutdownHook(new Thread { + /** Cleanup as the JVM shuts down. */ + override def run = { + try { + ProcessController.shutdown() + } catch { + case _ => /*ignore */ + } + runningCommandLines.foreach(commandLine => + try { + commandLine.shutdown() + } catch { + case _ => /* ignore */ + }) + } + }) +} diff --git a/scala/test/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunctionUnitTest.scala b/scala/test/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunctionUnitTest.scala deleted file mode 100644 index bcd2e254e..000000000 --- a/scala/test/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunctionUnitTest.scala +++ /dev/null @@ -1,266 +0,0 @@ -package org.broadinstitute.sting.queue.extensions.gatk - -import collection.JavaConversions._ -import java.io.File -import org.testng.Assert -import org.broadinstitute.sting.BaseTest -import org.broadinstitute.sting.utils.interval.IntervalUtils -import org.broadinstitute.sting.queue.QException -import net.sf.picard.reference.IndexedFastaSequenceFile -import org.testng.annotations.{Test, BeforeMethod} -import org.broadinstitute.sting.utils.GenomeLocParser - -class IntervalScatterFunctionUnitTest extends BaseTest { - private def reference = new File(BaseTest.b36KGReference) - private var header: IndexedFastaSequenceFile = _ - private var genomeLocParser: GenomeLocParser = _ - - @BeforeMethod - def setup() { - header = new IndexedFastaSequenceFile(reference) - genomeLocParser = new GenomeLocParser(header.getSequenceDictionary()) - } - - @Test - def testCountContigs = { - Assert.assertEquals(List("1"), IntervalScatterFunction.distinctContigs(reference, List(BaseTest.validationDataLocation + "chr1_b36_pilot3.interval_list"))) - Assert.assertEquals(List("1","2","3"), IntervalScatterFunction.distinctContigs(reference, List("1:1-1", "2:1-1", "3:2-2"))) - Assert.assertEquals(List("1","2","3"), IntervalScatterFunction.distinctContigs(reference, List("2:1-1", "1:1-1", "3:2-2"))) - } - - @Test - def testBasicScatter = { - val chr1 = genomeLocParser.parseGenomeInterval("1") - val chr2 = genomeLocParser.parseGenomeInterval("2") - val chr3 = genomeLocParser.parseGenomeInterval("3") - - val files = (1 to 3).toList.map(index => new File(testDir + "basic." + index + ".intervals")) - - IntervalScatterFunction.scatter(reference, List("1", "2", "3"), files, false) - - val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false) - val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false) - val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false) - - Assert.assertEquals(1, locs1.size) - Assert.assertEquals(1, locs2.size) - Assert.assertEquals(1, locs3.size) - - Assert.assertEquals(chr1, locs1.get(0)) - Assert.assertEquals(chr2, locs2.get(0)) - Assert.assertEquals(chr3, locs3.get(0)) - } - - @Test - def testScatterLessFiles = { - val chr1 = genomeLocParser.parseGenomeInterval("1") - val chr2 = genomeLocParser.parseGenomeInterval("2") - val chr3 = genomeLocParser.parseGenomeInterval("3") - val chr4 = genomeLocParser.parseGenomeInterval("4") - - val files = (1 to 3).toList.map(index => new File(testDir + "less." + index + ".intervals")) - - IntervalScatterFunction.scatter(reference, List("1", "2", "3", "4"), files, false) - - val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false) - val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false) - val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false) - - Assert.assertEquals(2, locs1.size) - Assert.assertEquals(1, locs2.size) - Assert.assertEquals(1, locs3.size) - - Assert.assertEquals(chr1, locs1.get(0)) - Assert.assertEquals(chr2, locs1.get(1)) - Assert.assertEquals(chr3, locs2.get(0)) - Assert.assertEquals(chr4, locs3.get(0)) - } - - @Test(expectedExceptions=Array(classOf[QException])) - def testScatterMoreFiles = { - val files = (1 to 3).toList.map(index => new File(testDir + "more." + index + ".intervals")) - IntervalScatterFunction.scatter(reference, List("1", "2"), files, false) - } - - @Test - def testScatterIntervals = { - val intervals = List("1:1-2", "1:4-5", "2:1-1", "3:2-2") - val chr1a = genomeLocParser.parseGenomeInterval("1:1-2") - val chr1b = genomeLocParser.parseGenomeInterval("1:4-5") - val chr2 = genomeLocParser.parseGenomeInterval("2:1-1") - val chr3 = genomeLocParser.parseGenomeInterval("3:2-2") - - val files = (1 to 3).toList.map(index => new File(testDir + "split." + index + ".intervals")) - - IntervalScatterFunction.scatter(reference, intervals, files, true) - - val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false) - val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false) - val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false) - - Assert.assertEquals(2, locs1.size) - Assert.assertEquals(1, locs2.size) - Assert.assertEquals(1, locs3.size) - - Assert.assertEquals(chr1a, locs1.get(0)) - Assert.assertEquals(chr1b, locs1.get(1)) - Assert.assertEquals(chr2, locs2.get(0)) - Assert.assertEquals(chr3, locs3.get(0)) - } - - @Test - def testScatterOrder = { - val intervals = List("2:1-1", "1:1-1", "3:2-2") - val chr1 = genomeLocParser.parseGenomeInterval("1:1-1") - val chr2 = genomeLocParser.parseGenomeInterval("2:1-1") - val chr3 = genomeLocParser.parseGenomeInterval("3:2-2") - - val files = (1 to 3).toList.map(index => new File(testDir + "split." + index + ".intervals")) - - IntervalScatterFunction.scatter(reference, intervals, files, true) - - val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false) - val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false) - val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false) - - Assert.assertEquals(1, locs1.size) - Assert.assertEquals(1, locs2.size) - Assert.assertEquals(1, locs3.size) - - Assert.assertEquals(chr1, locs1.get(0)) - Assert.assertEquals(chr2, locs2.get(0)) - Assert.assertEquals(chr3, locs3.get(0)) - } - - @Test - def testBasicScatterByContig = { - val chr1 = genomeLocParser.parseGenomeInterval("1") - val chr2 = genomeLocParser.parseGenomeInterval("2") - val chr3 = genomeLocParser.parseGenomeInterval("3") - - val files = (1 to 3).toList.map(index => new File(testDir + "contig_basic." + index + ".intervals")) - - IntervalScatterFunction.scatter(reference, List("1", "2", "3"), files, true) - - val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false) - val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false) - val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false) - - Assert.assertEquals(1, locs1.size) - Assert.assertEquals(1, locs2.size) - Assert.assertEquals(1, locs3.size) - - Assert.assertEquals(chr1, locs1.get(0)) - Assert.assertEquals(chr2, locs2.get(0)) - Assert.assertEquals(chr3, locs3.get(0)) - } - - @Test - def testScatterByContigLessFiles = { - val chr1 = genomeLocParser.parseGenomeInterval("1") - val chr2 = genomeLocParser.parseGenomeInterval("2") - val chr3 = genomeLocParser.parseGenomeInterval("3") - val chr4 = genomeLocParser.parseGenomeInterval("4") - - val files = (1 to 3).toList.map(index => new File(testDir + "contig_less." + index + ".intervals")) - - IntervalScatterFunction.scatter(reference, List("1", "2", "3", "4"), files, true) - - val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false) - val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false) - val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false) - - Assert.assertEquals(1, locs1.size) - Assert.assertEquals(1, locs2.size) - Assert.assertEquals(2, locs3.size) - - Assert.assertEquals(chr1, locs1.get(0)) - Assert.assertEquals(chr2, locs2.get(0)) - Assert.assertEquals(chr3, locs3.get(0)) - Assert.assertEquals(chr4, locs3.get(1)) - } - - @Test(expectedExceptions=Array(classOf[QException])) - def testScatterByContigMoreFiles = { - val files = (1 to 3).toList.map(index => new File(testDir + "contig_more." + index + ".intervals")) - IntervalScatterFunction.scatter(reference, List("1", "2"), files, true) - } - - @Test - def testScatterByContigIntervalsStart = { - val intervals = List("1:1-2", "1:4-5", "2:1-1", "3:2-2") - val chr1a = genomeLocParser.parseGenomeInterval("1:1-2") - val chr1b = genomeLocParser.parseGenomeInterval("1:4-5") - val chr2 = genomeLocParser.parseGenomeInterval("2:1-1") - val chr3 = genomeLocParser.parseGenomeInterval("3:2-2") - - val files = (1 to 3).toList.map(index => new File(testDir + "contig_split_start." + index + ".intervals")) - - IntervalScatterFunction.scatter(reference, intervals, files, true) - - val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false) - val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false) - val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false) - - Assert.assertEquals(2, locs1.size) - Assert.assertEquals(1, locs2.size) - Assert.assertEquals(1, locs3.size) - - Assert.assertEquals(chr1a, locs1.get(0)) - Assert.assertEquals(chr1b, locs1.get(1)) - Assert.assertEquals(chr2, locs2.get(0)) - Assert.assertEquals(chr3, locs3.get(0)) - } - - @Test - def testScatterByContigIntervalsMiddle = { - val intervals = List("1:1-1", "2:1-2", "2:4-5", "3:2-2") - val chr1 = genomeLocParser.parseGenomeInterval("1:1-1") - val chr2a = genomeLocParser.parseGenomeInterval("2:1-2") - val chr2b = genomeLocParser.parseGenomeInterval("2:4-5") - val chr3 = genomeLocParser.parseGenomeInterval("3:2-2") - - val files = (1 to 3).toList.map(index => new File(testDir + "contig_split_middle." + index + ".intervals")) - - IntervalScatterFunction.scatter(reference, intervals, files, true) - - val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false) - val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false) - val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false) - - Assert.assertEquals(1, locs1.size) - Assert.assertEquals(2, locs2.size) - Assert.assertEquals(1, locs3.size) - - Assert.assertEquals(chr1, locs1.get(0)) - Assert.assertEquals(chr2a, locs2.get(0)) - Assert.assertEquals(chr2b, locs2.get(1)) - Assert.assertEquals(chr3, locs3.get(0)) - } - - @Test - def testScatterByContigIntervalsEnd = { - val intervals = List("1:1-1", "2:2-2", "3:1-2", "3:4-5") - val chr1 = genomeLocParser.parseGenomeInterval("1:1-1") - val chr2 = genomeLocParser.parseGenomeInterval("2:2-2") - val chr3a = genomeLocParser.parseGenomeInterval("3:1-2") - val chr3b = genomeLocParser.parseGenomeInterval("3:4-5") - - val files = (1 to 3).toList.map(index => new File(testDir + "contig_split_end." + index + ".intervals")) - - IntervalScatterFunction.scatter(reference, intervals, files, true) - - val locs1 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(0).toString), false) - val locs2 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(1).toString), false) - val locs3 = IntervalUtils.parseIntervalArguments(genomeLocParser,List(files(2).toString), false) - - Assert.assertEquals(1, locs1.size) - Assert.assertEquals(1, locs2.size) - Assert.assertEquals(2, locs3.size) - - Assert.assertEquals(chr1, locs1.get(0)) - Assert.assertEquals(chr2, locs2.get(0)) - Assert.assertEquals(chr3a, locs3.get(0)) - Assert.assertEquals(chr3b, locs3.get(1)) - } -} diff --git a/scala/test/org/broadinstitute/sting/queue/util/IOUtilsUnitTest.scala b/scala/test/org/broadinstitute/sting/queue/util/IOUtilsUnitTest.scala index 06e5f4122..f0e5153a4 100644 --- a/scala/test/org/broadinstitute/sting/queue/util/IOUtilsUnitTest.scala +++ b/scala/test/org/broadinstitute/sting/queue/util/IOUtilsUnitTest.scala @@ -9,65 +9,53 @@ import org.testng.annotations.Test class IOUtilsUnitTest extends BaseTest { @Test def testGoodTempDir = { - val tmpDir = System.getProperty("java.io.tmpdir") - try { - System.setProperty("java.io.tmpdir", "/tmp/queue") - IOUtils.checkTempDir - } finally { - System.setProperty("java.io.tmpdir", tmpDir) - } + IOUtils.checkTempDir(new File("/tmp/queue")) } @Test(expectedExceptions=Array(classOf[UserException.BadTmpDir])) def testBadTempDir = { - val tmpDir = System.getProperty("java.io.tmpdir") - try { - System.setProperty("java.io.tmpdir", "/tmp") - IOUtils.checkTempDir - } finally { - System.setProperty("java.io.tmpdir", tmpDir) - } + IOUtils.checkTempDir(new File("/tmp")) } @Test def testAbsoluteSubDir = { - var subDir = IOUtils.subDir(IOUtils.CURRENT_DIR, new File("/path/to/file")) - Assert.assertEquals(new File("/path/to/file"), subDir) + var subDir = IOUtils.absolute(new File("."), new File("/path/to/file")) + Assert.assertEquals(subDir, new File("/path/to/file")) - subDir = IOUtils.subDir(new File("/different/path"), new File("/path/to/file")) - Assert.assertEquals(new File("/path/to/file"), subDir) + subDir = IOUtils.absolute(new File("/different/path"), new File("/path/to/file")) + Assert.assertEquals(subDir, new File("/path/to/file")) - subDir = IOUtils.subDir(new File("/different/path"), IOUtils.CURRENT_DIR) - Assert.assertEquals(new File("/different/path"), subDir) + subDir = IOUtils.absolute(new File("/different/path"), new File(".")) + Assert.assertEquals(subDir, new File("/different/path")) } @Test def testRelativeSubDir = { - var subDir = IOUtils.subDir(IOUtils.CURRENT_DIR, new File("path/to/file")) - Assert.assertEquals(new File("path/to/file").getCanonicalFile, subDir.getCanonicalFile) + var subDir = IOUtils.absolute(new File("."), new File("path/to/file")) + Assert.assertEquals(subDir.getCanonicalFile, new File("path/to/file").getCanonicalFile) - subDir = IOUtils.subDir(new File("/different/path"), new File("path/to/file")) - Assert.assertEquals(new File("/different/path/path/to/file"), subDir) + subDir = IOUtils.absolute(new File("/different/path"), new File("path/to/file")) + Assert.assertEquals(subDir, new File("/different/path/path/to/file")) } @Test def testDottedSubDir = { - var subDir = IOUtils.subDir(IOUtils.CURRENT_DIR, new File("path/../to/file")) - Assert.assertEquals(new File("path/../to/./file").getCanonicalFile, subDir.getCanonicalFile) + var subDir = IOUtils.absolute(new File("."), new File("path/../to/file")) + Assert.assertEquals(subDir.getCanonicalFile, new File("path/../to/./file").getCanonicalFile) - subDir = IOUtils.subDir(IOUtils.CURRENT_DIR, new File("/path/../to/file")) - Assert.assertEquals(new File("/path/../to/file"), subDir) + subDir = IOUtils.absolute(new File("."), new File("/path/../to/file")) + Assert.assertEquals(subDir, new File("/path/../to/file")) - subDir = IOUtils.subDir(new File("/different/../path"), new File("path/to/file")) - Assert.assertEquals(new File("/different/../path/path/to/file"), subDir) + subDir = IOUtils.absolute(new File("/different/../path"), new File("path/to/file")) + Assert.assertEquals(subDir, new File("/different/../path/path/to/file")) - subDir = IOUtils.subDir(new File("/different/./path"), new File("/path/../to/file")) - Assert.assertEquals(new File("/path/../to/file"), subDir) + subDir = IOUtils.absolute(new File("/different/./path"), new File("/path/../to/file")) + Assert.assertEquals(subDir, new File("/path/../to/file")) } @Test def testTempDir = { - val tempDir = IOUtils.tempDir("Q-Unit-Test") + val tempDir = IOUtils.tempDir("Q-Unit-Test", "", new File("queueTempDirToDelete")) Assert.assertTrue(tempDir.exists) Assert.assertFalse(tempDir.isFile) Assert.assertTrue(tempDir.isDirectory) @@ -79,43 +67,43 @@ class IOUtilsUnitTest extends BaseTest { @Test def testDirLevel = { var dir = IOUtils.dirLevel(new File("/path/to/directory"), 1) - Assert.assertEquals(new File("/path"), dir) + Assert.assertEquals(dir, new File("/path")) dir = IOUtils.dirLevel(new File("/path/to/directory"), 2) - Assert.assertEquals(new File("/path/to"), dir) + Assert.assertEquals(dir, new File("/path/to")) dir = IOUtils.dirLevel(new File("/path/to/directory"), 3) - Assert.assertEquals(new File("/path/to/directory"), dir) + Assert.assertEquals(dir, new File("/path/to/directory")) dir = IOUtils.dirLevel(new File("/path/to/directory"), 4) - Assert.assertEquals(new File("/path/to/directory"), dir) + Assert.assertEquals(dir, new File("/path/to/directory")) } @Test def testAbsolute = { var dir = IOUtils.absolute(new File("/path/./to/./directory/.")) - Assert.assertEquals(new File("/path/to/directory"), dir) + Assert.assertEquals(dir, new File("/path/to/directory")) dir = IOUtils.absolute(new File("/")) - Assert.assertEquals(new File("/"), dir) + Assert.assertEquals(dir, new File("/")) dir = IOUtils.absolute(new File("/.")) - Assert.assertEquals(new File("/"), dir) + Assert.assertEquals(dir, new File("/")) dir = IOUtils.absolute(new File("/././.")) - Assert.assertEquals(new File("/"), dir) + Assert.assertEquals(dir, new File("/")) dir = IOUtils.absolute(new File("/./directory/.")) - Assert.assertEquals(new File("/directory"), dir) + Assert.assertEquals(dir, new File("/directory")) dir = IOUtils.absolute(new File("/./directory/./")) - Assert.assertEquals(new File("/directory"), dir) + Assert.assertEquals(dir, new File("/directory")) dir = IOUtils.absolute(new File("/./directory./")) - Assert.assertEquals(new File("/directory."), dir) + Assert.assertEquals(dir, new File("/directory.")) dir = IOUtils.absolute(new File("/./.directory/")) - Assert.assertEquals(new File("/.directory"), dir) + Assert.assertEquals(dir, new File("/.directory")) } @Test @@ -127,8 +115,8 @@ class IOUtilsUnitTest extends BaseTest { "chr22_random 257318 3156435963 50 51", "chrX_random 1719168 3156698441 50 51") val tail = IOUtils.tail(new File(BaseTest.hg18Reference + ".fai"), 5) - Assert.assertEquals(5, tail.size) + Assert.assertEquals(tail.size, 5) for (i <- 0 until 5) - Assert.assertEquals(lines(i), tail(i)) + Assert.assertEquals(tail(i), lines(i)) } }