From ccdb4a0313d3d1cffe22ff6ff21ac070844e6119 Mon Sep 17 00:00:00 2001 From: hanna Date: Sun, 23 Aug 2009 00:56:02 +0000 Subject: [PATCH] General-purpose management of output streams. git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@1454 348d0f76-0448-11de-a6fe-93d51630548a --- .../sting/gatk/CommandLineExecutable.java | 40 ++- .../sting/gatk/GenomeAnalysisEngine.java | 47 ++-- .../sting/gatk/OutputTracker.java | 192 ------------- .../executive/HierarchicalMicroScheduler.java | 122 ++++----- .../gatk/executive/LinearMicroScheduler.java | 12 +- .../sting/gatk/executive/MicroScheduler.java | 8 + .../sting/gatk/executive/OutputMergeTask.java | 78 ++++++ .../sting/gatk/executive/OutputMerger.java | 167 ------------ .../sting/gatk/executive/ShardTraverser.java | 61 ++++- .../sting/gatk/io/DirectOutputTracker.java | 56 ++++ .../sting/gatk/io/OutputTracker.java | 156 +++++++++++ .../gatk/io/ThreadLocalOutputTracker.java | 109 ++++++++ .../gatk/io/storage/OutputStreamStorage.java | 139 ++++++++++ .../gatk/io/storage/SAMFileWriterStorage.java | 77 ++++++ .../sting/gatk/io/storage/Storage.java | 45 +++ .../sting/gatk/io/storage/StorageFactory.java | 87 ++++++ .../OutputStreamArgumentTypeDescriptor.java | 95 +++++++ .../sting/gatk/io/stubs/OutputStreamStub.java | 138 ++++++++++ .../SAMFileReaderArgumentTypeDescriptor.java | 79 ++++++ .../SAMFileWriterArgumentTypeDescriptor.java | 121 +++++++++ .../gatk/io/stubs/SAMFileWriterStub.java | 130 +++++++++ .../sting/gatk/io/stubs/Stub.java | 46 ++++ .../org/broadinstitute/sting/utils/Utils.java | 12 + .../utils/cmdLine/ArgumentDefinition.java | 116 ++++++++ .../utils/cmdLine/ArgumentDefinitions.java | 61 ----- .../sting/utils/cmdLine/ArgumentMatches.java | 62 +++-- .../sting/utils/cmdLine/ArgumentSource.java | 13 +- .../utils/cmdLine/ArgumentTypeDescriptor.java | 257 ++++++++---------- .../utils/cmdLine/CommandLineProgram.java | 20 +- .../sting/utils/cmdLine/ParsingEngine.java | 37 ++- .../sting/utils/sam/SAMFileWriterBuilder.java | 101 ------- .../gatk/{ => io}/OutputTrackerTest.java | 61 +++-- .../indels/CleanedReadInjectorTest.java | 1 - 33 files changed, 1881 insertions(+), 865 deletions(-) delete mode 100755 java/src/org/broadinstitute/sting/gatk/OutputTracker.java create mode 100755 java/src/org/broadinstitute/sting/gatk/executive/OutputMergeTask.java delete mode 100755 java/src/org/broadinstitute/sting/gatk/executive/OutputMerger.java create mode 100644 java/src/org/broadinstitute/sting/gatk/io/DirectOutputTracker.java create mode 100755 java/src/org/broadinstitute/sting/gatk/io/OutputTracker.java create mode 100644 java/src/org/broadinstitute/sting/gatk/io/ThreadLocalOutputTracker.java create mode 100644 java/src/org/broadinstitute/sting/gatk/io/storage/OutputStreamStorage.java create mode 100644 java/src/org/broadinstitute/sting/gatk/io/storage/SAMFileWriterStorage.java create mode 100644 java/src/org/broadinstitute/sting/gatk/io/storage/Storage.java create mode 100644 java/src/org/broadinstitute/sting/gatk/io/storage/StorageFactory.java create mode 100644 java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamArgumentTypeDescriptor.java create mode 100644 java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamStub.java create mode 100644 java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileReaderArgumentTypeDescriptor.java create mode 100644 java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java create mode 100644 java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterStub.java create mode 100644 java/src/org/broadinstitute/sting/gatk/io/stubs/Stub.java create mode 100644 java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentDefinition.java delete mode 100644 java/src/org/broadinstitute/sting/utils/sam/SAMFileWriterBuilder.java rename java/test/org/broadinstitute/sting/gatk/{ => io}/OutputTrackerTest.java (62%) diff --git a/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java b/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java index 16cd32875..d5cb9c0a9 100644 --- a/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java +++ b/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java @@ -1,19 +1,20 @@ package org.broadinstitute.sting.gatk; import org.broadinstitute.sting.utils.cmdLine.CommandLineProgram; -import org.broadinstitute.sting.utils.cmdLine.ArgumentSource; +import org.broadinstitute.sting.utils.cmdLine.ArgumentTypeDescriptor; import org.broadinstitute.sting.utils.StingException; import org.broadinstitute.sting.utils.xReadLines; -import org.broadinstitute.sting.utils.sam.SAMFileWriterBuilder; -import org.broadinstitute.sting.utils.sam.SAMFileReaderBuilder; import org.broadinstitute.sting.gatk.walkers.Walker; +import org.broadinstitute.sting.gatk.io.stubs.OutputStreamArgumentTypeDescriptor; +import org.broadinstitute.sting.gatk.io.stubs.SAMFileWriterArgumentTypeDescriptor; +import org.broadinstitute.sting.gatk.io.stubs.SAMFileReaderArgumentTypeDescriptor; import java.io.File; import java.io.FileNotFoundException; import java.util.List; import java.util.ArrayList; - -import net.sf.samtools.SAMFileWriter; +import java.util.Collection; +import java.util.Arrays; /* * Copyright (c) 2009 The Broad Institute @@ -87,6 +88,17 @@ public abstract class CommandLineExecutable extends CommandLineProgram { return 0; } + /** + * Subclasses of CommandLinePrograms can provide their own types of command-line arguments. + * @return A collection of type descriptors generating implementation-dependent placeholders. + */ + protected Collection getArgumentTypeDescriptors() { + return Arrays.asList( new SAMFileReaderArgumentTypeDescriptor(GATKEngine), + new SAMFileWriterArgumentTypeDescriptor(GATKEngine), + new OutputStreamArgumentTypeDescriptor(GATKEngine) ); + } + + /** * GATK can add arguments dynamically based on analysis type. * @@ -108,24 +120,6 @@ public abstract class CommandLineExecutable extends CommandLineProgram { return new Class[] { GATKEngine.getWalkerByName(getAnalysisName()).getClass() }; } - /** - * Allows arguments to be hijacked by subclasses of the program before being placed - * into plugin classes. - * @return True if the particular field should be hijacked; false otherwise. - */ - protected boolean intercept( ArgumentSource source, Object targetInstance, Object value ) { - if( !(Walker.class.isAssignableFrom(source.clazz)) ) - return false; - - if( value instanceof SAMFileReaderBuilder || value instanceof SAMFileWriterBuilder ) { - GATKEngine.setAdditionalIO( source.field, value ); - return true; - } - - return false; - } - - @Override protected String getArgumentSourceName( Class argumentSource ) { return WalkerManager.getWalkerName((Class) argumentSource); diff --git a/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java b/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java index 26aaefc1d..705696836 100755 --- a/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java +++ b/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java @@ -40,14 +40,16 @@ import org.broadinstitute.sting.gatk.refdata.ReferenceOrderedData; import org.broadinstitute.sting.gatk.refdata.ReferenceOrderedDatum; import org.broadinstitute.sting.gatk.walkers.*; import org.broadinstitute.sting.gatk.filters.ZeroMappingQualityReadFilter; +import org.broadinstitute.sting.gatk.io.OutputTracker; import org.broadinstitute.sting.utils.*; import org.broadinstitute.sting.utils.fasta.IndexedFastaSequenceFile; import org.broadinstitute.sting.utils.cmdLine.ArgumentException; +import org.broadinstitute.sting.utils.cmdLine.ArgumentSource; +import org.broadinstitute.sting.gatk.io.stubs.Stub; import java.io.File; import java.io.FileNotFoundException; import java.util.*; -import java.lang.reflect.Field; public class GenomeAnalysisEngine { @@ -73,8 +75,12 @@ public class GenomeAnalysisEngine { // our argument collection private GATKArgumentCollection argCollection; - /** Collection of output streams used by the walker. */ - private OutputTracker outputTracker = new OutputTracker(); + /** Collection of inputs used by the walker. */ + private Map inputs = new HashMap(); + + /** Collection of outputs used by the walker. */ + private Collection> outputs = new ArrayList>(); + /** our log, which we want to capture anything from this class */ private static Logger logger = Logger.getLogger(GenomeAnalysisEngine.class); @@ -120,7 +126,7 @@ public class GenomeAnalysisEngine { MicroScheduler microScheduler = createMicroscheduler(my_walker); // create the output streams - initializeOutputStreams(my_walker); + initializeOutputStreams(my_walker, microScheduler.getOutputTracker()); GenomeLocSortedSet locs = null; if (argCollection.intervals != null) { @@ -134,12 +140,20 @@ public class GenomeAnalysisEngine { } /** - * Add additional, externally managed IO streams for walker consumption. - * @param walkerField Field in the walker into which to inject the value. + * Add additional, externally managed IO streams for walker input. + * @param argumentSource Field in the walker into which to inject the value. * @param value Instance to inject. */ - public void setAdditionalIO( Field walkerField, Object value ) { - outputTracker.addAdditionalOutput( walkerField, value ); + public void addInput( ArgumentSource argumentSource, Object value ) { + inputs.put(argumentSource,value); + } + + /** + * Add additional, externally managed IO streams for walker output. + * @param stub Instance to inject. + */ + public void addOutput( Stub stub ) { + outputs.add(stub); } /** @@ -593,21 +607,18 @@ public class GenomeAnalysisEngine { * * @param walker the walker to initialize output streams for */ - private void initializeOutputStreams(Walker walker) { + private void initializeOutputStreams(Walker walker, OutputTracker outputTracker) { if( argCollection.outErrFileName != null ) outputTracker.initializeCoreIO( argCollection.outErrFileName, argCollection.outErrFileName ); else outputTracker.initializeCoreIO( argCollection.outFileName, argCollection.errFileName ); - outputTracker.prepareWalker(walker); - } - /** - * Gets the output tracker. Tracks data available to a given walker. - * - * @return The output tracker. - */ - public OutputTracker getOutputTracker() { - return outputTracker; + for( Map.Entry input: inputs.entrySet() ) + outputTracker.addInput(input.getKey(),input.getValue()); + for( Stub stub: outputs ) + outputTracker.addOutput(stub); + + outputTracker.prepareWalker(walker); } public SAMFileHeader getSAMFileHeader() { diff --git a/java/src/org/broadinstitute/sting/gatk/OutputTracker.java b/java/src/org/broadinstitute/sting/gatk/OutputTracker.java deleted file mode 100755 index bf65f26a0..000000000 --- a/java/src/org/broadinstitute/sting/gatk/OutputTracker.java +++ /dev/null @@ -1,192 +0,0 @@ -package org.broadinstitute.sting.gatk; - -import org.broadinstitute.sting.utils.StingException; -import org.broadinstitute.sting.utils.JVMUtils; -import org.broadinstitute.sting.utils.sam.SAMFileWriterBuilder; -import org.broadinstitute.sting.utils.sam.SAMFileReaderBuilder; -import org.broadinstitute.sting.utils.io.RedirectingOutputStream; -import org.broadinstitute.sting.gatk.walkers.Walker; - -import java.io.FileOutputStream; -import java.io.PrintStream; -import java.io.FileNotFoundException; -import java.io.OutputStream; -import java.io.PrintWriter; -import java.lang.reflect.Field; -import java.util.Map; -import java.util.HashMap; - -import net.sf.samtools.SAMFileWriter; -/** - * User: hanna - * Date: Apr 30, 2009 - * Time: 9:40:09 AM - * BROAD INSTITUTE SOFTWARE COPYRIGHT NOTICE AND AGREEMENT - * Software and documentation are copyright 2005 by the Broad Institute. - * All rights are reserved. - * - * Users acknowledge that this software is supplied without any warranty or support. - * The Broad Institute is not responsible for its use, misuse, or - * functionality. - */ - -/** - * Manages the output and err streams that are created specifically for walker - * output. - */ - -public class OutputTracker { - /** - * The streams to which walker users should be writing directly. - */ - protected OutputStream globalOut; - protected OutputStream globalErr; - - /** - * Thread-local versions of the output stream. - */ - protected ThreadLocal localOut = new ThreadLocal(); - protected ThreadLocal localErr = new ThreadLocal(); - - protected Map additionalIO = new HashMap(); - - /** - * Create an object to manage output given filenames for the output and error files. - * If no files are specified, returns null. - * @param outFileName Name of the output file. - * @param errFileName Name of the error file. - */ - public void initializeCoreIO( String outFileName, String errFileName ) { - // If the two output streams match and are non-null, initialize them identically. - // Otherwise, initialize them separately. - if( outFileName != null && outFileName.equals(errFileName) ) { - FileOutputStream outputFile = prepareOutputFile( outFileName ); - globalOut = globalErr = outputFile; - } - else { - globalOut = (outFileName != null) ? prepareOutputFile( outFileName ) : System.out; - globalErr = (errFileName != null) ? prepareOutputFile( errFileName ) : System.err; - } - } - - public void prepareWalker( Walker walker ) { - Field out = JVMUtils.findField( walker.getClass(), "out" ); - Field err = JVMUtils.findField( walker.getClass(), "err" ); - - JVMUtils.setField( out, walker, new PrintStream(getOutStream()) ); - JVMUtils.setField( err, walker, new PrintStream(getErrStream()) ); - - for( Map.Entry io: additionalIO.entrySet() ) { - Field targetField = io.getKey(); - Object targetValue = io.getValue(); - - // Ghastly hacks: reaches in and finishes building out the SAMFileReader / SAMFileWriter. - // TODO: Generalize this, and move it to its own initialization step. - if( targetValue instanceof SAMFileReaderBuilder) { - SAMFileReaderBuilder builder = (SAMFileReaderBuilder)targetValue; - builder.setValidationStringency(GenomeAnalysisEngine.instance.getArguments().strictnessLevel); - targetValue = builder.build(); - } - - if( targetValue instanceof SAMFileWriterBuilder ) { - SAMFileWriterBuilder builder = (SAMFileWriterBuilder)targetValue; - builder.setSAMFileHeader(GenomeAnalysisEngine.instance.getDataSource().getHeader()); - targetValue = builder.build(); - } - - JVMUtils.setField( targetField, walker, targetValue ); - } - } - - /** - * Gets the output stream for the walker. - * @return Output stream; should be either file-backed or System.out. - */ - public OutputStream getGlobalOutStream() { - return globalOut; - } - - /** - * Gets the error stream for the walker. - * @return Error stream; should be either file-backed or System.err. - */ - public OutputStream getGlobalErrStream() { - return globalErr; - } - - /** - * Retrieve an output stream that will always return the most appropriate - * writer for this thread. - */ - public OutputStream getOutStream() { - // Create an anonymous inner class which will just return the most - // appropriate OutputStream from those streams stored in this class. - return new RedirectingOutputStream( - new RedirectingOutputStream.OutputStreamProvider() { - public OutputStream getOutputStream() { - return localOut.get() != null ? localOut.get() : globalOut; - } - } ); - } - - /** - * Retrieve the most appropriate output stream for this thread. - * Will retrieve thread-local if available; otherwise, it'll read the - * global stream. - */ - public OutputStream getErrStream() { - // Create an anonymous inner class which will just return the most - // appropriate OutputStream from those streams stored in this class. - return new RedirectingOutputStream( - new RedirectingOutputStream.OutputStreamProvider() { - public OutputStream getOutputStream() { - return localErr.get() != null ? localErr.get() : globalErr; - } - } ); - } - - /** - * Set the (thread-)local version of the given output and error files. - * @param out output stream to which to write. - * @param err error stream to which to write. - */ - public void setLocalStreams( OutputStream out, OutputStream err ) { - localOut.set( out ); - localErr.set( err ); - } - - /** - * Provide a mechanism for injecting supplemental streams for external management. - * @param field Field into which to inject this stream. - * @param stream Stream to manage. - */ - public void addAdditionalOutput( Field field, Object stream ) { - additionalIO.put(field,stream); - } - - /** - * Remove pointers to alternate, local output streams. - */ - public void cleanup() { - localOut.remove(); - localErr.remove(); - } - - /** - * Given a (non-null) filename, open a file for output. - * @param fileName Filename for the stream. Should not be null. - * @return An open output stream associated with the given file. - */ - private FileOutputStream prepareOutputFile( String fileName ) { - FileOutputStream outputFile = null; - - try { - outputFile = new FileOutputStream( fileName ); - } - catch( FileNotFoundException ex ) { - throw new StingException("Unable to open output file " + fileName, ex); - } - - return outputFile; - } -} diff --git a/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java b/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java index 547d8eb5a..e1366e13e 100755 --- a/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java +++ b/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java @@ -6,21 +6,14 @@ import org.broadinstitute.sting.gatk.datasources.shards.ShardStrategy; import org.broadinstitute.sting.gatk.datasources.shards.Shard; import org.broadinstitute.sting.gatk.datasources.simpleDataSources.SAMDataSource; import org.broadinstitute.sting.gatk.datasources.simpleDataSources.ReferenceOrderedDataSource; -import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; -import org.broadinstitute.sting.gatk.OutputTracker; -import org.broadinstitute.sting.gatk.Reads; -import org.broadinstitute.sting.gatk.refdata.ReferenceOrderedDatum; -import org.broadinstitute.sting.gatk.refdata.ReferenceOrderedData; +import org.broadinstitute.sting.gatk.io.*; import org.broadinstitute.sting.utils.StingException; -import org.broadinstitute.sting.utils.GenomeLocSortedSet; import org.broadinstitute.sting.utils.fasta.IndexedFastaSequenceFile; import org.broadinstitute.sting.utils.threading.ThreadPoolMonitor; import javax.management.MBeanServer; import javax.management.ObjectName; import javax.management.JMException; -import java.io.File; -import java.util.List; import java.util.Queue; import java.util.LinkedList; import java.util.Collection; @@ -30,14 +23,6 @@ import java.util.concurrent.Future; import java.util.concurrent.FutureTask; import java.lang.management.ManagementFactory; -/** - * Created by IntelliJ IDEA. - * User: mhanna - * Date: Apr 26, 2009 - * Time: 5:41:04 PM - * To change this template use File | Settings | File Templates. - */ - /** * A microscheduler that schedules shards according to a tree-like structure. * Requires a special walker tagged with a 'TreeReducible' interface. @@ -52,9 +37,20 @@ public class HierarchicalMicroScheduler extends MicroScheduler implements Hierar /** Manage currently running threads. */ private ExecutorService threadPool; - private Queue traverseTasks = new LinkedList(); - private Queue reduceTasks = new LinkedList(); - private Queue outputMergeTasks = new LinkedList(); + /** + * A thread local output tracker for managing output per-thread. + */ + private ThreadLocalOutputTracker outputTracker = new ThreadLocalOutputTracker(); + + private final Queue traverseTasks = new LinkedList(); + private final Queue reduceTasks = new LinkedList(); + + /** + * Keep a queue of shard traversals, and constantly monitor it to see what output + * merge tasks remain. + * TODO: Integrate this into the reduce tree. + */ + private final Queue outputMergeTasks = new LinkedList(); /** How many total tasks were in the queue at the start of run. */ private int totalTraversals = 0; @@ -115,7 +111,7 @@ public class HierarchicalMicroScheduler extends MicroScheduler implements Hierar while (isShardTraversePending() || isTreeReducePending()) { // Too many files sitting around taking up space? Merge them. if (isMergeLimitExceeded()) - mergeExistingOutput(); + mergeExistingOutput(false); // Wait for the next slot in the queue to become free. waitForFreeQueueSlot(); @@ -130,7 +126,7 @@ public class HierarchicalMicroScheduler extends MicroScheduler implements Hierar // Merge any lingering output files. If these files aren't ready, // sit around and wait for them, then merge them. - mergeRemainingOutput(); + mergeExistingOutput(true); threadPool.shutdown(); @@ -149,6 +145,13 @@ public class HierarchicalMicroScheduler extends MicroScheduler implements Hierar return result; } + /** + * @{inheritDoc} + */ + public OutputTracker getOutputTracker() { + return outputTracker; + } + /** * Returns true if there are unscheduled shard traversal waiting to run. * @@ -188,62 +191,43 @@ public class HierarchicalMicroScheduler extends MicroScheduler implements Hierar * @return True if the merging needs to take priority. False otherwise. */ protected boolean isMergeLimitExceeded() { - if (outputMergeTasks.size() < MAX_OUTSTANDING_OUTPUT_MERGES) - return false; - - // If any of the first MAX_OUTSTANDING merges aren't ready, the merge limit - // has not been exceeded. - OutputMerger[] outputMergers = outputMergeTasks.toArray(new OutputMerger[0]); - for (int i = 0; i < MAX_OUTSTANDING_OUTPUT_MERGES; i++) { - if (!outputMergers[i].isComplete()) - return false; + int pendingTasks = 0; + for( ShardTraverser shardTraverse: outputMergeTasks ) { + if( !shardTraverse.isComplete() ) + break; + pendingTasks++; } - - // Everything's ready? - return true; - } - - /** - * Returns whether there is output waiting to be merged into the global output - * streams right now. - * - * @return True if this output is ready to be merged. False otherwise. - */ - protected boolean isOutputMergeReady() { - if (outputMergeTasks.size() > 0) - return outputMergeTasks.peek().isComplete(); - else - return false; + return (outputMergeTasks.size() >= MAX_OUTSTANDING_OUTPUT_MERGES); } /** * Merging all output that's sitting ready in the OutputMerger queue into * the final data streams. */ - protected void mergeExistingOutput() { + protected void mergeExistingOutput( boolean wait ) { long startTime = System.currentTimeMillis(); - OutputTracker outputTracker = GenomeAnalysisEngine.instance.getOutputTracker(); - while (isOutputMergeReady()) - outputMergeTasks.remove().mergeInto(outputTracker.getGlobalOutStream(), outputTracker.getGlobalErrStream()); + // Create a list of the merge tasks that will be performed in this run of the mergeExistingOutput(). + Queue mergeTasksInSession = new LinkedList(); + while( !outputMergeTasks.isEmpty() ) { + ShardTraverser traverser = outputMergeTasks.peek(); - long endTime = System.currentTimeMillis(); + // If the next traversal isn't done and we're not supposed to wait, we've found our working set. Continue. + if( !traverser.isComplete() && !wait ) + break; - totalOutputMergeTime += ( endTime - startTime ); - } + outputMergeTasks.remove(); + mergeTasksInSession.add(traverser); + } - /** Merge any output that hasn't yet been taken care of by the blocking thread. */ - protected void mergeRemainingOutput() { - long startTime = System.currentTimeMillis(); + // Actually run through, merging the tasks in the working queue. + for( ShardTraverser traverser: mergeTasksInSession ) { + if( !traverser.isComplete() ) + traverser.waitForComplete(); - OutputTracker outputTracker = GenomeAnalysisEngine.instance.getOutputTracker(); - while (outputMergeTasks.size() > 0) { - OutputMerger outputMerger = outputMergeTasks.remove(); - synchronized (outputMerger) { - if (!outputMerger.isComplete()) - outputMerger.waitForOutputComplete(); - } - outputMerger.mergeInto(outputTracker.getGlobalOutStream(), outputTracker.getGlobalErrStream()); + OutputMergeTask mergeTask = traverser.getOutputMergeTask(); + if( mergeTask != null ) + mergeTask.merge(); } long endTime = System.currentTimeMillis(); @@ -262,26 +246,24 @@ public class HierarchicalMicroScheduler extends MicroScheduler implements Hierar throw new IllegalStateException("Cannot traverse; no pending traversals exist."); Shard shard = traverseTasks.remove(); - OutputMerger outputMerger = new OutputMerger(); ShardTraverser traverser = new ShardTraverser(this, traversalEngine, walker, shard, getShardDataProvider(shard), - outputMerger); + outputTracker); Future traverseResult = threadPool.submit(traverser); // Add this traverse result to the reduce tree. The reduce tree will call a callback to throw its entries on the queue. reduceTree.addEntry(traverseResult); + outputMergeTasks.add(traverser); // No more data? Let the reduce tree know so it can finish processing what it's got. if (!isShardTraversePending()) reduceTree.complete(); - outputMergeTasks.add(outputMerger); - return traverseResult; } @@ -372,7 +354,9 @@ public class HierarchicalMicroScheduler extends MicroScheduler implements Hierar /** {@inheritDoc} */ public int getNumberOfTasksInIOQueue() { - return outputMergeTasks.size(); + synchronized( outputMergeTasks ) { + return outputMergeTasks.size(); + } } /** {@inheritDoc} */ diff --git a/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java b/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java index d318ab861..d525f3b7c 100644 --- a/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java +++ b/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java @@ -6,6 +6,8 @@ import org.broadinstitute.sting.gatk.datasources.shards.ShardStrategy; import org.broadinstitute.sting.gatk.datasources.simpleDataSources.ReferenceOrderedDataSource; import org.broadinstitute.sting.gatk.datasources.simpleDataSources.SAMDataSource; import org.broadinstitute.sting.gatk.walkers.Walker; +import org.broadinstitute.sting.gatk.io.DirectOutputTracker; +import org.broadinstitute.sting.gatk.io.OutputTracker; import org.broadinstitute.sting.utils.fasta.IndexedFastaSequenceFile; import java.util.Collection; @@ -13,6 +15,11 @@ import java.util.Collection; /** A micro-scheduling manager for single-threaded execution of a traversal. */ public class LinearMicroScheduler extends MicroScheduler { + /** + * A direct output tracker for directly managing output. + */ + private DirectOutputTracker outputTracker = new DirectOutputTracker(); + /** * Create a new linear microscheduler to process the given reads and reference. * @@ -55,5 +62,8 @@ public class LinearMicroScheduler extends MicroScheduler { return accumulator; } - + /** + * @{inheritDoc} + */ + public OutputTracker getOutputTracker() { return outputTracker; } } diff --git a/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java b/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java index 32f3af31f..32712412d 100755 --- a/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java +++ b/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java @@ -33,6 +33,8 @@ import org.broadinstitute.sting.gatk.datasources.simpleDataSources.ReferenceOrde import org.broadinstitute.sting.gatk.datasources.simpleDataSources.SAMDataSource; import org.broadinstitute.sting.gatk.traversals.*; import org.broadinstitute.sting.gatk.walkers.*; +import org.broadinstitute.sting.gatk.io.OutputTracker; +import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.utils.fasta.IndexedFastaSequenceFile; import java.util.*; @@ -116,6 +118,12 @@ public abstract class MicroScheduler { */ public abstract Object execute(Walker walker, ShardStrategy shardStrategy, int iterations ); + /** + * Retrieves the object responsible for tracking and managing output. + * @return An output tracker, for loading data in and extracting results. Will not be null. + */ + public abstract OutputTracker getOutputTracker(); + /** * Gets an window into all the data that can be viewed as a single shard. diff --git a/java/src/org/broadinstitute/sting/gatk/executive/OutputMergeTask.java b/java/src/org/broadinstitute/sting/gatk/executive/OutputMergeTask.java new file mode 100755 index 000000000..76e0c1c8a --- /dev/null +++ b/java/src/org/broadinstitute/sting/gatk/executive/OutputMergeTask.java @@ -0,0 +1,78 @@ +package org.broadinstitute.sting.gatk.executive; + +import org.broadinstitute.sting.gatk.io.storage.Storage; +import org.broadinstitute.sting.gatk.io.OutputTracker; + +import java.util.Collection; +import java.util.ArrayList; + +/** + * User: hanna + * Date: Apr 30, 2009 + * Time: 4:04:38 PM + * BROAD INSTITUTE SOFTWARE COPYRIGHT NOTICE AND AGREEMENT + * Software and documentation are copyright 2005 by the Broad Institute. + * All rights are reserved. + * + * Users acknowledge that this software is supplied without any warranty or support. + * The Broad Institute is not responsible for its use, misuse, or + * functionality. + */ + +/** + * Hold pointers to the output and error streams, and state to indicate whether + * a write is complete. Not generally thread-safe. Calls to isComplete()/complete() + * can be made at any time from any thread, but complete() should be called on the + * thread which is doing the writing. + */ +public class OutputMergeTask { + /** + * The output streams which should be written to. + */ + private final Collection> mergeOperations = new ArrayList>(); + + /** + * Add a new merge operation to this merge task. + * @param targetStream Target for stream output. + * @param temporaryStorage Temporary storage. + * @param Type of the output stream. + */ + public void addMergeOperation( StreamType targetStream, Storage temporaryStorage ) { + mergeOperations.add( new MergeOperation(targetStream,temporaryStorage) ); + } + + /** + * Merge data from output streams into target storage. + */ + public synchronized void merge() { + for( MergeOperation mergeOperation: mergeOperations ) + mergeOperation.temporaryStorage.mergeInto(mergeOperation.targetStream); + } + + /** + * Represents a single file needed to be merged. + * @param Type of the file to be merged. + */ + private class MergeOperation { + /** + * Destination for the temporary file's output. + */ + public final StreamType targetStream; + + /** + * Temporary storage location for the file. + */ + public final Storage temporaryStorage; + + /** + * Create a new merge file object with the given output stream and storage placeholder. + * @param targetStream Target for temporary data. + * @param temporaryStorage The temporary data itself. + */ + public MergeOperation( StreamType targetStream, Storage temporaryStorage ) { + this.targetStream = targetStream; + this.temporaryStorage = temporaryStorage; + } + } +} + diff --git a/java/src/org/broadinstitute/sting/gatk/executive/OutputMerger.java b/java/src/org/broadinstitute/sting/gatk/executive/OutputMerger.java deleted file mode 100755 index a06a02266..000000000 --- a/java/src/org/broadinstitute/sting/gatk/executive/OutputMerger.java +++ /dev/null @@ -1,167 +0,0 @@ -package org.broadinstitute.sting.gatk.executive; - -import org.broadinstitute.sting.utils.StingException; -import org.broadinstitute.sting.utils.io.LazyFileOutputStream; - -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.nio.channels.FileChannel; -import java.nio.channels.Channels; -import java.nio.channels.WritableByteChannel; - -/** - * User: hanna - * Date: Apr 30, 2009 - * Time: 4:04:38 PM - * BROAD INSTITUTE SOFTWARE COPYRIGHT NOTICE AND AGREEMENT - * Software and documentation are copyright 2005 by the Broad Institute. - * All rights are reserved. - * - * Users acknowledge that this software is supplied without any warranty or support. - * The Broad Institute is not responsible for its use, misuse, or - * functionality. - */ - -/** - * Hold pointers to the output and error streams, and state to indicate whether - * a write is complete. Not generally thread-safe. Calls to isComplete()/complete() - * can be made at any time from any thread, but complete() should be called on the - * thread which is doing the writing. - */ -public class OutputMerger { - /** - * Is writing to these streams complete? - */ - private boolean complete = false; - - /** - * The printstreams which should be written to. - */ - private LazyFileOutputStream out = null; - private LazyFileOutputStream err = null; - - public OutputMerger() { - out = new LazyFileOutputStream( outStreamFactory ); - err = new LazyFileOutputStream( errStreamFactory ); - } - - /** - * Creates a generic output stream for temporary data. - */ - private static class TempFileFactory implements LazyFileOutputStream.FileFactory { - private final String prefix; - public TempFileFactory( String prefix ) { this.prefix = prefix; } - public File create() throws IOException { return File.createTempFile(prefix,null); } - } - - /** - * Creates a stream for temporary out (not err) data. - */ - private static final TempFileFactory outStreamFactory = new TempFileFactory("gatkout_"); - - /** - * Creates a stream for temporary err data. - */ - private static final TempFileFactory errStreamFactory = new TempFileFactory("gatkerr_"); - - /** - * Waits for any the given OutputMerger to be ready for merging. - */ - public synchronized void waitForOutputComplete() { - try { - wait(); - } - catch( InterruptedException ex ) { - throw new StingException("Interrupted while waiting for more output to be finalized.",ex); - } - } - - /** - * Is this output complete? - * @return True if output complete. False otherwise. - */ - public synchronized boolean isComplete() { - return complete; - } - - /** - * Indicate that no more data will be written to these output streams. - */ - public synchronized void complete() { - if( isComplete() ) - throw new IllegalStateException("Tried to complete an output merge twice."); - - try { - if( out.isCreated() ) { - out.flush(); - out.close(); - } - if( err.isCreated() ) { - err.flush(); - err.close(); - } - } - catch( IOException ex ) { - throw new StingException( "Unable to close sharding output files", ex ); - } - - this.complete = true; - - // Notify waiting listeners that this shard is complete and ready for merging. - notifyAll(); - } - - public void mergeInto( OutputStream globalOut, OutputStream globalErr ) { - synchronized(this) { - if( !isComplete() ) - throw new StingException("Tried to merge incomplete stream into output"); - } - - if( out.isCreated() ) transferContentsToTarget( out.getBackingFile(), globalOut ); - if( err.isCreated() ) transferContentsToTarget( err.getBackingFile(), globalErr ); - } - - /** - * Copy the contents of the given file into the specified output stream. - * @param source Source of data to copy. - * @param target Target for copied data. - */ - private void transferContentsToTarget( File source, OutputStream target ) { - FileInputStream sourceStream = null; - try { - sourceStream = new FileInputStream( source ); - FileChannel sourceChannel = sourceStream.getChannel(); - - WritableByteChannel targetChannel = Channels.newChannel( target ); - sourceChannel.transferTo( 0, sourceChannel.size(), targetChannel ); - - sourceStream.close(); - source.delete(); - } - catch( FileNotFoundException ex ) { - throw new StingException("Unable to open input stream for file: " + source,ex); - } - catch( IOException ex ) { - throw new StingException("Unable to transfer contents of file:" + source,ex); - } - } - - /** - * Return the stream where output is sent. - * @return output stream. - */ - public OutputStream getOutStream() { - return out; - } - - /** - * Gets the stream where error info is sent. - * @return error stream. - */ - public OutputStream getErrStream() { - return err; - } -} diff --git a/java/src/org/broadinstitute/sting/gatk/executive/ShardTraverser.java b/java/src/org/broadinstitute/sting/gatk/executive/ShardTraverser.java index d6bd6fc1d..8eada9df1 100755 --- a/java/src/org/broadinstitute/sting/gatk/executive/ShardTraverser.java +++ b/java/src/org/broadinstitute/sting/gatk/executive/ShardTraverser.java @@ -3,9 +3,9 @@ package org.broadinstitute.sting.gatk.executive; import org.broadinstitute.sting.gatk.datasources.providers.ShardDataProvider; import org.broadinstitute.sting.gatk.datasources.shards.Shard; import org.broadinstitute.sting.gatk.traversals.TraversalEngine; -import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; -import org.broadinstitute.sting.gatk.OutputTracker; +import org.broadinstitute.sting.gatk.io.ThreadLocalOutputTracker; import org.broadinstitute.sting.gatk.walkers.Walker; +import org.broadinstitute.sting.utils.StingException; import java.util.concurrent.Callable; /** @@ -29,36 +29,43 @@ public class ShardTraverser implements Callable { private TraversalEngine traversalEngine; private Shard shard; private ShardDataProvider dataProvider; - private OutputMerger output; + private ThreadLocalOutputTracker outputTracker; + private OutputMergeTask outputMergeTask; + + /** + * Is this traversal complete? + */ + private boolean complete = false; public ShardTraverser( HierarchicalMicroScheduler microScheduler, TraversalEngine traversalEngine, Walker walker, Shard shard, ShardDataProvider dataProvider, - OutputMerger output ) { + ThreadLocalOutputTracker outputTracker ) { this.microScheduler = microScheduler; this.walker = walker; this.traversalEngine = traversalEngine; this.shard = shard; this.dataProvider = dataProvider; - this.output = output; + this.outputTracker = outputTracker; } public Object call() { long startTime = System.currentTimeMillis(); Object accumulator = walker.reduceInit(); - OutputTracker outputTracker = GenomeAnalysisEngine.instance.getOutputTracker(); - outputTracker.setLocalStreams( output.getOutStream(), output.getErrStream() ); - try { accumulator = traversalEngine.traverse( walker, shard, dataProvider, accumulator ); } finally { dataProvider.close(); - output.complete(); - outputTracker.cleanup(); + outputMergeTask = outputTracker.closeStorage(); + + synchronized(this) { + complete = true; + notifyAll(); + } } long endTime = System.currentTimeMillis(); @@ -67,4 +74,38 @@ public class ShardTraverser implements Callable { return accumulator; } + + /** + * Has this traversal completed? + * @return True if completed, false otherwise. + */ + public boolean isComplete() { + synchronized(this) { + return complete; + } + } + + /** + * Waits for any the given OutputMerger to be ready for merging. + */ + public void waitForComplete() { + try { + synchronized(this) { + if( isComplete() ) + return; + wait(); + } + } + catch( InterruptedException ex ) { + throw new StingException("Interrupted while waiting for more output to be finalized.",ex); + } + } + + /** + * Gets the output merge task associated with the given shard. + * @return OutputMergeTask if one exists; null if nothing needs to be merged. + */ + public OutputMergeTask getOutputMergeTask() { + return outputMergeTask; + } } diff --git a/java/src/org/broadinstitute/sting/gatk/io/DirectOutputTracker.java b/java/src/org/broadinstitute/sting/gatk/io/DirectOutputTracker.java new file mode 100644 index 000000000..5fcf296b6 --- /dev/null +++ b/java/src/org/broadinstitute/sting/gatk/io/DirectOutputTracker.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2009 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.io; + +import org.broadinstitute.sting.gatk.io.stubs.Stub; +import org.broadinstitute.sting.gatk.io.storage.StorageFactory; +import org.broadinstitute.sting.gatk.io.storage.Storage; + +import java.util.Map; +import java.util.HashMap; + +/** + * Javadoc goes here. + * + * @author mhanna + * @version 0.1 + */ +public class DirectOutputTracker extends OutputTracker { + /** + * Direct storage for output streams. + */ + private final Map storage = new HashMap(); + + public T getStorage( Stub stub ) { + Storage target = storage.get(stub); + if( target == null ) { + target = StorageFactory.createStorage(stub); + storage.put(stub, target); + } + return (T)target; + } + +} diff --git a/java/src/org/broadinstitute/sting/gatk/io/OutputTracker.java b/java/src/org/broadinstitute/sting/gatk/io/OutputTracker.java new file mode 100755 index 000000000..69a0be1f7 --- /dev/null +++ b/java/src/org/broadinstitute/sting/gatk/io/OutputTracker.java @@ -0,0 +1,156 @@ +package org.broadinstitute.sting.gatk.io; + +import org.broadinstitute.sting.utils.JVMUtils; +import org.broadinstitute.sting.utils.StingException; +import org.broadinstitute.sting.utils.cmdLine.ArgumentSource; +import org.broadinstitute.sting.utils.sam.SAMFileReaderBuilder; +import org.broadinstitute.sting.gatk.walkers.Walker; +import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; +import org.broadinstitute.sting.gatk.io.stubs.OutputStreamStub; +import org.broadinstitute.sting.gatk.io.stubs.Stub; +import org.broadinstitute.sting.gatk.io.storage.StorageFactory; +import org.broadinstitute.sting.gatk.io.storage.Storage; + +import java.io.*; +import java.lang.reflect.Field; +import java.util.Map; +import java.util.HashMap; + +/** + * User: hanna + * Date: Apr 30, 2009 + * Time: 9:40:09 AM + * BROAD INSTITUTE SOFTWARE COPYRIGHT NOTICE AND AGREEMENT + * Software and documentation are copyright 2005 by the Broad Institute. + * All rights are reserved. + * + * Users acknowledge that this software is supplied without any warranty or support. + * The Broad Institute is not responsible for its use, misuse, or + * functionality. + */ + +/** + * Manages the output and err streams that are created specifically for walker + * output. + */ +public abstract class OutputTracker { + /** + * The streams to which walker users should be reading directly. + */ + protected Map inputs = new HashMap(); + + /** + * The streams to which walker users should be writing directly. + */ + protected Map outputs = new HashMap(); + + /** + * Special-purpose stub. Provides a connection to output streams. + */ + protected OutputStreamStub outStub = null; + + /** + * Special-purpose stream. Provides a connection to error streams. + */ + protected OutputStreamStub errStub = null; + + /** + * Create an object to manage output given filenames for the output and error files. + * If no files are specified, returns null. + * @param outFileName Name of the output file. + * @param errFileName Name of the error file. + */ + public void initializeCoreIO( String outFileName, String errFileName ) { + // If the two output streams match and are non-null, initialize them identically. + // Otherwise, initialize them separately. + if( outFileName != null && outFileName.equals(errFileName) ) { + outStub = errStub = new OutputStreamStub(new File(outFileName)); + outStub.register(this); + outputs.put(outStub,null); + } + else { + outStub = (outFileName != null) ? new OutputStreamStub(new File(outFileName)) + : new OutputStreamStub(System.out); + outStub.register(this); + outputs.put(outStub,null); + + errStub = (errFileName != null) ? new OutputStreamStub(new File(errFileName)) + : new OutputStreamStub(System.err); + errStub.register(this); + outputs.put(errStub,null); + } + } + + /** + * Gets the output storage associated with a given stub. + * @param stub The stub for which to find / create the right output stream. + * @param Type of the stream to create. + * @return Storage object with a facade of type T. + */ + public abstract T getStorage( Stub stub ); + + public void prepareWalker( Walker walker ) { + installStub( walker, "out", new PrintStream(outStub) ); + installStub( walker, "err", new PrintStream(errStub) ); + + for( Map.Entry io: inputs.entrySet() ) { + ArgumentSource targetField = io.getKey(); + Object targetValue = io.getValue(); + + // Ghastly hack: reaches in and finishes building out the SAMFileReader. + // TODO: Generalize this, and move it to its own initialization step. + if( targetValue instanceof SAMFileReaderBuilder) { + SAMFileReaderBuilder builder = (SAMFileReaderBuilder)targetValue; + builder.setValidationStringency(GenomeAnalysisEngine.instance.getArguments().strictnessLevel); + targetValue = builder.build(); + } + + JVMUtils.setField( targetField.field, walker, targetValue ); + } + } + + /** + * Provide a mechanism for injecting supplemental streams for external management. + * @param argumentSource source Class / field into which to inject this stream. + * @param stub Stream to manage. + */ + public void addInput( ArgumentSource argumentSource, Object stub ) { + inputs.put(argumentSource,stub); + } + + /** + * Provide a mechanism for injecting supplemental streams for external management. + * @param stub Stream to manage. + */ + public void addOutput(Stub stub) { + stub.register(this); + outputs.put(stub,null); + } + + /** + * Collects the target stream for this data. + * @param stub The stub for this stream. + * @param type of stub. + * @return An instantiated file into which data can be written. + */ + protected T getTargetStream( Stub stub ) { + if( !outputs.containsKey(stub) ) + throw new StingException("OutputTracker was not notified that this stub exists: " + stub); + Storage storage = outputs.get(stub); + if( storage == null ) { + storage = StorageFactory.createStorage(stub); + outputs.put(stub,storage); + } + return (T)storage; + } + + /** + * Install an OutputStreamStub into the given fieldName of the given walker. + * @param walker Walker into which to inject the field name. + * @param fieldName Name of the field into which to inject the stub. + */ + private void installStub( Walker walker, String fieldName, OutputStream outputStream ) { + Field field = JVMUtils.findField( walker.getClass(), fieldName ); + JVMUtils.setField( field, walker, outputStream ); + } +} diff --git a/java/src/org/broadinstitute/sting/gatk/io/ThreadLocalOutputTracker.java b/java/src/org/broadinstitute/sting/gatk/io/ThreadLocalOutputTracker.java new file mode 100644 index 000000000..1799b71bf --- /dev/null +++ b/java/src/org/broadinstitute/sting/gatk/io/ThreadLocalOutputTracker.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2009 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.io; + +import org.broadinstitute.sting.gatk.io.stubs.Stub; +import org.broadinstitute.sting.gatk.io.storage.StorageFactory; +import org.broadinstitute.sting.gatk.io.storage.Storage; +import org.broadinstitute.sting.gatk.executive.OutputMergeTask; +import org.broadinstitute.sting.utils.StingException; + +import java.util.*; +import java.io.File; +import java.io.IOException; + +/** + * An output tracker that tracks its output per-thread. + * + * @author mhanna + * @version 0.1 + */ +public class ThreadLocalOutputTracker extends OutputTracker { + /** + * Thread-local storage for output streams. + */ + private ThreadLocal> storage = new ThreadLocal>(); + + public T getStorage( Stub stub ) { + Map threadLocalOutputStreams = storage.get(); + + if( threadLocalOutputStreams == null ) { + threadLocalOutputStreams = new HashMap(); + storage.set( threadLocalOutputStreams ); + } + + Storage target = threadLocalOutputStreams.get(stub); + if( target == null ) { + target = StorageFactory.createStorage(stub, createTempFile(stub)); + threadLocalOutputStreams.put(stub, target); + } + + return (T)target; + } + + /** + * Close down any existing temporary files which have been opened. + */ + public OutputMergeTask closeStorage() { + Map threadLocalStorage = storage.get(); + + if( threadLocalStorage == null || threadLocalStorage.isEmpty() ) + return null; + + OutputMergeTask outputMergeTask = new OutputMergeTask(); + for( Map.Entry entry: threadLocalStorage.entrySet() ) { + Stub stub = entry.getKey(); + Storage storageEntry = entry.getValue(); + + storageEntry.close(); + outputMergeTask.addMergeOperation(getTargetStream(stub),storageEntry); + } + + threadLocalStorage.clear(); + + return outputMergeTask; + } + + /** + * Creates a temporary file for a stub of the given type. + * @param stub Stub for which to create a temporary file. + * @param Type of the stub to accept. + * @return A temp file, or throw an exception if the temp file cannot be created. + */ + private File createTempFile( Stub stub ) { + File tempFile = null; + + try { + tempFile = File.createTempFile( stub.getClass().getName(), null ); + tempFile.deleteOnExit(); + } + catch( IOException ex ) { + throw new StingException("Unable to create temporary file for stub: " + stub.getClass().getName() ); + } + + return tempFile; + } +} diff --git a/java/src/org/broadinstitute/sting/gatk/io/storage/OutputStreamStorage.java b/java/src/org/broadinstitute/sting/gatk/io/storage/OutputStreamStorage.java new file mode 100644 index 000000000..8f71eaa57 --- /dev/null +++ b/java/src/org/broadinstitute/sting/gatk/io/storage/OutputStreamStorage.java @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2009 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.io.storage; + +import org.broadinstitute.sting.utils.StingException; +import org.broadinstitute.sting.gatk.io.stubs.OutputStreamStub; +import org.broadinstitute.sting.gatk.io.storage.Storage; + +import java.io.*; +import java.nio.channels.FileChannel; +import java.nio.channels.WritableByteChannel; +import java.nio.channels.Channels; + +public class OutputStreamStorage extends OutputStream implements Storage { + /** + * File to which data will temporarily be written. + */ + private final File file; + + /** + * Stream to which data in this shard will be written. + */ + private final OutputStream outputStream; + + /** + * Create a new storage area with the given stub. + * @param stub + */ + public OutputStreamStorage( OutputStreamStub stub ) { + if( stub.getOutputFile() != null ) { + this.file = stub.getOutputFile(); + this.outputStream = initializeOutputStream(stub.getOutputFile()); + } + else if( stub.getOutputStream() != null ) { + this.file = null; + this.outputStream = stub.getOutputStream(); + } + else + throw new StingException("Not enough information to create storage for an OutputStream; need either a file or an existing output stream"); + } + + public OutputStreamStorage( OutputStreamStub stub, File file ) { + this.file = file; + this.outputStream = initializeOutputStream(file); + } + + private OutputStream initializeOutputStream( File file ) { + try { + return new FileOutputStream( file ); + } + catch(FileNotFoundException ex) { + throw new StingException("Unable to open output stream for file: " + file); + } + } + + /** + * @{inheritDoc} + */ + public void flush() throws IOException { + outputStream.close(); + } + + /** + * @{inheritDoc} + */ + public void close() { + try { + outputStream.close(); + } + catch( IOException ex ) { + throw new StingException( "Unable to close output stream" ); + } + } + + /** + * @{inheritDoc} + */ + public void write( byte[] b ) throws IOException { + outputStream.write(b); + } + + /** + * @{inheritDoc} + */ + public void write( byte[] b, int off, int len ) throws IOException { + outputStream.write(b, off, len); + } + + /** + * @{inheritDoc} + */ + public void write( int b ) throws IOException { + outputStream.write(b); + } + + + public void mergeInto( OutputStream targetStream ) { + FileInputStream sourceStream = null; + try { + sourceStream = new FileInputStream( file ); + FileChannel sourceChannel = sourceStream.getChannel(); + + WritableByteChannel targetChannel = Channels.newChannel( targetStream ); + sourceChannel.transferTo( 0, sourceChannel.size(), targetChannel ); + + sourceStream.close(); + file.delete(); + } + catch( FileNotFoundException ex ) { + throw new StingException("Unable to open input stream for file: " + file,ex); + } + catch( IOException ex ) { + throw new StingException("Unable to transfer contents of file:" + file,ex); + } + } +} diff --git a/java/src/org/broadinstitute/sting/gatk/io/storage/SAMFileWriterStorage.java b/java/src/org/broadinstitute/sting/gatk/io/storage/SAMFileWriterStorage.java new file mode 100644 index 000000000..9fe56e1a8 --- /dev/null +++ b/java/src/org/broadinstitute/sting/gatk/io/storage/SAMFileWriterStorage.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2009 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.io.storage; + +import net.sf.samtools.*; +import net.sf.samtools.util.CloseableIterator; + +import java.io.*; + +import org.broadinstitute.sting.gatk.io.stubs.SAMFileWriterStub; +import org.broadinstitute.sting.gatk.io.storage.Storage; + +/** + * Provides temporary storage for SAMFileWriters. + * + * @author mhanna + * @version 0.1 + */ +public class SAMFileWriterStorage implements SAMFileWriter, Storage { + private final File file; + private final SAMFileWriter writer; + + public SAMFileWriterStorage( SAMFileWriterStub stub ) { + this(stub,stub.getSAMFile()); + } + + public SAMFileWriterStorage( SAMFileWriterStub stub, File file ) { + this.file = file; + this.writer = new SAMFileWriterFactory().makeBAMWriter( stub.getSAMFileHeader(), true, file ); + } + + public void addAlignment( SAMRecord read ) { + writer.addAlignment(read); + } + + public void close() { + writer.close(); + } + + public void mergeInto( SAMFileWriter targetStream ) { + SAMFileReader reader = new SAMFileReader( file ); + try { + CloseableIterator iterator = reader.iterator(); + while( iterator.hasNext() ) + targetStream.addAlignment( iterator.next() ); + iterator.close(); + } + finally { + reader.close(); + file.delete(); + } + } + +} \ No newline at end of file diff --git a/java/src/org/broadinstitute/sting/gatk/io/storage/Storage.java b/java/src/org/broadinstitute/sting/gatk/io/storage/Storage.java new file mode 100644 index 000000000..96a465c16 --- /dev/null +++ b/java/src/org/broadinstitute/sting/gatk/io/storage/Storage.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2009 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.io.storage; + +/** + * An interface representing the temporary storage of data. + * + * @author mhanna + * @version 0.1 + */ +public interface Storage { + /** + * Writing to the temporary storage is done. Close down the file. + */ + public void close(); + + /** + * Merges the stream backing up this temporary storage into the target. + * @param target Target stream for the temporary storage. May not be null. + */ + public void mergeInto( StreamType target ); +} diff --git a/java/src/org/broadinstitute/sting/gatk/io/storage/StorageFactory.java b/java/src/org/broadinstitute/sting/gatk/io/storage/StorageFactory.java new file mode 100644 index 000000000..7a76d4b5e --- /dev/null +++ b/java/src/org/broadinstitute/sting/gatk/io/storage/StorageFactory.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2009 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.io.storage; + +import org.broadinstitute.sting.gatk.io.stubs.Stub; +import org.broadinstitute.sting.gatk.io.stubs.OutputStreamStub; +import org.broadinstitute.sting.gatk.io.stubs.SAMFileWriterStub; +import org.broadinstitute.sting.gatk.io.storage.SAMFileWriterStorage; +import org.broadinstitute.sting.gatk.io.storage.Storage; +import org.broadinstitute.sting.gatk.io.storage.OutputStreamStorage; +import org.broadinstitute.sting.utils.StingException; + +import java.io.File; + +/** + * Construct storage of the required type. + * + * @author mhanna + * @version 0.1 + */ +public class StorageFactory { + /** + * Disable storage factory construction. + */ + private StorageFactory() {} + + /** + * Gets the output storage associated with a given stub. + * @param stub The stub for which to find / create the right output stream. + * @param Type of the stream to create. + * @return Storage object with a facade of type T. + */ + public static Storage createStorage( Stub stub ) { + return createStorage( stub, null ); + } + + /** + * Gets the output storage associated with a given stub. + * @param stub The stub for which to find / create the right output stream. + * @param file The filename to which to write the file. + * @param Type of the stream to create. + * @return Storage object with a facade of type T. + */ + public static Storage createStorage( Stub stub, File file ) { + Storage storage = null; + + if(stub instanceof OutputStreamStub) { + if( file != null ) + storage = new OutputStreamStorage((OutputStreamStub)stub,file); + else + storage = new OutputStreamStorage((OutputStreamStub)stub); + } + else if(stub instanceof SAMFileWriterStub) { + if( file != null ) + storage = new SAMFileWriterStorage((SAMFileWriterStub)stub,file); + else + storage = new SAMFileWriterStorage((SAMFileWriterStub)stub); + } + else + throw new StingException("Unsupported stub type: " + stub.getClass().getName()); + + return storage; + } +} diff --git a/java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamArgumentTypeDescriptor.java b/java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamArgumentTypeDescriptor.java new file mode 100644 index 000000000..1e6a8fec5 --- /dev/null +++ b/java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamArgumentTypeDescriptor.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2009 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.io.stubs; + +import org.broadinstitute.sting.utils.cmdLine.*; +import org.broadinstitute.sting.utils.StingException; +import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; + +import java.io.OutputStream; +import java.io.File; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; + +/** + * Insert an OutputStreamStub instead of a full-fledged concrete OutputStream implementations. + */ +public class OutputStreamArgumentTypeDescriptor extends ArgumentTypeDescriptor { + /** + * The engine into which output stubs should be fed. + */ + private GenomeAnalysisEngine engine; + + /** + * Create a new OutputStream argument, notifying the given engine when that argument has been created. + * @param engine Engine to add SAMFileWriter output to. + */ + public OutputStreamArgumentTypeDescriptor( GenomeAnalysisEngine engine ) { + this.engine = engine; + } + + @Override + public boolean supports( Class type ) { + return getConstructorForClass(type) != null; + } + + @Override + public Object parse( ArgumentSource source, Class type, ArgumentMatches matches ) { + ArgumentDefinition definition = createDefaultArgumentDefinition(source); + String fileName = getArgumentValue( definition, matches ); + + OutputStreamStub stub = new OutputStreamStub(new File(fileName)); + + engine.addOutput(stub); + + try { + return getConstructorForClass(type).newInstance(stub); + } + catch( InstantiationException ex ) { + throw new StingException("Could not instantiate class with OutputStream constructor: " + type.getName()); + } + catch( IllegalAccessException ex ) { + throw new StingException("Could not access class with OutputStream constructor: " + type.getName()); + } + catch( InvocationTargetException ex ) { + throw new StingException("Could not invoke constructor for class with OutputStream constructor: " + type.getName()); + } + } + + /** + * Retrieves the constructor for an object that takes exactly one argument: an output stream. + * @param type Type for which to go constructor spelunking. + * @return Constructor, if available. Null, if not. + */ + private Constructor getConstructorForClass( Class type ) { + try { + return type.getConstructor( OutputStream.class ); + } + catch( NoSuchMethodException ex ) { + return null; + } + } +} diff --git a/java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamStub.java b/java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamStub.java new file mode 100644 index 000000000..5722d6ca2 --- /dev/null +++ b/java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamStub.java @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2009 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.io.stubs; + +import org.broadinstitute.sting.gatk.io.OutputTracker; + +import java.io.OutputStream; +import java.io.IOException; +import java.io.File; + +/** + * A stub for routing and management of anything backed by an OutputStream. + * + * @author mhanna + * @version 0.1 + */ +public class OutputStreamStub extends OutputStream implements Stub { + /** + * The file that this stub should write to. Should be passed along to + * whatever happens to create storage for this stub. Might be null, if + * this stub connects directly to an existing stream. + */ + private final File targetFile; + + /** + * The stream that this stub should write to. Should be passed along to + * whatever happens to create storage for this stub. Might be null, if + * this stub connects directly to an existing stream. + */ + private final OutputStream targetStream; + + /** + * Connects this stub with an external stream capable of serving the + * requests of the consumer of this stub. + */ + private OutputTracker outputTracker = null; + + /** + * Specify that this target output stream should write to the given file. + * @param targetFile Target file to which to write. Should not be null. + */ + public OutputStreamStub( File targetFile ) { + this.targetFile = targetFile; + this.targetStream = null; + } + + /** + * Specify that this target output stream should write to the given stream. + * @param targetStream Target stream to which to write. Should not be null. + */ + public OutputStreamStub( OutputStream targetStream ) { + this.targetFile = null; + this.targetStream = targetStream; + } + + + /** + * Return the target file to which this data should be written. + * @return Target file. No sanity checking will have been performed by the file object. + */ + public File getOutputFile() { + return targetFile; + } + + /** + * Return the target stream to which this data should be written. + * @return Target stream. No sanity checking will have been performed by the file object. + */ + public OutputStream getOutputStream() { + return targetStream; + } + + /** + * Registers the given streamConnector with this stub. + * @param outputTracker The connector used to provide an appropriate stream. + */ + public void register( OutputTracker outputTracker ) { + this.outputTracker = outputTracker; + } + + /** + * @{inheritDoc} + */ + public void flush() throws IOException { + outputTracker.getStorage(this).close(); + } + + /** + * @{inheritDoc} + */ + public void close() throws IOException { + outputTracker.getStorage(this).close(); + } + + /** + * @{inheritDoc} + */ + public void write( byte[] b ) throws IOException { + outputTracker.getStorage(this).write(b); + } + + /** + * @{inheritDoc} + */ + public void write( byte[] b, int off, int len ) throws IOException { + outputTracker.getStorage(this).write(b, off, len); + } + + /** + * @{inheritDoc} + */ + public void write( int b ) throws IOException { + outputTracker.getStorage(this).write(b); + } +} diff --git a/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileReaderArgumentTypeDescriptor.java b/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileReaderArgumentTypeDescriptor.java new file mode 100644 index 000000000..41bf96c4d --- /dev/null +++ b/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileReaderArgumentTypeDescriptor.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2009 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.io.stubs; + +import org.broadinstitute.sting.utils.cmdLine.ArgumentTypeDescriptor; +import org.broadinstitute.sting.utils.cmdLine.ArgumentSource; +import org.broadinstitute.sting.utils.cmdLine.ArgumentMatches; +import org.broadinstitute.sting.utils.sam.SAMFileReaderBuilder; +import org.broadinstitute.sting.utils.StingException; +import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; +import net.sf.samtools.SAMFileReader; + +import java.io.File; + +/** + * Describe how to parse SAMFileReaders. + */ +public class SAMFileReaderArgumentTypeDescriptor extends ArgumentTypeDescriptor { + /** + * The engine into which output stubs should be fed. + */ + private GenomeAnalysisEngine engine; + + /** + * Create a new SAMFileReader argument, notifying the given engine when that argument has been created. + * @param engine + */ + public SAMFileReaderArgumentTypeDescriptor( GenomeAnalysisEngine engine ) { + this.engine = engine; + } + + + @Override + public boolean supports( Class type ) { + return SAMFileReader.class.isAssignableFrom(type); + } + + @Override + public Object parse( ArgumentSource source, Class type, ArgumentMatches matches ) { + SAMFileReaderBuilder builder = new SAMFileReaderBuilder(); + + String readerFileName = getArgumentValue( createDefaultArgumentDefinition(source), matches ); + + if( readerFileName == null ) + throw new StingException("SAM file compression was supplied, but no associated writer was supplied with it."); + + builder.setSAMFile(new File(readerFileName)); + + engine.addInput(source, builder); + + // MASSIVE KLUDGE! SAMFileReader is tricky to implement and we don't yet have a stub. Return null, then + // let the output tracker load it in. + // TODO: Add a stub for SAMFileReader. + return null; + } +} diff --git a/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java b/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java new file mode 100644 index 000000000..f53935d6d --- /dev/null +++ b/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2009 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.io.stubs; + +import org.broadinstitute.sting.utils.cmdLine.*; +import org.broadinstitute.sting.utils.StingException; +import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; +import net.sf.samtools.SAMFileWriter; + +import java.util.List; +import java.util.Arrays; +import java.io.File; + +/** + * Insert a SAMFileWriterStub instead of a full-fledged concrete OutputStream implementations. + */ +public class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor { + private static final String COMPRESSION_FULLNAME = "bam_compression"; + private static final String COMPRESSION_SHORTNAME = "compress"; + + /** + * The engine into which output stubs should be fed. + */ + private GenomeAnalysisEngine engine; + + /** + * Create a new SAMFileWriter argument, notifying the given engine when that argument has been created. + * @param engine Engine to add SAMFileWriter output to. + */ + public SAMFileWriterArgumentTypeDescriptor( GenomeAnalysisEngine engine ) { + this.engine = engine; + } + + + @Override + public boolean supports( Class type ) { + return SAMFileWriter.class.isAssignableFrom(type); + } + + @Override + public List createArgumentDefinitions( ArgumentSource source ) { + return Arrays.asList( createBAMArgumentDefinition(source), + createBAMCompressionArgumentDefinition(source) ); + } + + @Override + public Object parse( ArgumentSource source, Class type, ArgumentMatches matches ) { + String writerFileName = getArgumentValue( createBAMArgumentDefinition(source), matches ); + if( writerFileName == null ) + throw new StingException("SAM file compression was supplied, but not associated writer was supplied with it."); + + SAMFileWriterStub stub = new SAMFileWriterStub(engine, new File(writerFileName)); + + String compressionLevelText = getArgumentValue( createBAMCompressionArgumentDefinition(source), matches ); + Integer compressionLevel = compressionLevelText != null ? Integer.valueOf(compressionLevelText) : null; + if( compressionLevel != null ) + stub.setCompressionLevel(compressionLevel); + + engine.addOutput(stub); + + return stub; + } + + /** + * Gets the definition of the argument representing the BAM file itself. + * @param source Argument source for the BAM file. Must not be null. + * @return Argument definition for the BAM file itself. Will not be null. + */ + private ArgumentDefinition createBAMArgumentDefinition(ArgumentSource source) { + Argument description = this.getArgumentDescription(source); + + String fullName = description.fullName().trim().length() > 0 ? description.fullName().trim() : "outputBAM"; + String shortName = description.shortName().trim().length() > 0 ? description.shortName().trim() : "ob"; + + return new ArgumentDefinition( source, + fullName, + shortName, + getDoc(source), + isRequired(source), + getExclusiveOf(source), + getValidationRegex(source) ); + } + + /** + * Creates the optional compression level argument for the BAM file. + * @param source Argument source for the BAM file. Must not be null. + * @return Argument definition for the BAM file itself. Will not be null. + */ + private ArgumentDefinition createBAMCompressionArgumentDefinition(ArgumentSource source) { + return new ArgumentDefinition( source, + COMPRESSION_FULLNAME, + COMPRESSION_SHORTNAME, + "Compression level to use for writing BAM files", + false, + null, + null ); + } +} diff --git a/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterStub.java b/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterStub.java new file mode 100644 index 000000000..3d03ac28c --- /dev/null +++ b/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterStub.java @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2009 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.io.stubs; + +import net.sf.samtools.SAMFileWriter; +import net.sf.samtools.SAMRecord; +import net.sf.samtools.SAMFileHeader; + +import java.io.File; + +import org.broadinstitute.sting.gatk.io.OutputTracker; +import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; + +/** + * A stub for routing and management of SAM file reading and writing. + * + * @author mhanna + * @version 0.1 + */ +public class SAMFileWriterStub implements Stub, SAMFileWriter { + /** + * Engine to use for collecting attributes for the output SAM file. + */ + private final GenomeAnalysisEngine engine; + + /** + * The sam file that this stub should write to. Should be passed along to + * whatever happens to create the StreamConnector. + */ + private final File samFile; + + /** + * The validation stringency to apply when reading this file. + */ + private Integer compressionLevel = null; + + /** + * Connects this stub with an external stream capable of serving the + * requests of the consumer of this stub. + */ + private OutputTracker outputTracker = null; + + /** + * Create a new stub given the requested SAM file and compression level. + * @param engine source of header data, maybe other data about input files. + * @param samFile SAM file to (ultimately) cerate. + */ + public SAMFileWriterStub( GenomeAnalysisEngine engine, File samFile ) { + this.engine = engine; + this.samFile = samFile; + } + + /** + * Retrieves the SAM file to (ultimately) be created. + * @return The SAM file. Must not be null. + */ + public File getSAMFile() { + return samFile; + } + + /** + * Retrieves the header to use when creating the new SAM file. + * @return header to use when creating the new SAM file. + */ + public SAMFileHeader getSAMFileHeader() { + return engine.getSAMFileHeader(); + } + + /** + * Retrieves the desired compression level for + * @return The current compression level. Could be null if the user doesn't care. + */ + public Integer getCompressionLevel() { + return compressionLevel; + } + + /** + * Sets the desired compression level. + * @param compressionLevel The suggested compression level. + */ + public void setCompressionLevel( Integer compressionLevel ) { + this.compressionLevel = compressionLevel; + } + + /** + * Registers the given streamConnector with this stub. + * @param outputTracker The connector used to provide an appropriate stream. + */ + public void register( OutputTracker outputTracker ) { + this.outputTracker = outputTracker; + } + + /** + * @{inheritDoc} + */ + public void addAlignment( SAMRecord alignment ) { + outputTracker.getStorage(this).addAlignment(alignment); + } + + /** + * @{inheritDoc} + */ + public void close() { + outputTracker.getStorage(this).close(); + } + +} diff --git a/java/src/org/broadinstitute/sting/gatk/io/stubs/Stub.java b/java/src/org/broadinstitute/sting/gatk/io/stubs/Stub.java new file mode 100644 index 000000000..b042144b6 --- /dev/null +++ b/java/src/org/broadinstitute/sting/gatk/io/stubs/Stub.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2009 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.io.stubs; + +import org.broadinstitute.sting.gatk.io.OutputTracker; + +/** + * A stub used for managing IO. Acts as a proxy for IO streams + * not yet created or streams that need significant external + * management. + * + * @author mhanna + * @version 0.1 + */ +public interface Stub { + /** + * Provides a facility to register this stream with the given + * StreamConnector. The stub should route each output method + * to the specified connector. + * @param outputTracker The connector used to provide an appropriate stream. + */ + public void register( OutputTracker outputTracker ); +} diff --git a/java/src/org/broadinstitute/sting/utils/Utils.java b/java/src/org/broadinstitute/sting/utils/Utils.java index 6ce45efc0..bd5dbe078 100755 --- a/java/src/org/broadinstitute/sting/utils/Utils.java +++ b/java/src/org/broadinstitute/sting/utils/Utils.java @@ -43,6 +43,18 @@ public class Utils { throw new StingException(msg); } + /** + * Compares two objects, either of which might be null. + * @param lhs One object to compare. + * @param rhs The other object to compare. + * @return True if the two objects are equal, false otherwise. + */ + public static boolean equals(Object lhs, Object rhs) { + if( lhs == null && rhs == null ) return true; + else if( lhs == null ) return false; + else return lhs.equals(rhs); + } + public static List cons(final T elt, final List l) { List l2 = new ArrayList(); l2.add(elt); diff --git a/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentDefinition.java b/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentDefinition.java new file mode 100644 index 000000000..2a742c2a9 --- /dev/null +++ b/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentDefinition.java @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2009 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.cmdLine; + +import org.broadinstitute.sting.utils.Utils; + +/** + * A specific argument definition. Maps one-to-one with a field in some class. + */ +public class ArgumentDefinition { + /** + * Full name of the argument. Must have a value. + */ + public final String fullName; + + /** + * Short name of the argument. Can be null. + */ + public final String shortName; + + /** + * Doc string for the argument. Displayed in help. + */ + public final String doc; + + /** + * Is this argument required? + */ + public final boolean required; + + /** + * Is this argument exclusive of other arguments? + */ + public final String exclusiveOf; + + /** + * Can we validate this regular expression? + */ + public final String validation; + + /** + * The target into which to inject arguments meeting this definition. + */ + public final ArgumentSource source; + + /** + * Creates a new argument definition. + * @param source Source information for defining the argument. + */ + public ArgumentDefinition( ArgumentSource source, + String fullName, + String shortName, + String doc, + boolean required, + String exclusiveOf, + String validation ) { + this.source = source; + + this.fullName = fullName; + this.shortName = shortName; + this.doc = doc; + this.required = required; + this.exclusiveOf = exclusiveOf; + this.validation = validation; + } + + @Override + public int hashCode() { + int hashCode = fullName.hashCode(); + if(shortName != null) hashCode ^= shortName.hashCode(); + if(doc != null) hashCode ^= doc.hashCode(); + hashCode ^= Boolean.valueOf(required).hashCode(); + if(exclusiveOf != null) hashCode ^= exclusiveOf.hashCode(); + if(validation != null) hashCode ^= validation.hashCode(); + return hashCode; + } + + public boolean equals( Object o ) { + if( o == null ) + return false; + if( !(o instanceof ArgumentDefinition) ) + return false; + + ArgumentDefinition other = (ArgumentDefinition)o; + + return Utils.equals(fullName,other.fullName) && + Utils.equals(shortName,other.shortName) && + Utils.equals(doc,other.doc) && + required == other.required && + Utils.equals(exclusiveOf,other.exclusiveOf) && + Utils.equals(validation,other.validation); + } +} diff --git a/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentDefinitions.java b/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentDefinitions.java index cd7a27032..74cb6b6e3 100755 --- a/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentDefinitions.java +++ b/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentDefinitions.java @@ -227,67 +227,6 @@ class ArgumentDefinitionGroup implements Iterable { } } -/** - * A specific argument definition. Maps one-to-one with a field in some class. - */ -class ArgumentDefinition { - /** - * Full name of the argument. Must have a value. - */ - public final String fullName; - - /** - * Short name of the argument. Can be null. - */ - public final String shortName; - - /** - * Doc string for the argument. Displayed in help. - */ - public final String doc; - - /** - * Is this argument required? - */ - public final boolean required; - - /** - * Is this argument exclusive of other arguments? - */ - public final String exclusiveOf; - - /** - * Can we validate this regular expression? - */ - public final String validation; - - /** - * The target into which to inject arguments meeting this definition. - */ - public final ArgumentSource source; - - /** - * Creates a new argument definition. - * @param source Source information for defining the argument. - */ - public ArgumentDefinition( ArgumentSource source, - String fullName, - String shortName, - String doc, - boolean required, - String exclusiveOf, - String validation ) { - this.source = source; - - this.fullName = fullName; - this.shortName = shortName; - this.doc = doc; - this.required = required; - this.exclusiveOf = exclusiveOf; - this.validation = validation; - } -} - /** * A Comparator-esque interface for finding argument definitions within a collection. */ diff --git a/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentMatches.java b/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentMatches.java index 53a4fed4e..985350744 100755 --- a/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentMatches.java +++ b/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentMatches.java @@ -31,7 +31,7 @@ public class ArgumentMatches implements Iterable { * Provide a place to put command-line argument values that don't seem to belong to * any particular command-line option. */ - public ArgumentMatch MissingArgument = new ArgumentMatch(); + ArgumentMatch MissingArgument = new ArgumentMatch(); /** * Get an iterator cycling through *unique* command-line argument <-> definition matches. @@ -41,12 +41,34 @@ public class ArgumentMatches implements Iterable { return getUniqueMatches().iterator(); } + /** + * Create an empty ArgumentMatches object. + */ + public ArgumentMatches() { + } + + /** + * Create a singleton ArgumentMatches object. + * @param match Match to incorporate. + */ + public ArgumentMatches( ArgumentMatch match ) { + mergeInto( match ); + } + + /** + * Returns the number of matches in this structure. + * @return Count of the matches in this structure. + */ + public int size() { + return argumentMatches.size(); + } + /** * Indicates whether the site contains a matched argument. * @param site Site at which to check. * @return True if the site has a match. False otherwise. */ - public boolean hasMatch( int site ) { + boolean hasMatch( int site ) { return argumentMatches.containsKey( site ); } @@ -56,7 +78,7 @@ public class ArgumentMatches implements Iterable { * @return The match present at the given site. * @throws IllegalArgumentException if site does not contain a match. */ - public ArgumentMatch getMatch( int site ) { + ArgumentMatch getMatch( int site ) { if( !argumentMatches.containsKey(site) ) throw new IllegalArgumentException( "Site does not contain an argument: " + site ); return argumentMatches.get(site); @@ -67,7 +89,7 @@ public class ArgumentMatches implements Iterable { * @param definition Definition to match. * @return True if a match exists; false otherwise. */ - public boolean hasMatch( ArgumentDefinition definition ) { + boolean hasMatch( ArgumentDefinition definition ) { return findMatches( definition ).size() > 0; } @@ -77,11 +99,11 @@ public class ArgumentMatches implements Iterable { * @return List of all matches. */ - public Collection findMatches( ArgumentSource argumentSource ) { - Collection matches = new HashSet(); + ArgumentMatches findMatches( ArgumentSource argumentSource ) { + ArgumentMatches matches = new ArgumentMatches(); for( ArgumentMatch argumentMatch: getUniqueMatches() ) { if( argumentMatch.definition != null && argumentMatch.definition.source.equals( argumentSource ) ) - matches.add( argumentMatch ); + matches.mergeInto( argumentMatch ); } return matches; } @@ -91,23 +113,23 @@ public class ArgumentMatches implements Iterable { * @param definition Argument definition to match. * @return List of all matches. */ - public Collection findMatches( ArgumentDefinition definition ) { - Collection matches = new HashSet(); - for( ArgumentMatch argumentMatch: getUniqueMatches() ) { + ArgumentMatches findMatches( ArgumentDefinition definition ) { + ArgumentMatches matches = new ArgumentMatches(); + for( ArgumentMatch argumentMatch: argumentMatches.values() ) { if( argumentMatch.definition == definition ) - matches.add( argumentMatch ); - } + matches.mergeInto( argumentMatch ); + } return matches; } /** * Find all successful matches (a 'successful' match is one paired with a definition). */ - public Collection findSuccessfulMatches() { - Collection matches = new HashSet(); - for( ArgumentMatch argumentMatch: getUniqueMatches() ) { + ArgumentMatches findSuccessfulMatches() { + ArgumentMatches matches = new ArgumentMatches(); + for( ArgumentMatch argumentMatch: argumentMatches.values() ) { if( argumentMatch.definition != null ) - matches.add( argumentMatch ); + matches.mergeInto( argumentMatch ); } return matches; } @@ -116,11 +138,11 @@ public class ArgumentMatches implements Iterable { * Find arguments that are unmatched to any definition. * @return Set of matches that have no associated definition. */ - public Collection findUnmatched() { - Collection matches = new HashSet(); - for( ArgumentMatch argumentMatch: getUniqueMatches() ) { + ArgumentMatches findUnmatched() { + ArgumentMatches matches = new ArgumentMatches(); + for( ArgumentMatch argumentMatch: argumentMatches.values() ) { if( argumentMatch.definition == null ) - matches.add( argumentMatch ); + matches.mergeInto( argumentMatch ); } return matches; } diff --git a/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentSource.java b/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentSource.java index f3d8f9837..fc3a0f6bf 100644 --- a/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentSource.java +++ b/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentSource.java @@ -30,7 +30,6 @@ import org.broadinstitute.sting.utils.StingException; import java.lang.reflect.Field; import java.util.Collection; import java.util.List; -import java.util.Collections; /** * Describes the source field which defines a command-line argument. @@ -51,11 +50,6 @@ public class ArgumentSource { */ public final Field field; - /** - * Descriptor for the argument. Contains name, validation info, etc. - */ - public final Argument descriptor; - /** * Create a new command-line argument target. * @param clazz Class containing the argument. @@ -64,9 +58,6 @@ public class ArgumentSource { public ArgumentSource( Class clazz, Field field ) { this.clazz = clazz; this.field = field; - this.descriptor = field.getAnnotation(Argument.class); - if( descriptor == null ) - throw new StingException("Cannot build out a command-line argument without a descriptor."); } /** @@ -101,7 +92,7 @@ public class ArgumentSource { */ public List createArgumentDefinitions() { ArgumentTypeDescriptor typeDescriptor = ArgumentTypeDescriptor.create( field.getType() ); - return typeDescriptor.createArgumentDefinitions( this, descriptor ); + return typeDescriptor.createArgumentDefinitions( this ); } /** @@ -109,7 +100,7 @@ public class ArgumentSource { * @param targetInstance Instance into which to inject the parsed value. * @param values String representation of all values passed. */ - public Object parse( ArgumentSource source, Object targetInstance, ArgumentMatch... values ) { + public Object parse( ArgumentSource source, Object targetInstance, ArgumentMatches values ) { Object value = null; if( !isFlag() ) { ArgumentTypeDescriptor typeDescriptor = ArgumentTypeDescriptor.create( field.getType() ); diff --git a/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentTypeDescriptor.java b/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentTypeDescriptor.java index 7b29a6ba1..5fa005759 100644 --- a/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentTypeDescriptor.java +++ b/java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentTypeDescriptor.java @@ -26,19 +26,13 @@ package org.broadinstitute.sting.utils.cmdLine; import org.broadinstitute.sting.utils.StingException; -import org.broadinstitute.sting.utils.sam.SAMFileWriterBuilder; -import org.broadinstitute.sting.utils.sam.SAMFileReaderBuilder; import org.apache.log4j.Logger; import java.lang.reflect.*; import java.util.*; -import java.io.File; - -import net.sf.samtools.SAMFileWriter; -import net.sf.samtools.SAMFileReader; /** - * An factory capable of providing parsers that can parse any type + * An descriptor capable of providing parsers that can parse any type * of supported command-line argument. * * @author mhanna @@ -53,11 +47,23 @@ public abstract class ArgumentTypeDescriptor { /** * Class reference to the different types of descriptors that the create method can create. + * The type of set used must be ordered (but not necessarily sorted). */ - private static List descriptors = Arrays.asList( new SAMFileReaderArgumentTypeDescriptor(), - new SAMFileWriterArgumentTypeDescriptor(), - new SimpleArgumentTypeDescriptor(), - new CompoundArgumentTypeDescriptor() ); + private static Set descriptors = new LinkedHashSet( Arrays.asList(new SimpleArgumentTypeDescriptor(), + new CompoundArgumentTypeDescriptor()) ); + + /** + * Adds new, user defined descriptors to the head of the descriptor list. + * @param argumentTypeDescriptors New descriptors to add. List can be empty, but should not be null. + */ + public static void addDescriptors( Collection argumentTypeDescriptors ) { + // We care about ordering; newly added descriptors should have priority over stock descriptors. + // Enforce this by creating a new *ordered* set, adding the new descriptors, then adding the old descriptors. + Set allDescriptors = new LinkedHashSet(); + allDescriptors.addAll( argumentTypeDescriptors ); + allDescriptors.addAll( descriptors ); + descriptors = allDescriptors; + } public static ArgumentTypeDescriptor create( Class type ) { for( ArgumentTypeDescriptor descriptor: descriptors ) { @@ -77,32 +83,41 @@ public abstract class ArgumentTypeDescriptor { /** * Given the given argument source and attributes, synthesize argument definitions for command-line arguments. * @param source Source class and field for the given argument. - * @param description Description of the fields that go into a given argument. * @return A list of command-line argument definitions supporting this field. */ - public List createArgumentDefinitions( ArgumentSource source, Argument description ) { - ArgumentDefinition definition = new ArgumentDefinition( source, - getFullName( source, description ), - getShortName( source, description ), - getDoc( source, description ), - isRequired( source, description ), - getExclusiveOf( source, description ), - getValidationRegex( source, description ) ); - return Collections.singletonList(definition); - } - - public Object parse( ArgumentSource source, ArgumentMatch... values ) { - return parse( source, source.field.getType(), values ); + public List createArgumentDefinitions( ArgumentSource source ) { + return Collections.singletonList(createDefaultArgumentDefinition(source)); } - protected abstract Object parse( ArgumentSource source, Class type, ArgumentMatch... values ); + public Object parse( ArgumentSource source, ArgumentMatches matches ) { + return parse( source, source.field.getType(), matches ); + } + + /** + * By default, argument sources create argument definitions with a set of default values. + * Use this method to create the one simple argument definition. + * @param source argument source for which to create a default definition. + * @return The default definition for this argument source. + */ + protected ArgumentDefinition createDefaultArgumentDefinition( ArgumentSource source ) { + return new ArgumentDefinition( source, + getFullName(source), + getShortName(source), + getDoc(source), + isRequired(source), + getExclusiveOf(source), + getValidationRegex(source) ); + } + + protected abstract Object parse( ArgumentSource source, Class type, ArgumentMatches matches ); /** * Retrieves the full name of the argument, specifiable with the '--' prefix. The full name can be * either specified explicitly with the fullName annotation parameter or implied by the field name. * @return full name of the argument. Never null. */ - protected String getFullName( ArgumentSource source, Argument description ) { + protected String getFullName( ArgumentSource source ) { + Argument description = getArgumentDescription(source); return description.fullName().trim().length() > 0 ? description.fullName().trim() : source.field.getName().toLowerCase(); } @@ -111,7 +126,8 @@ public abstract class ArgumentTypeDescriptor { * be specified or not; if left unspecified, no short name will be present. * @return short name of the argument. Null if no short name exists. */ - protected String getShortName( ArgumentSource source, Argument description ) { + protected String getShortName( ArgumentSource source ) { + Argument description = getArgumentDescription(source); return description.shortName().trim().length() > 0 ? description.shortName().trim() : null; } @@ -119,7 +135,8 @@ public abstract class ArgumentTypeDescriptor { * Documentation for this argument. Mandatory field. * @return Documentation for this argument. */ - protected String getDoc( ArgumentSource source, Argument description ) { + protected String getDoc( ArgumentSource source ) { + Argument description = getArgumentDescription(source); return description.doc(); } @@ -127,7 +144,8 @@ public abstract class ArgumentTypeDescriptor { * Returns whether this field is required. Note that flag fields are always forced to 'not required'. * @return True if the field is mandatory and not a boolean flag. False otherwise. */ - protected boolean isRequired( ArgumentSource source, Argument description ) { + protected boolean isRequired( ArgumentSource source ) { + Argument description = getArgumentDescription(source); return description.required() && !source.isFlag(); } @@ -135,7 +153,8 @@ public abstract class ArgumentTypeDescriptor { * Specifies other arguments which cannot be used in conjunction with tihs argument. Comma-separated list. * @return A comma-separated list of exclusive arguments, or null if none are present. */ - protected String getExclusiveOf( ArgumentSource source, Argument description ) { + protected String getExclusiveOf( ArgumentSource source ) { + Argument description = getArgumentDescription(source); return description.exclusiveOf().trim().length() > 0 ? description.exclusiveOf().trim() : null; } @@ -143,9 +162,51 @@ public abstract class ArgumentTypeDescriptor { * A regular expression which can be used for validation. * @return a JVM regex-compatible regular expression, or null to permit any possible value. */ - protected String getValidationRegex( ArgumentSource source, Argument description ) { + protected String getValidationRegex( ArgumentSource source ) { + Argument description = getArgumentDescription(source); return description.validation().trim().length() > 0 ? description.validation().trim() : null; } + + /** + * Gets the value of an argument with the given full name, from the collection of ArgumentMatches. + * If the argument matches multiple values, an exception will be thrown. + * @param definition Definition of the argument for which to find matches. + * @param matches The matches for the given argument. + * @return The value of the argument if available, or null if not present. + */ + protected String getArgumentValue( ArgumentDefinition definition, ArgumentMatches matches ) { + Collection argumentValues = getArgumentValues( definition, matches ); + if( argumentValues.size() > 1 ) + throw new StingException("Multiple values associated with given definition, but this argument expects only one: " + definition.fullName); + return argumentValues.size() > 0 ? argumentValues.iterator().next() : null; + } + + /** + * Gets the values of an argument with the given full name, from the collection of ArgumentMatches. + * @param definition Definition of the argument for which to find matches. + * @param matches The matches for the given argument. + * @return The value of the argument if available, or an empty collection if not present. + */ + protected Collection getArgumentValues( ArgumentDefinition definition, ArgumentMatches matches ) { + Collection values = new ArrayList(); + for( ArgumentMatch match: matches ) { + if( match.definition.equals(definition) ) + values.addAll(match.values()); + } + return values; + } + + /** + * Retrieves the argument description from the given argument source. Will throw an exception if + * the given ArgumentSource + * @param source source of the argument. + * @return Argument description annotation associated with the given field. + */ + protected Argument getArgumentDescription( ArgumentSource source ) { + if( !source.field.isAnnotationPresent(Argument.class) ) + throw new StingException("ArgumentAnnotation is not present for the argument field: " + source.field.getName()); + return source.field.getAnnotation(Argument.class); + } } /** @@ -171,10 +232,8 @@ class SimpleArgumentTypeDescriptor extends ArgumentTypeDescriptor { } @Override - protected Object parse( ArgumentSource source, Class type, ArgumentMatch... matches ) { - if( matches.length > 1 || matches[0].values().size() > 1 ) - throw new StingException("Simple argument parser is unable to parse multiple arguments."); - String value = matches[0].values().get(0); + protected Object parse( ArgumentSource source, Class type, ArgumentMatches matches ) { + String value = getArgumentValue( createDefaultArgumentDefinition(source), matches ); // lets go through the types we support try { @@ -228,14 +287,10 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor { } @Override - public Object parse( ArgumentSource source, Class type, ArgumentMatch... matches ) + public Object parse( ArgumentSource source, Class type, ArgumentMatches matches ) { Class componentType = null; - if( matches.length > 1 ) - throw new StingException("Simple argument parser is unable to combine multiple argument types into a compound argument."); - ArgumentMatch match = matches[0]; - if( Collection.class.isAssignableFrom(type) ) { // If this is a generic interface, pick a concrete implementation to create and pass back. @@ -272,8 +327,10 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor { throw new StingException("constructFromString:IllegalAccessException: Failed conversion " + e.getMessage()); } - for( ArgumentMatch value: match ) - collection.add( componentArgumentParser.parse(source,componentType,value) ); + for( ArgumentMatch match: matches ) { + for( ArgumentMatch value: match ) + collection.add( componentArgumentParser.parse(source,componentType,new ArgumentMatches(value)) ); + } return collection; @@ -281,11 +338,19 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor { else if( type.isArray() ) { componentType = type.getComponentType(); ArgumentTypeDescriptor componentArgumentParser = ArgumentTypeDescriptor.create( componentType ); - Object arr = Array.newInstance(componentType,match.values().size()); + + // Assemble a collection of individual values used in this computation. + Collection values = new ArrayList(); + for( ArgumentMatch match: matches ) { + for( ArgumentMatch value: match ) + values.add(value); + } + + Object arr = Array.newInstance(componentType,values.size()); int i = 0; - for( ArgumentMatch value: match ) - Array.set( arr,i++,componentArgumentParser.parse(source,componentType,value)); + for( ArgumentMatch value: values ) + Array.set( arr,i++,componentArgumentParser.parse(source,componentType,new ArgumentMatches(value))); return arr; } @@ -293,103 +358,3 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor { throw new StingException("Unsupported compound argument type: " + type); } } - -/** - * Handle SAMFileReaders. - */ -class SAMFileReaderArgumentTypeDescriptor extends ArgumentTypeDescriptor { - @Override - public boolean supports( Class type ) { - return SAMFileReader.class.isAssignableFrom(type); - } - - @Override - public Object parse( ArgumentSource source, Class type, ArgumentMatch... matches ) { - if( matches.length > 1 ) - throw new UnsupportedOperationException("Only an input file name and validation stringency can be supplied when creating a BAM file reader."); - - SAMFileReaderBuilder builder = new SAMFileReaderBuilder(); - - ArgumentMatch readerMatch = matches[0]; - - if( readerMatch == null ) - throw new StingException("SAM file compression was supplied, but not associated writer was supplied with it."); - if( readerMatch.values().size() > 1 ) - throw new StingException("Only one filename can be supplied per created BAM file"); - - builder.setSAMFile(new File(readerMatch.values().get(0).trim())); - - return builder; - } -} - -/** - * Handle SAMFileWriters. - */ -class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor { - private static final String COMPRESSION_FULLNAME = "bam_compression"; - private static final String COMPRESSION_SHORTNAME = "compress"; - - @Override - public boolean supports( Class type ) { - return SAMFileWriter.class.isAssignableFrom(type); - } - - @Override - public List createArgumentDefinitions( ArgumentSource source, Argument description ) { - String fullName = description.fullName().trim().length() > 0 ? description.fullName().trim() : "outputBAM"; - String shortName = description.shortName().trim().length() > 0 ? description.shortName().trim() : "ob"; - - ArgumentDefinition writerDefinition = new ArgumentDefinition( source, - fullName, - shortName, - getDoc( source, description ), - isRequired( source, description ), - getExclusiveOf( source, description ), - getValidationRegex( source, description ) ); - ArgumentDefinition compressionDefinition = new ArgumentDefinition( source, - COMPRESSION_FULLNAME, - COMPRESSION_SHORTNAME, - "Compression level to use for writing BAM files", - false, - null, - null ); - - return Arrays.asList( writerDefinition, compressionDefinition ); - } - - @Override - public Object parse( ArgumentSource source, Class type, ArgumentMatch... matches ) { - if( matches.length > 2 ) - throw new UnsupportedOperationException("Only an input file name and validation stringency can be supplied when creating a BAM file reader."); - - SAMFileWriterBuilder builder = new SAMFileWriterBuilder(); - - ArgumentMatch writerMatch = null; - ArgumentMatch compressionMatch = null; - - for( ArgumentMatch match: matches ) { - if( match.definition.fullName.equals(COMPRESSION_FULLNAME) ) - compressionMatch = match; - else - writerMatch = match; - } - - if( writerMatch == null ) - throw new StingException("SAM file compression was supplied, but not associated writer was supplied with it."); - if( writerMatch.values().size() > 1 ) - throw new StingException("Only one filename can be supplied per created BAM file"); - - builder.setSAMFile(new File(writerMatch.values().get(0).trim())); - - if( compressionMatch != null ) { - if( compressionMatch.values().size() > 1 ) - throw new StingException("Only one value can be supplied for BAM compression"); - int compressionLevel = Integer.valueOf(compressionMatch.values().get(0)); - builder.setCompressionLevel(compressionLevel); - } - - return builder; - } - -} diff --git a/java/src/org/broadinstitute/sting/utils/cmdLine/CommandLineProgram.java b/java/src/org/broadinstitute/sting/utils/cmdLine/CommandLineProgram.java index e309adc9b..7923ebbaf 100644 --- a/java/src/org/broadinstitute/sting/utils/cmdLine/CommandLineProgram.java +++ b/java/src/org/broadinstitute/sting/utils/cmdLine/CommandLineProgram.java @@ -7,6 +7,8 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.EnumSet; import java.util.Enumeration; +import java.util.Collections; +import java.util.Collection; /** * User: aaron @@ -95,6 +97,14 @@ public abstract class CommandLineProgram { ApplicationDetails.createDefaultRunningInstructions(getClass()) ); } + /** + * Subclasses of CommandLinePrograms can provide their own types of command-line arguments. + * @return A collection of type descriptors generating implementation-dependent placeholders. + */ + protected Collection getArgumentTypeDescriptors() { + return Collections.emptyList(); + } + /** * Will this application want to vary its argument list dynamically? * If so, parse the command-line options and then prompt the subclass to return @@ -109,16 +119,6 @@ public abstract class CommandLineProgram { */ protected Class[] getArgumentSources() { return new Class[] {}; } - /** - * Allows arguments to be hijacked by subclasses of the program before being placed - * into plugin classes. - * @param source Source class for the argument. - * @param targetInstance Instance into which the value should be ultimately injected. - * @param value Value to inject. - * @return True if the particular field has been hijacked; false otherwise. - */ - protected boolean intercept( ArgumentSource source, Object targetInstance, Object value ) { return false; } - /** * Name this argument source. Provides the (full) class name as a default. * @param source The argument source. diff --git a/java/src/org/broadinstitute/sting/utils/cmdLine/ParsingEngine.java b/java/src/org/broadinstitute/sting/utils/cmdLine/ParsingEngine.java index b4fe1c164..772fb7b7d 100755 --- a/java/src/org/broadinstitute/sting/utils/cmdLine/ParsingEngine.java +++ b/java/src/org/broadinstitute/sting/utils/cmdLine/ParsingEngine.java @@ -61,8 +61,13 @@ public class ParsingEngine { public ParsingEngine( CommandLineProgram clp ) { this.clp = clp; + parsingMethods.add( ParsingMethod.FullNameParsingMethod ); parsingMethods.add( ParsingMethod.ShortNameParsingMethod ); + + // Null check for unit tests. Perhaps we should mock up an empty CLP? + if( clp != null ) + ArgumentTypeDescriptor.addDescriptors( clp.getArgumentTypeDescriptors() ); } /** @@ -172,7 +177,7 @@ public class ParsingEngine { // Find invalid arguments. Invalid arguments will have a null argument definition. if( !skipValidationOf.contains(ValidationType.InvalidArgument) ) { - Collection invalidArguments = argumentMatches.findUnmatched(); + ArgumentMatches invalidArguments = argumentMatches.findUnmatched(); if( invalidArguments.size() > 0 ) throw new InvalidArgumentException( invalidArguments ); } @@ -183,7 +188,7 @@ public class ParsingEngine { argumentDefinitions.findArgumentDefinitions( null, ArgumentDefinitions.VerifiableDefinitionMatcher ); Collection> invalidValues = new ArrayList>(); for( ArgumentDefinition verifiableArgument: verifiableArguments ) { - Collection verifiableMatches = argumentMatches.findMatches( verifiableArgument ); + ArgumentMatches verifiableMatches = argumentMatches.findMatches( verifiableArgument ); for( ArgumentMatch verifiableMatch: verifiableMatches ) { for( String value: verifiableMatch.values() ) { if( !value.matches(verifiableArgument.validation) ) @@ -253,15 +258,14 @@ public class ParsingEngine { * @param argumentMatches Argument matches to load into the object. * @param target */ - private void loadMatchesIntoObject( ArgumentSource source, Object target, Collection argumentMatches ) { + private void loadMatchesIntoObject( ArgumentSource source, Object target, ArgumentMatches argumentMatches ) { // Nothing to load if( argumentMatches.size() == 0 ) return; if( source.clazz.isAssignableFrom(target.getClass()) ) { - Object value = source.parse( source, target, argumentMatches.toArray(new ArgumentMatch[0]) ); - if( clp == null || !clp.intercept(source, target, value) ) - JVMUtils.setField( source.field, target, value ); + Object value = source.parse( source, target, argumentMatches ); + JVMUtils.setField( source.field, target, value ); } } @@ -346,15 +350,32 @@ class MissingArgumentException extends ArgumentException { } } +class MissingArgumentValueException extends ArgumentException { + public MissingArgumentValueException( Collection missingArguments ) { + super( formatArguments(missingArguments) ); + } + + private static String formatArguments( Collection missingArguments ) { + StringBuilder sb = new StringBuilder(); + for( ArgumentDefinition missingArgument: missingArguments ) { + if( missingArgument.shortName != null ) + sb.append( String.format("%nValue for argument with name '--%s' (-%s) is missing.", missingArgument.fullName, missingArgument.shortName) ); + else + sb.append( String.format("%nValue for argument with name '--%s' is missing.", missingArgument.fullName) ); + } + return sb.toString(); + } +} + /** * An exception for undefined arguments. */ class InvalidArgumentException extends ArgumentException { - public InvalidArgumentException( Collection invalidArguments ) { + public InvalidArgumentException( ArgumentMatches invalidArguments ) { super( formatArguments(invalidArguments) ); } - private static String formatArguments( Collection invalidArguments ) { + private static String formatArguments( ArgumentMatches invalidArguments ) { StringBuilder sb = new StringBuilder(); for( ArgumentMatch invalidArgument: invalidArguments ) sb.append( String.format("%nArgument with name '%s' isn't defined.", invalidArgument.label) ); diff --git a/java/src/org/broadinstitute/sting/utils/sam/SAMFileWriterBuilder.java b/java/src/org/broadinstitute/sting/utils/sam/SAMFileWriterBuilder.java deleted file mode 100644 index 97339d66e..000000000 --- a/java/src/org/broadinstitute/sting/utils/sam/SAMFileWriterBuilder.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2009 The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.utils.sam; - -import net.sf.samtools.SAMFileHeader; -import net.sf.samtools.SAMFileWriter; -import net.sf.samtools.SAMFileWriterFactory; - -import java.io.File; - -import org.broadinstitute.sting.utils.StingException; - -/** - * Allows the user to steadily accumulate information about what - * components go into a SAM file writer, ultimately using this - * information to create a SAM file writer on demand. - * - * @author mhanna - * @version 0.1 - */ -public class SAMFileWriterBuilder { - /** - * Default compression level for newly constructed SAM files. - * Default to 5 (based on research by Alec Wysoker) - */ - public static final int DEFAULT_COMPRESSION_LEVEL = 5; - - /** - * To which file should output be written? - */ - private File samFile = null; - - /** - * Which header should be used when writing the SAM file? - */ - private SAMFileHeader header = null; - - /** - * What compression level should be used when building this file? - */ - private int compressionLevel = DEFAULT_COMPRESSION_LEVEL; - - /** - * Sets the handle of the sam file to which data should be written. - * @param samFile The SAM file into which data should flow. - */ - public void setSAMFile( File samFile ) { - this.samFile = samFile; - } - - /** - * Sets the header to be written at the head of this SAM file. - * @param header Header to write. - */ - public void setSAMFileHeader( SAMFileHeader header ) { - this.header = header; - } - - /** - * Sets the compression level to use when writing this BAM file. - * @param compressionLevel Compression level to use when writing this SAM file. - */ - public void setCompressionLevel( int compressionLevel ) { - this.compressionLevel = compressionLevel; - } - - /** - * Create the SAM writer, given the constituent parts accrued. - * @return Newly minted SAM file writer. - */ - public SAMFileWriter build() { - if( samFile == null ) - throw new StingException( "Filename for output sam file must be supplied."); - if( header == null ) - throw new StingException( "Header for output sam file must be supplied."); - return new SAMFileWriterFactory().makeBAMWriter( header, true, samFile, compressionLevel ); - } -} diff --git a/java/test/org/broadinstitute/sting/gatk/OutputTrackerTest.java b/java/test/org/broadinstitute/sting/gatk/io/OutputTrackerTest.java similarity index 62% rename from java/test/org/broadinstitute/sting/gatk/OutputTrackerTest.java rename to java/test/org/broadinstitute/sting/gatk/io/OutputTrackerTest.java index 462a24362..87b6d0501 100755 --- a/java/test/org/broadinstitute/sting/gatk/OutputTrackerTest.java +++ b/java/test/org/broadinstitute/sting/gatk/io/OutputTrackerTest.java @@ -1,10 +1,13 @@ -package org.broadinstitute.sting.gatk; +package org.broadinstitute.sting.gatk.io; import org.junit.Test; import org.junit.After; import org.junit.Assert; import org.broadinstitute.sting.utils.io.RedirectingOutputStream; import org.broadinstitute.sting.BaseTest; +import org.broadinstitute.sting.gatk.io.OutputTracker; +import org.broadinstitute.sting.gatk.io.DirectOutputTracker; +import org.broadinstitute.sting.gatk.io.stubs.OutputStreamStub; import java.io.File; import java.io.FileNotFoundException; @@ -42,26 +45,28 @@ public class OutputTrackerTest extends BaseTest { @Test public void testNullInputs() { - OutputTracker ot = new OutputTracker(); + OutputTracker ot = new DirectOutputTracker(); ot.initializeCoreIO(null,null); - Assert.assertTrue("OutputTracker: Output stream is of wrong type.", ot.getOutStream() instanceof RedirectingOutputStream ); - Assert.assertTrue("OutputTracker: Error stream is of wrong type.", ot.getErrStream() instanceof RedirectingOutputStream ); + Assert.assertNotNull("OutputTracker: Output stream is null.", ot.outStub ); + Assert.assertNotNull("OutputTracker: Error stream is null.", ot.errStub ); - RedirectingOutputStream outStream = (RedirectingOutputStream)ot.getOutStream(); - RedirectingOutputStream errStream = (RedirectingOutputStream)ot.getErrStream(); + OutputStreamStub outStream = ot.outStub; + Assert.assertNull("OutputTracker: Output file incorrectly initialized.", outStream.getOutputFile()); + Assert.assertSame("OutputTracker: Output stream incorrectly initialized.", System.out, outStream.getOutputStream()); - Assert.assertSame("OutputTracker: Output stream incorrectly initialized.", System.out, outStream.getBackingOutputStream()); - Assert.assertSame("OutputTracker: Error stream incorrectly initialized.", System.err, errStream.getBackingOutputStream()); + OutputStreamStub errStream = ot.errStub; + Assert.assertNull("OutputTracker: Error file incorrectly initialized.", errStream.getOutputFile()); + Assert.assertSame("OutputTracker: Error stream incorrectly initialized.", System.err, errStream.getOutputStream()); } @Test public void testOutputStreamAlone() throws FileNotFoundException { - OutputTracker ot = new OutputTracker(); + OutputTracker ot = new DirectOutputTracker(); ot.initializeCoreIO(OUTPUT_FILENAME,null); final String OUTPUT_TEXT = "out stream test"; - PrintWriter outWriter = new PrintWriter(ot.getOutStream()); + PrintWriter outWriter = new PrintWriter(ot.outStub); outWriter.append(OUTPUT_TEXT); outWriter.close(); @@ -71,18 +76,22 @@ public class OutputTrackerTest extends BaseTest { Assert.assertEquals("OutputTracker: Written output is incorrect", outText, OUTPUT_TEXT); - Assert.assertTrue("OutputTracker: Error stream is of wrong type.", ot.getErrStream() instanceof RedirectingOutputStream ); - RedirectingOutputStream errStream = (RedirectingOutputStream)ot.getErrStream(); - Assert.assertSame("OutputTracker: Error stream incorrectly initialized.", System.err, errStream.getBackingOutputStream()); + OutputStreamStub errStream = ot.errStub; + Assert.assertNull("OutputTracker: Error file incorrectly initialized.", errStream.getOutputFile()); + Assert.assertSame("OutputTracker: Error stream incorrectly initialized.", System.err, errStream.getOutputStream()); } @Test public void testErrorStreamAlone() throws FileNotFoundException { - OutputTracker ot = new OutputTracker(); + OutputTracker ot = new DirectOutputTracker(); ot.initializeCoreIO(null,ERROR_FILENAME); + OutputStreamStub outStream = ot.outStub; + Assert.assertNull("OutputTracker: Output file incorrectly initialized.", outStream.getOutputFile()); + Assert.assertSame("OutputTracker: Output stream incorrectly initialized.", System.out, outStream.getOutputStream()); + final String ERROR_TEXT = "err stream test"; - PrintWriter errWriter = new PrintWriter(ot.getErrStream()); + PrintWriter errWriter = new PrintWriter(ot.errStub); errWriter.append(ERROR_TEXT); errWriter.close(); @@ -90,24 +99,21 @@ public class OutputTrackerTest extends BaseTest { String errText = errScanner.nextLine(); Assert.assertFalse("Err stream has too much data", errScanner.hasNext()); - Assert.assertTrue("OutputTracker: Output stream is of wrong type.", ot.getOutStream() instanceof RedirectingOutputStream ); - RedirectingOutputStream outStream = (RedirectingOutputStream)ot.getOutStream(); - Assert.assertSame("OutputTracker: Output stream incorrectly initialized.", System.out, outStream.getBackingOutputStream()); Assert.assertEquals("OutputTracker: Written error text is incorrect", errText, ERROR_TEXT); } @Test public void testIndependentStreams() throws FileNotFoundException { - OutputTracker ot = new OutputTracker(); + OutputTracker ot = new DirectOutputTracker(); ot.initializeCoreIO(OUTPUT_FILENAME,ERROR_FILENAME); final String OUTPUT_TEXT = "out stream test"; - PrintWriter outWriter = new PrintWriter(ot.getOutStream()); + PrintWriter outWriter = new PrintWriter(ot.outStub); outWriter.append(OUTPUT_TEXT); outWriter.close(); final String ERROR_TEXT = "err stream test"; - PrintWriter errWriter = new PrintWriter(ot.getErrStream()); + PrintWriter errWriter = new PrintWriter(ot.errStub); errWriter.append(ERROR_TEXT); errWriter.close(); @@ -125,15 +131,16 @@ public class OutputTrackerTest extends BaseTest { @Test public void testIdenticalInputsGetIdenticalResults() { - OutputTracker ot = new OutputTracker(); + OutputTracker ot = new DirectOutputTracker(); ot.initializeCoreIO(OUTPUT_FILENAME,OUTPUT_FILENAME); - Assert.assertTrue("OutputTracker: Output stream is of wrong type.", ot.getOutStream() instanceof RedirectingOutputStream ); - Assert.assertTrue("OutputTracker: Error stream is of wrong type.", ot.getErrStream() instanceof RedirectingOutputStream ); + Assert.assertNotNull("OutputTracker: Output stream is null.", ot.outStub ); + Assert.assertNotNull("OutputTracker: Error stream is null.", ot.errStub ); - RedirectingOutputStream outStream = (RedirectingOutputStream)ot.getOutStream(); - RedirectingOutputStream errStream = (RedirectingOutputStream)ot.getErrStream(); + OutputStreamStub outStream = ot.outStub; + OutputStreamStub errStream = ot.errStub; - Assert.assertSame("OutputTracker: PrintStreams don't match", outStream.getBackingOutputStream(), errStream.getBackingOutputStream()); + Assert.assertSame("OutputTracker: files don't match", outStream.getOutputFile(), errStream.getOutputFile()); + Assert.assertSame("OutputTracker: streams don't match", outStream.getOutputStream(), errStream.getOutputStream()); } } diff --git a/java/test/org/broadinstitute/sting/gatk/walkers/indels/CleanedReadInjectorTest.java b/java/test/org/broadinstitute/sting/gatk/walkers/indels/CleanedReadInjectorTest.java index 405205b36..a848231c7 100644 --- a/java/test/org/broadinstitute/sting/gatk/walkers/indels/CleanedReadInjectorTest.java +++ b/java/test/org/broadinstitute/sting/gatk/walkers/indels/CleanedReadInjectorTest.java @@ -1,7 +1,6 @@ package org.broadinstitute.sting.gatk.walkers.indels; import org.broadinstitute.sting.BaseTest; -import org.broadinstitute.sting.gatk.OutputTracker; import org.broadinstitute.sting.utils.fasta.IndexedFastaSequenceFile; import org.broadinstitute.sting.utils.GenomeLocParser; import org.broadinstitute.sting.utils.sam.ArtificialSAMFileReader;