zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

General-purpose management of output streams. 

git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@1454 348d0f76-0448-11de-a6fe-93d51630548a
This commit is contained in:
hanna 2009-08-23 00:56:02 +00:00
parent b316abd20f
commit ccdb4a0313
33 changed files with 1881 additions and 865 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java
View File

@ -1,19 +1,20 @@
package org.broadinstitute.sting.gatk;
import org.broadinstitute.sting.utils.cmdLine.CommandLineProgram;
import org.broadinstitute.sting.utils.cmdLine.ArgumentSource;
import org.broadinstitute.sting.utils.cmdLine.ArgumentTypeDescriptor;
import org.broadinstitute.sting.utils.StingException;
import org.broadinstitute.sting.utils.xReadLines;
import org.broadinstitute.sting.utils.sam.SAMFileWriterBuilder;
import org.broadinstitute.sting.utils.sam.SAMFileReaderBuilder;
import org.broadinstitute.sting.gatk.walkers.Walker;
import org.broadinstitute.sting.gatk.io.stubs.OutputStreamArgumentTypeDescriptor;
import org.broadinstitute.sting.gatk.io.stubs.SAMFileWriterArgumentTypeDescriptor;
import org.broadinstitute.sting.gatk.io.stubs.SAMFileReaderArgumentTypeDescriptor;
import java.io.File;
import java.io.FileNotFoundException;
import java.util.List;
import java.util.ArrayList;
import net.sf.samtools.SAMFileWriter;
import java.util.Collection;
import java.util.Arrays;
/*
* Copyright (c) 2009 The Broad Institute
@ -87,6 +88,17 @@ public abstract class CommandLineExecutable extends CommandLineProgram {
return 0;
}
/**
* Subclasses of CommandLinePrograms can provide their own types of command-line arguments.
* @return A collection of type descriptors generating implementation-dependent placeholders.
*/
protected Collection<ArgumentTypeDescriptor> getArgumentTypeDescriptors() {
return Arrays.asList( new SAMFileReaderArgumentTypeDescriptor(GATKEngine),
new SAMFileWriterArgumentTypeDescriptor(GATKEngine),
new OutputStreamArgumentTypeDescriptor(GATKEngine) );
}
/**
* GATK can add arguments dynamically based on analysis type.
*
@ -108,24 +120,6 @@ public abstract class CommandLineExecutable extends CommandLineProgram {
return new Class[] { GATKEngine.getWalkerByName(getAnalysisName()).getClass() };
}
/**
* Allows arguments to be hijacked by subclasses of the program before being placed
* into plugin classes.
* @return True if the particular field should be hijacked; false otherwise.
*/
protected boolean intercept( ArgumentSource source, Object targetInstance, Object value ) {
if( !(Walker.class.isAssignableFrom(source.clazz)) )
return false;
if( value instanceof SAMFileReaderBuilder || value instanceof SAMFileWriterBuilder ) {
GATKEngine.setAdditionalIO( source.field, value );
return true;
}
return false;
}
@Override
protected String getArgumentSourceName( Class argumentSource ) {
return WalkerManager.getWalkerName((Class<Walker>) argumentSource);

47
java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java
View File

@ -40,14 +40,16 @@ import org.broadinstitute.sting.gatk.refdata.ReferenceOrderedData;
import org.broadinstitute.sting.gatk.refdata.ReferenceOrderedDatum;
import org.broadinstitute.sting.gatk.walkers.*;
import org.broadinstitute.sting.gatk.filters.ZeroMappingQualityReadFilter;
import org.broadinstitute.sting.gatk.io.OutputTracker;
import org.broadinstitute.sting.utils.*;
import org.broadinstitute.sting.utils.fasta.IndexedFastaSequenceFile;
import org.broadinstitute.sting.utils.cmdLine.ArgumentException;
import org.broadinstitute.sting.utils.cmdLine.ArgumentSource;
import org.broadinstitute.sting.gatk.io.stubs.Stub;
import java.io.File;
import java.io.FileNotFoundException;
import java.util.*;
import java.lang.reflect.Field;
public class GenomeAnalysisEngine {
@ -73,8 +75,12 @@ public class GenomeAnalysisEngine {
// our argument collection
private GATKArgumentCollection argCollection;
/** Collection of output streams used by the walker. */
private OutputTracker outputTracker = new OutputTracker();
/** Collection of inputs used by the walker. */
private Map<ArgumentSource,Object> inputs = new HashMap<ArgumentSource,Object>();
/** Collection of outputs used by the walker. */
private Collection<Stub<?>> outputs = new ArrayList<Stub<?>>();
/** our log, which we want to capture anything from this class */
private static Logger logger = Logger.getLogger(GenomeAnalysisEngine.class);
@ -120,7 +126,7 @@ public class GenomeAnalysisEngine {
MicroScheduler microScheduler = createMicroscheduler(my_walker);
// create the output streams
initializeOutputStreams(my_walker);
initializeOutputStreams(my_walker, microScheduler.getOutputTracker());
GenomeLocSortedSet locs = null;
if (argCollection.intervals != null) {
@ -134,12 +140,20 @@ public class GenomeAnalysisEngine {
}
/**
* Add additional, externally managed IO streams for walker consumption.
* @param walkerField Field in the walker into which to inject the value.
* Add additional, externally managed IO streams for walker input.
* @param argumentSource Field in the walker into which to inject the value.
* @param value Instance to inject.
*/
public void setAdditionalIO( Field walkerField, Object value ) {
outputTracker.addAdditionalOutput( walkerField, value );
public void addInput( ArgumentSource argumentSource, Object value ) {
inputs.put(argumentSource,value);
}
/**
* Add additional, externally managed IO streams for walker output.
* @param stub Instance to inject.
*/
public void addOutput( Stub<?> stub ) {
outputs.add(stub);
}
/**
@ -593,21 +607,18 @@ public class GenomeAnalysisEngine {
*
* @param walker the walker to initialize output streams for
*/
private void initializeOutputStreams(Walker walker) {
private void initializeOutputStreams(Walker walker, OutputTracker outputTracker) {
if( argCollection.outErrFileName != null )
outputTracker.initializeCoreIO( argCollection.outErrFileName, argCollection.outErrFileName );
else
outputTracker.initializeCoreIO( argCollection.outFileName, argCollection.errFileName );
outputTracker.prepareWalker(walker);
}
/**
* Gets the output tracker. Tracks data available to a given walker.
*
* @return The output tracker.
*/
public OutputTracker getOutputTracker() {
return outputTracker;
for( Map.Entry<ArgumentSource,Object> input: inputs.entrySet() )
outputTracker.addInput(input.getKey(),input.getValue());
for( Stub<?> stub: outputs )
outputTracker.addOutput(stub);
outputTracker.prepareWalker(walker);
}
public SAMFileHeader getSAMFileHeader() {

192
java/src/org/broadinstitute/sting/gatk/OutputTracker.java
View File

@ -1,192 +0,0 @@
package org.broadinstitute.sting.gatk;
import org.broadinstitute.sting.utils.StingException;
import org.broadinstitute.sting.utils.JVMUtils;
import org.broadinstitute.sting.utils.sam.SAMFileWriterBuilder;
import org.broadinstitute.sting.utils.sam.SAMFileReaderBuilder;
import org.broadinstitute.sting.utils.io.RedirectingOutputStream;
import org.broadinstitute.sting.gatk.walkers.Walker;
import java.io.FileOutputStream;
import java.io.PrintStream;
import java.io.FileNotFoundException;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.lang.reflect.Field;
import java.util.Map;
import java.util.HashMap;
import net.sf.samtools.SAMFileWriter;
/**
* User: hanna
* Date: Apr 30, 2009
* Time: 9:40:09 AM
* BROAD INSTITUTE SOFTWARE COPYRIGHT NOTICE AND AGREEMENT
* Software and documentation are copyright 2005 by the Broad Institute.
* All rights are reserved.
*
* Users acknowledge that this software is supplied without any warranty or support.
* The Broad Institute is not responsible for its use, misuse, or
* functionality.
*/
/**
* Manages the output and err streams that are created specifically for walker
* output.
*/
public class OutputTracker {
/**
* The streams to which walker users should be writing directly.
*/
protected OutputStream globalOut;
protected OutputStream globalErr;
/**
* Thread-local versions of the output stream.
*/
protected ThreadLocal<OutputStream> localOut = new ThreadLocal<OutputStream>();
protected ThreadLocal<OutputStream> localErr = new ThreadLocal<OutputStream>();
protected Map<Field,Object> additionalIO = new HashMap<Field,Object>();
/**
* Create an object to manage output given filenames for the output and error files.
* If no files are specified, returns null.
* @param outFileName Name of the output file.
* @param errFileName Name of the error file.
*/
public void initializeCoreIO( String outFileName, String errFileName ) {
// If the two output streams match and are non-null, initialize them identically.
// Otherwise, initialize them separately.
if( outFileName != null && outFileName.equals(errFileName) ) {
FileOutputStream outputFile = prepareOutputFile( outFileName );
globalOut = globalErr = outputFile;
}
else {
globalOut = (outFileName != null) ? prepareOutputFile( outFileName ) : System.out;
globalErr = (errFileName != null) ? prepareOutputFile( errFileName ) : System.err;
}
}
public void prepareWalker( Walker walker ) {
Field out = JVMUtils.findField( walker.getClass(), "out" );
Field err = JVMUtils.findField( walker.getClass(), "err" );
JVMUtils.setField( out, walker, new PrintStream(getOutStream()) );
JVMUtils.setField( err, walker, new PrintStream(getErrStream()) );
for( Map.Entry<Field,Object> io: additionalIO.entrySet() ) {
Field targetField = io.getKey();
Object targetValue = io.getValue();
// Ghastly hacks: reaches in and finishes building out the SAMFileReader / SAMFileWriter.
// TODO: Generalize this, and move it to its own initialization step.
if( targetValue instanceof SAMFileReaderBuilder) {
SAMFileReaderBuilder builder = (SAMFileReaderBuilder)targetValue;
builder.setValidationStringency(GenomeAnalysisEngine.instance.getArguments().strictnessLevel);
targetValue = builder.build();
}
if( targetValue instanceof SAMFileWriterBuilder ) {
SAMFileWriterBuilder builder = (SAMFileWriterBuilder)targetValue;
builder.setSAMFileHeader(GenomeAnalysisEngine.instance.getDataSource().getHeader());
targetValue = builder.build();
}
JVMUtils.setField( targetField, walker, targetValue );
}
}
/**
* Gets the output stream for the walker.
* @return Output stream; should be either file-backed or System.out.
*/
public OutputStream getGlobalOutStream() {
return globalOut;
}
/**
* Gets the error stream for the walker.
* @return Error stream; should be either file-backed or System.err.
*/
public OutputStream getGlobalErrStream() {
return globalErr;
}
/**
* Retrieve an output stream that will always return the most appropriate
* writer for this thread.
*/
public OutputStream getOutStream() {
// Create an anonymous inner class which will just return the most
// appropriate OutputStream from those streams stored in this class.
return new RedirectingOutputStream(
new RedirectingOutputStream.OutputStreamProvider() {
public OutputStream getOutputStream() {
return localOut.get() != null ? localOut.get() : globalOut;
}
} );
}
/**
* Retrieve the most appropriate output stream for this thread.
* Will retrieve thread-local if available; otherwise, it'll read the
* global stream.
*/
public OutputStream getErrStream() {
// Create an anonymous inner class which will just return the most
// appropriate OutputStream from those streams stored in this class.
return new RedirectingOutputStream(
new RedirectingOutputStream.OutputStreamProvider() {
public OutputStream getOutputStream() {
return localErr.get() != null ? localErr.get() : globalErr;
}
} );
}
/**
* Set the (thread-)local version of the given output and error files.
* @param out output stream to which to write.
* @param err error stream to which to write.
*/
public void setLocalStreams( OutputStream out, OutputStream err ) {
localOut.set( out );
localErr.set( err );
}
/**
* Provide a mechanism for injecting supplemental streams for external management.
* @param field Field into which to inject this stream.
* @param stream Stream to manage.
*/
public void addAdditionalOutput( Field field, Object stream ) {
additionalIO.put(field,stream);
}
/**
* Remove pointers to alternate, local output streams.
*/
public void cleanup() {
localOut.remove();
localErr.remove();
}
/**
* Given a (non-null) filename, open a file for output.
* @param fileName Filename for the stream. Should not be null.
* @return An open output stream associated with the given file.
*/
private FileOutputStream prepareOutputFile( String fileName ) {
FileOutputStream outputFile = null;
try {
outputFile = new FileOutputStream( fileName );
}
catch( FileNotFoundException ex ) {
throw new StingException("Unable to open output file " + fileName, ex);
}
return outputFile;
}
}

122
java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java
View File

@ -6,21 +6,14 @@ import org.broadinstitute.sting.gatk.datasources.shards.ShardStrategy;
import org.broadinstitute.sting.gatk.datasources.shards.Shard;
import org.broadinstitute.sting.gatk.datasources.simpleDataSources.SAMDataSource;
import org.broadinstitute.sting.gatk.datasources.simpleDataSources.ReferenceOrderedDataSource;
import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
import org.broadinstitute.sting.gatk.OutputTracker;
import org.broadinstitute.sting.gatk.Reads;
import org.broadinstitute.sting.gatk.refdata.ReferenceOrderedDatum;
import org.broadinstitute.sting.gatk.refdata.ReferenceOrderedData;
import org.broadinstitute.sting.gatk.io.*;
import org.broadinstitute.sting.utils.StingException;
import org.broadinstitute.sting.utils.GenomeLocSortedSet;
import org.broadinstitute.sting.utils.fasta.IndexedFastaSequenceFile;
import org.broadinstitute.sting.utils.threading.ThreadPoolMonitor;
import javax.management.MBeanServer;
import javax.management.ObjectName;
import javax.management.JMException;
import java.io.File;
import java.util.List;
import java.util.Queue;
import java.util.LinkedList;
import java.util.Collection;
@ -30,14 +23,6 @@ import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
import java.lang.management.ManagementFactory;
/**
* Created by IntelliJ IDEA.
* User: mhanna
* Date: Apr 26, 2009
* Time: 5:41:04 PM
* To change this template use File | Settings | File Templates.
*/
/**
* A microscheduler that schedules shards according to a tree-like structure.
* Requires a special walker tagged with a 'TreeReducible' interface.
@ -52,9 +37,20 @@ public class HierarchicalMicroScheduler extends MicroScheduler implements Hierar
/** Manage currently running threads. */
private ExecutorService threadPool;
private Queue<Shard> traverseTasks = new LinkedList<Shard>();
private Queue<TreeReduceTask> reduceTasks = new LinkedList<TreeReduceTask>();
private Queue<OutputMerger> outputMergeTasks = new LinkedList<OutputMerger>();
/**
* A thread local output tracker for managing output per-thread.
*/
private ThreadLocalOutputTracker outputTracker = new ThreadLocalOutputTracker();
private final Queue<Shard> traverseTasks = new LinkedList<Shard>();
private final Queue<TreeReduceTask> reduceTasks = new LinkedList<TreeReduceTask>();
/**
* Keep a queue of shard traversals, and constantly monitor it to see what output
* merge tasks remain.
* TODO: Integrate this into the reduce tree.
*/
private final Queue<ShardTraverser> outputMergeTasks = new LinkedList<ShardTraverser>();
/** How many total tasks were in the queue at the start of run. */
private int totalTraversals = 0;
@ -115,7 +111,7 @@ public class HierarchicalMicroScheduler extends MicroScheduler implements Hierar
while (isShardTraversePending() || isTreeReducePending()) {
// Too many files sitting around taking up space? Merge them.
if (isMergeLimitExceeded())
mergeExistingOutput();
mergeExistingOutput(false);
// Wait for the next slot in the queue to become free.
waitForFreeQueueSlot();
@ -130,7 +126,7 @@ public class HierarchicalMicroScheduler extends MicroScheduler implements Hierar
// Merge any lingering output files. If these files aren't ready,
// sit around and wait for them, then merge them.
mergeRemainingOutput();
mergeExistingOutput(true);
threadPool.shutdown();
@ -149,6 +145,13 @@ public class HierarchicalMicroScheduler extends MicroScheduler implements Hierar
return result;
}
/**
* @{inheritDoc}
*/
public OutputTracker getOutputTracker() {
return outputTracker;
}
/**
* Returns true if there are unscheduled shard traversal waiting to run.
*
@ -188,62 +191,43 @@ public class HierarchicalMicroScheduler extends MicroScheduler implements Hierar
* @return True if the merging needs to take priority. False otherwise.
*/
protected boolean isMergeLimitExceeded() {
if (outputMergeTasks.size() < MAX_OUTSTANDING_OUTPUT_MERGES)
return false;
// If any of the first MAX_OUTSTANDING merges aren't ready, the merge limit
// has not been exceeded.
OutputMerger[] outputMergers = outputMergeTasks.toArray(new OutputMerger[0]);
for (int i = 0; i < MAX_OUTSTANDING_OUTPUT_MERGES; i++) {
if (!outputMergers[i].isComplete())
return false;
int pendingTasks = 0;
for( ShardTraverser shardTraverse: outputMergeTasks ) {
if( !shardTraverse.isComplete() )
break;
pendingTasks++;
}
// Everything's ready?
return true;
}
/**
* Returns whether there is output waiting to be merged into the global output
* streams right now.
*
* @return True if this output is ready to be merged. False otherwise.
*/
protected boolean isOutputMergeReady() {
if (outputMergeTasks.size() > 0)
return outputMergeTasks.peek().isComplete();
else
return false;
return (outputMergeTasks.size() >= MAX_OUTSTANDING_OUTPUT_MERGES);
}
/**
* Merging all output that's sitting ready in the OutputMerger queue into
* the final data streams.
*/
protected void mergeExistingOutput() {
protected void mergeExistingOutput( boolean wait ) {
long startTime = System.currentTimeMillis();
OutputTracker outputTracker = GenomeAnalysisEngine.instance.getOutputTracker();
while (isOutputMergeReady())
outputMergeTasks.remove().mergeInto(outputTracker.getGlobalOutStream(), outputTracker.getGlobalErrStream());
// Create a list of the merge tasks that will be performed in this run of the mergeExistingOutput().
Queue<ShardTraverser> mergeTasksInSession = new LinkedList<ShardTraverser>();
while( !outputMergeTasks.isEmpty() ) {
ShardTraverser traverser = outputMergeTasks.peek();
long endTime = System.currentTimeMillis();
// If the next traversal isn't done and we're not supposed to wait, we've found our working set. Continue.
if( !traverser.isComplete() && !wait )
break;
totalOutputMergeTime += ( endTime - startTime );
}
outputMergeTasks.remove();
mergeTasksInSession.add(traverser);
}
/** Merge any output that hasn't yet been taken care of by the blocking thread. */
protected void mergeRemainingOutput() {
long startTime = System.currentTimeMillis();
// Actually run through, merging the tasks in the working queue.
for( ShardTraverser traverser: mergeTasksInSession ) {
if( !traverser.isComplete() )
traverser.waitForComplete();
OutputTracker outputTracker = GenomeAnalysisEngine.instance.getOutputTracker();
while (outputMergeTasks.size() > 0) {
OutputMerger outputMerger = outputMergeTasks.remove();
synchronized (outputMerger) {
if (!outputMerger.isComplete())
outputMerger.waitForOutputComplete();
}
outputMerger.mergeInto(outputTracker.getGlobalOutStream(), outputTracker.getGlobalErrStream());
OutputMergeTask mergeTask = traverser.getOutputMergeTask();
if( mergeTask != null )
mergeTask.merge();
}
long endTime = System.currentTimeMillis();
@ -262,26 +246,24 @@ public class HierarchicalMicroScheduler extends MicroScheduler implements Hierar
throw new IllegalStateException("Cannot traverse; no pending traversals exist.");
Shard shard = traverseTasks.remove();
OutputMerger outputMerger = new OutputMerger();
ShardTraverser traverser = new ShardTraverser(this,
traversalEngine,
walker,
shard,
getShardDataProvider(shard),
outputMerger);
outputTracker);
Future traverseResult = threadPool.submit(traverser);
// Add this traverse result to the reduce tree. The reduce tree will call a callback to throw its entries on the queue.
reduceTree.addEntry(traverseResult);
outputMergeTasks.add(traverser);
// No more data? Let the reduce tree know so it can finish processing what it's got.
if (!isShardTraversePending())
reduceTree.complete();
outputMergeTasks.add(outputMerger);
return traverseResult;
}
@ -372,7 +354,9 @@ public class HierarchicalMicroScheduler extends MicroScheduler implements Hierar
/** {@inheritDoc} */
public int getNumberOfTasksInIOQueue() {
return outputMergeTasks.size();
synchronized( outputMergeTasks ) {
return outputMergeTasks.size();
}
}
/** {@inheritDoc} */

12
java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java
View File

@ -6,6 +6,8 @@ import org.broadinstitute.sting.gatk.datasources.shards.ShardStrategy;
import org.broadinstitute.sting.gatk.datasources.simpleDataSources.ReferenceOrderedDataSource;
import org.broadinstitute.sting.gatk.datasources.simpleDataSources.SAMDataSource;
import org.broadinstitute.sting.gatk.walkers.Walker;
import org.broadinstitute.sting.gatk.io.DirectOutputTracker;
import org.broadinstitute.sting.gatk.io.OutputTracker;
import org.broadinstitute.sting.utils.fasta.IndexedFastaSequenceFile;
import java.util.Collection;
@ -13,6 +15,11 @@ import java.util.Collection;
/** A micro-scheduling manager for single-threaded execution of a traversal. */
public class LinearMicroScheduler extends MicroScheduler {
/**
* A direct output tracker for directly managing output.
*/
private DirectOutputTracker outputTracker = new DirectOutputTracker();
/**
* Create a new linear microscheduler to process the given reads and reference.
*
@ -55,5 +62,8 @@ public class LinearMicroScheduler extends MicroScheduler {
return accumulator;
}
/**
* @{inheritDoc}
*/
public OutputTracker getOutputTracker() { return outputTracker; }
}

8
java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java
View File

@ -33,6 +33,8 @@ import org.broadinstitute.sting.gatk.datasources.simpleDataSources.ReferenceOrde
import org.broadinstitute.sting.gatk.datasources.simpleDataSources.SAMDataSource;
import org.broadinstitute.sting.gatk.traversals.*;
import org.broadinstitute.sting.gatk.walkers.*;
import org.broadinstitute.sting.gatk.io.OutputTracker;
import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
import org.broadinstitute.sting.utils.fasta.IndexedFastaSequenceFile;
import java.util.*;
@ -116,6 +118,12 @@ public abstract class MicroScheduler {
*/
public abstract Object execute(Walker walker, ShardStrategy shardStrategy, int iterations );
/**
* Retrieves the object responsible for tracking and managing output.
* @return An output tracker, for loading data in and extracting results. Will not be null.
*/
public abstract OutputTracker getOutputTracker();
/**
* Gets an window into all the data that can be viewed as a single shard.

78
java/src/org/broadinstitute/sting/gatk/executive/OutputMergeTask.java 100755
View File

@ -0,0 +1,78 @@
package org.broadinstitute.sting.gatk.executive;
import org.broadinstitute.sting.gatk.io.storage.Storage;
import org.broadinstitute.sting.gatk.io.OutputTracker;
import java.util.Collection;
import java.util.ArrayList;
/**
* User: hanna
* Date: Apr 30, 2009
* Time: 4:04:38 PM
* BROAD INSTITUTE SOFTWARE COPYRIGHT NOTICE AND AGREEMENT
* Software and documentation are copyright 2005 by the Broad Institute.
* All rights are reserved.
*
* Users acknowledge that this software is supplied without any warranty or support.
* The Broad Institute is not responsible for its use, misuse, or
* functionality.
*/
/**
* Hold pointers to the output and error streams, and state to indicate whether
* a write is complete. Not generally thread-safe. Calls to isComplete()/complete()
* can be made at any time from any thread, but complete() should be called on the
* thread which is doing the writing.
*/
public class OutputMergeTask {
/**
* The output streams which should be written to.
*/
private final Collection<MergeOperation<?>> mergeOperations = new ArrayList<MergeOperation<?>>();
/**
* Add a new merge operation to this merge task.
* @param targetStream Target for stream output.
* @param temporaryStorage Temporary storage.
* @param <StreamType> Type of the output stream.
*/
public <StreamType> void addMergeOperation( StreamType targetStream, Storage<StreamType> temporaryStorage ) {
mergeOperations.add( new MergeOperation<StreamType>(targetStream,temporaryStorage) );
}
/**
* Merge data from output streams into target storage.
*/
public synchronized void merge() {
for( MergeOperation mergeOperation: mergeOperations )
mergeOperation.temporaryStorage.mergeInto(mergeOperation.targetStream);
}
/**
* Represents a single file needed to be merged.
* @param <StreamType> Type of the file to be merged.
*/
private class MergeOperation<StreamType> {
/**
* Destination for the temporary file's output.
*/
public final StreamType targetStream;
/**
* Temporary storage location for the file.
*/
public final Storage<StreamType> temporaryStorage;
/**
* Create a new merge file object with the given output stream and storage placeholder.
* @param targetStream Target for temporary data.
* @param temporaryStorage The temporary data itself.
*/
public MergeOperation( StreamType targetStream, Storage<StreamType> temporaryStorage ) {
this.targetStream = targetStream;
this.temporaryStorage = temporaryStorage;
}
}
}

167
java/src/org/broadinstitute/sting/gatk/executive/OutputMerger.java
View File

@ -1,167 +0,0 @@
package org.broadinstitute.sting.gatk.executive;
import org.broadinstitute.sting.utils.StingException;
import org.broadinstitute.sting.utils.io.LazyFileOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.nio.channels.FileChannel;
import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;
/**
* User: hanna
* Date: Apr 30, 2009
* Time: 4:04:38 PM
* BROAD INSTITUTE SOFTWARE COPYRIGHT NOTICE AND AGREEMENT
* Software and documentation are copyright 2005 by the Broad Institute.
* All rights are reserved.
*
* Users acknowledge that this software is supplied without any warranty or support.
* The Broad Institute is not responsible for its use, misuse, or
* functionality.
*/
/**
* Hold pointers to the output and error streams, and state to indicate whether
* a write is complete. Not generally thread-safe. Calls to isComplete()/complete()
* can be made at any time from any thread, but complete() should be called on the
* thread which is doing the writing.
*/
public class OutputMerger {
/**
* Is writing to these streams complete?
*/
private boolean complete = false;
/**
* The printstreams which should be written to.
*/
private LazyFileOutputStream out = null;
private LazyFileOutputStream err = null;
public OutputMerger() {
out = new LazyFileOutputStream( outStreamFactory );
err = new LazyFileOutputStream( errStreamFactory );
}
/**
* Creates a generic output stream for temporary data.
*/
private static class TempFileFactory implements LazyFileOutputStream.FileFactory {
private final String prefix;
public TempFileFactory( String prefix ) { this.prefix = prefix; }
public File create() throws IOException { return File.createTempFile(prefix,null); }
}
/**
* Creates a stream for temporary out (not err) data.
*/
private static final TempFileFactory outStreamFactory = new TempFileFactory("gatkout_");
/**
* Creates a stream for temporary err data.
*/
private static final TempFileFactory errStreamFactory = new TempFileFactory("gatkerr_");
/**
* Waits for any the given OutputMerger to be ready for merging.
*/
public synchronized void waitForOutputComplete() {
try {
wait();
}
catch( InterruptedException ex ) {
throw new StingException("Interrupted while waiting for more output to be finalized.",ex);
}
}
/**
* Is this output complete?
* @return True if output complete. False otherwise.
*/
public synchronized boolean isComplete() {
return complete;
}
/**
* Indicate that no more data will be written to these output streams.
*/
public synchronized void complete() {
if( isComplete() )
throw new IllegalStateException("Tried to complete an output merge twice.");
try {
if( out.isCreated() ) {
out.flush();
out.close();
}
if( err.isCreated() ) {
err.flush();
err.close();
}
}
catch( IOException ex ) {
throw new StingException( "Unable to close sharding output files", ex );
}
this.complete = true;
// Notify waiting listeners that this shard is complete and ready for merging.
notifyAll();
}
public void mergeInto( OutputStream globalOut, OutputStream globalErr ) {
synchronized(this) {
if( !isComplete() )
throw new StingException("Tried to merge incomplete stream into output");
}
if( out.isCreated() ) transferContentsToTarget( out.getBackingFile(), globalOut );
if( err.isCreated() ) transferContentsToTarget( err.getBackingFile(), globalErr );
}
/**
* Copy the contents of the given file into the specified output stream.
* @param source Source of data to copy.
* @param target Target for copied data.
*/
private void transferContentsToTarget( File source, OutputStream target ) {
FileInputStream sourceStream = null;
try {
sourceStream = new FileInputStream( source );
FileChannel sourceChannel = sourceStream.getChannel();
WritableByteChannel targetChannel = Channels.newChannel( target );
sourceChannel.transferTo( 0, sourceChannel.size(), targetChannel );
sourceStream.close();
source.delete();
}
catch( FileNotFoundException ex ) {
throw new StingException("Unable to open input stream for file: " + source,ex);
}
catch( IOException ex ) {
throw new StingException("Unable to transfer contents of file:" + source,ex);
}
}
/**
* Return the stream where output is sent.
* @return output stream.
*/
public OutputStream getOutStream() {
return out;
}
/**
* Gets the stream where error info is sent.
* @return error stream.
*/
public OutputStream getErrStream() {
return err;
}
}

61
java/src/org/broadinstitute/sting/gatk/executive/ShardTraverser.java
View File

@ -3,9 +3,9 @@ package org.broadinstitute.sting.gatk.executive;
import org.broadinstitute.sting.gatk.datasources.providers.ShardDataProvider;
import org.broadinstitute.sting.gatk.datasources.shards.Shard;
import org.broadinstitute.sting.gatk.traversals.TraversalEngine;
import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
import org.broadinstitute.sting.gatk.OutputTracker;
import org.broadinstitute.sting.gatk.io.ThreadLocalOutputTracker;
import org.broadinstitute.sting.gatk.walkers.Walker;
import org.broadinstitute.sting.utils.StingException;
import java.util.concurrent.Callable;
/**
@ -29,36 +29,43 @@ public class ShardTraverser implements Callable {
private TraversalEngine traversalEngine;
private Shard shard;
private ShardDataProvider dataProvider;
private OutputMerger output;
private ThreadLocalOutputTracker outputTracker;
private OutputMergeTask outputMergeTask;
/**
* Is this traversal complete?
*/
private boolean complete = false;
public ShardTraverser( HierarchicalMicroScheduler microScheduler,
TraversalEngine traversalEngine,
Walker walker,
Shard shard,
ShardDataProvider dataProvider,
OutputMerger output ) {
ThreadLocalOutputTracker outputTracker ) {
this.microScheduler = microScheduler;
this.walker = walker;
this.traversalEngine = traversalEngine;
this.shard = shard;
this.dataProvider = dataProvider;
this.output = output;
this.outputTracker = outputTracker;
}
public Object call() {
long startTime = System.currentTimeMillis();
Object accumulator = walker.reduceInit();
OutputTracker outputTracker = GenomeAnalysisEngine.instance.getOutputTracker();
outputTracker.setLocalStreams( output.getOutStream(), output.getErrStream() );
try {
accumulator = traversalEngine.traverse( walker, shard, dataProvider, accumulator );
}
finally {
dataProvider.close();
output.complete();
outputTracker.cleanup();
outputMergeTask = outputTracker.closeStorage();
synchronized(this) {
complete = true;
notifyAll();
}
}
long endTime = System.currentTimeMillis();
@ -67,4 +74,38 @@ public class ShardTraverser implements Callable {
return accumulator;
}
/**
* Has this traversal completed?
* @return True if completed, false otherwise.
*/
public boolean isComplete() {
synchronized(this) {
return complete;
}
}
/**
* Waits for any the given OutputMerger to be ready for merging.
*/
public void waitForComplete() {
try {
synchronized(this) {
if( isComplete() )
return;
wait();
}
}
catch( InterruptedException ex ) {
throw new StingException("Interrupted while waiting for more output to be finalized.",ex);
}
}
/**
* Gets the output merge task associated with the given shard.
* @return OutputMergeTask if one exists; null if nothing needs to be merged.
*/
public OutputMergeTask getOutputMergeTask() {
return outputMergeTask;
}
}

56
java/src/org/broadinstitute/sting/gatk/io/DirectOutputTracker.java 100644
View File

@ -0,0 +1,56 @@
/*
* Copyright (c) 2009 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.io;
import org.broadinstitute.sting.gatk.io.stubs.Stub;
import org.broadinstitute.sting.gatk.io.storage.StorageFactory;
import org.broadinstitute.sting.gatk.io.storage.Storage;
import java.util.Map;
import java.util.HashMap;
/**
* Javadoc goes here.
*
* @author mhanna
* @version 0.1
*/
public class DirectOutputTracker extends OutputTracker {
/**
* Direct storage for output streams.
*/
private final Map<Stub, Storage> storage = new HashMap<Stub,Storage>();
public <T> T getStorage( Stub<T> stub ) {
Storage target = storage.get(stub);
if( target == null ) {
target = StorageFactory.createStorage(stub);
storage.put(stub, target);
}
return (T)target;
}
}

156
java/src/org/broadinstitute/sting/gatk/io/OutputTracker.java 100755
View File

@ -0,0 +1,156 @@
package org.broadinstitute.sting.gatk.io;
import org.broadinstitute.sting.utils.JVMUtils;
import org.broadinstitute.sting.utils.StingException;
import org.broadinstitute.sting.utils.cmdLine.ArgumentSource;
import org.broadinstitute.sting.utils.sam.SAMFileReaderBuilder;
import org.broadinstitute.sting.gatk.walkers.Walker;
import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
import org.broadinstitute.sting.gatk.io.stubs.OutputStreamStub;
import org.broadinstitute.sting.gatk.io.stubs.Stub;
import org.broadinstitute.sting.gatk.io.storage.StorageFactory;
import org.broadinstitute.sting.gatk.io.storage.Storage;
import java.io.*;
import java.lang.reflect.Field;
import java.util.Map;
import java.util.HashMap;
/**
* User: hanna
* Date: Apr 30, 2009
* Time: 9:40:09 AM
* BROAD INSTITUTE SOFTWARE COPYRIGHT NOTICE AND AGREEMENT
* Software and documentation are copyright 2005 by the Broad Institute.
* All rights are reserved.
*
* Users acknowledge that this software is supplied without any warranty or support.
* The Broad Institute is not responsible for its use, misuse, or
* functionality.
*/
/**
* Manages the output and err streams that are created specifically for walker
* output.
*/
public abstract class OutputTracker {
/**
* The streams to which walker users should be reading directly.
*/
protected Map<ArgumentSource, Object> inputs = new HashMap<ArgumentSource,Object>();
/**
* The streams to which walker users should be writing directly.
*/
protected Map<Stub,Storage> outputs = new HashMap<Stub,Storage>();
/**
* Special-purpose stub. Provides a connection to output streams.
*/
protected OutputStreamStub outStub = null;
/**
* Special-purpose stream. Provides a connection to error streams.
*/
protected OutputStreamStub errStub = null;
/**
* Create an object to manage output given filenames for the output and error files.
* If no files are specified, returns null.
* @param outFileName Name of the output file.
* @param errFileName Name of the error file.
*/
public void initializeCoreIO( String outFileName, String errFileName ) {
// If the two output streams match and are non-null, initialize them identically.
// Otherwise, initialize them separately.
if( outFileName != null && outFileName.equals(errFileName) ) {
outStub = errStub = new OutputStreamStub(new File(outFileName));
outStub.register(this);
outputs.put(outStub,null);
}
else {
outStub = (outFileName != null) ? new OutputStreamStub(new File(outFileName))
: new OutputStreamStub(System.out);
outStub.register(this);
outputs.put(outStub,null);
errStub = (errFileName != null) ? new OutputStreamStub(new File(errFileName))
: new OutputStreamStub(System.err);
errStub.register(this);
outputs.put(errStub,null);
}
}
/**
* Gets the output storage associated with a given stub.
* @param stub The stub for which to find / create the right output stream.
* @param <T> Type of the stream to create.
* @return Storage object with a facade of type T.
*/
public abstract <T> T getStorage( Stub<T> stub );
public void prepareWalker( Walker walker ) {
installStub( walker, "out", new PrintStream(outStub) );
installStub( walker, "err", new PrintStream(errStub) );
for( Map.Entry<ArgumentSource,Object> io: inputs.entrySet() ) {
ArgumentSource targetField = io.getKey();
Object targetValue = io.getValue();
// Ghastly hack: reaches in and finishes building out the SAMFileReader.
// TODO: Generalize this, and move it to its own initialization step.
if( targetValue instanceof SAMFileReaderBuilder) {
SAMFileReaderBuilder builder = (SAMFileReaderBuilder)targetValue;
builder.setValidationStringency(GenomeAnalysisEngine.instance.getArguments().strictnessLevel);
targetValue = builder.build();
}
JVMUtils.setField( targetField.field, walker, targetValue );
}
}
/**
* Provide a mechanism for injecting supplemental streams for external management.
* @param argumentSource source Class / field into which to inject this stream.
* @param stub Stream to manage.
*/
public void addInput( ArgumentSource argumentSource, Object stub ) {
inputs.put(argumentSource,stub);
}
/**
* Provide a mechanism for injecting supplemental streams for external management.
* @param stub Stream to manage.
*/
public <T> void addOutput(Stub<T> stub) {
stub.register(this);
outputs.put(stub,null);
}
/**
* Collects the target stream for this data.
* @param stub The stub for this stream.
* @param <T> type of stub.
* @return An instantiated file into which data can be written.
*/
protected <T> T getTargetStream( Stub<T> stub ) {
if( !outputs.containsKey(stub) )
throw new StingException("OutputTracker was not notified that this stub exists: " + stub);
Storage<T> storage = outputs.get(stub);
if( storage == null ) {
storage = StorageFactory.createStorage(stub);
outputs.put(stub,storage);
}
return (T)storage;
}
/**
* Install an OutputStreamStub into the given fieldName of the given walker.
* @param walker Walker into which to inject the field name.
* @param fieldName Name of the field into which to inject the stub.
*/
private void installStub( Walker walker, String fieldName, OutputStream outputStream ) {
Field field = JVMUtils.findField( walker.getClass(), fieldName );
JVMUtils.setField( field, walker, outputStream );
}
}

109
java/src/org/broadinstitute/sting/gatk/io/ThreadLocalOutputTracker.java 100644
View File

@ -0,0 +1,109 @@
/*
* Copyright (c) 2009 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.io;
import org.broadinstitute.sting.gatk.io.stubs.Stub;
import org.broadinstitute.sting.gatk.io.storage.StorageFactory;
import org.broadinstitute.sting.gatk.io.storage.Storage;
import org.broadinstitute.sting.gatk.executive.OutputMergeTask;
import org.broadinstitute.sting.utils.StingException;
import java.util.*;
import java.io.File;
import java.io.IOException;
/**
* An output tracker that tracks its output per-thread.
*
* @author mhanna
* @version 0.1
*/
public class ThreadLocalOutputTracker extends OutputTracker {
/**
* Thread-local storage for output streams.
*/
private ThreadLocal<Map<Stub, Storage>> storage = new ThreadLocal<Map<Stub,Storage>>();
public <T> T getStorage( Stub<T> stub ) {
Map<Stub,Storage> threadLocalOutputStreams = storage.get();
if( threadLocalOutputStreams == null ) {
threadLocalOutputStreams = new HashMap<Stub,Storage>();
storage.set( threadLocalOutputStreams );
}
Storage target = threadLocalOutputStreams.get(stub);
if( target == null ) {
target = StorageFactory.createStorage(stub, createTempFile(stub));
threadLocalOutputStreams.put(stub, target);
}
return (T)target;
}
/**
* Close down any existing temporary files which have been opened.
*/
public OutputMergeTask closeStorage() {
Map<Stub,Storage> threadLocalStorage = storage.get();
if( threadLocalStorage == null || threadLocalStorage.isEmpty() )
return null;
OutputMergeTask outputMergeTask = new OutputMergeTask();
for( Map.Entry<Stub,Storage> entry: threadLocalStorage.entrySet() ) {
Stub stub = entry.getKey();
Storage storageEntry = entry.getValue();
storageEntry.close();
outputMergeTask.addMergeOperation(getTargetStream(stub),storageEntry);
}
threadLocalStorage.clear();
return outputMergeTask;
}
/**
* Creates a temporary file for a stub of the given type.
* @param stub Stub for which to create a temporary file.
* @param <T> Type of the stub to accept.
* @return A temp file, or throw an exception if the temp file cannot be created.
*/
private <T> File createTempFile( Stub<T> stub ) {
File tempFile = null;
try {
tempFile = File.createTempFile( stub.getClass().getName(), null );
tempFile.deleteOnExit();
}
catch( IOException ex ) {
throw new StingException("Unable to create temporary file for stub: " + stub.getClass().getName() );
}
return tempFile;
}
}

139
java/src/org/broadinstitute/sting/gatk/io/storage/OutputStreamStorage.java 100644
View File

@ -0,0 +1,139 @@
/*
* Copyright (c) 2009 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.io.storage;
import org.broadinstitute.sting.utils.StingException;
import org.broadinstitute.sting.gatk.io.stubs.OutputStreamStub;
import org.broadinstitute.sting.gatk.io.storage.Storage;
import java.io.*;
import java.nio.channels.FileChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.channels.Channels;
public class OutputStreamStorage extends OutputStream implements Storage<OutputStream> {
/**
* File to which data will temporarily be written.
*/
private final File file;
/**
* Stream to which data in this shard will be written.
*/
private final OutputStream outputStream;
/**
* Create a new storage area with the given stub.
* @param stub
*/
public OutputStreamStorage( OutputStreamStub stub ) {
if( stub.getOutputFile() != null ) {
this.file = stub.getOutputFile();
this.outputStream = initializeOutputStream(stub.getOutputFile());
}
else if( stub.getOutputStream() != null ) {
this.file = null;
this.outputStream = stub.getOutputStream();
}
else
throw new StingException("Not enough information to create storage for an OutputStream; need either a file or an existing output stream");
}
public OutputStreamStorage( OutputStreamStub stub, File file ) {
this.file = file;
this.outputStream = initializeOutputStream(file);
}
private OutputStream initializeOutputStream( File file ) {
try {
return new FileOutputStream( file );
}
catch(FileNotFoundException ex) {
throw new StingException("Unable to open output stream for file: " + file);
}
}
/**
* @{inheritDoc}
*/
public void flush() throws IOException {
outputStream.close();
}
/**
* @{inheritDoc}
*/
public void close() {
try {
outputStream.close();
}
catch( IOException ex ) {
throw new StingException( "Unable to close output stream" );
}
}
/**
* @{inheritDoc}
*/
public void write( byte[] b ) throws IOException {
outputStream.write(b);
}
/**
* @{inheritDoc}
*/
public void write( byte[] b, int off, int len ) throws IOException {
outputStream.write(b, off, len);
}
/**
* @{inheritDoc}
*/
public void write( int b ) throws IOException {
outputStream.write(b);
}
public void mergeInto( OutputStream targetStream ) {
FileInputStream sourceStream = null;
try {
sourceStream = new FileInputStream( file );
FileChannel sourceChannel = sourceStream.getChannel();
WritableByteChannel targetChannel = Channels.newChannel( targetStream );
sourceChannel.transferTo( 0, sourceChannel.size(), targetChannel );
sourceStream.close();
file.delete();
}
catch( FileNotFoundException ex ) {
throw new StingException("Unable to open input stream for file: " + file,ex);
}
catch( IOException ex ) {
throw new StingException("Unable to transfer contents of file:" + file,ex);
}
}
}

77
java/src/org/broadinstitute/sting/gatk/io/storage/SAMFileWriterStorage.java 100644
View File

@ -0,0 +1,77 @@
/*
* Copyright (c) 2009 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.io.storage;
import net.sf.samtools.*;
import net.sf.samtools.util.CloseableIterator;
import java.io.*;
import org.broadinstitute.sting.gatk.io.stubs.SAMFileWriterStub;
import org.broadinstitute.sting.gatk.io.storage.Storage;
/**
* Provides temporary storage for SAMFileWriters.
*
* @author mhanna
* @version 0.1
*/
public class SAMFileWriterStorage implements SAMFileWriter, Storage<SAMFileWriter> {
private final File file;
private final SAMFileWriter writer;
public SAMFileWriterStorage( SAMFileWriterStub stub ) {
this(stub,stub.getSAMFile());
}
public SAMFileWriterStorage( SAMFileWriterStub stub, File file ) {
this.file = file;
this.writer = new SAMFileWriterFactory().makeBAMWriter( stub.getSAMFileHeader(), true, file );
}
public void addAlignment( SAMRecord read ) {
writer.addAlignment(read);
}
public void close() {
writer.close();
}
public void mergeInto( SAMFileWriter targetStream ) {
SAMFileReader reader = new SAMFileReader( file );
try {
CloseableIterator<SAMRecord> iterator = reader.iterator();
while( iterator.hasNext() )
targetStream.addAlignment( iterator.next() );
iterator.close();
}
finally {
reader.close();
file.delete();
}
}
}

45
java/src/org/broadinstitute/sting/gatk/io/storage/Storage.java 100644
View File

@ -0,0 +1,45 @@
/*
* Copyright (c) 2009 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.io.storage;
/**
* An interface representing the temporary storage of data.
*
* @author mhanna
* @version 0.1
*/
public interface Storage<StreamType> {
/**
* Writing to the temporary storage is done. Close down the file.
*/
public void close();
/**
* Merges the stream backing up this temporary storage into the target.
* @param target Target stream for the temporary storage. May not be null.
*/
public void mergeInto( StreamType target );
}

87
java/src/org/broadinstitute/sting/gatk/io/storage/StorageFactory.java 100644
View File

@ -0,0 +1,87 @@
/*
* Copyright (c) 2009 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.io.storage;
import org.broadinstitute.sting.gatk.io.stubs.Stub;
import org.broadinstitute.sting.gatk.io.stubs.OutputStreamStub;
import org.broadinstitute.sting.gatk.io.stubs.SAMFileWriterStub;
import org.broadinstitute.sting.gatk.io.storage.SAMFileWriterStorage;
import org.broadinstitute.sting.gatk.io.storage.Storage;
import org.broadinstitute.sting.gatk.io.storage.OutputStreamStorage;
import org.broadinstitute.sting.utils.StingException;
import java.io.File;
/**
* Construct storage of the required type.
*
* @author mhanna
* @version 0.1
*/
public class StorageFactory {
/**
* Disable storage factory construction.
*/
private StorageFactory() {}
/**
* Gets the output storage associated with a given stub.
* @param stub The stub for which to find / create the right output stream.
* @param <T> Type of the stream to create.
* @return Storage object with a facade of type T.
*/
public static <T> Storage<T> createStorage( Stub<T> stub ) {
return createStorage( stub, null );
}
/**
* Gets the output storage associated with a given stub.
* @param stub The stub for which to find / create the right output stream.
* @param file The filename to which to write the file.
* @param <T> Type of the stream to create.
* @return Storage object with a facade of type T.
*/
public static <T> Storage<T> createStorage( Stub<T> stub, File file ) {
Storage storage = null;
if(stub instanceof OutputStreamStub) {
if( file != null )
storage = new OutputStreamStorage((OutputStreamStub)stub,file);
else
storage = new OutputStreamStorage((OutputStreamStub)stub);
}
else if(stub instanceof SAMFileWriterStub) {
if( file != null )
storage = new SAMFileWriterStorage((SAMFileWriterStub)stub,file);
else
storage = new SAMFileWriterStorage((SAMFileWriterStub)stub);
}
else
throw new StingException("Unsupported stub type: " + stub.getClass().getName());
return storage;
}
}

95
java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamArgumentTypeDescriptor.java 100644
View File

@ -0,0 +1,95 @@
/*
* Copyright (c) 2009 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.io.stubs;
import org.broadinstitute.sting.utils.cmdLine.*;
import org.broadinstitute.sting.utils.StingException;
import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
import java.io.OutputStream;
import java.io.File;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
/**
* Insert an OutputStreamStub instead of a full-fledged concrete OutputStream implementations.
*/
public class OutputStreamArgumentTypeDescriptor extends ArgumentTypeDescriptor {
/**
* The engine into which output stubs should be fed.
*/
private GenomeAnalysisEngine engine;
/**
* Create a new OutputStream argument, notifying the given engine when that argument has been created.
* @param engine Engine to add SAMFileWriter output to.
*/
public OutputStreamArgumentTypeDescriptor( GenomeAnalysisEngine engine ) {
this.engine = engine;
}
@Override
public boolean supports( Class type ) {
return getConstructorForClass(type) != null;
}
@Override
public Object parse( ArgumentSource source, Class type, ArgumentMatches matches ) {
ArgumentDefinition definition = createDefaultArgumentDefinition(source);
String fileName = getArgumentValue( definition, matches );
OutputStreamStub stub = new OutputStreamStub(new File(fileName));
engine.addOutput(stub);
try {
return getConstructorForClass(type).newInstance(stub);
}
catch( InstantiationException ex ) {
throw new StingException("Could not instantiate class with OutputStream constructor: " + type.getName());
}
catch( IllegalAccessException ex ) {
throw new StingException("Could not access class with OutputStream constructor: " + type.getName());
}
catch( InvocationTargetException ex ) {
throw new StingException("Could not invoke constructor for class with OutputStream constructor: " + type.getName());
}
}
/**
* Retrieves the constructor for an object that takes exactly one argument: an output stream.
* @param type Type for which to go constructor spelunking.
* @return Constructor, if available. Null, if not.
*/
private Constructor<OutputStream> getConstructorForClass( Class type ) {
try {
return type.getConstructor( OutputStream.class );
}
catch( NoSuchMethodException ex ) {
return null;
}
}
}

138
java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamStub.java 100644
View File

@ -0,0 +1,138 @@
/*
* Copyright (c) 2009 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.io.stubs;
import org.broadinstitute.sting.gatk.io.OutputTracker;
import java.io.OutputStream;
import java.io.IOException;
import java.io.File;
/**
* A stub for routing and management of anything backed by an OutputStream.
*
* @author mhanna
* @version 0.1
*/
public class OutputStreamStub extends OutputStream implements Stub<OutputStream> {
/**
* The file that this stub should write to. Should be passed along to
* whatever happens to create storage for this stub. Might be null, if
* this stub connects directly to an existing stream.
*/
private final File targetFile;
/**
* The stream that this stub should write to. Should be passed along to
* whatever happens to create storage for this stub. Might be null, if
* this stub connects directly to an existing stream.
*/
private final OutputStream targetStream;
/**
* Connects this stub with an external stream capable of serving the
* requests of the consumer of this stub.
*/
private OutputTracker outputTracker = null;
/**
* Specify that this target output stream should write to the given file.
* @param targetFile Target file to which to write. Should not be null.
*/
public OutputStreamStub( File targetFile ) {
this.targetFile = targetFile;
this.targetStream = null;
}
/**
* Specify that this target output stream should write to the given stream.
* @param targetStream Target stream to which to write. Should not be null.
*/
public OutputStreamStub( OutputStream targetStream ) {
this.targetFile = null;
this.targetStream = targetStream;
}
/**
* Return the target file to which this data should be written.
* @return Target file. No sanity checking will have been performed by the file object.
*/
public File getOutputFile() {
return targetFile;
}
/**
* Return the target stream to which this data should be written.
* @return Target stream. No sanity checking will have been performed by the file object.
*/
public OutputStream getOutputStream() {
return targetStream;
}
/**
* Registers the given streamConnector with this stub.
* @param outputTracker The connector used to provide an appropriate stream.
*/
public void register( OutputTracker outputTracker ) {
this.outputTracker = outputTracker;
}
/**
* @{inheritDoc}
*/
public void flush() throws IOException {
outputTracker.getStorage(this).close();
}
/**
* @{inheritDoc}
*/
public void close() throws IOException {
outputTracker.getStorage(this).close();
}
/**
* @{inheritDoc}
*/
public void write( byte[] b ) throws IOException {
outputTracker.getStorage(this).write(b);
}
/**
* @{inheritDoc}
*/
public void write( byte[] b, int off, int len ) throws IOException {
outputTracker.getStorage(this).write(b, off, len);
}
/**
* @{inheritDoc}
*/
public void write( int b ) throws IOException {
outputTracker.getStorage(this).write(b);
}
}

79
java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileReaderArgumentTypeDescriptor.java 100644
View File

@ -0,0 +1,79 @@
/*
* Copyright (c) 2009 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.io.stubs;
import org.broadinstitute.sting.utils.cmdLine.ArgumentTypeDescriptor;
import org.broadinstitute.sting.utils.cmdLine.ArgumentSource;
import org.broadinstitute.sting.utils.cmdLine.ArgumentMatches;
import org.broadinstitute.sting.utils.sam.SAMFileReaderBuilder;
import org.broadinstitute.sting.utils.StingException;
import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
import net.sf.samtools.SAMFileReader;
import java.io.File;
/**
* Describe how to parse SAMFileReaders.
*/
public class SAMFileReaderArgumentTypeDescriptor extends ArgumentTypeDescriptor {
/**
* The engine into which output stubs should be fed.
*/
private GenomeAnalysisEngine engine;
/**
* Create a new SAMFileReader argument, notifying the given engine when that argument has been created.
* @param engine
*/
public SAMFileReaderArgumentTypeDescriptor( GenomeAnalysisEngine engine ) {
this.engine = engine;
}
@Override
public boolean supports( Class type ) {
return SAMFileReader.class.isAssignableFrom(type);
}
@Override
public Object parse( ArgumentSource source, Class type, ArgumentMatches matches ) {
SAMFileReaderBuilder builder = new SAMFileReaderBuilder();
String readerFileName = getArgumentValue( createDefaultArgumentDefinition(source), matches );
if( readerFileName == null )
throw new StingException("SAM file compression was supplied, but no associated writer was supplied with it.");
builder.setSAMFile(new File(readerFileName));
engine.addInput(source, builder);
// MASSIVE KLUDGE! SAMFileReader is tricky to implement and we don't yet have a stub. Return null, then
// let the output tracker load it in.
// TODO: Add a stub for SAMFileReader.
return null;
}
}

121
java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java 100644
View File

@ -0,0 +1,121 @@
/*
* Copyright (c) 2009 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.io.stubs;
import org.broadinstitute.sting.utils.cmdLine.*;
import org.broadinstitute.sting.utils.StingException;
import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
import net.sf.samtools.SAMFileWriter;
import java.util.List;
import java.util.Arrays;
import java.io.File;
/**
* Insert a SAMFileWriterStub instead of a full-fledged concrete OutputStream implementations.
*/
public class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor {
private static final String COMPRESSION_FULLNAME = "bam_compression";
private static final String COMPRESSION_SHORTNAME = "compress";
/**
* The engine into which output stubs should be fed.
*/
private GenomeAnalysisEngine engine;
/**
* Create a new SAMFileWriter argument, notifying the given engine when that argument has been created.
* @param engine Engine to add SAMFileWriter output to.
*/
public SAMFileWriterArgumentTypeDescriptor( GenomeAnalysisEngine engine ) {
this.engine = engine;
}
@Override
public boolean supports( Class type ) {
return SAMFileWriter.class.isAssignableFrom(type);
}
@Override
public List<ArgumentDefinition> createArgumentDefinitions( ArgumentSource source ) {
return Arrays.asList( createBAMArgumentDefinition(source),
createBAMCompressionArgumentDefinition(source) );
}
@Override
public Object parse( ArgumentSource source, Class type, ArgumentMatches matches ) {
String writerFileName = getArgumentValue( createBAMArgumentDefinition(source), matches );
if( writerFileName == null )
throw new StingException("SAM file compression was supplied, but not associated writer was supplied with it.");
SAMFileWriterStub stub = new SAMFileWriterStub(engine, new File(writerFileName));
String compressionLevelText = getArgumentValue( createBAMCompressionArgumentDefinition(source), matches );
Integer compressionLevel = compressionLevelText != null ? Integer.valueOf(compressionLevelText) : null;
if( compressionLevel != null )
stub.setCompressionLevel(compressionLevel);
engine.addOutput(stub);
return stub;
}
/**
* Gets the definition of the argument representing the BAM file itself.
* @param source Argument source for the BAM file. Must not be null.
* @return Argument definition for the BAM file itself. Will not be null.
*/
private ArgumentDefinition createBAMArgumentDefinition(ArgumentSource source) {
Argument description = this.getArgumentDescription(source);
String fullName = description.fullName().trim().length() > 0 ? description.fullName().trim() : "outputBAM";
String shortName = description.shortName().trim().length() > 0 ? description.shortName().trim() : "ob";
return new ArgumentDefinition( source,
fullName,
shortName,
getDoc(source),
isRequired(source),
getExclusiveOf(source),
getValidationRegex(source) );
}
/**
* Creates the optional compression level argument for the BAM file.
* @param source Argument source for the BAM file. Must not be null.
* @return Argument definition for the BAM file itself. Will not be null.
*/
private ArgumentDefinition createBAMCompressionArgumentDefinition(ArgumentSource source) {
return new ArgumentDefinition( source,
COMPRESSION_FULLNAME,
COMPRESSION_SHORTNAME,
"Compression level to use for writing BAM files",
false,
null,
null );
}
}

130
java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterStub.java 100644
View File

@ -0,0 +1,130 @@
/*
* Copyright (c) 2009 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.io.stubs;
import net.sf.samtools.SAMFileWriter;
import net.sf.samtools.SAMRecord;
import net.sf.samtools.SAMFileHeader;
import java.io.File;
import org.broadinstitute.sting.gatk.io.OutputTracker;
import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
/**
* A stub for routing and management of SAM file reading and writing.
*
* @author mhanna
* @version 0.1
*/
public class SAMFileWriterStub implements Stub<SAMFileWriter>, SAMFileWriter {
/**
* Engine to use for collecting attributes for the output SAM file.
*/
private final GenomeAnalysisEngine engine;
/**
* The sam file that this stub should write to. Should be passed along to
* whatever happens to create the StreamConnector.
*/
private final File samFile;
/**
* The validation stringency to apply when reading this file.
*/
private Integer compressionLevel = null;
/**
* Connects this stub with an external stream capable of serving the
* requests of the consumer of this stub.
*/
private OutputTracker outputTracker = null;
/**
* Create a new stub given the requested SAM file and compression level.
* @param engine source of header data, maybe other data about input files.
* @param samFile SAM file to (ultimately) cerate.
*/
public SAMFileWriterStub( GenomeAnalysisEngine engine, File samFile ) {
this.engine = engine;
this.samFile = samFile;
}
/**
* Retrieves the SAM file to (ultimately) be created.
* @return The SAM file. Must not be null.
*/
public File getSAMFile() {
return samFile;
}
/**
* Retrieves the header to use when creating the new SAM file.
* @return header to use when creating the new SAM file.
*/
public SAMFileHeader getSAMFileHeader() {
return engine.getSAMFileHeader();
}
/**
* Retrieves the desired compression level for
* @return The current compression level. Could be null if the user doesn't care.
*/
public Integer getCompressionLevel() {
return compressionLevel;
}
/**
* Sets the desired compression level.
* @param compressionLevel The suggested compression level.
*/
public void setCompressionLevel( Integer compressionLevel ) {
this.compressionLevel = compressionLevel;
}
/**
* Registers the given streamConnector with this stub.
* @param outputTracker The connector used to provide an appropriate stream.
*/
public void register( OutputTracker outputTracker ) {
this.outputTracker = outputTracker;
}
/**
* @{inheritDoc}
*/
public void addAlignment( SAMRecord alignment ) {
outputTracker.getStorage(this).addAlignment(alignment);
}
/**
* @{inheritDoc}
*/
public void close() {
outputTracker.getStorage(this).close();
}
}

46
java/src/org/broadinstitute/sting/gatk/io/stubs/Stub.java 100644
View File

@ -0,0 +1,46 @@
/*
* Copyright (c) 2009 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.io.stubs;
import org.broadinstitute.sting.gatk.io.OutputTracker;
/**
* A stub used for managing IO. Acts as a proxy for IO streams
* not yet created or streams that need significant external
* management.
*
* @author mhanna
* @version 0.1
*/
public interface Stub<StreamType> {
/**
* Provides a facility to register this stream with the given
* StreamConnector. The stub should route each output method
* to the specified connector.
* @param outputTracker The connector used to provide an appropriate stream.
*/
public void register( OutputTracker outputTracker );
}

12
java/src/org/broadinstitute/sting/utils/Utils.java
View File

@ -43,6 +43,18 @@ public class Utils {
throw new StingException(msg);
}
/**
* Compares two objects, either of which might be null.
* @param lhs One object to compare.
* @param rhs The other object to compare.
* @return True if the two objects are equal, false otherwise.
*/
public static boolean equals(Object lhs, Object rhs) {
if( lhs == null && rhs == null ) return true;
else if( lhs == null ) return false;
else return lhs.equals(rhs);
}
public static <T> List<T> cons(final T elt, final List<T> l) {
List<T> l2 = new ArrayList<T>();
l2.add(elt);

116
java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentDefinition.java 100644
View File

@ -0,0 +1,116 @@
/*
* Copyright (c) 2009 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.utils.cmdLine;
import org.broadinstitute.sting.utils.Utils;
/**
* A specific argument definition. Maps one-to-one with a field in some class.
*/
public class ArgumentDefinition {
/**
* Full name of the argument. Must have a value.
*/
public final String fullName;
/**
* Short name of the argument. Can be null.
*/
public final String shortName;
/**
* Doc string for the argument. Displayed in help.
*/
public final String doc;
/**
* Is this argument required?
*/
public final boolean required;
/**
* Is this argument exclusive of other arguments?
*/
public final String exclusiveOf;
/**
* Can we validate this regular expression?
*/
public final String validation;
/**
* The target into which to inject arguments meeting this definition.
*/
public final ArgumentSource source;
/**
* Creates a new argument definition.
* @param source Source information for defining the argument.
*/
public ArgumentDefinition( ArgumentSource source,
String fullName,
String shortName,
String doc,
boolean required,
String exclusiveOf,
String validation ) {
this.source = source;
this.fullName = fullName;
this.shortName = shortName;
this.doc = doc;
this.required = required;
this.exclusiveOf = exclusiveOf;
this.validation = validation;
}
@Override
public int hashCode() {
int hashCode = fullName.hashCode();
if(shortName != null) hashCode ^= shortName.hashCode();
if(doc != null) hashCode ^= doc.hashCode();
hashCode ^= Boolean.valueOf(required).hashCode();
if(exclusiveOf != null) hashCode ^= exclusiveOf.hashCode();
if(validation != null) hashCode ^= validation.hashCode();
return hashCode;
}
public boolean equals( Object o ) {
if( o == null )
return false;
if( !(o instanceof ArgumentDefinition) )
return false;
ArgumentDefinition other = (ArgumentDefinition)o;
return Utils.equals(fullName,other.fullName) &&
Utils.equals(shortName,other.shortName) &&
Utils.equals(doc,other.doc) &&
required == other.required &&
Utils.equals(exclusiveOf,other.exclusiveOf) &&
Utils.equals(validation,other.validation);
}
}

61
java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentDefinitions.java
View File

@ -227,67 +227,6 @@ class ArgumentDefinitionGroup implements Iterable<ArgumentDefinition> {
}
}
/**
* A specific argument definition. Maps one-to-one with a field in some class.
*/
class ArgumentDefinition {
/**
* Full name of the argument. Must have a value.
*/
public final String fullName;
/**
* Short name of the argument. Can be null.
*/
public final String shortName;
/**
* Doc string for the argument. Displayed in help.
*/
public final String doc;
/**
* Is this argument required?
*/
public final boolean required;
/**
* Is this argument exclusive of other arguments?
*/
public final String exclusiveOf;
/**
* Can we validate this regular expression?
*/
public final String validation;
/**
* The target into which to inject arguments meeting this definition.
*/
public final ArgumentSource source;
/**
* Creates a new argument definition.
* @param source Source information for defining the argument.
*/
public ArgumentDefinition( ArgumentSource source,
String fullName,
String shortName,
String doc,
boolean required,
String exclusiveOf,
String validation ) {
this.source = source;
this.fullName = fullName;
this.shortName = shortName;
this.doc = doc;
this.required = required;
this.exclusiveOf = exclusiveOf;
this.validation = validation;
}
}
/**
* A Comparator-esque interface for finding argument definitions within a collection.
*/

62
java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentMatches.java
View File

@ -31,7 +31,7 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
* Provide a place to put command-line argument values that don't seem to belong to
* any particular command-line option.
*/
public ArgumentMatch MissingArgument = new ArgumentMatch();
ArgumentMatch MissingArgument = new ArgumentMatch();
/**
* Get an iterator cycling through *unique* command-line argument <-> definition matches.
@ -41,12 +41,34 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
return getUniqueMatches().iterator();
}
/**
* Create an empty ArgumentMatches object.
*/
public ArgumentMatches() {
}
/**
* Create a singleton ArgumentMatches object.
* @param match Match to incorporate.
*/
public ArgumentMatches( ArgumentMatch match ) {
mergeInto( match );
}
/**
* Returns the number of matches in this structure.
* @return Count of the matches in this structure.
*/
public int size() {
return argumentMatches.size();
}
/**
* Indicates whether the site contains a matched argument.
* @param site Site at which to check.
* @return True if the site has a match. False otherwise.
*/
public boolean hasMatch( int site ) {
boolean hasMatch( int site ) {
return argumentMatches.containsKey( site );
}
@ -56,7 +78,7 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
* @return The match present at the given site.
* @throws IllegalArgumentException if site does not contain a match.
*/
public ArgumentMatch getMatch( int site ) {
ArgumentMatch getMatch( int site ) {
if( !argumentMatches.containsKey(site) )
throw new IllegalArgumentException( "Site does not contain an argument: " + site );
return argumentMatches.get(site);
@ -67,7 +89,7 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
* @param definition Definition to match.
* @return True if a match exists; false otherwise.
*/
public boolean hasMatch( ArgumentDefinition definition ) {
boolean hasMatch( ArgumentDefinition definition ) {
return findMatches( definition ).size() > 0;
}
@ -77,11 +99,11 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
* @return List of all matches.
*/
public Collection<ArgumentMatch> findMatches( ArgumentSource argumentSource ) {
Collection<ArgumentMatch> matches = new HashSet<ArgumentMatch>();
ArgumentMatches findMatches( ArgumentSource argumentSource ) {
ArgumentMatches matches = new ArgumentMatches();
for( ArgumentMatch argumentMatch: getUniqueMatches() ) {
if( argumentMatch.definition != null && argumentMatch.definition.source.equals( argumentSource ) )
matches.add( argumentMatch );
matches.mergeInto( argumentMatch );
}
return matches;
}
@ -91,23 +113,23 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
* @param definition Argument definition to match.
* @return List of all matches.
*/
public Collection<ArgumentMatch> findMatches( ArgumentDefinition definition ) {
Collection<ArgumentMatch> matches = new HashSet<ArgumentMatch>();
for( ArgumentMatch argumentMatch: getUniqueMatches() ) {
ArgumentMatches findMatches( ArgumentDefinition definition ) {
ArgumentMatches matches = new ArgumentMatches();
for( ArgumentMatch argumentMatch: argumentMatches.values() ) {
if( argumentMatch.definition == definition )
matches.add( argumentMatch );
}
matches.mergeInto( argumentMatch );
}
return matches;
}
/**
* Find all successful matches (a 'successful' match is one paired with a definition).
*/
public Collection<ArgumentMatch> findSuccessfulMatches() {
Collection<ArgumentMatch> matches = new HashSet<ArgumentMatch>();
for( ArgumentMatch argumentMatch: getUniqueMatches() ) {
ArgumentMatches findSuccessfulMatches() {
ArgumentMatches matches = new ArgumentMatches();
for( ArgumentMatch argumentMatch: argumentMatches.values() ) {
if( argumentMatch.definition != null )
matches.add( argumentMatch );
matches.mergeInto( argumentMatch );
}
return matches;
}
@ -116,11 +138,11 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
* Find arguments that are unmatched to any definition.
* @return Set of matches that have no associated definition.
*/
public Collection<ArgumentMatch> findUnmatched() {
Collection<ArgumentMatch> matches = new HashSet<ArgumentMatch>();
for( ArgumentMatch argumentMatch: getUniqueMatches() ) {
ArgumentMatches findUnmatched() {
ArgumentMatches matches = new ArgumentMatches();
for( ArgumentMatch argumentMatch: argumentMatches.values() ) {
if( argumentMatch.definition == null )
matches.add( argumentMatch );
matches.mergeInto( argumentMatch );
}
return matches;
}

13
java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentSource.java
View File

@ -30,7 +30,6 @@ import org.broadinstitute.sting.utils.StingException;
import java.lang.reflect.Field;
import java.util.Collection;
import java.util.List;
import java.util.Collections;
/**
* Describes the source field which defines a command-line argument.
@ -51,11 +50,6 @@ public class ArgumentSource {
*/
public final Field field;
/**
* Descriptor for the argument. Contains name, validation info, etc.
*/
public final Argument descriptor;
/**
* Create a new command-line argument target.
* @param clazz Class containing the argument.
@ -64,9 +58,6 @@ public class ArgumentSource {
public ArgumentSource( Class clazz, Field field ) {
this.clazz = clazz;
this.field = field;
this.descriptor = field.getAnnotation(Argument.class);
if( descriptor == null )
throw new StingException("Cannot build out a command-line argument without a descriptor.");
}
/**
@ -101,7 +92,7 @@ public class ArgumentSource {
*/
public List<ArgumentDefinition> createArgumentDefinitions() {
ArgumentTypeDescriptor typeDescriptor = ArgumentTypeDescriptor.create( field.getType() );
return typeDescriptor.createArgumentDefinitions( this, descriptor );
return typeDescriptor.createArgumentDefinitions( this );
}
/**
@ -109,7 +100,7 @@ public class ArgumentSource {
* @param targetInstance Instance into which to inject the parsed value.
* @param values String representation of all values passed.
*/
public Object parse( ArgumentSource source, Object targetInstance, ArgumentMatch... values ) {
public Object parse( ArgumentSource source, Object targetInstance, ArgumentMatches values ) {
Object value = null;
if( !isFlag() ) {
ArgumentTypeDescriptor typeDescriptor = ArgumentTypeDescriptor.create( field.getType() );

257
java/src/org/broadinstitute/sting/utils/cmdLine/ArgumentTypeDescriptor.java
View File

@ -26,19 +26,13 @@
package org.broadinstitute.sting.utils.cmdLine;
import org.broadinstitute.sting.utils.StingException;
import org.broadinstitute.sting.utils.sam.SAMFileWriterBuilder;
import org.broadinstitute.sting.utils.sam.SAMFileReaderBuilder;
import org.apache.log4j.Logger;
import java.lang.reflect.*;
import java.util.*;
import java.io.File;
import net.sf.samtools.SAMFileWriter;
import net.sf.samtools.SAMFileReader;
/**
* An factory capable of providing parsers that can parse any type
* An descriptor capable of providing parsers that can parse any type
* of supported command-line argument.
*
* @author mhanna
@ -53,11 +47,23 @@ public abstract class ArgumentTypeDescriptor {
/**
* Class reference to the different types of descriptors that the create method can create.
* The type of set used must be ordered (but not necessarily sorted).
*/
private static List<ArgumentTypeDescriptor> descriptors = Arrays.asList( new SAMFileReaderArgumentTypeDescriptor(),
new SAMFileWriterArgumentTypeDescriptor(),
new SimpleArgumentTypeDescriptor(),
new CompoundArgumentTypeDescriptor() );
private static Set<ArgumentTypeDescriptor> descriptors = new LinkedHashSet<ArgumentTypeDescriptor>( Arrays.asList(new SimpleArgumentTypeDescriptor(),
new CompoundArgumentTypeDescriptor()) );
/**
* Adds new, user defined descriptors to the head of the descriptor list.
* @param argumentTypeDescriptors New descriptors to add. List can be empty, but should not be null.
*/
public static void addDescriptors( Collection<ArgumentTypeDescriptor> argumentTypeDescriptors ) {
// We care about ordering; newly added descriptors should have priority over stock descriptors.
// Enforce this by creating a new *ordered* set, adding the new descriptors, then adding the old descriptors.
Set<ArgumentTypeDescriptor> allDescriptors = new LinkedHashSet<ArgumentTypeDescriptor>();
allDescriptors.addAll( argumentTypeDescriptors );
allDescriptors.addAll( descriptors );
descriptors = allDescriptors;
}
public static ArgumentTypeDescriptor create( Class type ) {
for( ArgumentTypeDescriptor descriptor: descriptors ) {
@ -77,32 +83,41 @@ public abstract class ArgumentTypeDescriptor {
/**
* Given the given argument source and attributes, synthesize argument definitions for command-line arguments.
* @param source Source class and field for the given argument.
* @param description Description of the fields that go into a given argument.
* @return A list of command-line argument definitions supporting this field.
*/
public List<ArgumentDefinition> createArgumentDefinitions( ArgumentSource source, Argument description ) {
ArgumentDefinition definition = new ArgumentDefinition( source,
getFullName( source, description ),
getShortName( source, description ),
getDoc( source, description ),
isRequired( source, description ),
getExclusiveOf( source, description ),
getValidationRegex( source, description ) );
return Collections.singletonList(definition);
}
public Object parse( ArgumentSource source, ArgumentMatch... values ) {
return parse( source, source.field.getType(), values );
public List<ArgumentDefinition> createArgumentDefinitions( ArgumentSource source ) {
return Collections.singletonList(createDefaultArgumentDefinition(source));
}
protected abstract Object parse( ArgumentSource source, Class type, ArgumentMatch... values );
public Object parse( ArgumentSource source, ArgumentMatches matches ) {
return parse( source, source.field.getType(), matches );
}
/**
* By default, argument sources create argument definitions with a set of default values.
* Use this method to create the one simple argument definition.
* @param source argument source for which to create a default definition.
* @return The default definition for this argument source.
*/
protected ArgumentDefinition createDefaultArgumentDefinition( ArgumentSource source ) {
return new ArgumentDefinition( source,
getFullName(source),
getShortName(source),
getDoc(source),
isRequired(source),
getExclusiveOf(source),
getValidationRegex(source) );
}
protected abstract Object parse( ArgumentSource source, Class type, ArgumentMatches matches );
/**
* Retrieves the full name of the argument, specifiable with the '--' prefix. The full name can be
* either specified explicitly with the fullName annotation parameter or implied by the field name.
* @return full name of the argument. Never null.
*/
protected String getFullName( ArgumentSource source, Argument description ) {
protected String getFullName( ArgumentSource source ) {
Argument description = getArgumentDescription(source);
return description.fullName().trim().length() > 0 ? description.fullName().trim() : source.field.getName().toLowerCase();
}
@ -111,7 +126,8 @@ public abstract class ArgumentTypeDescriptor {
* be specified or not; if left unspecified, no short name will be present.
* @return short name of the argument. Null if no short name exists.
*/
protected String getShortName( ArgumentSource source, Argument description ) {
protected String getShortName( ArgumentSource source ) {
Argument description = getArgumentDescription(source);
return description.shortName().trim().length() > 0 ? description.shortName().trim() : null;
}
@ -119,7 +135,8 @@ public abstract class ArgumentTypeDescriptor {
* Documentation for this argument. Mandatory field.
* @return Documentation for this argument.
*/
protected String getDoc( ArgumentSource source, Argument description ) {
protected String getDoc( ArgumentSource source ) {
Argument description = getArgumentDescription(source);
return description.doc();
}
@ -127,7 +144,8 @@ public abstract class ArgumentTypeDescriptor {
* Returns whether this field is required. Note that flag fields are always forced to 'not required'.
* @return True if the field is mandatory and not a boolean flag. False otherwise.
*/
protected boolean isRequired( ArgumentSource source, Argument description ) {
protected boolean isRequired( ArgumentSource source ) {
Argument description = getArgumentDescription(source);
return description.required() && !source.isFlag();
}
@ -135,7 +153,8 @@ public abstract class ArgumentTypeDescriptor {
* Specifies other arguments which cannot be used in conjunction with tihs argument. Comma-separated list.
* @return A comma-separated list of exclusive arguments, or null if none are present.
*/
protected String getExclusiveOf( ArgumentSource source, Argument description ) {
protected String getExclusiveOf( ArgumentSource source ) {
Argument description = getArgumentDescription(source);
return description.exclusiveOf().trim().length() > 0 ? description.exclusiveOf().trim() : null;
}
@ -143,9 +162,51 @@ public abstract class ArgumentTypeDescriptor {
* A regular expression which can be used for validation.
* @return a JVM regex-compatible regular expression, or null to permit any possible value.
*/
protected String getValidationRegex( ArgumentSource source, Argument description ) {
protected String getValidationRegex( ArgumentSource source ) {
Argument description = getArgumentDescription(source);
return description.validation().trim().length() > 0 ? description.validation().trim() : null;
}
/**
* Gets the value of an argument with the given full name, from the collection of ArgumentMatches.
* If the argument matches multiple values, an exception will be thrown.
* @param definition Definition of the argument for which to find matches.
* @param matches The matches for the given argument.
* @return The value of the argument if available, or null if not present.
*/
protected String getArgumentValue( ArgumentDefinition definition, ArgumentMatches matches ) {
Collection<String> argumentValues = getArgumentValues( definition, matches );
if( argumentValues.size() > 1 )
throw new StingException("Multiple values associated with given definition, but this argument expects only one: " + definition.fullName);
return argumentValues.size() > 0 ? argumentValues.iterator().next() : null;
}
/**
* Gets the values of an argument with the given full name, from the collection of ArgumentMatches.
* @param definition Definition of the argument for which to find matches.
* @param matches The matches for the given argument.
* @return The value of the argument if available, or an empty collection if not present.
*/
protected Collection<String> getArgumentValues( ArgumentDefinition definition, ArgumentMatches matches ) {
Collection<String> values = new ArrayList<String>();
for( ArgumentMatch match: matches ) {
if( match.definition.equals(definition) )
values.addAll(match.values());
}
return values;
}
/**
* Retrieves the argument description from the given argument source. Will throw an exception if
* the given ArgumentSource
* @param source source of the argument.
* @return Argument description annotation associated with the given field.
*/
protected Argument getArgumentDescription( ArgumentSource source ) {
if( !source.field.isAnnotationPresent(Argument.class) )
throw new StingException("ArgumentAnnotation is not present for the argument field: " + source.field.getName());
return source.field.getAnnotation(Argument.class);
}
}
/**
@ -171,10 +232,8 @@ class SimpleArgumentTypeDescriptor extends ArgumentTypeDescriptor {
}
@Override
protected Object parse( ArgumentSource source, Class type, ArgumentMatch... matches ) {
if( matches.length > 1 || matches[0].values().size() > 1 )
throw new StingException("Simple argument parser is unable to parse multiple arguments.");
String value = matches[0].values().get(0);
protected Object parse( ArgumentSource source, Class type, ArgumentMatches matches ) {
String value = getArgumentValue( createDefaultArgumentDefinition(source), matches );
// lets go through the types we support
try {
@ -228,14 +287,10 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor {
}
@Override
public Object parse( ArgumentSource source, Class type, ArgumentMatch... matches )
public Object parse( ArgumentSource source, Class type, ArgumentMatches matches )
{
Class componentType = null;
if( matches.length > 1 )
throw new StingException("Simple argument parser is unable to combine multiple argument types into a compound argument.");
ArgumentMatch match = matches[0];
if( Collection.class.isAssignableFrom(type) ) {
// If this is a generic interface, pick a concrete implementation to create and pass back.
@ -272,8 +327,10 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor {
throw new StingException("constructFromString:IllegalAccessException: Failed conversion " + e.getMessage());
}
for( ArgumentMatch value: match )
collection.add( componentArgumentParser.parse(source,componentType,value) );
for( ArgumentMatch match: matches ) {
for( ArgumentMatch value: match )
collection.add( componentArgumentParser.parse(source,componentType,new ArgumentMatches(value)) );
}
return collection;
@ -281,11 +338,19 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor {
else if( type.isArray() ) {
componentType = type.getComponentType();
ArgumentTypeDescriptor componentArgumentParser = ArgumentTypeDescriptor.create( componentType );
Object arr = Array.newInstance(componentType,match.values().size());
// Assemble a collection of individual values used in this computation.
Collection<ArgumentMatch> values = new ArrayList<ArgumentMatch>();
for( ArgumentMatch match: matches ) {
for( ArgumentMatch value: match )
values.add(value);
}
Object arr = Array.newInstance(componentType,values.size());
int i = 0;
for( ArgumentMatch value: match )
Array.set( arr,i++,componentArgumentParser.parse(source,componentType,value));
for( ArgumentMatch value: values )
Array.set( arr,i++,componentArgumentParser.parse(source,componentType,new ArgumentMatches(value)));
return arr;
}
@ -293,103 +358,3 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor {
throw new StingException("Unsupported compound argument type: " + type);
}
}
/**
* Handle SAMFileReaders.
*/
class SAMFileReaderArgumentTypeDescriptor extends ArgumentTypeDescriptor {
@Override
public boolean supports( Class type ) {
return SAMFileReader.class.isAssignableFrom(type);
}
@Override
public Object parse( ArgumentSource source, Class type, ArgumentMatch... matches ) {
if( matches.length > 1 )
throw new UnsupportedOperationException("Only an input file name and validation stringency can be supplied when creating a BAM file reader.");
SAMFileReaderBuilder builder = new SAMFileReaderBuilder();
ArgumentMatch readerMatch = matches[0];
if( readerMatch == null )
throw new StingException("SAM file compression was supplied, but not associated writer was supplied with it.");
if( readerMatch.values().size() > 1 )
throw new StingException("Only one filename can be supplied per created BAM file");
builder.setSAMFile(new File(readerMatch.values().get(0).trim()));
return builder;
}
}
/**
* Handle SAMFileWriters.
*/
class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor {
private static final String COMPRESSION_FULLNAME = "bam_compression";
private static final String COMPRESSION_SHORTNAME = "compress";
@Override
public boolean supports( Class type ) {
return SAMFileWriter.class.isAssignableFrom(type);
}
@Override
public List<ArgumentDefinition> createArgumentDefinitions( ArgumentSource source, Argument description ) {
String fullName = description.fullName().trim().length() > 0 ? description.fullName().trim() : "outputBAM";
String shortName = description.shortName().trim().length() > 0 ? description.shortName().trim() : "ob";
ArgumentDefinition writerDefinition = new ArgumentDefinition( source,
fullName,
shortName,
getDoc( source, description ),
isRequired( source, description ),
getExclusiveOf( source, description ),
getValidationRegex( source, description ) );
ArgumentDefinition compressionDefinition = new ArgumentDefinition( source,
COMPRESSION_FULLNAME,
COMPRESSION_SHORTNAME,
"Compression level to use for writing BAM files",
false,
null,
null );
return Arrays.asList( writerDefinition, compressionDefinition );
}
@Override
public Object parse( ArgumentSource source, Class type, ArgumentMatch... matches ) {
if( matches.length > 2 )
throw new UnsupportedOperationException("Only an input file name and validation stringency can be supplied when creating a BAM file reader.");
SAMFileWriterBuilder builder = new SAMFileWriterBuilder();
ArgumentMatch writerMatch = null;
ArgumentMatch compressionMatch = null;
for( ArgumentMatch match: matches ) {
if( match.definition.fullName.equals(COMPRESSION_FULLNAME) )
compressionMatch = match;
else
writerMatch = match;
}
if( writerMatch == null )
throw new StingException("SAM file compression was supplied, but not associated writer was supplied with it.");
if( writerMatch.values().size() > 1 )
throw new StingException("Only one filename can be supplied per created BAM file");
builder.setSAMFile(new File(writerMatch.values().get(0).trim()));
if( compressionMatch != null ) {
if( compressionMatch.values().size() > 1 )
throw new StingException("Only one value can be supplied for BAM compression");
int compressionLevel = Integer.valueOf(compressionMatch.values().get(0));
builder.setCompressionLevel(compressionLevel);
}
return builder;
}
}

20
java/src/org/broadinstitute/sting/utils/cmdLine/CommandLineProgram.java
View File

@ -7,6 +7,8 @@ import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.EnumSet;
import java.util.Enumeration;
import java.util.Collections;
import java.util.Collection;
/**
* User: aaron
@ -95,6 +97,14 @@ public abstract class CommandLineProgram {
ApplicationDetails.createDefaultRunningInstructions(getClass()) );
}
/**
* Subclasses of CommandLinePrograms can provide their own types of command-line arguments.
* @return A collection of type descriptors generating implementation-dependent placeholders.
*/
protected Collection<ArgumentTypeDescriptor> getArgumentTypeDescriptors() {
return Collections.emptyList();
}
/**
* Will this application want to vary its argument list dynamically?
* If so, parse the command-line options and then prompt the subclass to return
@ -109,16 +119,6 @@ public abstract class CommandLineProgram {
*/
protected Class[] getArgumentSources() { return new Class[] {}; }
/**
* Allows arguments to be hijacked by subclasses of the program before being placed
* into plugin classes.
* @param source Source class for the argument.
* @param targetInstance Instance into which the value should be ultimately injected.
* @param value Value to inject.
* @return True if the particular field has been hijacked; false otherwise.
*/
protected boolean intercept( ArgumentSource source, Object targetInstance, Object value ) { return false; }
/**
* Name this argument source. Provides the (full) class name as a default.
* @param source The argument source.

37
java/src/org/broadinstitute/sting/utils/cmdLine/ParsingEngine.java
View File

@ -61,8 +61,13 @@ public class ParsingEngine {
public ParsingEngine( CommandLineProgram clp ) {
this.clp = clp;
parsingMethods.add( ParsingMethod.FullNameParsingMethod );
parsingMethods.add( ParsingMethod.ShortNameParsingMethod );
// Null check for unit tests. Perhaps we should mock up an empty CLP?
if( clp != null )
ArgumentTypeDescriptor.addDescriptors( clp.getArgumentTypeDescriptors() );
}
/**
@ -172,7 +177,7 @@ public class ParsingEngine {
// Find invalid arguments. Invalid arguments will have a null argument definition.
if( !skipValidationOf.contains(ValidationType.InvalidArgument) ) {
Collection<ArgumentMatch> invalidArguments = argumentMatches.findUnmatched();
ArgumentMatches invalidArguments = argumentMatches.findUnmatched();
if( invalidArguments.size() > 0 )
throw new InvalidArgumentException( invalidArguments );
}
@ -183,7 +188,7 @@ public class ParsingEngine {
argumentDefinitions.findArgumentDefinitions( null, ArgumentDefinitions.VerifiableDefinitionMatcher );
Collection<Pair<ArgumentDefinition,String>> invalidValues = new ArrayList<Pair<ArgumentDefinition,String>>();
for( ArgumentDefinition verifiableArgument: verifiableArguments ) {
Collection<ArgumentMatch> verifiableMatches = argumentMatches.findMatches( verifiableArgument );
ArgumentMatches verifiableMatches = argumentMatches.findMatches( verifiableArgument );
for( ArgumentMatch verifiableMatch: verifiableMatches ) {
for( String value: verifiableMatch.values() ) {
if( !value.matches(verifiableArgument.validation) )
@ -253,15 +258,14 @@ public class ParsingEngine {
* @param argumentMatches Argument matches to load into the object.
* @param target
*/
private void loadMatchesIntoObject( ArgumentSource source, Object target, Collection<ArgumentMatch> argumentMatches ) {
private void loadMatchesIntoObject( ArgumentSource source, Object target, ArgumentMatches argumentMatches ) {
// Nothing to load
if( argumentMatches.size() == 0 )
return;
if( source.clazz.isAssignableFrom(target.getClass()) ) {
Object value = source.parse( source, target, argumentMatches.toArray(new ArgumentMatch[0]) );
if( clp == null || !clp.intercept(source, target, value) )
JVMUtils.setField( source.field, target, value );
Object value = source.parse( source, target, argumentMatches );
JVMUtils.setField( source.field, target, value );
}
}
@ -346,15 +350,32 @@ class MissingArgumentException extends ArgumentException {
}
}
class MissingArgumentValueException extends ArgumentException {
public MissingArgumentValueException( Collection<ArgumentDefinition> missingArguments ) {
super( formatArguments(missingArguments) );
}
private static String formatArguments( Collection<ArgumentDefinition> missingArguments ) {
StringBuilder sb = new StringBuilder();
for( ArgumentDefinition missingArgument: missingArguments ) {
if( missingArgument.shortName != null )
sb.append( String.format("%nValue for argument with name '--%s' (-%s) is missing.", missingArgument.fullName, missingArgument.shortName) );
else
sb.append( String.format("%nValue for argument with name '--%s' is missing.", missingArgument.fullName) );
}
return sb.toString();
}
}
/**
* An exception for undefined arguments.
*/
class InvalidArgumentException extends ArgumentException {
public InvalidArgumentException( Collection<ArgumentMatch> invalidArguments ) {
public InvalidArgumentException( ArgumentMatches invalidArguments ) {
super( formatArguments(invalidArguments) );
}
private static String formatArguments( Collection<ArgumentMatch> invalidArguments ) {
private static String formatArguments( ArgumentMatches invalidArguments ) {
StringBuilder sb = new StringBuilder();
for( ArgumentMatch invalidArgument: invalidArguments )
sb.append( String.format("%nArgument with name '%s' isn't defined.", invalidArgument.label) );

101
java/src/org/broadinstitute/sting/utils/sam/SAMFileWriterBuilder.java
View File

@ -1,101 +0,0 @@
/*
* Copyright (c) 2009 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.utils.sam;
import net.sf.samtools.SAMFileHeader;
import net.sf.samtools.SAMFileWriter;
import net.sf.samtools.SAMFileWriterFactory;
import java.io.File;
import org.broadinstitute.sting.utils.StingException;
/**
* Allows the user to steadily accumulate information about what
* components go into a SAM file writer, ultimately using this
* information to create a SAM file writer on demand.
*
* @author mhanna
* @version 0.1
*/
public class SAMFileWriterBuilder {
/**
* Default compression level for newly constructed SAM files.
* Default to 5 (based on research by Alec Wysoker)
*/
public static final int DEFAULT_COMPRESSION_LEVEL = 5;
/**
* To which file should output be written?
*/
private File samFile = null;
/**
* Which header should be used when writing the SAM file?
*/
private SAMFileHeader header = null;
/**
* What compression level should be used when building this file?
*/
private int compressionLevel = DEFAULT_COMPRESSION_LEVEL;
/**
* Sets the handle of the sam file to which data should be written.
* @param samFile The SAM file into which data should flow.
*/
public void setSAMFile( File samFile ) {
this.samFile = samFile;
}
/**
* Sets the header to be written at the head of this SAM file.
* @param header Header to write.
*/
public void setSAMFileHeader( SAMFileHeader header ) {
this.header = header;
}
/**
* Sets the compression level to use when writing this BAM file.
* @param compressionLevel Compression level to use when writing this SAM file.
*/
public void setCompressionLevel( int compressionLevel ) {
this.compressionLevel = compressionLevel;
}
/**
* Create the SAM writer, given the constituent parts accrued.
* @return Newly minted SAM file writer.
*/
public SAMFileWriter build() {
if( samFile == null )
throw new StingException( "Filename for output sam file must be supplied.");
if( header == null )
throw new StingException( "Header for output sam file must be supplied.");
return new SAMFileWriterFactory().makeBAMWriter( header, true, samFile, compressionLevel );
}
}

61
java/test/org/broadinstitute/sting/gatk/OutputTrackerTest.java → java/test/org/broadinstitute/sting/gatk/io/OutputTrackerTest.java
View File

@ -1,10 +1,13 @@
package org.broadinstitute.sting.gatk;
package org.broadinstitute.sting.gatk.io;
import org.junit.Test;
import org.junit.After;
import org.junit.Assert;
import org.broadinstitute.sting.utils.io.RedirectingOutputStream;
import org.broadinstitute.sting.BaseTest;
import org.broadinstitute.sting.gatk.io.OutputTracker;
import org.broadinstitute.sting.gatk.io.DirectOutputTracker;
import org.broadinstitute.sting.gatk.io.stubs.OutputStreamStub;
import java.io.File;
import java.io.FileNotFoundException;
@ -42,26 +45,28 @@ public class OutputTrackerTest extends BaseTest {
@Test
public void testNullInputs() {
OutputTracker ot = new OutputTracker();
OutputTracker ot = new DirectOutputTracker();
ot.initializeCoreIO(null,null);
Assert.assertTrue("OutputTracker: Output stream is of wrong type.", ot.getOutStream() instanceof RedirectingOutputStream );
Assert.assertTrue("OutputTracker: Error stream is of wrong type.", ot.getErrStream() instanceof RedirectingOutputStream );
Assert.assertNotNull("OutputTracker: Output stream is null.", ot.outStub );
Assert.assertNotNull("OutputTracker: Error stream is null.", ot.errStub );
RedirectingOutputStream outStream = (RedirectingOutputStream)ot.getOutStream();
RedirectingOutputStream errStream = (RedirectingOutputStream)ot.getErrStream();
OutputStreamStub outStream = ot.outStub;
Assert.assertNull("OutputTracker: Output file incorrectly initialized.", outStream.getOutputFile());
Assert.assertSame("OutputTracker: Output stream incorrectly initialized.", System.out, outStream.getOutputStream());
Assert.assertSame("OutputTracker: Output stream incorrectly initialized.", System.out, outStream.getBackingOutputStream());
Assert.assertSame("OutputTracker: Error stream incorrectly initialized.", System.err, errStream.getBackingOutputStream());
OutputStreamStub errStream = ot.errStub;
Assert.assertNull("OutputTracker: Error file incorrectly initialized.", errStream.getOutputFile());
Assert.assertSame("OutputTracker: Error stream incorrectly initialized.", System.err, errStream.getOutputStream());
}
@Test
public void testOutputStreamAlone() throws FileNotFoundException {
OutputTracker ot = new OutputTracker();
OutputTracker ot = new DirectOutputTracker();
ot.initializeCoreIO(OUTPUT_FILENAME,null);
final String OUTPUT_TEXT = "out stream test";
PrintWriter outWriter = new PrintWriter(ot.getOutStream());
PrintWriter outWriter = new PrintWriter(ot.outStub);
outWriter.append(OUTPUT_TEXT);
outWriter.close();
@ -71,18 +76,22 @@ public class OutputTrackerTest extends BaseTest {
Assert.assertEquals("OutputTracker: Written output is incorrect", outText, OUTPUT_TEXT);
Assert.assertTrue("OutputTracker: Error stream is of wrong type.", ot.getErrStream() instanceof RedirectingOutputStream );
RedirectingOutputStream errStream = (RedirectingOutputStream)ot.getErrStream();
Assert.assertSame("OutputTracker: Error stream incorrectly initialized.", System.err, errStream.getBackingOutputStream());
OutputStreamStub errStream = ot.errStub;
Assert.assertNull("OutputTracker: Error file incorrectly initialized.", errStream.getOutputFile());
Assert.assertSame("OutputTracker: Error stream incorrectly initialized.", System.err, errStream.getOutputStream());
}
@Test
public void testErrorStreamAlone() throws FileNotFoundException {
OutputTracker ot = new OutputTracker();
OutputTracker ot = new DirectOutputTracker();
ot.initializeCoreIO(null,ERROR_FILENAME);
OutputStreamStub outStream = ot.outStub;
Assert.assertNull("OutputTracker: Output file incorrectly initialized.", outStream.getOutputFile());
Assert.assertSame("OutputTracker: Output stream incorrectly initialized.", System.out, outStream.getOutputStream());
final String ERROR_TEXT = "err stream test";
PrintWriter errWriter = new PrintWriter(ot.getErrStream());
PrintWriter errWriter = new PrintWriter(ot.errStub);
errWriter.append(ERROR_TEXT);
errWriter.close();
@ -90,24 +99,21 @@ public class OutputTrackerTest extends BaseTest {
String errText = errScanner.nextLine();
Assert.assertFalse("Err stream has too much data", errScanner.hasNext());
Assert.assertTrue("OutputTracker: Output stream is of wrong type.", ot.getOutStream() instanceof RedirectingOutputStream );
RedirectingOutputStream outStream = (RedirectingOutputStream)ot.getOutStream();
Assert.assertSame("OutputTracker: Output stream incorrectly initialized.", System.out, outStream.getBackingOutputStream());
Assert.assertEquals("OutputTracker: Written error text is incorrect", errText, ERROR_TEXT);
}
@Test
public void testIndependentStreams() throws FileNotFoundException {
OutputTracker ot = new OutputTracker();
OutputTracker ot = new DirectOutputTracker();
ot.initializeCoreIO(OUTPUT_FILENAME,ERROR_FILENAME);
final String OUTPUT_TEXT = "out stream test";
PrintWriter outWriter = new PrintWriter(ot.getOutStream());
PrintWriter outWriter = new PrintWriter(ot.outStub);
outWriter.append(OUTPUT_TEXT);
outWriter.close();
final String ERROR_TEXT = "err stream test";
PrintWriter errWriter = new PrintWriter(ot.getErrStream());
PrintWriter errWriter = new PrintWriter(ot.errStub);
errWriter.append(ERROR_TEXT);
errWriter.close();
@ -125,15 +131,16 @@ public class OutputTrackerTest extends BaseTest {
@Test
public void testIdenticalInputsGetIdenticalResults() {
OutputTracker ot = new OutputTracker();
OutputTracker ot = new DirectOutputTracker();
ot.initializeCoreIO(OUTPUT_FILENAME,OUTPUT_FILENAME);
Assert.assertTrue("OutputTracker: Output stream is of wrong type.", ot.getOutStream() instanceof RedirectingOutputStream );
Assert.assertTrue("OutputTracker: Error stream is of wrong type.", ot.getErrStream() instanceof RedirectingOutputStream );
Assert.assertNotNull("OutputTracker: Output stream is null.", ot.outStub );
Assert.assertNotNull("OutputTracker: Error stream is null.", ot.errStub );
RedirectingOutputStream outStream = (RedirectingOutputStream)ot.getOutStream();
RedirectingOutputStream errStream = (RedirectingOutputStream)ot.getErrStream();
OutputStreamStub outStream = ot.outStub;
OutputStreamStub errStream = ot.errStub;
Assert.assertSame("OutputTracker: PrintStreams don't match", outStream.getBackingOutputStream(), errStream.getBackingOutputStream());
Assert.assertSame("OutputTracker: files don't match", outStream.getOutputFile(), errStream.getOutputFile());
Assert.assertSame("OutputTracker: streams don't match", outStream.getOutputStream(), errStream.getOutputStream());
}
}

1
java/test/org/broadinstitute/sting/gatk/walkers/indels/CleanedReadInjectorTest.java
View File

@ -1,7 +1,6 @@
package org.broadinstitute.sting.gatk.walkers.indels;
import org.broadinstitute.sting.BaseTest;
import org.broadinstitute.sting.gatk.OutputTracker;
import org.broadinstitute.sting.utils.fasta.IndexedFastaSequenceFile;
import org.broadinstitute.sting.utils.GenomeLocParser;
import org.broadinstitute.sting.utils.sam.ArtificialSAMFileReader;