diff --git a/build.xml b/build.xml index 7c81c1f20..dbdafa3d9 100644 --- a/build.xml +++ b/build.xml @@ -1,5 +1,5 @@ - + + + + + + + + diff --git a/ivy.xml b/ivy.xml index 4f41904ba..f5ff15c30 100644 --- a/ivy.xml +++ b/ivy.xml @@ -1,3 +1,26 @@ + @@ -21,7 +44,6 @@ - @@ -40,7 +62,7 @@ - + diff --git a/public/java/src/org/broadinstitute/sting/commandline/Gather.java b/public/java/src/org/broadinstitute/sting/commandline/Gather.java index 59c3f50cb..d452f708e 100644 --- a/public/java/src/org/broadinstitute/sting/commandline/Gather.java +++ b/public/java/src/org/broadinstitute/sting/commandline/Gather.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -34,5 +34,6 @@ import java.lang.annotation.*; @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.FIELD}) public @interface Gather { - Class value(); + Class value() default Gather.class; + boolean enabled() default true; } diff --git a/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java b/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java index f954d7650..ede8e9340 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java @@ -443,7 +443,7 @@ public class GenomeAnalysisEngine { if(!readsDataSource.hasIndex() && intervals != null && !argCollection.allowIntervalsWithUnindexedBAM) throw new UserException.CommandLineException("Cannot perform interval processing when reads are present but no index is available."); - if(walker instanceof LocusWalker) { + if(walker instanceof LocusWalker || walker instanceof ActiveRegionWalker) { if (readsDataSource.getSortOrder() != SAMFileHeader.SortOrder.coordinate) throw new UserException.MissortedBAM(SAMFileHeader.SortOrder.coordinate, "Locus walkers can only traverse coordinate-sorted data. Please resort your input BAM file(s) or set the Sort Order tag in the header appropriately."); if(intervals == null) diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BlockInputStream.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BlockInputStream.java index e377f865d..a95eb7b8e 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BlockInputStream.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BlockInputStream.java @@ -31,13 +31,13 @@ import net.sf.samtools.util.BlockCompressedFilePointerUtil; import net.sf.samtools.util.BlockCompressedInputStream; import net.sf.samtools.util.RuntimeEOFException; import net.sf.samtools.util.SeekableStream; -import org.broad.tribble.util.BlockCompressedStreamConstants; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Arrays; +import java.util.Iterator; import java.util.LinkedList; /** @@ -120,6 +120,12 @@ public class BlockInputStream extends SeekableStream implements BAMInputStream { // TODO: Kill the region when all we want to do is start at the beginning of the stream and run to the end of the stream. this.position = new SAMReaderPosition(reader,this,new GATKBAMFileSpan(new GATKChunk(0,Long.MAX_VALUE))); + // The block offsets / block positions guarantee that the ending offset/position in the data structure maps to + // the point in the file just following the last read. These two arrays should never be empty; initializing + // to 0 to match the position above. + this.blockOffsets.add(0); + this.blockPositions.add(0L); + try { if(validate) { System.out.printf("BlockInputStream %s: BGZF block validation mode activated%n",this); @@ -143,34 +149,52 @@ public class BlockInputStream extends SeekableStream implements BAMInputStream { public long getFilePointer() { long filePointer; synchronized(lock) { - if(buffer.remaining() > 0) { - // If there's data in the buffer, figure out from whence it came. - final long blockAddress = blockPositions.size() > 0 ? blockPositions.get(0) : 0; - final int blockOffset = buffer.position(); - filePointer = blockAddress << 16 | blockOffset; - } - else { - // Otherwise, find the next position to load. - filePointer = position.getBlockAddress() << 16; - } + // Find the current block within the input stream. + int blockIndex; + for(blockIndex = 0; blockIndex+1 < blockOffsets.size() && buffer.position() >= blockOffsets.get(blockIndex + 1); blockIndex++) + ; + filePointer = blockPositions.get(blockIndex) + (buffer.position()-blockOffsets.get(blockIndex)); } - if(validatingInputStream != null && filePointer != validatingInputStream.getFilePointer()) - throw new ReviewedStingException(String.format("Position of input stream is invalid; expected (block address, block offset) = (%d,%d), got (%d,%d)", - BlockCompressedFilePointerUtil.getBlockAddress(filePointer),BlockCompressedFilePointerUtil.getBlockOffset(filePointer), - BlockCompressedFilePointerUtil.getBlockAddress(validatingInputStream.getFilePointer()),BlockCompressedFilePointerUtil.getBlockOffset(validatingInputStream.getFilePointer()))); +// if(validatingInputStream != null && filePointer != validatingInputStream.getFilePointer()) +// throw new ReviewedStingException(String.format("Position of input stream is invalid; expected (block address, block offset) = (%d,%d), got (%d,%d)", +// BlockCompressedFilePointerUtil.getBlockAddress(validatingInputStream.getFilePointer()),BlockCompressedFilePointerUtil.getBlockOffset(validatingInputStream.getFilePointer()), +// BlockCompressedFilePointerUtil.getBlockAddress(filePointer),BlockCompressedFilePointerUtil.getBlockOffset(filePointer))); return filePointer; } public void seek(long target) { - // TODO: Validate the seek point. //System.out.printf("Thread %s, BlockInputStream %s: seeking to block %d, offset %d%n",Thread.currentThread().getId(),this,BlockCompressedFilePointerUtil.getBlockAddress(target),BlockCompressedFilePointerUtil.getBlockOffset(target)); synchronized(lock) { clearBuffers(); - position.advancePosition(BlockCompressedFilePointerUtil.getBlockAddress(target)); - waitForBufferFill(); - buffer.position(BlockCompressedFilePointerUtil.getBlockOffset(target)); + + // Ensure that the position filled in by submitAccessPlan() is in sync with the seek target just specified. + position.advancePosition(target); + + // If the position advances past the end of the target, that must mean that we seeked to a point at the end + // of one of the chunk list's subregions. Make a note of our current position and punt on loading any data. + if(target < position.getBlockAddress() << 16) { + blockOffsets.clear(); + blockOffsets.add(0); + blockPositions.clear(); + blockPositions.add(target); + } + else { + waitForBufferFill(); + // A buffer fill will load the relevant data from the shard, but the buffer position still needs to be + // advanced as appropriate. + Iterator blockOffsetIterator = blockOffsets.descendingIterator(); + Iterator blockPositionIterator = blockPositions.descendingIterator(); + while(blockOffsetIterator.hasNext() && blockPositionIterator.hasNext()) { + final int blockOffset = blockOffsetIterator.next(); + final long blockPosition = blockPositionIterator.next(); + if((blockPosition >> 16) == (target >> 16) && (blockPosition&0xFFFF) < (target&0xFFFF)) { + buffer.position(blockOffset + (int)(target&0xFFFF)-(int)(blockPosition&0xFFFF)); + break; + } + } + } if(validatingInputStream != null) { try { @@ -191,14 +215,17 @@ public class BlockInputStream extends SeekableStream implements BAMInputStream { buffer.clear(); buffer.limit(0); + // Clear everything except the last block offset / position blockOffsets.clear(); - blockPositions.clear(); + blockOffsets.add(0); + while(blockPositions.size() > 1) + blockPositions.removeFirst(); } public boolean eof() { synchronized(lock) { // TODO: Handle multiple empty BGZF blocks at end of the file. - return position != null && position.getBlockAddress() >= length; + return position != null && (position.getBlockAddress() < 0 || position.getBlockAddress() >= length); } } @@ -216,7 +243,7 @@ public class BlockInputStream extends SeekableStream implements BAMInputStream { // Assume that the access plan is going to tell us to start where we are and move forward. // If this isn't the case, we'll soon receive a seek request and the buffer will be forced to reset. if(this.position != null && position.getBlockAddress() < this.position.getBlockAddress()) - position.advancePosition(this.position.getBlockAddress()); + position.advancePosition(this.position.getBlockAddress() << 16); } this.position = position; } @@ -225,14 +252,15 @@ public class BlockInputStream extends SeekableStream implements BAMInputStream { // Compact buffer to maximize storage space. int bytesToRemove = 0; - // Look ahead to see if we can compact away the first block in the series. - while(blockOffsets.size() > 1 && buffer.position() < blockOffsets.get(1)) { - bytesToRemove += blockOffsets.remove(); + // Look ahead to see if we can compact away the first blocks in the series. + while(blockOffsets.size() > 1 && buffer.position() >= blockOffsets.get(1)) { + blockOffsets.remove(); blockPositions.remove(); + bytesToRemove = blockOffsets.peek(); } // If we end up with an empty block at the end of the series, compact this as well. - if(buffer.remaining() == 0 && !blockOffsets.isEmpty() && buffer.position() >= blockOffsets.peek()) { + if(buffer.remaining() == 0 && blockOffsets.size() > 1 && buffer.position() >= blockOffsets.peek()) { bytesToRemove += buffer.position(); blockOffsets.remove(); blockPositions.remove(); @@ -241,11 +269,17 @@ public class BlockInputStream extends SeekableStream implements BAMInputStream { int finalBufferStart = buffer.position() - bytesToRemove; int finalBufferSize = buffer.remaining(); + // Position the buffer to remove the unneeded data, and compact it away. buffer.position(bytesToRemove); buffer.compact(); + // Reset the limits for reading. buffer.position(finalBufferStart); buffer.limit(finalBufferStart+finalBufferSize); + + // Shift everything in the offset buffer down to accommodate the bytes removed from the buffer. + for(int i = 0; i < blockOffsets.size(); i++) + blockOffsets.set(i,blockOffsets.get(i)-bytesToRemove); } /** @@ -253,6 +287,7 @@ public class BlockInputStream extends SeekableStream implements BAMInputStream { * MUST be called from a thread that is NOT the reader thread. * @param incomingBuffer The data being pushed into this input stream. * @param position target position for the data. + * @param filePosition the current position of the file pointer */ public void copyIntoBuffer(final ByteBuffer incomingBuffer, final SAMReaderPosition position, final long filePosition) { synchronized(lock) { @@ -262,7 +297,12 @@ public class BlockInputStream extends SeekableStream implements BAMInputStream { buffer.limit(buffer.capacity()); // Advance the position to take the most recent read into account. - long lastReadPosition = position.getBlockAddress(); + final long lastBlockAddress = position.getBlockAddress(); + final int blockOffsetStart = position.getFirstOffsetInBlock(); + final int blockOffsetEnd = position.getLastOffsetInBlock(); + + // Where did this read end? It either ended in the middle of a block (for a bounding chunk) or it ended at the start of the next block. + final long endOfRead = (blockOffsetEnd < incomingBuffer.remaining()) ? (lastBlockAddress << 16) | blockOffsetEnd : filePosition << 16; byte[] validBytes = null; if(validatingInputStream != null) { @@ -277,7 +317,7 @@ public class BlockInputStream extends SeekableStream implements BAMInputStream { incomingBuffer.position(pos); long currentFilePointer = validatingInputStream.getFilePointer(); - validatingInputStream.seek(lastReadPosition << 16); + validatingInputStream.seek(lastBlockAddress << 16); validatingInputStream.read(validBytes); validatingInputStream.seek(currentFilePointer); @@ -286,7 +326,7 @@ public class BlockInputStream extends SeekableStream implements BAMInputStream { } this.position = position; - position.advancePosition(filePosition); + position.advancePosition(filePosition << 16); if(buffer.remaining() < incomingBuffer.remaining()) { //System.out.printf("Thread %s: waiting for available space in buffer; buffer remaining = %d, incoming buffer remaining = %d%n",Thread.currentThread().getId(),buffer.remaining(),incomingBuffer.remaining()); @@ -294,12 +334,21 @@ public class BlockInputStream extends SeekableStream implements BAMInputStream { //System.out.printf("Thread %s: waited for available space in buffer; buffer remaining = %d, incoming buffer remaining = %d%n", Thread.currentThread().getId(), buffer.remaining(), incomingBuffer.remaining()); } - // Queue list of block offsets / block positions. + // Remove the last position in the list and add in the last read position, in case the two are different. + blockOffsets.removeLast(); blockOffsets.add(buffer.position()); - blockPositions.add(lastReadPosition); + blockPositions.removeLast(); + blockPositions.add(lastBlockAddress << 16 | blockOffsetStart); + // Stream the buffer into the data stream. + incomingBuffer.position(blockOffsetStart); + incomingBuffer.limit(Math.min(incomingBuffer.limit(),blockOffsetEnd)); buffer.put(incomingBuffer); + // Then, add the last position read to the very end of the list, just past the end of the last buffer. + blockOffsets.add(buffer.position()); + blockPositions.add(endOfRead); + // Set up the buffer for reading. buffer.flip(); bufferFilled = true; @@ -380,23 +429,21 @@ public class BlockInputStream extends SeekableStream implements BAMInputStream { lock.notify(); } - if(validatingInputStream != null) { - byte[] validBytes = new byte[length]; - try { - validatingInputStream.read(validBytes,offset,length); - for(int i = offset; i < offset+length; i++) { - if(bytes[i] != validBytes[i]) { - System.out.printf("Thread %s: preparing to throw an exception because contents don't match%n",Thread.currentThread().getId()); - throw new ReviewedStingException(String.format("Thread %s: blockInputStream %s attempting to return wrong set of bytes; mismatch at offset %d",Thread.currentThread().getId(),this,i)); - } - } - } - catch(IOException ex) { - throw new ReviewedStingException("Unable to validate against Picard input stream",ex); - } - } +// if(validatingInputStream != null) { +// byte[] validBytes = new byte[length]; +// try { +// validatingInputStream.read(validBytes,offset,length); +// for(int i = offset; i < offset+length; i++) { +// if(bytes[i] != validBytes[i]) +// throw new ReviewedStingException(String.format("Thread %s: blockInputStream %s attempting to return wrong set of bytes; mismatch at offset %d",Thread.currentThread().getId(),this,i)); +// } +// } +// catch(IOException ex) { +// throw new ReviewedStingException("Unable to validate against Picard input stream",ex); +// } +// } - return length - remaining; + return eof() ? -1 : length-remaining; } public void close() { @@ -424,7 +471,6 @@ public class BlockInputStream extends SeekableStream implements BAMInputStream { lock.wait(); } catch(InterruptedException ex) { - // TODO: handle me. throw new ReviewedStingException("Interrupt occurred waiting for buffer to fill",ex); } diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMReaderPosition.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMReaderPosition.java index f9f6539a7..0a6173c1e 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMReaderPosition.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMReaderPosition.java @@ -75,6 +75,22 @@ class SAMReaderPosition { return nextBlockAddress; } + /** + * Retrieves the first offset of interest in the block returned by getBlockAddress(). + * @return First block of interest in this segment. + */ + public int getFirstOffsetInBlock() { + return (nextBlockAddress == positionIterator.peek().getBlockStart()) ? positionIterator.peek().getBlockOffsetStart() : 0; + } + + /** + * Retrieves the last offset of interest in the block returned by getBlockAddress(). + * @return First block of interest in this segment. + */ + public int getLastOffsetInBlock() { + return (nextBlockAddress == positionIterator.peek().getBlockEnd()) ? positionIterator.peek().getBlockOffsetEnd() : 65536; + } + public void reset() { initialize(); } @@ -95,26 +111,27 @@ class SAMReaderPosition { * @param filePosition The current position within the file. */ void advancePosition(final long filePosition) { - nextBlockAddress = filePosition; + nextBlockAddress = filePosition >> 16; // Check the current file position against the iterator; if the iterator is before the current file position, // draw the iterator forward. Remember when performing the check that coordinates are half-open! - try { - while(positionIterator.hasNext() && isFilePositionPastEndOfChunk(filePosition,positionIterator.peek())) { - positionIterator.next(); - // Check to see if the iterator has more data available. - if(positionIterator.hasNext() && filePosition < positionIterator.peek().getBlockStart()) { - nextBlockAddress = positionIterator.peek().getBlockStart(); - break; - } + while(positionIterator.hasNext() && isFilePositionPastEndOfChunk(filePosition,positionIterator.peek())) { + positionIterator.next(); + + // If the block iterator has shot past the file pointer, bring the file pointer flush with the start of the current block. + if(positionIterator.hasNext() && filePosition < positionIterator.peek().getChunkStart()) { + nextBlockAddress = positionIterator.peek().getBlockStart(); + //System.out.printf("SAMReaderPosition: next block address advanced to %d%n",nextBlockAddress); + break; } } - catch(Exception ex) { - throw new ReviewedStingException(""); - } + + // If we've shot off the end of the block pointer, notify consumers that iteration is complete. + if(!positionIterator.hasNext()) + nextBlockAddress = -1; } private boolean isFilePositionPastEndOfChunk(final long filePosition, final GATKChunk chunk) { - return (filePosition > chunk.getBlockEnd() || (filePosition == chunk.getBlockEnd() && chunk.getBlockOffsetEnd() == 0)); + return filePosition >= chunk.getChunkEnd(); } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java index 39e1bdc72..eec440820 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java +++ b/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java @@ -11,7 +11,6 @@ import org.broadinstitute.sting.gatk.io.ThreadLocalOutputTracker; import org.broadinstitute.sting.gatk.walkers.TreeReducible; import org.broadinstitute.sting.gatk.walkers.Walker; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import org.broadinstitute.sting.utils.exceptions.UserException; import org.broadinstitute.sting.utils.threading.ThreadPoolMonitor; import java.util.Collection; diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java index ff5e1064b..774b532f3 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java +++ b/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java @@ -10,6 +10,7 @@ import org.broadinstitute.sting.gatk.datasources.reads.Shard; import org.broadinstitute.sting.gatk.datasources.rmd.ReferenceOrderedDataSource; import org.broadinstitute.sting.gatk.io.DirectOutputTracker; import org.broadinstitute.sting.gatk.io.OutputTracker; +import org.broadinstitute.sting.gatk.traversals.TraverseActiveRegions; import org.broadinstitute.sting.gatk.walkers.LocusWalker; import org.broadinstitute.sting.gatk.walkers.Walker; import org.broadinstitute.sting.utils.SampleUtils; @@ -55,7 +56,6 @@ public class LinearMicroScheduler extends MicroScheduler { traversalEngine.startTimersIfNecessary(); if(shard.getShardType() == Shard.ShardType.LOCUS) { - LocusWalker lWalker = (LocusWalker)walker; WindowMaker windowMaker = new WindowMaker(shard, engine.getGenomeLocParser(), getReadIterator(shard), shard.getGenomeLocs(), SampleUtils.getSAMFileSamples(engine)); for(WindowMaker.WindowMakerIterator iterator: windowMaker) { @@ -77,6 +77,12 @@ public class LinearMicroScheduler extends MicroScheduler { done = walker.isDone(); } + // Special function call to empty out the work queue. Ugly for now but will be cleaned up when we push this functionality more into the engine + if( traversalEngine instanceof TraverseActiveRegions ) { + final Object result = ((TraverseActiveRegions) traversalEngine).endTraversal(walker, accumulator.getReduceInit()); + accumulator.accumulate(null, result); // Assumes only used with StandardAccumulator + } + Object result = accumulator.finishTraversal(); printOnTraversalDone(result); diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java index d013db7e8..508099708 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java +++ b/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java @@ -128,6 +128,8 @@ public abstract class MicroScheduler implements MicroSchedulerMBean { traversalEngine = new TraverseDuplicates(); } else if (walker instanceof ReadPairWalker) { traversalEngine = new TraverseReadPairs(); + } else if (walker instanceof ActiveRegionWalker) { + traversalEngine = new TraverseActiveRegions(); } else { throw new UnsupportedOperationException("Unable to determine traversal type, the walker is an unknown type."); } diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/TreeReducer.java b/public/java/src/org/broadinstitute/sting/gatk/executive/TreeReducer.java index d36a3b576..6acaadd50 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/executive/TreeReducer.java +++ b/public/java/src/org/broadinstitute/sting/gatk/executive/TreeReducer.java @@ -27,8 +27,8 @@ import java.util.concurrent.Future; public class TreeReducer implements Callable { private HierarchicalMicroScheduler microScheduler; private TreeReducible walker; - private final Future lhs; - private final Future rhs; + private Future lhs; + private Future rhs; /** * Create a one-sided reduce. Result will be a simple pass-through of the result. @@ -99,6 +99,10 @@ public class TreeReducer implements Callable { long endTime = System.currentTimeMillis(); + // Constituent bits of this tree reduces are no longer required. Throw them away. + this.lhs = null; + this.rhs = null; + microScheduler.reportTreeReduceTime( endTime - startTime ); return result; diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java new file mode 100644 index 000000000..01bfe396a --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java @@ -0,0 +1,213 @@ +package org.broadinstitute.sting.gatk.traversals; + +import net.sf.samtools.SAMRecord; +import org.apache.log4j.Logger; +import org.broadinstitute.sting.gatk.WalkerManager; +import org.broadinstitute.sting.gatk.contexts.AlignmentContext; +import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.gatk.datasources.providers.*; +import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; +import org.broadinstitute.sting.gatk.walkers.ActiveRegionWalker; +import org.broadinstitute.sting.gatk.walkers.DataSource; +import org.broadinstitute.sting.gatk.walkers.Walker; +import org.broadinstitute.sting.utils.GenomeLoc; +import org.broadinstitute.sting.utils.activeregion.ActiveRegion; +import org.broadinstitute.sting.utils.pileup.PileupElement; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.Queue; + +/** + * Created by IntelliJ IDEA. + * User: rpoplin + * Date: 12/9/11 + */ + +public class TraverseActiveRegions extends TraversalEngine,LocusShardDataProvider> { + /** + * our log, which we want to capture anything from this class + */ + protected static Logger logger = Logger.getLogger(TraversalEngine.class); + + private final Queue workQueue = new LinkedList(); + private final LinkedHashSet myReads = new LinkedHashSet(); + + @Override + protected String getTraversalType() { + return "active regions"; + } + + @Override + public T traverse( final ActiveRegionWalker walker, + final LocusShardDataProvider dataProvider, + T sum) { + logger.debug(String.format("TraverseActiveRegion.traverse: Shard is %s", dataProvider)); + + LocusView locusView = getLocusView( walker, dataProvider ); + + int minStart = Integer.MAX_VALUE; + final LocusReferenceView referenceView = new LocusReferenceView( walker, dataProvider ); + + if ( locusView.hasNext() ) { // trivial optimization to avoid unnecessary processing when there's nothing here at all + + final ArrayList isActiveList = new ArrayList(); + + //ReferenceOrderedView referenceOrderedDataView = new ReferenceOrderedView( dataProvider ); + ReferenceOrderedView referenceOrderedDataView = null; + if ( WalkerManager.getWalkerDataSource(walker) != DataSource.REFERENCE_ORDERED_DATA ) + referenceOrderedDataView = new ManagingReferenceOrderedView( dataProvider ); + else + referenceOrderedDataView = (RodLocusView)locusView; + + // We keep processing while the next reference location is within the interval + while( locusView.hasNext() ) { + final AlignmentContext locus = locusView.next(); + GenomeLoc location = locus.getLocation(); + + dataProvider.getShard().getReadMetrics().incrementNumIterations(); + + if ( locus.hasExtendedEventPileup() ) { + // if the alignment context we received holds an "extended" pileup (i.e. pileup of insertions/deletions + // associated with the current site), we need to update the location. The updated location still starts + // at the current genomic position, but it has to span the length of the longest deletion (if any). + location = engine.getGenomeLocParser().setStop(location,location.getStop()+locus.getExtendedEventPileup().getMaxDeletionLength()); + + // it is possible that the new expanded location spans the current shard boundary; the next method ensures + // that when it is the case, the reference sequence held by the ReferenceView will be reloaded so that + // the view has all the bases we are gonna need. If the location fits within the current view bounds, + // the next call will not do anything to the view: + referenceView.expandBoundsToAccomodateLoc(location); + } + + // create reference context. Note that if we have a pileup of "extended events", the context will + // hold the (longest) stretch of deleted reference bases (if deletions are present in the pileup). + final ReferenceContext refContext = referenceView.getReferenceContext(location); + + // Iterate forward to get all reference ordered data covering this location + final RefMetaDataTracker tracker = referenceOrderedDataView.getReferenceOrderedDataAtLocus(locus.getLocation(), refContext); + + // Call the walkers isActive function for this locus and add them to the list to be integrated later + final boolean isActive = walker.isActive( tracker, refContext, locus ); + isActiveList.add( new ActiveRegion(location, isActive, engine.getGenomeLocParser()) ); + + // Grab all the previously unseen reads from this pileup and add them to the massive read list + for( final PileupElement p : locus.getBasePileup() ) { + final SAMRecord read = p.getRead(); + if( !myReads.contains(read) ) { + myReads.add(read); + } + } + + // If this is the last pileup for this shard then need to calculate the minimum alignment start so that + // we know which active regions in the work queue are now safe to process + if( !locusView.hasNext() ) { + for( final PileupElement p : locus.getBasePileup() ) { + final SAMRecord read = p.getRead(); + if( read.getAlignmentStart() < minStart ) { minStart = read.getAlignmentStart(); } + } + } + printProgress(dataProvider.getShard(),locus.getLocation()); + } + + // Take the individual isActive calls and integrate them into contiguous active regions and + // add these blocks of work to the work queue + final ArrayList activeRegions = integrateActiveList( isActiveList ); + logger.debug("Integrated " + isActiveList.size() + " isActive calls into " + activeRegions.size() + " regions." ); + workQueue.addAll( activeRegions ); + } + + while( workQueue.peek().getLocation().getStop() < minStart ) { + final ActiveRegion activeRegion = workQueue.remove(); + sum = processActiveRegion( activeRegion, myReads, workQueue, sum, walker ); + } + + return sum; + } + + // Special function called in LinearMicroScheduler to empty out the work queue. Ugly for now but will be cleaned up when we push this functionality more into the engine + public T endTraversal( final Walker walker, T sum) { + while( workQueue.peek() != null ) { + final ActiveRegion activeRegion = workQueue.remove(); + sum = processActiveRegion( activeRegion, myReads, workQueue, sum, (ActiveRegionWalker) walker ); + } + + return sum; + } + + private T processActiveRegion( final ActiveRegion activeRegion, final LinkedHashSet reads, final Queue workQueue, final T sum, final ActiveRegionWalker walker ) { + final ArrayList placedReads = new ArrayList(); + for( final SAMRecord read : reads ) { + final GenomeLoc readLoc = this.engine.getGenomeLocParser().createGenomeLoc( read ); + if( activeRegion.getLocation().overlapsP( readLoc ) ) { + // The region which the highest amount of overlap is chosen as the primary region for the read (tie breaking is done as right most region) + long maxOverlap = activeRegion.getLocation().sizeOfOverlap( readLoc ); + ActiveRegion bestRegion = activeRegion; + for( final ActiveRegion otherRegionToTest : workQueue ) { + if( otherRegionToTest.getLocation().sizeOfOverlap(readLoc) >= maxOverlap ) { + maxOverlap = otherRegionToTest.getLocation().sizeOfOverlap(readLoc); + bestRegion = otherRegionToTest; + } + } + bestRegion.add( (GATKSAMRecord) read, true ); + + // The read is also added to all other region in which it overlaps but marked as non-primary + if( !bestRegion.equals(activeRegion) ) { + activeRegion.add( (GATKSAMRecord) read, false ); + } + for( final ActiveRegion otherRegionToTest : workQueue ) { + if( !bestRegion.equals(otherRegionToTest) && otherRegionToTest.getLocation().overlapsP( readLoc ) ) { + activeRegion.add( (GATKSAMRecord) read, false ); + } + } + placedReads.add( read ); + } + } + reads.removeAll( placedReads ); // remove all the reads which have been placed into their active region + + logger.debug(">> Map call with " + activeRegion.getReads().size() + " " + (activeRegion.isActive ? "active" : "inactive") + " reads @ " + activeRegion.getLocation() + " with full extent: " + activeRegion.getReferenceLocation()); + final M x = walker.map( activeRegion, null ); // BUGBUG: tracker needs to be filled in and passed to the walker + return walker.reduce( x, sum ); + } + + /** + * Gets the best view of loci for this walker given the available data. + * @param walker walker to interrogate. + * @param dataProvider Data which which to drive the locus view. + * @return A view of the locus data, where one iteration of the locus view maps to one iteration of the traversal. + */ + private LocusView getLocusView( Walker walker, LocusShardDataProvider dataProvider ) { + DataSource dataSource = WalkerManager.getWalkerDataSource(walker); + if( dataSource == DataSource.READS ) + return new CoveredLocusView(dataProvider); + else if( dataSource == DataSource.REFERENCE ) //|| ! GenomeAnalysisEngine.instance.getArguments().enableRodWalkers ) + return new AllLocusView(dataProvider); + else if( dataSource == DataSource.REFERENCE_ORDERED_DATA ) + return new RodLocusView(dataProvider); + else + throw new UnsupportedOperationException("Unsupported traversal type: " + dataSource); + } + + // integrate active regions into contiguous chunks based on active status + private ArrayList integrateActiveList( final ArrayList activeList ) { + final ArrayList returnList = new ArrayList(); + ActiveRegion prevLocus = activeList.remove(0); + ActiveRegion startLocus = prevLocus; + for( final ActiveRegion thisLocus : activeList ) { + if( prevLocus.isActive != thisLocus.isActive ) { + returnList.add( new ActiveRegion( engine.getGenomeLocParser().createGenomeLoc(startLocus.getLocation().getContig(), startLocus.getLocation().getStart(), prevLocus.getLocation().getStart()), + prevLocus.isActive, engine.getGenomeLocParser() ) ); + startLocus = thisLocus; + } + prevLocus = thisLocus; + } + // output the last region if necessary + if( startLocus != prevLocus ) { + returnList.add( new ActiveRegion( engine.getGenomeLocParser().createGenomeLoc(startLocus.getLocation().getContig(), startLocus.getLocation().getStart(), prevLocus.getLocation().getStart()), + prevLocus.isActive, engine.getGenomeLocParser() ) ); + } + return returnList; + } +} diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java new file mode 100644 index 000000000..d2891c959 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java @@ -0,0 +1,29 @@ +package org.broadinstitute.sting.gatk.walkers; + +import org.broadinstitute.sting.gatk.contexts.AlignmentContext; +import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.gatk.refdata.ReadMetaDataTracker; +import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; +import org.broadinstitute.sting.utils.activeregion.ActiveRegion; + +/** + * Created by IntelliJ IDEA. + * User: rpoplin + * Date: 12/7/11 + */ + +@By(DataSource.READS) +@Requires({DataSource.READS, DataSource.REFERENCE_BASES}) +@PartitionBy(PartitionType.READ) +public abstract class ActiveRegionWalker extends Walker { + // Do we actually want to operate on the context? + public boolean filter(final RefMetaDataTracker tracker, final ReferenceContext ref, final AlignmentContext context) { + return true; // We are keeping all the reads + } + + // Determine active status over the AlignmentContext + public abstract boolean isActive(final RefMetaDataTracker tracker, final ReferenceContext ref, final AlignmentContext context); + + // Map over the ActiveRegion + public abstract MapType map(final ActiveRegion activeRegion, final ReadMetaDataTracker metaDataTracker); +} diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java index b30a25414..ace780dd0 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java @@ -31,6 +31,7 @@ import org.broadinstitute.sting.gatk.contexts.AlignmentContextUtils; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; import org.broadinstitute.sting.utils.BaseUtils; +import org.broadinstitute.sting.utils.GenomeLocParser; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.pileup.PileupElement; import org.broadinstitute.sting.utils.pileup.ReadBackedPileup; @@ -72,25 +73,28 @@ public abstract class GenotypeLikelihoodsCalculationModel implements Cloneable { this.logger = logger; } - /** - * Must be overridden by concrete subclasses - * - * @param tracker rod data - * @param ref reference context - * @param contexts stratified alignment contexts - * @param contextType stratified context type - * @param priors priors to use for GLs - * @param alternateAlleleToUse the alternate allele to use, null if not set - * @param useBAQedPileup should we use the BAQed pileup or the raw one? - * @return variant context where genotypes are no-called but with GLs - */ - public abstract VariantContext getLikelihoods(RefMetaDataTracker tracker, - ReferenceContext ref, - Map contexts, - AlignmentContextUtils.ReadOrientation contextType, - GenotypePriors priors, - Allele alternateAlleleToUse, - boolean useBAQedPileup); + /** + * Can be overridden by concrete subclasses + * + * @param tracker rod data + * @param ref reference context + * @param contexts stratified alignment contexts + * @param contextType stratified context type + * @param priors priors to use for GLs + * @param alternateAlleleToUse the alternate allele to use, null if not set + * @param useBAQedPileup should we use the BAQed pileup or the raw one? + * @param locParser Genome Loc Parser + * @return variant context where genotypes are no-called but with GLs + */ + public abstract VariantContext getLikelihoods(RefMetaDataTracker tracker, + ReferenceContext ref, + Map contexts, + AlignmentContextUtils.ReadOrientation contextType, + GenotypePriors priors, + Allele alternateAlleleToUse, + boolean useBAQedPileup, + GenomeLocParser locParser); + protected int getFilteredDepth(ReadBackedPileup pileup) { int count = 0; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java index fe2086d47..0756caf03 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java @@ -33,6 +33,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; import org.broadinstitute.sting.gatk.walkers.indels.PairHMMIndelErrorModel; import org.broadinstitute.sting.utils.BaseUtils; import org.broadinstitute.sting.utils.GenomeLoc; +import org.broadinstitute.sting.utils.GenomeLocParser; import org.broadinstitute.sting.utils.Haplotype; import org.broadinstitute.sting.utils.clipping.ReadClipper; import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants; @@ -54,17 +55,17 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood private final boolean getAlleleListFromVCF; private boolean DEBUG = false; - + private final boolean doMultiAllelicCalls; private boolean ignoreSNPAllelesWhenGenotypingIndels = false; - + private final int maxAlternateAlleles; private PairHMMIndelErrorModel pairModel; private static ThreadLocal>> indelLikelihoodMap = new ThreadLocal>>() { - protected synchronized HashMap> initialValue() { - return new HashMap>(); - } - }; + protected synchronized HashMap> initialValue() { + return new HashMap>(); + } + }; private LinkedHashMap haplotypeMap; @@ -81,12 +82,14 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood protected IndelGenotypeLikelihoodsCalculationModel(UnifiedArgumentCollection UAC, Logger logger) { super(UAC, logger); pairModel = new PairHMMIndelErrorModel(UAC.INDEL_GAP_OPEN_PENALTY,UAC.INDEL_GAP_CONTINUATION_PENALTY, - UAC.OUTPUT_DEBUG_INDEL_INFO, UAC.BANDED_INDEL_COMPUTATION); + UAC.OUTPUT_DEBUG_INDEL_INFO, !UAC.DONT_DO_BANDED_INDEL_COMPUTATION); alleleList = new ArrayList(); getAlleleListFromVCF = UAC.GenotypingMode == GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES; minIndelCountForGenotyping = UAC.MIN_INDEL_COUNT_FOR_GENOTYPING; HAPLOTYPE_SIZE = UAC.INDEL_HAPLOTYPE_SIZE; DEBUG = UAC.OUTPUT_DEBUG_INDEL_INFO; + maxAlternateAlleles = UAC.MAX_ALTERNATE_ALLELES; + doMultiAllelicCalls = UAC.MULTI_ALLELIC; haplotypeMap = new LinkedHashMap(); ignoreSNPAllelesWhenGenotypingIndels = UAC.IGNORE_SNP_ALLELES; @@ -95,7 +98,7 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood private ArrayList computeConsensusAlleles(ReferenceContext ref, Map contexts, - AlignmentContextUtils.ReadOrientation contextType) { + AlignmentContextUtils.ReadOrientation contextType, GenomeLocParser locParser) { Allele refAllele=null, altAllele=null; GenomeLoc loc = ref.getLocus(); ArrayList aList = new ArrayList(); @@ -114,7 +117,7 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood if (insCount < minIndelCountForGenotyping && delCount < minIndelCountForGenotyping) return aList; - + for ( Map.Entry sample : contexts.entrySet() ) { // todo -- warning, can be duplicating expensive partition here AlignmentContext context = AlignmentContextUtils.stratify(sample.getValue(), contextType); @@ -126,9 +129,9 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood for ( ExtendedEventPileupElement p : indelPileup.toExtendedIterable() ) { //SAMRecord read = p.getRead(); - GATKSAMRecord read = ReadClipper.hardClipAdaptorSequence(p.getRead()); + GATKSAMRecord read = ReadClipper.hardClipAdaptorSequence(p.getRead()); if (read == null) - continue; + continue; if(ReadUtils.is454Read(read)) { continue; } @@ -208,63 +211,69 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood } } -/* if (DEBUG) { - int icount = indelPileup.getNumberOfInsertions(); - int dcount = indelPileup.getNumberOfDeletions(); - if (icount + dcount > 0) - { - List> eventStrings = indelPileup.getEventStringsWithCounts(ref.getBases()); - System.out.format("#ins: %d, #del:%d\n", insCount, delCount); - - for (int i=0 ; i < eventStrings.size() ; i++ ) { - System.out.format("%s:%d,",eventStrings.get(i).first,eventStrings.get(i).second); - // int k=0; - } - System.out.println(); - } - } */ } + Collection vcs = new ArrayList(); int maxAlleleCnt = 0; String bestAltAllele = ""; + for (String s : consensusIndelStrings.keySet()) { - int curCnt = consensusIndelStrings.get(s); - if (curCnt > maxAlleleCnt) { - maxAlleleCnt = curCnt; - bestAltAllele = s; + int curCnt = consensusIndelStrings.get(s), stop = 0; + // if observed count if above minimum threshold, we will genotype this allele + if (curCnt < minIndelCountForGenotyping) + continue; + + if (s.startsWith("D")) { + // get deletion length + int dLen = Integer.valueOf(s.substring(1)); + // get ref bases of accurate deletion + int startIdxInReference = 1+loc.getStart()-ref.getWindow().getStart(); + stop = loc.getStart() + dLen; + byte[] refBases = Arrays.copyOfRange(ref.getBases(),startIdxInReference,startIdxInReference+dLen); + + if (Allele.acceptableAlleleBases(refBases)) { + refAllele = Allele.create(refBases,true); + altAllele = Allele.create(Allele.NULL_ALLELE_STRING, false); + } + } + else { + // insertion case + if (Allele.acceptableAlleleBases(s)) { + refAllele = Allele.create(Allele.NULL_ALLELE_STRING, true); + altAllele = Allele.create(s, false); + stop = loc.getStart(); + } } -// if (DEBUG) -// System.out.format("Key:%s, number: %d\n",s,consensusIndelStrings.get(s) ); - } //gdebug- - if (maxAlleleCnt < minIndelCountForGenotyping) - return aList; - if (bestAltAllele.startsWith("D")) { - // get deletion length - int dLen = Integer.valueOf(bestAltAllele.substring(1)); - // get ref bases of accurate deletion - int startIdxInReference = 1+loc.getStart()-ref.getWindow().getStart(); + ArrayList vcAlleles = new ArrayList(); + vcAlleles.add(refAllele); + vcAlleles.add(altAllele); - //System.out.println(new String(ref.getBases())); - byte[] refBases = Arrays.copyOfRange(ref.getBases(),startIdxInReference,startIdxInReference+dLen); + final VariantContextBuilder builder = new VariantContextBuilder().source(""); + builder.loc(loc.getContig(), loc.getStart(), stop); + builder.alleles(vcAlleles); + builder.referenceBaseForIndel(ref.getBase()); + builder.noGenotypes(); + if (doMultiAllelicCalls) + vcs.add(builder.make()); + else { + if (curCnt > maxAlleleCnt) { + maxAlleleCnt = curCnt; + vcs.clear(); + vcs.add(builder.make()); + } - if (Allele.acceptableAlleleBases(refBases)) { - refAllele = Allele.create(refBases,true); - altAllele = Allele.create(Allele.NULL_ALLELE_STRING, false); } } - else { - // insertion case - if (Allele.acceptableAlleleBases(bestAltAllele)) { - refAllele = Allele.create(Allele.NULL_ALLELE_STRING, true); - altAllele = Allele.create(bestAltAllele, false); - } - } - if (refAllele != null && altAllele != null) { - aList.add(0,refAllele); - aList.add(1,altAllele); - } + + if (vcs.isEmpty()) + return aList; // nothing else to do, no alleles passed minimum count criterion + + VariantContext mergedVC = VariantContextUtils.simpleMerge(locParser, vcs, null, VariantContextUtils.FilteredRecordMergeType.KEEP_IF_ANY_UNFILTERED, VariantContextUtils.GenotypeMergeType.UNSORTED, false, false, null, false, false); + + aList = new ArrayList(mergedVC.getAlleles()); + return aList; } @@ -277,7 +286,7 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood AlignmentContextUtils.ReadOrientation contextType, GenotypePriors priors, Allele alternateAlleleToUse, - boolean useBAQedPileup) { + boolean useBAQedPileup, GenomeLocParser locParser) { if ( tracker == null ) return null; @@ -294,17 +303,17 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood haplotypeMap.clear(); if (getAlleleListFromVCF) { - for( final VariantContext vc_input : tracker.getValues(UAC.alleles, loc) ) { - if( vc_input != null && - allowableTypes.contains(vc_input.getType()) && - ref.getLocus().getStart() == vc_input.getStart()) { - vc = vc_input; - break; - } - } - // ignore places where we don't have a variant - if ( vc == null ) - return null; + for( final VariantContext vc_input : tracker.getValues(UAC.alleles, loc) ) { + if( vc_input != null && + allowableTypes.contains(vc_input.getType()) && + ref.getLocus().getStart() == vc_input.getStart()) { + vc = vc_input; + break; + } + } + // ignore places where we don't have a variant + if ( vc == null ) + return null; alleleList.clear(); if (ignoreSNPAllelesWhenGenotypingIndels) { @@ -323,7 +332,7 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood } else { - alleleList = computeConsensusAlleles(ref,contexts, contextType); + alleleList = computeConsensusAlleles(ref,contexts, contextType, locParser); if (alleleList.isEmpty()) return null; } @@ -340,7 +349,7 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood if (alleleList.isEmpty()) return null; - + refAllele = alleleList.get(0); altAllele = alleleList.get(1); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java index eee89674a..81c766e4d 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java @@ -30,10 +30,7 @@ import org.broadinstitute.sting.gatk.contexts.AlignmentContext; import org.broadinstitute.sting.gatk.contexts.AlignmentContextUtils; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; -import org.broadinstitute.sting.utils.BaseUtils; -import org.broadinstitute.sting.utils.GenomeLoc; -import org.broadinstitute.sting.utils.MathUtils; -import org.broadinstitute.sting.utils.Utils; +import org.broadinstitute.sting.utils.*; import org.broadinstitute.sting.utils.baq.BAQ; import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants; import org.broadinstitute.sting.utils.exceptions.StingException; @@ -66,7 +63,8 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsC final AlignmentContextUtils.ReadOrientation contextType, final GenotypePriors priors, final Allele alternateAlleleToUse, - final boolean useBAQedPileup) { + final boolean useBAQedPileup, + final GenomeLocParser locParser) { if ( !(priors instanceof DiploidSNPGenotypePriors) ) throw new StingException("Only diploid-based SNP priors are supported in the SNP GL model"); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java index 5713432b4..16159393f 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java @@ -109,7 +109,7 @@ public class UnifiedArgumentCollection { * For advanced users only. */ @Advanced - @Argument(fullName = "multiallelic", shortName = "multiallelic", doc = "Allow the discovery of multiple alleles (SNPs only)", required = false) + @Argument(fullName = "multiallelic", shortName = "multiallelic", doc = "Allow the discovery of multiple alleles", required = false) public boolean MULTI_ALLELIC = false; /** @@ -146,8 +146,8 @@ public class UnifiedArgumentCollection { public int INDEL_HAPLOTYPE_SIZE = 80; @Hidden - @Argument(fullName = "bandedIndel", shortName = "bandedIndel", doc = "Banded Indel likelihood computation", required = false) - public boolean BANDED_INDEL_COMPUTATION = false; + @Argument(fullName = "noBandedIndel", shortName = "noBandedIndel", doc = "Don't do Banded Indel likelihood computation", required = false) + public boolean DONT_DO_BANDED_INDEL_COMPUTATION = false; @Hidden @Argument(fullName = "indelDebug", shortName = "indelDebug", doc = "Output indel debug info", required = false) @@ -184,7 +184,7 @@ public class UnifiedArgumentCollection { // todo- arguments to remove uac.IGNORE_SNP_ALLELES = IGNORE_SNP_ALLELES; - uac.BANDED_INDEL_COMPUTATION = BANDED_INDEL_COMPUTATION; + uac.DONT_DO_BANDED_INDEL_COMPUTATION = DONT_DO_BANDED_INDEL_COMPUTATION; uac.MULTI_ALLELIC = MULTI_ALLELIC; return uac; } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java index 34be88dbb..5d73e8d28 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java @@ -243,7 +243,7 @@ public class UnifiedGenotyperEngine { glcm.set(getGenotypeLikelihoodsCalculationObject(logger, UAC)); } - return glcm.get().get(model).getLikelihoods(tracker, refContext, stratifiedContexts, type, getGenotypePriors(model), alternateAlleleToUse, useBAQedPileup && BAQEnabledOnCMDLine); + return glcm.get().get(model).getLikelihoods(tracker, refContext, stratifiedContexts, type, getGenotypePriors(model), alternateAlleleToUse, useBAQedPileup && BAQEnabledOnCMDLine, genomeLocParser); } private VariantCallContext generateEmptyContext(RefMetaDataTracker tracker, ReferenceContext ref, Map stratifiedContexts, AlignmentContext rawContext) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/VariantSummary.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/VariantSummary.java index a271d3c35..3c7c6f00c 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/VariantSummary.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/VariantSummary.java @@ -46,6 +46,7 @@ import java.util.*; public class VariantSummary extends VariantEvaluator implements StandardEval { final protected static Logger logger = Logger.getLogger(VariantSummary.class); + /** Indels with size greater than this value are tallied in the CNV column */ private final static int MAX_INDEL_LENGTH = 50; private final static double MIN_CNV_OVERLAP = 0.5; private VariantEvalWalker walker; @@ -196,14 +197,16 @@ public class VariantSummary extends VariantEvaluator implements StandardEval { } private final boolean overlapsKnownCNV(VariantContext cnv) { - final GenomeLoc loc = walker.getGenomeLocParser().createGenomeLoc(cnv, true); - IntervalTree intervalTree = knownCNVs.get(loc.getContig()); + if ( knownCNVs != null ) { + final GenomeLoc loc = walker.getGenomeLocParser().createGenomeLoc(cnv, true); + IntervalTree intervalTree = knownCNVs.get(loc.getContig()); - final Iterator> nodeIt = intervalTree.overlappers(loc.getStart(), loc.getStop()); - while ( nodeIt.hasNext() ) { - final double overlapP = loc.reciprocialOverlapFraction(nodeIt.next().getValue()); - if ( overlapP > MIN_CNV_OVERLAP ) - return true; + final Iterator> nodeIt = intervalTree.overlappers(loc.getStart(), loc.getStop()); + while ( nodeIt.hasNext() ) { + final double overlapP = loc.reciprocialOverlapFraction(nodeIt.next().getValue()); + if ( overlapP > MIN_CNV_OVERLAP ) + return true; + } } return false; @@ -224,7 +227,7 @@ public class VariantSummary extends VariantEvaluator implements StandardEval { allVariantCounts.inc(type, ALL); // type specific calculations - if ( type == Type.SNP ) { + if ( type == Type.SNP && eval.isBiallelic() ) { titvTable = VariantContextUtils.isTransition(eval) ? transitionsPerSample : transversionsPerSample; titvTable.inc(type, ALL); } diff --git a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java index cdfc329e8..71640c66a 100644 --- a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java +++ b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -70,17 +70,18 @@ public abstract class ArgumentDefinitionField extends ArgumentField { " * Short name of %1$s%n" + " * @return Short name of %1$s%n" + " */%n" + - "def %3$s = this.%1$s%n" + + "%5$sdef %3$s = this.%1$s%n" + "%n" + "/**%n" + " * Short name of %1$s%n" + " * @param value Short name of %1$s%n" + " */%n" + - "def %4$s(value: %2$s) { this.%1$s = value }%n", + "%5$sdef %4$s(value: %2$s) { this.%1$s = value }%n", getFieldName(), getFieldType(), getShortFieldGetter(), - getShortFieldSetter()); + getShortFieldSetter(), + getPrivacy()); } protected static final String REQUIRED_TEMPLATE = " + required(\"%1$s\", %3$s, spaceSeparated=true, escape=true, format=%2$s)"; @@ -135,11 +136,8 @@ public abstract class ArgumentDefinitionField extends ArgumentField { new IntervalFileArgumentField(argumentDefinition), new IntervalStringArgumentField(argumentDefinition)); - // ROD Bindings are set by the RodBindField - } else if (RodBindArgumentField.ROD_BIND_FIELD.equals(argumentDefinition.fullName) && argumentDefinition.ioType == ArgumentIOType.INPUT) { - // TODO: Once everyone is using @Allows and @Requires correctly, we can stop blindly allowing Triplets - return Arrays.asList(new RodBindArgumentField(argumentDefinition), new InputIndexesArgumentField(argumentDefinition, Tribble.STANDARD_INDEX_EXTENSION)); - //return Collections.emptyList(); + } else if (NumThreadsArgumentField.NUM_THREADS_FIELD.equals(argumentDefinition.fullName)) { + return Arrays.asList(new NumThreadsArgumentField(argumentDefinition)); } else if ("input_file".equals(argumentDefinition.fullName) && argumentDefinition.ioType == ArgumentIOType.INPUT) { return Arrays.asList(new InputTaggedFileDefinitionField(argumentDefinition), new InputIndexesArgumentField(argumentDefinition, BAMIndex.BAMIndexSuffix, ".bam")); @@ -166,10 +164,13 @@ public abstract class ArgumentDefinitionField extends ArgumentField { fields.add(new OutputArgumentField(argumentDefinition, gatherClass)); - if (SAMFileWriter.class.isAssignableFrom(argumentDefinition.argumentType)) + if (SAMFileWriter.class.isAssignableFrom(argumentDefinition.argumentType)) { fields.add(new SAMFileWriterIndexArgumentField(argumentDefinition)); - else if (VCFWriter.class.isAssignableFrom(argumentDefinition.argumentType)) + fields.add(new SAMFileWriterMD5ArgumentField(argumentDefinition)); + } + else if (VCFWriter.class.isAssignableFrom(argumentDefinition.argumentType)) { fields.add(new VCFWriterIndexArgumentField(argumentDefinition)); + } return fields; @@ -228,7 +229,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField { @Override protected String getRawFieldName() { return super.getRawFieldName() + "String"; } @Override protected String getFullName() { return super.getFullName() + "String"; } @Override protected String getRawShortFieldName() { return super.getRawShortFieldName() + "String"; } - @Override protected String getFieldType() { return "List[String]"; } + @Override protected String getFieldType() { return "Seq[String]"; } @Override protected String getDefaultValue() { return "Nil"; } @Override public String getCommandLineTemplate() { return REPEAT_TEMPLATE; } @@ -250,7 +251,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField { } @Override protected Class getInnerType() { return File.class; } - @Override protected String getFieldType() { return isMultiValued() ? "List[File]" : "File"; } + @Override protected String getFieldType() { return isMultiValued() ? "Seq[File]" : "File"; } @Override protected String getDefaultValue() { return isMultiValued() ? "Nil" : "_"; } } @@ -294,7 +295,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField { } @Override protected Class getInnerType() { return mapType(argumentDefinition.componentType); } - @Override protected String getFieldType() { return String.format("List[%s]", getType(getInnerType())); } + @Override protected String getFieldType() { return String.format("Seq[%s]", getType(getInnerType())); } @Override protected String getDefaultValue() { return "Nil"; } @Override protected String getCommandLineTemplate() { return REPEAT_TEMPLATE; } } @@ -336,17 +337,16 @@ public abstract class ArgumentDefinitionField extends ArgumentField { } // Allows the user to specify the track name, track type, and the file. - public static class RodBindArgumentField extends ArgumentDefinitionField { - public static final String ROD_BIND_FIELD = "rodBind"; + public static class NumThreadsArgumentField extends OptionedArgumentField { + public static final String NUM_THREADS_FIELD = "num_threads"; - public RodBindArgumentField(ArgumentDefinition argumentDefinition) { - super(argumentDefinition); + public NumThreadsArgumentField(ArgumentDefinition argumentDefinition) { + super(argumentDefinition, false); } - @Override protected Class getInnerType() { return null; } // RodBind does not need to be imported. - @Override protected String getFieldType() { return "List[RodBind]"; } - @Override protected String getDefaultValue() { return "Nil"; } - @Override protected String getCommandLineTemplate() { - return " + repeat(\"%1$s\", %3$s, formatPrefix=RodBind.formatCommandLineParameter, spaceSeparated=true, escape=true, format=%2$s)"; + + @Override + protected String getFreezeFields() { + return String.format("if (num_threads.isDefined) nCoresRequest = num_threads%n"); } } @@ -356,7 +356,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField { super(argumentDefinition); } @Override protected Class getInnerType() { return null; } // TaggedFile does not need to be imported. - @Override protected String getFieldType() { return argumentDefinition.isMultiValued ? "List[File]" : "File"; } + @Override protected String getFieldType() { return argumentDefinition.isMultiValued ? "Seq[File]" : "File"; } @Override protected String getDefaultValue() { return argumentDefinition.isMultiValued ? "Nil" : "_"; } @Override protected String getCommandLineTemplate() { if (argumentDefinition.isMultiValued) { @@ -395,10 +395,11 @@ public abstract class ArgumentDefinitionField extends ArgumentField { } @Override protected String getFullName() { return this.indexFieldName; } @Override protected boolean isRequired() { return false; } - @Override protected String getFieldType() { return "List[File]"; } + @Override protected String getFieldType() { return "Seq[File]"; } @Override protected String getDefaultValue() { return "Nil"; } @Override protected Class getInnerType() { return File.class; } @Override protected String getRawFieldName() { return this.indexFieldName; } + @Override protected String getPrivacy() { return "private "; } @Override protected String getFreezeFields() { if (originalIsMultiValued) { if (originalSuffix == null) { @@ -434,53 +435,69 @@ public abstract class ArgumentDefinitionField extends ArgumentField { } } - // Tracks an automatically generated index - private static abstract class OutputIndexArgumentField extends ArgumentField { - protected final String indexFieldName; + // Tracks an automatically generated index, md5, etc. + private static abstract class AuxilliaryOutputArgumentField extends ArgumentField { protected final String originalFieldName; - public OutputIndexArgumentField(ArgumentDefinition originalArgumentDefinition) { - this.indexFieldName = originalArgumentDefinition.fullName + "Index"; + protected final String auxFieldName; + protected final String auxFieldLabel; + public AuxilliaryOutputArgumentField(ArgumentDefinition originalArgumentDefinition, String auxFieldLabel) { this.originalFieldName = originalArgumentDefinition.fullName; + this.auxFieldName = originalArgumentDefinition.fullName + auxFieldLabel; + this.auxFieldLabel = auxFieldLabel; } @Override protected Class getAnnotationIOClass() { return Output.class; } @Override public String getCommandLineAddition() { return ""; } - @Override protected String getDoc() { return "Automatically generated index for " + this.originalFieldName; } - @Override protected String getFullName() { return this.indexFieldName; } + @Override protected String getDoc() { return String.format("Automatically generated %s for %s", auxFieldLabel.toLowerCase(), this.originalFieldName); } + @Override protected String getFullName() { return this.auxFieldName; } @Override protected boolean isRequired() { return false; } @Override protected String getFieldType() { return "File"; } @Override protected String getDefaultValue() { return "_"; } @Override protected Class getInnerType() { return File.class; } - @Override protected String getRawFieldName() { return this.indexFieldName; } + @Override protected String getRawFieldName() { return this.auxFieldName; } + @Override protected String getPrivacy() { return "private "; } @Override public boolean isGather() { return true; } @Override protected String getGatherAnnotation() { - return String.format("@Gather(classOf[AutoIndexGatherFunction])%n"); + return String.format("@Gather(enabled=false)%n"); } } - private static class VCFWriterIndexArgumentField extends OutputIndexArgumentField { + private static class VCFWriterIndexArgumentField extends AuxilliaryOutputArgumentField { public VCFWriterIndexArgumentField(ArgumentDefinition originalArgumentDefinition) { - super(originalArgumentDefinition); + super(originalArgumentDefinition, "Index"); } @Override protected String getFreezeFields() { return String.format( ("if (%2$s != null)%n" + " if (!org.broadinstitute.sting.gatk.io.stubs.VCFWriterArgumentTypeDescriptor.isCompressed(%2$s.getPath))%n" + " %1$s = new File(%2$s.getPath + \"%3$s\")%n"), - indexFieldName, originalFieldName, Tribble.STANDARD_INDEX_EXTENSION); + auxFieldName, originalFieldName, Tribble.STANDARD_INDEX_EXTENSION); } } - private static class SAMFileWriterIndexArgumentField extends OutputIndexArgumentField { + private static class SAMFileWriterIndexArgumentField extends AuxilliaryOutputArgumentField { public SAMFileWriterIndexArgumentField(ArgumentDefinition originalArgumentDefinition) { - super(originalArgumentDefinition); + super(originalArgumentDefinition, "Index"); } @Override protected String getFreezeFields() { return String.format( ("if (%2$s != null)%n" + " if (!%3$s)%n" + " %1$s = new File(%2$s.getPath.stripSuffix(\".bam\") + \"%4$s\")%n"), - indexFieldName, originalFieldName, SAMFileWriterArgumentTypeDescriptor.DISABLE_INDEXING_FULLNAME, BAMIndex.BAMIndexSuffix); + auxFieldName, originalFieldName, SAMFileWriterArgumentTypeDescriptor.DISABLE_INDEXING_FULLNAME, BAMIndex.BAMIndexSuffix); + } + } + + private static class SAMFileWriterMD5ArgumentField extends AuxilliaryOutputArgumentField { + public SAMFileWriterMD5ArgumentField(ArgumentDefinition originalArgumentDefinition) { + super(originalArgumentDefinition, "MD5"); + } + @Override protected String getFreezeFields() { + return String.format( + ("if (%2$s != null)%n" + + " if (%3$s)%n" + + " %1$s = new File(%2$s.getPath + \"%4$s\")%n"), + auxFieldName, originalFieldName, SAMFileWriterArgumentTypeDescriptor.ENABLE_MD5_FULLNAME, ".md5"); } } diff --git a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java index e90933504..2428a13a8 100644 --- a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java +++ b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -56,7 +56,7 @@ public abstract class ArgumentField { return String.format("%n" + "/** %s */%n" + "@%s(fullName=\"%s\", shortName=\"%s\", doc=\"%s\", required=%s, exclusiveOf=\"%s\", validation=\"%s\")%n" + - "%svar %s: %s = %s%n" + + "%s%svar %s: %s = %s%n" + "%s", getDoc(), getAnnotationIOClass().getSimpleName(), @@ -66,7 +66,7 @@ public abstract class ArgumentField { isRequired(), getExclusiveOf(), getValidation(), - getGatherAnnotation(), getFieldName(), getFieldType(), getDefaultValue(), + getGatherAnnotation(), getPrivacy(), getFieldName(), getFieldType(), getDefaultValue(), getDefineAddition()); } @@ -143,6 +143,9 @@ public abstract class ArgumentField { /** @return True if this field uses @Gather. */ public boolean isGather() { return false; } + /** @return Privacy for the field. */ + protected String getPrivacy() { return ""; } + /** @return The raw field name, which will be checked against scala build in types. */ protected abstract String getRawFieldName(); /** @return The field name checked against reserved words. */ diff --git a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java index 9c40fb976..a3f80af1c 100644 --- a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java +++ b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -34,13 +34,11 @@ import org.broadinstitute.sting.commandline.ParsingEngine; import org.broadinstitute.sting.gatk.CommandLineGATK; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.gatk.WalkerManager; -import org.broadinstitute.sting.gatk.arguments.ValidationExclusion; import org.broadinstitute.sting.gatk.filters.FilterManager; import org.broadinstitute.sting.gatk.filters.ReadFilter; import org.broadinstitute.sting.gatk.io.stubs.OutputStreamArgumentTypeDescriptor; import org.broadinstitute.sting.gatk.io.stubs.SAMFileWriterArgumentTypeDescriptor; import org.broadinstitute.sting.gatk.io.stubs.VCFWriterArgumentTypeDescriptor; -import org.broadinstitute.sting.gatk.refdata.tracks.RMDTrackBuilder; import org.broadinstitute.sting.gatk.walkers.PartitionBy; import org.broadinstitute.sting.gatk.walkers.PartitionType; import org.broadinstitute.sting.gatk.walkers.Walker; @@ -85,7 +83,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram { "%n" + "/** A dynamicly generated list of classes that the GATK Extensions depend on, but are not be detected by default by BCEL. */%n" + "class %s {%n" + - "val types = List(%n%s)%n" + + "val types = Seq(%n%s)%n" + "}%n"; @Output(fullName="output_directory", shortName="outDir", doc="Directory to output the generated scala", required=true) @@ -95,10 +93,6 @@ public class GATKExtensionsGenerator extends CommandLineProgram { GenomeAnalysisEngine GATKEngine = new GenomeAnalysisEngine(); WalkerManager walkerManager = new WalkerManager(); FilterManager filterManager = new FilterManager(); - // HACK: We're currently relying on the fact that RMDTrackBuilder is used only from RMD type lookups, not - // RMD track location. Therefore, no sequence dictionary is required. In the future, we should separate - // RMD track lookups from track creation. - RMDTrackBuilder trackBuilder = new RMDTrackBuilder(null,null,ValidationExclusion.TYPE.ALL); /** * Required main method implementation. @@ -147,7 +141,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram { String clpConstructor = String.format("analysisName = \"%s\"%njavaMainClass = \"%s\"%n", clpClassName, clp.getName()); writeClass("org.broadinstitute.sting.queue.function.JavaCommandLineFunction", clpClassName, - false, clpConstructor, ArgumentDefinitionField.getArgumentFields(parser,clp), dependents, false); + false, clpConstructor, ArgumentDefinitionField.getArgumentFields(parser,clp), dependents); if (clp == CommandLineGATK.class) { for (Entry>> walkersByPackage: walkerManager.getWalkerNamesByPackage(false).entrySet()) { @@ -169,7 +163,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram { } writeClass(GATK_EXTENSIONS_PACKAGE_NAME + "." + clpClassName, walkerName, - isScatter, constructor, argumentFields, dependents, true); + isScatter, constructor, argumentFields, dependents); } catch (Exception e) { throw new ReviewedStingException("Error generating wrappers for walker " + walkerType, e); } @@ -242,8 +236,8 @@ public class GATKExtensionsGenerator extends CommandLineProgram { */ private void writeClass(String baseClass, String className, boolean isScatter, String constructor, List argumentFields, - Set> dependents, boolean isGATKWalker) throws IOException { - String content = getContent(CLASS_TEMPLATE, baseClass, className, constructor, isScatter, "", argumentFields, dependents, isGATKWalker); + Set> dependents) throws IOException { + String content = getContent(CLASS_TEMPLATE, baseClass, className, constructor, isScatter, "", argumentFields, dependents); writeFile(GATK_EXTENSIONS_PACKAGE_NAME + "." + className, content); } @@ -257,7 +251,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram { */ private void writeFilter(String className, List argumentFields, Set> dependents) throws IOException { String content = getContent(TRAIT_TEMPLATE, "org.broadinstitute.sting.queue.function.CommandLineFunction", - className, "", false, String.format(" + \" -read_filter %s\"", className), argumentFields, dependents, false); + className, "", false, String.format(" + \" -read_filter %s\"", className), argumentFields, dependents); writeFile(GATK_EXTENSIONS_PACKAGE_NAME + "." + className, content); } @@ -351,8 +345,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram { */ private static String getContent(String scalaTemplate, String baseClass, String className, String constructor, boolean isScatter, String commandLinePrefix, - List argumentFields, Set> dependents, - boolean isGATKWalker) { + List argumentFields, Set> dependents) { StringBuilder arguments = new StringBuilder(); StringBuilder commandLine = new StringBuilder(commandLinePrefix); @@ -376,9 +369,6 @@ public class GATKExtensionsGenerator extends CommandLineProgram { if (isGather) importSet.add("import org.broadinstitute.sting.commandline.Gather"); - // Needed for ShellUtils.escapeShellArgument() - importSet.add("import org.broadinstitute.sting.queue.util.ShellUtils"); - // Sort the imports so that the are always in the same order. List sortedImports = new ArrayList(importSet); Collections.sort(sortedImports); @@ -386,10 +376,8 @@ public class GATKExtensionsGenerator extends CommandLineProgram { StringBuffer freezeFieldOverride = new StringBuffer(); for (String freezeField: freezeFields) freezeFieldOverride.append(freezeField); - if (freezeFieldOverride.length() > 0 || isGATKWalker) { - freezeFieldOverride.insert(0, String.format("override def freezeFieldValues = {%nsuper.freezeFieldValues%n")); - if ( isGATKWalker ) - freezeFieldOverride.append(String.format("if ( num_threads.isDefined ) nCoresRequest = num_threads%n")); + if (freezeFieldOverride.length() > 0) { + freezeFieldOverride.insert(0, String.format("override def freezeFieldValues() {%nsuper.freezeFieldValues()%n")); freezeFieldOverride.append(String.format("}%n%n")); } diff --git a/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java b/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java index 345161416..6941b888b 100644 --- a/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java +++ b/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java @@ -145,7 +145,7 @@ public class GenomeLoc implements Comparable, Serializable, HasGenome } return new GenomeLoc(getContig(), this.contigIndex, - Math.min(getStart(), that.getStart()), + Math.min( getStart(), that.getStart() ), Math.max( getStop(), that.getStop()) ); } @@ -465,4 +465,8 @@ public class GenomeLoc implements Comparable, Serializable, HasGenome private final static double overlapPercent(final GenomeLoc gl1, final GenomeLoc gl2) { return (1.0 * gl1.intersect(gl2).size()) / gl1.size(); } + + public long sizeOfOverlap( final GenomeLoc that ) { + return ( this.overlapsP(that) ? Math.min( getStop(), that.getStop() ) - Math.max( getStart(), that.getStart() ) : 0L ); + } } diff --git a/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRead.java b/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRead.java new file mode 100644 index 000000000..8d08a29b6 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRead.java @@ -0,0 +1,19 @@ +package org.broadinstitute.sting.utils.activeregion; + +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; + +/** + * Created by IntelliJ IDEA. + * User: rpoplin + * Date: 1/4/12 + */ + +public class ActiveRead { + final public GATKSAMRecord read; + final public boolean isPrimaryRegion; + + ActiveRead( final GATKSAMRecord read, final boolean isPrimaryRegion ) { + this.read = read; + this.isPrimaryRegion = isPrimaryRegion; + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRegion.java b/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRegion.java new file mode 100644 index 000000000..e8908480c --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRegion.java @@ -0,0 +1,55 @@ +package org.broadinstitute.sting.utils.activeregion; + +import net.sf.picard.reference.IndexedFastaSequenceFile; +import org.broadinstitute.sting.utils.GenomeLoc; +import org.broadinstitute.sting.utils.GenomeLocParser; +import org.broadinstitute.sting.utils.HasGenomeLocation; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; + +import java.util.ArrayList; + +/** + * Created by IntelliJ IDEA. + * User: rpoplin + * Date: 1/4/12 + */ + +public class ActiveRegion implements HasGenomeLocation { + + private final ArrayList reads = new ArrayList(); + private byte[] reference = null; + private final GenomeLoc loc; + private GenomeLoc referenceLoc = null; + private final GenomeLocParser genomeLocParser; + public final boolean isActive; + + public ActiveRegion( final GenomeLoc loc, final boolean isActive, final GenomeLocParser genomeLocParser ) { + this.loc = loc; + this.isActive = isActive; + this.genomeLocParser = genomeLocParser; + referenceLoc = loc; + } + + // add each read to the bin and extend the reference genome loc if needed + public void add( final GATKSAMRecord read, final boolean isPrimaryRegion ) { + referenceLoc = referenceLoc.union( genomeLocParser.createGenomeLoc( read ) ); + reads.add( new ActiveRead(read, isPrimaryRegion) ); + } + + public ArrayList getReads() { return reads; } + + public byte[] getReference( final IndexedFastaSequenceFile referenceReader ) { + // set up the reference if we haven't done so yet + if ( reference == null ) { + reference = referenceReader.getSubsequenceAt(referenceLoc.getContig(), referenceLoc.getStart(), referenceLoc.getStop()).getBases(); + } + + return reference; + } + + public GenomeLoc getLocation() { return loc; } + + public GenomeLoc getReferenceLocation() { return referenceLoc; } + + public int size() { return reads.size(); } +} \ No newline at end of file diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java index 646ede836..d89c9ef6f 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java @@ -265,7 +265,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest { WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec( baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "indelAllelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,100,000", 1, - Arrays.asList("fa4f3ee67d98b64102a8a3ec81a3bc81")); + Arrays.asList("c60a44ba94a80a0cb1fba8b6f90a13cd")); executeTest("test MultiSample Pilot2 indels with alleles passed in", spec1); } @@ -275,7 +275,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest { baseCommandIndels + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "indelAllelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,100,000", 1, - Arrays.asList("df90890e43d735573a3b3e4f289ca46b")); + Arrays.asList("36ce53ae4319718ad9c8ae391deebc8c")); executeTest("test MultiSample Pilot2 indels with alleles passed in and emitting all sites", spec2); } @@ -285,7 +285,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest { WalkerTest.WalkerTestSpec spec3 = new WalkerTest.WalkerTestSpec( baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "ALL.wgs.union_v2.20101123.indels.sites.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,080,000", 1, - Arrays.asList("cff6dd0f4eb1ef0b6fc476da6ffead19")); + Arrays.asList("d356cbaf240d7025d1aecdabaff3a3e0")); executeTest("test MultiSample Pilot2 indels with complicated records", spec3); } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalIntegrationTest.java index b3555b145..5c3a43c96 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalIntegrationTest.java @@ -450,4 +450,21 @@ public class VariantEvalIntegrationTest extends WalkerTest { ); executeTest("testIntervalStrat", spec); } + + @Test + public void testModernVCFWithLargeIndels() { + WalkerTestSpec spec = new WalkerTestSpec( + buildCommandLine( + "-T VariantEval", + "-R " + b37KGReference, + "-eval " + validationDataLocation + "/NA12878.HiSeq.WGS.b37_decoy.indel.recalibrated.vcf", + "-L 20", + "-D " + b37dbSNP132, + "-o %s" + ), + 1, + Arrays.asList("a6f8b32fa732632da13dfe3ddcc73cef") + ); + executeTest("testModernVCFWithLargeIndels", spec); + } } diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/DataProcessingPipeline.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/DataProcessingPipeline.scala index 621afe817..e26541e98 100755 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/DataProcessingPipeline.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/DataProcessingPipeline.scala @@ -29,14 +29,14 @@ class DataProcessingPipeline extends QScript { var reference: File = _ @Input(doc="dbsnp ROD to use (must be in VCF format)", fullName="dbsnp", shortName="D", required=true) - var dbSNP: List[File] = List() + var dbSNP: Seq[File] = Seq() /**************************************************************************** * Optional Parameters ****************************************************************************/ @Input(doc="extra VCF files to use as reference indels for Indel Realignment", fullName="extra_indels", shortName="indels", required=false) - var indels: List[File] = List() + var indels: Seq[File] = Seq() @Input(doc="The path to the binary of bwa (usually BAM files have already been mapped - but if you want to remap this is the option)", fullName="path_to_bwa", shortName="bwa", required=false) var bwaPath: File = _ @@ -118,13 +118,13 @@ class DataProcessingPipeline extends QScript { // Because the realignment only happens after these scripts are executed, in case you are using // bwa realignment, this function will operate over the original bam files and output over the // (to be realigned) bam files. - def createSampleFiles(bamFiles: List[File], realignedBamFiles: List[File]): Map[String, List[File]] = { + def createSampleFiles(bamFiles: Seq[File], realignedBamFiles: Seq[File]): Map[String, Seq[File]] = { // Creating a table with SAMPLE information from each input BAM file - val sampleTable = scala.collection.mutable.Map.empty[String, List[File]] + val sampleTable = scala.collection.mutable.Map.empty[String, Seq[File]] val realignedIterator = realignedBamFiles.iterator for (bam <- bamFiles) { - val rBam = realignedIterator.next // advance to next element in the realignedBam list so they're in sync. + val rBam = realignedIterator.next() // advance to next element in the realignedBam list so they're in sync. val samReader = new SAMFileReader(bam) val header = samReader.getFileHeader @@ -138,12 +138,12 @@ class DataProcessingPipeline extends QScript { for (rg <- readGroups) { val sample = rg.getSample if (!sampleTable.contains(sample)) - sampleTable(sample) = List(rBam) + sampleTable(sample) = Seq(rBam) else if ( !sampleTable(sample).contains(rBam)) sampleTable(sample) :+= rBam } } - return sampleTable.toMap + sampleTable.toMap } // Rebuilds the Read Group string to give BWA @@ -161,8 +161,8 @@ class DataProcessingPipeline extends QScript { // Takes a list of processed BAM files and realign them using the BWA option requested (bwase or bwape). // Returns a list of realigned BAM files. - def performAlignment(bams: List[File]): List[File] = { - var realignedBams: List[File] = List() + def performAlignment(bams: Seq[File]): Seq[File] = { + var realignedBams: Seq[File] = Seq() var index = 1 for (bam <- bams) { // first revert the BAM file to the original qualities @@ -194,10 +194,10 @@ class DataProcessingPipeline extends QScript { realignedBams :+= rgRealignedBamFile index = index + 1 } - return realignedBams + realignedBams } - def getIndelCleaningModel(): ConsensusDeterminationModel = { + def getIndelCleaningModel: ConsensusDeterminationModel = { if (cleaningModel == "KNOWNS_ONLY") ConsensusDeterminationModel.KNOWNS_ONLY else if (cleaningModel == "USE_SW") @@ -206,17 +206,17 @@ class DataProcessingPipeline extends QScript { ConsensusDeterminationModel.USE_READS } - def revertBams(bams: List[File], removeAlignmentInformation: Boolean): List[File] = { - var revertedBAMList: List[File] = List() + def revertBams(bams: Seq[File], removeAlignmentInformation: Boolean): Seq[File] = { + var revertedBAMList: Seq[File] = Seq() for (bam <- bams) revertedBAMList :+= revertBAM(bam, removeAlignmentInformation) - return revertedBAMList + revertedBAMList } def revertBAM(bam: File, removeAlignmentInformation: Boolean): File = { val revertedBAM = swapExt(bam, ".bam", ".reverted.bam") add(revert(bam, revertedBAM, removeAlignmentInformation)) - return revertedBAM + revertedBAM } /**************************************************************************** @@ -224,22 +224,22 @@ class DataProcessingPipeline extends QScript { ****************************************************************************/ - def script = { + def script() { // final output list of processed bam files - var cohortList: List[File] = List() + var cohortList: Seq[File] = Seq() // sets the model for the Indel Realigner - cleanModelEnum = getIndelCleaningModel() + cleanModelEnum = getIndelCleaningModel // keep a record of the number of contigs in the first bam file in the list - val bams = QScriptUtils.createListFromFile(input) + val bams = QScriptUtils.createSeqFromFile(input) if (nContigs < 0) nContigs = QScriptUtils.getNumberOfContigs(bams(0)) val realignedBAMs = if (useBWApe || useBWAse || useBWAsw) {performAlignment(bams)} else {revertBams(bams, false)} // generate a BAM file per sample joining all per lane files if necessary - val sampleBAMFiles: Map[String, List[File]] = createSampleFiles(bams, realignedBAMs) + val sampleBAMFiles: Map[String, Seq[File]] = createSampleFiles(bams, realignedBAMs) // if this is a 'knowns only' indel realignment run, do it only once for all samples. val globalIntervals = new File(outputDir + projectName + ".intervals") @@ -317,7 +317,7 @@ class DataProcessingPipeline extends QScript { this.maxRecordsInRam = 100000 } - case class target (inBams: List[File], outIntervals: File) extends RealignerTargetCreator with CommandLineGATKArgs { + case class target (inBams: Seq[File], outIntervals: File) extends RealignerTargetCreator with CommandLineGATKArgs { if (cleanModelEnum != ConsensusDeterminationModel.KNOWNS_ONLY) this.input_file = inBams this.out = outIntervals @@ -330,7 +330,7 @@ class DataProcessingPipeline extends QScript { this.jobName = queueLogDir + outIntervals + ".target" } - case class clean (inBams: List[File], tIntervals: File, outBam: File) extends IndelRealigner with CommandLineGATKArgs { + case class clean (inBams: Seq[File], tIntervals: File, outBam: File) extends IndelRealigner with CommandLineGATKArgs { this.input_file = inBams this.targetIntervals = tIntervals this.out = outBam @@ -347,11 +347,11 @@ class DataProcessingPipeline extends QScript { case class cov (inBam: File, outRecalFile: File) extends CountCovariates with CommandLineGATKArgs { this.knownSites ++= qscript.dbSNP - this.covariate ++= List("ReadGroupCovariate", "QualityScoreCovariate", "CycleCovariate", "DinucCovariate") + this.covariate ++= Seq("ReadGroupCovariate", "QualityScoreCovariate", "CycleCovariate", "DinucCovariate") this.input_file :+= inBam this.recal_file = outRecalFile if (!defaultPlatform.isEmpty) this.default_platform = defaultPlatform - if (!qscript.intervalString.isEmpty()) this.intervalsString ++= List(qscript.intervalString) + if (!qscript.intervalString.isEmpty) this.intervalsString ++= Seq(qscript.intervalString) else if (qscript.intervals != null) this.intervals :+= qscript.intervals this.scatterCount = nContigs this.analysisName = queueLogDir + outRecalFile + ".covariates" @@ -363,7 +363,7 @@ class DataProcessingPipeline extends QScript { this.recal_file = inRecalFile this.baq = CalculationMode.CALCULATE_AS_NECESSARY this.out = outBam - if (!qscript.intervalString.isEmpty()) this.intervalsString ++= List(qscript.intervalString) + if (!qscript.intervalString.isEmpty) this.intervalsString ++= Seq(qscript.intervalString) else if (qscript.intervals != null) this.intervals :+= qscript.intervals this.no_pg_tag = qscript.testMode this.scatterCount = nContigs @@ -395,7 +395,7 @@ class DataProcessingPipeline extends QScript { this.jobName = queueLogDir + outBam + ".dedup" } - case class joinBams (inBams: List[File], outBam: File) extends MergeSamFiles with ExternalCommonArgs { + case class joinBams (inBams: Seq[File], outBam: File) extends MergeSamFiles with ExternalCommonArgs { this.input = inBams this.output = outBam this.analysisName = queueLogDir + outBam + ".joinBams" @@ -495,7 +495,7 @@ class DataProcessingPipeline extends QScript { this.jobName = queueLogDir + outBam + ".bwasw" } - case class writeList(inBams: List[File], outBamList: File) extends ListWriterFunction { + case class writeList(inBams: Seq[File], outBamList: File) extends ListWriterFunction { this.inputFiles = inBams this.listFile = outBamList this.analysisName = queueLogDir + outBamList + ".bamList" diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala index c06601a2d..b50bf3d67 100755 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala @@ -46,20 +46,24 @@ class MethodsDevelopmentCallingPipeline extends QScript { val bamList: File, val goldStandard_VCF: File, val intervals: String, - val titvTarget: Double, - val trancheTarget: Double, + val indelTranchTarget: Double, + val snpTrancheTarget: Double, val isLowpass: Boolean, val isExome: Boolean, val nSamples: Int) { val name = qscript.outputDir + baseName val clusterFile = new File(name + ".clusters") - val rawVCF = new File(name + ".raw.vcf") + val rawSnpVCF = new File(name + ".raw.vcf") val rawIndelVCF = new File(name + ".raw.indel.vcf") val filteredIndelVCF = new File(name + ".filtered.indel.vcf") - val recalibratedVCF = new File(name + ".recalibrated.vcf") - val tranchesFile = new File(name + ".tranches") - val vqsrRscript = name + ".vqsr.r" - val recalFile = new File(name + ".tranches.recal") + val recalibratedSnpVCF = new File(name + ".snp.recalibrated.vcf") + val recalibratedIndelVCF = new File(name + ".indel.recalibrated.vcf") + val tranchesSnpFile = new File(name + ".snp.tranches") + val tranchesIndelFile = new File(name + ".indel.tranches") + val vqsrSnpRscript = name + ".snp.vqsr.r" + val vqsrIndelRscript = name + ".indel.vqsr.r" + val recalSnpFile = new File(name + ".snp.tranches.recal") + val recalIndelFile = new File(name + ".indel.tranches.recal") val goldStandardRecalibratedVCF = new File(name + "goldStandard.recalibrated.vcf") val goldStandardTranchesFile = new File(name + "goldStandard.tranches") val goldStandardRecalFile = new File(name + "goldStandard.tranches.recal") @@ -88,6 +92,7 @@ class MethodsDevelopmentCallingPipeline extends QScript { val training_1000G = "/humgen/1kg/processing/official_release/phase1/projectConsensus/phase1.wgs.projectConsensus.v2b.recal.highQuality.vcf" val badSites_1000G = "/humgen/1kg/processing/official_release/phase1/projectConsensus/phase1.wgs.projectConsensus.v2b.recal.terrible.vcf" val projectConsensus_1000G = "/humgen/1kg/processing/official_release/phase1/projectConsensus/ALL.wgs.projectConsensus_v2b.20101123.snps.sites.vcf" + val indelGoldStandardCallset = "/humgen/gsa-hpprojects/GATK/data/Comparisons/Unvalidated/GoldStandardIndel/gold.standard.indel.MillsAnd1000G.b37.vcf" val lowPass: Boolean = true val exome: Boolean = true @@ -101,69 +106,69 @@ class MethodsDevelopmentCallingPipeline extends QScript { "NA12878_gold" -> new Target("NA12878.goldStandard", hg19, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/dev/carneiro/NA12878/data/goldStandard.list"), new File("/humgen/gsa-hpprojects/dev/carneiro/NA12878/analysis/snps/NA12878.HiSeq19.filtered.vcf"), // ** There is no gold standard for the gold standard ** - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.noChrY.hg19.intervals", 2.14, 99.0, lowPass, !exome, 391), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.noChrY.hg19.intervals", 90.0, 99.0, lowPass, !exome, 391), "NA12878_wgs_b37" -> new Target("NA12878.HiSeq.WGS.b37", hg19, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/NA12878.HiSeq.WGS.bwa.cleaned.recal.hg19.bam"), new File("/humgen/gsa-hpprojects/dev/carneiro/NA12878/analysis/snps/NA12878.HiSeq19.filtered.vcf"), - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.noChrY.hg19.intervals", 2.14, 99.0, !lowPass, !exome, 1), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.noChrY.hg19.intervals", 90.0, 99.0, !lowPass, !exome, 1), "NA12878_wgs_decoy" -> new Target("NA12878.HiSeq.WGS.b37_decoy", b37_decoy, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WGS.b37_decoy.NA12878.clean.dedup.recal.bam"), new File("/humgen/gsa-hpprojects/dev/carneiro/NA12878/analysis/snps/NA12878.HiSeq19.filtered.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.noChrY.hg19.intervals", 2.14, 99.0, !lowPass, !exome, 1), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.noChrY.hg19.intervals", 90.0, 99.0, !lowPass, !exome, 1), "NA12878_wgs_hg18" -> new Target("NA12878.HiSeq.WGS.hg18", hg18, dbSNP_hg18_129, hapmap_hg18, "/humgen/gsa-hpprojects/dev/depristo/oneOffProjects/1000GenomesProcessingPaper/wgs.v13/HiSeq.WGS.cleaned.indels.10.mask", new File("/humgen/gsa-hpprojects/NA12878Collection/bams/NA12878.HiSeq.WGS.bwa.cleaned.recal.bam"), new File("/home/radon01/depristo/work/oneOffProjects/1000GenomesProcessingPaper/wgs.v13/HiSeq.WGS.cleaned.ug.snpfiltered.indelfiltered.vcf"), - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg18.intervals", 2.14, 99.0, !lowPass, !exome, 1), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg18.intervals", 90.0, 99.0, !lowPass, !exome, 1), "NA12878_wex_b37" -> new Target("NA12878.HiSeq.WEx.b37", hg19, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/seq/picard_aggregation/C339/NA12878/v3/NA12878.bam"), new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 3.3, 98.0, !lowPass, exome, 1), + "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 90.0, 98.0, !lowPass, exome, 1), "NA12878_wex_hg18" -> new Target("NA12878.HiSeq.WEx.hg18", hg18, dbSNP_hg18_129, hapmap_hg18, "/humgen/gsa-hpprojects/dev/depristo/oneOffProjects/1000GenomesProcessingPaper/wgs.v13/GA2.WEx.cleaned.indels.10.mask", new File("/humgen/gsa-hpprojects/NA12878Collection/bams/NA12878.WEx.cleaned.recal.bam"), new File("/home/radon01/depristo/work/oneOffProjects/1000GenomesProcessingPaper/wgs.v13/GA2.WEx.cleaned.ug.snpfiltered.indelfiltered.vcf"), - "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.targets.interval_list", 3.3, 98.0, !lowPass, exome, 1), + "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.targets.interval_list", 90.0, 98.0, !lowPass, exome, 1), "NA12878_wex_decoy" -> new Target("NA12878.HiSeq.WEx.b37_decoy", b37_decoy, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WEx.b37_decoy.NA12878.clean.dedup.recal.bam"), new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 3.3, 98.0, !lowPass, exome, 1), + "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 90.0, 98.0, !lowPass, exome, 1), "CEUTrio_wex_b37" -> new Target("CEUTrio.HiSeq.WEx.b37", hg19, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WEx.bwa.cleaned.recal.bam"), new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"), - "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 3.3, 98.0, !lowPass, exome, 3), + "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 90.0, 98.0, !lowPass, exome, 3), "CEUTrio_wgs_b37" -> new Target("CEUTrio.HiSeq.WGS.b37", hg19, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WGS.bwa.cleaned.recal.bam"), new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 2.3, 99.0, !lowPass, !exome, 3), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 90.0, 99.0, !lowPass, !exome, 3), "CEUTrio_wex_decoy" -> new Target("CEUTrio.HiSeq.WEx.b37_decoy", b37_decoy, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WEx.b37_decoy.list"), new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 3.3, 98.0, !lowPass, exome, 3), + "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 90.0, 98.0, !lowPass, exome, 3), "CEUTrio_wgs_decoy" -> new Target("CEUTrio.HiSeq.WGS.b37_decoy", b37_decoy, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WGS.b37_decoy.list"), new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 2.3, 99.0, !lowPass, !exome, 3), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 90.0, 99.0, !lowPass, !exome, 3), "GA2hg19" -> new Target("NA12878.GA2.hg19", hg19, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/NA12878.GA2.WGS.bwa.cleaned.hg19.bam"), new File("/humgen/gsa-hpprojects/dev/carneiro/NA12878/analysis/snps/NA12878.GA2.hg19.filtered.vcf"), - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 2.14, 99.0, !lowPass, !exome, 1), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 90.0, 99.0, !lowPass, !exome, 1), "FIN" -> new Target("FIN", b37, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/1kg/processing/pipeline_test_bams/FIN.79sample.Nov2010.chr20.bam"), new File("/humgen/gsa-hpprojects/dev/data/AugChr20Calls_v4_3state/ALL.august.v4.chr20.filtered.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.hg19.intervals", 2.3, 99.0, lowPass, !exome, 79), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.hg19.intervals", 90.0, 99.0, lowPass, !exome, 79), "TGPWExGdA" -> new Target("1000G.WEx.GdA", b37, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/1kg/processing/pipeline_test_bams/Barcoded_1000G_WEx_Reduced_Plate_1.cleaned.list"), // BUGBUG: reduce from 60 to 20 people new File("/humgen/gsa-scr1/delangel/NewUG/calls/AugustRelease.filtered_Q50_QD5.0_SB0.0.allSamples.SNPs_hg19.WEx_UG_newUG_MQC.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 2.6, 99.0, !lowPass, exome, 96), + "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 90.0, 99.0, !lowPass, exome, 96), "LowPassN60" -> new Target("lowpass.N60", b36, dbSNP_b36, hapmap_b36, indelMask_b36, new File("/humgen/1kg/analysis/bamsForDataProcessingPapers/lowpass_b36/lowpass.chr20.cleaned.matefixed.bam"), // the bam list to call from new File("/home/radon01/depristo/work/oneOffProjects/VQSRCutByNRS/lowpass.N60.chr20.filtered.vcf"), // the gold standard VCF file to run through the VQSR - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.b36.intervals", 2.3, 99.0, lowPass, !exome, 60), // chunked interval list to use with Queue's scatter/gather functionality + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.b36.intervals", 90.0, 99.0, lowPass, !exome, 60), // chunked interval list to use with Queue's scatter/gather functionality "LowPassEUR363Nov" -> new Target("EUR.nov2010", b37, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/1kg/processing/pipeline_test_bams/EUR.363sample.Nov2010.chr20.bam"), new File("/humgen/gsa-hpprojects/dev/data/AugChr20Calls_v4_3state/ALL.august.v4.chr20.filtered.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.hg19.intervals", 2.3, 99.0, lowPass, !exome, 363) + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.hg19.intervals", 90.0, 99.0, lowPass, !exome, 363) ) @@ -181,15 +186,15 @@ class MethodsDevelopmentCallingPipeline extends QScript { val goldStandard = true for (target <- targets) { if( !skipCalling ) { - if (!noIndels) add(new indelCall(target), new indelFilter(target), new indelEvaluation(target)) + if (!noIndels) add(new indelCall(target), new indelRecal(target), new indelCut(target), new indelEvaluation(target)) add(new snpCall(target)) - add(new VQSR(target, !goldStandard)) - add(new applyVQSR(target, !goldStandard)) + add(new snpRecal(target, !goldStandard)) + add(new snpCut(target, !goldStandard)) add(new snpEvaluation(target)) } if ( runGoldStandard ) { - add(new VQSR(target, goldStandard)) - add(new applyVQSR(target, goldStandard)) + add(new snpRecal(target, goldStandard)) + add(new snpCut(target, goldStandard)) } } } @@ -222,7 +227,7 @@ class MethodsDevelopmentCallingPipeline extends QScript { this.min_base_quality_score = minimumBaseQuality if (qscript.deletions >= 0) this.max_deletion_fraction = qscript.deletions - this.out = t.rawVCF + this.out = t.rawSnpVCF this.glm = org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeLikelihoodsCalculationModel.Model.SNP this.baq = if (noBAQ || t.isExome) {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF} else {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY} this.analysisName = t.name + "_UGs" @@ -257,79 +262,114 @@ class MethodsDevelopmentCallingPipeline extends QScript { this.jobName = queueLogDir + t.name + ".indelfilter" } - // 3.) Variant Quality Score Recalibration - Generate Recalibration table - class VQSR(t: Target, goldStandard: Boolean) extends VariantRecalibrator with UNIVERSAL_GATK_ARGS { + class VQSRBase(t: Target) extends VariantRecalibrator with UNIVERSAL_GATK_ARGS { this.nt = 2 this.reference_sequence = t.reference this.intervalsString ++= List(t.intervals) - this.input :+= ( if ( goldStandard ) { t.goldStandard_VCF } else { t.rawVCF } ) - this.resource :+= new TaggedFile( t.hapmapFile, "training=true,truth=true,prior=15.0" ) - this.resource :+= new TaggedFile( omni_b37, "training=true,truth=true,prior=12.0" ) - this.resource :+= new TaggedFile( training_1000G, "training=true,prior=10.0" ) + this.allPoly = true + this.tranche ++= List("100.0", "99.9", "99.5", "99.3", "99.0", "98.9", "98.8", "98.5", "98.4", "98.3", "98.2", "98.1", "98.0", "97.9", "97.8", "97.5", "97.0", "95.0", "90.0") + } + + class snpRecal(t: Target, goldStandard: Boolean) extends VQSRBase(t) with UNIVERSAL_GATK_ARGS { + this.input :+= ( if ( goldStandard ) { t.goldStandard_VCF } else { t.rawSnpVCF } ) + this.resource :+= new TaggedFile( t.hapmapFile, "known=false,training=true,truth=true,prior=15.0" ) + this.resource :+= new TaggedFile( omni_b37, "known=false,training=true,truth=true,prior=12.0" ) + this.resource :+= new TaggedFile( training_1000G, "known=false,training=true,prior=10.0" ) this.resource :+= new TaggedFile( t.dbsnpFile, "known=true,prior=2.0" ) this.resource :+= new TaggedFile( projectConsensus_1000G, "prior=8.0" ) this.use_annotation ++= List("QD", "HaplotypeScore", "MQRankSum", "ReadPosRankSum", "MQ", "FS") - if(t.nSamples >= 10) { // InbreedingCoeff is a population-wide statistic that requires at least 10 samples to calculate - this.use_annotation ++= List("InbreedingCoeff") - } - if(!t.isExome) { + if(t.nSamples >= 10) + this.use_annotation ++= List("InbreedingCoeff") // InbreedingCoeff is a population-wide statistic that requires at least 10 samples to calculate + if(!t.isExome) this.use_annotation ++= List("DP") - } else { // exome specific parameters + else { // exome specific parameters this.resource :+= new TaggedFile( badSites_1000G, "bad=true,prior=2.0" ) this.mG = 6 - if(t.nSamples <= 3) { // very few exome samples means very few variants + if(t.nSamples <= 3) { // very few exome samples means very few variants this.mG = 4 this.percentBad = 0.04 } } - this.tranches_file = if ( goldStandard ) { t.goldStandardTranchesFile } else { t.tranchesFile } - this.recal_file = if ( goldStandard ) { t.goldStandardRecalFile } else { t.recalFile } - this.allPoly = true - this.tranche ++= List("100.0", "99.9", "99.5", "99.3", "99.0", "98.9", "98.8", "98.5", "98.4", "98.3", "98.2", "98.1", "98.0", "97.9", "97.8", "97.5", "97.0", "95.0", "90.0") - this.rscript_file = t.vqsrRscript - this.analysisName = t.name + "_VQSR" - this.jobName = queueLogDir + t.name + ".VQSR" + this.tranches_file = if ( goldStandard ) { t.goldStandardTranchesFile } else { t.tranchesSnpFile } + this.recal_file = if ( goldStandard ) { t.goldStandardRecalFile } else { t.recalSnpFile } + this.rscript_file = t.vqsrSnpRscript + this.mode = org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibratorArgumentCollection.Mode.SNP + this.analysisName = t.name + "_VQSRs" + this.jobName = queueLogDir + t.name + ".snprecal" } + class indelRecal(t: Target) extends VQSRBase(t) with UNIVERSAL_GATK_ARGS { + this.input :+= t.rawIndelVCF + this.resource :+= new TaggedFile( indelGoldStandardCallset, "known=true,training=true,truth=true,prior=12.0" ) + this.use_annotation ++= List("QD", "HaplotypeScore", "ReadPosRankSum", "FS") + if(t.nSamples >= 10) + this.use_annotation ++= List("InbreedingCoeff") // InbreedingCoeff is a population-wide statistic that requires at least 10 samples to calculate + this.tranches_file = t.tranchesIndelFile + this.recal_file = t.recalIndelFile + this.rscript_file = t.vqsrIndelRscript + this.mode = org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibratorArgumentCollection.Mode.INDEL + this.analysisName = t.name + "_VQSRi" + this.jobName = queueLogDir + t.name + ".indelrecal" + } + + // 4.) Apply the recalibration table to the appropriate tranches - class applyVQSR (t: Target, goldStandard: Boolean) extends ApplyRecalibration with UNIVERSAL_GATK_ARGS { + class applyVQSRBase (t: Target) extends ApplyRecalibration with UNIVERSAL_GATK_ARGS { this.memoryLimit = 6 this.reference_sequence = t.reference this.intervalsString ++= List(t.intervals) - this.input :+= ( if ( goldStandard ) { t.goldStandard_VCF } else { t.rawVCF } ) - this.tranches_file = if ( goldStandard ) { t.goldStandardTranchesFile } else { t.tranchesFile} - this.recal_file = if ( goldStandard ) { t.goldStandardRecalFile } else { t.recalFile } - this.ts_filter_level = t.trancheTarget - this.out = t.recalibratedVCF - this.analysisName = t.name + "_AVQSR" - this.jobName = queueLogDir + t.name + ".applyVQSR" } + class snpCut (t: Target, goldStandard: Boolean) extends applyVQSRBase(t) { + this.input :+= ( if ( goldStandard ) { t.goldStandard_VCF } else { t.rawSnpVCF } ) + this.tranches_file = if ( goldStandard ) { t.goldStandardTranchesFile } else { t.tranchesSnpFile} + this.recal_file = if ( goldStandard ) { t.goldStandardRecalFile } else { t.recalSnpFile } + this.ts_filter_level = t.snpTrancheTarget + this.mode = org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibratorArgumentCollection.Mode.SNP + this.out = t.recalibratedSnpVCF + this.analysisName = t.name + "_AVQSRs" + this.jobName = queueLogDir + t.name + ".snpcut" + } + + class indelCut (t: Target) extends applyVQSRBase(t) { + this.input :+= t.rawIndelVCF + this.tranches_file = t.tranchesIndelFile + this.recal_file = t.recalIndelFile + this.ts_filter_level = t.indelTranchTarget + this.mode = org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibratorArgumentCollection.Mode.INDEL + this.out = t.recalibratedIndelVCF + this.analysisName = t.name + "_AVQSRi" + this.jobName = queueLogDir + t.name + ".indelcut" + } + + // 5.) Variant Evaluation Base(OPTIONAL) class EvalBase(t: Target) extends VariantEval with UNIVERSAL_GATK_ARGS { this.memoryLimit = 3 - this.reference_sequence = t.reference this.comp :+= new TaggedFile(t.hapmapFile, "hapmap" ) - this.intervalsString ++= List(t.intervals) this.D = new File(t.dbsnpFile) + this.reference_sequence = t.reference + this.intervalsString ++= List(t.intervals) this.sample = samples } // 5a.) SNP Evaluation (OPTIONAL) based on the cut vcf class snpEvaluation(t: Target) extends EvalBase(t) { if (t.reference == b37 || t.reference == hg19) this.comp :+= new TaggedFile( omni_b37, "omni" ) - this.eval :+= t.recalibratedVCF + this.eval :+= t.recalibratedSnpVCF this.out = t.evalFile this.analysisName = t.name + "_VEs" - this.jobName = queueLogDir + t.name + ".snp.eval" + this.jobName = queueLogDir + t.name + ".snpeval" } // 5b.) Indel Evaluation (OPTIONAL) class indelEvaluation(t: Target) extends EvalBase(t) { - this.eval :+= t.filteredIndelVCF - this.evalModule :+= "IndelStatistics" + this.eval :+= t.recalibratedIndelVCF + this.comp :+= new TaggedFile(indelGoldStandardCallset, "indelGS" ) + this.noEV = true + this.evalModule = List("CompOverlap", "CountVariants", "TiTvVariantEvaluator", "ValidationReport", "IndelStatistics") this.out = t.evalIndelFile this.analysisName = t.name + "_VEi" - this.jobName = queueLogDir + queueLogDir + t.name + ".indel.eval" + this.jobName = queueLogDir + queueLogDir + t.name + ".indeleval" } } diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala index 4896eaed3..2f954713e 100755 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala @@ -53,9 +53,9 @@ class PacbioProcessingPipeline extends QScript { val queueLogDir: String = ".qlog/" - def script = { + def script() { - val fileList: List[File] = QScriptUtils.createListFromFile(input) + val fileList: Seq[File] = QScriptUtils.createSeqFromFile(input) for (file: File <- fileList) { diff --git a/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala b/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala index 32913deb4..7a22e700b 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -36,6 +36,7 @@ import org.broadinstitute.sting.utils.io.IOUtils import org.broadinstitute.sting.utils.help.ApplicationDetails import java.util.{ResourceBundle, Arrays} import org.broadinstitute.sting.utils.text.TextFormattingUtils +import org.apache.commons.io.FilenameUtils /** * Entry point of Queue. Compiles and runs QScripts passed in to the command line. @@ -61,6 +62,7 @@ object QCommandLine extends Logging { CommandLineProgram.start(qCommandLine, argv) try { Runtime.getRuntime.removeShutdownHook(shutdownHook) + qCommandLine.shutdown() } catch { case _ => /* ignore, example 'java.lang.IllegalStateException: Shutdown in progress' */ } @@ -78,10 +80,10 @@ object QCommandLine extends Logging { class QCommandLine extends CommandLineProgram with Logging { @Input(fullName="script", shortName="S", doc="QScript scala file", required=true) @ClassType(classOf[File]) - private var scripts = List.empty[File] + var scripts = Seq.empty[File] @ArgumentCollection - private val settings = new QGraphSettings + val settings = new QGraphSettings private val qScriptManager = new QScriptManager private val qGraph = new QGraph @@ -91,7 +93,7 @@ class QCommandLine extends CommandLineProgram with Logging { private lazy val pluginManager = { qScriptClasses = IOUtils.tempDir("Q-Classes-", "", settings.qSettings.tempDirectory) qScriptManager.loadScripts(scripts, qScriptClasses) - new PluginManager[QScript](classOf[QScript], List(qScriptClasses.toURI.toURL)) + new PluginManager[QScript](classOf[QScript], Seq(qScriptClasses.toURI.toURL)) } QFunction.parsingEngine = new ParsingEngine(this) @@ -101,12 +103,16 @@ class QCommandLine extends CommandLineProgram with Logging { * functions, and then builds and runs a QGraph based on the dependencies. */ def execute = { + if (settings.qSettings.runName == null) + settings.qSettings.runName = FilenameUtils.removeExtension(scripts.head.getName) + qGraph.settings = settings val allQScripts = pluginManager.createAllTypes(); for (script <- allQScripts) { logger.info("Scripting " + pluginManager.getName(script.getClass.asSubclass(classOf[QScript]))) loadArgumentsIntoObject(script) + script.qSettings = settings.qSettings try { script.script() } catch { @@ -120,22 +126,34 @@ class QCommandLine extends CommandLineProgram with Logging { // Execute the job graph qGraph.run() + val functionsAndStatus = qGraph.getFunctionsAndStatus + val success = qGraph.success + // walk over each script, calling onExecutionDone for (script <- allQScripts) { - script.onExecutionDone(qGraph.getFunctionsAndStatus(script.functions), qGraph.success) - if ( ! settings.disableJobReport ) { - val jobStringName = (QScriptUtils.?(settings.jobReportFile)).getOrElse(settings.qSettings.jobNamePrefix + ".jobreport.txt") + val scriptFunctions = functionsAndStatus.filterKeys(f => script.functions.contains(f)) + script.onExecutionDone(scriptFunctions, success) + } - if (!shuttingDown) { - val reportFile = new File(jobStringName) - logger.info("Writing JobLogging GATKReport to file " + reportFile) - QJobReport.printReport(qGraph.getFunctionsAndStatus(script.functions), reportFile) + logger.info("Script %s with %d total jobs".format(if (success) "completed successfully" else "failed", functionsAndStatus.size)) - if ( settings.run ) { - val pdfFile = new File(jobStringName + ".pdf") - logger.info("Plotting JobLogging GATKReport to file " + pdfFile) - QJobReport.plotReport(reportFile, pdfFile) - } + if (!settings.disableJobReport) { + val jobStringName = { + if (settings.jobReportFile != null) + settings.jobReportFile + else + settings.qSettings.runName + ".jobreport.txt" + } + + if (!shuttingDown) { + val reportFile = IOUtils.absolute(settings.qSettings.runDirectory, jobStringName) + logger.info("Writing JobLogging GATKReport to file " + reportFile) + QJobReport.printReport(functionsAndStatus, reportFile) + + if (settings.run) { + val pdfFile = IOUtils.absolute(settings.qSettings.runDirectory, FilenameUtils.removeExtension(jobStringName) + ".pdf") + logger.info("Plotting JobLogging GATKReport to file " + pdfFile) + QJobReport.plotReport(reportFile, pdfFile) } } } @@ -179,20 +197,20 @@ class QCommandLine extends CommandLineProgram with Logging { override def getApplicationDetails : ApplicationDetails = { new ApplicationDetails(createQueueHeader(), - List.empty[String], + Seq.empty[String], ApplicationDetails.createDefaultRunningInstructions(getClass.asInstanceOf[Class[CommandLineProgram]]), "") } - private def createQueueHeader() : List[String] = { - List(String.format("Queue v%s, Compiled %s", getQueueVersion, getBuildTimestamp), - "Copyright (c) 2011 The Broad Institute", + private def createQueueHeader() : Seq[String] = { + Seq(String.format("Queue v%s, Compiled %s", getQueueVersion, getBuildTimestamp), + "Copyright (c) 2012 The Broad Institute", "Please view our documentation at http://www.broadinstitute.org/gsa/wiki", "For support, please view our support site at http://getsatisfaction.com/gsa") } private def getQueueVersion : String = { - var stingResources : ResourceBundle = TextFormattingUtils.loadResourceBundle("StingText") + val stingResources : ResourceBundle = TextFormattingUtils.loadResourceBundle("StingText") if ( stingResources.containsKey("org.broadinstitute.sting.queue.QueueVersion.version") ) { stingResources.getString("org.broadinstitute.sting.queue.QueueVersion.version") @@ -203,7 +221,7 @@ class QCommandLine extends CommandLineProgram with Logging { } private def getBuildTimestamp : String = { - var stingResources : ResourceBundle = TextFormattingUtils.loadResourceBundle("StingText") + val stingResources : ResourceBundle = TextFormattingUtils.loadResourceBundle("StingText") if ( stingResources.containsKey("build.timestamp") ) { stingResources.getString("build.timestamp") diff --git a/public/scala/src/org/broadinstitute/sting/queue/QScript.scala b/public/scala/src/org/broadinstitute/sting/queue/QScript.scala index fce65c997..6f887ea00 100755 --- a/public/scala/src/org/broadinstitute/sting/queue/QScript.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/QScript.scala @@ -27,7 +27,6 @@ package org.broadinstitute.sting.queue import engine.JobRunInfo import org.broadinstitute.sting.queue.function.QFunction import annotation.target.field -import io.Source import util.{StringFileConversions, PrimitiveOptionConversions, Logging} /** @@ -53,6 +52,11 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon type ArgumentCollection = org.broadinstitute.sting.commandline.ArgumentCollection @field type Gather = org.broadinstitute.sting.commandline.Gather @field + /** + * Default settings for QFunctions + */ + var qSettings: QSettings = _ + /** * Builds the CommandLineFunctions that will be used to run this script and adds them to this.functions directly or using the add() utility method. */ @@ -60,18 +64,14 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon /** * A default handler for the onExecutionDone() function. By default this doesn't do anything - * except print out a fine status message. */ def onExecutionDone(jobs: Map[QFunction, JobRunInfo], success: Boolean) { - logger.info("Script %s with %d total jobs".format(if (success) "completed successfully" else "failed", jobs.size)) - // this is too much output - // for ( (f, info) <- jobs ) logger.info(" %s %s".format(f.jobName, info)) } /** * The command line functions that will be executed for this QScript. */ - var functions = List.empty[QFunction] + var functions = Seq.empty[QFunction] /** * Exchanges the extension on a file. @@ -98,22 +98,20 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon * Adds one or more command line functions to be run. * @param functions Functions to add. */ - def add(functions: QFunction*) = { + def add(functions: QFunction*) { functions.foreach(function => function.addOrder = QScript.nextAddOrder) this.functions ++= functions } - def addAll(functions: List[QFunction]) { + def addAll(functions: Seq[QFunction]) { functions.foreach( f => add(f) ) } - - def extractFileEntries(in: File): List[File] = Source.fromFile(in).getLines().toList } object QScript { private var addOrder = 0 private def nextAddOrder = { addOrder += 1 - List(addOrder) + Seq(addOrder) } } diff --git a/public/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala b/public/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala index 512a9f8dd..74487917f 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala @@ -20,7 +20,7 @@ class QScriptManager() extends Logging { * Compiles and loads the scripts in the files into the current classloader. * Heavily based on scala/src/compiler/scala/tools/ant/Scalac.scala */ - def loadScripts(scripts: List[File], tempDir: File) { + def loadScripts(scripts: Seq[File], tempDir: File) { if (scripts.size > 0) { val settings = new Settings((error: String) => logger.error(error)) settings.deprecation.value = true @@ -36,7 +36,7 @@ class QScriptManager() extends Logging { logger.info("Compiling %s QScript%s".format(scripts.size, plural(scripts.size))) logger.debug("Compilation directory: " + settings.outdir.value) - run.compileFiles(scripts.map(new PlainFile(_))) + run.compileFiles(scripts.toList.map(new PlainFile(_))) reporter.printSummary() if (reporter.hasErrors) { diff --git a/public/scala/src/org/broadinstitute/sting/queue/QSettings.scala b/public/scala/src/org/broadinstitute/sting/queue/QSettings.scala index e8ac26a57..d9fed4ce8 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/QSettings.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/QSettings.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -25,15 +25,14 @@ package org.broadinstitute.sting.queue import java.io.File -import org.broadinstitute.sting.commandline.{ArgumentCollection, Argument} -import org.broadinstitute.sting.queue.util.{SystemUtils, EmailSettings} +import org.broadinstitute.sting.commandline.Argument /** * Default settings settable on the command line and passed to CommandLineFunctions. */ class QSettings { - @Argument(fullName="job_name_prefix", shortName="jobPrefix", doc="Default name prefix for compute farm jobs.", required=false) - var jobNamePrefix: String = QSettings.processNamePrefix + @Argument(fullName="run_name", shortName="runName", doc="A name for this run used for various status messages.", required=false) + var runName: String = _ @Argument(fullName="job_project", shortName="jobProject", doc="Default project for compute farm jobs.", required=false) var jobProject: String = _ @@ -45,13 +44,13 @@ class QSettings { var jobPriority: Option[Int] = None @Argument(fullName="job_native_arg", shortName="jobNative", doc="Native arguments to pass to the job runner.", required=false) - var jobNativeArgs: List[String] = Nil + var jobNativeArgs: Seq[String] = Nil @Argument(fullName="job_resource_request", shortName="jobResReq", doc="Resource requests to pass to the job runner.", required=false) - var jobResourceRequests: List[String] = Nil + var jobResourceRequests: Seq[String] = Nil @Argument(fullName="job_environment_name", shortName="jobEnv", doc="Environment names for the job runner.", required=false) - var jobEnvironmentNames: List[String] = Nil + var jobEnvironmentNames: Seq[String] = Nil @Argument(fullName="memory_limit", shortName="memLimit", doc="Default memory limit for jobs, in gigabytes.", required=false) var memoryLimit: Option[Double] = None @@ -77,15 +76,4 @@ class QSettings { @Argument(fullName="job_scatter_gather_directory", shortName="jobSGDir", doc="Default directory to place scatter gather output for compute farm jobs.", required=false) var jobScatterGatherDirectory: File = _ - - @ArgumentCollection - val emailSettings = new EmailSettings -} - -/** - * Default settings settable on the command line and passed to CommandLineFunctions. - */ -object QSettings { - /** A semi-unique job prefix using the host name and the process id. */ - private val processNamePrefix = "Q-" + SystemUtils.pidAtHost } diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/FunctionEdge.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/FunctionEdge.scala index 55ed94267..8225d28ab 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/FunctionEdge.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/FunctionEdge.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.engine import org.broadinstitute.sting.queue.function.QFunction @@ -28,15 +52,18 @@ class FunctionEdge(val function: QFunction, val inputs: QNode, val outputs: QNod val myRunInfo: JobRunInfo = JobRunInfo.default // purely for dryRun testing + /** + * When using reset status this variable tracks the old status + */ + var resetFromStatus: RunnerStatus.Value = null + /** * Initializes with the current status of the function. */ private var currentStatus = { - val isDone = function.isDone - val isFail = function.isFail - if (isFail.isDefined && isFail.get) + if (function.isFail) RunnerStatus.FAILED - else if (isDone.isDefined && isDone.get) + else if (function.isDone) RunnerStatus.DONE else RunnerStatus.PENDING @@ -136,13 +163,15 @@ class FunctionEdge(val function: QFunction, val inputs: QNode, val outputs: QNod * Resets the edge to pending status. */ def resetToPending(cleanOutputs: Boolean) { + if (resetFromStatus == null) + resetFromStatus = currentStatus currentStatus = RunnerStatus.PENDING if (cleanOutputs) function.deleteOutputs() runner = null } - override def dotString = function.dotString + override def shortDescription = function.shortDescription /** * Returns the path to the file to use for logging errors. diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/InProcessRunner.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/InProcessRunner.scala index d006cde4b..be5622360 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/InProcessRunner.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/InProcessRunner.scala @@ -3,7 +3,8 @@ package org.broadinstitute.sting.queue.engine import org.broadinstitute.sting.queue.function.InProcessFunction import java.util.Date import org.broadinstitute.sting.utils.Utils -import org.apache.commons.io.FileUtils +import org.apache.commons.io.{IOUtils, FileUtils} +import java.io.PrintStream /** * Runs a function that executes in process and does not fork out an external process. @@ -16,12 +17,24 @@ class InProcessRunner(val function: InProcessFunction) extends JobRunner[InProce getRunInfo.exechosts = Utils.resolveHostname() runStatus = RunnerStatus.RUNNING - function.run() + function.jobOutputStream = new PrintStream(FileUtils.openOutputStream(function.jobOutputFile)) + function.jobErrorStream = { + if (function.jobErrorFile != null) + new PrintStream(FileUtils.openOutputStream(function.jobErrorFile)) + else + function.jobOutputStream + } + try { + function.run() + function.jobOutputStream.println("%s%nDone.".format(function.description)) + } finally { + IOUtils.closeQuietly(function.jobOutputStream) + if (function.jobErrorFile != null) + IOUtils.closeQuietly(function.jobErrorStream) + } - getRunInfo.doneTime = new Date() - val content = "%s%nDone.".format(function.description) - FileUtils.writeStringToFile(function.jobOutputFile, content) runStatus = RunnerStatus.DONE + getRunInfo.doneTime = new Date() } def status = runStatus diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/MappingEdge.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/MappingEdge.scala index 1d56009f3..17f0561fa 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/MappingEdge.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/MappingEdge.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.engine /** @@ -10,5 +34,5 @@ class MappingEdge(val inputs: QNode, val outputs: QNode) extends QEdge { * @return */ override def toString = "" - override def dotString = "" + override def shortDescription = "" } diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/QEdge.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/QEdge.scala index 1608e3c08..e40a86867 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/QEdge.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/QEdge.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.engine /** @@ -15,9 +39,9 @@ trait QEdge { def outputs: QNode /** - * The function description in .dot files + * The short description */ - def dotString = "" + def shortDescription = "" override def hashCode = inputs.hashCode + outputs.hashCode diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala index 42ddf9104..cee2c6e56 100755 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -30,7 +30,6 @@ import scala.collection.JavaConversions._ import org.jgrapht.alg.CycleDetector import org.jgrapht.EdgeFactory import org.jgrapht.ext.DOTExporter -import java.io.File import org.jgrapht.event.{TraversalListenerAdapter, EdgeTraversalEvent} import org.broadinstitute.sting.queue.QException import org.broadinstitute.sting.queue.function.{InProcessFunction, CommandLineFunction, QFunction} @@ -40,7 +39,8 @@ import collection.immutable.{TreeSet, TreeMap} import org.broadinstitute.sting.queue.function.scattergather.{ScatterFunction, CloneFunction, GatherFunction, ScatterGatherableFunction} import java.util.Date import org.broadinstitute.sting.utils.Utils -import org.broadinstitute.sting.utils.io.IOUtils +import org.apache.commons.io.{FileUtils, IOUtils} +import java.io.{OutputStreamWriter, File} /** * The internal dependency tracker between sets of function input and output files. @@ -69,7 +69,7 @@ class QGraph extends Logging { private val commandLinePluginManager = new CommandLinePluginManager private var commandLineManager: CommandLineJobManager[CommandLineJobRunner] = _ private val inProcessManager = new InProcessJobManager - private def managers = List[Any](inProcessManager, commandLineManager) + private def managers = Seq[Any](inProcessManager, commandLineManager) private class StatusCounts { var pending = 0 @@ -88,9 +88,14 @@ class QGraph extends Logging { runningLock.synchronized { if (running) { command.qSettings = settings.qSettings - command.freeze - val inputs = getQNode(command.inputs.toList.sorted(fileOrdering)) - val outputs = getQNode(command.outputs.toList.sorted(fileOrdering)) + command.freeze() + val inputFiles = command.inputs + var outputFiles = command.outputs + outputFiles :+= command.jobOutputFile + if (command.jobErrorFile != null) + outputFiles :+= command.jobErrorFile + val inputs = getQNode(inputFiles.sorted(fileOrdering)) + val outputs = getQNode(outputFiles.sorted(fileOrdering)) addEdge(new FunctionEdge(command, inputs, outputs)) } } @@ -106,8 +111,8 @@ class QGraph extends Logging { def run() { runningLock.synchronized { if (running) { - IOUtils.checkTempDir(settings.qSettings.tempDirectory) - fillGraph + org.broadinstitute.sting.utils.io.IOUtils.checkTempDir(settings.qSettings.tempDirectory) + fillGraph() val isReady = numMissingValues == 0 if (this.jobGraph.edgeSet.isEmpty) { @@ -133,11 +138,11 @@ class QGraph extends Logging { } } - private def fillGraph { + private def fillGraph() { logger.info("Generating graph.") - fill - if (settings.dotFile != null) - renderToDot(settings.dotFile) + fill() + if (settings.graphvizFile != null) + renderGraph(settings.graphvizFile) validate() if (running && numMissingValues == 0) { @@ -145,7 +150,7 @@ class QGraph extends Logging { if (!scatterGathers.isEmpty) { logger.info("Generating scatter gather jobs.") - var addedFunctions = List.empty[QFunction] + var addedFunctions = Seq.empty[QFunction] for (scatterGather <- scatterGathers) { val functions = scatterGather.asInstanceOf[FunctionEdge] .function.asInstanceOf[ScatterGatherableFunction] @@ -161,10 +166,10 @@ class QGraph extends Logging { addedFunctions.foreach(function => if (running) this.add(function)) logger.info("Regenerating graph.") - fill - val scatterGatherDotFile = if (settings.expandedDotFile != null) settings.expandedDotFile else settings.dotFile + fill() + val scatterGatherDotFile = if (settings.graphvizScatterGatherFile != null) settings.graphvizScatterGatherFile else settings.graphvizFile if (scatterGatherDotFile != null) - renderToDot(scatterGatherDotFile) + renderGraph(scatterGatherDotFile) validate() } } @@ -187,8 +192,8 @@ class QGraph extends Logging { * @param edge Graph edge to examine for the previous functions. * @return A list of prior function edges. */ - private def previousFunctions(edge: QEdge): List[FunctionEdge] = { - var previous = List.empty[FunctionEdge] + private def previousFunctions(edge: QEdge): Seq[FunctionEdge] = { + var previous = Seq.empty[FunctionEdge] val source = this.jobGraph.getEdgeSource(edge) for (incomingEdge <- this.jobGraph.incomingEdgesOf(source)) { incomingEdge match { @@ -208,8 +213,8 @@ class QGraph extends Logging { * @param edge Graph edge to examine for the next functions. * @return A list of prior function edges. */ - private def nextFunctions(edge: QEdge): List[FunctionEdge] = { - var next = List.empty[FunctionEdge] + private def nextFunctions(edge: QEdge): Seq[FunctionEdge] = { + var next = Seq.empty[FunctionEdge] val target = this.jobGraph.getEdgeTarget(edge) for (outgoingEdge <- this.jobGraph.outgoingEdgesOf(target)) { outgoingEdge match { @@ -238,7 +243,7 @@ class QGraph extends Logging { */ private def fillIn() { // clone since edgeSet is backed by the graph - asScalaSet(jobGraph.edgeSet).clone.foreach(edge => { + asScalaSet(jobGraph.edgeSet).clone().foreach(edge => { if (running) edge match { case cmd: FunctionEdge => { addCollectionOutputs(cmd.outputs) @@ -249,7 +254,7 @@ class QGraph extends Logging { }) } - private def getReadyJobs(): Set[FunctionEdge] = { + private def getReadyJobs: Set[FunctionEdge] = { jobGraph.edgeSet.filter{ case f: FunctionEdge => this.previousFunctions(f).forall(_.status == RunnerStatus.DONE) && f.status == RunnerStatus.PENDING @@ -317,33 +322,39 @@ class QGraph extends Logging { updateGraphStatus(false) - var readyJobs = getReadyJobs() + var readyJobs = getReadyJobs while (running && readyJobs.size > 0) { logger.debug("+++++++") - foreachFunction(readyJobs.toList, edge => { + foreachFunction(readyJobs.toSeq, edge => { if (running) { edge.myRunInfo.startTime = new Date() edge.getRunInfo.exechosts = Utils.resolveHostname() logEdge(edge) edge.myRunInfo.doneTime = new Date() - edge.markAsDone + edge.markAsDone() } }) - readyJobs = getReadyJobs() + readyJobs = getReadyJobs } } private def logEdge(edge: FunctionEdge) { logger.info("-------") + logger.info("%-8s %s".format(StringUtils.capitalize(edge.status.toString) + ":", edge.function.description)) if (logger.isDebugEnabled) { - logger.debug("Inputs: " + edge.inputs) + logger.debug("Inputs: " + edge.inputs) + logger.debug("Outputs: " + edge.outputs) + logger.debug("Done+: " + edge.function.doneOutputs.filter(_.exists())) + logger.debug("Done-: " + edge.function.doneOutputs.filterNot(_.exists())) + logger.debug("CmdDir: " + edge.function.commandDirectory) + logger.debug("Temp?: " + edge.function.isIntermediate) + logger.debug("Prev: " + + (if (edge.resetFromStatus == null) "none" else StringUtils.capitalize(edge.resetFromStatus.toString)) + + " (reset = " + (edge.resetFromStatus != null && edge.resetFromStatus != edge.status) + ")" ) } - logger.info(StringUtils.capitalize(edge.status.toString) + ": " + edge.function.description) - if (logger.isDebugEnabled) - logger.debug(edge.function.commandDirectory + " > " + edge.function.description) - logger.info("Log: " + edge.function.jobOutputFile.getAbsolutePath) + logger.info("Log: " + edge.function.jobOutputFile.getAbsolutePath) if (edge.function.jobErrorFile != null) - logger.info("Error: " + edge.function.jobErrorFile.getAbsolutePath) + logger.info("Error: " + edge.function.jobErrorFile.getAbsolutePath) } /** @@ -380,7 +391,7 @@ class QGraph extends Logging { updateGraphStatus(true) var readyJobs = TreeSet.empty[FunctionEdge](functionOrdering) - readyJobs ++= getReadyJobs() + readyJobs ++= getReadyJobs runningJobs = Set.empty[FunctionEdge] var lastRunningCheck = System.currentTimeMillis var logNextStatusCounts = true @@ -407,7 +418,7 @@ class QGraph extends Logging { statusCounts.running += startedJobs.size if (logNextStatusCounts) - logStatusCounts + logStatusCounts() logNextStatusCounts = false deleteCleanup(lastRunningCheck) @@ -456,10 +467,10 @@ class QGraph extends Logging { checkRetryJobs(failedJobs) } - readyJobs ++= getReadyJobs() + readyJobs ++= getReadyJobs } - logStatusCounts + logStatusCounts() deleteCleanup(-1) } catch { case e => @@ -476,7 +487,7 @@ class QGraph extends Logging { private def nextRunningCheck(lastRunningCheck: Long) = ((30 * 1000L) - (System.currentTimeMillis - lastRunningCheck)) - private def logStatusCounts { + private def logStatusCounts() { logger.info("%d Pend, %d Run, %d Fail, %d Done".format( statusCounts.pending, statusCounts.running, statusCounts.failed, statusCounts.done)) } @@ -532,7 +543,8 @@ class QGraph extends Logging { } if (edge.status == RunnerStatus.DONE || edge.status == RunnerStatus.SKIPPED) { - logger.debug("Already done: " + edge.function.description) + if (logger.isDebugEnabled) + logEdge(edge) addCleanup(edge) } } @@ -546,12 +558,12 @@ class QGraph extends Logging { } /** - * Checks if the function should have their outptus removed after they finish running - * @param edges Function to check + * Checks if the function should have their outputs removed after they finish running + * @param edge Function to check */ private def addCleanup(edge: FunctionEdge) { if (!settings.keepIntermediates) - if (edge.function.isIntermediate && edge.function.deleteIntermediateOutputs) + if (edge.function.isIntermediate) cleanupJobs += edge } @@ -601,14 +613,16 @@ class QGraph extends Logging { * From the previous edges, resets any that are marked as skipped to pending. * If those that are reset have skipped edges, those skipped edges are recursively also set * to pending. + * Any edges after this edge are also reset to pending. * @param edge Dependent edge. * @param previous Previous edges that provide inputs to edge. - * @param cleanOutputs If true will clean up the output files when resetting skipped jobs to pending. + * @param cleanOutputs If true will clean up the output files when resetting jobs to pending. */ - private def resetPreviousSkipped(edge: FunctionEdge, previous: List[FunctionEdge], cleanOutputs: Boolean) { - for (previousEdge <- previous.filter(_.status == RunnerStatus.SKIPPED)) { - previousEdge.resetToPending(cleanOutputs) - resetPreviousSkipped(previousEdge, this.previousFunctions(previousEdge), cleanOutputs) + private def resetPreviousSkipped(edge: FunctionEdge, previous: Seq[FunctionEdge], cleanOutputs: Boolean) { + val edges = previous.filter(_.status == RunnerStatus.SKIPPED) ++ this.nextFunctions(edge).filter(_.status != RunnerStatus.PENDING) + for (resetEdge <- edges) { + resetEdge.resetToPending(cleanOutputs) + resetPreviousSkipped(resetEdge, this.previousFunctions(resetEdge), cleanOutputs) } } @@ -628,9 +642,9 @@ class QGraph extends Logging { val emailMessage = new EmailMessage emailMessage.from = settings.statusEmailFrom emailMessage.to = settings.statusEmailTo - emailMessage.subject = "Queue function: Started: " + settings.qSettings.jobNamePrefix - addStartedFunctions(emailMessage, started.toList) - emailMessage.trySend(settings.qSettings.emailSettings) + emailMessage.subject = "Queue function: Started: " + settings.qSettings.runName + addStartedFunctions(emailMessage, started.toSeq) + emailMessage.trySend(settings.emailSettings) } } @@ -639,9 +653,9 @@ class QGraph extends Logging { val emailMessage = new EmailMessage emailMessage.from = settings.statusEmailFrom emailMessage.to = settings.statusEmailTo - emailMessage.subject = "Queue function: Failure: " + settings.qSettings.jobNamePrefix - addFailedFunctions(emailMessage, failed.toList) - emailMessage.trySend(settings.qSettings.emailSettings) + emailMessage.subject = "Queue function: Failure: " + settings.qSettings.runName + addFailedFunctions(emailMessage, failed.toSeq) + emailMessage.trySend(settings.emailSettings) } } @@ -665,7 +679,7 @@ class QGraph extends Logging { private def emailStatus() { if (running && settings.statusEmailTo.size > 0) { - var failed = List.empty[FunctionEdge] + var failed = Seq.empty[FunctionEdge] foreachFunction(edge => { if (edge.status == RunnerStatus.FAILED) { failed :+= edge @@ -677,16 +691,16 @@ class QGraph extends Logging { emailMessage.to = settings.statusEmailTo emailMessage.body = getStatus + nl if (failed.size == 0) { - emailMessage.subject = "Queue run: Success: " + settings.qSettings.jobNamePrefix + emailMessage.subject = "Queue run: Success: " + settings.qSettings.runName } else { - emailMessage.subject = "Queue run: Failure: " + settings.qSettings.jobNamePrefix + emailMessage.subject = "Queue run: Failure: " + settings.qSettings.runName addFailedFunctions(emailMessage, failed) } - emailMessage.trySend(settings.qSettings.emailSettings) + emailMessage.trySend(settings.emailSettings) } } - private def addStartedFunctions(emailMessage: EmailMessage, started: List[FunctionEdge]) { + private def addStartedFunctions(emailMessage: EmailMessage, started: Seq[FunctionEdge]) { if (emailMessage.body == null) emailMessage.body = "" emailMessage.body += """ @@ -697,7 +711,7 @@ class QGraph extends Logging { started.map(edge => emailDescription(edge)).mkString(nl+nl)) } - private def addFailedFunctions(emailMessage: EmailMessage, failed: List[FunctionEdge]) { + private def addFailedFunctions(emailMessage: EmailMessage, failed: Seq[FunctionEdge]) { val logs = failed.flatMap(edge => logFiles(edge)) if (emailMessage.body == null) @@ -725,7 +739,7 @@ class QGraph extends Logging { } private def logFiles(edge: FunctionEdge) = { - var failedOutputs = List.empty[File] + var failedOutputs = Seq.empty[File] failedOutputs :+= edge.function.jobOutputFile if (edge.function.jobErrorFile != null) failedOutputs :+= edge.function.jobErrorFile @@ -762,14 +776,14 @@ class QGraph extends Logging { private def getStatus = { val buffer = new StringBuilder doStatus(status => buffer.append(status).append(nl)) - buffer.toString + buffer.toString() } /** * Gets job statuses by traversing the graph and looking for status-related files */ - private def doStatus(statusFunc: String => Unit) = { - var statuses = List.empty[AnalysisStatus] + private def doStatus(statusFunc: String => Unit) { + var statuses = Seq.empty[AnalysisStatus] var maxWidth = 0 foreachFunction(edge => { val name = edge.function.analysisName @@ -860,7 +874,7 @@ class QGraph extends Logging { private def newGraph = new SimpleDirectedGraph[QNode, QEdge](new EdgeFactory[QNode, QEdge] { def createEdge(input: QNode, output: QNode) = new MappingEdge(input, output)}) - private def getQNode(files: List[File]) = { + private def getQNode(files: Seq[File]) = { nodeMap.get(files) match { case Some(node) => node @@ -888,7 +902,7 @@ class QGraph extends Logging { if (inputs.files.size > 1) for (file <- inputs.files) { if (running) { - val input = getQNode(List(file)) + val input = getQNode(Seq(file)) if (!jobGraph.containsEdge(input, inputs)) addEdge(new MappingEdge(input, inputs)) } @@ -903,7 +917,7 @@ class QGraph extends Logging { if (outputs.files.size > 1) for (file <- outputs.files) { if (running) { - val output = getQNode(List(file)) + val output = getQNode(Seq(file)) if (!jobGraph.containsEdge(outputs, output)) addEdge(new MappingEdge(outputs, output)) } @@ -937,37 +951,36 @@ class QGraph extends Logging { /** * Utility function for running a method over all function edges. - * @param edgeFunction Function to run for each FunctionEdge. + * @param f Function to run for each FunctionEdge. */ private def foreachFunction(f: (FunctionEdge) => Unit) { - foreachFunction(jobGraph.edgeSet.toList.filter(_.isInstanceOf[FunctionEdge]).asInstanceOf[List[FunctionEdge]], f) + foreachFunction(jobGraph.edgeSet.toSeq.filter(_.isInstanceOf[FunctionEdge]).asInstanceOf[Seq[FunctionEdge]], f) } /** * Utility function for running a method over a list of function edges. - * @param edegs Edges to traverse. - * @param edgeFunction Function to run for each FunctionEdge. + * @param edges Edges to traverse. + * @param f Function to run for each FunctionEdge. */ - private def foreachFunction(edges: List[FunctionEdge], f: (FunctionEdge) => Unit) { + private def foreachFunction(edges: Seq[FunctionEdge], f: (FunctionEdge) => Unit) { edges.sorted(functionOrdering).foreach(edge => if (running) f(edge)) } /** - * Utility function for running a method over all function edges. - * @param edgeFunction Function to run for each FunctionEdge. + * Utility function returning all function edges. */ - private def getFunctionEdges: List[FunctionEdge] = { - jobGraph.edgeSet.toList.filter(_.isInstanceOf[FunctionEdge]).asInstanceOf[List[FunctionEdge]] + private def getFunctionEdges: Seq[FunctionEdge] = { + jobGraph.edgeSet.toSeq.filter(_.isInstanceOf[FunctionEdge]).asInstanceOf[Seq[FunctionEdge]] } /** * Utility function for running a method over all functions, but traversing the nodes in order of dependency. - * @param edgeFunction Function to run for each FunctionEdge. + * @param f Function to run for each FunctionEdge. */ private def traverseFunctions(f: (FunctionEdge) => Unit) { val iterator = new TopologicalOrderIterator(this.jobGraph) iterator.addTraversalListener(new TraversalListenerAdapter[QNode, QEdge] { - override def edgeTraversed(event: EdgeTraversalEvent[QNode, QEdge]) = { + override def edgeTraversed(event: EdgeTraversalEvent[QNode, QEdge]) { if (running) { event.getEdge match { case functionEdge: FunctionEdge => f(functionEdge) @@ -980,23 +993,44 @@ class QGraph extends Logging { } /** - * Outputs the graph to a .dot file. + * Outputs the graph to a .gv DOT file. + * http://www.graphviz.org/Documentation.php * http://en.wikipedia.org/wiki/DOT_language - * @param file Path to output the .dot file. + * @param file Path to output the .gv file. */ - private def renderToDot(file: java.io.File) { - val out = new java.io.FileWriter(file) - - // todo -- we need a nice way to visualize the key pieces of information about commands. Perhaps a - // todo -- visualizeString() command, or something that shows inputs / outputs - val ve = new org.jgrapht.ext.EdgeNameProvider[QEdge] { - def getEdgeName(function: QEdge) = if (function.dotString == null) "" else function.dotString.replace("\"", "\\\"") + private def renderGraph(file: java.io.File) { + val vertexIDProvider = new org.jgrapht.ext.VertexNameProvider[QNode] { + def getVertexName(node: QNode) = node.id.toString } - //val iterator = new TopologicalOrderIterator(qGraph.jobGraph) - (new DOTExporter(new org.jgrapht.ext.IntegerNameProvider[QNode](), null, ve)).export(out, jobGraph) + val vertexLabelProvider = new org.jgrapht.ext.VertexNameProvider[QNode] { + // The QGraph fills in with single file nodes between nodes that contain more than one file. + // We only need to display the single element nodes. + def getVertexName(node: QNode) = { + if (!node.files.isEmpty && node.files.tail.isEmpty) + node.files.head.getName + else + "" + } + } - out.close + val edgeNameProvider = new org.jgrapht.ext.EdgeNameProvider[QEdge] { + def getEdgeName(edge: QEdge) = { + if (edge.shortDescription != null) + edge.shortDescription.replace("\"", "\\\"") + else + "" + } + } + + val exporter = new DOTExporter(vertexIDProvider, vertexLabelProvider, edgeNameProvider) + + val out = new OutputStreamWriter(FileUtils.openOutputStream(file)) + try { + exporter.export(out, jobGraph) + } finally { + IOUtils.closeQuietly(out) + } } /** @@ -1054,7 +1088,7 @@ class QGraph extends Logging { */ def isShutdown = !running - def getFunctionsAndStatus(functions: List[QFunction]): Map[QFunction, JobRunInfo] = { + def getFunctionsAndStatus: Map[QFunction, JobRunInfo] = { getFunctionEdges.map(edge => (edge.function, edge.getRunInfo)).toMap } diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/QGraphSettings.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/QGraphSettings.scala index 56d6975a5..6d81d4bd7 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/QGraphSettings.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/QGraphSettings.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -26,7 +26,7 @@ package org.broadinstitute.sting.queue.engine import java.io.File import org.broadinstitute.sting.queue.QSettings -import org.broadinstitute.sting.queue.util.SystemUtils +import org.broadinstitute.sting.queue.util.{EmailSettings, SystemUtils} import org.broadinstitute.sting.commandline.{Advanced, ArgumentCollection, Argument} /** @@ -58,16 +58,16 @@ class QGraphSettings { var keepIntermediates = false @Argument(fullName="status_email_to", shortName="statusTo", doc="Email address to send emails to upon completion or on error.", required=false) - var statusEmailTo: List[String] = Nil + var statusEmailTo: Seq[String] = Nil @Argument(fullName="status_email_from", shortName="statusFrom", doc="Email address to send emails from upon completion or on error.", required=false) var statusEmailFrom: String = System.getProperty("user.name") + "@" + SystemUtils.mailName - @Argument(fullName="dot_graph", shortName="dot", doc="Outputs the queue graph to a .dot file. See: http://en.wikipedia.org/wiki/DOT_language", required=false) - var dotFile: File = _ + @Argument(fullName="graphviz", shortName="gv", doc="Outputs the queue graph to a Graphviz .gv file. See: http://www.graphviz.org/Documentation.php", required=false) + var graphvizFile: File = _ - @Argument(fullName="expanded_dot_graph", shortName="expandedDot", doc="Outputs the queue graph of scatter gather to a .dot file. Otherwise overwrites the dot_graph", required=false) - var expandedDotFile: File = _ + @Argument(fullName="graphviz_scatter_gather", shortName="gvsg", doc="Outputs the scatter/gather queue graph to a Graphviz .gv file. Otherwise overwrites the --graphviz file.", required=false) + var graphvizScatterGatherFile: File = _ @Argument(fullName="jobReport", shortName="jobReport", doc="File where we will write the Queue job report", required=false) var jobReportFile: String = _ @@ -76,6 +76,9 @@ class QGraphSettings { @Argument(fullName="disableJobReport", shortName="disabpleJobReport", doc="If provided, we will not create a job report", required=false) var disableJobReport: Boolean = false + @ArgumentCollection + val emailSettings = new EmailSettings + @ArgumentCollection val qSettings = new QSettings } diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/QNode.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/QNode.scala index a86c08aae..a5c039a53 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/QNode.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/QNode.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.engine import java.io.File @@ -6,7 +30,7 @@ import java.io.File * Represents a state between QFunctions the directed acyclic QGraph * @param files The list of files that represent this node state ordered by file name. */ -class QNode (val id: Int, val files: List[File]) { +class QNode (val id: Int, val files: Seq[File]) { override def equals(obj: Any) = { obj match { case other: QNode => this.id == other.id @@ -16,5 +40,5 @@ class QNode (val id: Int, val files: List[File]) { override def hashCode = id - override def toString = files.toString + override def toString = files.toString() } diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/gridengine/GridEngineJobRunner.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/gridengine/GridEngineJobRunner.scala index fca92a7a1..239f83482 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/gridengine/GridEngineJobRunner.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/gridengine/GridEngineJobRunner.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -69,7 +69,7 @@ class GridEngineJobRunner(session: Session, function: CommandLineFunction) exten if ( function.nCoresRequest.getOrElse(1) > 1 ) { if ( function.qSettings.dontRequestMultipleCores ) logger.warn("Sending multicore job %s to farm without requesting appropriate number of cores (%d)".format( - function.jobName, function.nCoresRequest.get)) + function.shortDescription, function.nCoresRequest.get)) else nativeSpec += " -pe %s %d".format(function.qSettings.parallelEnvironmentName, function.nCoresRequest.get) } diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/lsf/Lsf706JobRunner.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/lsf/Lsf706JobRunner.scala index 5ef78500c..de996d187 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/lsf/Lsf706JobRunner.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/lsf/Lsf706JobRunner.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -35,8 +35,8 @@ import org.broadinstitute.sting.queue.engine.{RunnerStatus, CommandLineJobRunner import java.util.regex.Pattern import java.lang.StringBuffer import java.util.Date -import com.sun.jna.{Pointer, Structure, StringArray, NativeLong} -import com.sun.jna.ptr.{PointerByReference, IntByReference} +import com.sun.jna.{Structure, StringArray, NativeLong} +import com.sun.jna.ptr.IntByReference /** * Runs jobs on an LSF compute cluster. @@ -60,7 +60,6 @@ class Lsf706JobRunner(val function: CommandLineFunction) extends CommandLineJobR /** * Dispatches the function on the LSF cluster. - * @param function Command to run. */ def start() { Lsf706JobRunner.lsfLibLock.synchronized { @@ -110,7 +109,7 @@ class Lsf706JobRunner(val function: CommandLineFunction) extends CommandLineJobR if ( function.nCoresRequest.getOrElse(1) > 1 ) { if ( function.qSettings.dontRequestMultipleCores ) logger.warn("Sending multicore job %s to farm without requesting appropriate number of cores (%d)".format( - function.jobName, function.nCoresRequest.get)) + function.shortDescription, function.nCoresRequest.get)) else { request.numProcessors = function.nCoresRequest.get request.maxNumProcessors = request.numProcessors @@ -298,7 +297,7 @@ object Lsf706JobRunner extends Logging { runner.getRunInfo.doneTime = new Date(jobInfo.endTime.longValue * 1000) val exHostsRaw = jobInfo.exHosts.getStringArray(0) //logger.warn("exHostsRaw = " + exHostsRaw) - val exHostsList = exHostsRaw.toList + val exHostsList = exHostsRaw.toSeq //logger.warn("exHostsList = " + exHostsList) val exHosts = exHostsList.reduceLeft(_ + "," + _) //logger.warn("exHosts = " + exHosts) @@ -363,7 +362,7 @@ object Lsf706JobRunner extends Logging { /** * Returns the run limit in seconds for the queue. * If the queue name is null returns the length of the default queue. - * @param queue Name of the queue or null for the default queue. + * @param queueName Name of the queue or null for the default queue. * @return the run limit in seconds for the queue. */ private def getRlimitRun(queueName: String) = { diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala index 9751012a4..6cd4b06bc 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala @@ -53,6 +53,9 @@ class BamGatherFunction extends GatherFunction with PicardBamFunction { val disableIndex = QFunction.findField(originalFunction.getClass, SAMFileWriterArgumentTypeDescriptor.DISABLE_INDEXING_FULLNAME) this.createIndex = Some(!originalGATK.getFieldValue(disableIndex).asInstanceOf[Boolean]) - super.freezeFieldValues + val enableMD5 = QFunction.findField(originalFunction.getClass, SAMFileWriterArgumentTypeDescriptor.ENABLE_MD5_FULLNAME) + this.createMD5 = Some(originalGATK.getFieldValue(enableMD5).asInstanceOf[Boolean]) + + super.freezeFieldValues() } } diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala index 9e47f64a1..42a027be1 100755 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -32,9 +32,8 @@ import net.sf.samtools.SAMFileHeader import java.util.Collections import org.broadinstitute.sting.utils.{GenomeLoc, GenomeLocSortedSet, GenomeLocParser} -case class GATKIntervals(reference: File, intervals: List[String]) { +case class GATKIntervals(reference: File, intervals: Seq[String]) { private lazy val referenceDataSource = new ReferenceDataSource(reference) -// private var splitsBySize = Map.empty[Int, java.util.List[java.lang.Integer]] lazy val samFileHeader = { val header = new SAMFileHeader @@ -53,13 +52,5 @@ case class GATKIntervals(reference: File, intervals: List[String]) { Collections.unmodifiableList(parsedLocs) } - lazy val contigs = locs.map(_.getContig).distinct.toList - -// def getSplits(size: Int) = { -// splitsBySize.getOrElse(size, { -// val splits: java.util.List[java.lang.Integer] = IntervalUtils.splitFixedIntervals(locs, size) -// splitsBySize += size -> splits -// splits -// }) -// } + lazy val contigs = locs.map(_.getContig).distinct.toSeq } diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKScatterFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKScatterFunction.scala index c9adff026..28c3f41e9 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKScatterFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKScatterFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -26,7 +26,6 @@ package org.broadinstitute.sting.queue.extensions.gatk import org.broadinstitute.sting.utils.interval.IntervalUtils import java.io.File -import collection.JavaConversions._ import org.broadinstitute.sting.utils.io.IOUtils import org.broadinstitute.sting.queue.function.scattergather.{CloneFunction, ScatterFunction} import org.broadinstitute.sting.commandline.Output @@ -39,7 +38,7 @@ trait GATKScatterFunction extends ScatterFunction { private final val intervalsStringField = "intervalsString" @Output(doc="Scatter function outputs") - var scatterOutputFiles: List[File] = Nil + var scatterOutputFiles: Seq[File] = Nil /** The original GATK function. */ protected var originalGATK: CommandLineGATK = _ @@ -48,7 +47,7 @@ trait GATKScatterFunction extends ScatterFunction { protected var referenceSequence: File = _ /** The list of interval files ("/path/to/interval.list") or interval strings ("chr1", "chr2") to parse into smaller parts. */ - protected var intervals: List[String] = Nil + protected var intervals: Seq[String] = Nil /** Whether the last scatter job should also include any unmapped reads. */ protected var includeUnmapped: Boolean = _ @@ -57,7 +56,7 @@ trait GATKScatterFunction extends ScatterFunction { this.originalGATK = this.originalFunction.asInstanceOf[CommandLineGATK] this.referenceSequence = this.originalGATK.reference_sequence if (this.originalGATK.intervals.isEmpty && (this.originalGATK.intervalsString == null || this.originalGATK.intervalsString.isEmpty)) { - this.intervals ++= GATKScatterFunction.getGATKIntervals(this.referenceSequence, List.empty[String]).contigs + this.intervals ++= GATKScatterFunction.getGATKIntervals(this.referenceSequence, Seq.empty[String]).contigs } else { this.intervals ++= this.originalGATK.intervals.map(_.toString) this.intervals ++= this.originalGATK.intervalsString.filterNot(interval => IntervalUtils.isUnmapped(interval)) @@ -70,16 +69,16 @@ trait GATKScatterFunction extends ScatterFunction { } override def initCloneInputs(cloneFunction: CloneFunction, index: Int) { - cloneFunction.setFieldValue(this.intervalsField, List(new File("scatter.intervals"))) + cloneFunction.setFieldValue(this.intervalsField, Seq(new File("scatter.intervals"))) if (index == this.scatterCount && this.includeUnmapped) - cloneFunction.setFieldValue(this.intervalsStringField, List("unmapped")) + cloneFunction.setFieldValue(this.intervalsStringField, Seq("unmapped")) else - cloneFunction.setFieldValue(this.intervalsStringField, List.empty[String]) + cloneFunction.setFieldValue(this.intervalsStringField, Seq.empty[String]) } override def bindCloneInputs(cloneFunction: CloneFunction, index: Int) { val scatterPart = cloneFunction.getFieldValue(this.intervalsField) - .asInstanceOf[List[File]] + .asInstanceOf[Seq[File]] .map(file => IOUtils.absolute(cloneFunction.commandDirectory, file)) cloneFunction.setFieldValue(this.intervalsField, scatterPart) this.scatterOutputFiles ++= scatterPart @@ -100,9 +99,9 @@ trait GATKScatterFunction extends ScatterFunction { } object GATKScatterFunction { - var gatkIntervals = List.empty[GATKIntervals] + var gatkIntervals = Seq.empty[GATKIntervals] - def getGATKIntervals(reference: File, intervals: List[String]) = { + def getGATKIntervals(reference: File, intervals: Seq[String]) = { gatkIntervals.find(gi => gi.reference == reference && gi.intervals == intervals) match { case Some(gi) => gi case None => diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/RodBind.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/RodBind.scala deleted file mode 100644 index deb83bf5a..000000000 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/RodBind.scala +++ /dev/null @@ -1,41 +0,0 @@ -package org.broadinstitute.sting.queue.extensions.gatk - -import java.io.File -import org.broadinstitute.sting.utils.io.FileExtension -import java.lang.String - -/** - * Used to provide -B rodBind arguments to the GATK. - */ -class RodBind(var trackName: String, var trackType: String, path: String, val tag: String) extends File(path) with FileExtension { - def this(trackName: String, trackType: String, path: String) = - this(trackName, trackType, path, null) - def this(trackName: String, trackType: String, file: File, tag: String) = - this(trackName, trackType, file.getPath, tag) - def this(trackName: String, trackType: String, file: File) = - this(trackName, trackType, file.getPath, null) - require(trackName != null, "RodBind trackName cannot be null") - require(trackType != null, "RodBind trackType cannot be null") - def withPath(newPath: String) = new RodBind(trackName, trackType, newPath, tag) -} - -/** - * Used to provide -B rodBind arguments to the GATK. - */ -object RodBind { - def apply(trackName: String, trackType: String, path: String, tag: String) = new RodBind(trackName, trackType, path, tag) - def apply(trackName: String, trackType: String, path: String) = new RodBind(trackName, trackType, path, null) - def apply(trackName: String, trackType: String, file: File, tag: String) = new RodBind(trackName, trackType, file, tag) - def apply(trackName: String, trackType: String, file: File) = new RodBind(trackName, trackType, file, null) - - def formatCommandLineParameter( cmdLineParam: String, value: Any ) = { - value match { - case rodBind: RodBind if (rodBind.tag != null) => - "%s:%s,%s,%s".format(cmdLineParam, rodBind.trackName, rodBind.trackType, rodBind.tag) - case rodBind: RodBind => - "%s:%s,%s".format(cmdLineParam, rodBind.trackName, rodBind.trackType) - case x => - "" - } - } -} diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/AddOrReplaceReadGroups.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/AddOrReplaceReadGroups.scala index 93735e4ac..2faa65908 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/AddOrReplaceReadGroups.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/AddOrReplaceReadGroups.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.extensions.picard import org.broadinstitute.sting.commandline._ @@ -15,7 +39,7 @@ class AddOrReplaceReadGroups extends org.broadinstitute.sting.queue.function.Jav javaMainClass = "net.sf.picard.sam.AddOrReplaceReadGroups" @Input(doc="The input SAM or BAM files to analyze. Must be coordinate sorted.", shortName = "input", fullName = "input_bam_files", required = true) - var input: List[File] = Nil + var input: Seq[File] = Nil @Output(doc="The output BAM file with the modified/added read groups", shortName = "output", fullName = "output_bam_file", required = true) var output: File = _ diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MarkDuplicates.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MarkDuplicates.scala index d73c556af..06c6e3fdc 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MarkDuplicates.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MarkDuplicates.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.extensions.picard import org.broadinstitute.sting.commandline._ @@ -15,7 +39,7 @@ class MarkDuplicates extends org.broadinstitute.sting.queue.function.JavaCommand javaMainClass = "net.sf.picard.sam.MarkDuplicates" @Input(doc="The input SAM or BAM files to analyze. Must be coordinate sorted.", shortName = "input", fullName = "input_bam_files", required = true) - var input: List[File] = Nil + var input: Seq[File] = Nil @Output(doc="The output file to write marked records to", shortName = "output", fullName = "output_bam_file", required = true) var output: File = _ diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MergeSamFiles.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MergeSamFiles.scala index 036932cc6..8c2377577 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MergeSamFiles.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MergeSamFiles.scala @@ -1,9 +1,32 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.extensions.picard import org.broadinstitute.sting.commandline._ import java.io.File -import org.broadinstitute.sting.queue.QScript._ /* * Created by IntelliJ IDEA. @@ -16,7 +39,7 @@ class MergeSamFiles extends org.broadinstitute.sting.queue.function.JavaCommandL javaMainClass = "net.sf.picard.sam.MergeSamFiles" @Input(doc="The input SAM or BAM files to analyze. Must be coordinate sorted.", shortName = "input", fullName = "input_bam_files", required = true) - var input: List[File] = Nil + var input: Seq[File] = Nil @Output(doc="The output merged BAM file", shortName = "output", fullName = "output_bam_file", required = true) var output: File = _ diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamFunction.scala index 76856dc36..defb43e4e 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -41,20 +41,22 @@ trait PicardBamFunction extends JavaCommandLineFunction { var sortOrder = SortOrder.coordinate var compressionLevel: Option[Int] = None var createIndex: Option[Boolean] = None + var createMD5: Option[Boolean] = None var maxRecordsInRam: Option[Int] = None var assumeSorted: Option[Boolean] = None - protected def inputBams: List[File] + protected def inputBams: Seq[File] protected def outputBam: File abstract override def commandLine = super.commandLine + - repeat("INPUT=", inputBams, spaceSeparated=false) + - required("TMP_DIR=" + jobTempDir) + - optional("OUTPUT=", outputBam, spaceSeparated=false) + - optional("COMPRESSION_LEVEL=", compressionLevel, spaceSeparated=false) + - optional("VALIDATION_STRINGENCY=", validationStringency, spaceSeparated=false) + - optional("SO=", sortOrder, spaceSeparated=false) + - optional("MAX_RECORDS_IN_RAM=", maxRecordsInRam, spaceSeparated=false) + - optional("ASSUME_SORTED=", assumeSorted, spaceSeparated=false) + - optional("CREATE_INDEX=", createIndex, spaceSeparated=false) + repeat("INPUT=", inputBams, spaceSeparated=false) + + required("TMP_DIR=" + jobTempDir) + + optional("OUTPUT=", outputBam, spaceSeparated=false) + + optional("COMPRESSION_LEVEL=", compressionLevel, spaceSeparated=false) + + optional("VALIDATION_STRINGENCY=", validationStringency, spaceSeparated=false) + + optional("SO=", sortOrder, spaceSeparated=false) + + optional("MAX_RECORDS_IN_RAM=", maxRecordsInRam, spaceSeparated=false) + + optional("ASSUME_SORTED=", assumeSorted, spaceSeparated=false) + + optional("CREATE_INDEX=", createIndex, spaceSeparated=false) + + optional("CREATE_MD5_FILE=", createMD5, spaceSeparated=false) } diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ReorderSam.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ReorderSam.scala index b1968bee5..46188586e 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ReorderSam.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ReorderSam.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.extensions.picard import org.broadinstitute.sting.commandline._ @@ -14,7 +38,7 @@ class ReorderSam extends org.broadinstitute.sting.queue.function.JavaCommandLine javaMainClass = "net.sf.picard.sam.ReorderSam" @Input(doc="Input file (bam or sam) to extract reads from.", shortName = "input", fullName = "input_bam_files", required = true) - var input: List[File] = Nil + var input: Seq[File] = Nil @Output(doc="Output file (bam or sam) to write extracted reads to.", shortName = "output", fullName = "output_bam_file", required = true) var output: File = _ diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/RevertSam.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/RevertSam.scala index 60d8bfaf8..c2161b551 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/RevertSam.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/RevertSam.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.extensions.picard import org.broadinstitute.sting.commandline._ @@ -15,7 +39,7 @@ class RevertSam extends org.broadinstitute.sting.queue.function.JavaCommandLineF javaMainClass = "net.sf.picard.sam.RevertSam" @Input(shortName = "input", fullName = "input_bam_files", required = true, doc = "The input SAM or BAM files to revert.") - var input: List[File] = Nil + var input: Seq[File] = Nil @Output(shortName = "output", fullName = "output_bam_file", required = true, doc = "The reverted BAM or SAM output file.") var output: File = _ @@ -33,7 +57,7 @@ class RevertSam extends org.broadinstitute.sting.queue.function.JavaCommandLineF var removeAlignmentInformation: Boolean = true @Argument(shortName = "atc", fullName = "attributes_to_clear", required = false, doc = "When removing alignment information, the set of optional tags to remove.") - var attributesToClear: List[String] = Nil + var attributesToClear: Seq[String] = Nil @Argument(shortName = "sa", fullName = "sample_alias", required = false, doc = "The sample alias to use in the reverted output file. This will override the existing sample alias in the file and is used only if all the read groups in the input file have the same sample alias.") var sampleAlias: String = null diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SamToFastq.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SamToFastq.scala index 3eb4e8e06..6c658b105 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SamToFastq.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SamToFastq.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.extensions.picard import org.broadinstitute.sting.commandline._ @@ -15,7 +39,7 @@ class SamToFastq extends org.broadinstitute.sting.queue.function.JavaCommandLine javaMainClass = "net.sf.picard.sam.SamToFastq" @Input(shortName = "input", fullName = "input_bam_files", required = true, doc = "Input SAM/BAM file to extract reads from.") - var input: List[File] = Nil + var input: Seq[File] = Nil @Output(shortName = "fastq", fullName = "output_fastq_file", required = true, doc = "Output fastq file (single-end fastq or, if paired, first end of the pair fastq).") var fastq: File = _ diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SortSam.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SortSam.scala index a56093be8..9257cc7c2 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SortSam.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SortSam.scala @@ -1,9 +1,32 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.extensions.picard import org.broadinstitute.sting.commandline._ import java.io.File -import org.broadinstitute.sting.queue.QScript._ /* * Created by IntelliJ IDEA. @@ -16,7 +39,7 @@ class SortSam extends org.broadinstitute.sting.queue.function.JavaCommandLineFun javaMainClass = "net.sf.picard.sam.SortSam" @Input(doc="The input SAM or BAM files to sort.", shortName = "input", fullName = "input_bam_files", required = true) - var input: List[File] = Nil + var input: Seq[File] = Nil @Output(doc="The sorted BAM or SAM output file.", shortName = "output", fullName = "output_bam_file", required = true) var output: File = _ diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ValidateSamFile.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ValidateSamFile.scala index 030e4b07d..43d4ab442 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ValidateSamFile.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ValidateSamFile.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.extensions.picard import org.broadinstitute.sting.commandline._ @@ -17,7 +41,7 @@ class ValidateSamFile extends org.broadinstitute.sting.queue.function.JavaComman javaMainClass = "net.sf.picard.sam.ValidateSamFile" @Input(doc="The input SAM or BAM files to analyze. Must be coordinate sorted.", shortName = "input", fullName = "input_bam_files", required = true) - var input: List[File] = Nil + var input: Seq[File] = Nil @Output(doc="Send output to a file instead of stdout", shortName = "output", fullName = "output_file", required = false) var output: File = _ @@ -26,7 +50,7 @@ class ValidateSamFile extends org.broadinstitute.sting.queue.function.JavaComman var MODE: Mode = Mode.VERBOSE @Argument(doc="List of validation error types to ignore.", shortName = "ignore", fullName = "ignore_error_types", required = false) - var IGNORE: List[String] = Nil + var IGNORE: Seq[String] = Nil @Argument(doc = "The maximum number of lines output in verbose mode.", shortName = "max", fullName = "max_output", required = false) var MAX_OUTPUT: Int = 100 diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsIndexFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsIndexFunction.scala index 83a03b904..1ad758b58 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsIndexFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsIndexFunction.scala @@ -52,6 +52,4 @@ class SamtoolsIndexFunction extends SamtoolsCommandLineFunction { required("index") + required(bamFile) + required(bamFileIndex) - - override def dotString = "Index: %s".format(bamFile.getName) } diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsMergeFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsMergeFunction.scala index aff9a25c0..1949d9add 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsMergeFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsMergeFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -34,7 +34,7 @@ class SamtoolsMergeFunction extends SamtoolsCommandLineFunction { analysisName = "samtools merge" @Input(doc="BAM file input") - var inputBams: List[File] = Nil + var inputBams: Seq[File] = Nil @Output(doc="BAM file output") var outputBam: File = _ @@ -43,10 +43,10 @@ class SamtoolsMergeFunction extends SamtoolsCommandLineFunction { var region: String = _ @Input(doc="BAM file input indexes") - var inputBamIndexes: List[File] = Nil + var inputBamIndexes: Seq[File] = Nil - override def freezeFieldValues = { - super.freezeFieldValues + override def freezeFieldValues() { + super.freezeFieldValues() inputBamIndexes ++= inputBams .filter(orig => orig != null && orig.getName.endsWith(".bam")) .flatMap(orig => Array( diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala index 167dcb593..eff4a2ba9 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.function import org.broadinstitute.sting.queue.util._ @@ -27,13 +51,13 @@ trait CommandLineFunction extends QFunction with Logging { var jobQueue: String = _ /** Native arguments to pass to the job runner */ - var jobNativeArgs: List[String] = Nil + var jobNativeArgs: Seq[String] = Nil /** Native arguments to pass to the job runner */ - var jobResourceRequests: List[String] = Nil + var jobResourceRequests: Seq[String] = Nil /** Environment names to pass to the job runner */ - var jobEnvironmentNames: List[String] = Nil + var jobEnvironmentNames: Seq[String] = Nil override def copySettingsTo(function: QFunction) { super.copySettingsTo(function) @@ -270,7 +294,7 @@ trait CommandLineFunction extends QFunction with Logging { } // Trim leading and trailing whitespace off our three tokens, and unwrap Some(x) to x for the param - val trimmedValues : List[String] = List((if ( prefix != null ) prefix.trim else ""), + val trimmedValues : Seq[String] = Seq((if ( prefix != null ) prefix.trim else ""), (param match { case Some(x) => paramFormat.format(x).trim case x => paramFormat.format(x).trim diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/InProcessFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/InProcessFunction.scala index 783eef1bf..653b87b2f 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/InProcessFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/InProcessFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -24,10 +24,24 @@ package org.broadinstitute.sting.queue.function +import java.io.PrintStream + + /** * Runs a function in process. */ trait InProcessFunction extends QFunction { + analysisName = this.getClass.getSimpleName + def run() - def description = this.getClass.getSimpleName + " " + this.commandOutputs.mkString(" ") + + /** + * During run() this stream will write to the stdout. + */ + var jobOutputStream: PrintStream = null + + /** + * Write errors to this stream run(). + */ + var jobErrorStream: PrintStream = null } diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/JavaCommandLineFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/JavaCommandLineFunction.scala index 5b19cf9b6..534d68069 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/JavaCommandLineFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/JavaCommandLineFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -42,7 +42,7 @@ trait JavaCommandLineFunction extends CommandLineFunction { * Class path for the main class. * Defaults to the current classpath. */ - var javaClasspath: List[String] = Nil + var javaClasspath: Seq[String] = Nil /** * Memory limit for the java executable, or if None will use the default memoryLimit. @@ -82,5 +82,5 @@ trait JavaCommandLineFunction extends CommandLineFunction { object JavaCommandLineFunction { val currentClasspath = System.getProperty("java.class.path") - .split(File.pathSeparatorChar).map(path => IOUtils.absolute(new File(path)).getPath).toList + .split(File.pathSeparatorChar).map(path => IOUtils.absolute(new File(path)).getPath).toSeq } diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/ListWriterFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/ListWriterFunction.scala index f60302ef4..becc64f04 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/ListWriterFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/ListWriterFunction.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.function import org.broadinstitute.sting.commandline.{Input, Output} @@ -9,10 +33,12 @@ import org.apache.commons.io.IOUtils * Custom formats can override addFile. */ class ListWriterFunction extends InProcessFunction { - @Input(doc="input files") var inputFiles: List[File] = Nil + analysisName = "WriteList" + + @Input(doc="input files") var inputFiles: Seq[File] = Nil @Output(doc="output file") var listFile: File = _ - def run { + def run() { val writer = new PrintWriter(listFile) try { for (inputFile <- inputFiles) diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala index 59f2ada44..dee1acfac 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -29,7 +29,7 @@ import java.lang.annotation.Annotation import org.broadinstitute.sting.commandline._ import org.broadinstitute.sting.queue.{QException, QSettings} import collection.JavaConversions._ -import org.broadinstitute.sting.queue.function.scattergather.SimpleTextGatherFunction +import java.lang.IllegalStateException import org.broadinstitute.sting.queue.util._ import org.broadinstitute.sting.utils.io.IOUtils @@ -39,13 +39,18 @@ import org.broadinstitute.sting.utils.io.IOUtils * Inputs are matched to other outputs by using .equals() */ trait QFunction extends Logging with QJobReport { - /** A short description of this step in the graph */ + /** + * A short description of what this class of function does. + * By default does not include the output specific to this function. + * See shortDescription for a description of what this instance of the function outputs. + */ var analysisName: String = "" - /** Prefix for automatic job name creation */ - var jobNamePrefix: String = _ - - /** The name name of the job */ + /** + * The name name of the job, must be file system safe and unique to the graph. + * Defaults to "runName-". + * Use shortDescription for an alternative that is display friendly. + */ var jobName: String = _ /** Default settings */ @@ -58,7 +63,7 @@ trait QFunction extends Logging with QJobReport { var jobTempDir: File = null /** Order the function was added to the graph. */ - var addOrder: List[Int] = Nil + var addOrder: Seq[Int] = Nil /** Job priority */ var jobPriority: Option[Int] = None @@ -78,12 +83,6 @@ trait QFunction extends Logging with QJobReport { */ var isIntermediate = false - /** - * If true and isIntermediate is true, the files listed - * via outputs will deleted after the command completes. - */ - var deleteIntermediateOutputs = true - // ------------------------------------------------------- // // job run information @@ -95,8 +94,6 @@ trait QFunction extends Logging with QJobReport { * @param function QFunction to copy values to. */ override def copySettingsTo(function: QFunction) { - function.analysisName = this.analysisName - function.jobName = this.jobName function.qSettings = this.qSettings function.commandDirectory = this.commandDirectory function.jobTempDir = this.jobTempDir @@ -105,79 +102,80 @@ trait QFunction extends Logging with QJobReport { function.jobRestartable = this.jobRestartable function.updateJobRun = this.updateJobRun function.isIntermediate = this.isIntermediate - function.deleteIntermediateOutputs = this.deleteIntermediateOutputs function.reportGroup = this.reportGroup function.reportFeatures = this.reportFeatures } /** File to redirect any output. Defaults to .out */ - @Output(doc="File to redirect any output", required=false) - @Gather(classOf[SimpleTextGatherFunction]) var jobOutputFile: File = _ /** File to redirect any errors. Defaults to .out */ - @Output(doc="File to redirect any errors", required=false) - @Gather(classOf[SimpleTextGatherFunction]) var jobErrorFile: File = _ /** * Description of this command line function. */ - def description: String + def description: String = "%s: %s > %s".format(analysisName, inputs, outputs) /** - * The function description in .dot files + * A short description of the function. */ - def dotString = jobName + " => " + description + def shortDescription = { + firstOutput match { + case file: File => analysisName + ": " + file.getName + case _ => analysisName + } + } /** - * Returns true if the function is done, false if it's - * not done and None if the done status is unknown. + * Returns true if the function is done. */ - def isDone = { + def isDone: Boolean = { val files = doneOutputs if (files.size == 0) - None - else - Some(files.forall(_.exists)) + throw new IllegalStateException("Function should have at least one output: " + analysisName) + files.forall(_.exists) } /** - * Returns true if the function has failed, false if it - * has not failed and None if the fail status is unknown. + * Returns true if the function has failed. */ - def isFail = { + def isFail: Boolean = { val files = failOutputs if (files.size == 0) - None - else - Some(files.exists(_.exists)) + throw new IllegalStateException("Function should have at least one output: " + analysisName) + files.exists(_.exists) } /** - * Returns true if the file is a log file for this function. + * Returns files to track for hidden done/fail files. + * @return Seq[String] files. */ - protected def isLogFile(file: File) = - file == jobOutputFile || file == jobErrorFile + protected def statusPaths = { + var paths = outputs + paths :+= jobOutputFile + if (jobErrorFile != null) + paths :+= jobErrorFile + paths + } + + /** + * Returns prefixes for hidden done/fail files. + * @return prefixes. + */ + private def statusPrefixes = statusPaths.map(file => file.getParentFile + "/." + file.getName) /** * Returns the output files for this function. - * @return Set[File] outputs for this function. + * @return outputs for this function. */ - private def statusPaths = - commandOutputs.map(file => file.getParentFile + "/." + file.getName) - - /** - * Returns the output files for this function. - * @return Set[File] outputs for this function. - */ - def doneOutputs = statusPaths.map(path => new File(path + ".done")) + def doneOutputs: Seq[File] = statusPrefixes.map(path => new File(path + ".done")) /** * Returns the output files for this function. - * @return Set[File] outputs for this function. + * @return outputs for this function. */ - def failOutputs = statusPaths.map(path => new File(path + ".fail")) + def failOutputs: Seq[File] = statusPrefixes.map(path => new File(path + ".fail")) /** The complete list of fields on this CommandLineFunction. */ def functionFields = QFunction.classFields(this.functionFieldClass).functionFields @@ -195,21 +193,21 @@ trait QFunction extends Logging with QJobReport { /** * Returns the input files for this function. - * @return Set[File] inputs for this function. + * @return inputs for this function. */ - def inputs = getFieldFiles(inputFields) + def inputs: Seq[File] = getFieldFiles(inputFields) /** * Returns the output files for this function. - * @return Set[File] outputs for this function. + * @return outputs for this function. */ - def outputs = getFieldFiles(outputFields) + def outputs: Seq[File] = getFieldFiles(outputFields) /** - * Returns the non-log outputs for this function. - * @return the non-log outputs for this function. + * Returns the first output file. + * @return first output for this function. */ - def commandOutputs = outputs.filterNot(file => isLogFile(file)) + def firstOutput: File = outputs.headOption.getOrElse(null) /** * Returns the set of directories where files may be written. @@ -218,6 +216,9 @@ trait QFunction extends Logging with QJobReport { var dirs = Set.empty[File] dirs += commandDirectory dirs += jobTempDir + dirs += jobOutputFile.getParentFile + if (jobErrorFile != null) + dirs += jobErrorFile.getParentFile dirs ++= outputs.map(_.getParentFile) dirs } @@ -235,7 +236,7 @@ trait QFunction extends Logging with QJobReport { * Deletes the output files and all the status files for this function. */ def deleteOutputs() { - commandOutputs.foreach(file => IOUtils.tryDelete(file)) + outputs.foreach(file => IOUtils.tryDelete(file)) doneOutputs.foreach(file => IOUtils.tryDelete(file)) failOutputs.foreach(file => IOUtils.tryDelete(file)) } @@ -252,63 +253,63 @@ trait QFunction extends Logging with QJobReport { /** * Returns fields that do not have values which are required. - * @return List[String] names of fields missing values. + * @return Seq[String] names of fields missing values. */ - def missingFields: List[String] = { + def missingFields: Seq[String] = { val missingInputs = missingFields(inputFields, classOf[Input]) val missingOutputs = missingFields(outputFields, classOf[Output]) val missingArguments = missingFields(argumentFields, classOf[Argument]) - (missingInputs | missingOutputs | missingArguments).toList.sorted + (missingInputs ++ missingOutputs ++ missingArguments).distinct.sorted } /** * Returns fields that do not have values which are required. * @param sources Fields to check. * @param annotation Annotation. - * @return Set[String] names of fields missing values. + * @return names of fields missing values. */ - private def missingFields(sources: List[ArgumentSource], annotation: Class[_ <: Annotation]): Set[String] = { - var missing = Set.empty[String] + private def missingFields(sources: Seq[ArgumentSource], annotation: Class[_ <: Annotation]): Seq[String] = { + var missing: Seq[String] = Nil for (source <- sources) { if (isRequired(source, annotation)) if (!hasFieldValue(source)) if (!exclusiveOf(source, annotation).exists(otherSource => hasFieldValue(otherSource))) - missing += "@%s: %s - %s".format(annotation.getSimpleName, source.field.getName, doc(source, annotation)) + missing :+= "@%s: %s - %s".format(annotation.getSimpleName, source.field.getName, doc(source, annotation)) } missing } /** - * Gets the files from the fields. The fields must be a File, a FileExtension, or a List or Set of either. + * Gets the files from the fields. The fields must be a File, a FileExtension, or a Seq or Set of either. * @param fields Fields to get files. - * @return Set[File] for the fields. + * @return for the fields. */ - private def getFieldFiles(fields: List[ArgumentSource]): Set[File] = { - var files = Set.empty[File] + private def getFieldFiles(fields: Seq[ArgumentSource]): Seq[File] = { + var files: Seq[File] = Nil for (field <- fields) files ++= getFieldFiles(field) - files + files.distinct } /** - * Gets the files from the field. The field must be a File, a FileExtension, or a List or Set of either. - * @param fields Field to get files. - * @return Set[File] for the field. + * Gets the files from the field. The field must be a File, a FileExtension, or a Seq or Set of either. + * @param field Field to get files. + * @return for the field. */ - def getFieldFiles(field: ArgumentSource): Set[File] = { - var files = Set.empty[File] + def getFieldFiles(field: ArgumentSource): Seq[File] = { + var files: Seq[File] = Nil CollectionUtils.foreach(getFieldValue(field), (fieldValue) => { val file = fieldValueToFile(field, fieldValue) if (file != null) - files += file + files :+= file }) - files + files.distinct } /** - * Gets the file from the field. The field must be a File or a FileExtension and not a List or Set. + * Gets the file from the field. The field must be a File or a FileExtension and not a Seq or Set. * @param field Field to get the file. - * @return File for the field. + * @return for the field. */ def getFieldFile(field: ArgumentSource): File = fieldValueToFile(field, getFieldValue(field)) @@ -340,14 +341,15 @@ trait QFunction extends Logging with QJobReport { * Sets all field values. */ def freezeFieldValues() { - if (jobNamePrefix == null) - jobNamePrefix = qSettings.jobNamePrefix - if (jobName == null) - jobName = QFunction.nextJobName(jobNamePrefix) + jobName = qSettings.runName + "-" + this.addOrder.mkString("-") - if (jobOutputFile == null) - jobOutputFile = new File(jobName + ".out") + if (jobOutputFile == null) { + jobOutputFile = firstOutput match { + case file: File => new File(file.getParentFile, file.getName + ".out") + case _ => new File(jobName + ".out") + } + } if (jobTempDir == null) jobTempDir = qSettings.tempDirectory @@ -378,6 +380,10 @@ trait QFunction extends Logging with QJobReport { fieldValue = CollectionUtils.updated(fieldValue, canon).asInstanceOf[AnyRef] this.setFieldValue(field, fieldValue) } + + this.jobOutputFile = canon(this.jobOutputFile).asInstanceOf[File] + if (this.jobErrorFile != null) + this.jobErrorFile = canon(this.jobErrorFile).asInstanceOf[File] } /** @@ -443,7 +449,7 @@ trait QFunction extends Logging with QJobReport { /** * Returns false if the value is null or an empty collection. - * @param value Value to test for null, or a collection to test if it is empty. + * @param param Value to test for null, or a collection to test if it is empty. * @return false if the value is null, or false if the collection is empty, otherwise true. */ protected def hasValue(param: Any) = CollectionUtils.isNotNullOrNotEmpty(param) @@ -472,28 +478,15 @@ trait QFunction extends Logging with QJobReport { } object QFunction { - /** Job index counter for this run of Queue. */ - private var jobIndex = 0 - var parsingEngine: ParsingEngine = _ - /** - * Returns the next job name using the prefix. - * @param prefix Prefix of the job name. - * @return the next job name. - */ - private def nextJobName(prefix: String) = { - jobIndex += 1 - prefix + "-" + jobIndex - } - /** * The list of fields defined on a class * @param clazz The class to lookup fields. */ private class ClassFields(clazz: Class[_]) { /** The complete list of fields on this CommandLineFunction. */ - val functionFields: List[ArgumentSource] = parsingEngine.extractArgumentSources(clazz).toList + val functionFields: Seq[ArgumentSource] = parsingEngine.extractArgumentSources(clazz).toSeq /** The @Input fields on this CommandLineFunction. */ val inputFields = functionFields.filter(source => ReflectionUtils.hasAnnotation(source.field, classOf[Input])) /** The @Output fields on this CommandLineFunction. */ diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/CloneFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/CloneFunction.scala index b5cef3d5c..5b4f2b7e6 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/CloneFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/CloneFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -25,7 +25,6 @@ package org.broadinstitute.sting.queue.function.scattergather import org.broadinstitute.sting.commandline.ArgumentSource -import java.io.File import org.broadinstitute.sting.queue.function.{QFunction, CommandLineFunction} /** @@ -62,9 +61,8 @@ class CloneFunction extends CommandLineFunction { } } - override def commandOutputs = withScatterPart(() => originalFunction.commandOutputs) - override def dotString = withScatterPart(() => originalFunction.dotString) override def description = withScatterPart(() => originalFunction.description) + override def shortDescription = withScatterPart(() => originalFunction.shortDescription) override protected def functionFieldClass = originalFunction.getClass def commandLine = withScatterPart(() => originalFunction.commandLine) @@ -75,30 +73,22 @@ class CloneFunction extends CommandLineFunction { } override def getFieldValue(source: ArgumentSource): AnyRef = { - source.field.getName match { - case "jobOutputFile" => jobOutputFile - case "jobErrorFile" => jobErrorFile - case _ => overriddenFields.get(source) match { - case Some(value) => value.asInstanceOf[AnyRef] - case None => { - val value = originalFunction.getFieldValue(source) - overriddenFields += source -> value - value - } + overriddenFields.get(source) match { + case Some(value) => value.asInstanceOf[AnyRef] + case None => { + val value = originalFunction.getFieldValue(source) + overriddenFields += source -> value + value } } } - def setFieldValue(field: String, value: Any): Unit = { + def setFieldValue(field: String, value: Any) { val source = QFunction.findField(originalFunction.getClass, field) setFieldValue(source, value) } - override def setFieldValue(source: ArgumentSource, value: Any): Unit = { - source.field.getName match { - case "jobOutputFile" => jobOutputFile = value.asInstanceOf[File] - case "jobErrorFile" => jobErrorFile = value.asInstanceOf[File] - case _ => overriddenFields += source -> value - } + override def setFieldValue(source: ArgumentSource, value: Any) { + overriddenFields += source -> value } } diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/AutoIndexGatherFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ConcatenateLogsFunction.scala similarity index 53% rename from public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/AutoIndexGatherFunction.scala rename to public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ConcatenateLogsFunction.scala index 7fb96e074..9261dd767 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/AutoIndexGatherFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ConcatenateLogsFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -22,15 +22,34 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -package org.broadinstitute.sting.queue.extensions.gatk +package org.broadinstitute.sting.queue.function.scattergather -import org.broadinstitute.sting.queue.function.scattergather.GatherFunction import org.broadinstitute.sting.queue.function.InProcessFunction +import org.broadinstitute.sting.queue.QException +import org.broadinstitute.sting.commandline.Input +import org.apache.commons.io.FileUtils +import java.io.File +import collection.JavaConversions._ /** - * A no-op for index files that were automatically generated during the gather step. - * TODO: Allow graph to know that this isn't needed, and/or that one gather job can actually gather N-outputs, and/or look more into generic source->sinks. + * Concatenate log files to the jobOutputFile. */ -class AutoIndexGatherFunction extends InProcessFunction with GatherFunction { - def run() {} +class ConcatenateLogsFunction extends InProcessFunction { + analysisName = "Concat" + + @Input(doc="Parts to gather back into the original output") + var logs: Seq[File] = Nil + + override def description = "%s: %s > %s".format(analysisName, logs, jobOutputFile) + override def shortDescription = analysisName + ": " + jobOutputFile.getName + + def run() { + val missing = org.broadinstitute.sting.utils.io.IOUtils.waitFor(logs, 120) + if (!missing.isEmpty) + throw new QException("Unable to find log: " + missing.mkString(", ")) + logs.foreach(log => { + FileUtils.copyFile(log, this.jobOutputStream) + this.jobOutputStream.println() + }) + } } diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GatherFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GatherFunction.scala index 6b8b5d143..c8b9d52fb 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GatherFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GatherFunction.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.function.scattergather import java.io.File @@ -11,22 +35,31 @@ import collection.JavaConversions._ * Base class for Gather command line functions. */ trait GatherFunction extends QFunction { + analysisName = "Gather" + var originalFunction: ScatterGatherableFunction = _ @Output(doc="The original output of the scattered function") var originalOutput: File = _ @Input(doc="Parts to gather back into the original output") - var gatherParts: List[File] = Nil - - @Input(doc="Other log files that will be gathered before this output", required=false) - var originalLogFiles: List[File] = Nil + var gatherParts: Seq[File] = Nil /** * Called to initialize the gather function values after all other values have been setup for this function. */ def init() {} + /** + * Don't include this @Gather's log file when tracking .done. + * The done files for original log file being produced will do. + * + * The logs from the scatter/gather jobs are concatenated together into the original log. + * Without removing the logs a .done file would be created for the logs. If a SGF is switched + * from scatterCount=1 to >1 then this Gather would be "missing" its logs and re-run. + */ + override protected def statusPaths = outputs + /** * Waits for gather parts to propagate over NFS or throws an exception. */ diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GathererFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GathererFunction.scala index c1204fd1d..536bbf5fc 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GathererFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GathererFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -32,11 +32,12 @@ import collection.JavaConversions._ * Runs a Gatherer in process. */ class GathererFunction(gathererClass: Class[_ <: Gatherer]) extends InProcessFunction with GatherFunction { + analysisName = this.gathererClass.getSimpleName + def run() { val gatherer = gathererClass.newInstance if (gatherer.waitForInputs) - waitForGatherParts + waitForGatherParts() gatherer.gather(this.gatherParts, this.originalOutput) } - override def description = this.gathererClass.getSimpleName + " " + this.commandOutputs.mkString(" ") } diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterFunction.scala index 632e2d39f..a40747671 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterFunction.scala @@ -32,14 +32,17 @@ import org.broadinstitute.sting.queue.function.QFunction * Base class for Scatter functions. */ trait ScatterFunction extends QFunction { + analysisName = "Scatter" + var originalFunction: ScatterGatherableFunction = _ @Input(doc="Original inputs to scatter") var originalInputs: Set[File] = _ + override def shortDescription = analysisName + ": %s ...".format(firstOutput.getName) + /** * Called to initialize scatter function values after all other values have been setup for this function. - * @param originalFunction The original function to with inputs bind to this scatter function. */ def init() {} diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala index 402da4a7a..921928bce 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -30,6 +30,7 @@ import org.broadinstitute.sting.commandline.{Gatherer, Gather, ArgumentSource} import org.broadinstitute.sting.queue.function.{QFunction, CommandLineFunction} import org.broadinstitute.sting.queue.QException import org.broadinstitute.sting.utils.io.IOUtils +import collection.immutable.ListMap /** * A function that can be run faster by splitting it up into pieces and then joining together the results. @@ -47,28 +48,28 @@ trait ScatterGatherableFunction extends CommandLineFunction { /** * Function that returns the class to use for gathering a directory. If it returns null then @Gather annotation will be used. - * @param gatherField Field that is to be gathered. + * PartialFunction param gatherField Field that is to be gathered. * @return The class of the GatherFunction to be used or null. */ var gatherClass: PartialFunction[ArgumentSource, Class[_ <: GatherFunction]] = _ /** * Allows external modification of the ScatterFunction that will create the scatter pieces in the temporary directories. - * @param scatterFunction The function that will create the scatter pieces in the temporary directories. + * PartialFunction param scatterFunction The function that will create the scatter pieces in the temporary directories. */ var setupScatterFunction: PartialFunction[ScatterFunction, Unit] = _ /** * Allows external modification of the GatherFunction that will collect the gather pieces in the temporary directories. - * @param gatherFunction The function that will merge the gather pieces from the temporary directories. - * @param gatherField The output field being gathered. + * PartialFunction param gatherFunction The function that will merge the gather pieces from the temporary directories. + * PartialFunction param gatherField The output field being gathered. */ var setupGatherFunction: PartialFunction[(GatherFunction, ArgumentSource), Unit] = _ /** * Allows external modification of the cloned function. - * @param cloneFunction A clone wrapper of this ScatterGatherableFunction - * @param index The one based index (from 1..scatterCount inclusive) of the scatter piece. + * PartialFunction param cloneFunction A clone wrapper of this ScatterGatherableFunction + * PartialFunction param index The one based index (from 1..scatterCount inclusive) of the scatter piece. */ var setupCloneFunction: PartialFunction[(CloneFunction, Int), Unit] = _ @@ -108,8 +109,9 @@ trait ScatterGatherableFunction extends CommandLineFunction { scatterFunction.originalFunction = this scatterFunction.originalInputs = inputFiles scatterFunction.commandDirectory = this.scatterGatherTempDir("scatter") - scatterFunction.isIntermediate = true + scatterFunction.jobOutputFile = new File("scatter.out") scatterFunction.addOrder = this.addOrder :+ 1 + scatterFunction.isIntermediate = true initScatterFunction(scatterFunction) scatterFunction.absoluteCommandDirectory() @@ -121,69 +123,61 @@ trait ScatterGatherableFunction extends CommandLineFunction { * Returns a list of scatter / gather and clones of this function * that can be run in parallel to produce the same output as this * command line function. - * @return List[QFunction] to run instead of this function. + * @return Seq[QFunction] to run instead of this function. */ def generateFunctions() = { - var functions = List.empty[QFunction] - - // Only gather up fields that will have a value - val outputFieldsWithValues = this.outputFields.filter(hasFieldValue(_)) - - // Create the scatter function based on @Scatter - functions :+= scatterFunction - // Ask the scatter function how many clones to create. val numClones = scatterFunction.scatterCount - // List of the log files that are output by this function. - var logFiles = List(this.jobOutputFile) - if (this.jobErrorFile != null) - logFiles :+= this.jobErrorFile - // Create the gather functions for each output field - var gatherFunctions = Map.empty[ArgumentSource, GatherFunction] - var gatherOutputs = Map.empty[ArgumentSource, File] + var gatherFunctions = ListMap.empty[ArgumentSource, GatherFunction] + var gatherOutputs = ListMap.empty[ArgumentSource, File] var gatherAddOrder = numClones + 2 + + // Only track fields that will have a value + val outputFieldsWithValues = this.outputFields.filter(hasFieldValue(_)) + for (gatherField <- outputFieldsWithValues) { - val gatherOutput = getFieldFile(gatherField) + gatherOutputs += gatherField -> getFieldFile(gatherField) + } + + // Only gather fields that are @Gather(enabled=true) + val outputFieldsWithGathers = outputFieldsWithValues.filter(hasGatherFunction(_)) + + for (gatherField <- outputFieldsWithGathers) { + val gatherOutput = gatherOutputs(gatherField) val gatherFunction = this.newGatherFunction(gatherField) this.copySettingsTo(gatherFunction) gatherFunction.originalFunction = this gatherFunction.originalOutput = gatherOutput gatherFunction.commandDirectory = this.scatterGatherTempDir("gather-" + gatherField.field.getName) - // If this is a gather for a log file, make the gather intermediate just in case the log file name changes - // Otherwise have the regular output function wait on the log files to gather - if (isLogFile(gatherOutput)) { - gatherFunction.isIntermediate = true - // Only delete the log files if the original function is an intermediate - // and the intermediate files are supposed to be deleted - gatherFunction.deleteIntermediateOutputs = this.isIntermediate && this.deleteIntermediateOutputs - } else { - gatherFunction.originalLogFiles = logFiles - } + gatherFunction.jobOutputFile = new File("gather-" + gatherOutput.getName + ".out") gatherFunction.addOrder = this.addOrder :+ gatherAddOrder initGatherFunction(gatherFunction, gatherField) gatherFunction.absoluteCommandDirectory() gatherFunction.init() - functions :+= gatherFunction gatherFunctions += gatherField -> gatherFunction - gatherOutputs += gatherField -> gatherOutput gatherAddOrder += 1 } // Create the clone functions for running the parallel jobs - var cloneFunctions = List.empty[CloneFunction] + var cloneFunctions = Seq.empty[CloneFunction] + val dirFormat = "temp_%%0%dd_of_%d".format(numClones.toString.length(), numClones) for (i <- 1 to numClones) { val cloneFunction = this.newCloneFunction() this.copySettingsTo(cloneFunction) cloneFunction.originalFunction = this + cloneFunction.analysisName = this.analysisName cloneFunction.cloneIndex = i - cloneFunction.commandDirectory = this.scatterGatherTempDir("temp-"+i) + cloneFunction.commandDirectory = this.scatterGatherTempDir(dirFormat.format(i)) + cloneFunction.jobOutputFile = new File(this.jobOutputFile.getName) + if (this.jobErrorFile != null) + cloneFunction.jobErrorFile = new File(this.jobErrorFile.getName) cloneFunction.addOrder = this.addOrder :+ (i+1) cloneFunction.isIntermediate = true @@ -200,17 +194,39 @@ trait ScatterGatherableFunction extends CommandLineFunction { // If the command directory is relative, insert the run directory ahead of it. cloneFunction.absoluteCommandDirectory() - // Get absolute paths to the files and bind the sg functions to the clone function via the absolute paths. + // Allow the scatter function to set the specific input for this clone scatterFunction.bindCloneInputs(cloneFunction, i) + + // Set each of the clone outputs to be absolute paths. for (gatherField <- outputFieldsWithValues) { val gatherPart = IOUtils.absolute(cloneFunction.commandDirectory, cloneFunction.getFieldFile(gatherField)) cloneFunction.setFieldValue(gatherField, gatherPart) - gatherFunctions(gatherField).gatherParts :+= gatherPart + } + + // For the outputs that are being gathered add this clone's output to be gathered. + for (gatherField <- outputFieldsWithGathers) { + gatherFunctions(gatherField).gatherParts :+= cloneFunction.getFieldFile(gatherField) } cloneFunctions :+= cloneFunction } - functions ++= cloneFunctions + + // Track the functions starting with the scatter function. + var functions: Seq[QFunction] = Seq(scatterFunction) ++ cloneFunctions ++ gatherFunctions.values + + // Make all log file paths absolute. + for (function <- functions) { + function.jobOutputFile = IOUtils.absolute(function.commandDirectory, function.jobOutputFile) + if (function.jobErrorFile != null) + function.jobErrorFile = IOUtils.absolute(function.commandDirectory, function.jobErrorFile) + } + + val jobOutputGather = gatherLogFile(_.jobOutputFile, functions, gatherAddOrder) + if (this.jobErrorFile != null) { + val jobErrorGather = gatherLogFile(_.jobErrorFile, functions, gatherAddOrder + 1) + functions :+= jobErrorGather + } + functions :+= jobOutputGather // Return all the various created functions. functions @@ -237,6 +253,25 @@ trait ScatterGatherableFunction extends CommandLineFunction { this.setupScatterFunction(scatterFunction) } + /** + * Returns true if the field should be gathered. + * @param gatherField Field that defined @Gather. + * @return true if the field should be gathered. + */ + protected def hasGatherFunction(gatherField: ArgumentSource) : Boolean = { + // Check if there is a function that will return the gather class for this field. + if (this.gatherClass != null && this.gatherClass.isDefinedAt(gatherField)) + true + + // Check for an annotation defining the gather class. + else if (ReflectionUtils.hasAnnotation(gatherField.field, classOf[Gather])) + ReflectionUtils.getAnnotation(gatherField.field, classOf[Gather]).enabled + + // Nothing else to disable this field. + else + true + } + /** * Creates a new GatherFunction for the gatherField. * @param gatherField Field that defined @Gather. @@ -255,16 +290,18 @@ trait ScatterGatherableFunction extends CommandLineFunction { if (ReflectionUtils.hasAnnotation(gatherField.field, classOf[Gather])) { gatherClass = ReflectionUtils.getAnnotation(gatherField.field, classOf[Gather]).value } else { - throw new QException("Missing @Gather annotation: " + gatherField.field) + throw new QException("Missing @Gather annotation on %s".format(gatherField.field)) } } - if (classOf[GatherFunction].isAssignableFrom(gatherClass)) { + if (gatherClass == classOf[GatherFunction]) { + throw new QException("@Gather did not specify class type on %s".format(gatherField.field)) + } else if (classOf[GatherFunction].isAssignableFrom(gatherClass)) { gatherClass.newInstance.asInstanceOf[GatherFunction] } else if (classOf[Gatherer].isAssignableFrom(gatherClass)) { new GathererFunction(gatherClass.asSubclass(classOf[Gatherer])) } else { - throw new QException("Unsupported @Gather class type: " + gatherClass) + throw new QException("Unsupported @Gather class type on %s: %s".format(gatherField.field, gatherClass)) } } @@ -298,10 +335,27 @@ trait ScatterGatherableFunction extends CommandLineFunction { this.setupCloneFunction(cloneFunction, index) } + /** + * Gathers up the logs files from other functions. + * @param logFile Takes the QFunction and return the log file. + * @param functions The functions for which the logs will be concatenated. + * @param addOrder The order this function should be added in the graph. + */ + private def gatherLogFile(logFile: (QFunction) => File, functions: Seq[QFunction], addOrder: Int) = { + val gatherLogFunction = new ConcatenateLogsFunction + this.copySettingsTo(gatherLogFunction) + gatherLogFunction.logs = functions.map(logFile).filter(_ != null) + gatherLogFunction.jobOutputFile = logFile(this) + gatherLogFunction.commandDirectory = this.scatterGatherTempDir() + gatherLogFunction.addOrder = this.addOrder :+ addOrder + gatherLogFunction.isIntermediate = false + gatherLogFunction + } + /** * Returns a temporary directory under this scatter gather directory. - * @param Sub directory under the scatter gather directory. + * @param subDir directory under the scatter gather directory. * @return temporary directory under this scatter gather directory. */ - private def scatterGatherTempDir(subDir: String) = IOUtils.absolute(this.scatterGatherDirectory, this.jobName + "-sg/" + subDir) + private def scatterGatherTempDir(subDir: String = "") = IOUtils.absolute(this.scatterGatherDirectory, this.jobName + "-sg/" + subDir) } diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala b/public/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala index cda981d29..2ef7aa06f 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -34,12 +34,12 @@ import scala.collection.JavaConversions._ */ class EmailMessage extends Logging { var from: String = _ - var to: List[String] = Nil - var cc: List[String] = Nil - var bcc: List[String] = Nil + var to: Seq[String] = Nil + var cc: Seq[String] = Nil + var bcc: Seq[String] = Nil var subject: String = _ var body: String = _ - var attachments: List[File] = Nil + var attachments: Seq[File] = Nil /** * Sends the email and throws an exception if the email can't be sent. @@ -111,10 +111,10 @@ class EmailMessage extends Logging { /** * Converts the email addresses to a collection of InternetAddress which can bypass client side validation, * specifically that the domain is specified. - * @param addresses List of email addresses. + * @param addresses Seq of email addresses. * @return java.util.List of InternetAddress'es */ - private def convert(addresses: List[String]): java.util.List[InternetAddress] = { + private def convert(addresses: Seq[String]): java.util.List[InternetAddress] = { addresses.map(address => new InternetAddress(address, false)) } diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/QJobReport.scala b/public/scala/src/org/broadinstitute/sting/queue/util/QJobReport.scala index 73d1c028a..e548e5c5e 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/util/QJobReport.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/util/QJobReport.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -28,9 +28,10 @@ import org.broadinstitute.sting.queue.function.QFunction import org.broadinstitute.sting.gatk.report.{GATKReportTable, GATKReport} import org.broadinstitute.sting.utils.exceptions.UserException import org.broadinstitute.sting.queue.engine.JobRunInfo -import java.io.{FileOutputStream, PrintStream, File} +import java.io.{PrintStream, File} import org.broadinstitute.sting.utils.R.{RScriptLibrary, RScriptExecutor} import org.broadinstitute.sting.utils.io.Resource +import org.apache.commons.io.{IOUtils, FileUtils} /** * A mixin to add Job info to the class @@ -67,7 +68,7 @@ trait QJobReport extends Logging { def getReportGroup = self.analysisName.replaceAll(GATKReportTable.INVALID_TABLE_NAME_REGEX, "_") def getReportFeatures = reportFeatures - def getReportFeatureNames: List[String] = getReportFeatures.keys.toList + def getReportFeatureNames: Seq[String] = getReportFeatures.keys.toSeq def getReportFeature(key: String): String = { getReportFeatures.get(key) match { case Some(x) => x @@ -102,9 +103,12 @@ object QJobReport { def printReport(jobsRaw: Map[QFunction, JobRunInfo], dest: File) { val jobs = jobsRaw.filter(_._2.isFilledIn).filter(_._1.includeInReport) jobs foreach {case (qf, info) => qf.setRunInfo(info)} - val stream = new PrintStream(new FileOutputStream(dest)) - printJobLogging(jobs.keys.toList, stream) - stream.close() + val stream = new PrintStream(FileUtils.openOutputStream(dest)) + try { + printJobLogging(jobs.keys.toSeq, stream) + } finally { + IOUtils.closeQuietly(stream) + } } def plotReport(reportFile: File, pdfFile: File) { @@ -129,7 +133,7 @@ object QJobReport { * Prints the JobLogging logs to a GATKReport. First splits up the * logs by group, and for each group generates a GATKReportTable */ - private def printJobLogging(logs: List[QFunction], stream: PrintStream) { + private def printJobLogging(logs: Seq[QFunction], stream: PrintStream) { // create the report val report: GATKReport = new GATKReport @@ -151,11 +155,11 @@ object QJobReport { report.print(stream) } - private def groupLogs(logs: List[QFunction]): Map[String, List[QFunction]] = { + private def groupLogs(logs: Seq[QFunction]): Map[String, Seq[QFunction]] = { logs.groupBy(_.getReportGroup) } - private def logKeys(logs: List[QFunction]): Set[String] = { + private def logKeys(logs: Seq[QFunction]): Set[String] = { // the keys should be the same for each log, but we will check that val keys = Set[String](logs(0).getReportFeatureNames : _*) diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala b/public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala index 3b1b2ece1..5d76f39ed 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.util import java.io.File @@ -12,23 +36,22 @@ import collection.JavaConversions._ * User: carneiro * Date: 7/14/11 * Time: 4:57 PM - * To change this template use File | Settings | File Templates. */ object QScriptUtils { /** - * Takes a bam list file and produces a scala list with each file allowing the bam list + * Takes a bam list file and produces a scala sequence with each file allowing the bam list * to have empty lines and comment lines (lines starting with #). */ - def createListFromFile(in: File):List[File] = { + def createSeqFromFile(in: File):Seq[File] = { // If the file provided ends with .bam, .fasta or .fq, it is not a bam list, we treat it as a single file. // and return a list with only this file. if (in.toString.endsWith(".bam") || in.toString.endsWith(".fasta") || in.toString.endsWith(".fq")) - return List(in) + return Seq(in) - var list: List[File] = List() - for (file <- fromFile(in).getLines) + var list: Seq[File] = Seq() + for (file <- fromFile(in).getLines()) if (!file.startsWith("#") && !file.isEmpty ) list :+= new File(file.trim()) list.sortWith(_.compareTo(_) < 0) @@ -55,8 +78,4 @@ object QScriptUtils { } false } - - - def ?[A <: AnyRef](ref: A): Option[A] = - if (ref eq null) None else Some(ref) -} \ No newline at end of file +} diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/ReflectionUtils.scala b/public/scala/src/org/broadinstitute/sting/queue/util/ReflectionUtils.scala index f6a174dd6..980a22e8e 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/util/ReflectionUtils.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/util/ReflectionUtils.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.util import org.broadinstitute.sting.queue.QException @@ -64,17 +88,17 @@ object ReflectionUtils { /** * Returns all the declared fields on a class in order of sub type to super type. * @param clazz Base class to start looking for fields. - * @return List[Field] found on the class and all super classes. + * @return Seq[Field] found on the class and all super classes. */ - def getAllFields(clazz: Class[_]) = getAllTypes(clazz).map(_.getDeclaredFields).flatMap(_.toList) + def getAllFields(clazz: Class[_]) = getAllTypes(clazz).map(_.getDeclaredFields).flatMap(_.toSeq) /** * Gets all the types on a class in order of sub type to super type. * @param clazz Base class. - * @return List[Class] including the class and all super classes. + * @return Seq[Class] including the class and all super classes. */ def getAllTypes(clazz: Class[_]) = { - var types = List.empty[Class[_]] + var types = Seq.empty[Class[_]] var c = clazz while (c != null) { types :+= c diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala b/public/scala/src/org/broadinstitute/sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala index 58341a0a5..6b615e6d9 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.util import collection.JavaConversions._ @@ -14,32 +38,34 @@ class ScalaCompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor { /** * Checks if the class type is a scala collection. * @param classType Class type to check. - * @return true if the class is a List, Set, or an Option. + * @return true if the class is a Seq, Set, or an Option. */ def supports(classType: Class[_]) = isCompound(classType) /** * Checks if the class type is a scala collection. * @param source Argument source to check. - * @return true if the source is a List, Set, or an Option. + * @return true if the source is a Seq, Set, or an Option. */ override def isMultiValued(source: ArgumentSource) = isCompound(source.field.getType) /** * Checks if the class type is a scala collection. * @param classType Class type to check. - * @return true if the class is a List, Set, or an Option. + * @return true if the class is a Seq, Set, or an Option. */ private def isCompound(classType: Class[_]) = { - classOf[List[_]].isAssignableFrom(classType) || + classOf[Seq[_]].isAssignableFrom(classType) || + classOf[List[_]].isAssignableFrom(classType) || // see comment below re: List vs. Seq classOf[Set[_]].isAssignableFrom(classType) || classOf[Option[_]].isAssignableFrom(classType) } /** * Parses the argument matches based on the class type of the argument source's field. + * @param parsingEngine Parsing engine. * @param source Argument source that contains the field being populated. - * @param classType Class type being parsed. + * @param typeType Type of the argument source's field. * @param argumentMatches The argument match strings that were found for this argument source. * @return The parsed object. */ @@ -51,7 +77,15 @@ class ScalaCompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor { val componentType = ReflectionUtils.getCollectionType(source.field) val componentArgumentParser = parsingEngine.selectBestTypeDescriptor(componentType) - if (classOf[List[_]].isAssignableFrom(classType)) { + if (classOf[Seq[_]].isAssignableFrom(classType)) { + var seq = Seq.empty[Any] + for (argumentMatch <- argumentMatches) + for (value <- argumentMatch) + seq :+= componentArgumentParser.parse(parsingEngine, source, componentType, new ArgumentMatches(value)) + seq + } else if (classOf[List[_]].isAssignableFrom(classType)) { + // QScripts should be using the interface Seq instead of the class List. + // Leaving this here for now for legacy support until the effects of switching have been tested for a while. -ks var list = List.empty[Any] for (argumentMatch <- argumentMatches) for (value <- argumentMatch) diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/StringFileConversions.scala b/public/scala/src/org/broadinstitute/sting/queue/util/StringFileConversions.scala index 62240b604..2c6d62ae9 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/util/StringFileConversions.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/util/StringFileConversions.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -44,7 +44,7 @@ object StringFileConversions { // and mixins all correct so this doesn't have to be duplicated with concrete implementations? // http://programming-scala.labs.oreilly.com/ch12.html is your friend. - implicit def stringsAsFiles(x: List[Comparable[_ >: String with File <: Comparable[_ >: String with File <: Serializable] with Serializable] with Serializable]): List[File] = { + implicit def stringsAsFiles(x: Seq[Comparable[_ >: String with File <: Comparable[_ >: String with File <: Serializable] with Serializable] with Serializable]): Seq[File] = { x.map(_ match { case string: String => stringAsFile(string) case file: File => file @@ -52,7 +52,23 @@ object StringFileConversions { }) } - implicit def filesAsStrings(x: List[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable]): List[String] = { + implicit def filesAsStrings(x: Seq[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable]): Seq[String] = { + x.map(_ match { + case file: File => fileAsString(file) + case string: String => string + case null => null + }) + } + + implicit def stringsAsFilesList(x: List[Comparable[_ >: String with File <: Comparable[_ >: String with File <: Serializable] with Serializable] with Serializable]): List[File] = { + x.map(_ match { + case string: String => stringAsFile(string) + case file: File => file + case null => null + }) + } + + implicit def filesAsStringsList(x: List[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable]): List[String] = { x.map(_ match { case file: File => fileAsString(file) case string: String => string @@ -91,14 +107,22 @@ trait StringFileConversions { StringFileConversions.fileAsString(x) } - implicit def stringsAsFiles(x: List[Comparable[_ >: String with File <: Comparable[_ >: String with File <: Serializable] with Serializable] with Serializable]): List[File] = { + implicit def stringsAsFiles(x: Seq[Comparable[_ >: String with File <: Comparable[_ >: String with File <: Serializable] with Serializable] with Serializable]): Seq[File] = { StringFileConversions.stringsAsFiles(x) } - implicit def filesAsStrings(x: List[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable]): List[String] = { + implicit def filesAsStrings(x: Seq[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable]): Seq[String] = { StringFileConversions.filesAsStrings(x) } + implicit def stringsAsFilesList(x: List[Comparable[_ >: String with File <: Comparable[_ >: String with File <: Serializable] with Serializable] with Serializable]): List[File] = { + StringFileConversions.stringsAsFilesList(x) + } + + implicit def filesAsStringsList(x: List[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable]): List[String] = { + StringFileConversions.filesAsStringsList(x) + } + implicit def stringsAsFiles(x: Set[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable] with Serializable]): Set[File] = { StringFileConversions.stringsAsFiles(x) } diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/SystemUtils.scala b/public/scala/src/org/broadinstitute/sting/queue/util/SystemUtils.scala index 9002def78..ed149f8a4 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/util/SystemUtils.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/util/SystemUtils.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -24,7 +24,6 @@ package org.broadinstitute.sting.queue.util -import java.lang.management.ManagementFactory import java.net.InetAddress import java.io.File import io.Source @@ -56,6 +55,4 @@ object SystemUtils extends Logging { else hostName.split('.').takeRight(2).mkString(".") } - - val pidAtHost = ManagementFactory.getRuntimeMXBean.getName.split('.').head } diff --git a/public/scala/test/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervalsUnitTest.scala b/public/scala/test/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervalsUnitTest.scala index 38abe24ef..db0d187c9 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervalsUnitTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervalsUnitTest.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -40,9 +40,6 @@ class GATKIntervalsUnitTest { createSetFromSequenceDictionary(new ReferenceDataSource(hg18Reference).getReference.getSequenceDictionary).toList private final lazy val hg19Reference = new File(BaseTest.hg19Reference) - private final lazy val hg19GenomeLocParser = new GenomeLocParser(new CachingIndexedFastaSequenceFile(hg19Reference)) - private final lazy val hg19ReferenceLocs = GenomeLocSortedSet. - createSetFromSequenceDictionary(new ReferenceDataSource(hg19Reference).getReference.getSequenceDictionary).toList @Test def testWithIntervals() { @@ -50,16 +47,14 @@ class GATKIntervalsUnitTest { val chr2 = hg18GenomeLocParser.parseGenomeLoc("chr2:2-3") val chr3 = hg18GenomeLocParser.parseGenomeLoc("chr3:3-5") - val gi = new GATKIntervals(hg18Reference, List("chr1:1-1", "chr2:2-3", "chr3:3-5")) - Assert.assertEquals(gi.locs.toList, List(chr1, chr2, chr3)) - Assert.assertEquals(gi.contigs, List("chr1", "chr2", "chr3")) -// Assert.assertEquals(gi.getSplits(2).toList, List(2, 3)) -// Assert.assertEquals(gi.getSplits(3).toList, List(1, 2, 3)) + val gi = new GATKIntervals(hg18Reference, Seq("chr1:1-1", "chr2:2-3", "chr3:3-5")) + Assert.assertEquals(gi.locs.toSeq, Seq(chr1, chr2, chr3)) + Assert.assertEquals(gi.contigs, Seq("chr1", "chr2", "chr3")) } - @Test(timeOut = 30000) + @Test(timeOut = 30000L) def testIntervalFile() { - var gi = new GATKIntervals(hg19Reference, List(BaseTest.hg19Intervals)) + var gi = new GATKIntervals(hg19Reference, Seq(BaseTest.hg19Intervals)) Assert.assertEquals(gi.locs.size, 189894) // Timeout check is because of bad: // for(Item item: javaConvertedScalaList) @@ -74,14 +69,12 @@ class GATKIntervalsUnitTest { val gi = new GATKIntervals(hg18Reference, Nil) Assert.assertEquals(gi.locs, hg18ReferenceLocs) Assert.assertEquals(gi.contigs.size, hg18ReferenceLocs.size) -// Assert.assertEquals(gi.getSplits(2).toList, List(10, 45)) -// Assert.assertEquals(gi.getSplits(4).toList, List(5, 10, 16, 45)) } @Test def testContigCounts() { Assert.assertEquals(new GATKIntervals(hg18Reference, Nil).contigs, hg18ReferenceLocs.map(_.getContig)) - Assert.assertEquals(new GATKIntervals(hg18Reference, List("chr1", "chr2", "chr3")).contigs, List("chr1", "chr2", "chr3")) - Assert.assertEquals(new GATKIntervals(hg18Reference, List("chr1:1-2", "chr1:4-5", "chr2:1-1", "chr3:2-2")).contigs, List("chr1", "chr2", "chr3")) + Assert.assertEquals(new GATKIntervals(hg18Reference, Seq("chr1", "chr2", "chr3")).contigs, Seq("chr1", "chr2", "chr3")) + Assert.assertEquals(new GATKIntervals(hg18Reference, Seq("chr1:1-2", "chr1:4-5", "chr2:1-1", "chr3:2-2")).contigs, Seq("chr1", "chr2", "chr3")) } } diff --git a/public/scala/test/org/broadinstitute/sting/queue/function/CommandLineFunctionUnitTest.scala b/public/scala/test/org/broadinstitute/sting/queue/function/CommandLineFunctionUnitTest.scala index eb50c3a2e..9c5b648d2 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/function/CommandLineFunctionUnitTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/function/CommandLineFunctionUnitTest.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.function import org.testng.Assert @@ -114,20 +138,20 @@ class CommandLineFunctionUnitTest extends CommandLineFunction { @DataProvider( name = "repeatTestData" ) def repeatDataProvider = { - Array(Array("", List("a", "bc", "d"), "", " ", true, true, " 'a' 'bc' 'd' "), - Array("", List("a", "bc", "d"), "", " ", true, false, " a bc d "), - Array("", List("a", "bc", "d"), "", "", true, true, " 'a''bc''d' "), - Array("", List("a", "bc", "d"), "", "", true, false, " abcd "), - Array("-f", List("file1", "file2", "file3"), "", " ", true, true, " '-f' 'file1' '-f' 'file2' '-f' 'file3' "), - Array("-f", List("file1", "file2", "file3"), "", " ", true, false, " -f file1 -f file2 -f file3 "), - Array("-f", List("file1", "file2", "file3"), "", " ", false, true, " '-ffile1' '-ffile2' '-ffile3' "), - Array("-f", List("file1", "file2", "file3"), "", " ", false, false, " -ffile1 -ffile2 -ffile3 "), - Array("-f", List("file1", "file2", "file3"), "", "", false, true, " '-ffile1''-ffile2''-ffile3' "), - Array("-f", List("file1", "file2", "file3"), "", "", false, false, " -ffile1-ffile2-ffile3 "), - Array("-f", List("file1", "file2", "file3"), "suffix", " ", true, true, " '-f' 'file1' 'suffix' '-f' 'file2' 'suffix' '-f' 'file3' 'suffix' "), - Array("-f", List("file1", "file2", "file3"), "suffix", " ", true, false, " -f file1 suffix -f file2 suffix -f file3 suffix "), - Array("-f", List("file1", "file2", "file3"), "suffix", " ", false, true, " '-ffile1suffix' '-ffile2suffix' '-ffile3suffix' "), - Array("-f", List("file1", "file2", "file3"), "suffix", " ", false, false, " -ffile1suffix -ffile2suffix -ffile3suffix "), + Array(Array("", Seq("a", "bc", "d"), "", " ", true, true, " 'a' 'bc' 'd' "), + Array("", Seq("a", "bc", "d"), "", " ", true, false, " a bc d "), + Array("", Seq("a", "bc", "d"), "", "", true, true, " 'a''bc''d' "), + Array("", Seq("a", "bc", "d"), "", "", true, false, " abcd "), + Array("-f", Seq("file1", "file2", "file3"), "", " ", true, true, " '-f' 'file1' '-f' 'file2' '-f' 'file3' "), + Array("-f", Seq("file1", "file2", "file3"), "", " ", true, false, " -f file1 -f file2 -f file3 "), + Array("-f", Seq("file1", "file2", "file3"), "", " ", false, true, " '-ffile1' '-ffile2' '-ffile3' "), + Array("-f", Seq("file1", "file2", "file3"), "", " ", false, false, " -ffile1 -ffile2 -ffile3 "), + Array("-f", Seq("file1", "file2", "file3"), "", "", false, true, " '-ffile1''-ffile2''-ffile3' "), + Array("-f", Seq("file1", "file2", "file3"), "", "", false, false, " -ffile1-ffile2-ffile3 "), + Array("-f", Seq("file1", "file2", "file3"), "suffix", " ", true, true, " '-f' 'file1' 'suffix' '-f' 'file2' 'suffix' '-f' 'file3' 'suffix' "), + Array("-f", Seq("file1", "file2", "file3"), "suffix", " ", true, false, " -f file1 suffix -f file2 suffix -f file3 suffix "), + Array("-f", Seq("file1", "file2", "file3"), "suffix", " ", false, true, " '-ffile1suffix' '-ffile2suffix' '-ffile3suffix' "), + Array("-f", Seq("file1", "file2", "file3"), "suffix", " ", false, false, " -ffile1suffix -ffile2suffix -ffile3suffix "), Array("-f", null, "", " ", true, true, ""), Array("-f", Nil, "", " ", true, true, "") ) @@ -148,11 +172,11 @@ class CommandLineFunctionUnitTest extends CommandLineFunction { @DataProvider( name = "repeatWithPrefixFormattingTestData" ) def repeatWithPrefixFormattingDataProvider = { - Array(Array("-f", List("file1", "file2", "file3"), "", " ", true, true, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value), + Array(Array("-f", Seq("file1", "file2", "file3"), "", " ", true, true, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value), " '-f:tagfile1' 'file1' '-f:tagfile2' 'file2' '-f:tagfile3' 'file3' "), - Array("-f", List("file1", "file2", "file3"), "", " ", true, false, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value), + Array("-f", Seq("file1", "file2", "file3"), "", " ", true, false, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value), " -f:tagfile1 file1 -f:tagfile2 file2 -f:tagfile3 file3 "), - Array("", List("file1", "file2", "file3"), "", " ", true, true, (prefix: String, value: Any) => "-%s".format(value), + Array("", Seq("file1", "file2", "file3"), "", " ", true, true, (prefix: String, value: Any) => "-%s".format(value), " '-file1' 'file1' '-file2' 'file2' '-file3' 'file3' "), Array("-f", null, "", " ", true, true, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value), ""), diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala index aedbc1cd3..f0feb207b 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -47,10 +47,10 @@ object PipelineTest extends BaseTest with Logging { final val allJobRunners = { val commandLinePluginManager = new CommandLinePluginManager - commandLinePluginManager.getPlugins.map(commandLinePluginManager.getName(_)).toList + commandLinePluginManager.getPlugins.map(commandLinePluginManager.getName(_)).toSeq } - final val defaultJobRunners = List("Lsf706", "GridEngine") + final val defaultJobRunners = Seq("Lsf706", "GridEngine") /** * Returns the top level output path to this test. diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTestEvalSpec.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTestEvalSpec.scala index 33b8c1c39..3996f2ca3 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTestEvalSpec.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTestEvalSpec.scala @@ -1,14 +1,38 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.pipeline /** * Data validations to evaluate on a GATKReport. */ class PipelineTestEvalSpec { - /** List of eval modules to output. */ + /** Eval modules to output. */ var evalReport: String = _ /** Validations to assert. */ - var validations: List[PipelineValidation[_]] = Nil + var validations: Seq[PipelineValidation[_]] = Nil } /** A VariantEval JEXL and range of values to validate. */ diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTestSpec.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTestSpec.scala index a7b3f3a47..090024698 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTestSpec.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTestSpec.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.pipeline class PipelineTestSpec(var name: String = null) { @@ -9,7 +33,7 @@ class PipelineTestSpec(var name: String = null) { var jobQueue: String = _ /** Job runners to run the test. Default is null which means use the default. */ - var jobRunners: List[String] = _ + var jobRunners: Seq[String] = _ /** Expected MD5 results for each file path. */ var fileMD5s = Map.empty[String, String] diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/HelloWorldPipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/HelloWorldPipelineTest.scala index f320cb3a6..a43727ba6 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/HelloWorldPipelineTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/HelloWorldPipelineTest.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -38,11 +38,11 @@ class HelloWorldPipelineTest { } @Test - def testHelloWorldWithPrefix() { + def testHelloWorldWithRunName() { val spec = new PipelineTestSpec - spec.name = "HelloWorldWithPrefix" + spec.name = "HelloWorldWithRunName" spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" + - " -jobPrefix HelloWorld" + " -runName HelloWorld" spec.jobRunners = PipelineTest.allJobRunners PipelineTest.executeTest(spec) } @@ -73,7 +73,7 @@ class HelloWorldPipelineTest { spec.name = "HelloWorldWithLsfResource" spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" + " -jobResReq rusage[iodine_io=1] -jobResReq select[swp>0] -jobResReq order[swp]" - spec.jobRunners = List("Lsf706") + spec.jobRunners = Seq("Lsf706") PipelineTest.executeTest(spec) } @@ -83,7 +83,7 @@ class HelloWorldPipelineTest { spec.name = "HelloWorldWithLsfResourceAndMemoryLimit" spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" + " -memLimit 1.25 -jobResReq rusage[iodine_io=1] -jobResReq select[swp>0] -jobResReq order[swp]" - spec.jobRunners = List("Lsf706") + spec.jobRunners = Seq("Lsf706") PipelineTest.executeTest(spec) } @@ -93,7 +93,7 @@ class HelloWorldPipelineTest { spec.name = "HelloWorldWithLsfEnvironment" spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" + " -jobEnv tv" - spec.jobRunners = List("Lsf706") + spec.jobRunners = Seq("Lsf706") PipelineTest.executeTest(spec) } @@ -103,7 +103,7 @@ class HelloWorldPipelineTest { spec.name = "HelloWorldWithGridEngineResource" spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" + " -jobResReq s_core=1000M" - spec.jobRunners = List("GridEngine") + spec.jobRunners = Seq("GridEngine") PipelineTest.executeTest(spec) } @@ -113,7 +113,7 @@ class HelloWorldPipelineTest { spec.name = "HelloWorldWithGridEngineResourceAndMemoryLimit" spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" + " -memLimit 1.25 -jobResReq s_core=1000M" - spec.jobRunners = List("GridEngine") + spec.jobRunners = Seq("GridEngine") PipelineTest.executeTest(spec) } @@ -123,7 +123,7 @@ class HelloWorldPipelineTest { spec.name = "HelloWorldWithGridEngineEnvironment" spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" + " -jobEnv \"make 1\"" - spec.jobRunners = List("GridEngine") + spec.jobRunners = Seq("GridEngine") PipelineTest.executeTest(spec) } } diff --git a/public/scala/test/org/broadinstitute/sting/queue/util/StringFileConversionsUnitTest.scala b/public/scala/test/org/broadinstitute/sting/queue/util/StringFileConversionsUnitTest.scala index a735edebe..4d364040a 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/util/StringFileConversionsUnitTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/util/StringFileConversionsUnitTest.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -50,40 +50,40 @@ class StringFileConversionsUnitTest { @Test def testStringToFileList() { - var files = List(new File("foo")) + var files = Seq(new File("foo")) files :+= "bar" - Assert.assertEquals(files, List(new File("foo"), new File("bar"))) + Assert.assertEquals(files, Seq(new File("foo"), new File("bar"))) - files = List(new File("foo")) + files = Seq(new File("foo")) files :+= null.asInstanceOf[String] - Assert.assertEquals(files, List(new File("foo"), null)) + Assert.assertEquals(files, Seq(new File("foo"), null)) - files = List[File](null) + files = Seq[File](null) files :+= "foo" - Assert.assertEquals(files, List(null, new File("foo"))) + Assert.assertEquals(files, Seq(null, new File("foo"))) - files = List[File](null) + files = Seq[File](null) files :+= null.asInstanceOf[String] - Assert.assertEquals(files, List(null, null)) + Assert.assertEquals(files, Seq(null, null)) } @Test def testFileToStringList() { - var strings = List("foo") + var strings = Seq("foo") strings :+= new File("bar") - Assert.assertEquals(strings, List("foo", "bar")) + Assert.assertEquals(strings, Seq("foo", "bar")) - strings = List("foo") + strings = Seq("foo") strings :+= null.asInstanceOf[File] - Assert.assertEquals(strings, List("foo", null)) + Assert.assertEquals(strings, Seq("foo", null)) - strings = List[String](null) + strings = Seq[String](null) strings :+= new File("foo") - Assert.assertEquals(strings, List(null, "foo")) + Assert.assertEquals(strings, Seq(null, "foo")) - strings = List[String](null) + strings = Seq[String](null) strings :+= null.asInstanceOf[File] - Assert.assertEquals(strings, List(null, null)) + Assert.assertEquals(strings, Seq(null, null)) } @Test @@ -126,40 +126,40 @@ class StringFileConversionsUnitTest { @Test def testStringListToFileList() { - var files = List(new File("foo")) - files ++= List("bar") - Assert.assertEquals(files, List(new File("foo"), new File("bar"))) + var files = Seq(new File("foo")) + files ++= Seq("bar") + Assert.assertEquals(files, Seq(new File("foo"), new File("bar"))) - files = List(new File("foo")) - files ++= List[String](null) - Assert.assertEquals(files, List(new File("foo"), null)) + files = Seq(new File("foo")) + files ++= Seq[String](null) + Assert.assertEquals(files, Seq(new File("foo"), null)) - files = List[File](null) - files ++= List("foo") - Assert.assertEquals(files, List(null, new File("foo"))) + files = Seq[File](null) + files ++= Seq("foo") + Assert.assertEquals(files, Seq(null, new File("foo"))) - files = List[File](null) - files ++= List[String](null) - Assert.assertEquals(files, List(null, null)) + files = Seq[File](null) + files ++= Seq[String](null) + Assert.assertEquals(files, Seq(null, null)) } @Test def testFileListToStringList() { - var strings = List("foo") - strings ++= List(new File("bar")) - Assert.assertEquals(strings, List("foo", "bar")) + var strings = Seq("foo") + strings ++= Seq(new File("bar")) + Assert.assertEquals(strings, Seq("foo", "bar")) - strings = List("foo") - strings ++= List[File](null) - Assert.assertEquals(strings, List("foo", null)) + strings = Seq("foo") + strings ++= Seq[File](null) + Assert.assertEquals(strings, Seq("foo", null)) - strings = List[String](null) - strings ++= List(new File("foo")) - Assert.assertEquals(strings, List(null, "foo")) + strings = Seq[String](null) + strings ++= Seq(new File("foo")) + Assert.assertEquals(strings, Seq(null, "foo")) - strings = List[String](null) - strings ++= List[File](null) - Assert.assertEquals(strings, List(null, null)) + strings = Seq[String](null) + strings ++= Seq[File](null) + Assert.assertEquals(strings, Seq(null, null)) } @Test