diff --git a/java/src/org/broadinstitute/sting/gatk/datasources/simpleDataSources/ReferenceOrderedDataSource.java b/java/src/org/broadinstitute/sting/gatk/datasources/simpleDataSources/ReferenceOrderedDataSource.java index eba717cd0..55f4d1f3f 100755 --- a/java/src/org/broadinstitute/sting/gatk/datasources/simpleDataSources/ReferenceOrderedDataSource.java +++ b/java/src/org/broadinstitute/sting/gatk/datasources/simpleDataSources/ReferenceOrderedDataSource.java @@ -1,12 +1,13 @@ package org.broadinstitute.sting.gatk.datasources.simpleDataSources; +import org.broad.tribble.FeatureReader; import org.broadinstitute.sting.gatk.datasources.shards.Shard; import org.broadinstitute.sting.gatk.refdata.SeekableRODIterator; -import org.broadinstitute.sting.gatk.refdata.tracks.QueryableTrack; import org.broadinstitute.sting.gatk.refdata.tracks.RMDTrack; +import org.broadinstitute.sting.gatk.refdata.tracks.builders.TribbleRMDTrackBuilder; +import org.broadinstitute.sting.gatk.refdata.utils.FeatureToGATKFeatureIterator; import org.broadinstitute.sting.gatk.refdata.utils.FlashBackIterator; import org.broadinstitute.sting.gatk.refdata.utils.LocationAwareSeekableRODIterator; -import org.broadinstitute.sting.gatk.refdata.utils.RODRecordList; import org.broadinstitute.sting.gatk.walkers.ReadWalker; import org.broadinstitute.sting.gatk.walkers.Walker; import org.broadinstitute.sting.utils.GenomeLoc; @@ -41,7 +42,7 @@ public class ReferenceOrderedDataSource implements SimpleDataSource { /** * A pool of iterators for navigating through the genome. */ - private final ReferenceOrderedDataPool iteratorPool; + private final ResourcePool iteratorPool; /** * Create a new reference-ordered data source. @@ -49,8 +50,10 @@ public class ReferenceOrderedDataSource implements SimpleDataSource { */ public ReferenceOrderedDataSource( Walker walker, RMDTrack rod) { this.rod = rod; - if (rod.supportsQuery()) iteratorPool = null; - else iteratorPool = new ReferenceOrderedDataPool( walker, rod ); + if (rod.supportsQuery()) + iteratorPool = new ReferenceOrderedQueryDataPool(new TribbleRMDTrackBuilder(), rod); + else + iteratorPool = new ReferenceOrderedDataPool( walker, rod ); } /** @@ -75,11 +78,8 @@ public class ReferenceOrderedDataSource implements SimpleDataSource { * @return Iterator through the data. */ public LocationAwareSeekableRODIterator seek( Shard shard ) { - if (iteratorPool == null) // use query - return getQuery(shard.getGenomeLocs() == null || shard.getGenomeLocs().size() == 0 ? null : shard.getGenomeLocs()); DataStreamSegment dataStreamSegment = shard.getGenomeLocs().size() != 0 ? new MappedStreamSegment(shard.getGenomeLocs().get(0)) : new EntireStream(); - LocationAwareSeekableRODIterator RODIterator = iteratorPool.iterator(dataStreamSegment); - return RODIterator; + return iteratorPool.iterator(dataStreamSegment); } /** @@ -90,30 +90,17 @@ public class ReferenceOrderedDataSource implements SimpleDataSource { * @return Iterator through the data. */ public LocationAwareSeekableRODIterator seek(GenomeLoc loc) { - if (iteratorPool == null) // use query - return getQuery(loc == null ? null : Arrays.asList(loc)); DataStreamSegment dataStreamSegment = loc != null ? new MappedStreamSegment(loc) : new EntireStream(); - LocationAwareSeekableRODIterator RODIterator = iteratorPool.iterator(dataStreamSegment); - return RODIterator; + return iteratorPool.iterator(dataStreamSegment); } - /** - * assuming the ROD is a queryable ROD, use that interface to get an iterator to the selected region - * @param loc the region to query for - * @return a LocationAwareSeekableRODIterator over the selected region - */ - private LocationAwareSeekableRODIterator getQuery(List loc) { - if (loc == null) // for the mono shard case - return new SeekableRODIterator(rod.getIterator()); - return new StitchingLocationAwareSeekableRODIterator(loc,(QueryableTrack)rod); - } /** * Close the specified iterator, returning it to the pool. * @param iterator Iterator to close. */ public void close( LocationAwareSeekableRODIterator iterator ) { - if (iteratorPool != null) iteratorPool.release(iterator); + iteratorPool.release(iterator); } } @@ -189,78 +176,54 @@ class ReferenceOrderedDataPool extends ResourcePool { - // the list of intervals we're iterating over - private final LinkedList locationList; + // the reference-ordered data itself. + private final RMDTrack rod; - // The reference-ordered data itself. - private final QueryableTrack rod; + // our tribble track builder + private final TribbleRMDTrackBuilder builder; - // the current iterator - private SeekableRODIterator iterator; - - StitchingLocationAwareSeekableRODIterator(List list, QueryableTrack rmd) { - rod = rmd; - locationList = new LinkedList(); - locationList.addAll(list); - fetchNextInterval(); + public ReferenceOrderedQueryDataPool( TribbleRMDTrackBuilder builder, RMDTrack rod ) { + this.rod = rod; + this.builder = builder; } @Override - public GenomeLoc peekNextLocation() { - if (iterator == null) return null; - return iterator.peekNextLocation(); + protected FeatureReader createNewResource() { + return builder.createFeatureReader(rod.getType(),rod.getFile()); } @Override - public GenomeLoc position() { - if (iterator == null) return null; - return iterator.position(); + protected FeatureReader selectBestExistingResource(DataStreamSegment segment, List availableResources) { + for (FeatureReader reader : availableResources) + if (reader != null) return reader; + return null; } @Override - public RODRecordList seekForward(GenomeLoc interval) { - RODRecordList list = iterator.seekForward(interval); - if (list == null) { // we were unable to seek the current interval to the location - fetchNextInterval(); - list = iterator.seekForward(interval); - } - return list; - } - - @Override - public boolean hasNext() { - if (iterator == null) return false; - return iterator.hasNext(); - } - - @Override - public RODRecordList next() { - if (!hasNext()) throw new IllegalStateException("StitchingLocationAwareSeekableRODIterator: We do not have a next"); - RODRecordList list = iterator.next(); - if (!iterator.hasNext()) fetchNextInterval(); - return list; - } - - @Override - public void remove() { - throw new UnsupportedOperationException("\"Thou shall not remove()!\" - Software Engineering Team"); - } - - private void fetchNextInterval() { - if (locationList != null && locationList.size() > 0) { - GenomeLoc loc = locationList.getFirst(); - locationList.removeFirst(); - if (rod == null) throw new StingException("Unable to query(), target rod is null, next location = " + ((locationList != null) ? locationList.getFirst() : "null")); - try { - iterator = new SeekableRODIterator(rod.query(loc)); - } catch (IOException e) { - throw new StingException("Unable to query iterator with location " + loc + " and rod name of " + ((RMDTrack)rod).getName()); + protected LocationAwareSeekableRODIterator createIteratorFromResource(DataStreamSegment position, FeatureReader resource) { + try { + if (position instanceof MappedStreamSegment) { + GenomeLoc pos = ((MappedStreamSegment) position).locus; + //System.err.println("Querying position1 " + pos.getContig() + " start " + pos.getStart() + " stop " + pos.getStop()); + return new SeekableRODIterator(new FeatureToGATKFeatureIterator(resource.query(pos.getContig(), (int) pos.getStart(), (int) pos.getStop()),rod.getName())); + } else { + return new SeekableRODIterator(new FeatureToGATKFeatureIterator(resource.iterator(),rod.getName())); } + } catch (IOException e) { + throw new StingException("Unable to create iterator for rod named " + rod.getName()); + } + } + + @Override + protected void closeResource(FeatureReader resource) { + try { + resource.close(); + } catch (IOException e) { + throw new StingException("Unable to close reader for rod named " + rod.getName()); } } } diff --git a/java/src/org/broadinstitute/sting/gatk/datasources/simpleDataSources/ResourcePool.java b/java/src/org/broadinstitute/sting/gatk/datasources/simpleDataSources/ResourcePool.java index 91d92a7b2..90564444a 100755 --- a/java/src/org/broadinstitute/sting/gatk/datasources/simpleDataSources/ResourcePool.java +++ b/java/src/org/broadinstitute/sting/gatk/datasources/simpleDataSources/ResourcePool.java @@ -50,26 +50,27 @@ abstract class ResourcePool { public I iterator( DataStreamSegment segment ) { // Grab the first iterator in the list whose position is before the requested position. T selectedResource = null; - synchronized(this) { - selectedResource = selectBestExistingResource( segment, availableResources ); + synchronized (this) { + selectedResource = selectBestExistingResource(segment, availableResources); // No iterator found? Create another. It is expected that // each iterator created will have its own file handle. - if( selectedResource == null ) { + if (selectedResource == null) { selectedResource = createNewResource(); - addNewResource( selectedResource ); + addNewResource(selectedResource); } // Remove the iterator from the list of available iterators. availableResources.remove(selectedResource); + + + I iterator = createIteratorFromResource(segment, selectedResource); + + // Make a note of this assignment for proper releasing later. + resourceAssignments.put(iterator, selectedResource); + + return iterator; } - - I iterator = createIteratorFromResource( segment, selectedResource ); - - // Make a note of this assignment for proper releasing later. - resourceAssignments.put( iterator, selectedResource ); - - return iterator; } /** diff --git a/java/src/org/broadinstitute/sting/gatk/io/storage/GenotypeWriterStorage.java b/java/src/org/broadinstitute/sting/gatk/io/storage/GenotypeWriterStorage.java index a0351a81e..145169d09 100755 --- a/java/src/org/broadinstitute/sting/gatk/io/storage/GenotypeWriterStorage.java +++ b/java/src/org/broadinstitute/sting/gatk/io/storage/GenotypeWriterStorage.java @@ -73,6 +73,7 @@ public abstract class GenotypeWriterStorage implements this.stream = null; writer = GenotypeWriterFactory.create(stub.getFormat(), file); Set samples = SampleUtils.getSAMFileSamples(stub.getSAMFileHeader()); + // TODO: this line is a problem, creating with an empty hashset eliminates any genotype FORMAT fields in the calls (besides GT) GenotypeWriterFactory.writeHeader(writer, stub.getSAMFileHeader(), samples, new HashSet()); } diff --git a/java/src/org/broadinstitute/sting/gatk/io/storage/VCFGenotypeWriterStorage.java b/java/src/org/broadinstitute/sting/gatk/io/storage/VCFGenotypeWriterStorage.java index d142a5ffb..873e51b1c 100644 --- a/java/src/org/broadinstitute/sting/gatk/io/storage/VCFGenotypeWriterStorage.java +++ b/java/src/org/broadinstitute/sting/gatk/io/storage/VCFGenotypeWriterStorage.java @@ -64,11 +64,11 @@ public class VCFGenotypeWriterStorage extends GenotypeWriterStorage - * Class PeekableRODIterator - *

- * the methods attached to a peekable ROD iterator - */ -public interface PeekableRODIterator extends Iterator> { - public GenomeLoc peekNextLocation(); - - public RODRecordList seekForward(GenomeLoc interval); -} - diff --git a/java/src/org/broadinstitute/sting/gatk/iterators/PushbackIterator.java b/java/src/org/broadinstitute/sting/gatk/iterators/PushbackIterator.java index b8633c401..d3b6e3e14 100755 --- a/java/src/org/broadinstitute/sting/gatk/iterators/PushbackIterator.java +++ b/java/src/org/broadinstitute/sting/gatk/iterators/PushbackIterator.java @@ -30,7 +30,6 @@ public class PushbackIterator implements Iterator, Iterable { /** * Retrieves, but does not remove, the head of this iterator. * @return T the next element in the iterator - * @throws NoSuchElementException - if the iterator doesn't have a next element */ public T element() { T x = next(); @@ -40,7 +39,6 @@ public class PushbackIterator implements Iterator, Iterable { /** * @return the next element in the iteration. - * @throws NoSuchElementException - iteration has no more elements. */ public T next() { if (pushedElement != null) { diff --git a/java/src/org/broadinstitute/sting/gatk/refdata/SeekableRODIterator.java b/java/src/org/broadinstitute/sting/gatk/refdata/SeekableRODIterator.java index 3992c129b..d670c9bfc 100644 --- a/java/src/org/broadinstitute/sting/gatk/refdata/SeekableRODIterator.java +++ b/java/src/org/broadinstitute/sting/gatk/refdata/SeekableRODIterator.java @@ -1,5 +1,6 @@ package org.broadinstitute.sting.gatk.refdata; +import net.sf.samtools.util.CloseableIterator; import org.broadinstitute.sting.gatk.iterators.PushbackIterator; import org.broadinstitute.sting.gatk.refdata.utils.GATKFeature; import org.broadinstitute.sting.gatk.refdata.utils.LocationAwareSeekableRODIterator; @@ -78,7 +79,7 @@ public class SeekableRODIterator implements LocationAwareSeekableRODIterator { // This implementation tracks the query history and makes next() illegal after a seekforward query of length > 1, // but re-enables next() again after a length-1 query. - public SeekableRODIterator(Iterator it) { + public SeekableRODIterator(CloseableIterator it) { this.it = new PushbackIterator(it); records = new LinkedList(); // the following is a trick: we would like the iterator to know the actual name assigned to @@ -169,7 +170,7 @@ public class SeekableRODIterator implements LocationAwareSeekableRODIterator { if ( r.getLocation().getStart() < curr_position ) throw new StingException("LocationAwareSeekableRODIterator: track "+r.getName() + - " is out of coordinate order on contig "+r.getLocation().getContig()); + " is out of coordinate order on contig "+r.getLocation() + " compared to " + curr_contig + ":" + curr_position); if ( r.getLocation().getStart() > curr_position ) break; // next record starts after the current position; we do not need it yet @@ -334,4 +335,8 @@ public class SeekableRODIterator implements LocationAwareSeekableRODIterator { } + @Override + public void close() { + if (this.it != null) ((CloseableIterator)this.it.getUnderlyingIterator()).close(); + } } diff --git a/java/src/org/broadinstitute/sting/gatk/refdata/tracks/FeatureReaderTrack.java b/java/src/org/broadinstitute/sting/gatk/refdata/tracks/FeatureReaderTrack.java index e88f1b43b..b70cc9921 100644 --- a/java/src/org/broadinstitute/sting/gatk/refdata/tracks/FeatureReaderTrack.java +++ b/java/src/org/broadinstitute/sting/gatk/refdata/tracks/FeatureReaderTrack.java @@ -23,6 +23,7 @@ package org.broadinstitute.sting.gatk.refdata.tracks; +import net.sf.samtools.util.CloseableIterator; import org.broad.tribble.FeatureReader; import org.broadinstitute.sting.gatk.refdata.utils.FeatureToGATKFeatureIterator; import org.broadinstitute.sting.gatk.refdata.utils.GATKFeature; @@ -64,7 +65,7 @@ public class FeatureReaderTrack extends RMDTrack implements QueryableTrack { * but other more advanced tracks support the query interface */ @Override - public Iterator getIterator() { + public CloseableIterator getIterator() { try { return new FeatureToGATKFeatureIterator(reader.iterator(),this.getName()); } catch (IOException e) { @@ -83,22 +84,22 @@ public class FeatureReaderTrack extends RMDTrack implements QueryableTrack { } @Override - public Iterator query(GenomeLoc interval) throws IOException { + public CloseableIterator query(GenomeLoc interval) throws IOException { return new FeatureToGATKFeatureIterator(reader.query(interval.getContig(),(int)interval.getStart(),(int)interval.getStop()),this.getName()); } @Override - public Iterator query(GenomeLoc interval, boolean contained) throws IOException { + public CloseableIterator query(GenomeLoc interval, boolean contained) throws IOException { return new FeatureToGATKFeatureIterator(reader.query(interval.getContig(),(int)interval.getStart(),(int)interval.getStop(), contained),this.getName()); } @Override - public Iterator query(String contig, int start, int stop) throws IOException { + public CloseableIterator query(String contig, int start, int stop) throws IOException { return new FeatureToGATKFeatureIterator(reader.query(contig,start,stop),this.getName()); } @Override - public Iterator query(String contig, int start, int stop, boolean contained) throws IOException { + public CloseableIterator query(String contig, int start, int stop, boolean contained) throws IOException { return new FeatureToGATKFeatureIterator(reader.query(contig,start,stop, contained),this.getName()); } diff --git a/java/src/org/broadinstitute/sting/gatk/refdata/tracks/QueryableTrack.java b/java/src/org/broadinstitute/sting/gatk/refdata/tracks/QueryableTrack.java index a8dc82de3..19050ae11 100644 --- a/java/src/org/broadinstitute/sting/gatk/refdata/tracks/QueryableTrack.java +++ b/java/src/org/broadinstitute/sting/gatk/refdata/tracks/QueryableTrack.java @@ -23,6 +23,7 @@ package org.broadinstitute.sting.gatk.refdata.tracks; +import net.sf.samtools.util.CloseableIterator; import org.broadinstitute.sting.gatk.refdata.utils.GATKFeature; import org.broadinstitute.sting.utils.GenomeLoc; @@ -37,9 +38,9 @@ import java.util.Iterator; * a decorator interface for tracks that are queryable */ public interface QueryableTrack { - public Iterator query(final GenomeLoc interval) throws IOException; - public Iterator query(final GenomeLoc interval, final boolean contained) throws IOException; - public Iterator query(final String contig, final int start, final int stop) throws IOException; - public Iterator query(final String contig, final int start, final int stop, final boolean contained) throws IOException; + public CloseableIterator query(final GenomeLoc interval) throws IOException; + public CloseableIterator query(final GenomeLoc interval, final boolean contained) throws IOException; + public CloseableIterator query(final String contig, final int start, final int stop) throws IOException; + public CloseableIterator query(final String contig, final int start, final int stop, final boolean contained) throws IOException; public void close(); } diff --git a/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrack.java b/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrack.java index d37d16860..aedac5305 100644 --- a/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrack.java +++ b/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrack.java @@ -23,6 +23,7 @@ package org.broadinstitute.sting.gatk.refdata.tracks; +import net.sf.samtools.util.CloseableIterator; import org.broadinstitute.sting.gatk.refdata.utils.GATKFeature; import java.io.File; @@ -73,7 +74,7 @@ public abstract class RMDTrack { * @return how to get an iterator of the underlying data. This is all a track has to support, * but other more advanced tracks support the query interface */ - public abstract Iterator getIterator(); + public abstract CloseableIterator getIterator(); /** * helper function for determining if we are the same track diff --git a/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RODRMDTrack.java b/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RODRMDTrack.java index 0ee93193c..88a19005c 100644 --- a/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RODRMDTrack.java +++ b/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RODRMDTrack.java @@ -23,6 +23,7 @@ package org.broadinstitute.sting.gatk.refdata.tracks; +import net.sf.samtools.util.CloseableIterator; import org.broadinstitute.sting.gatk.refdata.ReferenceOrderedData; import org.broadinstitute.sting.gatk.refdata.utils.GATKFeature; import org.broadinstitute.sting.gatk.refdata.utils.GATKFeatureIterator; @@ -64,7 +65,7 @@ public class RODRMDTrack extends RMDTrack { * but other more advanced tracks support the query interface */ @Override - public Iterator getIterator() { + public CloseableIterator getIterator() { return new GATKFeatureIterator(data.iterator()); } diff --git a/java/src/org/broadinstitute/sting/gatk/refdata/tracks/builders/TribbleRMDTrackBuilder.java b/java/src/org/broadinstitute/sting/gatk/refdata/tracks/builders/TribbleRMDTrackBuilder.java index c46f012b6..3d64aaa7c 100644 --- a/java/src/org/broadinstitute/sting/gatk/refdata/tracks/builders/TribbleRMDTrackBuilder.java +++ b/java/src/org/broadinstitute/sting/gatk/refdata/tracks/builders/TribbleRMDTrackBuilder.java @@ -26,11 +26,10 @@ package org.broadinstitute.sting.gatk.refdata.tracks.builders; import org.apache.log4j.Logger; -import org.broad.tribble.Feature; -import org.broad.tribble.FeatureCodec; -import org.broad.tribble.FeatureReader; +import org.broad.tribble.*; import org.broad.tribble.index.linear.LinearIndex; import org.broad.tribble.index.linear.LinearIndexCreator; +import org.broad.tribble.readers.BasicFeatureReader; import org.broadinstitute.sting.gatk.refdata.tracks.FeatureReaderTrack; import org.broadinstitute.sting.gatk.refdata.tracks.RMDTrack; import org.broadinstitute.sting.gatk.refdata.tracks.RMDTrackCreationException; @@ -53,16 +52,6 @@ import java.util.Map; * This class keeps track of the available codecs, and knows how to put together a track of * that gets iterators from the FeatureReader using Tribble. * - * Here's an example run command to find SNPs 200 base pairs up and downstream of the target file. - * - * java -jar dist/GenomeAnalysisTK.jar \ - * -R /broad/1KG/reference/human_b36_both.fasta \ - * -L 1:1863 \ - * -L MT:16520 \ - * -db /humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/dbSNP/dbsnp_129_b36.rod \ - * -dbw 200 \ - * -l INFO \ - * -T DbSNPWindowCounter */ public class TribbleRMDTrackBuilder extends PluginManager implements RMDTrackBuilder { /** @@ -120,11 +109,11 @@ public class TribbleRMDTrackBuilder extends PluginManager implemen // check to see if the input file has an index if (requireIndex(inputFile)) { logger.warn("Creating Tribble Index for file " + inputFile); - LinearIndex index = createIndex(inputFile, this.createByType(targetClass)); - reader = new FeatureReader(inputFile,index, this.createByType(targetClass)); + LinearIndex index = createIndex(inputFile, this.createByType(targetClass), true); + reader = new BasicFeatureReader(inputFile,index, this.createByType(targetClass)); } else { - reader = new FeatureReader(inputFile,this.createByType(targetClass)); + reader = new BasicFeatureReader(inputFile,this.createByType(targetClass)); } } catch (FileNotFoundException e) { throw new StingException("Unable to create reader with file " + inputFile, e); @@ -138,18 +127,19 @@ public class TribbleRMDTrackBuilder extends PluginManager implemen * create an index for the input file * @param inputFile the input file * @param codec the codec to use + * @param onDisk write the index to disk? * @return a linear index for the specified type * @throws IOException if we cannot write the index file */ - public static LinearIndex createIndex(File inputFile, FeatureCodec codec) throws IOException { + public static LinearIndex createIndex(File inputFile, FeatureCodec codec, boolean onDisk) throws IOException { LinearIndexCreator create = new LinearIndexCreator(inputFile, codec); // if we can write the index, we should, but if not just create it in memory File indexFile = new File(inputFile.getAbsoluteFile() + linearIndexExtension); - if (indexFile.getParentFile().canWrite() && (!indexFile.exists() || indexFile.canWrite())) + if (indexFile.getParentFile().canWrite() && (!indexFile.exists() || indexFile.canWrite()) && onDisk) return create.createIndex(); else { - logger.info("Unable to write to location " + indexFile + " for index file, creating index in memory only"); + if (onDisk) logger.info("Unable to write to location " + indexFile + " for index file, creating index in memory only"); return create.createIndex(null); } diff --git a/java/src/org/broadinstitute/sting/gatk/refdata/utils/FeatureToGATKFeatureIterator.java b/java/src/org/broadinstitute/sting/gatk/refdata/utils/FeatureToGATKFeatureIterator.java index 99a93f72c..f2fb8ffd5 100644 --- a/java/src/org/broadinstitute/sting/gatk/refdata/utils/FeatureToGATKFeatureIterator.java +++ b/java/src/org/broadinstitute/sting/gatk/refdata/utils/FeatureToGATKFeatureIterator.java @@ -23,6 +23,7 @@ package org.broadinstitute.sting.gatk.refdata.utils; +import net.sf.samtools.util.CloseableIterator; import org.broad.tribble.Feature; import java.util.Iterator; @@ -36,11 +37,11 @@ import java.util.Iterator; * * a wrapper on Tribble feature iterators so that they produce GATKFeatures (which produce GenomeLocs) */ -public class FeatureToGATKFeatureIterator implements Iterator { - private final Iterator iterator; +public class FeatureToGATKFeatureIterator implements CloseableIterator { + private final CloseableIterator iterator; private final String name; - public FeatureToGATKFeatureIterator(Iterator iter, String name) { + public FeatureToGATKFeatureIterator(CloseableIterator iter, String name) { this.name = name; this.iterator = iter; } @@ -59,4 +60,9 @@ public class FeatureToGATKFeatureIterator implements Iterator { public void remove() { throw new UnsupportedOperationException("Why does Iterator have this method? We always throw an exception here"); } + + @Override + public void close() { + this.iterator.close(); + } } diff --git a/java/src/org/broadinstitute/sting/gatk/refdata/utils/GATKFeatureIterator.java b/java/src/org/broadinstitute/sting/gatk/refdata/utils/GATKFeatureIterator.java index c366cffaa..17c9fa718 100644 --- a/java/src/org/broadinstitute/sting/gatk/refdata/utils/GATKFeatureIterator.java +++ b/java/src/org/broadinstitute/sting/gatk/refdata/utils/GATKFeatureIterator.java @@ -23,6 +23,7 @@ package org.broadinstitute.sting.gatk.refdata.utils; +import net.sf.samtools.util.CloseableIterator; import org.broadinstitute.sting.gatk.refdata.ReferenceOrderedDatum; import java.util.Iterator; @@ -36,7 +37,7 @@ import java.util.Iterator; * * Takes a RODatum iterator and makes it an iterator of GATKFeatures. Shazam! */ -public class GATKFeatureIterator implements Iterator { +public class GATKFeatureIterator implements CloseableIterator { private final Iterator iter; public GATKFeatureIterator(Iterator iter) { this.iter = iter; @@ -56,4 +57,9 @@ public class GATKFeatureIterator implements Iterator { public void remove() { throw new UnsupportedOperationException("Remove not supported"); } + + @Override + public void close() { + // do nothing, our underlying iterator doesn't support this + } } diff --git a/java/src/org/broadinstitute/sting/gatk/refdata/utils/LocationAwareSeekableRODIterator.java b/java/src/org/broadinstitute/sting/gatk/refdata/utils/LocationAwareSeekableRODIterator.java index df2cc8b33..29e42eab6 100644 --- a/java/src/org/broadinstitute/sting/gatk/refdata/utils/LocationAwareSeekableRODIterator.java +++ b/java/src/org/broadinstitute/sting/gatk/refdata/utils/LocationAwareSeekableRODIterator.java @@ -1,5 +1,6 @@ package org.broadinstitute.sting.gatk.refdata.utils; +import net.sf.samtools.util.CloseableIterator; import org.broadinstitute.sting.gatk.refdata.ReferenceOrderedDatum; import org.broadinstitute.sting.utils.GenomeLoc; @@ -13,11 +14,10 @@ import java.util.List; *

* combine iteration with a position aware interface */ -public interface LocationAwareSeekableRODIterator extends Iterator { +public interface LocationAwareSeekableRODIterator extends CloseableIterator { public GenomeLoc peekNextLocation(); public GenomeLoc position(); - public RODRecordList seekForward(GenomeLoc interval); - + public RODRecordList seekForward(GenomeLoc interval); } diff --git a/java/src/org/broadinstitute/sting/gatk/walkers/sequenom/PickSequenomProbes.java b/java/src/org/broadinstitute/sting/gatk/walkers/sequenom/PickSequenomProbes.java index 7e0eff304..7ca79fdd6 100755 --- a/java/src/org/broadinstitute/sting/gatk/walkers/sequenom/PickSequenomProbes.java +++ b/java/src/org/broadinstitute/sting/gatk/walkers/sequenom/PickSequenomProbes.java @@ -25,6 +25,7 @@ package org.broadinstitute.sting.gatk.walkers.sequenom; +import net.sf.samtools.util.CloseableIterator; import org.broad.tribble.dbsnp.DbSNPFeature; import org.broadinstitute.sting.gatk.contexts.AlignmentContext; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; @@ -73,7 +74,7 @@ public class PickSequenomProbes extends RodWalker { ReferenceOrderedData snp_mask; if ( SNP_MASK.contains(DbSNPHelper.STANDARD_DBSNP_TRACK_NAME)) { TribbleRMDTrackBuilder builder = new TribbleRMDTrackBuilder(); - Iterator iter = builder.createInstanceOfTrack(DbSNPFeature.class,"snp_mask",new java.io.File(SNP_MASK)).getIterator(); + CloseableIterator iter = builder.createInstanceOfTrack(DbSNPFeature.class,"snp_mask",new java.io.File(SNP_MASK)).getIterator(); snpMaskIterator = new SeekableRODIterator(iter); } else { diff --git a/java/src/org/broadinstitute/sting/oneoffprojects/walkers/DbSNPWindowCounter.java b/java/src/org/broadinstitute/sting/oneoffprojects/walkers/DbSNPWindowCounter.java index 30b84a3dd..b9ed2cf3b 100644 --- a/java/src/org/broadinstitute/sting/oneoffprojects/walkers/DbSNPWindowCounter.java +++ b/java/src/org/broadinstitute/sting/oneoffprojects/walkers/DbSNPWindowCounter.java @@ -1,9 +1,11 @@ package org.broadinstitute.sting.oneoffprojects.walkers; +import net.sf.samtools.util.CloseableIterator; import org.broad.tribble.FeatureIterator; import org.broad.tribble.FeatureReader; import org.broad.tribble.dbsnp.DbSNPCodec; import org.broad.tribble.dbsnp.DbSNPFeature; +import org.broad.tribble.util.CloseableTribbleIterator; import org.broadinstitute.sting.commandline.Argument; import org.broadinstitute.sting.gatk.contexts.AlignmentContext; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; @@ -49,7 +51,7 @@ public class DbSNPWindowCounter extends LocusWalker { public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) { - FeatureIterator dbSNPs; + CloseableTribbleIterator dbSNPs; // our upstream and downstream window locations int windowStart = (int)Math.max(context.getLocation().getStart()-windowSize,0); diff --git a/java/src/org/broadinstitute/sting/utils/genotype/vcf/VCFGenotypeWriter.java b/java/src/org/broadinstitute/sting/utils/genotype/vcf/VCFGenotypeWriter.java index a7f705272..9cd3b875e 100644 --- a/java/src/org/broadinstitute/sting/utils/genotype/vcf/VCFGenotypeWriter.java +++ b/java/src/org/broadinstitute/sting/utils/genotype/vcf/VCFGenotypeWriter.java @@ -7,7 +7,7 @@ import org.broadinstitute.sting.utils.genotype.GenotypeWriter; import java.util.Set; /** - * An extension of eth GenotypeWriter interface with support + * An extension of the GenotypeWriter interface with support * for adding header lines. * * @author mhanna diff --git a/java/src/org/broadinstitute/sting/utils/genotype/vcf/VCFReader.java b/java/src/org/broadinstitute/sting/utils/genotype/vcf/VCFReader.java index 5a22f8466..109fd0dcc 100644 --- a/java/src/org/broadinstitute/sting/utils/genotype/vcf/VCFReader.java +++ b/java/src/org/broadinstitute/sting/utils/genotype/vcf/VCFReader.java @@ -22,6 +22,7 @@ import java.util.zip.GZIPInputStream; import org.broad.tribble.FeatureReader; import org.broad.tribble.index.linear.LinearIndex; +import org.broad.tribble.readers.BasicFeatureReader; import org.broad.tribble.vcf.*; import org.broadinstitute.sting.gatk.refdata.tracks.builders.TribbleRMDTrackBuilder; import org.broadinstitute.sting.utils.StingException; @@ -45,7 +46,16 @@ public class VCFReader implements Iterator, Iterable { * @param vcfFile the vcf file to write */ public VCFReader(File vcfFile) { - initialize(vcfFile, null); + initialize(vcfFile, null, true); + } + + /** + * Create a VCF reader, given a VCF file + * + * @param vcfFile the vcf file to write + */ + public VCFReader(File vcfFile, boolean createIndexOnDisk) { + initialize(vcfFile, null, createIndexOnDisk); } /** @@ -54,22 +64,21 @@ public class VCFReader implements Iterator, Iterable { * @param vcfFile the vcf file to write */ public VCFReader(File vcfFile, VCFCodec.LineTransform transform) { - initialize(vcfFile, transform); + initialize(vcfFile, transform, true); } - private void initialize(File vcfFile, VCFCodec.LineTransform transform) { + /** + * initialize the VCF reader + * @param vcfFile the VCF file to open + * @param transform the line transformer to use, if any + * @param createIndexOnDisk do we need to create an index on disk? + */ + private void initialize(File vcfFile, VCFCodec.LineTransform transform, boolean createIndexOnDisk) { VCFCodec codec = new VCFCodec(); - LinearIndex index = null; - if (TribbleRMDTrackBuilder.requireIndex(vcfFile)) { - try { - index = TribbleRMDTrackBuilder.createIndex(vcfFile, new VCFCodec()); - } catch (IOException e) { - throw new StingException("Unable to make required index for file " + vcfFile + " do you have write permissions to the directory?"); - } - } + LinearIndex index = createIndex(vcfFile, createIndexOnDisk); if (transform != null) codec.setTransformer(transform); try { - vcfReader = new FeatureReader(vcfFile,index,codec); + vcfReader = new BasicFeatureReader(vcfFile,index,codec); iterator= vcfReader.iterator(); } catch (FileNotFoundException e) { throw new StingException("Unable to read VCF File from " + vcfFile, e); @@ -79,6 +88,24 @@ public class VCFReader implements Iterator, Iterable { mHeader = codec.getHeader(); } + /** + * create an index given: + * @param vcfFile the vcf file + * @param createIndexOnDisk do we create the index on disk (or only in memory?) + * @return an instance of an index + */ + private LinearIndex createIndex(File vcfFile, boolean createIndexOnDisk) { + LinearIndex index = null; + if (TribbleRMDTrackBuilder.requireIndex(vcfFile)) { + try { + index = TribbleRMDTrackBuilder.createIndex(vcfFile, new VCFCodec(), createIndexOnDisk); + } catch (IOException e) { + throw new StingException("Unable to make required index for file " + vcfFile + " do you have write permissions to the directory?"); + } + } + return index; + } + /** @return true if we have another VCF record to return */ public boolean hasNext() { @@ -110,6 +137,9 @@ public class VCFReader implements Iterator, Iterable { return this; } + /** + * close the files + */ public void close() { if (vcfReader != null) try { vcfReader.close(); diff --git a/java/test/org/broadinstitute/sting/gatk/datasources/providers/ReadBasedReferenceOrderedViewUnitTest.java b/java/test/org/broadinstitute/sting/gatk/datasources/providers/ReadBasedReferenceOrderedViewUnitTest.java index 22742b469..5dc52fcba 100644 --- a/java/test/org/broadinstitute/sting/gatk/datasources/providers/ReadBasedReferenceOrderedViewUnitTest.java +++ b/java/test/org/broadinstitute/sting/gatk/datasources/providers/ReadBasedReferenceOrderedViewUnitTest.java @@ -142,6 +142,11 @@ class FakePeekingRODIterator implements LocationAwareSeekableRODIterator { public void remove() { throw new IllegalStateException("GRRR"); } + + @Override + public void close() { + // nothing to do + } } class FakeRODRecordList extends AbstractList implements RODRecordList { diff --git a/java/test/org/broadinstitute/sting/gatk/iterators/LocusIteratorByStateUnitTest.java b/java/test/org/broadinstitute/sting/gatk/iterators/LocusIteratorByStateUnitTest.java index 1f8d536ff..804d26f09 100644 --- a/java/test/org/broadinstitute/sting/gatk/iterators/LocusIteratorByStateUnitTest.java +++ b/java/test/org/broadinstitute/sting/gatk/iterators/LocusIteratorByStateUnitTest.java @@ -5,6 +5,7 @@ import net.sf.picard.filter.SamRecordFilter; import net.sf.samtools.SAMFileHeader; import net.sf.samtools.SAMFileReader; import net.sf.samtools.SAMRecord; +import net.sf.samtools.util.CloseableIterator; import org.broadinstitute.sting.BaseTest; import org.broadinstitute.sting.gatk.Reads; import org.broadinstitute.sting.gatk.arguments.ValidationExclusion; @@ -17,6 +18,7 @@ import org.junit.Test; import java.io.File; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; /** @@ -47,7 +49,7 @@ public class LocusIteratorByStateUnitTest extends BaseTest { reads.setMaxPileupSize(MAX_READS); // create the iterator by state with the fake reads and fake records - li = new LocusIteratorByState(records.iterator(), reads); + li = new LocusIteratorByState(new FakeCloseableIterator(records.iterator()), reads); // inject the testing version of the locus iterator watcher li.setLocusOverflowTracker(new LocusIteratorOverride(MAX_READS)); @@ -73,7 +75,7 @@ public class LocusIteratorByStateUnitTest extends BaseTest { reads.setMaxPileupSize(MAX_READS); // create the iterator by state with the fake reads and fake records - li = new LocusIteratorByState(records.iterator(), reads); + li = new LocusIteratorByState(new FakeCloseableIterator(records.iterator()), reads); // inject the testing version of the locus iterator watcher li.setLocusOverflowTracker(new LocusIteratorOverride(MAX_READS)); @@ -102,4 +104,32 @@ class TestReads extends Reads { public void setMaxPileupSize(int maxSize) { this.maximumReadsAtLocus = maxSize; } +} + +class FakeCloseableIterator implements CloseableIterator { + Iterator iterator; + + public FakeCloseableIterator(Iterator it) { + iterator = it; + } + + @Override + public void close() { + return; + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public T next() { + return iterator.next(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException("Don't remove!"); + } } \ No newline at end of file diff --git a/java/test/org/broadinstitute/sting/gatk/refdata/tracks/builders/TribbleRMDTrackBuilderUnitTest.java b/java/test/org/broadinstitute/sting/gatk/refdata/tracks/builders/TribbleRMDTrackBuilderUnitTest.java index b5a5d5ac3..b02026880 100644 --- a/java/test/org/broadinstitute/sting/gatk/refdata/tracks/builders/TribbleRMDTrackBuilderUnitTest.java +++ b/java/test/org/broadinstitute/sting/gatk/refdata/tracks/builders/TribbleRMDTrackBuilderUnitTest.java @@ -61,7 +61,7 @@ public class TribbleRMDTrackBuilderUnitTest extends BaseTest { public void testBuilderIndexUnwriteable() { File vcfFile = new File(validationDataLocation + "/ROD_validation/mixedup.vcf"); try { - builder.createIndex(vcfFile,new VCFCodec()); + builder.createIndex(vcfFile,new VCFCodec(), true); } catch (IOException e) { Assert.fail("Unable to make index because of IO exception " + e.getMessage()); } diff --git a/java/test/org/broadinstitute/sting/gatk/refdata/utils/FlashBackIteratorUnitTest.java b/java/test/org/broadinstitute/sting/gatk/refdata/utils/FlashBackIteratorUnitTest.java index 6f421e6d4..14c57ab7f 100644 --- a/java/test/org/broadinstitute/sting/gatk/refdata/utils/FlashBackIteratorUnitTest.java +++ b/java/test/org/broadinstitute/sting/gatk/refdata/utils/FlashBackIteratorUnitTest.java @@ -186,6 +186,11 @@ class FakeSeekableRODIterator implements LocationAwareSeekableRODIterator { public void remove() { throw new IllegalStateException("GRRR"); } + + @Override + public void close() { + // nothing to do + } } diff --git a/settings/repository/org.broad/tribble-79.xml b/settings/repository/org.broad/tribble-79.xml deleted file mode 100644 index 9d57d65d0..000000000 --- a/settings/repository/org.broad/tribble-79.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/settings/repository/org.broad/tribble-79.jar b/settings/repository/org.broad/tribble-80.jar similarity index 81% rename from settings/repository/org.broad/tribble-79.jar rename to settings/repository/org.broad/tribble-80.jar index 2d2da49da..dba37922d 100644 Binary files a/settings/repository/org.broad/tribble-79.jar and b/settings/repository/org.broad/tribble-80.jar differ diff --git a/settings/repository/org.broad/tribble-80.xml b/settings/repository/org.broad/tribble-80.xml new file mode 100644 index 000000000..e5bc6a855 --- /dev/null +++ b/settings/repository/org.broad/tribble-80.xml @@ -0,0 +1,3 @@ + + +