diff --git a/build.xml b/build.xml index 9a66d4699..d3e25d424 100644 --- a/build.xml +++ b/build.xml @@ -955,8 +955,8 @@ - - + + diff --git a/public/java/src/org/broadinstitute/sting/gatk/io/storage/VCFWriterStorage.java b/public/java/src/org/broadinstitute/sting/gatk/io/storage/VCFWriterStorage.java index 4ca7b935f..25d339ee6 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/io/storage/VCFWriterStorage.java +++ b/public/java/src/org/broadinstitute/sting/gatk/io/storage/VCFWriterStorage.java @@ -2,7 +2,7 @@ package org.broadinstitute.sting.gatk.io.storage; import net.sf.samtools.util.BlockCompressedOutputStream; import org.apache.log4j.Logger; -import org.broad.tribble.source.BasicFeatureSource; +import org.broad.tribble.AbstractFeatureReader; import org.broadinstitute.sting.gatk.io.stubs.VCFWriterStub; import org.broadinstitute.sting.utils.codecs.vcf.StandardVCFWriter; import org.broadinstitute.sting.utils.codecs.vcf.VCFCodec; @@ -114,7 +114,7 @@ public class VCFWriterStorage implements Storage, VCFWriter { String sourceFilePath = file.getAbsolutePath(); String targetFilePath = target.file != null ? target.file.getAbsolutePath() : "/dev/stdin"; logger.debug(String.format("Merging %s into %s",sourceFilePath,targetFilePath)); - BasicFeatureSource source = BasicFeatureSource.getFeatureSource(file.getAbsolutePath(), new VCFCodec(), false); + AbstractFeatureReader source = AbstractFeatureReader.getFeatureReader(file.getAbsolutePath(), new VCFCodec(), false); for ( VariantContext vc : source.iterator() ) { target.writer.add(vc); diff --git a/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrack.java b/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrack.java index 7aa112961..96bc874e0 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrack.java +++ b/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrack.java @@ -26,11 +26,10 @@ package org.broadinstitute.sting.gatk.refdata.tracks; import net.sf.samtools.SAMSequenceDictionary; import net.sf.samtools.util.CloseableIterator; import org.apache.log4j.Logger; +import org.broad.tribble.AbstractFeatureReader; +import org.broad.tribble.CloseableTribbleIterator; import org.broad.tribble.Feature; import org.broad.tribble.FeatureCodec; -import org.broad.tribble.FeatureSource; -import org.broad.tribble.iterators.CloseableTribbleIterator; -import org.broad.tribble.source.PerformanceLoggingFeatureSource; import org.broadinstitute.sting.gatk.refdata.utils.FeatureToGATKFeatureIterator; import org.broadinstitute.sting.gatk.refdata.utils.GATKFeature; import org.broadinstitute.sting.utils.GenomeLoc; @@ -57,7 +56,7 @@ public class RMDTrack { private final File file; // the associated file we create the reader from // our feature reader - allows queries - private FeatureSource reader; + private AbstractFeatureReader reader; // our sequence dictionary, which can be null private final SAMSequenceDictionary dictionary; @@ -92,7 +91,7 @@ public class RMDTrack { * @param dict the sam sequence dictionary * @param codec the feature codec we use to decode this type */ - public RMDTrack(Class type, String name, File file, FeatureSource reader, SAMSequenceDictionary dict, GenomeLocParser genomeLocParser, FeatureCodec codec) { + public RMDTrack(Class type, String name, File file, AbstractFeatureReader reader, SAMSequenceDictionary dict, GenomeLocParser genomeLocParser, FeatureCodec codec) { this.type = type; this.name = name; this.file = file; @@ -116,8 +115,6 @@ public class RMDTrack { public CloseableIterator query(GenomeLoc interval) throws IOException { CloseableTribbleIterator iter = reader.query(interval.getContig(),interval.getStart(),interval.getStop()); - if ( RMDTrackBuilder.MEASURE_TRIBBLE_QUERY_PERFORMANCE ) - logger.warn("Query " + getName() + ":" + ((PerformanceLoggingFeatureSource)reader).getPerformanceLog()); return new FeatureToGATKFeatureIterator(genomeLocParser, iter, this.getName()); } @@ -130,10 +127,6 @@ public class RMDTrack { reader = null; } - public FeatureSource getReader() { - return reader; - } - /** * get the sequence dictionary from the track, if available * @return a SAMSequenceDictionary if available, null if unavailable diff --git a/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrackBuilder.java b/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrackBuilder.java index edb514984..2a83b9d34 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrackBuilder.java +++ b/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrackBuilder.java @@ -26,14 +26,12 @@ package org.broadinstitute.sting.gatk.refdata.tracks; import net.sf.samtools.SAMSequenceDictionary; import org.apache.log4j.Logger; +import org.broad.tribble.AbstractFeatureReader; import org.broad.tribble.FeatureCodec; -import org.broad.tribble.FeatureSource; import org.broad.tribble.Tribble; import org.broad.tribble.TribbleException; import org.broad.tribble.index.Index; import org.broad.tribble.index.IndexFactory; -import org.broad.tribble.source.BasicFeatureSource; -import org.broad.tribble.source.PerformanceLoggingFeatureSource; import org.broad.tribble.util.LittleEndianOutputStream; import org.broadinstitute.sting.commandline.Tags; import org.broadinstitute.sting.gatk.arguments.ValidationExclusion; @@ -121,7 +119,7 @@ public class RMDTrackBuilder { // extends PluginManager { throw new UserException.BadArgumentValue("-B",fileDescriptor.getType()); // return a feature reader track - Pair pair; + Pair pair; if (inputFile.getAbsolutePath().endsWith(".gz")) pair = createTabixIndexedFeatureSource(descriptor, name, inputFile); else @@ -155,11 +153,11 @@ public class RMDTrackBuilder { // extends PluginManager { * @param inputFile the file to load * @return a feature reader implementation */ - private Pair createTabixIndexedFeatureSource(FeatureManager.FeatureDescriptor descriptor, String name, File inputFile) { + private Pair createTabixIndexedFeatureSource(FeatureManager.FeatureDescriptor descriptor, String name, File inputFile) { // we might not know the index type, try loading with the default reader constructor logger.info("Attempting to blindly load " + inputFile + " as a tabix indexed file"); try { - return new Pair(BasicFeatureSource.getFeatureSource(inputFile.getAbsolutePath(), createCodec(descriptor, name)),null); + return new Pair(AbstractFeatureReader.getFeatureReader(inputFile.getAbsolutePath(), createCodec(descriptor, name)),null); } catch (TribbleException e) { throw new UserException(e.getMessage(), e); } @@ -183,12 +181,12 @@ public class RMDTrackBuilder { // extends PluginManager { * @param storageType How the RMD is streamed into the input file. * @return the input file as a FeatureReader */ - private Pair getFeatureSource(FeatureManager.FeatureDescriptor descriptor, + private Pair getFeatureSource(FeatureManager.FeatureDescriptor descriptor, String name, File inputFile, RMDStorageType storageType) { // Feature source and sequence dictionary to use as the ultimate reference - FeatureSource featureSource = null; + AbstractFeatureReader featureSource = null; SAMSequenceDictionary sequenceDictionary = null; // Detect whether or not this source should be indexed. @@ -215,10 +213,7 @@ public class RMDTrackBuilder { // extends PluginManager { sequenceDictionary = IndexDictionaryUtils.getSequenceDictionaryFromProperties(index); } - if ( MEASURE_TRIBBLE_QUERY_PERFORMANCE ) - featureSource = new PerformanceLoggingFeatureSource(inputFile.getAbsolutePath(), index, createCodec(descriptor, name)); - else - featureSource = new BasicFeatureSource(inputFile.getAbsolutePath(), index, createCodec(descriptor, name)); + featureSource = AbstractFeatureReader.getFeatureReader(inputFile.getAbsolutePath(), createCodec(descriptor, name), index); } catch (TribbleException e) { throw new UserException(e.getMessage()); @@ -228,10 +223,10 @@ public class RMDTrackBuilder { // extends PluginManager { } } else { - featureSource = BasicFeatureSource.getFeatureSource(inputFile.getAbsolutePath(),createCodec(descriptor, name),false); + featureSource = AbstractFeatureReader.getFeatureReader(inputFile.getAbsolutePath(), createCodec(descriptor, name), false); } - return new Pair(featureSource,sequenceDictionary); + return new Pair(featureSource,sequenceDictionary); } /** @@ -358,7 +353,7 @@ public class RMDTrackBuilder { // extends PluginManager { private Index createIndexInMemory(File inputFile, FeatureCodec codec) { // this can take a while, let them know what we're doing logger.info("Creating Tribble index in memory for file " + inputFile); - Index idx = IndexFactory.createIndex(inputFile, codec, IndexFactory.IndexBalanceApproach.FOR_SEEK_TIME); + Index idx = IndexFactory.createDynamicIndex(inputFile, codec, IndexFactory.IndexBalanceApproach.FOR_SEEK_TIME); validateAndUpdateIndexSequenceDictionary(inputFile, idx, dict); return idx; } diff --git a/public/java/src/org/broadinstitute/sting/gatk/refdata/utils/FeatureToGATKFeatureIterator.java b/public/java/src/org/broadinstitute/sting/gatk/refdata/utils/FeatureToGATKFeatureIterator.java index 104ba87b5..cfc1c36c6 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/refdata/utils/FeatureToGATKFeatureIterator.java +++ b/public/java/src/org/broadinstitute/sting/gatk/refdata/utils/FeatureToGATKFeatureIterator.java @@ -24,8 +24,8 @@ package org.broadinstitute.sting.gatk.refdata.utils; import net.sf.samtools.util.CloseableIterator; +import org.broad.tribble.CloseableTribbleIterator; import org.broad.tribble.Feature; -import org.broad.tribble.iterators.CloseableTribbleIterator; import org.broadinstitute.sting.utils.GenomeLocParser; diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/IndexingVCFWriter.java b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/IndexingVCFWriter.java index 71ec4ce1b..0cca77e4a 100644 --- a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/IndexingVCFWriter.java +++ b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/IndexingVCFWriter.java @@ -28,12 +28,10 @@ import com.google.java.contract.Ensures; import com.google.java.contract.Requires; import net.sf.samtools.SAMSequenceDictionary; import org.broad.tribble.Tribble; -import org.broad.tribble.TribbleException; import org.broad.tribble.index.DynamicIndexCreator; import org.broad.tribble.index.Index; import org.broad.tribble.index.IndexFactory; import org.broad.tribble.util.LittleEndianOutputStream; -import org.broad.tribble.util.PositionalStream; import org.broadinstitute.sting.gatk.refdata.tracks.IndexDictionaryUtils; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; @@ -142,3 +140,31 @@ public abstract class IndexingVCFWriter implements VCFWriter { } } } + +class PositionalStream extends OutputStream { + OutputStream out = null; + private long position = 0; + + public PositionalStream(OutputStream out) { + this.out = out; + } + + public void write(final byte[] bytes) throws IOException { + write(bytes, 0, bytes.length); + } + + public void write(final byte[] bytes, int startIndex, int numBytes) throws IOException { + //System.out.println("write: " + bytes + " " + numBytes); + position += numBytes; + out.write(bytes, startIndex, numBytes); + } + + public void write(int c) throws IOException { + System.out.println("write byte: " + c); + //System.out.printf("Position %d for %c\n", position, (char)c); + position++; + out.write(c); + } + + public long getPosition() { return position; } +} \ No newline at end of file diff --git a/public/java/test/org/broadinstitute/sting/WalkerTest.java b/public/java/test/org/broadinstitute/sting/WalkerTest.java index f477fedc9..1c5f8431c 100755 --- a/public/java/test/org/broadinstitute/sting/WalkerTest.java +++ b/public/java/test/org/broadinstitute/sting/WalkerTest.java @@ -76,7 +76,7 @@ public class WalkerTest extends BaseTest { public static void assertOnDiskIndexEqualToNewlyCreatedIndex(final File indexFile, final String name, final File resultFile) { System.out.println("Verifying on-the-fly index " + indexFile + " for test " + name + " using file " + resultFile); - Index indexFromOutputFile = IndexFactory.createIndex(resultFile, new VCFCodec()); + Index indexFromOutputFile = IndexFactory.createDynamicIndex(resultFile, new VCFCodec()); Index dynamicIndex = IndexFactory.loadIndex(indexFile.getAbsolutePath()); if ( ! indexFromOutputFile.equalsIgnoreProperties(dynamicIndex) ) { diff --git a/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/IndexFactoryUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/IndexFactoryUnitTest.java index 55bd4783b..8da11c3d7 100755 --- a/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/IndexFactoryUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/IndexFactoryUnitTest.java @@ -1,10 +1,10 @@ package org.broadinstitute.sting.utils.codecs.vcf; import net.sf.samtools.SAMSequenceDictionary; +import org.broad.tribble.AbstractFeatureReader; +import org.broad.tribble.CloseableTribbleIterator; import org.broad.tribble.Tribble; import org.broad.tribble.index.*; -import org.broad.tribble.iterators.CloseableTribbleIterator; -import org.broad.tribble.source.BasicFeatureSource; import org.broadinstitute.sting.BaseTest; import org.broadinstitute.sting.WalkerTest; import org.broadinstitute.sting.utils.exceptions.UserException; @@ -45,14 +45,14 @@ public class IndexFactoryUnitTest extends BaseTest { // @Test public void testOnTheFlyIndexing1() throws IOException { - Index indexFromInputFile = IndexFactory.createIndex(inputFile, new VCFCodec()); + Index indexFromInputFile = IndexFactory.createDynamicIndex(inputFile, new VCFCodec()); if ( outputFileIndex.exists() ) { System.err.println("Deleting " + outputFileIndex); outputFileIndex.delete(); } for ( int maxRecords : Arrays.asList(0, 1, 10, 100, 1000, -1)) { - BasicFeatureSource source = new BasicFeatureSource(inputFile.getAbsolutePath(), indexFromInputFile, new VCFCodec()); + AbstractFeatureReader source = AbstractFeatureReader.getFeatureReader(inputFile.getAbsolutePath(), new VCFCodec(), indexFromInputFile); int counter = 0; VCFWriter writer = new StandardVCFWriter(outputFile, dict); @@ -66,7 +66,7 @@ public class IndexFactoryUnitTest extends BaseTest { // test that the input index is the same as the one created from the identical input file // test that the dynamic index is the same as the output index, which is equal to the input index - WalkerTest.assertOnDiskIndexEqualToNewlyCreatedIndex(outputFileIndex, "unittest", outputFile); + //WalkerTest.assertOnDiskIndexEqualToNewlyCreatedIndex(outputFileIndex, "unittest", outputFile); } } } diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/VCF_BAM_utilities.scala b/public/scala/src/org/broadinstitute/sting/queue/util/VCF_BAM_utilities.scala index f2592e05c..bb2d4f864 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/util/VCF_BAM_utilities.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/util/VCF_BAM_utilities.scala @@ -4,14 +4,14 @@ import java.io.File import org.apache.commons.io.FilenameUtils import scala.io.Source._ import net.sf.samtools.SAMFileReader -import org.broad.tribble.source.BasicFeatureSource import org.broadinstitute.sting.utils.codecs.vcf.{VCFHeader, VCFCodec} import scala.collection.JavaConversions._ +import org.broad.tribble.AbstractFeatureReader object VCF_BAM_utilities { def getSamplesFromVCF(vcfFile: File): List[String] = { - return BasicFeatureSource.getFeatureSource(vcfFile.getPath(), new VCFCodec()).getHeader().asInstanceOf[VCFHeader].getGenotypeSamples().toList + return AbstractFeatureReader.getFeatureReader(vcfFile.getPath(), new VCFCodec()).getHeader().asInstanceOf[VCFHeader].getGenotypeSamples().toList } def getSamplesInBAM(bam: File): List[String] = { diff --git a/settings/repository/org.broad/tribble-53.jar b/settings/repository/org.broad/tribble-94.jar similarity index 58% rename from settings/repository/org.broad/tribble-53.jar rename to settings/repository/org.broad/tribble-94.jar index 02865df43..5f534f08b 100644 Binary files a/settings/repository/org.broad/tribble-53.jar and b/settings/repository/org.broad/tribble-94.jar differ diff --git a/settings/repository/org.broad/tribble-53.xml b/settings/repository/org.broad/tribble-94.xml similarity index 51% rename from settings/repository/org.broad/tribble-53.xml rename to settings/repository/org.broad/tribble-94.xml index cae6cf15a..cd5fcd99f 100644 --- a/settings/repository/org.broad/tribble-53.xml +++ b/settings/repository/org.broad/tribble-94.xml @@ -1,3 +1,3 @@ - +