diff --git a/public/java/src/org/broadinstitute/sting/gatk/downsampling/ReservoirDownsampler.java b/public/java/src/org/broadinstitute/sting/gatk/downsampling/ReservoirDownsampler.java index 0d7a0dd14..4331fd723 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/downsampling/ReservoirDownsampler.java +++ b/public/java/src/org/broadinstitute/sting/gatk/downsampling/ReservoirDownsampler.java @@ -29,9 +29,7 @@ import net.sf.samtools.SAMRecord; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; +import java.util.*; /** * Reservoir Downsampler: Selects n reads out of a stream whose size is not known in advance, with @@ -42,10 +40,25 @@ import java.util.List; * @author David Roazen */ public class ReservoirDownsampler implements ReadsDownsampler { + private final int targetSampleSize; - private ArrayList reservoir; + /** + * if true, this downsampler will be optimized for the case + * where most of the time we won't fill up anything like the + * targetSampleSize elements. If this is false, we will allocate + * internal buffers to targetSampleSize initially, which minimizes + * the cost of allocation if we often use targetSampleSize or more + * elements. + */ + private final boolean expectFewOverflows; - private int targetSampleSize; + /** + * At times this can be a linked list or an array list, depending on how we're accessing the + * data and whether or not we're expecting few overflows + */ + private List reservoir; + + private boolean isLinkedList; private int totalReadsSeen; @@ -56,17 +69,35 @@ public class ReservoirDownsampler implements ReadsDownsampl * * @param targetSampleSize Size of the reservoir used by this downsampler. Number of items retained * after downsampling will be min(totalReads, targetSampleSize) + * @param expectFewOverflows if true, this downsampler will be optimized for the case + * where most of the time we won't fill up anything like the + * targetSampleSize elements. If this is false, we will allocate + * internal buffers to targetSampleSize initially, which minimizes + * the cost of allocation if we often use targetSampleSize or more + * elements. */ - public ReservoirDownsampler ( int targetSampleSize ) { + public ReservoirDownsampler ( final int targetSampleSize, final boolean expectFewOverflows) { if ( targetSampleSize <= 0 ) { throw new ReviewedStingException("Cannot do reservoir downsampling with a sample size <= 0"); } this.targetSampleSize = targetSampleSize; + this.expectFewOverflows = expectFewOverflows; clear(); reset(); } + /** + * Construct a ReservoirDownsampler + * + * @param targetSampleSize Size of the reservoir used by this downsampler. Number of items retained + * after downsampling will be min(totalReads, targetSampleSize) + */ + public ReservoirDownsampler ( int targetSampleSize ) { + this(targetSampleSize, false); + } + + public void submit ( T newRead ) { totalReadsSeen++; @@ -74,7 +105,12 @@ public class ReservoirDownsampler implements ReadsDownsampl reservoir.add(newRead); } else { - int randomSlot = GenomeAnalysisEngine.getRandomGenerator().nextInt(totalReadsSeen); + if ( isLinkedList ) { + reservoir = new ArrayList(reservoir); + isLinkedList = false; + } + + final int randomSlot = GenomeAnalysisEngine.getRandomGenerator().nextInt(totalReadsSeen); if ( randomSlot < targetSampleSize ) { reservoir.set(randomSlot, newRead); } @@ -93,10 +129,15 @@ public class ReservoirDownsampler implements ReadsDownsampl } public List consumeFinalizedItems() { - // pass by reference rather than make a copy, for speed - List downsampledItems = reservoir; - clear(); - return downsampledItems; + if ( reservoir.isEmpty() ) { + // if there's nothing here, don't both allocating a new list completely + return Collections.emptyList(); + } else { + // pass by reference rather than make a copy, for speed + List downsampledItems = reservoir; + clear(); + return downsampledItems; + } } public boolean hasPendingItems() { @@ -119,9 +160,18 @@ public class ReservoirDownsampler implements ReadsDownsampl // NO-OP } + /** + * Clear the data structures used to hold information + */ public void clear() { - reservoir = new ArrayList(targetSampleSize); - totalReadsSeen = 0; // an internal stat used by the downsampling process, so not cleared by reset() below + // if we aren't expecting many overflows, allocate a linked list not an arraylist + reservoir = expectFewOverflows ? new LinkedList() : new ArrayList(targetSampleSize); + + // it's a linked list if we allocate one + isLinkedList = expectFewOverflows; + + // an internal stat used by the downsampling process, so not cleared by reset() below + totalReadsSeen = 0; } public void reset() { diff --git a/public/java/src/org/broadinstitute/sting/utils/locusiterator/SamplePartitioner.java b/public/java/src/org/broadinstitute/sting/utils/locusiterator/SamplePartitioner.java index 7dada292b..9bb474e4d 100644 --- a/public/java/src/org/broadinstitute/sting/utils/locusiterator/SamplePartitioner.java +++ b/public/java/src/org/broadinstitute/sting/utils/locusiterator/SamplePartitioner.java @@ -62,16 +62,17 @@ class SamplePartitioner { * will throw an exception. Duplicates in the list of samples will be ignored * * @param LIBSDownsamplingInfo do we want to downsample, and if so to what coverage? - * @param samples the complete list of samples we're going to partition reads into + * @param samples the complete list of samples we're going to partition reads into. Can be + * empty, but in that case this code cannot function properly if you + * attempt to add data to it. */ @Ensures({ "readsBySample != null", - "! readsBySample.isEmpty()", "readsBySample.size() == new HashSet(samples).size()" }) public SamplePartitioner(final LIBSDownsamplingInfo LIBSDownsamplingInfo, final List samples) { if ( LIBSDownsamplingInfo == null ) throw new IllegalArgumentException("LIBSDownsamplingInfo cannot be null"); - if ( samples == null || samples.isEmpty() ) throw new IllegalArgumentException("samples must be a non-null, non-empty list but got " + samples); + if ( samples == null ) throw new IllegalArgumentException("samples must be a non-null list"); readsBySample = new LinkedHashMap>(samples.size()); for ( final String sample : samples ) { @@ -89,7 +90,7 @@ class SamplePartitioner { @Ensures("result != null") private Downsampler createDownsampler(final LIBSDownsamplingInfo LIBSDownsamplingInfo) { return LIBSDownsamplingInfo.isPerformDownsampling() - ? new ReservoirDownsampler(LIBSDownsamplingInfo.getToCoverage()) + ? new ReservoirDownsampler(LIBSDownsamplingInfo.getToCoverage(), true) : new PassThroughDownsampler(); }