From 46c14ec63fde1d2d35e61eb801799d470ad44790 Mon Sep 17 00:00:00 2001 From: hanna Date: Thu, 25 Mar 2010 17:41:22 +0000 Subject: [PATCH] New, much less memory intensive implementation of BAM file sharding. Streams indices together with the expectation that bins will be present in the bin sparse array, which avoids the problem of having to hold the sparse bin array stored in every BAM file index in memory at the same time. git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@3075 348d0f76-0448-11de-a6fe-93d51630548a --- .../BlockDelimitedReadShardStrategy.java | 18 +- .../IndexDelimitedLocusShardStrategy.java | 10 +- .../datasources/shards/IntervalSharder.java | 519 ++++++++++-------- ... picard-private-parts-1333-sharding-4.jar} | Bin 288615 -> 288615 bytes ... picard-private-parts-1333-sharding-4.xml} | 2 +- ...rding.jar => picard-1.16.364-sharding.jar} | Bin 407397 -> 407397 bytes ...rding.xml => picard-1.16.364-sharding.xml} | 2 +- ...sharding.jar => sam-1.16.364-sharding.jar} | Bin 453399 -> 456344 bytes ...sharding.xml => sam-1.16.364-sharding.xml} | 2 +- 9 files changed, 319 insertions(+), 234 deletions(-) rename settings/repository/edu.mit.broad/{picard-private-parts-1333-sharding-3.jar => picard-private-parts-1333-sharding-4.jar} (94%) rename settings/repository/edu.mit.broad/{picard-private-parts-1333-sharding-3.xml => picard-private-parts-1333-sharding-4.xml} (55%) rename settings/repository/net.sf/{picard-1.16.363-sharding.jar => picard-1.16.364-sharding.jar} (95%) rename settings/repository/net.sf/{picard-1.16.363-sharding.xml => picard-1.16.364-sharding.xml} (76%) rename settings/repository/net.sf/{sam-1.16.363-sharding.jar => sam-1.16.364-sharding.jar} (94%) rename settings/repository/net.sf/{sam-1.16.363-sharding.xml => sam-1.16.364-sharding.xml} (52%) diff --git a/java/src/org/broadinstitute/sting/gatk/datasources/shards/BlockDelimitedReadShardStrategy.java b/java/src/org/broadinstitute/sting/gatk/datasources/shards/BlockDelimitedReadShardStrategy.java index 032c93b54..12feb4c74 100644 --- a/java/src/org/broadinstitute/sting/gatk/datasources/shards/BlockDelimitedReadShardStrategy.java +++ b/java/src/org/broadinstitute/sting/gatk/datasources/shards/BlockDelimitedReadShardStrategy.java @@ -27,7 +27,12 @@ public class BlockDelimitedReadShardStrategy extends ReadShardStrategy { /** * The data source used to shard. */ - protected final BlockDrivenSAMDataSource dataSource; + private final BlockDrivenSAMDataSource dataSource; + + /** + * The intervals to be processed. + */ + private final GenomeLocSortedSet locations; /** * The cached shard to be returned next. Prefetched in the peekable iterator style. @@ -63,10 +68,13 @@ public class BlockDelimitedReadShardStrategy extends ReadShardStrategy { this.dataSource = (BlockDrivenSAMDataSource)dataSource; this.position = this.dataSource.getCurrentPosition(); - if(locations != null) - filePointers.addAll(IntervalSharder.shardIntervals(this.dataSource,locations.toList())); + this.locations = locations; + + if(locations != null) + filePointerIterator = IntervalSharder.shardIntervals(this.dataSource,locations.toList()); + else + filePointerIterator = filePointers.iterator(); - filePointerIterator = filePointers.iterator(); if(filePointerIterator.hasNext()) currentFilePointer = filePointerIterator.next(); @@ -99,7 +107,7 @@ public class BlockDelimitedReadShardStrategy extends ReadShardStrategy { nextShard = null; SamRecordFilter filter = null; - if(!filePointers.isEmpty()) { + if(locations != null) { Map> selectedReaders = new HashMap>(); while(selectedReaders.size() == 0 && currentFilePointer != null) { shardPosition = currentFilePointer.chunks; diff --git a/java/src/org/broadinstitute/sting/gatk/datasources/shards/IndexDelimitedLocusShardStrategy.java b/java/src/org/broadinstitute/sting/gatk/datasources/shards/IndexDelimitedLocusShardStrategy.java index d428064d3..20c44b994 100755 --- a/java/src/org/broadinstitute/sting/gatk/datasources/shards/IndexDelimitedLocusShardStrategy.java +++ b/java/src/org/broadinstitute/sting/gatk/datasources/shards/IndexDelimitedLocusShardStrategy.java @@ -12,7 +12,6 @@ import org.broadinstitute.sting.gatk.datasources.simpleDataSources.SAMReaderID; import java.util.*; import net.sf.samtools.Chunk; -import net.sf.samtools.SAMFileReader; import net.sf.samtools.SAMFileHeader; import net.sf.samtools.SAMSequenceRecord; @@ -51,9 +50,6 @@ public class IndexDelimitedLocusShardStrategy implements ShardStrategy { */ private final BlockDrivenSAMDataSource reads; - /** our storage of the genomic locations they'd like to shard over */ - private final List filePointers = new ArrayList(); - /** * An iterator through the available file pointers. */ @@ -90,13 +86,13 @@ public class IndexDelimitedLocusShardStrategy implements ShardStrategy { else intervals = locations.toList(); - this.reads = (BlockDrivenSAMDataSource)reads; - filePointers.addAll(IntervalSharder.shardIntervals(this.reads,intervals)); + this.filePointerIterator = IntervalSharder.shardIntervals(this.reads,intervals); } else { final int maxShardSize = 100000; this.reads = null; + List filePointers = new ArrayList(); if(locations == null) { for(SAMSequenceRecord refSequenceRecord: reference.getSequenceDictionary().getSequences()) { for(int shardStart = 1; shardStart <= refSequenceRecord.getSequenceLength(); shardStart += maxShardSize) { @@ -109,9 +105,9 @@ public class IndexDelimitedLocusShardStrategy implements ShardStrategy { for(GenomeLoc interval: locations) filePointers.add(new FilePointer(interval)); } + filePointerIterator = filePointers.iterator(); } - filePointerIterator = filePointers.iterator(); } /** diff --git a/java/src/org/broadinstitute/sting/gatk/datasources/shards/IntervalSharder.java b/java/src/org/broadinstitute/sting/gatk/datasources/shards/IntervalSharder.java index 91520f842..311c8a94f 100644 --- a/java/src/org/broadinstitute/sting/gatk/datasources/shards/IntervalSharder.java +++ b/java/src/org/broadinstitute/sting/gatk/datasources/shards/IntervalSharder.java @@ -1,9 +1,6 @@ package org.broadinstitute.sting.gatk.datasources.shards; -import org.broadinstitute.sting.utils.GenomeLoc; -import org.broadinstitute.sting.utils.GenomeLocParser; -import org.broadinstitute.sting.utils.StingException; -import org.broadinstitute.sting.utils.GenomeLocSortedSet; +import org.broadinstitute.sting.utils.*; import org.broadinstitute.sting.gatk.datasources.simpleDataSources.BlockDrivenSAMDataSource; import org.broadinstitute.sting.gatk.datasources.simpleDataSources.SAMReaderID; @@ -19,192 +16,113 @@ import net.sf.picard.util.PeekableIterator; * @version 0.1 */ public class IntervalSharder { - protected static List shardIntervals(final BlockDrivenSAMDataSource dataSource, final List loci) { - Map> filePointersByReader = new HashMap>(); - for(SAMReaderID id: dataSource.getReaderIDs()) { - PreloadedBAMFileIndex index = dataSource.getIndex(id); - // Gather bins for the given loci, splitting loci as necessary so that each falls into exactly one lowest-level bin.\ - filePointersByReader.put(id,shardIntervalsOverIndex(dataSource,id,index,loci,index.getNumIndexLevels()-1)); - index.close(); - } - return combineFilePointers(filePointersByReader); + public static Iterator shardIntervals(final BlockDrivenSAMDataSource dataSource, final List loci) { + return new FilePointerIterator(dataSource,loci); } /** - * Combine adjacent file pointers into a structure that can be streamed in. - * @param filePointersByReader File pointers broken down by reader. - * @return A large structure of file pointers. + * A lazy-loading iterator over file pointers. */ - private static List combineFilePointers(Map> filePointersByReader) { - PeekableIterator mergingIterator = new PeekableIterator(new FilePointerMergingIterator(filePointersByReader)); + private static class FilePointerIterator implements Iterator { + final BlockDrivenSAMDataSource dataSource; + final PeekableIterator locusIterator; + final Queue cachedFilePointers = new LinkedList(); - List overlappingFilePointers = new ArrayList(); - List mergedFilePointers = new ArrayList(); - - while(mergingIterator.hasNext()) { - GenomeLoc bounds = null; - - // Load up a segment where file pointers overlap - while(mergingIterator.hasNext() && (overlappingFilePointers.size() == 0 || mergingIterator.peek().getBounds().overlapsP(bounds))) { - FilePointer filePointer = mergingIterator.next(); - if(bounds != null) - bounds = GenomeLocParser.createGenomeLoc(bounds.getContig(), - Math.min(bounds.getStart(),filePointer.getBounds().getStart()), - Math.max(bounds.getStop(),filePointer.getBounds().getStop())); - else - bounds = filePointer.getBounds(); - overlappingFilePointers.add(filePointer); - } - - // determine the complete set of unique locations defining this set. - List overlappingLocations = new ArrayList(); - for(FilePointer filePointer: overlappingFilePointers) - overlappingLocations.addAll(filePointer.locations); - Collections.sort(overlappingLocations); - overlappingLocations = GenomeLocSortedSet.mergeOverlappingLocations(overlappingLocations); - - while(!overlappingLocations.isEmpty()) { - long overlapStart = overlappingLocations.get(0).getStart(); - long overlapStop = overlappingLocations.get(overlappingLocations.size()-1).getStop(); - - for(FilePointer overlappingFilePointer: overlappingFilePointers) { - if(overlappingFilePointer.getBounds().getStop() < overlapStart) - continue; - if(overlappingFilePointer.getBounds().getStart() > overlapStart) overlapStop = Math.min(overlapStop,overlappingFilePointer.getBounds().getStart()-1); - if(overlappingFilePointer.getBounds().getStop() < overlapStop) overlapStop = Math.min(overlapStop,overlappingFilePointer.getBounds().getStop()); - } - - // Find the overlapping genome locs. - List segmentOverlap = new ArrayList(); - for(GenomeLoc overlappingLocation: overlappingLocations) { - if(overlappingLocation.getStop() <= overlapStop) { - // segment is completely before end of overlap. - segmentOverlap.add(overlappingLocation); - } - else if(overlappingLocation.getStart() <= overlapStop) { - // segment is partially before end of overlap. - segmentOverlap.add(GenomeLocParser.setStop(overlappingLocation,overlapStop)); - break; - } - else { - // segment starts after overlap ends. - break; - } - } - - // Trim the overlapping genome locs of the overlapping locations list. - while(!overlappingLocations.isEmpty() && overlappingLocations.get(0).getStart() <= overlapStop) { - GenomeLoc location = overlappingLocations.remove(0); - if(location.getStop() > overlapStop) - overlappingLocations.add(0,GenomeLocParser.setStart(location,overlapStop+1)); - } - - // Merge together all file pointers that overlap with these bounds. - GenomeLoc overlapBounds = GenomeLocParser.createGenomeLoc(segmentOverlap.get(0).getContigIndex(),overlapStart,overlapStop); - FilePointer mergedFilePointer = null; - for(FilePointer overlappingFilePointer: overlappingFilePointers) { - if(overlappingFilePointer.getBounds().overlapsP(overlapBounds)) - mergedFilePointer = overlappingFilePointer.merge(mergedFilePointer,segmentOverlap); - } - - // Add the resulting file pointer and clear state. - mergedFilePointers.add(mergedFilePointer); - } - - // reset - overlappingFilePointers.clear(); - } - - return mergedFilePointers; - } - - private static class FilePointerMergingIterator implements Iterator { - private PriorityQueue> filePointerQueue; - - public FilePointerMergingIterator(Map> filePointers) { - filePointerQueue = new PriorityQueue>(filePointers.size(),new FilePointerMergingComparator()); - for(List filePointersByReader: filePointers.values()) - filePointerQueue.add(new PeekableIterator(filePointersByReader.iterator())); + public FilePointerIterator(final BlockDrivenSAMDataSource dataSource, final List loci) { + this.dataSource = dataSource; + locusIterator = new PeekableIterator(loci.iterator()); + advance(); } public boolean hasNext() { - return !filePointerQueue.isEmpty(); + return !cachedFilePointers.isEmpty(); } public FilePointer next() { - if(!hasNext()) throw new NoSuchElementException("FilePointerMergingIterator is out of elements"); - PeekableIterator nextIterator = filePointerQueue.remove(); - FilePointer nextFilePointer = nextIterator.next(); - if(nextIterator.hasNext()) - filePointerQueue.add(nextIterator); - return nextFilePointer; + if(!hasNext()) + throw new NoSuchElementException("FilePointerIterator iteration is complete"); + FilePointer filePointer = cachedFilePointers.remove(); + if(cachedFilePointers.isEmpty()) + advance(); + return filePointer; } - public void remove() { throw new UnsupportedOperationException("Cannot remove from a merging iterator."); } + public void remove() { + throw new UnsupportedOperationException("Cannot remove from a FilePointerIterator"); + } - private class FilePointerMergingComparator implements Comparator> { - public int compare(PeekableIterator lhs, PeekableIterator rhs) { - if(!lhs.hasNext() && !rhs.hasNext()) return 0; - if(!rhs.hasNext()) return -1; - if(!lhs.hasNext()) return 1; - return lhs.peek().getBounds().compareTo(rhs.peek().getBounds()); + private void advance() { + List nextBatch = new ArrayList(); + String contig = null; + + while(locusIterator.hasNext() && nextBatch.isEmpty()) { + contig = null; + while(locusIterator.hasNext() && (contig == null || locusIterator.peek().getContig().equals(contig))) { + GenomeLoc nextLocus = locusIterator.next(); + contig = nextLocus.getContig(); + nextBatch.add(nextLocus); + } } + + if(nextBatch.size() > 0) + cachedFilePointers.addAll(shardIntervalsOnContig(dataSource,contig,nextBatch)); } } - - private static List shardIntervalsOverIndex(final BlockDrivenSAMDataSource dataSource, final SAMReaderID id, final PreloadedBAMFileIndex index, final List loci, final int binsDeeperThan) { + + private static List shardIntervalsOnContig(final BlockDrivenSAMDataSource dataSource, final String contig, final List loci) { // Gather bins for the given loci, splitting loci as necessary so that each falls into exactly one lowest-level bin. List filePointers = new ArrayList(); FilePointer lastFilePointer = null; - Bin lastBin = null; + BAMOverlap lastBAMOverlap = null; + + Map readerToIndexMap = new HashMap(); + BinMergingIterator binMerger = new BinMergingIterator(); + for(SAMReaderID id: dataSource.getReaderIDs()) { + final SAMSequenceRecord referenceSequence = dataSource.getHeader(id).getSequence(contig); + final PreloadedBAMFileIndex index = dataSource.getIndex(id); + binMerger.addReader(id, + index, + referenceSequence.getSequenceIndex(), + index.getBinsOverlapping(referenceSequence.getSequenceIndex(),1,referenceSequence.getSequenceLength()).iterator()); + // Cache the reader for later data lookup. + readerToIndexMap.put(id,index); + } + PeekableIterator binIterator = new PeekableIterator(binMerger); for(GenomeLoc location: loci) { - // If crossing contigs, be sure to reset the filepointer that's been accumulating shard data. - if(lastFilePointer != null && lastFilePointer.referenceSequence != location.getContigIndex()) { - filePointers.add(lastFilePointer); - lastFilePointer = null; - lastBin = null; - } + if(!location.getContig().equals(contig)) + throw new StingException("Location outside bounds of contig"); int locationStart = (int)location.getStart(); final int locationStop = (int)location.getStop(); - List bins = findBinsAtLeastAsDeepAs(index,getOverlappingBins(dataSource,id,index,location),binsDeeperThan); + // Advance to first bin. + while(binIterator.peek().stop < locationStart) + binIterator.next(); - // Recursive stopping condition -- algorithm is at the zero point and no bins have been found. - if(binsDeeperThan == 0 && bins.size() == 0) { - filePointers.add(new FilePointer(location)); - continue; - } - - // No bins found; step up a level and search again. - if(bins.size() == 0) { - if(lastFilePointer != null && lastFilePointer.locations.size() > 0) { - filePointers.add(lastFilePointer); - lastFilePointer = null; - lastBin = null; - } - - filePointers.addAll(shardIntervalsOverIndex(dataSource,id,index,Collections.singletonList(location),binsDeeperThan-1)); - continue; - } + // Add all relevant bins to a list. If the given bin extends beyond the end of the current interval, make + // sure the extending bin is not pruned from the list. + List bamOverlaps = new ArrayList(); + while(binIterator.hasNext() && binIterator.peek().stop <= locationStop) + bamOverlaps.add(binIterator.next()); + if(binIterator.hasNext() && binIterator.peek().start <= locationStop) + bamOverlaps.add(binIterator.peek()); // Bins found; try to match bins with locations. - Collections.sort(bins); - Iterator binIterator = bins.iterator(); + Iterator bamOverlapIterator = bamOverlaps.iterator(); while(locationStop >= locationStart) { - int binStart = lastFilePointer!=null ? index.getFirstLocusInBin(lastBin) : 0; - int binStop = lastFilePointer!=null ? index.getLastLocusInBin(lastBin) : 0; + int binStart = lastFilePointer!=null ? lastFilePointer.overlap.start : 0; + int binStop = lastFilePointer!=null ? lastFilePointer.overlap.stop : 0; - while(binStop < locationStart && binIterator.hasNext()) { + while(binStop < locationStart && bamOverlapIterator.hasNext()) { if(lastFilePointer != null && lastFilePointer.locations.size() > 0) filePointers.add(lastFilePointer); - lastBin = binIterator.next(); - lastFilePointer = new FilePointer(id,lastBin.referenceSequence,getFilePointersBounding(index,lastBin)); - binStart = index.getFirstLocusInBin(lastBin); - binStop = index.getLastLocusInBin(lastBin); + lastBAMOverlap = bamOverlapIterator.next(); + lastFilePointer = new FilePointer(contig,lastBAMOverlap); + binStart = lastFilePointer.overlap.start; + binStop = lastFilePointer.overlap.stop; } if(locationStart < binStart) { @@ -212,13 +130,13 @@ public class IntervalSharder { if(lastFilePointer != null && lastFilePointer.locations.size() > 0) { filePointers.add(lastFilePointer); lastFilePointer = null; - lastBin = null; + lastBAMOverlap = null; } final int regionStop = Math.min(locationStop,binStart-1); GenomeLoc subset = GenomeLocParser.createGenomeLoc(location.getContig(),locationStart,regionStop); - filePointers.addAll(shardIntervalsOverIndex(dataSource,id,index,Collections.singletonList(subset),binsDeeperThan-1)); + lastFilePointer = new FilePointer(subset); locationStart = regionStop + 1; } @@ -227,20 +145,21 @@ public class IntervalSharder { if(lastFilePointer != null && lastFilePointer.locations.size() > 0) { filePointers.add(lastFilePointer); lastFilePointer = null; - lastBin = null; + lastBAMOverlap = null; } GenomeLoc subset = GenomeLocParser.createGenomeLoc(location.getContig(),locationStart,locationStop); - filePointers.addAll(shardIntervalsOverIndex(dataSource,id,index,Collections.singletonList(subset),binsDeeperThan-1)); + filePointers.add(new FilePointer(subset)); locationStart = locationStop + 1; } else { + if(lastFilePointer == null) + throw new StingException("Illegal state: initializer failed to create cached file pointer."); + // The start of the region overlaps the bin. Add the overlapping subset. final int regionStop = Math.min(locationStop,binStop); - lastFilePointer.addLocation(GenomeLocParser.createGenomeLoc(location.getContig(), - locationStart, - regionStop)); + lastFilePointer.addLocation(GenomeLocParser.createGenomeLoc(location.getContig(),locationStart,regionStop)); locationStart = regionStop + 1; } } @@ -249,48 +168,204 @@ public class IntervalSharder { if(lastFilePointer != null && lastFilePointer.locations.size() > 0) filePointers.add(lastFilePointer); + // Lookup the locations for every file pointer in the index. + for(SAMReaderID id: dataSource.getReaderIDs()) { + PreloadedBAMFileIndex index = readerToIndexMap.get(id); + for(FilePointer filePointer: filePointers) + filePointer.addChunks(id,index.getChunksOverlapping(filePointer.overlap.getBin(id))); + index.close(); + } + return filePointers; } - private static List findBinsAtLeastAsDeepAs(final PreloadedBAMFileIndex index, final List bins, final int deepestBinLevel) { - List deepestBins = new ArrayList(); - for(Bin bin: bins) { - if(index.getLevelForBin(bin) >= deepestBinLevel) - deepestBins.add(bin); + private static class BinMergingIterator implements Iterator { + private PriorityQueue binQueue = new PriorityQueue(); + private Queue pendingOverlaps = new LinkedList(); + + public void addReader(final SAMReaderID id, final PreloadedBAMFileIndex index, final int referenceSequence, Iterator bins) { + binQueue.add(new BinQueueState(id,index,referenceSequence,new LowestLevelBinFilteringIterator(index,bins))); + } + + public boolean hasNext() { + return pendingOverlaps.size() > 0 || !binQueue.isEmpty(); + } + + public BAMOverlap next() { + if(!hasNext()) + throw new NoSuchElementException("No elements left in merging iterator"); + if(pendingOverlaps.isEmpty()) + advance(); + return pendingOverlaps.remove(); + } + + public void advance() { + List bins = new ArrayList(); + int boundsStart, boundsStop; + + // Prime the pump + if(binQueue.isEmpty()) + return; + bins.add(getNextBin()); + boundsStart = bins.get(0).getStart(); + boundsStop = bins.get(0).getStop(); + + // Accumulate all the bins that overlap the current bin, in sorted order. + while(!binQueue.isEmpty() && peekNextBin().getStart() <= boundsStop) { + ReaderBin bin = getNextBin(); + bins.add(bin); + boundsStart = Math.min(boundsStart,bin.getStart()); + boundsStop = Math.max(boundsStop,bin.getStop()); + } + + List> range = new ArrayList>(); + int start = bins.get(0).getStart(); + int stop = bins.get(0).getStop(); + while(start <= boundsStop) { + // Find the next stopping point. + for(ReaderBin bin: bins) { + stop = Math.min(stop,bin.getStop()); + if(start < bin.getStart()) + stop = Math.min(stop,bin.getStart()-1); + } + + range.add(new Pair(start,stop)); + // If the last entry added included the last element, stop. + if(stop >= boundsStop) + break; + + // Find the next start. + start = stop + 1; + for(ReaderBin bin: bins) { + if(start >= bin.getStart() && start <= bin.getStop()) + break; + else if(start < bin.getStart()) { + start = bin.getStart(); + break; + } + } + } + + // Add the next series of BAM overlaps to the window. + for(Pair window: range) { + BAMOverlap bamOverlap = new BAMOverlap(window.first,window.second); + for(ReaderBin bin: bins) + bamOverlap.addBin(bin.id,bin.bin); + pendingOverlaps.add(bamOverlap); + } + } + + public void remove() { throw new UnsupportedOperationException("Cannot remove from a merging iterator."); } + + private ReaderBin peekNextBin() { + if(binQueue.isEmpty()) + throw new NoSuchElementException("No more bins are available"); + BinQueueState current = binQueue.peek(); + return new ReaderBin(current.id,current.index,current.referenceSequence,current.bins.peek()); + } + + private ReaderBin getNextBin() { + if(binQueue.isEmpty()) + throw new NoSuchElementException("No more bins are available"); + BinQueueState current = binQueue.remove(); + ReaderBin readerBin = new ReaderBin(current.id,current.index,current.referenceSequence,current.bins.next()); + if(current.bins.hasNext()) + binQueue.add(current); + return readerBin; + } + + private class ReaderBin { + public final SAMReaderID id; + public final PreloadedBAMFileIndex index; + public final int referenceSequence; + public final Bin bin; + + public ReaderBin(final SAMReaderID id, final PreloadedBAMFileIndex index, final int referenceSequence, final Bin bin) { + this.id = id; + this.index = index; + this.referenceSequence = referenceSequence; + this.bin = bin; + } + + public int getStart() { + return index.getFirstLocusInBin(bin); + } + + public int getStop() { + return index.getLastLocusInBin(bin); + } + } + + private class BinQueueState implements Comparable { + public final SAMReaderID id; + public final PreloadedBAMFileIndex index; + public final int referenceSequence; + public final PeekableIterator bins; + + public BinQueueState(final SAMReaderID id, final PreloadedBAMFileIndex index, final int referenceSequence, final Iterator bins) { + this.id = id; + this.index = index; + this.referenceSequence = referenceSequence; + this.bins = new PeekableIterator(bins); + } + + public int compareTo(BinQueueState other) { + if(!this.bins.hasNext() && !other.bins.hasNext()) return 0; + if(!this.bins.hasNext()) return -1; + if(!this.bins.hasNext()) return 1; + + int thisStart = this.index.getFirstLocusInBin(this.bins.peek()); + int otherStart = other.index.getFirstLocusInBin(other.bins.peek()); + + // Straight integer subtraction works here because lhsStart, rhsStart always positive. + if(thisStart != otherStart) + return thisStart - otherStart; + + int thisStop = this.index.getLastLocusInBin(this.bins.peek()); + int otherStop = other.index.getLastLocusInBin(other.bins.peek()); + + // Straight integer subtraction works here because lhsStop, rhsStop always positive. + return thisStop - otherStop; + } } - return deepestBins; } /** - * Gets a list of the bins in each BAM file that overlap with the given interval list. - * @param location Location for which to determine the bin. - * @return A map of reader back to bin. + * Filters out bins not at the lowest level in the tree. */ - private static List getOverlappingBins(final BlockDrivenSAMDataSource dataSource, final SAMReaderID id, final PreloadedBAMFileIndex index, final GenomeLoc location) { - // All readers will have the same bin structure, so just use the first bin as an example. - final SAMFileHeader fileHeader = dataSource.getHeader(id); - int referenceIndex = fileHeader.getSequenceIndex(location.getContig()); - if (referenceIndex != -1) { - return index.getBinsContaining(referenceIndex,(int)location.getStart(),(int)location.getStop()); + private static class LowestLevelBinFilteringIterator implements Iterator { + private PreloadedBAMFileIndex index; + private Iterator wrappedIterator; + + private Bin nextBin; + + public LowestLevelBinFilteringIterator(final PreloadedBAMFileIndex index, Iterator iterator) { + this.index = index; + this.wrappedIterator = iterator; + advance(); } - return Collections.emptyList(); - } - /** - * Gets the file pointers bounded by this bin, grouped by the reader of origination. - * @param bin The bin for which to load data. - * @return A map of the file pointers bounding the bin. - */ - private static List getFilePointersBounding(final PreloadedBAMFileIndex index, final Bin bin) { - if(bin != null) { - List chunks = index.getSearchBins(bin); - return chunks != null ? chunks : Collections.emptyList(); + public boolean hasNext() { + return nextBin != null; } - else - return Collections.emptyList(); - } + public Bin next() { + Bin bin = nextBin; + advance(); + return bin; + } + public void remove() { throw new UnsupportedOperationException("Remove operation is not supported"); } + + private void advance() { + nextBin = null; + while(wrappedIterator.hasNext() && nextBin == null) { + Bin bin = wrappedIterator.next(); + if(index.getLevelForBin(bin) == index.getNumIndexLevels()-1) + nextBin = bin; + } + } + } } /** @@ -298,47 +373,53 @@ public class IntervalSharder { */ class FilePointer { protected final Map> chunks = new HashMap>(); - protected final int referenceSequence; + protected final String referenceSequence; + protected final BAMOverlap overlap; protected final List locations; - public FilePointer(SAMReaderID id, int referenceSequence, List chunks) { - this.referenceSequence = referenceSequence; - this.chunks.put(id,chunks); - this.locations = new ArrayList(); - } - - public FilePointer(GenomeLoc location) { - referenceSequence = location.getContigIndex(); + public FilePointer(final GenomeLoc location) { + referenceSequence = location.getContig(); + overlap = null; locations = Collections.singletonList(location); } - /** - * Private constructor for merge operation. - * @param referenceSequence Sequence to merge. - * @param locations Merged locations. - */ - private FilePointer(final int referenceSequence, final List locations) { + public FilePointer(final String referenceSequence,final BAMOverlap overlap) { this.referenceSequence = referenceSequence; - this.locations = locations; - } - - public FilePointer merge(FilePointer other, List locations) { - FilePointer merged = new FilePointer(referenceSequence,locations); - merged.chunks.putAll(this.chunks); - if(other != null) - merged.chunks.putAll(other.chunks); - return merged; + this.overlap = overlap; + this.locations = new ArrayList(); } public void addLocation(GenomeLoc location) { locations.add(location); } - public GenomeLoc getBounds() { - final long boundaryStart = locations.get(0).getStart(); - final long boundaryStop = locations.get(locations.size()-1).getStop(); - return GenomeLocParser.createGenomeLoc(locations.get(0).getContigIndex(),boundaryStart,boundaryStop); + public void addChunks(SAMReaderID id, List chunks) { + this.chunks.put(id,chunks); + } +} + +/** + * Models a bin at which all BAM files in the merged input stream overlap. + */ +class BAMOverlap { + public final int start; + public final int stop; + + private final Map bins = new HashMap(); + + public BAMOverlap(final int start, final int stop) { + this.start = start; + this.stop = stop; + } + + public void addBin(final SAMReaderID id, final Bin bin) { + bins.put(id,bin); + } + + public Bin getBin(final SAMReaderID id) { + return bins.get(id); } } + diff --git a/settings/repository/edu.mit.broad/picard-private-parts-1333-sharding-3.jar b/settings/repository/edu.mit.broad/picard-private-parts-1333-sharding-4.jar similarity index 94% rename from settings/repository/edu.mit.broad/picard-private-parts-1333-sharding-3.jar rename to settings/repository/edu.mit.broad/picard-private-parts-1333-sharding-4.jar index 05ba590d337625b7415254bae1bbcb3b96596b54..f8fc13d6d1d6c2f9d9c8d04154cc873cb97d0244 100644 GIT binary patch delta 3002 zcmYLKYfx3!6~24##pO|m-~$X)P|yop1TV<@As{01ekmY90YR_?qQ;~JjTvp#rkYU5 zP1u@;p`)oL5uNdBP9jla6R4v!m1e3l1__CVXw?SoMB9qC=d879`N8?lT5EsnTaSJ2 z_g{E@|Am)FQLX?tlkxZLo2{K5A!4Kci0{P5U;sY;9%MZ1(^)7!)0oY8IySLBKtqff zQ-@)_}t)s%{VE4c#c`@7niyE{a?3kPR0V%zaaZJ0T#zVKtEBn2R-e9NhfTP z$Zd;!i6Q8Xs-%#av6)V2sY%*tMsOp&3I#c`a^6BAj(;g-#H`Ch-2_42zvT1Q_QdbQ09r2(Z3v3 z%$LTUo-i~PR5YhmaFZzm^``|71V@BSzreiZJ8|G_@1A9r?MdLSZRKn!won^zVQx`zS?0v=VOXx2*L%<+*5q#X)E9ALauI#)>vNpt?p&CnjZfpzG@)Q$ew)_K zdk&v3_Ufa`oWlcAkZg+N*O0X+iu+Q^mLlFw^YJX@UDWOOp5nKu(!D?9Z%~tCHu60b z_t_@yAmMG#bBn;w5CKwC%q8$(ZSe|as~Q)tact4y#;*d3 zby-7ZypwoOPSwKIDGGl1zF0@w_FzJI)7E>M;W2(WH7|cSN8aJN(9DGX6C`sj z5Rc!$T^2}1P#Xe%@|p#vVQ7!nXP2SXZyW5I9?fqr5E0(%HVAvhNTNeJ{{ zKRG89<{9`zpHiDG-@Hx659C9Yf($VsII8-9I7Y=m@Dk7j5!I20kG=TOv0>W`FGQtKc(YMRGX%D5yYiFbeQC$>$bA4W_@m5Dug3m(g%;Qmda_7XuOawl4;T z(BBfPSkJ_&gaL8NvLQ|hz7(gk8VM8PRl?Weq1UizB?;h8%4-vK<;Jh}=|otB(T@|U zPOT&frch9Qvfep^NlS)21m+ZIMX)^uK1PtT2s8u_7O4s3r%pCLRUN;Z3M()sI}NfB z*wa+-%{0hIOF}xVLvS!1auM81r+&*Ni`7hiw^;G~YcVXxn4%2*xKV~aTR+*G0WYID zXNju(-6e|S!6eAegmR2Ikf~xGXDSH=OW{LwIcKSu+AKBulUbBdb6*DZ8?XH)8>Uj5 z_UBC2nWNSrKUXoC^I$3JE6jsryx#43V5MWd`D&j$3Sc^!<`z<)wM~T(N2Z=4IF6#$ z6vKH03re60L3@d6;`I_GGrm;y<+r732BW2_0a450X*|`vT)FNpSF)F^fD&}=TLA+I z)|A08g7k9Lq_@iTkDvZ|lA=NlqozU)|Cu1y0Kb`O{!L6JF4}&YaG8?4Qml(*1%~5 zPPO_(4Y4a~Nr&tzRq zUw`WL^`~AQg*gn|OzQ9Z7oY3!2w~sJM|=l9h6C{N&mi@%&mp1sOeHq;bZjRBfQA?| z`q|6Of%yKsISilgnQg2`I$kxmx*=Ha)$fESK0cai+7`ffTfxoV(3K-N>Z9z|ZVA5$@ zB&x7QzCeOCj`|=`^#le2?p90d0^bcZ`z4c=4gFu zuGZSo{hj%LW%32F;A-zb;)XuwjvS>J)%xBJE&ty8Upb>?%rX1}RMa9&HhMbeegDFvky;4 zAk)L?bWL#awd7silAxFIvqUUO*c^TuLsfV#Ka8a=;mpTNRmfLiDhXd`Of${$V}2ho+Ma zq>*~4CXMgB-o%_trg!W?)VuQz|n~0 zedHj{Se*YAxr4hUPQ6aX-7wEvhsfuy*qNPu{EYrB3uNvgMd*orhk;>YN>(auKH=FQTt~Yqm2#kOMQt21Ak-V5mxq{vuE195YBVn6uMO%Lihe!QJJwD8tvsfA&0mxtXi{8(PqQl5H&`0(~baOK-dz>|-*&_83w-d5_a zH==(p-RF)iNbjKUXwF-9`j=VgclvkK-yPA|ar*2$M5PyKpKhA@BK<<^xpA2uq1cxt z1>RfxoK2=mPm?Kuk4#VtA1ecIF?xjx+&9trcWMP(xK{%A?z6}}p9gB;erVrT2;$ZR zP1hg^az zWXYje0o1XNwBMgLvv`;xS2X!QwyXKi%XywPt4r(0e^dpYBEFIB$IC6wHnVyy#@fo< zqmd?|k475rnC)wT^D0=B9tQYTr7#m~@Gz%GFw`Sw%H`mz}-I(Fq z>=m7@a+1ADQC3gG=K)1h)`%JJB;J!#bud>%{EI!ItA6<&+oHb*_otb+E0TJe;Sqj9 zg_l2^)2E}0pouW~6U1{Y5Rc!${T5h(pe6+T_%#d6!MA@~U>O2iFeD&o3Wih!$AaM) z0vWLsfjtDG5S$BvBm~mgk1q^`#i}DbD->!F^o2qLo=a9g9%Ti8d@Bl#g03(q#E7Xd$U=~^1U|td2#4&)WNFCrS~yhT(cN&UMN4@E6d~x1 zfC6F_c}WUn$Re`l4!asM&M%78KqJ(OXL z<57koA%8Xe9*>+e4WF7!-4lK?QzsPeYe4_Ti(h2HEZxhlY@vm|3|ojOub@7$20&tF0ENaXq+d(Lb$#HOgC@t$S4R2{9W zkvf!fw_y>JHLzG~oL6f!t*I93(fe<;Mq{JvjIG&UXDsem9ejp9CpN?T$hfKAX#NNF KMqyqJQ2JjLxOeUV diff --git a/settings/repository/edu.mit.broad/picard-private-parts-1333-sharding-3.xml b/settings/repository/edu.mit.broad/picard-private-parts-1333-sharding-4.xml similarity index 55% rename from settings/repository/edu.mit.broad/picard-private-parts-1333-sharding-3.xml rename to settings/repository/edu.mit.broad/picard-private-parts-1333-sharding-4.xml index 1b36b12ab..acccf2e28 100644 --- a/settings/repository/edu.mit.broad/picard-private-parts-1333-sharding-3.xml +++ b/settings/repository/edu.mit.broad/picard-private-parts-1333-sharding-4.xml @@ -1,3 +1,3 @@ - + diff --git a/settings/repository/net.sf/picard-1.16.363-sharding.jar b/settings/repository/net.sf/picard-1.16.364-sharding.jar similarity index 95% rename from settings/repository/net.sf/picard-1.16.363-sharding.jar rename to settings/repository/net.sf/picard-1.16.364-sharding.jar index 60f977b53b5fff20ae66d6621ca0300ef5c34cf4..fc60bbb51dee15abb0bb487843e1fbc5fac80975 100644 GIT binary patch delta 2845 zcmXw)3vg7`8OQIr_na@go82`_Ht&$l=Do@0v3U(KFDzId;Soy&ghZ4EIy}OoJ_;>N zMMX;kh7j2!$Ot&?pp0RF7_S7hm9ah$rnQt2EQ#$z!19V(J50o2&-u=Q8Rq`-z0UXj zzjNAFsSs)E9`OiMi5MJ4GRk=F1u2FF3g-YeBda)8MJu9Q7DmHpuoxSa) zs1Gf92Uk#RVtCeA^#U-A1<(^Y7~p${E$Dmy&~lJbJ(2!0jBf8gNfAN=yVY4AoH(su z=byL(U8NsgRgnJa+v@bs&fY=K`G34+A#|Uw#Rhx+Iv7U7ox!M5iGOubg*|mqcx>MOz(a;G-#!pe7NYVtt0y25?d&RFwR|Uq6^^!xXfkS=7 zfz$G1$BA-|94B@XhP{54^rBq$L+8nxs&L%rB#aZ zxe;^&hi3GSL|d zTIbXh;uYAcrrlm_1--a|JC_rmDI|tbNcf}ksGnzsEVundtyNDjIZ$uqdXc6yS)~@z z`dQYi5u}W#taT>RuBWXw57KCtm7;73JFJ%*(bo5=)#37ruIgI8OLc4x8ph@z@9q#U z(%(i2<<0pJXV1N8bpSb(Q!cM8%7Fy00jYBayH?RVA$=}+>rpI8d-q(-5Yls_Yke;|@eqJU@ zMF&^{e}}+5cGFHql=8EqtVfAgxe`ayOEyR}aoWh>RzifpzWe-Z3= zzhLc3`rQArqv|8{`6%0?GS(AV5o3uT5Pro06L91#Cp5|*5#H_u{4VpT6YwL)KXF2h zoQ21@U=~6{5~T6RT#$!2SriYm(RCyq>JYw;*QZbN!*sN4_rp|#fBNAk z2$=~w>z4`8hL*o4=t;8@^?;ib;pgbOmZ%E~Y41bJ(@DCYFOzguWimW~uHPlYT!eoo z!_N?^Q?x8wQ}j5)DKH6Lp;WEvb9Z4lRVzO^4SsaB@pEnAHsxr${q}6!52wHKjvqk!~%L!}go(${R zPla`z)MD|b$n<5!;%Z4aU92?>l<4I?U!n_MF44PaDAo6Hf2nds%k%;lm+9>OyWohx zG%VZ_(PJHq=)=E{XyIp;!xU_pmx~y=lN;q)r^yxK6_kDLs}L7T!tDw@dP}8V@|%@f egM=!r!Gth~Eg%Y3UER{bAJRKrLB delta 2845 zcmXw42XItX6#ei2|8BCI-8IW@ddOzeH`(+uMA8(H5=xW^2#FvD97-rE2nwS#6$6A2 z*(b;dI69Ou3=rd&fQ_+I#)6EXB9%olvNm$8{@5Qvfn<`FeE2yH!~ zSP4tZI zFPRLAG8tO)_j<(k9u3+*-8UP8tk(RFJY+X?9Muu{I=9MNpC36P!RC)dD7tdLyd**V zx9`j9Kb*Rao-^OSry*!LGZ-6e{kqGKhTB~Msgn5poTMziq9z7i;yD{D2m)ijjk&OS z<8wRf#&~0wi%mi04PMT!Vz0)(L)a@Cj`UXPu#pSn6=h+(Ig(q9iv*`}a5!5LkBQ9{ z*uzIeNrWuxhIFFPyiALwJJ(P^yJ z)zK22aj04h^19Pizf5fRsm(HCdx3gGcDH4aIy@SAJvC|$F5GZWR|B$vwi)VIY3N4y-Y}TFaGPEgh@9Dfg1_g0@5|zh$LX zZ%1BVvlb(55u3EvE78{et=4G8a<$iEL)Ul}WsO&~G@2Lbn?9zCnsf*^CY;lny~zH( zU$cZE`}B&W2P-s1snEnH>$zK0Wq@1sr$dO5W6?)YH#Yr%94gzXUy=&OChLDmnr>=Si0ss@_?j}3 z&*}+|P?J`+{m|%lgd>al#eetlFyn=b`c2%ZvHVYcl+^v0mzT*dELpt7;tYlzXReRjnVOx{Rg1B)h*xqPeAznw6^RIXA9O=X2%y>t^s{ zQa;~oK6L|_ZZciIg^Tx z3-DGc+WBqVg_|^P?dI2{_sU*=QXcXn2l-rCHp}1}GqCK3Kl27Ded<5_pv>}PAKxY; zmNJ+VqM08MaWNDI1p`Eq1uBCd5z$})d@qZD1@I*&j$5D1%{z?&+z zSz#KQ?^t0ng1Ru6h2YCDn1mqN26GU!*x(TaS8ebpg0XfOgWwH2h2OGM!bcr|7hkkF z%LY48fl+sy@UGqtLR^4I>cz4C;f|k?-D!MFz zdhSjjSDA-8Z1Iq)e|Sii!HG0iQz8}pJCO!_z)Nau@sf0XURaN{)+SNri%CQtoJ@_a zO{S=G$#hP$Q%IIoDYVM26q2qa73N{WU8&TwC5=Q_m_|`wq*2eFbZT{eI<5OiI*shg zpv;XKH0jX{65QjXJd1o3b<_tHSX9fTgyS=*pLa4zReKhx@pKl{A^B7mjg#Uh=@$EG z_G5m^6PrypWmYyFtrOX#h9`&SdMSqzUdW-jDsqXuGgrC-c_ha4Jc{nP3qtc@B(_tV zPh;)Pr{XvBi9DtNh9gH55a((E=`^&EuHcSBI#9O?Y4qA6TJpO^q(MY6X|SxA!cP?w Qr?7-vTT4jh8znIBe{&ifH2?qr diff --git a/settings/repository/net.sf/picard-1.16.363-sharding.xml b/settings/repository/net.sf/picard-1.16.364-sharding.xml similarity index 76% rename from settings/repository/net.sf/picard-1.16.363-sharding.xml rename to settings/repository/net.sf/picard-1.16.364-sharding.xml index 31c6ab92b..b731a711d 100644 --- a/settings/repository/net.sf/picard-1.16.363-sharding.xml +++ b/settings/repository/net.sf/picard-1.16.364-sharding.xml @@ -1,3 +1,3 @@ - + diff --git a/settings/repository/net.sf/sam-1.16.363-sharding.jar b/settings/repository/net.sf/sam-1.16.364-sharding.jar similarity index 94% rename from settings/repository/net.sf/sam-1.16.363-sharding.jar rename to settings/repository/net.sf/sam-1.16.364-sharding.jar index 3ebadb087864b1fcdc736f939f74d3ca50f376bc..7453d10ea71115c8532d0c77df34cdd5913dd036 100644 GIT binary patch delta 11595 zcmb_CX?Rq{veh%^^qHAV_I0w534}lv2zx>ZO8}9uhRqAOAVLr%36O*x8RCYBqB2S= zDsUADw6pyuMb=0rbg!WybUV&8?mAD|3;xBYy3Rw`Sqj)yDMY4PvR(yS@uM8z6!Bpr-;Q z@NT@nA%Tjdn-&RdN`6lP1!?W=vS(%cMoa>`I{c(7;_M$Z34D;#GeiQv3|So*fFiP^ zT-hDgXaH+8v2M2>v3OjhcyAl$>#N6nq=vjSesOD3YJH2PMSc=kwrXgc1pc$ViAw@8 zUw*CyAWinX#GgOR=K%R!1YOW0{JKvl>NlaJG!efSXGD4i+aVNMJ0J|&5Y*Ozk9Kxw z4;>tk0h#o(qXAhq=;Qz!WE;?#kX;<$2VD*5W`i6D1VANj&i~Pk`WC9z3k4k{)xkiMn6!H?=GF> zF3vBUG%epfu6R4XB*s&q(xDWXay%tGT>nE9PoiXL$zphRiPF+X%9x&iu2!{J0!ULmlW$%~LBNSUxdIVamIa-PV>0t#H zj9@aPm|C=>E<`vsLBNQRz;X!MiocY()Re|Hn3BU}Y9UuOITR8H4-RUPG6EIX?jK2Z zZ(u629odbhwGo0}u06FHth6Z7a^|8;#8@amy$T@>CN+@QqJa^~Ua3JjXu~v9b~=8O z_WD>fVhGu%q2@gr@X<{^R%juqT0*+~ZUqVW?LaP}n1S42xt+Wz&)u;$Y+)xW0JS*r zfnf>lTo=J(3PzWdeEP>HTX1|EXy(Ahh_j;`G@?hm0d5B_aDoAi4QOIOB7^#TgZ>?5 zF~cFQK0ie?&+Yf;(4aiUT;7~rVY*Cb;(Iqs5(h3=oUDZ?o58}xw-+oSGS}#oYh)Hm z63cFw3CQz*C7?iP>cph$bb>zw{O>Vdmvx*F2uTe7KG0Qw7mO}?FkN&Bu-HVa`*^s4 z!Hv-@lW=R513M+(g~sP-H-?*l1^pBiL1TuIE^=`+6cvQnV9}*Db9+LtQ-yE`G;Ick zaV8RW#FYbFIb0zm55wd@pFqisd?Qe@qcEm~VZnF-Xlr`CeG_OSFw%78IwU|jeK*Ne z)ZJ}j0`Mb{Zvzyduk2Xuos?a&og8}OC`=k_-Jc!wTq z{NPMwhcDYcR4-&MURgO{Rxf9 zwYt$r#AlNODqu4`wm3Zad#eK~VH^EMr~Gv)Jb~`X3@A1O`-RnRd z_K{wn8t|C|6UKfTeSpLqq{ksUbb`YUsDdLzf0Q1_{NOm8pvOrX`MCjK5W`b;NKS^+ zH0n!woT0~A1I{^+MKw*fhI+m-;A;cEDW6fuUa~q6{K9}~2AubvnjagijV><6(AEm_ zi}1@bA<E!XHqJ5SCMnF$;qj@q7`B@lco5$uwd%9ws z7MBlUKkUQR$LC=Dbr*&|l_U`vSH*brULc*`G;WNLdBH@h?&aC zLmCX$>j$GVnS(drra+s3XgZq>LA7VjW=*=0JlY$gQTiw}6-}>y{RH~M{V*DbKY+b{ zpoQK9&VqkKr5PnTX01;0y|}3MrBbwUf12`+A=3O`aSpjImOp z6Vp^aLRK6T2;&gaq(s~)Zb1xVc&K6UpegU^IX?2Hqr4d?ZzhhN_5UZ&LIFbaE6;wI z^cRNC2&*(5SMC9rjC~wu zIs&F3WS}d~LkrLxkK{m=@gO1dLhOAdf(pko!un$#H* zq3MWbIDO?ZO6*J&OUs2-{7q=iZb39-wOFy+4VZ=5fEh3wpMVLZbuHRmQ{8|mBN%aW z*l!S3i?yJI`T=_$4n+}jv4BToq@lxogo^v29*J14T6RMKeQku)nB`zg!6G~@B_=j* zBeclU#Gc1loF~P_!)@IBsv~<}wfMX-1zYP8^-DUq6k}U9#$82nM?Qxg>G2kin zWzgBWO^`6692#eF>TJy*%WxSVLmR^OC#WsMEzO-}L(#g+_A#`x;P3W|)J|+AX*@w5 zNt>AEaB&w(?w~}ju06B5_I($>Ei(2sFs!SA$7*9c7~w`xz^o27dxkxnWY{)9#+KJ1 z&;-M6=^1vMR)l>cWM*-MHbO_Lk2D#J_tB{fu-b-lgghT(bXB08H)Gn_f=jm*qii#J z`*!q?9q19e(QEhM#P;I4?}L-L>NRixzJ|lN0aU>aIEo8(442_J3xyLb4k^e7iTxe& zKrW9um?a6I>KtPB_4ruzT0gKRM4S1G(w%ISB``vgC$c!z7%-FJ{G#Q}{ zW6R-k)W;S=W@!1j442=ZLQATHp=1lUQ7GB%8IEvb=^IL(jBtB;h9kvgL||-YQkdV4 zqbIPw1uT3t3<3_nqOpF%i-g~CS#IGH`~m6c5Sh#ZLzxxs$2FgT@FWbBVrIk4Wrs(Y z1C}s9Sizi7!Te!23xLB2pI|}o1q*?%Sf~dRVHiupJ-jQ6V!fG*4P?=5FpFWMSS*{! z8nLM?p4OOAAu$wBPA_A~*r5wXnTod_c+yt!)(brv;|+bL0*^yP=MQgU1}8cPtik&L zqVvNLY%b}X_{8mGsf?5*&<$RZgI8cEW4Q=+mS#q+AB!-f)`2tb1g}XIu9{JcF3M)q zxgJ7lyY^#+RAZqtHD`IS3?VDpD;Jg{q@mTi!3v664;qWZKe3aepD&RCY_Q&_H3hJZ z4a7N7{My-IRDt5xAvYU}U&VytSB;#BIfs0D1_be&5Wz5p)t-YPs&*YSh8trH-vkTp zNu5T4KkhR11W1PKCj&j978x5pNZfkS%#g@B@C+twsf!{(I1TZ#Ams&b?7$1t_tK=WDbGZ>zOoqIM7G~ASEUz{!T%A)d+bjc}l4+-K%XKBPrOWb0xMZ;O(?zYmw*Y)^5r-FO z5n|qF)({cDpL22lLiUgtm4oMJpIeqHW#*rN? zyF92@YLnYOh*t^J+;mWn5c{gQzhYc5+&bPN*)=2HdkT-gc1*q$D8I|4mB^O?<@?)cXO;A|85#>yT>EOnm2S>qT8uJV%H7&t)%TyV zT6@K%#bhl{={s+#78oX{j~-k930LVsO4DEb1`FZfLRNZB~?m^Mi&BbJ=f+Dc9LpVA&vT|;WLRzz%IyEHplm*F$n(YbldX~B z`Wf!5?^4|C9OVrSsG5WVkxvBzvkRexM;2*0TK^QJUOl%v-W4zaA2AAJc zs=L+Nd8X=2(LL6G`05KB-~tzum+3L$AhX)bKb)igtPH;9QGK)WvWfHc8xc~!{AK$0 zDk?^-(TA#8uKQ5$qv&pZq;J=y_D}0`k6y%OxafVaJu3f=7AQ8=;Dc4@mn5TW6?%aZ zbN3GYPBnbhZoP+suI|&HQP|ms^|=bVc|zyvc1pjXw^KoyeMvW0G3lBf-d=Lt9dNS}ebeUJ=xrzoA%e9LPN}Qefsp-d`E{$pU`ECYj8cx_;YJxQ^6%NhaRv zBV0T;hbMY%8G+?rP2~&KQcj-DUsO}IJ;Ix-#I-K(Cs&#Vkfy06YVi`ci-A{A{&>MV zsId9xEBqBTuIuZ()@8-Fg(9bdXVUvZF{grerQ9X9RPc0CQe3a#Z6wfiGw&o(V>a_~ z64<|)50gOR7Cu}8o-KT!1gf_1K@w=XmG_sx)U7;E0(-aeF%oE5$tOzS)k=P!1a4LG z@e&xn&1AS^8y_I0hix~3{O!Dt?D=pz$|4Q59lWQs-R;+d0Lfy+4xVVMdsQTYD|xh= zqSohby*}~*R+hbZQ6%i(;Vv4}Qr@%Y^<|bYt9q$QlKSr8F&4k8l1x0jgUh$7Vmb9* z_)+$HtJnK2zQV=b=ebkPpv^Y4h$0KcJVC9HFh_Tv4#y}BCsz@ z5O?o36?urdAHA1}xkFfd%RiYcJGT1sW2-Q^P)n$1Oy*80L_O#RO z2xb^+t$c_BCL!_M7qk>HV2`P^&jqs&am#69-X7jjDqXP$?cZ~;XuX$*i^)&2U@AG( zthsCe-Y>ZSN9px)_+sMqDqD>THjyI7dmjA%E#H@`aj=5xuI0;pyB zNDVw*Hzr2R`xHGQ`BUzcV-9EDdHu}{0BTKq#w$=|I8wTiR^h{AAu4KwRA z(@z$BhD`CTvXm$B_q&kkp*s9o!oS&3;jy|(P_8*UF6m__CRJ=nKBCTK6KOk1b?*oI z_Qs*OjM$p{P<@iC7+zn&0>!@lruEP6M_G;UHDw*((Q-{%9H42BuJ1KOmItR%dcYjD zfbiYMO1yX7FfK}vpOoJv^+q1PX98-z2C4X3v2*`m2@;LIzzKT}ng(fc5ZR45Uyrwm z^fUNcZRSC96&4YG)>VbCJjhd}r+r7gH?Nzc4)H{(z^ggWPPv8lq2{YVw;T1OL1Mun zllQVi$mOv{V$cy>e5@m5CGUSz@2w=+>uohZM6^9@PEecGWy?xTP1KtC8o0e8f9{C@ z@g$N_q$zA8c)W1l-uufWG=0 zYoHi#%$$+`kFGwi<9`cK^UVv7UTZLDInpPPZEsiRf2JSj(8RVHYm}&{wK~P-)7&Ws z%o!UL^9tTxQlo!tsQaue8x$o6gp17MTvmi)&~e-?nkQ}}i~6t4e#3W53$l>`HTs8#y3fk8 zW#*V>qTrNiRXOdZ*F?>!znJ!051x*%LN3&Nqj0z!p`tMMG)`;EkHjKLj5^IDWE7qp z_r&>&7`@bz>Qun*m1Qk&5os(c&!gmg_7j5<7LilKBbR(>8q579cD~Y5c6wJ#j21;- zn(;j};En?3#>*jW?cw%kWm$2%+vL^BIu~cYHuuV!_*IuTp%+l=aC-=9vwLXWm_V<4 znNytn+O*8bGpNAv@%31Fh!bni@L;L*HbSp2l4#)`kN^5*J!^*g`)6Ih;}tJm^Zsc8 z-k7fF`<*p+jN7NaKi7OW<|@f}4iv6RQ>O!GQRtv~br@Yto~G;K{i!(B-@h^A%6$%L zhRs)k=yWLVJ!dMG;@5B9cywWESw2>Hc#)z%_pI&^>(23Ja*n5nRp-SGP*K&UCM~KF zmGQPhRlcs<#Y1%{51~>w5-Li}Tci7lt<|P(2MP7aWfZzL)k<6YOV^Cs#c zLItMRqx{6W^QP4{#ua<=p|g)xzjSOse_uM56Y-U1kZulx+I3iAQp@%kG%}~b zpvn$dEzuyU-@6x3iLnC|KBx~*5O^1Wl|&sr?jqtJ8+;qTq&Loi%0FqoSh4t%Og!pQ zOY@QW@b!64(2E|9#hQpR>-I_g=%<`^3Uqq2ffSXP1sf z#R~LW^REf>JY$SsDg_`X$JUkN*8uOf?br8c7?vn!cA3ZYG1c6?cjrs(5UTX z5B$X2<5_gBj}@B1atoxu3KOiff)NVnVHI7z@_^OwH9>3W@{I)?u+|7LKh*@^n&CSu z*kGL%DeFy8XayfAFvItB|A8(W=(3T>KaxxbqXT12dNx_X1wYYcvo#l=w^(5-Y@=s{ zT(I2)J4~?C3XxDi&$~?UvlXIYw+GaRU##fE9`ftoCfI9*TCk72+D|GDkgbDsIYg|# zdcbdRm@Y-cUu=RSq~>=E3LPb`V|WRJImJP%)F{7nWINXGf=Vy zjl`Q_ShRRHnMFH~`bLSxIV?yxrm*&+G>2taajxx5aE?JsXV+vuku!zGn@uqtI&`Sj zJJnh2A84IXeP|&fr?U3uH29AVdcs@cqGAjYv!}9{CYRu{4KBeI z8(f8JHs}StZEzh*OmNc%w@A%xc*_QFz?&wxV}rYJ&j$BlFatm5yvu&##Z(q-EQJTc z;m<+>(%_*D9>HUB>4^>ghNlcd#jf{QXcnRVp*gg$!86i02nO4rKUsSYFK{t!P{zQ< z7&F+IktqzkDreeh*qE_4rZR407*rcGGm8xbEVMBz$#^iEjd?OJTt<@h7I~W#St8XUR-m8JrIW>!)v-&IGxpGUBs^GEU0={ zqy-`uUAsYdyc3iGJ&=>p8cT_1=u&PMNmB9pI;d#C3aQ|S>Vwe$sj(p|fL&L^D5xZi zxOlu1P5l8ZJHAE00HlmdWcQbZ)e2z@x-PsC6l^B|V$}@R9o_zW2CLSR>^Tq}g4W;0 zg`o+gRZJia`aoY4?}u0qQ2d?&XH}k}(~QE4*L3mr2dv#Y=;oK2>*Xu}ThKL{C7#v2A8q)3YvFg%A)#@f-O2kQ20Gu`Y>Kzr|@ zy@@D0>Hp83fpVDU=h`-tbz-<$mRanhTw1Ms@PY{t218&dw$Eu8xnT%-;B!5A2Yol= zOo!vYESz_5oYx4%adiJJT#puD!ZMkLD?oERjxl~$QoXSpvk@{tW7q?u5HiBAkN~44 z-|rBM*)ayxTmV+Bd?67Ui)04l&?!n8&FpwIOX2DOmS+&do}eyBPJCZliZX#mF8(Qz;IBenKEP+4K^ zjZnK;>_#x`1uwd7g1E4yV2%xQL7mvzn;@_`UF#+(TOsl-7epnicD43qs9}INg~(?U za^o2!o9*V!kicLOq!E7)&EvQemTQ);I13uT^48&@+TXoi|1WDHDWDmQ^?y(6P zXo^>ton2kFUfNX#CYb}wK3D|V3W*gJM$`R>Gu--YS$ zGlp(Arphm{5Cggb;jdvYtcCrk;Q$wwigyvhqzYsE`6G`-f zz&w3{^3!1kN?gZEnFb%AmZQ)Va%ZAy3;LfSk!Ff)MTVv7rRu?2P`0NK&F&o4>w^_nqaP;#QnXrgD4Ll6Q@X3U2yTKVl{ue^jU3=IUavq9 zT*H^cUr-%xKuv5>3AkZMhdbD4?_wlN@iqMb?LUUE;0b((9ixE3= z>6mzv^yIT(8`P4|jCz_9C-LD&%hhg%8o`_#&v*6sCb{=4^hhbLimW_ao zsOci`q?l_BZMvDQE<0&gE2ETzD+(dA7U|)%6{+JvAkih~W zlLf(Wghyd5Qd1d>-ezEI>D6w*-B5kVrLMxU#Oungf;(XBFVt1A9bgp7w=-iaA~Loj zB4hQ4#9;GV1dDN98{<~%bJ-5cUrZW?YMUy$CYXE`^E1FL-4|t8`}kt2NpB{6l%*-F zH~I;3Yg?}`(TgSe_UZaO)~FtBHnbg2EJk1{J{e#cW+>pi7gVQuBP)ir1p3ybxn<5}G1DLq=_#-+8hC@DzvFJ@7q`Ekx& zzZJ1g^4RJMJI*F+(C4Sv>-^Z{(JFxE0H2h2BOB-mIm@BCVE*TbU zUJbrth~N@8v&4|AP04c4&`QIlJvOweDXH@!jN`SFt+l3ci#+|JeI%cKzHwgz zGoGzeR$5ZpIM*}`6aCNg0OzTO4$)+valbrvEBlSJq9ruBQVOl%5vJ zSJF=jLG5aiX`cRKmDLAp#D|Gxy?Che?!+ptrn#zvNzHwy zsmvtTVtQK7)mPE+tDIK1_y+fPiJ9tM&E?to>UM4MQs=9W1Es*nb)NtHgAL#Z8w$vI zF8xlmxW=qeciW`y#+%d(tvVYR(xX8IAe zjRqY&rhcMPd!1INYS8J6D%ZYP-uyI7(Q=e*63@ugl zz^jkdfQHhw@HmcN%qSVc;`vppgw$l-&m^HCt++``+)NM^B(>hp z2TNe>e%?a@#sj>k1o|G}=@KYBz`II7Impu_ka3WAk-+MMe1HTz5Aop=$T`IOO5n^P zK2!qjf7KPP_?35-)=LQ>$J+kJQzbC(H{I5y-*{Vz`Smal6|WuULE^<>UR9zTMLg87 z=P&2?*8-duHh73WMcg-%UJ1uvA9{H_HWnKA(XQfGkk=afeV#W96Wo#JNeVZ7=#C;mLAszC6N1jrUWTNbJEP#hD|zk0pp!GP|SZZ39`bi29xTN;kqT zoIJJ)i-AU@n^N-tWTuZt4eAca${aJMOGjq#H_|Q&fYzzT*#Uq5j>S(S!A)?}KJCkvX5$7XO^&HW}5H1DZ`bi<71irNga!EBDIX^*t=0EjKVc1)42`;O0?qP8_Hk|(cM}X%MBFhwP_2$(KGkz-q?XGt zOf$<#Ul3{HVvUs03&Vc0oOFvw3-=JoAx0K6f7*U%(=Ew)Hn$eGTpEqt97s4w{O#m) zHy=j@>x&3cReX3=cgA`Gu?w)kPaw9?*T6+5+@eT<-b6ZM)<`Y5(Il3f&@C2!mNMmA z{B%U4mYc=80mR=(DzN^63L@=oh~*JRd@+1s^c1Nlby`27wdp2lAwrzw z@;xrr5&Fy#6Z~Jj;Wn=_4V* z=OQiGTWGkkh72IqAaN%ftu8yO=kX^|(`N)@+0ux0WXjx8t(nP;G$p`c%tO0-1)xBHM5MIvCD0+ zXoqDTNj-n^K%dahJpgFXhpzHpLEdOjMT{C?3=o?y@Ic9H4n6QzvK@eiJ8uWOBCFKd ze}gLeMk`!+U(}--#uj+3W&&|ibIS*d=yR2#?If&Mx+x0ptufoh zmU1OUsKju6pqP6J({cD`SctYudeTQ;LXsn!ND<=gOWa>3!U6NX)ZhK^{e;8)tubaC zu_nlXykr&vDjs~`R;wAaMC2776!&s%lGkDXbZ=0AU6zLX^ncPM#N5l=L%enw8yh9n z+n3S16`x3DL<*PKLc+!zB5bH=enRp5FFOlY=fqdXcPowf3eRYngN~6{GpQ#N%2Ps%EQX5 z1v*g|k!KfC>yaM)FRx-M&A+ax(`Xm2@<8eR-$YBluhZ&W!^Oeff=IiD=t+-t^wKv} z5ApFeJ=n{UX4qCuBwpuBg&8@Bc9>)S{8HQM;9Coq4HOd=d4BiZDhnRR$cYi zR?G<+NtH~vuGcycR}H6QiB0UiuDgHwI@){GM^Z$@Up!Kpx{z}@YYlc&8nxV%92-Ea zVIt=*oCC#w5dwxiS%eK1hyUW$q=L#L)~lPj;IX304cyF;Z08#&J9-=;!bElwS4I9b zJjT2d6S17w#!n`;V6o1It@_*zTocOHSdUKAhGM4DNOp5{$@}C|kSNSotIMsEwAi}@ zRrQ^zk+SFG@-8jGRzc3JBT~;L8i~$rqNGGyqy;yT(rK+mDQ=;1k$h7RS2~ea{-BZS z%uqbUuI;M!{*>#`ZtYlO9`2H8)Nu3f?vF1CT`t%6H|hIvt43do|Jp)#xieEZ=K8pNzTD3M+_4qMKXi4v33) zuutdT<~8IBcdUC`Bq19O_ktT)T)~PThf-9lcz#7e6$lk>CcwJywQ|=?y-P1ka?hj>!xUZyU6eEyk**(c0agEOED#cabYOdEADdXJg9JXy?W?Zk*`|Xn6f1?Ar{ku27MAO z{}tp7$iZ(t(|9Yf>;vvYj!XW8 - +