zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Removed used routines in duputils; duplicatequals to archive; docs for new duplicate traversal code; general code cleanup; bug fixes for combineduplicates; integration tests for combine duplicates walker 

git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@2468 348d0f76-0448-11de-a6fe-93d51630548a
This commit is contained in:
depristo 2009-12-29 19:46:29 +00:00
parent 29f94119d1
commit 87e863b48d
8 changed files with 108 additions and 476 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
java/src/org/broadinstitute/sting/gatk/traversals/TraverseDuplicates.java
View File

@ -61,6 +61,7 @@ public class TraverseDuplicates extends TraversalEngine {
/** descriptor of the type */ /** descriptor of the type */
private static final String DUPS_STRING = "dups"; private static final String DUPS_STRING = "dups";
/** Turn this to true to enable logger.debug output */
private final boolean DEBUG = false; private final boolean DEBUG = false;
private List<SAMRecord> readsAtLoc(final SAMRecord read, PushbackIterator<SAMRecord> iter) { private List<SAMRecord> readsAtLoc(final SAMRecord read, PushbackIterator<SAMRecord> iter) {
@ -73,11 +74,9 @@ public class TraverseDuplicates extends TraversalEngine {
// the next read starts too late // the next read starts too late
if (site2.getStart() != site.getStart()) { if (site2.getStart() != site.getStart()) {
//System.out.printf("site = %s, site2 = %s%n", site, site2);
iter.pushback(read2); iter.pushback(read2);
break; break;
} else { } else {
//System.out.printf("Read is a duplicate: %s%n", read.format());
l.add(read2); l.add(read2);
} }
} }
@ -85,11 +84,20 @@ public class TraverseDuplicates extends TraversalEngine {
return l; return l;
} }
/**
* Creates a set of lists of reads, where each list contains reads from the same underlying molecule according
* to their duplicate flag and their (and mate, if applicable) start/end positions.
*
* @param reads the list of reads to split into unique molecular samples
* @return
*/
protected Set<List<SAMRecord>> uniqueReadSets(List<SAMRecord> reads) { protected Set<List<SAMRecord>> uniqueReadSets(List<SAMRecord> reads) {
Set<List<SAMRecord>> readSets = new HashSet<List<SAMRecord>>(); Set<List<SAMRecord>> readSets = new HashSet<List<SAMRecord>>();
// for each read, find duplicates, and either add the read to its duplicate list or start a new one
for ( SAMRecord read : reads ) { for ( SAMRecord read : reads ) {
List<SAMRecord> readSet = findDuplicateReads(read, readSets); List<SAMRecord> readSet = findDuplicateReads(read, readSets);
if ( readSet == null ) { if ( readSet == null ) {
readSets.add(new ArrayList<SAMRecord>(Arrays.asList(read))); // copy so I can add to the list readSets.add(new ArrayList<SAMRecord>(Arrays.asList(read))); // copy so I can add to the list
} else { } else {
@ -100,6 +108,15 @@ public class TraverseDuplicates extends TraversalEngine {
return readSets; return readSets;
} }
/**
* Find duplicate reads for read in the set of unique reads. This is effective a duplicate marking algorithm,
* but it relies for safety's sake on the file itself being marked by a true duplicate marking algorithm. Pair
* and single-end read aware.
*
* @param read
* @param readSets
* @return The list of duplicate reads that read is a member of, or null if it's the only one of its kind
*/
protected List<SAMRecord> findDuplicateReads(SAMRecord read, Set<List<SAMRecord>> readSets ) { protected List<SAMRecord> findDuplicateReads(SAMRecord read, Set<List<SAMRecord>> readSets ) {
if ( read.getReadPairedFlag() ) { if ( read.getReadPairedFlag() ) {
// paired // paired
@ -107,8 +124,6 @@ public class TraverseDuplicates extends TraversalEngine {
for (List<SAMRecord> reads : readSets) { for (List<SAMRecord> reads : readSets) {
SAMRecord key = reads.get(0); SAMRecord key = reads.get(0);
//if (DEBUG)
// logger.debug(String.format("Examining reads at %s vs. %s at %s / %s vs. %s / %s%n", key.getReadName(), read.getReadName(), keyLoc, keyMateLoc, readLoc, readMateLoc));
// read and key start at the same place, and either the this read and the key // read and key start at the same place, and either the this read and the key
// share a mate location or the read is flagged as a duplicate // share a mate location or the read is flagged as a duplicate
@ -191,19 +206,11 @@ public class TraverseDuplicates extends TraversalEngine {
Shard shard, Shard shard,
ShardDataProvider dataProvider, ShardDataProvider dataProvider,
T sum) { T sum) {
//logger.debug(String.format("TraverseDuplicates.traverse Genomic interval is %s", shard.getGenomeLoc())); // safety first :-)
if (!(walker instanceof DuplicateWalker)) if (!(walker instanceof DuplicateWalker))
throw new IllegalArgumentException("Walker isn't a duplicate walker!"); throw new IllegalArgumentException("Walker isn't a duplicate walker!");
DuplicateWalker<M, T> dupWalker = (DuplicateWalker<M, T>) walker; DuplicateWalker<M, T> dupWalker = (DuplicateWalker<M, T>) walker;
// while we still have more reads
// ok, here's the idea. We get all the reads that start at the same position in the genome
// We then split the list of reads into sublists of reads:
// -> those with the same mate pair position, for paired reads
// -> those flagged as unpaired and duplicated but having the same start and end and
FilteringIterator filterIter = new FilteringIterator(new ReadView(dataProvider).iterator(), new duplicateStreamFilterFunc()); FilteringIterator filterIter = new FilteringIterator(new ReadView(dataProvider).iterator(), new duplicateStreamFilterFunc());
PushbackIterator<SAMRecord> iter = new PushbackIterator<SAMRecord>(filterIter); PushbackIterator<SAMRecord> iter = new PushbackIterator<SAMRecord>(filterIter);
@ -219,7 +226,7 @@ public class TraverseDuplicates extends TraversalEngine {
GenomeLoc site = GenomeLocParser.createGenomeLoc(read); GenomeLoc site = GenomeLocParser.createGenomeLoc(read);
Set<List<SAMRecord>> readSets = uniqueReadSets(readsAtLoc(read, iter)); Set<List<SAMRecord>> readSets = uniqueReadSets(readsAtLoc(read, iter));
logger.debug(String.format("*** TraverseDuplicates.traverse at %s with %d read sets", site, readSets.size())); if ( DEBUG ) logger.debug(String.format("*** TraverseDuplicates.traverse at %s with %d read sets", site, readSets.size()));
// Jump forward in the reference to this locus location // Jump forward in the reference to this locus location
AlignmentContext locus = new AlignmentContext(site, new ReadBackedPileup(site)); AlignmentContext locus = new AlignmentContext(site, new ReadBackedPileup(site));
@ -227,6 +234,7 @@ public class TraverseDuplicates extends TraversalEngine {
// update the number of duplicate sets we've seen // update the number of duplicate sets we've seen
TraversalStatistics.nRecords++; TraversalStatistics.nRecords++;
// actually call filter and map, accumulating sum
final boolean keepMeP = dupWalker.filter(site, locus, readSets); final boolean keepMeP = dupWalker.filter(site, locus, readSets);
if (keepMeP) { if (keepMeP) {
M x = dupWalker.map(site, locus, readSets); M x = dupWalker.map(site, locus, readSets);

15
java/src/org/broadinstitute/sting/gatk/walkers/DuplicateWalker.java
View File

@ -22,21 +22,6 @@ public abstract class DuplicateWalker<MapType, ReduceType> extends Walker<MapTyp
return true; // We are keeping all the reads return true; // We are keeping all the reads
} }
/**
* Called by the traversal engine to decide whether to send non-duplicates as lists of
* singleton reads to the map function. By default it's false.
*
* @return true if you want to see non duplicates during the traversal
*/
public boolean mapUniqueReadsTooP() { return false; }
/**
* Called by the traversal engine to decide whether to call map() at loci without duplicate reads
*
* @return true if you want to see non duplicates during the traversal
*/
public boolean mapAtLociWithoutDuplicates() { return true; }
public abstract MapType map(GenomeLoc loc, AlignmentContext context, Set<List<SAMRecord>> readSets ); public abstract MapType map(GenomeLoc loc, AlignmentContext context, Set<List<SAMRecord>> readSets );
// Given result of map function // Given result of map function

244
java/src/org/broadinstitute/sting/oneoffprojects/walkers/DuplicateQualsWalker.java
View File

@ -1,244 +0,0 @@
/*
* Copyright (c) 2009 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.oneoffprojects.walkers;
import net.sf.samtools.SAMRecord;
import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
import org.broadinstitute.sting.gatk.walkers.DuplicateWalker;
import org.broadinstitute.sting.utils.BaseUtils;
import org.broadinstitute.sting.utils.GenomeLoc;
import org.broadinstitute.sting.utils.Pair;
import org.broadinstitute.sting.utils.QualityUtils;
import org.broadinstitute.sting.utils.cmdLine.Argument;
import org.broadinstitute.sting.utils.duplicates.DupUtils;
import org.broadinstitute.sting.utils.duplicates.DuplicateComp;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
class MismatchCounter {
long nObs = 0;
long nMismatches = 0;
public void inc(long incNObs, long incNMismatches) {
nObs += incNObs;
nMismatches += incNMismatches;
}
public void inc(boolean mismatchP) {
inc(1, mismatchP ? 1 : 0);
}
public double mismatchRate() {
return (double)nMismatches / nObs;
}
public byte empiricalQualScore() {
return QualityUtils.probToQual(1 - mismatchRate(), 0);
}
public String headerString() {
return "mismatchRate\tempiricalQ\tnObs\tnMismatches";
}
public String toString() {
return String.format("%.10f\t%d\t%d\t%6d", mismatchRate(), empiricalQualScore(), nObs, nMismatches);
}
}
class QualityTracker {
final private int MAX_QUAL_SCORE = 100;
MismatchCounter[][] mismatchesByQ = new MismatchCounter[MAX_QUAL_SCORE][MAX_QUAL_SCORE];
public QualityTracker() {
for ( int i = 0; i < MAX_QUAL_SCORE; i++ ) {
for ( int j = 0; j < MAX_QUAL_SCORE; j++ ) {
mismatchesByQ[i][j] = new MismatchCounter();
}
}
}
public void inc(int b1Qi, int b2Qi, boolean mismatchP, boolean orderDependent) {
int b1Q = orderDependent ? b1Qi : Math.max(b1Qi, b2Qi);
int b2Q = orderDependent ? b2Qi : Math.min(b1Qi, b2Qi);
if ( b1Q > MAX_QUAL_SCORE ) throw new RuntimeException("Unexpectedly large base quality " + b1Q);
if ( b2Q > MAX_QUAL_SCORE ) throw new RuntimeException("Unexpectedly large base quality " + b2Q);
mismatchesByQ[b1Q][b2Q].inc(mismatchP);
}
public void inc(DuplicateComp dc, boolean orderDependent) {
inc(dc.getQLarger(), dc.getQSmaller(), dc.isMismatchP(), orderDependent);
}
public int probMismatchQ1Q2(int q1, int q2) {
double e1 = 1 - QualityUtils.qualToProb(q1);
double e2 = 1 - QualityUtils.qualToProb(q2);
double eMM = e1 * (1 - e2) + (1 - e1) * e2 - 1/3 * e1 * e2;
return QualityUtils.probToQual(1 - eMM, 0.0);
}
public void printToStream(PrintStream out, boolean filterUnobserved) {
out.printf("Q1\tQ2\tQmin\t%s%n", mismatchesByQ[0][0].headerString());
for ( int i = 0; i < MAX_QUAL_SCORE; i++ ) {
for ( int j = 0; j < MAX_QUAL_SCORE; j++ ) {
MismatchCounter mc = mismatchesByQ[i][j];
//System.out.printf("MC = %s%n", mc);
if ( filterUnobserved && mc.nObs == 0 )
continue;
out.printf("%d\t%d\t%d\t%s\t%n", i, j, probMismatchQ1Q2(i,j), mc.toString());
}
}
}
}
public class DuplicateQualsWalker extends DuplicateWalker<List<DuplicateComp>, QualityTracker> {
@Argument(fullName="filterUnobservedQuals", required=false, doc="Show only quality bins with at least one observation in the data")
public boolean FILTER_UNOBSERVED_QUALS = false;
@Argument(fullName="maxPairwiseCompsPerDupSet", required=false, doc="Maximumize number of pairwise comparisons to perform among duplicate read sets")
public int MAX_PAIRSIZE_COMPS_PER_DUPLICATE_SET = 100;
@Argument(fullName="combinedQuals", required=false, doc="Combine and assess pairwise base qualities")
public boolean COMBINE_QUALS = false;
@Argument(fullName="combineAllDups", required=false, doc="Combine and assess pairwise base qualities")
public boolean COMBINE_ALL_DUPS = false;
@Argument(fullName="orderDependent", required=false, doc="")
public boolean orderDependent = false;
@Argument(fullName="compareToUniqueReads", required=false, doc="If true, then we will compare only to unique (i.e., non-duplicated molecules) at the same duplicate site")
public boolean compareToUniqueReads = false;
@Argument(fullName="comparePairToSingleton", required=false, doc="If true, then we will compare a combined dup to a random other read in the duplicate set, not a combined pair itself")
public boolean comparePairToSingleton = false;
final boolean DEBUG = false;
final private boolean ACTUALLY_DO_WORK = true;
public void onTraversalDone(QualityTracker result) {
result.printToStream(out, FILTER_UNOBSERVED_QUALS);
}
public QualityTracker reduceInit() {
return new QualityTracker();
}
public QualityTracker reduce(List<DuplicateComp> dupComps, QualityTracker tracker) {
for ( DuplicateComp dc : dupComps ) {
tracker.inc(dc, orderDependent);
}
return tracker;
}
// Print out data for regression
public List<DuplicateComp> map(GenomeLoc loc, AlignmentContext context, Set<List<SAMRecord>> readSets ) {
//logger.info(String.format("%s has %d duplicates and %d non-duplicates", loc, duplicateReads.size(), uniqueReads.size()));
List<DuplicateComp> pairwiseComps = new ArrayList<DuplicateComp>();
// todo -- fixme -- the logic here is all wrong given new interface
// if ( ! ACTUALLY_DO_WORK )
// return pairwiseComps;
//
// if ( COMBINE_QUALS ) {
// Pair<SAMRecord, SAMRecord> combinedReads = DupUtils.combinedReadPair( duplicateReads );
// if ( combinedReads != null ) {
// SAMRecord combined1 = combinedReads.first;
// SAMRecord combined2 = combinedReads.second;
//
// if ( comparePairToSingleton )
// pairwiseComps = addPairwiseMatches( pairwiseComps, combined1, duplicateReads.get(2), uniqueReads );
// else
// pairwiseComps = addPairwiseMatches( pairwiseComps, combined1, combined2, uniqueReads );
// }
// } else {
// int nComparisons = 0;
// for ( SAMRecord read1 : duplicateReads ) {
// for ( SAMRecord read2 : duplicateReads ) {
// if ( read1.hashCode() < read2.hashCode() && DupUtils.usableDuplicate(read1, read2) ) {
// // the hashcode insures we don't do A vs. B and B vs. A
// //System.out.printf("Comparing %s against %s%n", read1, read2);
// nComparisons++;
// pairwiseComps = addPairwiseMatches( pairwiseComps, read1, read2, uniqueReads );
// if ( nComparisons > MAX_PAIRSIZE_COMPS_PER_DUPLICATE_SET )
// break;
// }
// }
// }
// }
return pairwiseComps;
}
private List<DuplicateComp> addPairwiseMatches(List<DuplicateComp> comps,
SAMRecord read1, SAMRecord read2,
List<SAMRecord> uniqueReads ) {
if ( compareToUniqueReads ) {
// we want to compare to a read in the unique read set
if ( uniqueReads.size() > 0 ) { // there's actually something to compare to
SAMRecord uniqueRead = uniqueReads.get(0); // might as well get the first one
return pairwiseMatches(comps, read1, uniqueRead);
} else {
return comps;
}
} else {
// default, just do read1 vs. read2
return pairwiseMatches(comps, read1, read2);
}
}
/**
* Calculates the pairwise mismatches between reads read1 and read2 and adds the result to the comps list.
* Doesn't contain any logic deciding what to compare, just does read1 and read2
*
* @param comps
* @param read1
* @param read2
* @return
*/
private List<DuplicateComp> pairwiseMatches(List<DuplicateComp> comps, SAMRecord read1, SAMRecord read2 ) {
byte[] read1Bases = read1.getReadBases();
byte[] read1Quals = read1.getBaseQualities();
byte[] read2Bases = read2.getReadBases();
byte[] read2Quals = read2.getBaseQualities();
for ( int i = 0; i < read1Bases.length; i++) {
byte qual1 = read1Quals[i];
byte qual2 = read2Quals[i];
boolean mismatchP = ! BaseUtils.basesAreEqual(read1Bases[i], read2Bases[i]);
DuplicateComp dc = new DuplicateComp(qual1, qual2, mismatchP);
comps.add(dc);
}
return comps;
}
}

29
java/src/org/broadinstitute/sting/playground/gatk/walkers/duplicates/CombineDuplicatesWalker.java
View File

@ -21,21 +21,10 @@ public class CombineDuplicatesWalker extends DuplicateWalker<List<SAMRecord>, SA
@Argument(fullName="outputBAM", shortName="outputBAM", required=false, doc="BAM File to write combined duplicates to") @Argument(fullName="outputBAM", shortName="outputBAM", required=false, doc="BAM File to write combined duplicates to")
public SAMFileWriter outputBAM = null; public SAMFileWriter outputBAM = null;
@Argument(fullName="includeUniqueReads", shortName="includeUniqueReads", required=false, doc="If true, also writes out non-duplicate reads in file")
public boolean INCLUDE_UNIQUE_READS = true;
@Argument(fullName="maxQ", shortName="maxQ", required=false, @Argument(fullName="maxQ", shortName="maxQ", required=false,
doc="The maximum Q score allowed for combined reads, reflects the background error rate giving rise to perfect bases that don't correspond to the reference") doc="The maximum Q score allowed for combined reads, reflects the background error rate giving rise to perfect bases that don't correspond to the reference")
public int MAX_QUALITY_SCORE = 50; public int MAX_QUALITY_SCORE = 50;
/**
* do we want to include unqiue reads?
* @return the user specified command line argument INCLUDE_UNIQUE_READS
*/
public boolean mapUniqueReadsTooP() {
return INCLUDE_UNIQUE_READS;
}
/** /**
* start the walker with the command line argument specified SAMFileWriter * start the walker with the command line argument specified SAMFileWriter
* @return a sam file writer, which may be null * @return a sam file writer, which may be null
@ -59,11 +48,15 @@ public class CombineDuplicatesWalker extends DuplicateWalker<List<SAMRecord>, SA
return output; return output;
} }
/** /**
* We don't want to see loci without duplicates, since * when we're done, print out the collected stats
* @return * @param result the result of the traversal engine, to be printed out
*/ */
public boolean mapAtLociWithoutDuplicates() { return false; } public void onTraversalDone(SAMFileWriter result) {
return; // don't do anything
}
/** /**
* Build a combined read given the input list of non-unique reads. If there's just one read in the * Build a combined read given the input list of non-unique reads. If there's just one read in the
@ -86,10 +79,12 @@ public class CombineDuplicatesWalker extends DuplicateWalker<List<SAMRecord>, SA
combinedRead = reads.get(0); combinedRead = reads.get(0);
} else { } else {
// actually call the combine function // actually call the combine function
//for (SAMRecord read : duplicateReads ) { // for (SAMRecord read : reads ) {
// System.out.printf("Read %s%n", read.format()); // out.printf("Combining Read %s%n", read.format());
//} // }
//
combinedRead = DupUtils.combineDuplicates(reads, MAX_QUALITY_SCORE); combinedRead = DupUtils.combineDuplicates(reads, MAX_QUALITY_SCORE);
//out.printf(" => into %s%n", combinedRead.format());
} }
if ( combinedRead.getDuplicateReadFlag() ) if ( combinedRead.getDuplicateReadFlag() )

2
java/src/org/broadinstitute/sting/playground/gatk/walkers/duplicates/CountDuplicatesWalker.java
View File

@ -48,7 +48,7 @@ class DuplicateCount {
/** /**
* Count the number of unique reads, duplicates, and the average depth of unique reads and duplicates at all positions. * Count the number of unique reads, duplicates, and the average depth of unique reads and duplicates at all positions.
* @author aaron * @author mark DePristo
*/ */
public class CountDuplicatesWalker extends DuplicateWalker<DuplicateCount, DuplicateCount> { public class CountDuplicatesWalker extends DuplicateWalker<DuplicateCount, DuplicateCount> {
@Argument(fullName="quiet", required=false, doc="If true, per locus information isn't printex") @Argument(fullName="quiet", required=false, doc="If true, per locus information isn't printex")

212
java/src/org/broadinstitute/sting/utils/duplicates/DupUtils.java
View File

@ -2,109 +2,21 @@ package org.broadinstitute.sting.utils.duplicates;
import net.sf.samtools.SAMRecord; import net.sf.samtools.SAMRecord;
import org.broadinstitute.sting.utils.*; import org.broadinstitute.sting.utils.*;
import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
import org.broadinstitute.sting.utils.pileup.PileupElement;
import org.broadinstitute.sting.oneoffprojects.multisamplecaller.BasicPileup;
import java.util.List; import java.util.List;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
public class DupUtils { public class DupUtils {
public static boolean usableDuplicate( SAMRecord read1, SAMRecord read2 ) { private static SAMRecord tmpCopyRead(SAMRecord read) {
return read1 != read2 && read1.getReadLength() == read2.getReadLength(); try {
} return (SAMRecord)read.clone();
} catch ( CloneNotSupportedException e ) {
public static Pair<SAMRecord, SAMRecord> combinedReadPair( List<SAMRecord> duplicateReads ) { throw new StingException("Unexpected Clone failure!");
if ( duplicateReads.size() < 4 )
return null;
SAMRecord c1 = combine2Duplicates(duplicateReads.get(0),duplicateReads.get(1));
SAMRecord c2 = combine2Duplicates(duplicateReads.get(2),duplicateReads.get(3));
return new Pair<SAMRecord, SAMRecord>(c1, c2);
}
public static SAMRecord sample3rdRead( List<SAMRecord> duplicateReads, SAMRecord read1, SAMRecord read2 ) {
if ( duplicateReads.size() <= 2 ) {
// no third unique read is available
return null;
} else {
for ( SAMRecord read3 : duplicateReads ) {
if ( usableDuplicate(read1, read3) && usableDuplicate(read2, read3) )
return read3;
}
return null;
}
}
public static SAMRecord tmpCopyRead(SAMRecord read) {
SAMRecord copy = new SAMRecord(read.getHeader());
copy.setReadName(read.getReadName());
//copy.setReadString(final String value) {
copy.setReadBases(read.getReadBases());
copy.setBaseQualities(read.getBaseQualities());
copy.setReferenceName(read.getReferenceName());
copy.setReferenceIndex(read.getReferenceIndex());
copy.setMateReferenceName(read.getMateReferenceName());
copy.setMateReferenceIndex(read.getMateReferenceIndex());
copy.setAlignmentStart(read.getAlignmentStart());
//copy.setAlignmentEnd(read.getAlignmentEnd());
copy.setMateAlignmentStart(read.getMateAlignmentStart());
copy.setInferredInsertSize(read.getInferredInsertSize());
copy.setMappingQuality(read.getMappingQuality());
copy.setCigar(read.getCigar());
copy.setFlags(read.getFlags());
return copy;
}
public static SAMRecord combine2Duplicates(SAMRecord read1, SAMRecord read2) {
byte[] read1Bases = read1.getReadBases();
byte[] read1Quals = read1.getBaseQualities();
byte[] read2Bases = read2.getReadBases();
byte[] read2Quals = read2.getBaseQualities();
byte[] bases = new byte[read1Bases.length];
byte[] quals = new byte[read1Bases.length];
SAMRecord c = tmpCopyRead(read1);
for ( int i = 0; i < read1Bases.length; i++) {
byte base1 = read1Bases[i];
byte base2 = read2Bases[i];
byte qual1 = read1Quals[i];
byte qual2 = read2Quals[i];
Pair<Byte, Integer> combined = combine2BasesAndQuals(base1, base2, qual1, qual2);
bases[i] = combined.getFirst();
quals[i] = QualityUtils.boundQual(combined.getSecond());
// if ( DEBUG )
// logger.debug(String.format("Combining %s (Q%2d) with %s (Q%2d) -> %s (Q%2d)%s%n",
// (char)base1, qual1, (char)base2, qual2, (char)bases[i], quals[i],
// base1 == base2 ? "" : " [MISMATCH]"));
} }
c.setReadBases(bases);
c.setBaseQualities(quals);
return c;
}
public static Pair<Byte, Integer> combine2BasesAndQuals(byte base1, byte base2, int qual1, int qual2) {
byte cbase;
int cqual;
if ( base1 == base2 ) {
// agreement
cbase = base1;
cqual = qual1 + qual2;
} else {
// disagreement
cbase = qual1 > qual2 ? base1 : base2;
//cqual = Math.max(qual1, qual2);
cqual = Math.max(qual1, qual2) - Math.min(qual1, qual2);
}
return new Pair<Byte, Integer>(cbase, cqual);
} }
public static SAMRecord combineDuplicates(List<SAMRecord> duplicates, int maxQScore) { public static SAMRecord combineDuplicates(List<SAMRecord> duplicates, int maxQScore) {
@ -114,6 +26,7 @@ public class DupUtils {
// make the combined read by copying the first read and setting the // make the combined read by copying the first read and setting the
// bases and quals to new arrays // bases and quals to new arrays
SAMRecord comb = tmpCopyRead(duplicates.get(0)); SAMRecord comb = tmpCopyRead(duplicates.get(0));
//SAMRecord comb = tmpCopyRead(duplicates.get(0));
comb.setDuplicateReadFlag(false); comb.setDuplicateReadFlag(false);
int readLen = comb.getReadBases().length; int readLen = comb.getReadBases().length;
byte[] bases = new byte[readLen]; byte[] bases = new byte[readLen];
@ -125,8 +38,6 @@ public class DupUtils {
// System.out.printf("dup base %c %d%n", (char)read.getReadBases()[i], read.getBaseQualities()[i]); // System.out.printf("dup base %c %d%n", (char)read.getReadBases()[i], read.getBaseQualities()[i]);
//} //}
Pair<Byte, Byte> baseAndQual = combineBaseProbs(duplicates, i, maxQScore); Pair<Byte, Byte> baseAndQual = combineBaseProbs(duplicates, i, maxQScore);
// baseAndQual = combineBasesByConsensus(duplicates, i);
// baseAndQual = combineDupBasesAtI(duplicates, i);
bases[i] = baseAndQual.getFirst(); bases[i] = baseAndQual.getFirst();
quals[i] = baseAndQual.getSecond(); quals[i] = baseAndQual.getSecond();
} }
@ -142,6 +53,7 @@ public class DupUtils {
char bestBase = 0; char bestBase = 0;
double bestProb = Double.NEGATIVE_INFINITY; double bestProb = Double.NEGATIVE_INFINITY;
double sumProbs = 0; double sumProbs = 0;
for ( int i = 0; i < 4; i++ ) { for ( int i = 0; i < 4; i++ ) {
sumProbs += Math.pow(10, probs[i]); sumProbs += Math.pow(10, probs[i]);
//System.out.printf("Bestprob is %f > %f%n", bestProb, probs[i]); //System.out.printf("Bestprob is %f > %f%n", bestProb, probs[i]);
@ -150,20 +62,20 @@ public class DupUtils {
bestProb = probs[i]; bestProb = probs[i];
} }
} }
Arrays.sort(probs); Arrays.sort(probs);
double normalizedP = Math.pow(10, bestProb) / sumProbs; double normalizedP = Math.pow(10, bestProb) / sumProbs;
double normalizedQ = 1 - normalizedP;
double eps = Math.pow(10, -maxQScore/10.0); double eps = Math.pow(10, -maxQScore/10.0);
byte qual = QualityUtils.probToQual(normalizedP, eps); byte qual = QualityUtils.probToQual(normalizedP, eps);
if ( false ) { // if ( false ) {
System.out.printf("Best base is %s %.8f%n", bestBase, bestProb); // System.out.printf("Best base is %s %.8f%n", bestBase, bestProb);
System.out.printf("2nd base is %.8f%n", probs[1]); // System.out.printf("2nd base is %.8f%n", probs[1]);
System.out.printf("normalized P %.8f%n", normalizedP); // System.out.printf("normalized P %.8f%n", normalizedP);
System.out.printf("normalized Q %.8f%n", 1 - normalizedP); // System.out.printf("normalized Q %.8f%n", 1 - normalizedP);
System.out.printf("max Q %2d%n", maxQScore); // System.out.printf("max Q %2d%n", maxQScore);
System.out.printf("eps %.8f%n", eps); // System.out.printf("eps %.8f%n", eps);
System.out.printf("encoded Q %2d%n", qual); // System.out.printf("encoded Q %2d%n", qual);
} // }
return new Pair<Byte, Byte>((byte)bestBase, qual); return new Pair<Byte, Byte>((byte)bestBase, qual);
} }
@ -176,93 +88,31 @@ public class DupUtils {
System.out.printf("%n"); System.out.printf("%n");
} }
private static List<Integer> constantOffset( List<SAMRecord> reads, int i ) {
List<Integer> l = new ArrayList<Integer>(reads.size());
for ( SAMRecord read : reads ) {
l.add(i);
}
return l;
}
// TODO -- get rid of all this crappy, obsolete pileup code
@Deprecated
private static ArrayList<Byte> getBasesAsArrayList( List<SAMRecord> reads, List<Integer> offsets ) {
ArrayList<Byte> bases = new ArrayList<Byte>(reads.size());
for (byte value : getBasesAsArray(reads, offsets))
bases.add(value);
return bases;
}
@Deprecated
private static ArrayList<Byte> getQualsAsArrayList( List<SAMRecord> reads, List<Integer> offsets ) {
ArrayList<Byte> quals = new ArrayList<Byte>(reads.size());
for (byte value : getQualsAsArray(reads, offsets))
quals.add(value);
return quals;
}
@Deprecated
public static byte[] getBasesAsArray( List<SAMRecord> reads, List<Integer> offsets ) {
byte array[] = new byte[reads.size()];
int index = 0;
for ( int i = 0; i < reads.size(); i++ ) {
SAMRecord read = reads.get(i);
int offset = offsets.get(i);
if ( offset == -1 ) {
array[index++] = ((byte)'D');
} else {
array[index++] = read.getReadBases()[offset];
}
}
return array;
}
@Deprecated
private static byte[] getQualsAsArray( List<SAMRecord> reads, List<Integer> offsets ) {
byte array[] = new byte[reads.size()];
int index = 0;
for ( int i = 0; i < reads.size(); i++ ) {
SAMRecord read = reads.get(i);
int offset = offsets.get(i);
// skip deletion sites
if ( offset == -1 ) {
array[index++] = ((byte)0);
} else {
array[index++] = read.getBaseQualities()[offset];
}
}
return array;
}
private static Pair<Byte, Byte> combineBaseProbs(List<SAMRecord> duplicates, int readOffset, int maxQScore) { private static Pair<Byte, Byte> combineBaseProbs(List<SAMRecord> duplicates, int readOffset, int maxQScore) {
List<Integer> offsets = constantOffset(duplicates, readOffset); GenomeLoc loc = GenomeLocParser.createGenomeLoc(duplicates.get(0));
ArrayList<Byte> bases = getBasesAsArrayList(duplicates, offsets); ReadBackedPileup pileup = new ReadBackedPileup(loc, duplicates, readOffset);
ArrayList<Byte> quals = getQualsAsArrayList(duplicates, offsets);
final boolean debug = false; final boolean debug = false;
// calculate base probs // calculate base probs
double[] qualSums = {0.0, 0.0, 0.0, 0.0}; double[] qualSums = {0.0, 0.0, 0.0, 0.0};
if ( debug ) print4BaseQuals("start", qualSums); if ( debug ) print4BaseQuals("start", qualSums);
for ( int i = 0; i < bases.size(); i++ ) {
char base = (char)(byte)bases.get(i); for (PileupElement e : pileup ) {
int baseIndex = BaseUtils.simpleBaseToBaseIndex(base); int baseIndex = e.getBaseIndex();
byte qual = quals.get(i); byte qual = e.getQual();
double pqual = QualityUtils.qualToProb(qual); double pqual = QualityUtils.qualToProb(qual);
for ( int j = 0; j < 4; j++) { for ( int j = 0; j < 4; j++) {
qualSums[j] += Math.log10(j == baseIndex ? pqual : (1 - pqual)/3); qualSums[j] += Math.log10(j == baseIndex ? pqual : (1 - pqual)/3);
} }
if ( debug ) print4BaseQuals(String.format("%c Q%2d", base, qual), qualSums);
if ( debug ) print4BaseQuals(String.format("%c Q%2d", e.getBase(), qual), qualSums);
} }
if ( debug ) print4BaseQuals("final", qualSums); if ( debug ) print4BaseQuals("final", qualSums);
Pair<Byte, Byte> combined = baseProbs2BaseAndQual(qualSums, maxQScore); Pair<Byte, Byte> combined = baseProbs2BaseAndQual(qualSums, maxQScore);
// if ( debug ) if ( debug ) System.out.printf("%s => %c Q%s%n", pileup.getPileupString('N'), (char)(byte)combined.getFirst(), combined.getSecond());
// System.out.printf("%s %s => %c Q%s%n",
// BasicPileup.basePileupAsString(duplicates, offsets),
// BasicPileup.qualPileupAsString(duplicates, offsets),
// (char)(byte)combined.getFirst(), combined.getSecond());
return combined; return combined;
} }
} }

23
java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileup.java
View File

@ -39,6 +39,10 @@ public class ReadBackedPileup implements Iterable<PileupElement> {
this(loc, readsOffsets2Pileup(reads, offsets)); this(loc, readsOffsets2Pileup(reads, offsets));
} }
public ReadBackedPileup(GenomeLoc loc, List<SAMRecord> reads, int offset ) {
this(loc, readsOffsets2Pileup(reads, offset));
}
/** /**
* Create a new version of a read backed pileup at loc without any aligned reads * Create a new version of a read backed pileup at loc without any aligned reads
* *
@ -122,6 +126,25 @@ public class ReadBackedPileup implements Iterable<PileupElement> {
return pileup; return pileup;
} }
/**
* Helper routine for converting reads and a single offset to a PileupElement list.
*
* @param reads
* @param offset
* @return
*/
private static ArrayList<PileupElement> readsOffsets2Pileup(List<SAMRecord> reads, int offset ) {
if ( reads == null ) throw new StingException("Illegal null read list in ReadBackedPileup2");
if ( offset < 0 ) throw new StingException("Illegal offset < 0 ReadBackedPileup2");
ArrayList<PileupElement> pileup = new ArrayList<PileupElement>(reads.size());
for ( int i = 0; i < reads.size(); i++ ) {
pileup.add( new PileupElement( reads.get(i), offset ) );
}
return pileup;
}
// -------------------------------------------------------- // --------------------------------------------------------
// //
// Special 'constructors' // Special 'constructors'

21
java/test/org/broadinstitute/sting/playground/gatk/walkers/duplicates/DuplicatesWalkersIntegrationTest.java
View File

@ -8,7 +8,7 @@ import java.util.Arrays;
import java.util.List; import java.util.List;
public class DuplicatesWalkersIntegrationTest extends WalkerTest { public class DuplicatesWalkersIntegrationTest extends WalkerTest {
public void testClipper(String name, String args, String md5) { public void testCounter(String name, String args, String md5) {
WalkerTestSpec spec = new WalkerTestSpec( WalkerTestSpec spec = new WalkerTestSpec(
"-T CountDuplicates " + "-T CountDuplicates " +
"-R /seq/references/Homo_sapiens_assembly18/v0/Homo_sapiens_assembly18.fasta " + "-R /seq/references/Homo_sapiens_assembly18/v0/Homo_sapiens_assembly18.fasta " +
@ -20,6 +20,21 @@ public class DuplicatesWalkersIntegrationTest extends WalkerTest {
List<File> result = executeTest(name, spec).getFirst(); List<File> result = executeTest(name, spec).getFirst();
} }
@Test public void testChr110Mb() { testClipper("testChr1-10mb", "-L chr1:1-10,000,000 --quiet", ""); } @Test public void testChr110Mb() { testCounter("testChr1-10mb", "-L chr1:1-10,000,000 --quiet", "fa8bfdd0b62a13a543bae90f7c674db7"); }
@Test public void testIntervalVerbose() { testClipper("testIntervalVerbose", "-L chr1:6,527,154-6,528,292", ""); } @Test public void testIntervalVerbose() { testCounter("testIntervalVerbose", "-L chr1:6,527,154-6,528,292", "1ebcc10b85af16805a54391721776657"); }
public void testCombiner(String name, String args, String md51, String md52) {
WalkerTestSpec spec = new WalkerTestSpec(
"-T CombineDuplicates " +
"-R /seq/references/Homo_sapiens_assembly18/v0/Homo_sapiens_assembly18.fasta " +
"-I /humgen/gsa-hpprojects/GATK/data/Validation_Data/TCGA-06-0188.aligned.duplicates_marked.bam " +
"-o %s --outputBAM %s " + args,
2, // just one output file
Arrays.asList("tmp", "bam"),
Arrays.asList(md51, md52));
List<File> result = executeTest(name, spec).getFirst();
}
@Test public void testIntervalCombine() { testCombiner("testIntervalCombine", "-L chr1:6,527,154-6,528,292 -maxQ 50", "d41d8cd98f00b204e9800998ecf8427e", "bbde777437fcf4386687e0d11547d0f3"); }
@Test public void testIntervalCombineQ60() { testCombiner("testIntervalCombine", "-L chr1:6,527,154-6,528,292 -maxQ 60", "d41d8cd98f00b204e9800998ecf8427e", "93993440eb0455208c7f9881d1115a3c"); }
} }