From 54121eb082eb84920770d573ff143efbe32f0340 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Wed, 6 Jul 2011 16:05:08 -0400 Subject: [PATCH 01/10] Catch malformed bams that cause the writer to run in infinite loops --- .../gatk/walkers/indels/ConstrainedMateFixingManager.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/ConstrainedMateFixingManager.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/ConstrainedMateFixingManager.java index aebb0e3eb..df1f4f908 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/ConstrainedMateFixingManager.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/ConstrainedMateFixingManager.java @@ -5,6 +5,7 @@ import net.sf.samtools.*; import org.apache.log4j.Logger; import org.broadinstitute.sting.utils.GenomeLoc; import org.broadinstitute.sting.utils.GenomeLocParser; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; import java.util.*; @@ -113,9 +114,10 @@ public class ConstrainedMateFixingManager { HashMap forMateMatching = new HashMap(); TreeSet waitingReads = new TreeSet(comparer); - private T remove(TreeSet treeSet) { - final T first = treeSet.first(); - treeSet.remove(first); + private SAMRecord remove(TreeSet treeSet) { + final SAMRecord first = treeSet.first(); + if ( !treeSet.remove(first) ) + throw new UserException("Error caching SAM record " + first.getReadName() + ", which is usually caused by malformed SAM/BAM files in which multiple identical copies of a read are present."); return first; } From 50111db2b7920ba5b3cfe659e33f7135ebd277f5 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Thu, 7 Jul 2011 15:02:48 -0400 Subject: [PATCH 03/10] Fixing non-determinism in single-threaded VQSR by moving references to cern.Normal over to the static random generator available in GenomeAnalysisEngine --- .../GaussianMixtureModel.java | 24 ++++++++++--------- .../VariantDataManager.java | 3 +-- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/GaussianMixtureModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/GaussianMixtureModel.java index acc1f24cc..41fea0896 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/GaussianMixtureModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/GaussianMixtureModel.java @@ -26,7 +26,6 @@ package org.broadinstitute.sting.gatk.walkers.variantrecalibration; import Jama.Matrix; -import cern.jet.random.Normal; import org.apache.log4j.Logger; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.utils.MathUtils; @@ -97,7 +96,7 @@ public class GaussianMixtureModel { int ttt = 0; while( ttt++ < numIterations ) { - // Estep: assign each variant to the nearest cluster + // E step: assign each variant to the nearest cluster for( final VariantDatum datum : data ) { double minDistance = Double.MAX_VALUE; MultivariateGaussian minGaussian = null; @@ -112,7 +111,7 @@ public class GaussianMixtureModel { datum.assignment = minGaussian; } - // Mstep: update gaussian means based on assigned variants + // M step: update gaussian means based on assigned variants for( final MultivariateGaussian gaussian : gaussians ) { gaussian.zeroOutMu(); int numAssigned = 0; @@ -216,26 +215,29 @@ public class GaussianMixtureModel { } public double evaluateDatumMarginalized( final VariantDatum datum ) { - int numVals = 0; + int numSamples = 0; double sumPVarInGaussian = 0.0; - int numIter = 10; + final int numIterPerMissingAnnotation = 10; // Trade off here between speed of computation and accuracy of the marginalization final double[] pVarInGaussianLog10 = new double[gaussians.size()]; + // for each dimension for( int iii = 0; iii < datum.annotations.length; iii++ ) { - // marginalize over the missing dimension by drawing X random values for the missing annotation and averaging the lod + // if it is missing marginalize over the missing dimension by drawing X random values for the missing annotation and averaging the lod if( datum.isNull[iii] ) { - for( int ttt = 0; ttt < numIter; ttt++ ) { - datum.annotations[iii] = Normal.staticNextDouble(0.0, 1.0); + for( int ttt = 0; ttt < numIterPerMissingAnnotation; ttt++ ) { + datum.annotations[iii] = GenomeAnalysisEngine.getRandomGenerator().nextGaussian(); // draw a random sample from the standard normal distribution + // evaluate this random data point int gaussianIndex = 0; for( final MultivariateGaussian gaussian : gaussians ) { pVarInGaussianLog10[gaussianIndex++] = gaussian.pMixtureLog10 + gaussian.evaluateDatumLog10( datum ); } - sumPVarInGaussian += Math.pow(10.0, MathUtils.log10sumLog10(pVarInGaussianLog10)); - numVals++; + // add this sample's probability to the pile in order to take an average in the end + sumPVarInGaussian += Math.pow(10.0, MathUtils.log10sumLog10(pVarInGaussianLog10)); // p = 10 ^ Sum(pi_k * p(v|n,k)) + numSamples++; } } } - return Math.log10( sumPVarInGaussian / ((double) numVals) ); + return Math.log10( sumPVarInGaussian / ((double) numSamples) ); } } \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantDataManager.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantDataManager.java index f0a78280d..e1a076e76 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantDataManager.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantDataManager.java @@ -25,7 +25,6 @@ package org.broadinstitute.sting.gatk.walkers.variantrecalibration; -import cern.jet.random.Normal; import org.apache.log4j.Logger; import org.broadinstitute.sting.utils.variantcontext.VariantContext; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; @@ -91,7 +90,7 @@ public class VariantDataManager { meanVector[iii] = theMean; varianceVector[iii] = theSTD; for( final VariantDatum datum : data ) { - datum.annotations[iii] = ( datum.isNull[iii] ? Normal.staticNextDouble(0.0, 1.0) : ( datum.annotations[iii] - theMean ) / theSTD ); + datum.annotations[iii] = ( datum.isNull[iii] ? GenomeAnalysisEngine.getRandomGenerator().nextGaussian() : ( datum.annotations[iii] - theMean ) / theSTD ); // Each data point is now [ (x - mean) / standard deviation ] if( annotationKeys.get(iii).toLowerCase().contains("ranksum") && datum.isNull[iii] && datum.annotations[iii] > 0.0 ) { datum.annotations[iii] /= 3.0; From 212e9a1a0cf766f54f70d17d0786db570f8111ae Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Thu, 7 Jul 2011 15:18:57 -0400 Subject: [PATCH 04/10] Fixing unstable build after stable commit --- .../gatk/walkers/variantrecalibration/GaussianMixtureModel.java | 1 + 1 file changed, 1 insertion(+) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/GaussianMixtureModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/GaussianMixtureModel.java index b4a8c4c32..17461de2f 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/GaussianMixtureModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/GaussianMixtureModel.java @@ -26,6 +26,7 @@ package org.broadinstitute.sting.gatk.walkers.variantrecalibration; import Jama.Matrix; +import cern.jet.random.Normal; import org.apache.log4j.Logger; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.utils.MathUtils; From 4cfe0dd857891f6c70f4d0e58b60d6c4c401a0cd Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Thu, 7 Jul 2011 23:01:03 -0400 Subject: [PATCH 09/10] Test for bad alleles so that we don't generate IndexOutOfBoundsExceptions --- .../sting/utils/codecs/vcf/AbstractVCFCodec.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java index cd0f52c68..01344a117 100755 --- a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java +++ b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java @@ -225,7 +225,7 @@ public abstract class AbstractVCFCodec implements FeatureCodec, NameAwareCodec, loc = pos + alleles.get(0).length() - 1; } else if ( !isSingleNucleotideEvent(alleles) ) { ArrayList newAlleles = new ArrayList(); - loc = clipAlleles(pos, ref, alleles, newAlleles); + loc = clipAlleles(pos, ref, alleles, newAlleles, lineNo); alleles = newAlleles; } @@ -504,7 +504,7 @@ public abstract class AbstractVCFCodec implements FeatureCodec, NameAwareCodec, * @param clippedAlleles output list of clipped alleles * @return a list of alleles, clipped to the reference */ - protected static long clipAlleles(long position, String ref, List unclippedAlleles, List clippedAlleles) { + protected static long clipAlleles(long position, String ref, List unclippedAlleles, List clippedAlleles, int lineNo) { // Note that the computation of forward clipping here is meant only to see whether there is a common // base to all alleles, and to correctly compute reverse clipping, @@ -522,6 +522,8 @@ public abstract class AbstractVCFCodec implements FeatureCodec, NameAwareCodec, } if (a.length() - reverseClipped <= forwardClipping || a.length() - forwardClipping == 0) clipping = false; + else if (ref.length() == reverseClipped) + generateException("bad alleles encountered", lineNo); else if (a.getBases()[a.length()-reverseClipped-1] != ref.getBytes()[ref.length()-reverseClipped-1]) clipping = false; }