From 1da4ad9607a887bf520ceb1d896ca6512166b6bf Mon Sep 17 00:00:00 2001
From: Ami Levy Moonshine <ami@broadinstitute.org>
Date: Mon, 22 Oct 2012 10:45:53 -0400
Subject: [PATCH 01/40] new class to test the quals of reduced bam vs
 non-reduced bam

---
 .../gatk/walkers/qc/AssessReducedQuals.java   | 116 ++++++++++++++++++
 1 file changed, 116 insertions(+)
 create mode 100644 public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java
new file mode 100644
index 000000000..c8ec33364
--- /dev/null
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java
@@ -0,0 +1,116 @@
+package org.broadinstitute.sting.gatk.walkers.qc;
+
+import org.broadinstitute.sting.commandline.Output;
+import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
+import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
+import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
+import org.broadinstitute.sting.gatk.walkers.LocusWalker;
+import org.broadinstitute.sting.gatk.walkers.TreeReducible;
+import org.broadinstitute.sting.utils.GenomeLoc;
+import org.broadinstitute.sting.utils.pileup.PileupElement;
+import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
+
+import java.io.PrintStream;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: ami
+ * Date: 10/19/12
+ * Time: 9:09 AM
+ * To change this template use File | Settings | File Templates.
+ */
+public class AssessReducedQuals extends LocusWalker<GenomeLoc, GenomeLoc> implements TreeReducible<GenomeLoc> {
+
+    private static final String original = "original";
+    private static final String reduced = "reduced";
+
+    @Output
+    protected PrintStream out;
+
+    @Override
+    public boolean includeReadsWithDeletionAtLoci() { return true; }
+
+    public void initialize() {}    //todo: why we need that?
+
+    @Override
+    public GenomeLoc map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
+        if ( tracker == null )
+            return null;
+
+        int originalQualsIndex = 0;
+        int reducedQualsIndex = 1;
+        double epsilon = 0;
+        double[] quals = getPileupQuals(context.getBasePileup());
+        return (quals[originalQualsIndex] - quals[reducedQualsIndex] >= epsilon || quals[originalQualsIndex] - quals[reducedQualsIndex] <= -1*epsilon) ? ref.getLocus() : null;
+
+    }
+
+    private double[] getPileupQuals(final ReadBackedPileup readPileup) {
+
+        int originalQualsIndex = 0;
+        int reducedQualsIndex = 1;
+        double[] quals = new double[2];
+
+        for( PileupElement p : readPileup){
+            if ( (int)p.getQual() > 2 && p.getMappingQual() > 0 && !p.isDeletion() ){
+                if (p.getRead().isReducedRead()){
+                    double tempQual = (double)(p.getQual()) * p.getRepresentativeCount();
+                    quals[reducedQualsIndex] += tempQual;
+                }
+                else
+                {
+                    double tempQual = (double)(p.getQual());
+                    quals[originalQualsIndex] += tempQual;
+                }
+            }
+
+        }
+
+
+        return quals;
+    }
+
+    //public void onTraversalDone(GenomeLoc sum) {
+    //    if ( sum != null )
+    //        out.println(sum);
+    //}
+
+    @Override
+    public GenomeLoc treeReduce(GenomeLoc lhs, GenomeLoc rhs) {
+        if ( lhs == null )
+            return rhs;
+
+        if ( rhs == null )
+            return lhs;
+
+        // if contiguous, just merge them
+        if ( lhs.contiguousP(rhs) )
+            return getToolkit().getGenomeLocParser().createGenomeLoc(lhs.getContig(), lhs.getStart(), rhs.getStop());
+
+        // otherwise, print the lhs and start over with the rhs
+        out.println(lhs);
+        return rhs;
+    }
+
+    @Override
+    public GenomeLoc reduceInit() {
+        return null;
+    }
+
+    @Override
+    public GenomeLoc reduce(GenomeLoc value, GenomeLoc sum) {
+        if ( value == null )
+            return sum;
+
+        if ( sum == null )
+            return value;
+
+        // if contiguous, just merge them
+        if ( sum.contiguousP(value) )
+            return getToolkit().getGenomeLocParser().createGenomeLoc(sum.getContig(), sum.getStart(), value.getStop());
+
+        // otherwise, print the sum and start over with the value
+        out.println(sum);
+        return value;
+    }
+}

From ccae6a5b9264350050695de05dad098884ea99f9 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Mon, 22 Oct 2012 11:48:34 -0400
Subject: [PATCH 02/40] Fixed the RR bug I (knowingly) introduced last week:
 turns out we can't trust a context size's worth of data from the previous
 marking.  I think Mauricio warned me about this but I forgot.

---
 .../gatk/walkers/compression/reducereads/SlidingWindow.java  | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/compression/reducereads/SlidingWindow.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/compression/reducereads/SlidingWindow.java
index 63524ae82..32abe8ef6 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/compression/reducereads/SlidingWindow.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/compression/reducereads/SlidingWindow.java
@@ -276,7 +276,10 @@ public class SlidingWindow {
 
         final int windowHeaderStartLocation = getStartLocation(windowHeader);
         final int sizeOfMarkedRegion = stop - windowHeaderStartLocation + contextSize + 1;
-        final int lastPositionMarked = markedSites.updateRegion(windowHeaderStartLocation, sizeOfMarkedRegion);
+
+        // copy over as many bits as we can from the previous calculation.  Note that we can't trust the
+        // last (contextSize - 1) worth of bits because we may not have actually looked at variant regions there.
+        final int lastPositionMarked = markedSites.updateRegion(windowHeaderStartLocation, sizeOfMarkedRegion) - contextSize - 1;
         final int locationToProcess = Math.min(lastPositionMarked, stop - contextSize);
 
         // update the iterator to the correct position

From 97dc3664c9ca9bd42d98e1a236219bd4e849fa26 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Mon, 22 Oct 2012 12:04:23 -0400
Subject: [PATCH 03/40] Fixed yet another NPE related to the
 ArgumentTypeDescriptor vs. ArgumentMatchValue. Added integration test based
 on GSA-621.

---
 .../gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java |  2 +-
 .../genotyper/UnifiedGenotyperIntegrationTest.java     | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/public/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java b/public/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java
index 43350ccc1..f521c959d 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java
@@ -151,7 +151,7 @@ public class VCFWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor {
                 ? new VariantContextWriterStub(engine, writerFile, argumentSources)
                 : new VariantContextWriterStub(engine, defaultOutputStream, argumentSources);
 
-        stub.setCompressed(isCompressed(writerFileName.asString()));
+        stub.setCompressed(isCompressed(writerFileName == null ? null: writerFileName.asString()));
         stub.setDoNotWriteGenotypes(argumentIsPresent(createSitesOnlyArgumentDefinition(),matches));
         stub.setSkipWritingCommandLineHeader(argumentIsPresent(createNoCommandLineHeaderArgumentDefinition(),matches));
         stub.setForceBCF(argumentIsPresent(createBCFArgumentDefinition(),matches));
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index 044d70fc2..3a2d6f697 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -7,6 +7,7 @@ import org.testng.annotations.Test;
 
 import java.io.File;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
 // ********************************************************************************** //
@@ -18,6 +19,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     private final static String baseCommand = "-T UnifiedGenotyper -R " + b36KGReference + " -nosl --no_cmdline_in_header -glm BOTH -minIndelFrac 0.0 --dbsnp " + b36dbSNP129;
     private final static String baseCommandIndels = "-T UnifiedGenotyper -R " + b36KGReference + " -nosl --no_cmdline_in_header -glm INDEL -mbq 20 -minIndelFrac 0.0 --dbsnp " + b36dbSNP129;
     private final static String baseCommandIndelsb37 = "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm INDEL -mbq 20 --dbsnp " + b37dbSNP132;
+    private final static String baseCommandNoCmdLineHeaderStdout = "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -I " + privateTestDir + "bamExample.ReducedRead.ADAnnotation.bam";
 
     // --------------------------------------------------------------------------------------------------------------
     //
@@ -175,6 +177,14 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test using comp track", spec);
     }
 
+    @Test
+    public void testNoCmdLineHeaderStdout() {
+        WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
+                baseCommandNoCmdLineHeaderStdout + " -glm INDEL -L 1:67,225,396-67,288,518", 0,
+                Collections.<String>emptyList());
+        executeTest("testNoCmdLineHeaderStdout", spec);
+    }
+
     @Test
     public void testOutputParameterSitesOnly() {
         testOutputParameters("-sites_only", "97ba874eafc9884a4de027a84c036311");

From 90f59803fd8c02bf7c2c38f3e909b72ab2c35cf8 Mon Sep 17 00:00:00 2001
From: Mark DePristo <depristo@broadinstitute.org>
Date: Mon, 22 Oct 2012 09:58:43 -0400
Subject: [PATCH 04/40] MaxAltAlleles now defaults to 6, no more
 MaxAltAllelesForIndels

-- Updated StandardCallerArgumentCollection to remove MaxAltAllelesForIndels. Previous argument is deprecated with meaningful doc message for people to use maxAltAlleles
-- All constructores, factory methods, and test builders and their users updated to provide just a single argument
-- Updating MD5s for integration tests that change due to genotyping more alleles
-- Adding more alleles to genotyping results in slight changes in the QUAL value for multi-allelic loci where one or more alleles aren't polymorphic.  That's simply due to the way that alternative hypotheses contribute as reference evidence against each true allele.  The effect can be large (new qual = old qual / 2 in one case here).
-- If we want more precision in our estimates we could decide (Eric, should we discuss?) to actually separately do a discovery phase in the genotyping, eliminate all variants not considered polymorphic, and then do a final round of calling to get the exact QUAL value for only those that are segregating.  This would have the value of having the QUAL stay constant as more alleles are genotyped, at the cost of some code complexity increase and runtime.  Might be worth it through
---
 .../genotyper/afcalc/AFCalcTestBuilder.java   |  2 +-
 .../afcalc/GeneralPloidyExactAFCalc.java      | 14 ++++-----
 .../genotyper/afcalc/AFCalcUnitTest.java      |  4 +--
 ...neralPloidyAFCalculationModelUnitTest.java |  2 +-
 .../HaplotypeCallerIntegrationTest.java       |  2 +-
 .../StandardCallerArgumentCollection.java     | 13 ++++++---
 .../gatk/walkers/genotyper/afcalc/AFCalc.java | 10 ++-----
 .../genotyper/afcalc/AFCalcFactory.java       | 29 ++++++++-----------
 .../genotyper/afcalc/DiploidExactAFCalc.java  | 14 ++++-----
 .../walkers/genotyper/afcalc/ExactAFCalc.java |  4 +--
 .../IndependentAllelesDiploidExactAFCalc.java | 17 ++++++-----
 .../afcalc/OriginalDiploidExactAFCalc.java    |  4 +--
 .../afcalc/ReferenceDiploidExactAFCalc.java   |  4 +--
 .../GLBasedSampleSelector.java                |  2 +-
 .../UnifiedGenotyperIntegrationTest.java      |  4 +--
 15 files changed, 59 insertions(+), 66 deletions(-)

diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalcTestBuilder.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalcTestBuilder.java
index cfb67164d..6f3740ab3 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalcTestBuilder.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalcTestBuilder.java
@@ -54,7 +54,7 @@ public class AFCalcTestBuilder {
     }
 
     public AFCalc makeModel() {
-        return AFCalcFactory.createAFCalc(modelType, nSamples, getNumAltAlleles(), getNumAltAlleles(), 2);
+        return AFCalcFactory.createAFCalc(modelType, nSamples, getNumAltAlleles(), 2);
     }
 
     public double[] makePriors() {
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/GeneralPloidyExactAFCalc.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/GeneralPloidyExactAFCalc.java
index 2b247430c..b248c8759 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/GeneralPloidyExactAFCalc.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/GeneralPloidyExactAFCalc.java
@@ -40,22 +40,20 @@ public class GeneralPloidyExactAFCalc extends ExactAFCalc {
     private final static double MAX_LOG10_ERROR_TO_STOP_EARLY = 6; // we want the calculation to be accurate to 1 / 10^6
     private final static boolean VERBOSE = false;
 
-    protected GeneralPloidyExactAFCalc(final int nSamples, final int maxAltAlleles, final int maxAltAllelesForIndels, final int ploidy) {
-        super(nSamples, maxAltAlleles, maxAltAllelesForIndels, ploidy);
+    protected GeneralPloidyExactAFCalc(final int nSamples, final int maxAltAlleles, final int ploidy) {
+        super(nSamples, maxAltAlleles, ploidy);
         this.ploidy = ploidy;
     }
 
     @Override
     protected VariantContext reduceScope(VariantContext vc) {
-        final int maxAltAlleles = vc.getType().equals(VariantContext.Type.INDEL) ? maxAlternateAllelesForIndels : maxAlternateAllelesToGenotype;
-
         // don't try to genotype too many alternate alleles
-        if ( vc.getAlternateAlleles().size() > maxAltAlleles) {
-            logger.warn("this tool is currently set to genotype at most " + maxAltAlleles + " alternate alleles in a given context, but the context at " + vc.getChr() + ":" + vc.getStart() + " has " + (vc.getAlternateAlleles().size()) + " alternate alleles so only the top alleles will be used; see the --max_alternate_alleles argument");
+        if ( vc.getAlternateAlleles().size() > getMaxAltAlleles()) {
+            logger.warn("this tool is currently set to genotype at most " + getMaxAltAlleles() + " alternate alleles in a given context, but the context at " + vc.getChr() + ":" + vc.getStart() + " has " + (vc.getAlternateAlleles().size()) + " alternate alleles so only the top alleles will be used; see the --max_alternate_alleles argument");
 
-            final List<Allele> alleles = new ArrayList<Allele>(maxAltAlleles + 1);
+            final List<Allele> alleles = new ArrayList<Allele>(getMaxAltAlleles() + 1);
             alleles.add(vc.getReference());
-            alleles.addAll(chooseMostLikelyAlternateAlleles(vc, maxAltAlleles, ploidy));
+            alleles.addAll(chooseMostLikelyAlternateAlleles(vc, getMaxAltAlleles(), ploidy));
 
             VariantContextBuilder builder = new VariantContextBuilder(vc);
             builder.alleles(alleles);
diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalcUnitTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalcUnitTest.java
index ef4318a40..2d346e548 100644
--- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalcUnitTest.java
+++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalcUnitTest.java
@@ -125,7 +125,7 @@ public class AFCalcUnitTest extends BaseTest {
         final List<Genotype> triAllelicSamples = Arrays.asList(AA2, AB2, BB2, AC2, BC2, CC2);
 
         for ( final int nSamples : Arrays.asList(1, 2, 3, 4) ) {
-            List<AFCalc> calcs = AFCalcFactory.createAFCalcs( Arrays.asList( AFCalcFactory.Calculation.values() ), 4, 2, 2, 2);
+            List<AFCalc> calcs = AFCalcFactory.createAFCalcs( Arrays.asList( AFCalcFactory.Calculation.values() ), 4, 2, 2);
 
             final int nPriorValues = 2*nSamples+1;
             final double[] flatPriors = MathUtils.normalizeFromLog10(new double[nPriorValues], true);  // flat priors
@@ -215,7 +215,7 @@ public class AFCalcUnitTest extends BaseTest {
                 samples.addAll(Collections.nCopies(nNonInformative, testData.nonInformative));
 
                 final int nSamples = samples.size();
-                List<AFCalc> calcs = AFCalcFactory.createAFCalcs(Arrays.asList(AFCalcFactory.Calculation.values()), 4, 2, 2, 2);
+                List<AFCalc> calcs = AFCalcFactory.createAFCalcs(Arrays.asList(AFCalcFactory.Calculation.values()), 4, 2, 2);
 
                 final double[] priors = MathUtils.normalizeFromLog10(new double[2*nSamples+1], true);  // flat priors
 
diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/GeneralPloidyAFCalculationModelUnitTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/GeneralPloidyAFCalculationModelUnitTest.java
index 1b3a4c0c0..3df2f7883 100644
--- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/GeneralPloidyAFCalculationModelUnitTest.java
+++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/GeneralPloidyAFCalculationModelUnitTest.java
@@ -140,7 +140,7 @@ public class GeneralPloidyAFCalculationModelUnitTest extends BaseTest {
         final int len = GeneralPloidyGenotypeLikelihoods.getNumLikelihoodElements(1 + cfg.numAltAlleles, cfg.ploidy * cfg.GLs.size());
         double[] priors = new double[len];  // flat priors
 
-        final GeneralPloidyExactAFCalc calc = new GeneralPloidyExactAFCalc(cfg.GLs.size(), 1 + cfg.numAltAlleles, 1 + cfg.numAltAlleles, cfg.ploidy);
+        final GeneralPloidyExactAFCalc calc = new GeneralPloidyExactAFCalc(cfg.GLs.size(), 1 + cfg.numAltAlleles, cfg.ploidy);
         calc.combineSinglePools(cfg.GLs, 1 + cfg.numAltAlleles, cfg.ploidy, priors);
         int nameIndex = 1;
         for ( int allele = 0; allele < cfg.numAltAlleles; allele++, nameIndex+=2 ) {
diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
index 870967f09..1d4fad19e 100644
--- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
+++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
@@ -31,7 +31,7 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
 
     @Test
     public void testHaplotypeCallerMultiSampleGGA() {
-        HCTest(CEUTRIO_BAM, "--max_alternate_alleles_for_indels 3 -gt_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "combined.phase1.chr20.raw.indels.sites.vcf", "53caa950535749f99d3c5b9bb61c7b60");
+        HCTest(CEUTRIO_BAM, "--max_alternate_alleles 3 -gt_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "combined.phase1.chr20.raw.indels.sites.vcf", "53caa950535749f99d3c5b9bb61c7b60");
     }
 
     private void HCTestComplexVariants(String bam, String args, String md5) {
diff --git a/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
index 84dfa694b..aa3a1e6ac 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
@@ -55,20 +55,25 @@ public class StandardCallerArgumentCollection {
      * then only this many alleles will be used.  Note that genotyping sites with many alternate alleles is both CPU and memory intensive and it
      * scales exponentially based on the number of alternate alleles.  Unless there is a good reason to change the default value, we highly recommend
      * that you not play around with this parameter.
+     *
+     * As of GATK 2.2 the genotyper can handle a very large number of events, so the default maximum has been increased to 6.
      */
     @Advanced
     @Argument(fullName = "max_alternate_alleles", shortName = "maxAltAlleles", doc = "Maximum number of alternate alleles to genotype", required = false)
-    public int MAX_ALTERNATE_ALLELES = 3;
+    public int MAX_ALTERNATE_ALLELES = 6;
 
     /**
      * If there are more than this number of alternate alleles presented to the genotyper (either through discovery or GENOTYPE_GIVEN ALLELES),
      * then only this many alleles will be used.  Note that genotyping sites with many alternate alleles is both CPU and memory intensive and it
      * scales exponentially based on the number of alternate alleles.  Unless there is a good reason to change the default value, we highly recommend
      * that you not play around with this parameter.
+     *
+     * This argument has been retired in GATK 2.2.  Please specify just maxAltAlleles from now on
      */
-    @Advanced
-    @Argument(fullName = "max_alternate_alleles_for_indels", shortName = "maxAltAllelesForIndels", doc = "Maximum number of alternate alleles to genotype for indels only", required = false)
-    public int MAX_ALTERNATE_ALLELES_FOR_INDELS = 2;
+    @Deprecated
+    @Hidden
+    @Argument(fullName = "max_alternate_alleles_for_indels", shortName = "maxAltAllelesForIndels", doc = "This argument has been retired in GATK 2.2.  Please specify just maxAltAlleles from now on, which will apply to any variant, regardless of type", required = false)
+    public int MAX_ALTERNATE_ALLELES_FOR_INDELS = -1;
 
     /**
      * If this fraction is greater is than zero, the caller will aggressively attempt to remove contamination through biased down-sampling of reads.
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalc.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalc.java
index e3abdeb24..f783267bc 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalc.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalc.java
@@ -45,7 +45,6 @@ public abstract class AFCalc implements Cloneable {
 
     protected final int nSamples;
     protected final int maxAlternateAllelesToGenotype;
-    protected final int maxAlternateAllelesForIndels;
 
     protected Logger logger = defaultLogger;
 
@@ -60,19 +59,16 @@ public abstract class AFCalc implements Cloneable {
      *
      * @param nSamples number of samples, must be > 0
      * @param maxAltAlleles maxAltAlleles for SNPs
-     * @param maxAltAllelesForIndels for indels
      * @param ploidy the ploidy, must be > 0
      */
-    protected AFCalc(final int nSamples, final int maxAltAlleles, final int maxAltAllelesForIndels, final int ploidy) {
+    protected AFCalc(final int nSamples, final int maxAltAlleles, final int ploidy) {
         if ( nSamples < 0 ) throw new IllegalArgumentException("nSamples must be greater than zero " + nSamples);
         if ( maxAltAlleles < 1 ) throw new IllegalArgumentException("maxAltAlleles must be greater than zero " + maxAltAlleles);
-        if ( maxAltAllelesForIndels < 1 ) throw new IllegalArgumentException("maxAltAllelesForIndels must be greater than zero " + maxAltAllelesForIndels);
         if ( ploidy < 1 ) throw new IllegalArgumentException("ploidy must be > 0 but got " + ploidy);
 
         this.nSamples = nSamples;
         this.maxAlternateAllelesToGenotype = maxAltAlleles;
-        this.maxAlternateAllelesForIndels = maxAltAllelesForIndels;
-        this.stateTracker = new StateTracker(Math.max(maxAltAlleles, maxAltAllelesForIndels));
+        this.stateTracker = new StateTracker(maxAltAlleles);
     }
 
     /**
@@ -191,7 +187,7 @@ public abstract class AFCalc implements Cloneable {
     // ---------------------------------------------------------------------------
 
     public int getMaxAltAlleles() {
-        return Math.max(maxAlternateAllelesToGenotype, maxAlternateAllelesForIndels);
+        return maxAlternateAllelesToGenotype;
     }
 
     protected StateTracker getStateTracker() {
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalcFactory.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalcFactory.java
index 80de555ca..efb16101e 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalcFactory.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalcFactory.java
@@ -91,7 +91,7 @@ public class AFCalcFactory {
     public static AFCalc createAFCalc(final UnifiedArgumentCollection UAC,
                                       final int nSamples,
                                       final Logger logger) {
-        final int maxAltAlleles = Math.max(UAC.MAX_ALTERNATE_ALLELES, UAC.MAX_ALTERNATE_ALLELES_FOR_INDELS);
+        final int maxAltAlleles = UAC.MAX_ALTERNATE_ALLELES;
         if ( ! UAC.AFmodel.usableForParams(UAC.samplePloidy, maxAltAlleles) ) {
             logger.info("Requested ploidy " + UAC.samplePloidy + " maxAltAlleles " + maxAltAlleles + " not supported by requested model " + UAC.AFmodel + " looking for an option");
             final List<Calculation> supportingCalculations = new LinkedList<Calculation>();
@@ -109,7 +109,7 @@ public class AFCalcFactory {
             logger.info("Selecting model " + UAC.AFmodel);
         }
 
-        final AFCalc calc = createAFCalc(UAC.AFmodel, nSamples, UAC.MAX_ALTERNATE_ALLELES, UAC.MAX_ALTERNATE_ALLELES_FOR_INDELS, UAC.samplePloidy);
+        final AFCalc calc = createAFCalc(UAC.AFmodel, nSamples, UAC.MAX_ALTERNATE_ALLELES, UAC.samplePloidy);
 
         if ( logger != null ) calc.setLogger(logger);
         if ( UAC.exactCallsLog != null ) calc.enableProcessLog(UAC.exactCallsLog);
@@ -126,7 +126,7 @@ public class AFCalcFactory {
      * @return an initialized AFCalc
      */
     public static AFCalc createAFCalc(final int nSamples) {
-        return createAFCalc(chooseBestCalculation(nSamples, 2, 1), nSamples, 2, 2, 2);
+        return createAFCalc(chooseBestCalculation(nSamples, 2, 1), nSamples, 2, 2);
     }
 
     /**
@@ -139,7 +139,7 @@ public class AFCalcFactory {
      * @return an initialized AFCalc
      */
     public static AFCalc createAFCalc(final Calculation calc, final int nSamples, final int maxAltAlleles) {
-        return createAFCalc(calc, nSamples, maxAltAlleles, maxAltAlleles, 2);
+        return createAFCalc(calc, nSamples, maxAltAlleles, 2);
     }
 
     /**
@@ -147,14 +147,12 @@ public class AFCalcFactory {
      *
      * @param nSamples the number of samples we'll be using
      * @param maxAltAlleles the max. alt alleles to consider for SNPs
-     * @param maxAltAllelesForIndels the max. alt alleles to consider for non-SNPs
      * @param ploidy the sample ploidy.  Must be consistent with the calc
      *
      * @return an initialized AFCalc
      */
-    public static AFCalc createAFCalc(final int nSamples, final int maxAltAlleles, final int maxAltAllelesForIndels, final int ploidy) {
-        final int maxAlt = Math.max(maxAltAlleles, maxAltAllelesForIndels);
-        return createAFCalc(chooseBestCalculation(nSamples, ploidy, maxAlt), nSamples, maxAltAlleles, maxAltAllelesForIndels, ploidy);
+    public static AFCalc createAFCalc(final int nSamples, final int maxAltAlleles, final int ploidy) {
+        return createAFCalc(chooseBestCalculation(nSamples, ploidy, maxAltAlleles), nSamples, maxAltAlleles, ploidy);
     }
 
     /**
@@ -181,20 +179,17 @@ public class AFCalcFactory {
      * @param calc the calculation to use
      * @param nSamples the number of samples we'll be using
      * @param maxAltAlleles the max. alt alleles to consider for SNPs
-     * @param maxAltAllelesForIndels the max. alt alleles to consider for non-SNPs
      * @param ploidy the sample ploidy.  Must be consistent with the calc
      *
      * @return an initialized AFCalc
      */
-    public static AFCalc createAFCalc(final Calculation calc, final int nSamples, final int maxAltAlleles, final int maxAltAllelesForIndels, final int ploidy) {
+    public static AFCalc createAFCalc(final Calculation calc, final int nSamples, final int maxAltAlleles, final int ploidy) {
         if ( calc == null ) throw new IllegalArgumentException("Calculation cannot be null");
         if ( nSamples < 0 ) throw new IllegalArgumentException("nSamples must be greater than zero " + nSamples);
         if ( maxAltAlleles < 1 ) throw new IllegalArgumentException("maxAltAlleles must be greater than zero " + maxAltAlleles);
-        if ( maxAltAllelesForIndels < 1 ) throw new IllegalArgumentException("maxAltAllelesForIndels must be greater than zero " + maxAltAllelesForIndels);
         if ( ploidy < 1 ) throw new IllegalArgumentException("sample ploidy must be greater than zero " + ploidy);
 
-        final int maxAlt = Math.max(maxAltAlleles, maxAltAllelesForIndels);
-        if ( ! calc.usableForParams(ploidy, maxAlt) )
+        if ( ! calc.usableForParams(ploidy, maxAltAlleles) )
             throw new IllegalArgumentException("AFCalc " + calc + " does not support requested ploidy " + ploidy);
 
         final Class<? extends AFCalc> afClass = getClassByName(calc.className);
@@ -202,19 +197,19 @@ public class AFCalcFactory {
             throw new IllegalArgumentException("Unexpected AFCalc " + calc);
 
         try {
-            Object args[] = new Object[]{nSamples, maxAltAlleles, maxAltAllelesForIndels, ploidy};
-            Constructor c = afClass.getDeclaredConstructor(int.class, int.class, int.class, int.class);
+            Object args[] = new Object[]{nSamples, maxAltAlleles, ploidy};
+            Constructor c = afClass.getDeclaredConstructor(int.class, int.class, int.class);
             return (AFCalc)c.newInstance(args);
         } catch (Exception e) {
             throw new ReviewedStingException("Could not instantiate AFCalc " + calc, e);
         }
     }
 
-    protected static List<AFCalc> createAFCalcs(final List<Calculation> calcs, final int nSamples, final int maxAltAlleles, final int maxAltAllelesForIndels, final int ploidy) {
+    protected static List<AFCalc> createAFCalcs(final List<Calculation> calcs, final int nSamples, final int maxAltAlleles, final int ploidy) {
         final List<AFCalc> AFCalcs = new LinkedList<AFCalc>();
 
         for ( final Calculation calc : calcs )
-            AFCalcs.add(createAFCalc(calc, nSamples, maxAltAlleles, maxAltAllelesForIndels, ploidy));
+            AFCalcs.add(createAFCalc(calc, nSamples, maxAltAlleles, ploidy));
 
         return AFCalcs;
     }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/DiploidExactAFCalc.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/DiploidExactAFCalc.java
index 6b345dcf5..4895c84d9 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/DiploidExactAFCalc.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/DiploidExactAFCalc.java
@@ -31,8 +31,8 @@ import org.broadinstitute.sting.utils.variantcontext.*;
 import java.util.*;
 
 public abstract class DiploidExactAFCalc extends ExactAFCalc {
-    public DiploidExactAFCalc(final int nSamples, final int maxAltAlleles, final int maxAltAllelesForIndels, final int ploidy) {
-        super(nSamples, maxAltAlleles, maxAltAllelesForIndels, ploidy);
+    public DiploidExactAFCalc(final int nSamples, final int maxAltAlleles, final int ploidy) {
+        super(nSamples, maxAltAlleles, ploidy);
         if ( ploidy != 2 ) throw new IllegalArgumentException("ploidy must be two for DiploidExactAFCalc and subclasses but saw " + ploidy);
     }
 
@@ -75,16 +75,14 @@ public abstract class DiploidExactAFCalc extends ExactAFCalc {
 
     @Override
     protected VariantContext reduceScope(final VariantContext vc) {
-        final int myMaxAltAllelesToGenotype = vc.getType().equals(VariantContext.Type.INDEL) ? maxAlternateAllelesForIndels : maxAlternateAllelesToGenotype;
-
         // don't try to genotype too many alternate alleles
-        if ( vc.getAlternateAlleles().size() > myMaxAltAllelesToGenotype ) {
-            logger.warn("this tool is currently set to genotype at most " + myMaxAltAllelesToGenotype + " alternate alleles in a given context, but the context at " + vc.getChr() + ":" + vc.getStart() + " has " + (vc.getAlternateAlleles().size()) + " alternate alleles so only the top alleles will be used; see the --max_alternate_alleles argument");
+        if ( vc.getAlternateAlleles().size() > getMaxAltAlleles() ) {
+            logger.warn("this tool is currently set to genotype at most " + getMaxAltAlleles() + " alternate alleles in a given context, but the context at " + vc.getChr() + ":" + vc.getStart() + " has " + (vc.getAlternateAlleles().size()) + " alternate alleles so only the top alleles will be used; see the --max_alternate_alleles argument");
 
             VariantContextBuilder builder = new VariantContextBuilder(vc);
-            List<Allele> alleles = new ArrayList<Allele>(myMaxAltAllelesToGenotype + 1);
+            List<Allele> alleles = new ArrayList<Allele>(getMaxAltAlleles() + 1);
             alleles.add(vc.getReference());
-            alleles.addAll(chooseMostLikelyAlternateAlleles(vc, myMaxAltAllelesToGenotype));
+            alleles.addAll(chooseMostLikelyAlternateAlleles(vc, getMaxAltAlleles()));
             builder.alleles(alleles);
             builder.genotypes(VariantContextUtils.subsetDiploidAlleles(vc, alleles, false));
             return builder.make();
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/ExactAFCalc.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/ExactAFCalc.java
index df0793352..ab230d398 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/ExactAFCalc.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/ExactAFCalc.java
@@ -39,8 +39,8 @@ import java.util.ArrayList;
 abstract class ExactAFCalc extends AFCalc {
     protected static final int HOM_REF_INDEX = 0;  // AA likelihoods are always first
 
-    protected ExactAFCalc(final int nSamples, int maxAltAlleles, int maxAltAllelesForIndels, final int ploidy) {
-        super(nSamples, maxAltAlleles, maxAltAllelesForIndels, ploidy);
+    protected ExactAFCalc(final int nSamples, int maxAltAlleles, final int ploidy) {
+        super(nSamples, maxAltAlleles, ploidy);
     }
 
     /**
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/IndependentAllelesDiploidExactAFCalc.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/IndependentAllelesDiploidExactAFCalc.java
index 804b560b4..857b7e59e 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/IndependentAllelesDiploidExactAFCalc.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/IndependentAllelesDiploidExactAFCalc.java
@@ -89,7 +89,7 @@ import java.util.*;
     /**
      * The min. confidence of an allele to be included in the joint posterior.
      */
-    private final static double MIN_LOG10_CONFIDENCE_TO_INCLUDE_ALLELE_IN_POSTERIOR = Math.log10(1e-20);
+    private final static double MIN_LOG10_CONFIDENCE_TO_INCLUDE_ALLELE_IN_POSTERIOR = Math.log10(1e-10);
 
     private final static int[] BIALLELIC_NON_INFORMATIVE_PLS = new int[]{0,0,0};
     private final static List<Allele> BIALLELIC_NOCALL = Arrays.asList(Allele.NO_CALL, Allele.NO_CALL);
@@ -111,9 +111,9 @@ import java.util.*;
      */
     final AFCalc biAlleleExactModel;
 
-    protected IndependentAllelesDiploidExactAFCalc(int nSamples, int maxAltAlleles, int maxAltAllelesForIndels, final int ploidy) {
-        super(nSamples, maxAltAlleles, maxAltAllelesForIndels, ploidy);
-        biAlleleExactModel = new ReferenceDiploidExactAFCalc(nSamples, 1, 1, ploidy);
+    protected IndependentAllelesDiploidExactAFCalc(int nSamples, int maxAltAlleles, final int ploidy) {
+        super(nSamples, maxAltAlleles, ploidy);
+        biAlleleExactModel = new ReferenceDiploidExactAFCalc(nSamples, 1, ploidy);
     }
 
     /**
@@ -336,12 +336,13 @@ import java.util.*;
             // MLE of altI allele is simply the MLE of this allele in altAlleles
             alleleCountsOfMLE[altI] = sortedResultWithThetaNPriors.getAlleleCountAtMLE(altAllele);
 
-            log10PriorsOfAC[0] += sortedResultWithThetaNPriors.getLog10PriorOfAFEq0();
-            log10PriorsOfAC[1] += sortedResultWithThetaNPriors.getLog10PriorOfAFGT0();
-
             // the AF > 0 case requires us to store the normalized likelihood for later summation
-            if ( sortedResultWithThetaNPriors.getLog10PosteriorOfAFGT0() > MIN_LOG10_CONFIDENCE_TO_INCLUDE_ALLELE_IN_POSTERIOR )
+            if ( sortedResultWithThetaNPriors.getLog10PosteriorOfAFGT0() > MIN_LOG10_CONFIDENCE_TO_INCLUDE_ALLELE_IN_POSTERIOR ) {
                 log10PosteriorOfACEq0Sum += sortedResultWithThetaNPriors.getLog10PosteriorOfAFEq0();
+                log10PriorsOfAC[0] += sortedResultWithThetaNPriors.getLog10PriorOfAFEq0();
+                log10PriorsOfAC[1] += sortedResultWithThetaNPriors.getLog10PriorOfAFGT0();
+            }
+
             log10PosteriorOfACGt0Sum += sortedResultWithThetaNPriors.getLog10PosteriorOfAFGT0();
 
             // bind pNonRef for allele to the posterior value of the AF > 0 with the new adjusted prior
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/OriginalDiploidExactAFCalc.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/OriginalDiploidExactAFCalc.java
index ac4634666..fc26111e0 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/OriginalDiploidExactAFCalc.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/OriginalDiploidExactAFCalc.java
@@ -13,8 +13,8 @@ import java.util.Map;
  * Original bi-allelic ~O(N) implementation.  Kept here for posterity and reference
  */
 class OriginalDiploidExactAFCalc extends DiploidExactAFCalc {
-    protected OriginalDiploidExactAFCalc(int nSamples, int maxAltAlleles, int maxAltAllelesForIndels, final int ploidy) {
-        super(nSamples, maxAltAlleles, maxAltAllelesForIndels, ploidy);
+    protected OriginalDiploidExactAFCalc(int nSamples, int maxAltAlleles, final int ploidy) {
+        super(nSamples, maxAltAlleles, ploidy);
     }
 
     @Override
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/ReferenceDiploidExactAFCalc.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/ReferenceDiploidExactAFCalc.java
index b4e7b2ab1..97e5fed3b 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/ReferenceDiploidExactAFCalc.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/ReferenceDiploidExactAFCalc.java
@@ -1,7 +1,7 @@
 package org.broadinstitute.sting.gatk.walkers.genotyper.afcalc;
 
 public class ReferenceDiploidExactAFCalc extends DiploidExactAFCalc {
-    protected ReferenceDiploidExactAFCalc(int nSamples, int maxAltAlleles, int maxAltAllelesForIndels, final int ploidy) {
-        super(nSamples, maxAltAlleles, maxAltAllelesForIndels, ploidy);
+    protected ReferenceDiploidExactAFCalc(int nSamples, int maxAltAlleles, final int ploidy) {
+        super(nSamples, maxAltAlleles, ploidy);
     }
 }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/validationsiteselector/GLBasedSampleSelector.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/validationsiteselector/GLBasedSampleSelector.java
index 48a2d2700..d71d0c9c8 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/validationsiteselector/GLBasedSampleSelector.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/validationsiteselector/GLBasedSampleSelector.java
@@ -55,7 +55,7 @@ public class GLBasedSampleSelector extends SampleSelector {
         // do we want to apply a prior? maybe user-spec?
         if ( flatPriors == null ) {
             flatPriors = new double[1+2*samples.size()];
-            AFCalculator = AFCalcFactory.createAFCalc(samples.size(), 4, 4, 2);
+            AFCalculator = AFCalcFactory.createAFCalc(samples.size(), 4, 2);
         }
         final AFCalcResult result = AFCalculator.getLog10PNonRef(subContext, flatPriors);
         // do we want to let this qual go up or down?
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index 3a2d6f697..a1701e3e5 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -62,7 +62,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testMultipleSNPAlleles() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + privateTestDir + "multiallelic.snps.bam -o %s -L " + privateTestDir + "multiallelic.snps.intervals", 1,
-                Arrays.asList("73c9b926c5e971a113de347a64fdcf20"));
+                Arrays.asList("543f68e42034bf44cfb24da8c9204320"));
         executeTest("test Multiple SNP alleles", spec);
     }
 
@@ -78,7 +78,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testReverseTrim() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm INDEL -I " + validationDataLocation + "CEUTrio.HiSeq.b37.chr20.10_11mb.bam -o %s -L 20:10289124 -L 20:10090289", 1,
-                Arrays.asList("7f8d13690cb7d4173787afa00c496f12"));
+                Arrays.asList("5ce03dd9ca2d9324c1d4a9d64389beb5"));
         executeTest("test reverse trim", spec);
     }
 

From 008df545754ce9dee93a0f0de29056385bea73b2 Mon Sep 17 00:00:00 2001
From: Ryan Poplin <rpoplin@broadinstitute.org>
Date: Mon, 22 Oct 2012 14:21:52 -0400
Subject: [PATCH 05/40] Bug fix in GATKSAMRecord.getSoftEnd() for reads that
 are entirely clipped.

---
 .../sting/utils/sam/GATKSAMRecord.java        | 21 ++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java
index 1feb76517..9fdb48b34 100755
--- a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java
+++ b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java
@@ -60,8 +60,9 @@ public class GATKSAMRecord extends BAMRecord {
     private String mReadString = null;
     private GATKSAMReadGroupRecord mReadGroup = null;
     private byte[] reducedReadCounts = null;
-    private int softStart = -1;
-    private int softEnd = -1;
+    private final static int UNINITIALIZED = -1;
+    private int softStart = UNINITIALIZED;
+    private int softEnd = UNINITIALIZED;
 
     // because some values can be null, we don't want to duplicate effort
     private boolean retrievedReadGroup = false;
@@ -386,7 +387,7 @@ public class GATKSAMRecord extends BAMRecord {
      * @return the unclipped start of the read taking soft clips (but not hard clips) into account
      */
     public int getSoftStart() {
-        if (softStart < 0) {
+        if ( softStart == UNINITIALIZED ) {
             softStart = getAlignmentStart();
             for (final CigarElement cig : getCigar().getCigarElements()) {
                 final CigarOperator op = cig.getOperator();
@@ -408,17 +409,23 @@ public class GATKSAMRecord extends BAMRecord {
      * @return the unclipped end of the read taking soft clips (but not hard clips) into account
      */
     public int getSoftEnd() {
-        if ( softEnd < 0 ) {
+        if ( softEnd == UNINITIALIZED ) {
+            boolean foundAlignedBase = false;
             softEnd = getAlignmentEnd();
             final List<CigarElement> cigs = getCigar().getCigarElements();
-            for (int i=cigs.size() - 1; i>=0; --i) {
+            for (int i = cigs.size() - 1; i >= 0; --i) {
                 final CigarElement cig = cigs.get(i);
                 final CigarOperator op = cig.getOperator();
 
-                if (op == CigarOperator.SOFT_CLIP)
+                if (op == CigarOperator.SOFT_CLIP) // assumes the soft clip that we found is at the end of the aligned read
                     softEnd += cig.getLength();
-                else if (op != CigarOperator.HARD_CLIP)
+                else if (op != CigarOperator.HARD_CLIP) {
+                    foundAlignedBase = true;
                     break;
+                }
+            }
+            if( !foundAlignedBase ) { // for example 64H14S, the soft end is actually the same as the alignment end
+                softEnd = getAlignmentEnd();
             }
         }
 

From fd59e7d5f65784182918a9f86f888cb3515bf1af Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Mon, 22 Oct 2012 16:27:31 -0400
Subject: [PATCH 06/40] Better error message when generic types are erased from
 scala collections.

---
 .../broadinstitute/sting/utils/exceptions/UserException.java   | 3 +++
 .../sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java b/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java
index 6b97f8f9f..c1f408bc7 100755
--- a/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java
+++ b/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java
@@ -352,6 +352,9 @@ public class UserException extends ReviewedStingException {
     }
 
     public static class CannotExecuteQScript extends UserException {
+        public CannotExecuteQScript(String message) {
+            super(String.format("Unable to execute QScript: " + message));
+        }
         public CannotExecuteQScript(String message, Exception e) {
             super(String.format("Unable to execute QScript: " + message), e);
         }
diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala b/public/scala/src/org/broadinstitute/sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala
index 0d8edc25d..54e89ec58 100644
--- a/public/scala/src/org/broadinstitute/sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala
+++ b/public/scala/src/org/broadinstitute/sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala
@@ -28,6 +28,7 @@ import collection.JavaConversions._
 import org.broadinstitute.sting.queue.QException
 import java.lang.Class
 import org.broadinstitute.sting.commandline.{ArgumentMatches, ArgumentSource, ArgumentTypeDescriptor, ParsingEngine}
+import org.broadinstitute.sting.utils.exceptions.UserException
 import java.lang.reflect.Type
 
 /**
@@ -75,6 +76,8 @@ class ScalaCompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor {
   
   def parse(parsingEngine: ParsingEngine, source: ArgumentSource, classType: Class[_], argumentMatches: ArgumentMatches) = {
     val componentType = ReflectionUtils.getCollectionType(source.field)
+    if (componentType == classOf[java.lang.Object])
+      throw new UserException.CannotExecuteQScript("Please also include a @ClassType(classOf[<primitive type>]) annotation on field: " + source.field + ". Example: @ClassType(classOf[Double]). The scala generic type for the field was subjected to java/scala type erasure and is not available via reflection.")
     val componentArgumentParser = parsingEngine.selectBestTypeDescriptor(componentType)
 
     if (classOf[Seq[_]].isAssignableFrom(classType)) {

From 15b28e61cd6f73c9ddbab1e06e8b0cb08e62572e Mon Sep 17 00:00:00 2001
From: Mark DePristo <depristo@broadinstitute.org>
Date: Mon, 22 Oct 2012 16:54:38 -0400
Subject: [PATCH 07/40] Retiring TraverseReads and TraverseLoci after testing
 confirms nano scheduler version in single threaded version is fine

-- There's been no report of problems with the nano scheduled version of TraverseLoci and TraverseReads, so I'm removing the old versions since they are no longer needed
-- Removing unnecessary intermediate base classes
-- GSA-515 / Nanoscheduler GSA-549 / https://jira.broadinstitute.org/browse/GSA-549
---
 .../sting/gatk/executive/MicroScheduler.java  |  12 +-
 .../gatk/traversals/TraverseLociBase.java     | 103 ----------------
 .../gatk/traversals/TraverseLociLinear.java   |  47 --------
 .../gatk/traversals/TraverseLociNano.java     |  85 +++++++++++++-
 .../gatk/traversals/TraverseReadPairs.java    |   2 +-
 .../sting/gatk/traversals/TraverseReads.java  | 111 ------------------
 .../traversals/TraverseReadsUnitTest.java     |  11 +-
 7 files changed, 89 insertions(+), 282 deletions(-)
 delete mode 100755 public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseLociBase.java
 delete mode 100755 public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseLociLinear.java
 delete mode 100755 public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseReads.java

diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java
index 223e11680..df4ed9ef8 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java
@@ -74,8 +74,6 @@ import java.util.*;
  *
  */
 public abstract class MicroScheduler implements MicroSchedulerMBean {
-    // TODO -- remove me and retire non nano scheduled versions of traversals
-    private final static boolean USE_NANOSCHEDULER_FOR_EVERYTHING = true;
     protected static final Logger logger = Logger.getLogger(MicroScheduler.class);
 
     /**
@@ -238,15 +236,9 @@ public abstract class MicroScheduler implements MicroSchedulerMBean {
     @Ensures("result != null")
     private TraversalEngine createTraversalEngine(final Walker walker, final ThreadAllocation threadAllocation) {
         if (walker instanceof ReadWalker) {
-            if ( USE_NANOSCHEDULER_FOR_EVERYTHING || threadAllocation.getNumCPUThreadsPerDataThread() > 1 )
-                return new TraverseReadsNano(threadAllocation.getNumCPUThreadsPerDataThread());
-            else
-                return new TraverseReads();
+            return new TraverseReadsNano(threadAllocation.getNumCPUThreadsPerDataThread());
         } else if (walker instanceof LocusWalker) {
-            if ( USE_NANOSCHEDULER_FOR_EVERYTHING || threadAllocation.getNumCPUThreadsPerDataThread() > 1 )
-                return new TraverseLociNano(threadAllocation.getNumCPUThreadsPerDataThread());
-            else
-                return new TraverseLociLinear();
+            return new TraverseLociNano(threadAllocation.getNumCPUThreadsPerDataThread());
         } else if (walker instanceof DuplicateWalker) {
             return new TraverseDuplicates();
         } else if (walker instanceof ReadPairWalker) {
diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseLociBase.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseLociBase.java
deleted file mode 100755
index 30e78ef5c..000000000
--- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseLociBase.java
+++ /dev/null
@@ -1,103 +0,0 @@
-package org.broadinstitute.sting.gatk.traversals;
-
-import org.apache.log4j.Logger;
-import org.broadinstitute.sting.gatk.WalkerManager;
-import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
-import org.broadinstitute.sting.gatk.datasources.providers.*;
-import org.broadinstitute.sting.gatk.walkers.DataSource;
-import org.broadinstitute.sting.gatk.walkers.LocusWalker;
-import org.broadinstitute.sting.gatk.walkers.Walker;
-import org.broadinstitute.sting.utils.GenomeLoc;
-import org.broadinstitute.sting.utils.pileup.ReadBackedPileupImpl;
-
-/**
- * A simple solution to iterating over all reference positions over a series of genomic locations.
- */
-public abstract class TraverseLociBase<M,T> extends TraversalEngine<M,T,LocusWalker<M,T>,LocusShardDataProvider> {
-    /**
-     * our log, which we want to capture anything from this class
-     */
-    protected static final Logger logger = Logger.getLogger(TraversalEngine.class);
-
-    @Override
-    public final String getTraversalUnits() {
-        return "sites";
-    }
-
-    protected static class TraverseResults<T> {
-        final int numIterations;
-        final T reduceResult;
-
-        public TraverseResults(int numIterations, T reduceResult) {
-            this.numIterations = numIterations;
-            this.reduceResult = reduceResult;
-        }
-    }
-
-    protected abstract TraverseResults<T> traverse( final LocusWalker<M,T> walker,
-                                                    final LocusView locusView,
-                                                    final LocusReferenceView referenceView,
-                                                    final ReferenceOrderedView referenceOrderedDataView,
-                                                    final T sum);
-
-    @Override
-    public T traverse( LocusWalker<M,T> walker,
-                       LocusShardDataProvider dataProvider,
-                       T sum) {
-        logger.debug(String.format("TraverseLociBase.traverse: Shard is %s", dataProvider));
-
-        final LocusView locusView = getLocusView( walker, dataProvider );
-
-        if ( locusView.hasNext() ) { // trivial optimization to avoid unnecessary processing when there's nothing here at all
-            //ReferenceOrderedView referenceOrderedDataView = new ReferenceOrderedView( dataProvider );
-            ReferenceOrderedView referenceOrderedDataView = null;
-            if ( WalkerManager.getWalkerDataSource(walker) != DataSource.REFERENCE_ORDERED_DATA )
-                referenceOrderedDataView = new ManagingReferenceOrderedView( dataProvider );
-            else
-                referenceOrderedDataView = (RodLocusView)locusView;
-
-            final LocusReferenceView referenceView = new LocusReferenceView( walker, dataProvider );
-
-            final TraverseResults<T> result = traverse( walker, locusView, referenceView, referenceOrderedDataView, sum );
-            sum = result.reduceResult;
-            dataProvider.getShard().getReadMetrics().incrementNumIterations(result.numIterations);
-            updateCumulativeMetrics(dataProvider.getShard());
-        }
-
-        // We have a final map call to execute here to clean up the skipped based from the
-        // last position in the ROD to that in the interval
-        if ( WalkerManager.getWalkerDataSource(walker) == DataSource.REFERENCE_ORDERED_DATA && ! walker.isDone() ) {
-            // only do this if the walker isn't done!
-            final RodLocusView rodLocusView = (RodLocusView)locusView;
-            final long nSkipped = rodLocusView.getLastSkippedBases();
-            if ( nSkipped > 0 ) {
-                final GenomeLoc site = rodLocusView.getLocOneBeyondShard();
-                final AlignmentContext ac = new AlignmentContext(site, new ReadBackedPileupImpl(site), nSkipped);
-                final M x = walker.map(null, null, ac);
-                sum = walker.reduce(x, sum);
-            }
-        }
-
-        return sum;
-    }
-
-    /**
-     * Gets the best view of loci for this walker given the available data.  The view will function as a 'trigger track'
-     * of sorts, providing a consistent interface so that TraverseLociBase doesn't need to be reimplemented for any new datatype
-     * that comes along.
-     * @param walker walker to interrogate.
-     * @param dataProvider Data which which to drive the locus view.
-     * @return A view of the locus data, where one iteration of the locus view maps to one iteration of the traversal.
-     */
-    private LocusView getLocusView( Walker<M,T> walker, LocusShardDataProvider dataProvider ) {
-        final DataSource dataSource = WalkerManager.getWalkerDataSource(walker);
-        if( dataSource == DataSource.READS )
-            return new CoveredLocusView(dataProvider);
-        else if( dataSource == DataSource.REFERENCE ) //|| ! GenomeAnalysisEngine.instance.getArguments().enableRodWalkers )
-            return new AllLocusView(dataProvider);
-        else if( dataSource == DataSource.REFERENCE_ORDERED_DATA )
-            return new RodLocusView(dataProvider);
-        else
-            throw new UnsupportedOperationException("Unsupported traversal type: " + dataSource);
-    }
-}
diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseLociLinear.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseLociLinear.java
deleted file mode 100755
index 22381092f..000000000
--- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseLociLinear.java
+++ /dev/null
@@ -1,47 +0,0 @@
-package org.broadinstitute.sting.gatk.traversals;
-
-import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
-import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
-import org.broadinstitute.sting.gatk.datasources.providers.LocusReferenceView;
-import org.broadinstitute.sting.gatk.datasources.providers.LocusView;
-import org.broadinstitute.sting.gatk.datasources.providers.ReferenceOrderedView;
-import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
-import org.broadinstitute.sting.gatk.walkers.LocusWalker;
-import org.broadinstitute.sting.utils.GenomeLoc;
-
-/**
- * A simple solution to iterating over all reference positions over a series of genomic locations.
- */
-public class TraverseLociLinear<M,T> extends TraverseLociBase<M,T> {
-
-    @Override
-    protected TraverseResults<T> traverse(LocusWalker<M, T> walker, LocusView locusView, LocusReferenceView referenceView, ReferenceOrderedView referenceOrderedDataView, T sum) {
-        // We keep processing while the next reference location is within the interval
-        boolean done = false;
-        int numIterations = 0;
-
-        while( locusView.hasNext() && ! done ) {
-            numIterations++;
-            final AlignmentContext locus = locusView.next();
-            final GenomeLoc location = locus.getLocation();
-
-            // create reference context. Note that if we have a pileup of "extended events", the context will
-            // hold the (longest) stretch of deleted reference bases (if deletions are present in the pileup).
-            final ReferenceContext refContext = referenceView.getReferenceContext(location);
-
-            // Iterate forward to get all reference ordered data covering this location
-            final RefMetaDataTracker tracker = referenceOrderedDataView.getReferenceOrderedDataAtLocus(locus.getLocation(), refContext);
-
-            final boolean keepMeP = walker.filter(tracker, refContext, locus);
-            if (keepMeP) {
-                final M x = walker.map(tracker, refContext, locus);
-                sum = walker.reduce(x, sum);
-                done = walker.isDone();
-            }
-
-            printProgress(locus.getLocation());
-        }
-
-        return new TraverseResults<T>(numIterations, sum);
-    }
-}
diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseLociNano.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseLociNano.java
index 469625c30..84715e5fd 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseLociNano.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseLociNano.java
@@ -1,24 +1,26 @@
 package org.broadinstitute.sting.gatk.traversals;
 
+import org.broadinstitute.sting.gatk.WalkerManager;
 import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
-import org.broadinstitute.sting.gatk.datasources.providers.LocusReferenceView;
-import org.broadinstitute.sting.gatk.datasources.providers.LocusView;
-import org.broadinstitute.sting.gatk.datasources.providers.ReferenceOrderedView;
+import org.broadinstitute.sting.gatk.datasources.providers.*;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
+import org.broadinstitute.sting.gatk.walkers.DataSource;
 import org.broadinstitute.sting.gatk.walkers.LocusWalker;
+import org.broadinstitute.sting.gatk.walkers.Walker;
 import org.broadinstitute.sting.utils.GenomeLoc;
 import org.broadinstitute.sting.utils.nanoScheduler.NSMapFunction;
 import org.broadinstitute.sting.utils.nanoScheduler.NSProgressFunction;
 import org.broadinstitute.sting.utils.nanoScheduler.NSReduceFunction;
 import org.broadinstitute.sting.utils.nanoScheduler.NanoScheduler;
+import org.broadinstitute.sting.utils.pileup.ReadBackedPileupImpl;
 
 import java.util.Iterator;
 
 /**
  * A simple solution to iterating over all reference positions over a series of genomic locations.
  */
-public class TraverseLociNano<M,T> extends TraverseLociBase<M,T> {
+public class TraverseLociNano<M,T> extends TraversalEngine<M,T,LocusWalker<M,T>,LocusShardDataProvider> {
     /** our log, which we want to capture anything from this class */
     private static final boolean DEBUG = false;
 
@@ -30,6 +32,81 @@ public class TraverseLociNano<M,T> extends TraverseLociBase<M,T> {
     }
 
     @Override
+    public final String getTraversalUnits() {
+        return "sites";
+    }
+
+    protected static class TraverseResults<T> {
+        final int numIterations;
+        final T reduceResult;
+
+        public TraverseResults(int numIterations, T reduceResult) {
+            this.numIterations = numIterations;
+            this.reduceResult = reduceResult;
+        }
+    }
+
+    @Override
+    public T traverse( LocusWalker<M,T> walker,
+                       LocusShardDataProvider dataProvider,
+                       T sum) {
+        logger.debug(String.format("TraverseLoci.traverse: Shard is %s", dataProvider));
+
+        final LocusView locusView = getLocusView( walker, dataProvider );
+
+        if ( locusView.hasNext() ) { // trivial optimization to avoid unnecessary processing when there's nothing here at all
+            //ReferenceOrderedView referenceOrderedDataView = new ReferenceOrderedView( dataProvider );
+            ReferenceOrderedView referenceOrderedDataView = null;
+            if ( WalkerManager.getWalkerDataSource(walker) != DataSource.REFERENCE_ORDERED_DATA )
+                referenceOrderedDataView = new ManagingReferenceOrderedView( dataProvider );
+            else
+                referenceOrderedDataView = (RodLocusView)locusView;
+
+            final LocusReferenceView referenceView = new LocusReferenceView( walker, dataProvider );
+
+            final TraverseResults<T> result = traverse( walker, locusView, referenceView, referenceOrderedDataView, sum );
+            sum = result.reduceResult;
+            dataProvider.getShard().getReadMetrics().incrementNumIterations(result.numIterations);
+            updateCumulativeMetrics(dataProvider.getShard());
+        }
+
+        // We have a final map call to execute here to clean up the skipped based from the
+        // last position in the ROD to that in the interval
+        if ( WalkerManager.getWalkerDataSource(walker) == DataSource.REFERENCE_ORDERED_DATA && ! walker.isDone() ) {
+            // only do this if the walker isn't done!
+            final RodLocusView rodLocusView = (RodLocusView)locusView;
+            final long nSkipped = rodLocusView.getLastSkippedBases();
+            if ( nSkipped > 0 ) {
+                final GenomeLoc site = rodLocusView.getLocOneBeyondShard();
+                final AlignmentContext ac = new AlignmentContext(site, new ReadBackedPileupImpl(site), nSkipped);
+                final M x = walker.map(null, null, ac);
+                sum = walker.reduce(x, sum);
+            }
+        }
+
+        return sum;
+    }
+
+    /**
+     * Gets the best view of loci for this walker given the available data.  The view will function as a 'trigger track'
+     * of sorts, providing a consistent interface so that TraverseLoci doesn't need to be reimplemented for any new datatype
+     * that comes along.
+     * @param walker walker to interrogate.
+     * @param dataProvider Data which which to drive the locus view.
+     * @return A view of the locus data, where one iteration of the locus view maps to one iteration of the traversal.
+     */
+    private LocusView getLocusView( Walker<M,T> walker, LocusShardDataProvider dataProvider ) {
+        final DataSource dataSource = WalkerManager.getWalkerDataSource(walker);
+        if( dataSource == DataSource.READS )
+            return new CoveredLocusView(dataProvider);
+        else if( dataSource == DataSource.REFERENCE ) //|| ! GenomeAnalysisEngine.instance.getArguments().enableRodWalkers )
+            return new AllLocusView(dataProvider);
+        else if( dataSource == DataSource.REFERENCE_ORDERED_DATA )
+            return new RodLocusView(dataProvider);
+        else
+            throw new UnsupportedOperationException("Unsupported traversal type: " + dataSource);
+    }
+
     protected TraverseResults<T> traverse(final LocusWalker<M, T> walker,
                                           final LocusView locusView,
                                           final LocusReferenceView referenceView,
diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseReadPairs.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseReadPairs.java
index aef3cf7d0..8273e1328 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseReadPairs.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseReadPairs.java
@@ -42,7 +42,7 @@ public class TraverseReadPairs<M,T> extends TraversalEngine<M,T, ReadPairWalker<
     public T traverse(ReadPairWalker<M, T> walker,
                       ReadShardDataProvider dataProvider,
                       T sum) {
-        logger.debug(String.format("TraverseReads.traverse Covered dataset is %s", dataProvider));
+        logger.debug(String.format("TraverseReadsPairs.traverse Covered dataset is %s", dataProvider));
 
         if( !dataProvider.hasReads() )
             throw new IllegalArgumentException("Unable to traverse reads; no read data is available.");
diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseReads.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseReads.java
deleted file mode 100755
index d41d17bde..000000000
--- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseReads.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Copyright (c) 2009 The Broad Institute
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-package org.broadinstitute.sting.gatk.traversals;
-
-import net.sf.samtools.SAMRecord;
-import org.apache.log4j.Logger;
-import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
-import org.broadinstitute.sting.gatk.datasources.providers.ReadBasedReferenceOrderedView;
-import org.broadinstitute.sting.gatk.datasources.providers.ReadReferenceView;
-import org.broadinstitute.sting.gatk.datasources.providers.ReadShardDataProvider;
-import org.broadinstitute.sting.gatk.datasources.providers.ReadView;
-import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
-import org.broadinstitute.sting.gatk.walkers.ReadWalker;
-import org.broadinstitute.sting.utils.GenomeLoc;
-import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
-
-/**
- * @author aaron
- * @version 1.0
- * @date Apr 24, 2009
- * <p/>
- * Class TraverseReads
- * <p/>
- * This class handles traversing by reads in the new shardable style
- */
-public class TraverseReads<M,T> extends TraversalEngine<M,T,ReadWalker<M,T>,ReadShardDataProvider> {    
-    /** our log, which we want to capture anything from this class */
-    protected static final Logger logger = Logger.getLogger(TraverseReads.class);
-
-    @Override
-    public String getTraversalUnits() {
-        return "reads";
-    }
-
-    /**
-     * Traverse by reads, given the data and the walker
-     *
-     * @param walker the walker to traverse with
-     * @param dataProvider the provider of the reads data
-     * @param sum the value of type T, specified by the walker, to feed to the walkers reduce function
-     * @return the reduce variable of the read walker
-     */
-    public T traverse(ReadWalker<M,T> walker,
-                      ReadShardDataProvider dataProvider,
-                      T sum) {
-
-        logger.debug(String.format("TraverseReads.traverse Covered dataset is %s", dataProvider));
-
-        if( !dataProvider.hasReads() )
-            throw new IllegalArgumentException("Unable to traverse reads; no read data is available.");
-
-        final ReadView reads = new ReadView(dataProvider);
-        final ReadReferenceView reference = new ReadReferenceView(dataProvider);
-
-        // get the reference ordered data
-        final ReadBasedReferenceOrderedView rodView = new ReadBasedReferenceOrderedView(dataProvider);
-
-        boolean done = walker.isDone();
-        // while we still have more reads
-        for (final SAMRecord read : reads) {
-            if ( done ) break;
-
-            // ReferenceContext -- the reference bases covered by the read
-            final ReferenceContext refContext = ! read.getReadUnmappedFlag() && dataProvider.hasReference()
-                    ? reference.getReferenceContext(read)
-                    : null;
-
-            // update the number of reads we've seen
-            dataProvider.getShard().getReadMetrics().incrementNumIterations();
-
-            // if the read is mapped, create a metadata tracker
-            final RefMetaDataTracker tracker = read.getReferenceIndex() >= 0 ? rodView.getReferenceOrderedDataForRead(read) : null;
-
-            final boolean keepMeP = walker.filter(refContext, (GATKSAMRecord) read);
-            if (keepMeP) {
-                M x = walker.map(refContext, (GATKSAMRecord) read, tracker); // the tracker can be null
-                sum = walker.reduce(x, sum);
-            }
-
-            final GenomeLoc locus = read.getReferenceIndex() == SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX ? null : engine.getGenomeLocParser().createGenomeLoc(read.getReferenceName(),read.getAlignmentStart());
-
-            updateCumulativeMetrics(dataProvider.getShard());
-            printProgress(locus);
-
-            done = walker.isDone();
-        }
-        return sum;
-    }
-}
diff --git a/public/java/test/org/broadinstitute/sting/gatk/traversals/TraverseReadsUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/traversals/TraverseReadsUnitTest.java
index 46f77c283..bf1fc9e65 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/traversals/TraverseReadsUnitTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/traversals/TraverseReadsUnitTest.java
@@ -6,13 +6,12 @@ import org.broadinstitute.sting.BaseTest;
 import org.broadinstitute.sting.commandline.Tags;
 import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
 import org.broadinstitute.sting.gatk.datasources.providers.ReadShardDataProvider;
-import org.broadinstitute.sting.gatk.datasources.providers.ShardDataProvider;
 import org.broadinstitute.sting.gatk.datasources.reads.ReadShardBalancer;
 import org.broadinstitute.sting.gatk.datasources.reads.SAMDataSource;
 import org.broadinstitute.sting.gatk.datasources.reads.SAMReaderID;
 import org.broadinstitute.sting.gatk.datasources.reads.Shard;
 import org.broadinstitute.sting.gatk.resourcemanagement.ThreadAllocation;
-import org.broadinstitute.sting.gatk.walkers.Walker;
+import org.broadinstitute.sting.gatk.walkers.ReadWalker;
 import org.broadinstitute.sting.gatk.walkers.qc.CountReads;
 import org.broadinstitute.sting.utils.GenomeLocParser;
 import org.broadinstitute.sting.utils.exceptions.UserException;
@@ -62,9 +61,9 @@ public class TraverseReadsUnitTest extends BaseTest {
     private SAMReaderID bam = new SAMReaderID(new File(validationDataLocation + "index_test.bam"),new Tags()); // TCGA-06-0188.aligned.duplicates_marked.bam");
     private File refFile = new File(validationDataLocation + "Homo_sapiens_assembly17.fasta");
     private List<SAMReaderID> bamList;
-    private Walker countReadWalker;
+    private ReadWalker countReadWalker;
     private File output;
-    private TraverseReads traversalEngine = null;
+    private TraverseReadsNano traversalEngine = null;
 
     private IndexedFastaSequenceFile ref = null;
     private GenomeLocParser genomeLocParser = null;
@@ -107,7 +106,7 @@ public class TraverseReadsUnitTest extends BaseTest {
         bamList.add(bam);
         countReadWalker = new CountReads();
         
-        traversalEngine = new TraverseReads();
+        traversalEngine = new TraverseReadsNano(1);
         traversalEngine.initialize(engine);
     }
 
@@ -125,7 +124,7 @@ public class TraverseReadsUnitTest extends BaseTest {
                 fail("Shard == null");
             }
 
-            ShardDataProvider dataProvider = new ReadShardDataProvider(shard,genomeLocParser,dataSource.seek(shard),null,null);
+            ReadShardDataProvider dataProvider = new ReadShardDataProvider(shard,genomeLocParser,dataSource.seek(shard),null,null);
             accumulator = traversalEngine.traverse(countReadWalker, dataProvider, accumulator);
             dataProvider.close();
         }

From f838815343d971d3a9ff2eda0df931aa69654c52 Mon Sep 17 00:00:00 2001
From: Mark DePristo <depristo@broadinstitute.org>
Date: Tue, 23 Oct 2012 06:47:53 -0400
Subject: [PATCH 08/40] Updating MD5s for confidence ref site estimation in
 IndependentAllelesDiploidExactAFCalc

-- Included logic to only add priors for alleles with sufficient evidence to be called polymorphic.  If no alleles are poly make sure to add priors of first allele
---
 .../afcalc/IndependentAllelesDiploidExactAFCalc.java      | 8 ++++++++
 .../genotyper/UnifiedGenotyperIntegrationTest.java        | 4 ++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/IndependentAllelesDiploidExactAFCalc.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/IndependentAllelesDiploidExactAFCalc.java
index 857b7e59e..d0b801a20 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/IndependentAllelesDiploidExactAFCalc.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/IndependentAllelesDiploidExactAFCalc.java
@@ -329,6 +329,7 @@ import java.util.*;
         double log10PosteriorOfACEq0Sum = 0.0;
         double log10PosteriorOfACGt0Sum = 0.0;
 
+        boolean anyPoly = false;
         for ( final AFCalcResult sortedResultWithThetaNPriors : sortedResultsWithThetaNPriors ) {
             final Allele altAllele = sortedResultWithThetaNPriors.getAllelesUsedInGenotyping().get(1);
             final int altI = vc.getAlleles().indexOf(altAllele) - 1;
@@ -338,6 +339,7 @@ import java.util.*;
 
             // the AF > 0 case requires us to store the normalized likelihood for later summation
             if ( sortedResultWithThetaNPriors.getLog10PosteriorOfAFGT0() > MIN_LOG10_CONFIDENCE_TO_INCLUDE_ALLELE_IN_POSTERIOR ) {
+                anyPoly = true;
                 log10PosteriorOfACEq0Sum += sortedResultWithThetaNPriors.getLog10PosteriorOfAFEq0();
                 log10PriorsOfAC[0] += sortedResultWithThetaNPriors.getLog10PriorOfAFEq0();
                 log10PriorsOfAC[1] += sortedResultWithThetaNPriors.getLog10PriorOfAFGT0();
@@ -352,6 +354,12 @@ import java.util.*;
             nEvaluations += sortedResultWithThetaNPriors.nEvaluations;
         }
 
+        // If no alleles were polymorphic, make sure we have the proper priors (the defaults) for likelihood calculation
+        if ( ! anyPoly ) {
+            log10PriorsOfAC[0] = sortedResultsWithThetaNPriors.get(0).getLog10PriorOfAFEq0();
+            log10PriorsOfAC[1] = sortedResultsWithThetaNPriors.get(0).getLog10PriorOfAFGT0();
+        }
+
         // In principle, if B_p = x and C_p = y are the probabilities of being poly for alleles B and C,
         // the probability of being poly is (1 - B_p) * (1 - C_p) = (1 - x) * (1 - y).  We want to estimate confidently
         // log10((1 - x) * (1 - y)) which is log10(1 - x) + log10(1 - y).  This sum is log10PosteriorOfACEq0
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index a1701e3e5..dee54b9f8 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -192,12 +192,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
     @Test
     public void testOutputParameterAllConfident() {
-        testOutputParameters("--output_mode EMIT_ALL_CONFIDENT_SITES", "8800c58715c2bb434b69e1873cb77de6");
+        testOutputParameters("--output_mode EMIT_ALL_CONFIDENT_SITES", "f9ea04d96eeef29e71d37e60518c2579");
     }
 
     @Test
     public void testOutputParameterAllSites() {
-        testOutputParameters("--output_mode EMIT_ALL_SITES", "639df9f4c029792ac2e46069efc82b20");
+        testOutputParameters("--output_mode EMIT_ALL_SITES", "41c046d38ea328421df924e37e017645");
     }
 
     private void testOutputParameters(final String args, final String md5) {

From 6e421a72d642b1e2477af3528a2248859332cd0f Mon Sep 17 00:00:00 2001
From: Mark DePristo <depristo@broadinstitute.org>
Date: Tue, 23 Oct 2012 20:10:39 -0400
Subject: [PATCH 09/40] Add more exhaustive unit tests for input errors to
 NanoScheduler

-- Resolves issue GSA-515 / Nanoscheduler GSA-605 / Seems that -nct may deadlock as not reproducible
-- It seems that it's not an input error problem (or at least cannot be provoked with unit tests)
-- I'll keep an eye on this later
---
 .../nanoScheduler/NanoSchedulerUnitTest.java  | 100 +++++++++++++-----
 1 file changed, 73 insertions(+), 27 deletions(-)

diff --git a/public/java/test/org/broadinstitute/sting/utils/nanoScheduler/NanoSchedulerUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/nanoScheduler/NanoSchedulerUnitTest.java
index 2ba2da734..af2e18ad9 100644
--- a/public/java/test/org/broadinstitute/sting/utils/nanoScheduler/NanoSchedulerUnitTest.java
+++ b/public/java/test/org/broadinstitute/sting/utils/nanoScheduler/NanoSchedulerUnitTest.java
@@ -23,6 +23,7 @@ import java.util.List;
  * To change this template use File | Settings | File Templates.
  */
 public class NanoSchedulerUnitTest extends BaseTest {
+    private final static boolean DEBUG = false;
     private final static boolean debug = false;
     public static final int NANO_SCHEDULE_MAX_RUNTIME = 30000;
 
@@ -30,19 +31,22 @@ public class NanoSchedulerUnitTest extends BaseTest {
         @Override public Integer apply(Integer input) { return input * 2; }
     }
 
+    private static void maybeDelayMe(final int input) {
+        try {
+            if ( input % 7 == 0 ) {
+                final int milliToSleep = (input % 10);
+                //System.out.printf("Sleeping %d millseconds%n", milliToSleep);
+                Thread.sleep(milliToSleep);
+            }
+        } catch ( InterruptedException ex ) {
+            throw new RuntimeException(ex);
+        }
+    }
+
     private static class Map2xWithDelays extends Map2x {
         @Override public Integer apply(Integer input) {
-            try {
-                if ( input % 7 == 0 ) {
-                    final int milliToSleep = (input % 10);
-                    //System.out.printf("Sleeping %d millseconds%n", milliToSleep);
-                    Thread.sleep(milliToSleep);
-                }
-
-                return input * 2;
-            } catch ( InterruptedException ex ) {
-                throw new RuntimeException(ex);
-            }
+            maybeDelayMe(input);
+            return input * 2;
         }
     }
 
@@ -117,10 +121,12 @@ public class NanoSchedulerUnitTest extends BaseTest {
     }
 
     static NanoSchedulerBasicTest exampleTest = null;
+    static NanoSchedulerBasicTest exampleTestWithDelays = null;
 
     @BeforeSuite
     public void setUp() throws Exception {
         exampleTest = new NanoSchedulerBasicTest(10, 2, 1, 10, false);
+        exampleTestWithDelays = new NanoSchedulerBasicTest(10, 2, 1, 10, true);
     }
 
     @DataProvider(name = "NanoSchedulerBasicTest")
@@ -151,14 +157,14 @@ public class NanoSchedulerUnitTest extends BaseTest {
         return NanoSchedulerBasicTest.getTests(NanoSchedulerBasicTest.class);
     }
 
-    @Test(enabled = true, dataProvider = "NanoSchedulerBasicTest", timeOut = NANO_SCHEDULE_MAX_RUNTIME)
+    @Test(enabled = true && ! DEBUG, dataProvider = "NanoSchedulerBasicTest", timeOut = NANO_SCHEDULE_MAX_RUNTIME)
     public void testSingleThreadedNanoScheduler(final NanoSchedulerBasicTest test) throws InterruptedException {
         logger.warn("Running " + test);
         if ( test.nThreads == 1 )
             testNanoScheduler(test);
     }
 
-    @Test(enabled = true, dataProvider = "NanoSchedulerBasicTest", timeOut = NANO_SCHEDULE_MAX_RUNTIME, dependsOnMethods = "testSingleThreadedNanoScheduler")
+    @Test(enabled = true && ! DEBUG, dataProvider = "NanoSchedulerBasicTest", timeOut = NANO_SCHEDULE_MAX_RUNTIME, dependsOnMethods = "testSingleThreadedNanoScheduler")
     public void testMultiThreadedNanoScheduler(final NanoSchedulerBasicTest test) throws InterruptedException {
         logger.warn("Running " + test);
         if ( test.nThreads >= 1 )
@@ -195,7 +201,7 @@ public class NanoSchedulerUnitTest extends BaseTest {
         }
     }
 
-    @Test(enabled = true, dataProvider = "NanoSchedulerBasicTest", dependsOnMethods = "testMultiThreadedNanoScheduler", timeOut = NANO_SCHEDULE_MAX_RUNTIME)
+    @Test(enabled = true && ! DEBUG, dataProvider = "NanoSchedulerBasicTest", dependsOnMethods = "testMultiThreadedNanoScheduler", timeOut = NANO_SCHEDULE_MAX_RUNTIME)
     public void testNanoSchedulerInLoop(final NanoSchedulerBasicTest test) throws InterruptedException {
         if ( test.bufferSize > 1) {
             logger.warn("Running " + test);
@@ -213,7 +219,7 @@ public class NanoSchedulerUnitTest extends BaseTest {
         }
     }
 
-    @Test(timeOut = NANO_SCHEDULE_MAX_RUNTIME)
+    @Test(enabled = true && ! DEBUG, timeOut = NANO_SCHEDULE_MAX_RUNTIME)
     public void testShutdown() throws InterruptedException {
         final NanoScheduler<Integer, Integer, Integer> nanoScheduler = new NanoScheduler<Integer, Integer, Integer>(1, 2);
         Assert.assertFalse(nanoScheduler.isShutdown(), "scheduler should be alive");
@@ -221,38 +227,78 @@ public class NanoSchedulerUnitTest extends BaseTest {
         Assert.assertTrue(nanoScheduler.isShutdown(), "scheduler should be dead");
     }
 
-    @Test(expectedExceptions = IllegalStateException.class, timeOut = NANO_SCHEDULE_MAX_RUNTIME)
+    @Test(enabled = true && ! DEBUG, expectedExceptions = IllegalStateException.class, timeOut = NANO_SCHEDULE_MAX_RUNTIME)
     public void testShutdownExecuteFailure() throws InterruptedException {
         final NanoScheduler<Integer, Integer, Integer> nanoScheduler = new NanoScheduler<Integer, Integer, Integer>(1, 2);
         nanoScheduler.shutdown();
         nanoScheduler.execute(exampleTest.makeReader(), exampleTest.makeMap(), exampleTest.initReduce(), exampleTest.makeReduce());
     }
 
-    @Test(expectedExceptions = NullPointerException.class, timeOut = 10000, invocationCount = 50)
+    @DataProvider(name = "NanoSchedulerInputExceptionTest")
+    public Object[][] createNanoSchedulerInputExceptionTest() {
+        List<Object[]> tests = new ArrayList<Object[]>();
+
+
+        for ( final int bufSize : Arrays.asList(100) ) {
+            for ( final int nThreads : Arrays.asList(8) ) {
+                for ( final boolean addDelays : Arrays.asList(true, false) ) {
+                    final NanoSchedulerBasicTest test = new NanoSchedulerBasicTest(bufSize, nThreads, 1, 1000000, false);
+                    final int maxN = addDelays ? 10000 : 100000;
+                    for ( int nElementsBeforeError = 0; nElementsBeforeError < maxN; nElementsBeforeError += Math.max(nElementsBeforeError / 10, 1) ) {
+                        tests.add(new Object[]{nElementsBeforeError, test, addDelays});
+                    }
+                }
+            }
+        }
+
+        return tests.toArray(new Object[][]{});
+    }
+
+    @Test(enabled = true, expectedExceptions = NullPointerException.class, timeOut = 10000)
     public void testInputErrorIsThrown_NPE() throws InterruptedException {
-        executeTestErrorThrowingInput(new NullPointerException());
+        executeTestErrorThrowingInput(10, new NullPointerException(), exampleTest, false);
     }
 
-    @Test(expectedExceptions = ReviewedStingException.class, timeOut = 10000, invocationCount = 50)
+    @Test(enabled = true, expectedExceptions = ReviewedStingException.class, timeOut = 10000)
     public void testInputErrorIsThrown_RSE() throws InterruptedException {
-        executeTestErrorThrowingInput(new ReviewedStingException("test"));
+        executeTestErrorThrowingInput(10, new ReviewedStingException("test"), exampleTest, false);
     }
 
-    private void executeTestErrorThrowingInput(final RuntimeException ex) {
-        final NanoScheduler<Integer, Integer, Integer> nanoScheduler = new NanoScheduler<Integer, Integer, Integer>(1, 2);
-        nanoScheduler.execute(new ErrorThrowingIterator(ex), exampleTest.makeMap(), exampleTest.initReduce(), exampleTest.makeReduce());
+    @Test(enabled = true, expectedExceptions = NullPointerException.class, dataProvider = "NanoSchedulerInputExceptionTest", timeOut = 10000, invocationCount = 1)
+    public void testInputErrorDoesntDeadlock(final int nElementsBeforeError, final NanoSchedulerBasicTest test, final boolean addDelays ) throws InterruptedException {
+        executeTestErrorThrowingInput(nElementsBeforeError, new NullPointerException(), test, addDelays);
+    }
+
+    private void executeTestErrorThrowingInput(final int nElementsBeforeError, final RuntimeException ex, final NanoSchedulerBasicTest test, final boolean addDelays) {
+        logger.warn("executeTestErrorThrowingInput " + nElementsBeforeError + " ex=" + ex + " test=" + test + " addInputDelays=" + addDelays);
+        final NanoScheduler<Integer, Integer, Integer> nanoScheduler = test.makeScheduler();
+        nanoScheduler.execute(new ErrorThrowingIterator(nElementsBeforeError, ex, addDelays), test.makeMap(), test.initReduce(), test.makeReduce());
     }
 
     private static class ErrorThrowingIterator implements Iterator<Integer> {
+        final int nElementsBeforeError;
+        final boolean addDelays;
+        int i = 0;
         final RuntimeException ex;
 
-        private ErrorThrowingIterator(RuntimeException ex) {
+        private ErrorThrowingIterator(final int nElementsBeforeError, RuntimeException ex, boolean addDelays) {
+            this.nElementsBeforeError = nElementsBeforeError;
             this.ex = ex;
+            this.addDelays = addDelays;
         }
 
-        @Override public boolean hasNext() { throw ex; }
-        @Override public Integer next() { throw ex; }
-        @Override public void remove() { throw ex; }
+        @Override public boolean hasNext() { return true; }
+        @Override public Integer next() {
+            if ( i++ > nElementsBeforeError ) {
+                throw ex;
+            } else if ( addDelays ) {
+                maybeDelayMe(i);
+                return i;
+            } else {
+                return i;
+            }
+        }
+        @Override public void remove() { throw new UnsupportedOperationException("x"); }
     }
 
     public static void main(String [ ] args) {

From 5b7b42356b803e691cd864a178f3306bb9de551b Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Wed, 24 Oct 2012 14:02:50 -0400
Subject: [PATCH 10/40] Fix bug in GenotypeAndValidate where it doesn't check
 vc.hasAttribute() before using vc.getAttribute().

---
 .../validation/GenotypeAndValidate.java       | 37 ++++++++++---------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/GenotypeAndValidate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/GenotypeAndValidate.java
index 5beeeb3e6..832c0ede3 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/GenotypeAndValidate.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/GenotypeAndValidate.java
@@ -245,24 +245,21 @@ public class GenotypeAndValidate extends RodWalker<GenotypeAndValidate.CountedDa
     @Argument(fullName="condition_on_depth", shortName="depth", doc="Condition validation on a minimum depth of coverage by the reads", required=false)
     private int minDepth = -1;
 
-    /**
-     * If your VCF or BAM file has more than one sample and you only want to validate one, use this parameter to choose it.
-     */
-    @Hidden
-    @Argument(fullName ="sample", shortName ="sn", doc="Name of the sample to validate (in case your VCF/BAM has more than one sample)", required=false)
-    private String sample = "";
-
-    /**
+   /**
      * Print out discordance sites to standard out.
      */
     @Hidden
     @Argument(fullName ="print_interesting_sites", shortName ="print_interesting", doc="Print out interesting sites to standard out", required=false)
-    private boolean printInterestingSites;
+    private boolean printInterestingSites = false;
 
     private UnifiedGenotyperEngine snpEngine;
     private UnifiedGenotyperEngine indelEngine;
     private Set<String> samples;
 
+    private enum GVstatus {
+        T, F, NONE
+    }
+
     public static class CountedData {
         private long nAltCalledAlt = 0L;
         private long nAltCalledRef = 0L;
@@ -368,9 +365,10 @@ public class GenotypeAndValidate extends RodWalker<GenotypeAndValidate.CountedDa
         // Do not operate on variants that are not covered to the optional minimum depth
         if (!context.hasReads() || (minDepth > 0 && context.getBasePileup().getBases().length < minDepth)) {
             counter.nUncovered = 1L;
-            if (vcComp.getAttribute("GV").equals("T"))
+            final GVstatus status = getGVstatus(vcComp);
+            if ( status == GVstatus.T )
                 counter.nAltNotCalled = 1L;
-            else if (vcComp.getAttribute("GV").equals("F"))
+            else if ( status == GVstatus.F )
                 counter.nRefNotCalled = 1L;
             else
                 counter.nNoStatusNotCalled = 1L;
@@ -427,10 +425,11 @@ public class GenotypeAndValidate extends RodWalker<GenotypeAndValidate.CountedDa
 //            if (!vcComp.hasExtendedAttribute("GV"))
 //                throw new UserException.BadInput("Variant has no GV annotation in the INFO field. " + vcComp.getChr() + ":" + vcComp.getStart());
 
+            final GVstatus status = getGVstatus(vcComp);
             if (call.isCalledAlt(callConf)) {
-                if (vcComp.getAttribute("GV").equals("T"))
+                if ( status == GVstatus.T )
                     counter.nAltCalledAlt = 1L;
-                else if (vcComp.getAttribute("GV").equals("F")) {
+                else if ( status == GVstatus.F ) {
                     counter.nRefCalledAlt = 1L;
                     if ( printInterestingSites )
                         System.out.println("Truth=REF Call=ALT at " + call.getChr() + ":" + call.getStart());
@@ -439,12 +438,12 @@ public class GenotypeAndValidate extends RodWalker<GenotypeAndValidate.CountedDa
                     counter.nNoStatusCalledAlt = 1L;
             }
             else if (call.isCalledRef(callConf)) {
-                if (vcComp.getAttribute("GV").equals("T")) {
+                if ( status == GVstatus.T ) {
                     counter.nAltCalledRef = 1L;
                     if ( printInterestingSites )
                         System.out.println("Truth=ALT Call=REF at " + call.getChr() + ":" + call.getStart());
                 }
-                else if (vcComp.getAttribute("GV").equals("F"))
+                else if ( status == GVstatus.F )
                     counter.nRefCalledRef = 1L;
 
                 else
@@ -452,9 +451,9 @@ public class GenotypeAndValidate extends RodWalker<GenotypeAndValidate.CountedDa
             }
             else {
                 counter.nNotConfidentCalls = 1L;
-                if (vcComp.getAttribute("GV").equals("T"))
+                if ( status == GVstatus.T )
                     counter.nAltNotCalled = 1L;
-                else if (vcComp.getAttribute("GV").equals("F"))
+                else if ( status == GVstatus.F )
                     counter.nRefNotCalled = 1L;
                 else
                     counter.nNoStatusNotCalled = 1L;
@@ -475,6 +474,10 @@ public class GenotypeAndValidate extends RodWalker<GenotypeAndValidate.CountedDa
         return counter;
     }
 
+    private GVstatus getGVstatus(final VariantContext vc) {
+        return ( !vc.hasAttribute("GV") ) ? GVstatus.NONE : (vc.getAttribute("GV").equals("T") ? GVstatus.T : GVstatus.F);
+    }
+
     //---------------------------------------------------------------------------------------------------------------
     //
     // reduce

From 094db7bf2468352811764d4e98f38d637516a577 Mon Sep 17 00:00:00 2001
From: Ryan Poplin <rpoplin@broadinstitute.org>
Date: Wed, 24 Oct 2012 14:07:36 -0400
Subject: [PATCH 11/40] We now require at least 10 samples to merge variants
 into complex events in the HC. Added a new population based bam for the
 complex event integration test.

---
 .../sting/gatk/walkers/haplotypecaller/GenotypingEngine.java  | 2 +-
 .../walkers/haplotypecaller/LikelihoodCalculationEngine.java  | 2 +-
 .../haplotypecaller/HaplotypeCallerIntegrationTest.java       | 4 +---
 3 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/GenotypingEngine.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/GenotypingEngine.java
index 8738def50..d91df82e2 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/GenotypingEngine.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/GenotypingEngine.java
@@ -217,7 +217,7 @@ public class GenotypingEngine {
         }
 
         cleanUpSymbolicUnassembledEvents( haplotypes );
-        if( activeAllelesToGenotype.isEmpty() && haplotypes.get(0).getSampleKeySet().size() >= 3 ) { // if not in GGA mode and have at least 3 samples try to create MNP and complex events by looking at LD structure
+        if( activeAllelesToGenotype.isEmpty() && haplotypes.get(0).getSampleKeySet().size() >= 10 ) { // if not in GGA mode and have at least 10 samples try to create MNP and complex events by looking at LD structure
             mergeConsecutiveEventsBasedOnLD( haplotypes, startPosKeySet, ref, refLoc );
         }
         if( !activeAllelesToGenotype.isEmpty() ) { // we are in GGA mode!
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java
index 62554c4ab..e90d89a8c 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java
@@ -139,7 +139,7 @@ public class LikelihoodCalculationEngine {
     }
 
     private static int computeFirstDifferingPosition( final byte[] b1, final byte[] b2 ) {
-        for( int iii = 0; iii < b1.length && iii < b2.length; iii++ ){
+        for( int iii = 0; iii < b1.length && iii < b2.length; iii++ ) {
             if( b1[iii] != b2[iii] ) {
                 return iii;
             }
diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
index 1d4fad19e..a0849d0c1 100644
--- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
+++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
@@ -42,7 +42,7 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
 
     @Test
     public void testHaplotypeCallerMultiSampleComplex() {
-        HCTestComplexVariants(CEUTRIO_BAM, "", "30598abeeb0b0ae5816ffdbf0c4044fd");
+        HCTestComplexVariants(privateTestDir + "AFR.complex.variants.bam", "", "966d338f423c86a390d685aa6336ec69");
     }
 
     private void HCTestSymbolicVariants(String bam, String args, String md5) {
@@ -94,6 +94,4 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
                 Arrays.asList("79af83432dc4a1768b3ebffffc4d2b8f"));
         executeTest("HC calling on a ReducedRead BAM", spec);
     }
-
-
 }

From a27ee26481e94b182003212944d75001e6a1a4c0 Mon Sep 17 00:00:00 2001
From: Ryan Poplin <rpoplin@broadinstitute.org>
Date: Wed, 24 Oct 2012 14:08:39 -0400
Subject: [PATCH 12/40] updating HC integration test.

---
 .../walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
index a0849d0c1..71f03ea00 100644
--- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
+++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
@@ -21,7 +21,7 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
 
     @Test
     public void testHaplotypeCallerMultiSample() {
-        HCTest(CEUTRIO_BAM, "", "ee866a8694a6f6c77242041275350ab9");
+        HCTest(CEUTRIO_BAM, "", "1d826f852ec1457efbfa7ee828c5783a");
     }
 
     @Test

From 991658acf485c3c7634387eea0dab46018aa6894 Mon Sep 17 00:00:00 2001
From: David Roazen <droazen@broadinstitute.org>
Date: Mon, 22 Oct 2012 13:36:07 -0400
Subject: [PATCH 13/40] BQSR: use more granular locking for concurrency control

-With this change, BQSR performance scales properly by thread rather
 than gaining nothing from additional threads.
-Benefits are seen when using either -nt (HierarchicalMicroScheduler) or -nct
 (NanoScheduler)
-Removes high-level locks in the recalibration engines and NestedIntegerArray
 in favor of maximally-granular locks on and around manipulation of the leaf
 nodes of the NestedIntegerArray.
-NestedIntegerArray now creates all interior nodes upfront rather than on
 the fly to avoid the need for locking during tree traversals. This uses
 more memory in the initial part of BQSR runs, but the BQSR would eventually
 converge to use this memory anyway over the course of a typical run.

IMPORTANT NOTE: This does not mean it's safe to run the old BaseRecalibrator
walker with multiple threads. The BaseRecalibrator walker is and will never be
thread-safe, as it's a LocusWalker that uses read attributes to track
state information. ONLY the newer DelocalizedBaseRecalibrator can be made
thread-safe (and will hopefully be made so in my subsequent commits). This
commit addresses performance, not correctness.
---
 .../bqsr/AdvancedRecalibrationEngine.java     |  71 +++-----
 .../bqsr/StandardRecalibrationEngine.java     | 157 +++++++++++++++---
 .../LoggingNestedIntegerArray.java            |   6 +-
 .../utils/collections/NestedIntegerArray.java |  56 +++++--
 .../utils/threading/ThreadLocalArray.java     |  64 +++++++
 5 files changed, 270 insertions(+), 84 deletions(-)
 create mode 100644 public/java/src/org/broadinstitute/sting/utils/threading/ThreadLocalArray.java

diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/AdvancedRecalibrationEngine.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/AdvancedRecalibrationEngine.java
index b89f68e24..be8357679 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/AdvancedRecalibrationEngine.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/AdvancedRecalibrationEngine.java
@@ -28,26 +28,22 @@ package org.broadinstitute.sting.gatk.walkers.bqsr;
 import org.broadinstitute.sting.utils.recalibration.covariates.Covariate;
 import org.broadinstitute.sting.utils.BaseUtils;
 import org.broadinstitute.sting.utils.classloader.ProtectedPackageSource;
-import org.broadinstitute.sting.utils.collections.NestedIntegerArray;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.recalibration.EventType;
 import org.broadinstitute.sting.utils.recalibration.ReadCovariates;
-import org.broadinstitute.sting.utils.recalibration.RecalDatum;
 import org.broadinstitute.sting.utils.recalibration.RecalibrationTables;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
+import org.broadinstitute.sting.utils.threading.ThreadLocalArray;
 
 public class AdvancedRecalibrationEngine extends StandardRecalibrationEngine implements ProtectedPackageSource {
 
-    // optimizations: don't reallocate an array each time
-    private byte[] tempQualArray;
-    private boolean[] tempErrorArray;
-    private double[] tempFractionalErrorArray;
+    // optimization: only allocate temp arrays once per thread
+    private final ThreadLocal<byte[]> threadLocalTempQualArray = new ThreadLocalArray<byte[]>(EventType.values().length, byte.class);
+    private final ThreadLocal<boolean[]> threadLocalTempErrorArray = new ThreadLocalArray<boolean[]>(EventType.values().length, boolean.class);
+    private final ThreadLocal<double[]> threadLocalTempFractionalErrorArray = new ThreadLocalArray<double[]>(EventType.values().length, double.class);
 
     public void initialize(final Covariate[] covariates, final RecalibrationTables recalibrationTables) {
         super.initialize(covariates, recalibrationTables);
-        tempQualArray = new byte[EventType.values().length];
-        tempErrorArray = new boolean[EventType.values().length];
-        tempFractionalErrorArray = new double[EventType.values().length];
     }
 
     /**
@@ -60,10 +56,13 @@ public class AdvancedRecalibrationEngine extends StandardRecalibrationEngine imp
      * @param refBase       The reference base at this locus
      */
     @Override
-    public synchronized void updateDataForPileupElement(final PileupElement pileupElement, final byte refBase) {
+    public void updateDataForPileupElement(final PileupElement pileupElement, final byte refBase) {
         final int offset = pileupElement.getOffset();
         final ReadCovariates readCovariates = covariateKeySetFrom(pileupElement.getRead());
 
+        byte[] tempQualArray = threadLocalTempQualArray.get();
+        boolean[] tempErrorArray = threadLocalTempErrorArray.get();
+
         tempQualArray[EventType.BASE_SUBSTITUTION.index] = pileupElement.getQual();
         tempErrorArray[EventType.BASE_SUBSTITUTION.index] = !BaseUtils.basesAreEqual(pileupElement.getBase(), refBase);
         tempQualArray[EventType.BASE_INSERTION.index] = pileupElement.getBaseInsertionQual();
@@ -77,40 +76,29 @@ public class AdvancedRecalibrationEngine extends StandardRecalibrationEngine imp
             final byte qual = tempQualArray[eventIndex];
             final boolean isError = tempErrorArray[eventIndex];
 
-            final NestedIntegerArray<RecalDatum> rgRecalTable = recalibrationTables.getReadGroupTable();
-            final RecalDatum rgPreviousDatum = rgRecalTable.get(keys[0], eventIndex);
-            final RecalDatum rgThisDatum = createDatumObject(qual, isError);
-            if (rgPreviousDatum == null) // key doesn't exist yet in the map so make a new bucket and add it
-                rgRecalTable.put(rgThisDatum, keys[0], eventIndex);
-            else
-                rgPreviousDatum.combine(rgThisDatum);
+            // TODO: should this really be combine rather than increment?
+            combineDatumOrPutIfNecessary(recalibrationTables.getReadGroupTable(), qual, isError, keys[0], eventIndex);
 
-            final NestedIntegerArray<RecalDatum> qualRecalTable = recalibrationTables.getQualityScoreTable();
-            final RecalDatum qualPreviousDatum = qualRecalTable.get(keys[0], keys[1], eventIndex);
-            if (qualPreviousDatum == null)
-                qualRecalTable.put(createDatumObject(qual, isError), keys[0], keys[1], eventIndex);
-            else
-                qualPreviousDatum.increment(isError);
+            incrementDatumOrPutIfNecessary(recalibrationTables.getQualityScoreTable(), qual, isError, keys[0], keys[1], eventIndex);
 
             for (int i = 2; i < covariates.length; i++) {
                 if (keys[i] < 0)
                     continue;
-                final NestedIntegerArray<RecalDatum> covRecalTable = recalibrationTables.getTable(i);
-                final RecalDatum covPreviousDatum = covRecalTable.get(keys[0], keys[1], keys[i], eventIndex);
-                if (covPreviousDatum == null)
-                    covRecalTable.put(createDatumObject(qual, isError), keys[0], keys[1], keys[i], eventIndex);
-                else
-                    covPreviousDatum.increment(isError);
+
+                incrementDatumOrPutIfNecessary(recalibrationTables.getTable(i), qual, isError, keys[0], keys[1], keys[i], eventIndex);
             }
         }
     }
 
     @Override
-    public synchronized void updateDataForRead(final GATKSAMRecord read, final boolean[] skip, final double[] snpErrors, final double[] insertionErrors, final double[] deletionErrors ) {
+    public void updateDataForRead(final GATKSAMRecord read, final boolean[] skip, final double[] snpErrors, final double[] insertionErrors, final double[] deletionErrors ) {
         for( int offset = 0; offset < read.getReadBases().length; offset++ ) {
             if( !skip[offset] ) {
                 final ReadCovariates readCovariates = covariateKeySetFrom(read);
 
+                byte[] tempQualArray = threadLocalTempQualArray.get();
+                double[] tempFractionalErrorArray = threadLocalTempFractionalErrorArray.get();
+
                 tempQualArray[EventType.BASE_SUBSTITUTION.index] = read.getBaseQualities()[offset];
                 tempFractionalErrorArray[EventType.BASE_SUBSTITUTION.index] = snpErrors[offset];
                 tempQualArray[EventType.BASE_INSERTION.index] = read.getBaseInsertionQualities()[offset];
@@ -124,30 +112,15 @@ public class AdvancedRecalibrationEngine extends StandardRecalibrationEngine imp
                     final byte qual = tempQualArray[eventIndex];
                     final double isError = tempFractionalErrorArray[eventIndex];
 
-                    final NestedIntegerArray<RecalDatum> rgRecalTable = recalibrationTables.getReadGroupTable();
-                    final RecalDatum rgPreviousDatum = rgRecalTable.get(keys[0], eventIndex);
-                    final RecalDatum rgThisDatum = createDatumObject(qual, isError);
-                    if (rgPreviousDatum == null) // key doesn't exist yet in the map so make a new bucket and add it
-                        rgRecalTable.put(rgThisDatum, keys[0], eventIndex);
-                    else
-                        rgPreviousDatum.combine(rgThisDatum);
+                    combineDatumOrPutIfNecessary(recalibrationTables.getReadGroupTable(), qual, isError, keys[0], eventIndex);
 
-                    final NestedIntegerArray<RecalDatum> qualRecalTable = recalibrationTables.getQualityScoreTable();
-                    final RecalDatum qualPreviousDatum = qualRecalTable.get(keys[0], keys[1], eventIndex);
-                    if (qualPreviousDatum == null)
-                        qualRecalTable.put(createDatumObject(qual, isError), keys[0], keys[1], eventIndex);
-                    else
-                        qualPreviousDatum.increment(1.0, isError);
+                    incrementDatumOrPutIfNecessary(recalibrationTables.getQualityScoreTable(), qual, isError, keys[0], keys[1], eventIndex);
 
                     for (int i = 2; i < covariates.length; i++) {
                         if (keys[i] < 0)
                             continue;
-                        final NestedIntegerArray<RecalDatum> covRecalTable = recalibrationTables.getTable(i);
-                        final RecalDatum covPreviousDatum = covRecalTable.get(keys[0], keys[1], keys[i], eventIndex);
-                        if (covPreviousDatum == null)
-                            covRecalTable.put(createDatumObject(qual, isError), keys[0], keys[1], keys[i], eventIndex);
-                        else
-                            covPreviousDatum.increment(1.0, isError);
+
+                        incrementDatumOrPutIfNecessary(recalibrationTables.getTable(i), qual, isError, keys[0], keys[1], keys[i], eventIndex);
                     }
                 }
             }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/StandardRecalibrationEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/StandardRecalibrationEngine.java
index 4fe9c5323..1f7fbbf42 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/StandardRecalibrationEngine.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/StandardRecalibrationEngine.java
@@ -55,7 +55,7 @@ public class StandardRecalibrationEngine implements RecalibrationEngine, PublicP
      * @param refBase       The reference base at this locus
      */
     @Override
-    public synchronized void updateDataForPileupElement(final PileupElement pileupElement, final byte refBase) {
+    public void updateDataForPileupElement(final PileupElement pileupElement, final byte refBase) {
         final int offset = pileupElement.getOffset();
         final ReadCovariates readCovariates = covariateKeySetFrom(pileupElement.getRead());
 
@@ -65,35 +65,21 @@ public class StandardRecalibrationEngine implements RecalibrationEngine, PublicP
         final int[] keys = readCovariates.getKeySet(offset, EventType.BASE_SUBSTITUTION);
         final int eventIndex = EventType.BASE_SUBSTITUTION.index;
 
-        final NestedIntegerArray<RecalDatum> rgRecalTable = recalibrationTables.getReadGroupTable();
-        final RecalDatum rgPreviousDatum = rgRecalTable.get(keys[0], eventIndex);
-        final RecalDatum rgThisDatum = createDatumObject(qual, isError);
-        if (rgPreviousDatum == null) // key doesn't exist yet in the map so make a new bucket and add it
-            rgRecalTable.put(rgThisDatum, keys[0], eventIndex);
-        else
-            rgPreviousDatum.combine(rgThisDatum);
+        // TODO: should this really be combine rather than increment?
+        combineDatumOrPutIfNecessary(recalibrationTables.getReadGroupTable(), qual, isError, keys[0], eventIndex);
 
-        final NestedIntegerArray<RecalDatum> qualRecalTable = recalibrationTables.getQualityScoreTable();
-        final RecalDatum qualPreviousDatum = qualRecalTable.get(keys[0], keys[1], eventIndex);
-        if (qualPreviousDatum == null)
-            qualRecalTable.put(createDatumObject(qual, isError), keys[0], keys[1], eventIndex);
-        else
-            qualPreviousDatum.increment(isError);
+        incrementDatumOrPutIfNecessary(recalibrationTables.getQualityScoreTable(), qual, isError, keys[0], keys[1], eventIndex);
 
         for (int i = 2; i < covariates.length; i++) {
             if (keys[i] < 0)
                 continue;
-            final NestedIntegerArray<RecalDatum> covRecalTable = recalibrationTables.getTable(i);
-            final RecalDatum covPreviousDatum = covRecalTable.get(keys[0], keys[1], keys[i], eventIndex);
-            if (covPreviousDatum == null)
-                covRecalTable.put(createDatumObject(qual, isError), keys[0], keys[1], keys[i], eventIndex);
-            else
-                covPreviousDatum.increment(isError);
+
+            incrementDatumOrPutIfNecessary(recalibrationTables.getTable(i), qual, isError, keys[0], keys[1], keys[i], eventIndex);
         }
     }
 
     @Override
-    public synchronized void updateDataForRead( final GATKSAMRecord read, final boolean[] skip, final double[] snpErrors, final double[] insertionErrors, final double[] deletionErrors ) {
+    public void updateDataForRead( final GATKSAMRecord read, final boolean[] skip, final double[] snpErrors, final double[] insertionErrors, final double[] deletionErrors ) {
         throw new UnsupportedOperationException("Delocalized BQSR is not available in the GATK-lite version");
     }
 
@@ -121,4 +107,133 @@ public class StandardRecalibrationEngine implements RecalibrationEngine, PublicP
     protected ReadCovariates covariateKeySetFrom(GATKSAMRecord read) {
         return (ReadCovariates) read.getTemporaryAttribute(BaseRecalibrator.COVARS_ATTRIBUTE);
     }
+
+    /**
+     * Increments the RecalDatum at the specified position in the specified table, or put a new item there
+     * if there isn't already one.
+     *
+     * Does this in a thread-safe way WITHOUT being synchronized: relies on the behavior of NestedIntegerArray.put()
+     * to return false if another thread inserts a new item at our position in the middle of our put operation.
+     *
+     * @param table the table that holds/will hold our item
+     * @param qual qual for this event
+     * @param isError error status for this event
+     * @param keys location in table of our item
+     */
+    protected void incrementDatumOrPutIfNecessary( final NestedIntegerArray<RecalDatum> table, final byte qual, final boolean isError, final int... keys ) {
+        final RecalDatum existingDatum = table.get(keys);
+
+        // There are three cases here:
+        //
+        // 1. The original get succeeded (existingDatum != null) because there already was an item in this position.
+        //    In this case we can just increment the existing item.
+        //
+        // 2. The original get failed (existingDatum == null), and we successfully put a new item in this position
+        //    and are done.
+        //
+        // 3. The original get failed (existingDatum == null), AND the put fails because another thread comes along
+        //    in the interim and puts an item in this position. In this case we need to do another get after the
+        //    failed put to get the item the other thread put here and increment it.
+        if ( existingDatum == null ) {
+            // No existing item, try to put a new one
+            if ( ! table.put(createDatumObject(qual, isError), keys) ) {
+                // Failed to put a new item because another thread came along and put an item here first.
+                // Get the newly-put item and increment it (item is guaranteed to exist at this point)
+                table.get(keys).increment(isError);
+            }
+        }
+        else {
+            // Easy case: already an item here, so increment it
+            existingDatum.increment(isError);
+        }
+    }
+
+    /**
+     * Increments the RecalDatum at the specified position in the specified table, or put a new item there
+     * if there isn't already one.
+     *
+     * Does this in a thread-safe way WITHOUT being synchronized: relies on the behavior of NestedIntegerArray.put()
+     * to return false if another thread inserts a new item at our position in the middle of our put operation.
+     *
+     * @param table the table that holds/will hold our item
+     * @param qual qual for this event
+     * @param isError error value for this event
+     * @param keys location in table of our item
+     */
+    protected void incrementDatumOrPutIfNecessary( final NestedIntegerArray<RecalDatum> table, final byte qual, final double isError, final int... keys ) {
+        final RecalDatum existingDatum = table.get(keys);
+
+        if ( existingDatum == null ) {
+            // No existing item, try to put a new one
+            if ( ! table.put(createDatumObject(qual, isError), keys) ) {
+                // Failed to put a new item because another thread came along and put an item here first.
+                // Get the newly-put item and increment it (item is guaranteed to exist at this point)
+                table.get(keys).increment(1.0, isError);
+            }
+        }
+        else {
+            // Easy case: already an item here, so increment it
+            existingDatum.increment(1.0, isError);
+        }
+    }
+
+    /**
+     * Combines the RecalDatum at the specified position in the specified table with a new RecalDatum, or put a
+     * new item there if there isn't already one.
+     *
+     * Does this in a thread-safe way WITHOUT being synchronized: relies on the behavior of NestedIntegerArray.put()
+     * to return false if another thread inserts a new item at our position in the middle of our put operation.
+     *
+     * @param table the table that holds/will hold our item
+     * @param qual qual for this event
+     * @param isError error status for this event
+     * @param keys location in table of our item
+     */
+    protected void combineDatumOrPutIfNecessary( final NestedIntegerArray<RecalDatum> table, final byte qual, final boolean isError, final int... keys ) {
+        final RecalDatum existingDatum = table.get(keys);
+        final RecalDatum newDatum = createDatumObject(qual, isError);
+
+        if ( existingDatum == null ) {
+            // No existing item, try to put a new one
+            if ( ! table.put(newDatum, keys) ) {
+                // Failed to put a new item because another thread came along and put an item here first.
+                // Get the newly-put item and combine it with our item (item is guaranteed to exist at this point)
+                table.get(keys).combine(newDatum);
+            }
+        }
+        else {
+            // Easy case: already an item here, so combine it with our item
+            existingDatum.combine(newDatum);
+        }
+    }
+
+    /**
+     * Combines the RecalDatum at the specified position in the specified table with a new RecalDatum, or put a
+     * new item there if there isn't already one.
+     *
+     * Does this in a thread-safe way WITHOUT being synchronized: relies on the behavior of NestedIntegerArray.put()
+     * to return false if another thread inserts a new item at our position in the middle of our put operation.
+     *
+     * @param table the table that holds/will hold our item
+     * @param qual qual for this event
+     * @param isError error value for this event
+     * @param keys location in table of our item
+     */
+    protected void combineDatumOrPutIfNecessary( final NestedIntegerArray<RecalDatum> table, final byte qual, final double isError, final int... keys ) {
+        final RecalDatum existingDatum = table.get(keys);
+        final RecalDatum newDatum = createDatumObject(qual, isError);
+
+        if ( existingDatum == null ) {
+            // No existing item, try to put a new one
+            if ( ! table.put(newDatum, keys) ) {
+                // Failed to put a new item because another thread came along and put an item here first.
+                // Get the newly-put item and combine it with our item (item is guaranteed to exist at this point)
+                table.get(keys).combine(newDatum);
+            }
+        }
+        else {
+            // Easy case: already an item here, so combine it with our item
+            existingDatum.combine(newDatum);
+        }
+    }
 }
diff --git a/public/java/src/org/broadinstitute/sting/utils/collections/LoggingNestedIntegerArray.java b/public/java/src/org/broadinstitute/sting/utils/collections/LoggingNestedIntegerArray.java
index 6fda0245b..63927ac84 100644
--- a/public/java/src/org/broadinstitute/sting/utils/collections/LoggingNestedIntegerArray.java
+++ b/public/java/src/org/broadinstitute/sting/utils/collections/LoggingNestedIntegerArray.java
@@ -99,9 +99,7 @@ public class LoggingNestedIntegerArray<T> extends NestedIntegerArray<T> {
     }
 
     @Override
-    public void put( final T value, final int... keys ) {
-        super.put(value, keys);
-
+    public boolean put( final T value, final int... keys ) {
         StringBuilder logEntry = new StringBuilder();
 
         logEntry.append(logEntryLabel);
@@ -116,5 +114,7 @@ public class LoggingNestedIntegerArray<T> extends NestedIntegerArray<T> {
 
         // PrintStream methods all use synchronized blocks internally, so our logging is thread-safe
         log.println(logEntry.toString());
+
+        return super.put(value, keys);
     }
 }
diff --git a/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java b/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java
index 31d316555..9f1eb546d 100755
--- a/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java
+++ b/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java
@@ -50,6 +50,28 @@ public class NestedIntegerArray<T> {
         this.dimensions = dimensions.clone();
 
         data = new Object[dimensions[0]];
+        prepopulateArray(data, 0);
+    }
+
+    /**
+     * Recursively allocate the entire tree of arrays in all its dimensions.
+     *
+     * Doing this upfront uses more memory initially, but saves time over the course of the run
+     * and (crucially) avoids having to make threads wait while traversing the tree to check
+     * whether branches exist or not.
+     *
+     * @param subarray current node in the tree
+     * @param dimension current level in the tree
+     */
+    private void prepopulateArray( Object[] subarray, int dimension ) {
+        if ( dimension >= numDimensions - 1 ) {
+            return;
+        }
+
+        for ( int i = 0; i < subarray.length; i++ ) {
+            subarray[i] = new Object[dimensions[dimension + 1]];
+            prepopulateArray((Object[])subarray[i], dimension + 1);
+        }
     }
 
     public T get(final int... keys) {
@@ -59,14 +81,15 @@ public class NestedIntegerArray<T> {
         for( int i = 0; i < numNestedDimensions; i++ ) {
             if ( keys[i] >= dimensions[i] )
                 return null;
-            myData = (Object[])myData[keys[i]];
-            if ( myData == null )
-                return null;
+
+            myData = (Object[])myData[keys[i]];  // interior nodes in the tree will never be null, so we can safely traverse
+                                                 // down to the leaves
         }
+
         return (T)myData[keys[numNestedDimensions]];
     }
 
-    public synchronized void put(final T value, final int... keys) { // WARNING! value comes before the keys!
+    public boolean put(final T value, final int... keys) { // WARNING! value comes before the keys!
         if ( keys.length != numDimensions )
             throw new ReviewedStingException("Exactly " + numDimensions + " keys should be passed to this NestedIntegerArray but " + keys.length + " were provided");
 
@@ -75,15 +98,26 @@ public class NestedIntegerArray<T> {
         for ( int i = 0; i < numNestedDimensions; i++ ) {
             if ( keys[i] >= dimensions[i] )
                 throw new ReviewedStingException("Key " + keys[i] + " is too large for dimension " + i + " (max is " + (dimensions[i]-1) + ")");
-            Object[] temp = (Object[])myData[keys[i]];
-            if ( temp == null ) {
-                temp = new Object[dimensions[i+1]];
-                myData[keys[i]] = temp;
-            }
-            myData = temp;
+
+            myData = (Object[])myData[keys[i]];  // interior nodes in the tree will never be null, so we can safely traverse
+                                                 // down to the leaves
         }
 
-        myData[keys[numNestedDimensions]] = value;
+        synchronized ( myData ) {   // lock the bottom row while we examine and (potentially) update it
+
+            // Insert the new value only if there still isn't any existing value in this position
+            if ( myData[keys[numNestedDimensions]] == null ) {
+                myData[keys[numNestedDimensions]] = value;
+            }
+            else {
+                // Already have a value for this leaf (perhaps another thread came along and inserted one
+                // while we traversed the tree), so return false to notify the caller that we didn't put
+                // the item
+                return false;
+            }
+        }
+
+        return true;
     }
 
     public List<T> getAllValues() {
diff --git a/public/java/src/org/broadinstitute/sting/utils/threading/ThreadLocalArray.java b/public/java/src/org/broadinstitute/sting/utils/threading/ThreadLocalArray.java
new file mode 100644
index 000000000..cc50152ac
--- /dev/null
+++ b/public/java/src/org/broadinstitute/sting/utils/threading/ThreadLocalArray.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2012, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.threading;
+
+import java.lang.reflect.Array;
+
+/**
+ * ThreadLocal implementation for arrays
+ *
+ * Example usage:
+ *
+ * private ThreadLocal<byte[]> threadLocalByteArray = new ThreadLocalArray<byte[]>(length, byte.class);
+ * ....
+ * byte[] byteArray = threadLocalByteArray.get();
+ *
+ * @param <T> the type of the array itself (eg., int[], double[], etc.)
+ *
+ * @author David Roazen
+ */
+public class ThreadLocalArray<T> extends ThreadLocal<T> {
+    private int arraySize;
+    private Class arrayElementType;
+
+    /**
+     * Create a new ThreadLocalArray
+     *
+     * @param arraySize desired length of the array
+     * @param arrayElementType type of the elements within the array (eg., Byte.class, Integer.class, etc.)
+     */
+    public ThreadLocalArray( int arraySize, Class arrayElementType ) {
+        super();
+
+        this.arraySize = arraySize;
+        this.arrayElementType = arrayElementType;
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    protected T initialValue() {
+        return (T)Array.newInstance(arrayElementType, arraySize);
+    }
+}

From 32a6d7000a4dab5f208c9d5e78f35b314edbaec3 Mon Sep 17 00:00:00 2001
From: David Roazen <droazen@broadinstitute.org>
Date: Wed, 24 Oct 2012 13:06:34 -0400
Subject: [PATCH 14/40] Thread-safe ReadGroupCovariate

The ReadGroupCovariate class was not thread-safe. This led to horrible race conditions
in multithreaded runs of the BQSR where (for example) the same read group could get
inserted into the reverse lookup table twice with different IDs.

Should fix the intermittent crash reported in GSA-492.
---
 .../covariates/ReadGroupCovariate.java        | 29 +++++++++++++++----
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/utils/recalibration/covariates/ReadGroupCovariate.java b/public/java/src/org/broadinstitute/sting/utils/recalibration/covariates/ReadGroupCovariate.java
index 85568dac9..29c15adf7 100755
--- a/public/java/src/org/broadinstitute/sting/utils/recalibration/covariates/ReadGroupCovariate.java
+++ b/public/java/src/org/broadinstitute/sting/utils/recalibration/covariates/ReadGroupCovariate.java
@@ -66,7 +66,9 @@ public class ReadGroupCovariate implements RequiredCovariate {
     }
 
     @Override
-    public String formatKey(final int key) {
+    public synchronized String formatKey(final int key) {
+        // This method is synchronized so that we don't attempt to do a get()
+        // from the reverse lookup table while that table is being updated
         return readGroupReverseLookupTable.get(key);
     }
 
@@ -76,17 +78,32 @@ public class ReadGroupCovariate implements RequiredCovariate {
     }
 
     private int keyForReadGroup(final String readGroupId) {
-        if (!readGroupLookupTable.containsKey(readGroupId)) {
-            readGroupLookupTable.put(readGroupId, nextId);
-            readGroupReverseLookupTable.put(nextId, readGroupId);
-            nextId++;
+        // Rather than synchronize this entire method (which would be VERY expensive for walkers like the BQSR),
+        // synchronize only the table updates.
+
+        // Before entering the synchronized block, check to see if this read group is not in our tables.
+        // If it's not, either we will have to insert it, OR another thread will insert it first.
+        // This preliminary check avoids doing any synchronization most of the time.
+        if ( ! readGroupLookupTable.containsKey(readGroupId) ) {
+
+            synchronized ( this ) {
+
+                // Now we need to make sure the key is STILL not there, since another thread may have come along
+                // and inserted it while we were waiting to enter this synchronized block!
+                if ( ! readGroupLookupTable.containsKey(readGroupId) ) {
+                    readGroupLookupTable.put(readGroupId, nextId);
+                    readGroupReverseLookupTable.put(nextId, readGroupId);
+                    nextId++;
+                }
+            }
         }
 
         return readGroupLookupTable.get(readGroupId);
     }
 
     @Override
-    public int maximumKeyValue() {
+    public synchronized int maximumKeyValue() {
+        // Synchronized so that we don't query table size while the tables are being updated
         return readGroupLookupTable.size() - 1;
     }
 

From 02018ca76495b1405a3b1ec4086afdb470ef0b95 Mon Sep 17 00:00:00 2001
From: David Roazen <droazen@broadinstitute.org>
Date: Wed, 24 Oct 2012 15:06:13 -0400
Subject: [PATCH 15/40] Legacy BaseRecalibrator walker is neither TreeReducible
 nor NanoSchedulable

The old BaseRecalibrator walker is and never will be thread-safe, since it's a
LocusWalker that uses read attributes to track state.

ONLY the newer DelocalizedBaseRecalibrator is believed likely to be thread-safe
at this point. It is safe to run the DelocalizedBaseRecalibrator with -nct > 1
for testing purposes, but wait for further testing to be done before using it
for production purposes in multithreaded mode.
---
 .../sting/gatk/walkers/bqsr/BQSRIntegrationTest.java       | 5 -----
 .../sting/gatk/walkers/bqsr/BaseRecalibrator.java          | 7 +------
 2 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
index 26ee78484..90d67d393 100644
--- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
+++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
@@ -73,11 +73,6 @@ public class BQSRIntegrationTest extends WalkerTest {
                 params.getCommandLine(),
                 Arrays.asList(params.md5));
         executeTest("testBQSR-"+params.args, spec).getFirst();
-
-        WalkerTestSpec specNT2 = new WalkerTestSpec(
-                params.getCommandLine() + " -nt 2",
-                Arrays.asList(params.md5));
-        executeTest("testBQSR-nt2-"+params.args, specNT2).getFirst();
     }
 
     @Test
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/BaseRecalibrator.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/BaseRecalibrator.java
index 2c7a12a36..245c8e3e8 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/BaseRecalibrator.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/BaseRecalibrator.java
@@ -109,7 +109,7 @@ import java.util.ArrayList;
 @ReadFilters({MappingQualityZeroFilter.class, MappingQualityUnavailableFilter.class}) // only look at covered loci, not every loci of the reference file
 @Requires({DataSource.READS, DataSource.REFERENCE}) // filter out all reads with zero or unavailable mapping quality
 @PartitionBy(PartitionType.LOCUS) // this walker requires both -I input.bam and -R reference.fasta
-public class BaseRecalibrator extends LocusWalker<Long, Long> implements TreeReducible<Long>, NanoSchedulable {
+public class BaseRecalibrator extends LocusWalker<Long, Long> {
 
     @ArgumentCollection
     private final RecalibrationArgumentCollection RAC = new RecalibrationArgumentCollection(); // all the command line arguments for BQSR and it's covariates
@@ -277,11 +277,6 @@ public class BaseRecalibrator extends LocusWalker<Long, Long> implements TreeRed
         return sum;
     }
 
-    public Long treeReduce(Long sum1, Long sum2) {
-        sum1 += sum2;
-        return sum1;
-    }
-
     @Override
     public void onTraversalDone(Long result) {
         logger.info("Calculating quantized quality scores...");

From d9aa9855f874f80ddc629c3c7179c00108991c9e Mon Sep 17 00:00:00 2001
From: David Roazen <droazen@broadinstitute.org>
Date: Wed, 24 Oct 2012 15:29:13 -0400
Subject: [PATCH 16/40] Better comments in NestedIntegerArray

---
 .../utils/collections/NestedIntegerArray.java      | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java b/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java
index 9f1eb546d..c83de8ec7 100755
--- a/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java
+++ b/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java
@@ -89,6 +89,20 @@ public class NestedIntegerArray<T> {
         return (T)myData[keys[numNestedDimensions]];
     }
 
+    /**
+     * Insert a value at the position specified by the given keys.
+     *
+     * This method is THREAD-SAFE despite not being synchronized, however the caller MUST
+     * check the return value to see if the put succeeded. This method RETURNS FALSE if
+     * the value could not be inserted because there already was a value present
+     * at the specified location. In this case the caller should do a get() to get
+     * the already-existing value and (potentially) update it.
+     *
+     * @param value value to insert
+     * @param keys keys specifying the location of the value in the tree
+     * @return true if the value was inserted, false if it could not be inserted because there was already
+     *         a value at the specified position
+     */
     public boolean put(final T value, final int... keys) { // WARNING! value comes before the keys!
         if ( keys.length != numDimensions )
             throw new ReviewedStingException("Exactly " + numDimensions + " keys should be passed to this NestedIntegerArray but " + keys.length + " were provided");

From 9da7bbf689dd63ed6d3aed4a10b7eea1f48ace63 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Wed, 24 Oct 2012 15:49:07 -0400
Subject: [PATCH 17/40] Refactoring the PerReadAlleleLikelihoodMap in
 preparation for adding contntamination downsampling into protected only.

---
 .../gatk/walkers/genotyper/ErrorModel.java    |  3 +-
 ...dyGenotypeLikelihoodsCalculationModel.java |  6 +--
 ...GeneralPloidyIndelGenotypeLikelihoods.java |  4 +-
 ...elGenotypeLikelihoodsCalculationModel.java |  2 +-
 ...NPGenotypeLikelihoodsCalculationModel.java |  2 +-
 .../haplotypecaller/HaplotypeCaller.java      |  2 +-
 .../LikelihoodCalculationEngine.java          |  4 +-
 .../AdvancedPerReadAlleleLikelihoodMap.java   | 40 +++++++++++++++++
 .../gatk/walkers/annotator/AlleleBalance.java |  2 +-
 .../annotator/AlleleBalanceBySample.java      |  2 +-
 .../gatk/walkers/annotator/BaseCounts.java    |  2 +-
 .../annotator/BaseQualityRankSumTest.java     |  5 +--
 .../walkers/annotator/ChromosomeCounts.java   |  4 +-
 .../annotator/ClippingRankSumTest.java        |  2 +-
 .../walkers/annotator/DepthOfCoverage.java    |  2 +-
 .../annotator/DepthPerAlleleBySample.java     |  2 +-
 .../gatk/walkers/annotator/FisherStrand.java  |  2 +-
 .../gatk/walkers/annotator/GCContent.java     |  2 +-
 .../walkers/annotator/HaplotypeScore.java     |  3 +-
 .../gatk/walkers/annotator/HardyWeinberg.java |  2 +-
 .../walkers/annotator/HomopolymerRun.java     |  2 +-
 .../walkers/annotator/InbreedingCoeff.java    |  4 +-
 .../gatk/walkers/annotator/IndelType.java     |  2 +-
 .../sting/gatk/walkers/annotator/LowMQ.java   |  2 +-
 .../walkers/annotator/MVLikelihoodRatio.java  |  2 +-
 .../annotator/MappingQualityRankSumTest.java  |  5 +--
 .../walkers/annotator/MappingQualityZero.java |  3 +-
 .../annotator/MappingQualityZeroBySample.java |  2 +-
 .../annotator/MappingQualityZeroFraction.java |  2 +-
 .../gatk/walkers/annotator/NBaseCount.java    |  2 +-
 .../gatk/walkers/annotator/QualByDepth.java   |  4 +-
 .../walkers/annotator/RMSMappingQuality.java  |  4 +-
 .../gatk/walkers/annotator/RankSumTest.java   |  4 +-
 .../walkers/annotator/ReadPosRankSumTest.java |  2 +-
 .../gatk/walkers/annotator/SampleList.java    |  2 +-
 .../sting/gatk/walkers/annotator/SnpEff.java  |  2 +-
 .../walkers/annotator/SpanningDeletions.java  |  2 +-
 .../annotator/TandemRepeatAnnotator.java      |  2 +-
 .../annotator/TechnologyComposition.java      |  2 +-
 .../TransmissionDisequilibriumTest.java       |  2 +-
 .../annotator/VariantAnnotatorEngine.java     |  2 +-
 .../ActiveRegionBasedAnnotation.java          |  4 +-
 .../interfaces/GenotypeAnnotation.java        |  2 +-
 .../interfaces/InfoFieldAnnotation.java       |  2 +-
 .../GenotypeLikelihoodsCalculationModel.java  |  2 +-
 ...elGenotypeLikelihoodsCalculationModel.java |  4 +-
 ...NPGenotypeLikelihoodsCalculationModel.java |  2 +-
 .../genotyper/UnifiedGenotyperEngine.java     | 12 ++---
 .../indels/PairHMMIndelErrorModel.java        |  3 +-
 .../genotyper/PerReadAlleleLikelihoodMap.java | 44 ++++++++++---------
 .../StandardPerReadAlleleLikelihoodMap.java   | 44 +++++++++++++++++++
 51 files changed, 165 insertions(+), 101 deletions(-)
 create mode 100644 protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
 rename public/java/src/org/broadinstitute/sting/{gatk/walkers => utils}/genotyper/PerReadAlleleLikelihoodMap.java (76%)
 create mode 100644 public/java/src/org/broadinstitute/sting/utils/genotyper/StandardPerReadAlleleLikelihoodMap.java

diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ErrorModel.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ErrorModel.java
index 8042c15d8..a01973ad5 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ErrorModel.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ErrorModel.java
@@ -1,7 +1,6 @@
 package org.broadinstitute.sting.gatk.walkers.genotyper;
 
 import com.google.java.contract.Requires;
-import org.apache.commons.lang.ArrayUtils;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.walkers.indels.PairHMMIndelErrorModel;
 import org.broadinstitute.sting.utils.Haplotype;
@@ -60,7 +59,7 @@ public class ErrorModel  {
 
         boolean hasCalledAlleles = false;
 
-        final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap = new PerReadAlleleLikelihoodMap();
+        final org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap = org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap.getBestAvailablePerReadAlleleLikelihoodMap();
         if (refSampleVC != null) {
 
             for (Allele allele : refSampleVC.getAlleles()) {
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyGenotypeLikelihoodsCalculationModel.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyGenotypeLikelihoodsCalculationModel.java
index 2522fc16e..f6ad445c7 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyGenotypeLikelihoodsCalculationModel.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyGenotypeLikelihoodsCalculationModel.java
@@ -195,7 +195,7 @@ public abstract class GeneralPloidyGenotypeLikelihoodsCalculationModel extends G
                                          final List<Allele> allAllelesToUse,
                                          final boolean useBAQedPileup,
                                          final GenomeLocParser locParser,
-                                         final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+                                         final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
 
         HashMap<String, ErrorModel> perLaneErrorModels = getPerLaneErrorModels(tracker, ref, contexts);
         if (perLaneErrorModels == null && UAC.referenceSampleName != null)
@@ -231,7 +231,7 @@ public abstract class GeneralPloidyGenotypeLikelihoodsCalculationModel extends G
             ReadBackedPileup pileup = AlignmentContextUtils.stratify(sample.getValue(), contextType).getBasePileup();
             if (!perReadAlleleLikelihoodMap.containsKey(sample.getKey())){
                 // no likelihoods have been computed for this sample at this site
-                perReadAlleleLikelihoodMap.put(sample.getKey(), new PerReadAlleleLikelihoodMap());
+                perReadAlleleLikelihoodMap.put(sample.getKey(), org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap.getBestAvailablePerReadAlleleLikelihoodMap());
             }
 
             // create the GenotypeLikelihoods object
@@ -333,7 +333,7 @@ public abstract class GeneralPloidyGenotypeLikelihoodsCalculationModel extends G
                                                                                final boolean useBQAedPileup,
                                                                                final ReferenceContext ref,
                                                                                final boolean ignoreLaneInformation,
-                                                                               final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap);
+                                                                               final org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap);
 
     protected abstract List<Allele> getInitialAllelesToUse(final RefMetaDataTracker tracker,
                                                            final ReferenceContext ref,
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyIndelGenotypeLikelihoods.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyIndelGenotypeLikelihoods.java
index afbd49a08..4bcaa5ff9 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyIndelGenotypeLikelihoods.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyIndelGenotypeLikelihoods.java
@@ -27,7 +27,7 @@ public class GeneralPloidyIndelGenotypeLikelihoods extends GeneralPloidyGenotype
     double[][] readHaplotypeLikelihoods;
 
     final byte refBase;
-    final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap;
+    final org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap;
 
     public GeneralPloidyIndelGenotypeLikelihoods(final List<Allele> alleles,
                                                  final double[] logLikelihoods,
@@ -37,7 +37,7 @@ public class GeneralPloidyIndelGenotypeLikelihoods extends GeneralPloidyGenotype
                                                  final PairHMMIndelErrorModel pairModel,
                                                  final LinkedHashMap<Allele, Haplotype> haplotypeMap,
                                                  final ReferenceContext referenceContext,
-                                                 final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap) {
+                                                 final org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap) {
         super(alleles, logLikelihoods, ploidy, perLaneErrorModels, ignoreLaneInformation);
         this.pairModel = pairModel;
         this.haplotypeMap = haplotypeMap;
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyIndelGenotypeLikelihoodsCalculationModel.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyIndelGenotypeLikelihoodsCalculationModel.java
index f09a1ea3e..eb4cf1839 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyIndelGenotypeLikelihoodsCalculationModel.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyIndelGenotypeLikelihoodsCalculationModel.java
@@ -74,7 +74,7 @@ public class GeneralPloidyIndelGenotypeLikelihoodsCalculationModel extends Gener
                                                                                final boolean useBQAedPileup,
                                                                                final ReferenceContext ref,
                                                                                final boolean ignoreLaneInformation,
-                                                                               final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap){
+                                                                               final org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap){
         return new GeneralPloidyIndelGenotypeLikelihoods(alleles, logLikelihoods, ploidy,perLaneErrorModels,ignoreLaneInformation, pairModel, haplotypeMap, ref, perReadAlleleLikelihoodMap);
     }
 
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidySNPGenotypeLikelihoodsCalculationModel.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidySNPGenotypeLikelihoodsCalculationModel.java
index 4376ec601..9f2fdc096 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidySNPGenotypeLikelihoodsCalculationModel.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidySNPGenotypeLikelihoodsCalculationModel.java
@@ -50,7 +50,7 @@ public class GeneralPloidySNPGenotypeLikelihoodsCalculationModel extends General
                                                                                final boolean useBQAedPileup,
                                                                                final ReferenceContext ref,
                                                                                final boolean ignoreLaneInformation,
-                                                                               final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap){
+                                                                               final org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap){
         return new GeneralPloidySNPGenotypeLikelihoods(alleles, null, UAC.samplePloidy, perLaneErrorModels, useBQAedPileup, UAC.IGNORE_LANE_INFO);
     }
 
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java
index a08614deb..3293ac817 100755
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java
@@ -424,7 +424,7 @@ public class HaplotypeCaller extends ActiveRegionWalker<Integer, Integer> implem
                   : genotypingEngine.assignGenotypeLikelihoodsAndCallIndependentEvents( UG_engine, bestHaplotypes, fullReferenceWithPadding, getPaddedLoc(activeRegion), activeRegion.getLocation(), getToolkit().getGenomeLocParser(), activeAllelesToGenotype ) ) ) {
             if( DEBUG ) { System.out.println(callResult.getFirst().toStringWithoutGenotypes()); }
 
-            final Map<String, PerReadAlleleLikelihoodMap> stratifiedReadMap = LikelihoodCalculationEngine.partitionReadsBasedOnLikelihoods( getToolkit().getGenomeLocParser(), perSampleReadList, perSampleFilteredReadList, callResult );
+            final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> stratifiedReadMap = LikelihoodCalculationEngine.partitionReadsBasedOnLikelihoods( getToolkit().getGenomeLocParser(), perSampleReadList, perSampleFilteredReadList, callResult );
             final VariantContext annotatedCall = annotationEngine.annotateContext(stratifiedReadMap, callResult.getFirst());
             final Map<String, Object> myAttributes = new LinkedHashMap<String, Object>(annotatedCall.getAttributes());
 
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java
index 62554c4ab..a05e20bd3 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java
@@ -27,7 +27,7 @@ package org.broadinstitute.sting.gatk.walkers.haplotypecaller;
 
 import com.google.java.contract.Ensures;
 import com.google.java.contract.Requires;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.*;
 import org.broadinstitute.sting.utils.collections.Pair;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
@@ -350,7 +350,7 @@ public class LikelihoodCalculationEngine {
         final Map<String, PerReadAlleleLikelihoodMap> returnMap = new HashMap<String, PerReadAlleleLikelihoodMap>();
         final GenomeLoc callLoc = parser.createGenomeLoc(call.getFirst());
         for( final Map.Entry<String, ArrayList<GATKSAMRecord>> sample : perSampleReadList.entrySet() ) {
-            final PerReadAlleleLikelihoodMap likelihoodMap = new PerReadAlleleLikelihoodMap();
+            final PerReadAlleleLikelihoodMap likelihoodMap = PerReadAlleleLikelihoodMap.getBestAvailablePerReadAlleleLikelihoodMap();
 
             final ArrayList<GATKSAMRecord> readsForThisSample = sample.getValue();
             for( int iii = 0; iii < readsForThisSample.size(); iii++ ) {
diff --git a/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java b/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
new file mode 100644
index 000000000..db1bf8762
--- /dev/null
+++ b/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2011 The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+ * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package org.broadinstitute.sting.utils.genotyper;
+
+
+import org.broadinstitute.sting.utils.classloader.ProtectedPackageSource;
+import org.broadinstitute.sting.utils.pileup.PileupElement;
+import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
+import org.broadinstitute.sting.utils.variantcontext.Allele;
+
+import java.util.*;
+
+public class AdvancedPerReadAlleleLikelihoodMap extends StandardPerReadAlleleLikelihoodMap implements ProtectedPackageSource {
+
+    public PerReadAlleleLikelihoodMap createPerAlleleDownsampledMap(final double downsamplingFraction) {
+        return this;
+    }
+ }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalance.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalance.java
index a68f0df21..18bdb02ed 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalance.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalance.java
@@ -30,7 +30,7 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalanceBySample.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalanceBySample.java
index 1e1f65333..4d79c4112 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalanceBySample.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalanceBySample.java
@@ -6,7 +6,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ExperimentalAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.GenotypeAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFFormatHeaderLine;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseCounts.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseCounts.java
index 3cbca4f52..aef3e49cf 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseCounts.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseCounts.java
@@ -36,7 +36,7 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.BaseUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java
index 577b1cfdc..e59fc827d 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java
@@ -1,14 +1,11 @@
 package org.broadinstitute.sting.gatk.walkers.annotator;
 
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.IndelGenotypeLikelihoodsCalculationModel;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
-import org.broadinstitute.sting.utils.BaseUtils;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
-import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
 import java.util.*;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ChromosomeCounts.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ChromosomeCounts.java
index 4ae1a0bba..0c78c0204 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ChromosomeCounts.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ChromosomeCounts.java
@@ -34,13 +34,11 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ActiveRegionBa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLine;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFStandardHeaderLines;
-import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
-import org.broadinstitute.sting.utils.variantcontext.Allele;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 import org.broadinstitute.sting.utils.variantcontext.VariantContextUtils;
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ClippingRankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ClippingRankSumTest.java
index c74f98ca3..1dff4d1a3 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ClippingRankSumTest.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ClippingRankSumTest.java
@@ -1,6 +1,6 @@
 package org.broadinstitute.sting.gatk.walkers.annotator;
 
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthOfCoverage.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthOfCoverage.java
index 1cc88fc24..c9481f244 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthOfCoverage.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthOfCoverage.java
@@ -7,7 +7,7 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ActiveRegionBa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFStandardHeaderLines;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthPerAlleleBySample.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthPerAlleleBySample.java
index d7790969e..89a239e54 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthPerAlleleBySample.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthPerAlleleBySample.java
@@ -6,7 +6,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.GenotypeAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFFormatHeaderLine;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFStandardHeaderLines;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java
index ec0393cdc..bdf7baec9 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java
@@ -32,7 +32,7 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ActiveRegionBa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.QualityUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/GCContent.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/GCContent.java
index 3fe5c5837..07391c78c 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/GCContent.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/GCContent.java
@@ -7,7 +7,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ExperimentalAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.BaseUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java
index 6487eac50..ca7180510 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java
@@ -32,8 +32,7 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ActiveRegionBa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.IndelGenotypeLikelihoodsCalculationModel;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.BaseUtils;
 import org.broadinstitute.sting.utils.Haplotype;
 import org.broadinstitute.sting.utils.MathUtils;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HardyWeinberg.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HardyWeinberg.java
index 06fa04526..0340f457c 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HardyWeinberg.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HardyWeinberg.java
@@ -7,7 +7,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.WorkInProgressAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.QualityUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HomopolymerRun.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HomopolymerRun.java
index 5891cbc69..037b357ae 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HomopolymerRun.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HomopolymerRun.java
@@ -5,7 +5,7 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.GenomeLoc;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/InbreedingCoeff.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/InbreedingCoeff.java
index 9a4de3c36..dd058b469 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/InbreedingCoeff.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/InbreedingCoeff.java
@@ -8,12 +8,10 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ActiveRegionBa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
-import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
-import org.broadinstitute.sting.utils.variantcontext.Allele;
 import org.broadinstitute.sting.utils.variantcontext.Genotype;
 import org.broadinstitute.sting.utils.variantcontext.GenotypesContext;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/IndelType.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/IndelType.java
index 5f405cb46..c67d829c2 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/IndelType.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/IndelType.java
@@ -6,7 +6,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ExperimentalAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.IndelUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/LowMQ.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/LowMQ.java
index 4be601bc8..c9a4d0ee6 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/LowMQ.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/LowMQ.java
@@ -5,7 +5,7 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MVLikelihoodRatio.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MVLikelihoodRatio.java
index 85f61c91c..c9d5ca261 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MVLikelihoodRatio.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MVLikelihoodRatio.java
@@ -9,7 +9,7 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ExperimentalAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.RodRequiringAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.MendelianViolation;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityRankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityRankSumTest.java
index 787c9b29b..82596a501 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityRankSumTest.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityRankSumTest.java
@@ -1,14 +1,11 @@
 package org.broadinstitute.sting.gatk.walkers.annotator;
 
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.IndelGenotypeLikelihoodsCalculationModel;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
-import org.broadinstitute.sting.utils.QualityUtils;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
-import org.broadinstitute.sting.utils.sam.AlignmentUtils;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZero.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZero.java
index 74f9c0d0c..364bbdbb9 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZero.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZero.java
@@ -7,14 +7,13 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ActiveRegionBa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFStandardHeaderLines;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
-import org.broadinstitute.sting.utils.variantcontext.Allele;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
 import java.util.Arrays;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroBySample.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroBySample.java
index 44657a7e7..afb4ceb60 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroBySample.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroBySample.java
@@ -30,7 +30,7 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.GenotypeAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFFormatHeaderLine;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroFraction.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroFraction.java
index 21ee66ea2..5f9f3416d 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroFraction.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroFraction.java
@@ -6,7 +6,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ExperimentalAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/NBaseCount.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/NBaseCount.java
index 8e4edaf0e..3e6aa62a2 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/NBaseCount.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/NBaseCount.java
@@ -5,7 +5,7 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.BaseUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/QualByDepth.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/QualByDepth.java
index a48d4a678..d75947879 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/QualByDepth.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/QualByDepth.java
@@ -7,11 +7,9 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ActiveRegionBa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
-import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
-import org.broadinstitute.sting.utils.variantcontext.Allele;
 import org.broadinstitute.sting.utils.variantcontext.Genotype;
 import org.broadinstitute.sting.utils.variantcontext.GenotypesContext;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RMSMappingQuality.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RMSMappingQuality.java
index 680478da0..474b6b150 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RMSMappingQuality.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RMSMappingQuality.java
@@ -7,16 +7,14 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ActiveRegionBa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.QualityUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFStandardHeaderLines;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
-import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
-import org.broadinstitute.sting.utils.variantcontext.Allele;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
 import java.util.*;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java
index 7c7391812..0df7aff71 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java
@@ -7,15 +7,13 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ActiveRegionBasedAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.IndelGenotypeLikelihoodsCalculationModel;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.MannWhitneyU;
 import org.broadinstitute.sting.utils.QualityUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLine;
 import org.broadinstitute.sting.utils.collections.Pair;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
-import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 import org.broadinstitute.sting.utils.variantcontext.Genotype;
 import org.broadinstitute.sting.utils.variantcontext.GenotypesContext;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java
index de0ce2ce2..d01233bb2 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java
@@ -5,7 +5,7 @@ import net.sf.samtools.CigarElement;
 import net.sf.samtools.CigarOperator;
 import net.sf.samtools.SAMRecord;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.gatk.walkers.indels.PairHMMIndelErrorModel;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SampleList.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SampleList.java
index abe55de5a..33e895187 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SampleList.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SampleList.java
@@ -30,7 +30,7 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineCount;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SnpEff.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SnpEff.java
index f0bd7ecd9..b3b0be153 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SnpEff.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SnpEff.java
@@ -33,7 +33,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.RodRequiringAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.Utils;
 import org.broadinstitute.sting.utils.codecs.vcf.*;
 import org.broadinstitute.sting.utils.exceptions.UserException;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SpanningDeletions.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SpanningDeletions.java
index f6bb4e747..8e1140af1 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SpanningDeletions.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SpanningDeletions.java
@@ -6,7 +6,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TandemRepeatAnnotator.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TandemRepeatAnnotator.java
index 3d62f2d96..c72ba1c5f 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TandemRepeatAnnotator.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TandemRepeatAnnotator.java
@@ -30,7 +30,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineCount;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TechnologyComposition.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TechnologyComposition.java
index 43ef188a8..57b50c6e2 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TechnologyComposition.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TechnologyComposition.java
@@ -7,7 +7,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ExperimentalAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java
index c3e98c20f..be7288a7e 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java
@@ -8,7 +8,7 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ExperimentalAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.RodRequiringAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineCount;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java
index eae13e1b5..ee4f77752 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java
@@ -31,7 +31,7 @@ import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.*;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.*;
 import org.broadinstitute.sting.utils.exceptions.UserException;
 import org.broadinstitute.sting.utils.variantcontext.*;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/ActiveRegionBasedAnnotation.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/ActiveRegionBasedAnnotation.java
index 9c5710872..03fcba760 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/ActiveRegionBasedAnnotation.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/ActiveRegionBasedAnnotation.java
@@ -1,9 +1,7 @@
 package org.broadinstitute.sting.gatk.walkers.annotator.interfaces;
 
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
-import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
-import org.broadinstitute.sting.utils.variantcontext.Allele;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
 import java.util.List;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/GenotypeAnnotation.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/GenotypeAnnotation.java
index 6c55c1c00..6970908b5 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/GenotypeAnnotation.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/GenotypeAnnotation.java
@@ -3,7 +3,7 @@ package org.broadinstitute.sting.gatk.walkers.annotator.interfaces;
 import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFFormatHeaderLine;
 import org.broadinstitute.sting.utils.variantcontext.Genotype;
 import org.broadinstitute.sting.utils.variantcontext.GenotypeBuilder;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/InfoFieldAnnotation.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/InfoFieldAnnotation.java
index 738be9883..5b2dc310d 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/InfoFieldAnnotation.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/InfoFieldAnnotation.java
@@ -3,7 +3,7 @@ package org.broadinstitute.sting.gatk.walkers.annotator.interfaces;
 import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java
index 77da35c41..ae9b01f2d 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java
@@ -104,7 +104,7 @@ public abstract class GenotypeLikelihoodsCalculationModel implements Cloneable {
                                                    final List<Allele> allAllelesToUse,
                                                    final boolean useBAQedPileup,
                                                    final GenomeLocParser locParser,
-                                                   final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap);
+                                                   final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap);
 
 
     protected int getFilteredDepth(ReadBackedPileup pileup) {
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java
index 1761f9cec..e1092454a 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java
@@ -81,7 +81,7 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood
                                          final List<Allele> allAllelesToUse,
                                          final boolean useBAQedPileup,
                                          final GenomeLocParser locParser,
-                                         final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+                                         final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
 
         GenomeLoc loc = ref.getLocus();
 //        if (!ref.getLocus().equals(lastSiteVisited)) {
@@ -118,7 +118,7 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood
 
             if (!perReadAlleleLikelihoodMap.containsKey(sample.getKey())){
                 // no likelihoods have been computed for this sample at this site
-                perReadAlleleLikelihoodMap.put(sample.getKey(), new PerReadAlleleLikelihoodMap());
+                perReadAlleleLikelihoodMap.put(sample.getKey(), org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap.getBestAvailablePerReadAlleleLikelihoodMap());
             }
             final ReadBackedPileup pileup = context.getBasePileup();
             if (pileup != null) {
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java
index d053e13a8..54303e12c 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java
@@ -64,7 +64,7 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsC
                                          final List<Allele> allAllelesToUse,
                                          final boolean useBAQedPileup,
                                          final GenomeLocParser locParser,
-                                         final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+                                         final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
 
         perReadAlleleLikelihoodMap.clear(); // not used in SNP model, sanity check to delete any older data
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java
index a52b5dfe6..4d9f37098 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java
@@ -174,7 +174,7 @@ public class UnifiedGenotyperEngine {
 
         final List<GenotypeLikelihoodsCalculationModel.Model> models = getGLModelsToUse(tracker, refContext, rawContext);
 
-        final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap = new HashMap<String,PerReadAlleleLikelihoodMap>();
+        final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap = new HashMap<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap>();
 
         if ( models.isEmpty() ) {
             results.add(UAC.OutputMode == OUTPUT_MODE.EMIT_ALL_SITES && UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES ? generateEmptyContext(tracker, refContext, null, rawContext) : null);
@@ -209,7 +209,7 @@ public class UnifiedGenotyperEngine {
     public VariantContext calculateLikelihoods(final RefMetaDataTracker tracker,
                                                final ReferenceContext refContext,
                                                final AlignmentContext rawContext,
-                                               final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+                                               final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
         final List<GenotypeLikelihoodsCalculationModel.Model> models = getGLModelsToUse(tracker, refContext, rawContext);
         if ( models.isEmpty() ) {
             return null;
@@ -275,7 +275,7 @@ public class UnifiedGenotyperEngine {
                                                 final List<Allele> alternateAllelesToUse,
                                                 final boolean useBAQedPileup,
                                                 final GenotypeLikelihoodsCalculationModel.Model model,
-                                                final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+                                                final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
 
         // initialize the data for this thread if that hasn't been done yet
         if ( glcm.get() == null ) {
@@ -313,7 +313,7 @@ public class UnifiedGenotyperEngine {
         return new VariantCallContext(vc, false);
     }
 
-    public VariantCallContext calculateGenotypes(final VariantContext vc, final GenotypeLikelihoodsCalculationModel.Model model, final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+    public VariantCallContext calculateGenotypes(final VariantContext vc, final GenotypeLikelihoodsCalculationModel.Model model, final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
         return calculateGenotypes(null, null, null, null, vc, model, perReadAlleleLikelihoodMap);
     }
 
@@ -327,7 +327,7 @@ public class UnifiedGenotyperEngine {
                                                  final Map<String, AlignmentContext> stratifiedContexts,
                                                  final VariantContext vc,
                                                  final GenotypeLikelihoodsCalculationModel.Model model,
-                                                 final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+                                                 final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
         return calculateGenotypes(tracker, refContext, rawContext, stratifiedContexts, vc, model, false,perReadAlleleLikelihoodMap);
     }
 
@@ -346,7 +346,7 @@ public class UnifiedGenotyperEngine {
                                                  final AlignmentContext rawContext, Map<String, AlignmentContext> stratifiedContexts,
                                                  final VariantContext vc, final GenotypeLikelihoodsCalculationModel.Model model,
                                                  final boolean inheritAttributesFromInputVC,
-                                                 final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+                                                 final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
 
         boolean limitedContext = tracker == null || refContext == null || rawContext == null || stratifiedContexts == null;
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java
index 3d287057c..04a26672c 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java
@@ -27,7 +27,7 @@ package org.broadinstitute.sting.gatk.walkers.indels;
 
 import com.google.java.contract.Ensures;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.Haplotype;
 import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.clipping.ReadClipper;
@@ -42,7 +42,6 @@ import org.broadinstitute.sting.utils.sam.ReadUtils;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/PerReadAlleleLikelihoodMap.java b/public/java/src/org/broadinstitute/sting/utils/genotyper/PerReadAlleleLikelihoodMap.java
similarity index 76%
rename from public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/PerReadAlleleLikelihoodMap.java
rename to public/java/src/org/broadinstitute/sting/utils/genotyper/PerReadAlleleLikelihoodMap.java
index a83adc275..bae38283d 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/PerReadAlleleLikelihoodMap.java
+++ b/public/java/src/org/broadinstitute/sting/utils/genotyper/PerReadAlleleLikelihoodMap.java
@@ -22,26 +22,24 @@
  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
  * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
-package org.broadinstitute.sting.gatk.walkers.genotyper;
+package org.broadinstitute.sting.utils.genotyper;
 
 
-//import org.broadinstitute.sting.gatk.walkers.Requires;
-import com.google.java.contract.Requires;
+import org.broadinstitute.sting.utils.classloader.GATKLiteUtils;
+import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
+import java.lang.reflect.Constructor;
 import java.util.*;
 
-public class PerReadAlleleLikelihoodMap {
+public abstract class PerReadAlleleLikelihoodMap {
+
     public static final double INDEL_LIKELIHOOD_THRESH = 0.1;
 
-    private List<Allele> alleles;
-    private Map<GATKSAMRecord,Map<Allele,Double>> likelihoodReadMap;
-    public PerReadAlleleLikelihoodMap() {
-        likelihoodReadMap = new LinkedHashMap<GATKSAMRecord,Map<Allele,Double>>();
-        alleles = new ArrayList<Allele>();
-    }
+    protected List<Allele> alleles;
+    protected Map<GATKSAMRecord,Map<Allele,Double>> likelihoodReadMap;
 
     public void add(GATKSAMRecord read, Allele a, Double likelihood) {
         Map<Allele,Double> likelihoodMap;
@@ -95,16 +93,6 @@ public class PerReadAlleleLikelihoodMap {
     public int getNumberOfStoredElements() {
         return likelihoodReadMap.size();
     }
-    /**
-     * Returns list of reads greedily associated with a particular allele.
-     * Needs to loop for each read, and assign to each allele
-     * @param a                          Desired allele
-     * @return
-     */
-    @Requires("a!=null")
-    public List<GATKSAMRecord> getReadsAssociatedWithAllele(Allele a) {
-        return null;
-    }
 
     public Map<Allele,Double> getLikelihoodsAssociatedWithPileupElement(PileupElement p) {
         if (!likelihoodReadMap.containsKey(p.getRead()))
@@ -129,4 +117,18 @@ public class PerReadAlleleLikelihoodMap {
         }
         return (maxLike - prevMaxLike > INDEL_LIKELIHOOD_THRESH ? mostLikelyAllele : Allele.NO_CALL );
     }
- }
+
+    public abstract PerReadAlleleLikelihoodMap createPerAlleleDownsampledMap(final double downsamplingFraction);
+
+    public static PerReadAlleleLikelihoodMap getBestAvailablePerReadAlleleLikelihoodMap() {
+        final Class PerReadAlleleLikelihoodMapClass = GATKLiteUtils.getProtectedClassIfAvailable(PerReadAlleleLikelihoodMap.class);
+        try {
+            Constructor constructor = PerReadAlleleLikelihoodMapClass.getDeclaredConstructor((Class[])null);
+            constructor.setAccessible(true);
+            return (PerReadAlleleLikelihoodMap)constructor.newInstance();
+        }
+        catch (Exception e) {
+            throw new ReviewedStingException("Unable to create RecalibrationEngine class instance " + PerReadAlleleLikelihoodMapClass.getSimpleName());
+        }
+    }
+}
diff --git a/public/java/src/org/broadinstitute/sting/utils/genotyper/StandardPerReadAlleleLikelihoodMap.java b/public/java/src/org/broadinstitute/sting/utils/genotyper/StandardPerReadAlleleLikelihoodMap.java
new file mode 100644
index 000000000..7ac757783
--- /dev/null
+++ b/public/java/src/org/broadinstitute/sting/utils/genotyper/StandardPerReadAlleleLikelihoodMap.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2011 The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+ * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package org.broadinstitute.sting.utils.genotyper;
+
+
+import org.broadinstitute.sting.utils.classloader.PublicPackageSource;
+import org.broadinstitute.sting.utils.pileup.PileupElement;
+import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
+import org.broadinstitute.sting.utils.variantcontext.Allele;
+
+import java.util.*;
+
+public class StandardPerReadAlleleLikelihoodMap extends PerReadAlleleLikelihoodMap implements PublicPackageSource {
+
+    public StandardPerReadAlleleLikelihoodMap() {
+        likelihoodReadMap = new LinkedHashMap<GATKSAMRecord,Map<Allele,Double>>();
+        alleles = new ArrayList<Allele>();
+    }
+
+    // not implemented in the standard version
+    public PerReadAlleleLikelihoodMap createPerAlleleDownsampledMap(final double downsamplingFraction) { return this; }
+ }

From c6b57fffdaf2a86a252f08a91881621663e04bf6 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Wed, 24 Oct 2012 22:52:25 -0400
Subject: [PATCH 19/40] Added allele biased down-sampling capabilities to the
 PerReadAlleleLikelihoodMap object, which means that both the UG and HC can
 use this functionality.  Note that it's only available in protected, so
 GATK-lite users won't be allowed to enable it.  Needs more testing.

---
 .../AlleleBiasedDownsamplingUtils.java        | 195 ++++++++++++++++++
 .../gatk/walkers/genotyper/ErrorModel.java    |   3 +-
 .../haplotypecaller/HaplotypeCaller.java      |   3 +-
 .../LikelihoodCalculationEngine.java          |   8 +-
 .../AdvancedPerReadAlleleLikelihoodMap.java   |  35 +++-
 .../StandardCallerArgumentCollection.java     |  33 ++-
 ...elGenotypeLikelihoodsCalculationModel.java |   7 +-
 ...NPGenotypeLikelihoodsCalculationModel.java |  67 +-----
 .../genotyper/UnifiedGenotyperEngine.java     |  26 +--
 .../indels/PairHMMIndelErrorModel.java        |   6 +-
 .../genotyper/PerReadAlleleLikelihoodMap.java |   7 +-
 .../StandardPerReadAlleleLikelihoodMap.java   |   8 +-
 12 files changed, 289 insertions(+), 109 deletions(-)
 create mode 100755 protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java

diff --git a/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java b/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
new file mode 100755
index 000000000..e224a9c57
--- /dev/null
+++ b/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
@@ -0,0 +1,195 @@
+/*
+ * Copyright (c) 2010.
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+ * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.gatk.downsampling;
+
+import net.sf.samtools.SAMReadGroupRecord;
+import net.sf.samtools.SAMRecord;
+import org.broadinstitute.sting.utils.*;
+import org.broadinstitute.sting.utils.pileup.*;
+import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
+import org.broadinstitute.sting.utils.variantcontext.Allele;
+
+import java.io.PrintStream;
+import java.util.*;
+
+public class AlleleBiasedDownsamplingUtils {
+
+    private static final ArrayList<PileupElement>[] alleleStratifiedElements = new ArrayList[4];
+    static {
+        for ( int i = 0; i < 4; i++ )
+            alleleStratifiedElements[i] = new ArrayList<PileupElement>();
+    }
+
+    /**
+     * Computes an allele biased version of the given pileup
+     *
+     * @param pileup                    the original pileup
+     * @param downsamplingFraction      the fraction of total reads to remove per allele
+     * @param log                       logging output
+     * @return allele biased pileup
+     */
+    public synchronized static ReadBackedPileup createAlleleBiasedBasePileup(final ReadBackedPileup pileup, final double downsamplingFraction, final PrintStream log) {
+        // special case removal of all or no reads
+        if ( downsamplingFraction <= 0.0 )
+            return pileup;
+        if ( downsamplingFraction >= 1.0 )
+            return new ReadBackedPileupImpl(pileup.getLocation(), new ArrayList<PileupElement>());
+
+        // start by stratifying the reads by the alleles they represent at this position
+        for( final PileupElement pe : pileup ) {
+            final int baseIndex = BaseUtils.simpleBaseToBaseIndex(pe.getBase());
+            if ( baseIndex != -1 )
+                alleleStratifiedElements[baseIndex].add(pe);
+        }
+
+        // Down-sample *each* allele by the contamination fraction applied to the entire pileup.
+        // Unfortunately, we need to maintain the original pileup ordering of reads or FragmentUtils will complain later.
+        int numReadsToRemove = (int)(pileup.getNumberOfElements() * downsamplingFraction); // floor
+        final TreeSet<PileupElement> elementsToKeep = new TreeSet<PileupElement>(new Comparator<PileupElement>() {
+            @Override
+            public int compare(PileupElement element1, PileupElement element2) {
+                final int difference = element1.getRead().getAlignmentStart() - element2.getRead().getAlignmentStart();
+                return difference != 0 ? difference : element1.getRead().getReadName().compareTo(element2.getRead().getReadName());
+            }
+        });
+
+        for ( int i = 0; i < 4; i++ ) {
+            final ArrayList<PileupElement> alleleList = alleleStratifiedElements[i];
+            if ( alleleList.size() <= numReadsToRemove )
+                logAllElements(alleleList, log);
+            else
+                elementsToKeep.addAll(downsampleElements(alleleList, numReadsToRemove, log));
+        }
+
+        // clean up pointers so memory can be garbage collected if needed
+        for ( int i = 0; i < 4; i++ )
+            alleleStratifiedElements[i].clear();
+
+        return new ReadBackedPileupImpl(pileup.getLocation(), new ArrayList<PileupElement>(elementsToKeep));
+    }
+
+    /**
+     * Performs allele biased down-sampling on a pileup and computes the list of elements to keep
+     *
+     * @param elements                  original list of records
+     * @param numElementsToRemove       the number of records to remove
+     * @param log                       logging output
+     * @return the list of pileup elements TO KEEP
+     */
+    private static List<PileupElement> downsampleElements(final ArrayList<PileupElement> elements, final int numElementsToRemove, final PrintStream log) {
+        final int pileupSize = elements.size();
+        final BitSet itemsToRemove = new BitSet(pileupSize);
+        for ( Integer selectedIndex : MathUtils.sampleIndicesWithoutReplacement(pileupSize, numElementsToRemove) ) {
+            itemsToRemove.set(selectedIndex);
+        }
+
+        ArrayList<PileupElement> elementsToKeep = new ArrayList<PileupElement>(pileupSize - numElementsToRemove);
+        for ( int i = 0; i < pileupSize; i++ ) {
+            if ( itemsToRemove.get(i) )
+                logRead(elements.get(i).getRead(), log);
+            else
+                elementsToKeep.add(elements.get(i));
+        }
+
+        return elementsToKeep;
+    }
+
+    /**
+     * Computes reads to remove based on an allele biased down-sampling
+     *
+     * @param alleleReadMap             original list of records per allele
+     * @param downsamplingFraction      the fraction of total reads to remove per allele
+     * @param log                       logging output
+     * @return list of reads TO REMOVE from allele biased down-sampling
+     */
+    public static List<GATKSAMRecord> selectAlleleBiasedReads(final Map<Allele, List<GATKSAMRecord>> alleleReadMap, final double downsamplingFraction, final PrintStream log) {
+        int totalReads = 0;
+        for ( final List<GATKSAMRecord> reads : alleleReadMap.values() )
+            totalReads += reads.size();
+
+        // Down-sample *each* allele by the contamination fraction applied to the entire pileup.
+        int numReadsToRemove = (int)(totalReads * downsamplingFraction);
+        final List<GATKSAMRecord> readsToRemove = new ArrayList<GATKSAMRecord>(numReadsToRemove * alleleReadMap.size());
+        for ( final List<GATKSAMRecord> reads : alleleReadMap.values() ) {
+            if ( reads.size() <= numReadsToRemove ) {
+                readsToRemove.addAll(reads);
+                logAllReads(reads, log);
+            } else {
+                readsToRemove.addAll(downsampleReads(reads, numReadsToRemove, log));
+            }
+        }
+
+        return readsToRemove;
+    }
+
+    /**
+     * Performs allele biased down-sampling on a pileup and computes the list of elements to remove
+     *
+     * @param reads                     original list of records
+     * @param numElementsToRemove       the number of records to remove
+     * @param log                       logging output
+     * @return the list of pileup elements TO REMOVE
+     */
+    private static List<GATKSAMRecord> downsampleReads(final List<GATKSAMRecord> reads, final int numElementsToRemove, final PrintStream log) {
+        final int pileupSize = reads.size();
+        final BitSet itemsToRemove = new BitSet(pileupSize);
+        for ( Integer selectedIndex : MathUtils.sampleIndicesWithoutReplacement(pileupSize, numElementsToRemove) ) {
+            itemsToRemove.set(selectedIndex);
+        }
+
+        ArrayList<GATKSAMRecord> readsToRemove = new ArrayList<GATKSAMRecord>(pileupSize - numElementsToRemove);
+        for ( int i = 0; i < pileupSize; i++ ) {
+            if ( itemsToRemove.get(i) ) {
+                final GATKSAMRecord read = reads.get(i);
+                readsToRemove.add(read);
+                logRead(read, log);
+            }
+        }
+
+        return readsToRemove;
+    }
+
+    private static void logAllElements(final List<PileupElement> elements, final PrintStream log) {
+        if ( log != null ) {
+            for ( final PileupElement p : elements )
+                logRead(p.getRead(), log);
+        }
+    }
+
+    private static void logAllReads(final List<GATKSAMRecord> reads, final PrintStream log) {
+        if ( log != null ) {
+            for ( final GATKSAMRecord read : reads )
+                logRead(read, log);
+        }
+    }
+
+    private static void logRead(final SAMRecord read, final PrintStream log) {
+        if ( log != null ) {
+            final SAMReadGroupRecord readGroup = read.getReadGroup();
+            log.println(String.format("%s\t%s\t%s\t%s", read.getReadName(), readGroup.getSample(), readGroup.getLibrary(), readGroup.getPlatformUnit()));
+        }
+    }
+}
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ErrorModel.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ErrorModel.java
index a01973ad5..fc6d23382 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ErrorModel.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ErrorModel.java
@@ -5,6 +5,7 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.walkers.indels.PairHMMIndelErrorModel;
 import org.broadinstitute.sting.utils.Haplotype;
 import org.broadinstitute.sting.utils.MathUtils;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
@@ -59,7 +60,7 @@ public class ErrorModel  {
 
         boolean hasCalledAlleles = false;
 
-        final org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap = org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap.getBestAvailablePerReadAlleleLikelihoodMap();
+        final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap = PerReadAlleleLikelihoodMap.getBestAvailablePerReadAlleleLikelihoodMap();
         if (refSampleVC != null) {
 
             for (Allele allele : refSampleVC.getAlleles()) {
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java
index 3293ac817..5aba23faa 100755
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java
@@ -51,6 +51,7 @@ import org.broadinstitute.sting.utils.exceptions.UserException;
 import org.broadinstitute.sting.utils.fasta.CachingIndexedFastaSequenceFile;
 import org.broadinstitute.sting.utils.fragments.FragmentCollection;
 import org.broadinstitute.sting.utils.fragments.FragmentUtils;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.help.DocumentedGATKFeature;
 import org.broadinstitute.sting.utils.pairhmm.PairHMM;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
@@ -424,7 +425,7 @@ public class HaplotypeCaller extends ActiveRegionWalker<Integer, Integer> implem
                   : genotypingEngine.assignGenotypeLikelihoodsAndCallIndependentEvents( UG_engine, bestHaplotypes, fullReferenceWithPadding, getPaddedLoc(activeRegion), activeRegion.getLocation(), getToolkit().getGenomeLocParser(), activeAllelesToGenotype ) ) ) {
             if( DEBUG ) { System.out.println(callResult.getFirst().toStringWithoutGenotypes()); }
 
-            final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> stratifiedReadMap = LikelihoodCalculationEngine.partitionReadsBasedOnLikelihoods( getToolkit().getGenomeLocParser(), perSampleReadList, perSampleFilteredReadList, callResult );
+            final Map<String, PerReadAlleleLikelihoodMap> stratifiedReadMap = LikelihoodCalculationEngine.partitionReadsBasedOnLikelihoods( getToolkit().getGenomeLocParser(), perSampleReadList, perSampleFilteredReadList, callResult, UG_engine.getUAC().CONTAMINATION_FRACTION, UG_engine.getUAC().contaminationLog );
             final VariantContext annotatedCall = annotationEngine.annotateContext(stratifiedReadMap, callResult.getFirst());
             final Map<String, Object> myAttributes = new LinkedHashMap<String, Object>(annotatedCall.getAttributes());
 
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java
index a05e20bd3..e1a9ef7c8 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java
@@ -38,6 +38,7 @@ import org.broadinstitute.sting.utils.sam.ReadUtils;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
+import java.io.PrintStream;
 import java.util.*;
 
 public class LikelihoodCalculationEngine {
@@ -346,7 +347,9 @@ public class LikelihoodCalculationEngine {
     public static Map<String, PerReadAlleleLikelihoodMap> partitionReadsBasedOnLikelihoods( final GenomeLocParser parser,
                                                                                             final HashMap<String, ArrayList<GATKSAMRecord>> perSampleReadList,
                                                                                             final HashMap<String, ArrayList<GATKSAMRecord>> perSampleFilteredReadList,
-                                                                                            final Pair<VariantContext, HashMap<Allele,ArrayList<Haplotype>>> call) {
+                                                                                            final Pair<VariantContext, HashMap<Allele,ArrayList<Haplotype>>> call,
+                                                                                            final double downsamplingFraction,
+                                                                                            final PrintStream downsamplingLog ) {
         final Map<String, PerReadAlleleLikelihoodMap> returnMap = new HashMap<String, PerReadAlleleLikelihoodMap>();
         final GenomeLoc callLoc = parser.createGenomeLoc(call.getFirst());
         for( final Map.Entry<String, ArrayList<GATKSAMRecord>> sample : perSampleReadList.entrySet() ) {
@@ -370,6 +373,9 @@ public class LikelihoodCalculationEngine {
                 }
             }
 
+            // down-sample before adding filtered reads
+            likelihoodMap.performPerAlleleDownsampling(downsamplingFraction, downsamplingLog);
+
             // add all filtered reads to the NO_CALL list because they weren't given any likelihoods
             for( final GATKSAMRecord read : perSampleFilteredReadList.get(sample.getKey()) ) {
                 // only count the read if it overlaps the event, otherwise it is not added to the output read list at all
diff --git a/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java b/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
index db1bf8762..80618cfdb 100644
--- a/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
+++ b/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
@@ -25,16 +25,43 @@
 package org.broadinstitute.sting.utils.genotyper;
 
 
+import org.broadinstitute.sting.gatk.downsampling.AlleleBiasedDownsamplingUtils;
 import org.broadinstitute.sting.utils.classloader.ProtectedPackageSource;
-import org.broadinstitute.sting.utils.pileup.PileupElement;
+import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
+import java.io.PrintStream;
 import java.util.*;
 
 public class AdvancedPerReadAlleleLikelihoodMap extends StandardPerReadAlleleLikelihoodMap implements ProtectedPackageSource {
 
-    public PerReadAlleleLikelihoodMap createPerAlleleDownsampledMap(final double downsamplingFraction) {
-        return this;
+    public ReadBackedPileup createPerAlleleDownsampledBasePileup(final ReadBackedPileup pileup, final double downsamplingFraction, final PrintStream log) {
+        return AlleleBiasedDownsamplingUtils.createAlleleBiasedBasePileup(pileup, downsamplingFraction, log);
     }
- }
+
+    public void performPerAlleleDownsampling(final double downsamplingFraction, final PrintStream log) {
+        // special case removal of all or no reads
+        if ( downsamplingFraction <= 0.0 )
+            return;
+        if ( downsamplingFraction >= 1.0 ) {
+            likelihoodReadMap.clear();
+            return;
+        }
+
+        // start by stratifying the reads by the alleles they represent at this position
+        final Map<Allele, List<GATKSAMRecord>> alleleReadMap = new HashMap<Allele, List<GATKSAMRecord>>(alleles.size());
+        for ( Allele allele : alleles )
+            alleleReadMap.put(allele, new ArrayList<GATKSAMRecord>());
+
+        for ( Map.Entry<GATKSAMRecord, Map<Allele, Double>> entry : likelihoodReadMap.entrySet() ) {
+            final Allele bestAllele = getMostLikelyAllele(entry.getValue());
+            alleleReadMap.get(bestAllele).add(entry.getKey());
+        }
+
+        // compute the reads to remove and actually remove them
+        final List<GATKSAMRecord> readsToRemove = AlleleBiasedDownsamplingUtils.selectAlleleBiasedReads(alleleReadMap, downsamplingFraction, log);
+        for ( final GATKSAMRecord read : readsToRemove )
+            likelihoodReadMap.remove(read);
+    }
+}
diff --git a/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
index aa3a1e6ac..36456ca09 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
@@ -7,6 +7,7 @@ import org.broadinstitute.sting.gatk.walkers.genotyper.afcalc.AFCalcFactory;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
 import java.io.File;
+import java.io.PrintStream;
 
 /**
  * Created with IntelliJ IDEA.
@@ -63,26 +64,23 @@ public class StandardCallerArgumentCollection {
     public int MAX_ALTERNATE_ALLELES = 6;
 
     /**
-     * If there are more than this number of alternate alleles presented to the genotyper (either through discovery or GENOTYPE_GIVEN ALLELES),
-     * then only this many alleles will be used.  Note that genotyping sites with many alternate alleles is both CPU and memory intensive and it
-     * scales exponentially based on the number of alternate alleles.  Unless there is a good reason to change the default value, we highly recommend
-     * that you not play around with this parameter.
-     *
-     * This argument has been retired in GATK 2.2.  Please specify just maxAltAlleles from now on
+     * Controls the model used to calculate the probability that a site is variant plus the various sample genotypes in the data at a given locus.
      */
-    @Deprecated
-    @Hidden
-    @Argument(fullName = "max_alternate_alleles_for_indels", shortName = "maxAltAllelesForIndels", doc = "This argument has been retired in GATK 2.2.  Please specify just maxAltAlleles from now on, which will apply to any variant, regardless of type", required = false)
-    public int MAX_ALTERNATE_ALLELES_FOR_INDELS = -1;
+    @Advanced
+    @Argument(fullName = "p_nonref_model", shortName = "pnrm", doc = "Non-reference probability calculation model to employ", required = false)
+    public AFCalcFactory.Calculation AFmodel = AFCalcFactory.Calculation.getDefaultModel();
 
     /**
      * If this fraction is greater is than zero, the caller will aggressively attempt to remove contamination through biased down-sampling of reads.
      * Basically, it will ignore the contamination fraction of reads for each alternate allele.  So if the pileup contains N total bases, then we
      * will try to remove (N * contamination fraction) bases for each alternate allele.
      */
-    @Hidden
     @Argument(fullName = "contamination_percentage_to_filter", shortName = "contamination", doc = "Fraction of contamination in sequencing data (for all samples) to aggressively remove", required = false)
-    public double CONTAMINATION_PERCENTAGE = 0.0;
+    public double CONTAMINATION_FRACTION = 0.0;
+
+    @Hidden
+    @Argument(shortName = "logRemovedReadsFromContaminationFiltering", doc="contaminationLog", required=false)
+    public PrintStream contaminationLog = null;
 
     @Hidden
     @Argument(shortName = "logExactCalls", doc="x", required=false)
@@ -96,19 +94,12 @@ public class StandardCallerArgumentCollection {
         this.GenotypingMode = SCAC.GenotypingMode;
         this.heterozygosity = SCAC.heterozygosity;
         this.MAX_ALTERNATE_ALLELES = SCAC.MAX_ALTERNATE_ALLELES;
-        this.MAX_ALTERNATE_ALLELES_FOR_INDELS = SCAC.MAX_ALTERNATE_ALLELES_FOR_INDELS;
         this.OutputMode = SCAC.OutputMode;
         this.STANDARD_CONFIDENCE_FOR_CALLING = SCAC.STANDARD_CONFIDENCE_FOR_CALLING;
         this.STANDARD_CONFIDENCE_FOR_EMITTING = SCAC.STANDARD_CONFIDENCE_FOR_EMITTING;
-        this.CONTAMINATION_PERCENTAGE = SCAC.CONTAMINATION_PERCENTAGE;
+        this.CONTAMINATION_FRACTION = SCAC.CONTAMINATION_FRACTION;
+        this.contaminationLog = SCAC.contaminationLog;
         this.exactCallsLog = SCAC.exactCallsLog;
         this.AFmodel = SCAC.AFmodel;
     }
-
-    /**
-     * Controls the model used to calculate the probability that a site is variant plus the various sample genotypes in the data at a given locus.
-     */
-    @Advanced
-    @Argument(fullName = "p_nonref_model", shortName = "pnrm", doc = "Non-reference probability calculation model to employ", required = false)
-    public AFCalcFactory.Calculation AFmodel = AFCalcFactory.Calculation.getDefaultModel();
 }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java
index e1092454a..0d9f443e2 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java
@@ -35,6 +35,7 @@ import org.broadinstitute.sting.utils.BaseUtils;
 import org.broadinstitute.sting.utils.GenomeLoc;
 import org.broadinstitute.sting.utils.GenomeLocParser;
 import org.broadinstitute.sting.utils.Haplotype;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.variantcontext.*;
@@ -81,7 +82,7 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood
                                          final List<Allele> allAllelesToUse,
                                          final boolean useBAQedPileup,
                                          final GenomeLocParser locParser,
-                                         final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+                                         final Map<String, PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
 
         GenomeLoc loc = ref.getLocus();
 //        if (!ref.getLocus().equals(lastSiteVisited)) {
@@ -118,12 +119,12 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood
 
             if (!perReadAlleleLikelihoodMap.containsKey(sample.getKey())){
                 // no likelihoods have been computed for this sample at this site
-                perReadAlleleLikelihoodMap.put(sample.getKey(), org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap.getBestAvailablePerReadAlleleLikelihoodMap());
+                perReadAlleleLikelihoodMap.put(sample.getKey(), PerReadAlleleLikelihoodMap.getBestAvailablePerReadAlleleLikelihoodMap());
             }
             final ReadBackedPileup pileup = context.getBasePileup();
             if (pileup != null) {
                 final GenotypeBuilder b = new GenotypeBuilder(sample.getKey());
-                final double[] genotypeLikelihoods = pairModel.computeDiploidReadHaplotypeLikelihoods(pileup, haplotypeMap, ref, eventLength, perReadAlleleLikelihoodMap.get(sample.getKey()));
+                final double[] genotypeLikelihoods = pairModel.computeDiploidReadHaplotypeLikelihoods(pileup, haplotypeMap, ref, eventLength, perReadAlleleLikelihoodMap.get(sample.getKey()), UAC.CONTAMINATION_FRACTION, UAC.contaminationLog);
                 b.PL(genotypeLikelihoods);
                 b.DP(getFilteredDepth(pileup));
                 genotypes.add(b.make());
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java
index 54303e12c..791cdc325 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java
@@ -36,6 +36,7 @@ import org.broadinstitute.sting.utils.GenomeLocParser;
 import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.baq.BAQ;
 import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileupImpl;
@@ -48,13 +49,13 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsC
     private final boolean useAlleleFromVCF;
 
     private final double[] likelihoodSums = new double[4];
-    private final ArrayList<PileupElement>[] alleleStratifiedElements = new ArrayList[4];
+
+    private final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap;
 
     protected SNPGenotypeLikelihoodsCalculationModel(UnifiedArgumentCollection UAC, Logger logger) {
         super(UAC, logger);
         useAlleleFromVCF = UAC.GenotypingMode == GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES;
-        for ( int i = 0; i < 4; i++ )
-            alleleStratifiedElements[i] = new ArrayList<PileupElement>();
+        perReadAlleleLikelihoodMap = PerReadAlleleLikelihoodMap.getBestAvailablePerReadAlleleLikelihoodMap();
     }
 
     public VariantContext getLikelihoods(final RefMetaDataTracker tracker,
@@ -64,9 +65,9 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsC
                                          final List<Allele> allAllelesToUse,
                                          final boolean useBAQedPileup,
                                          final GenomeLocParser locParser,
-                                         final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+                                         final Map<String, PerReadAlleleLikelihoodMap> sampleLikelihoodMap) {
 
-        perReadAlleleLikelihoodMap.clear(); // not used in SNP model, sanity check to delete any older data
+        sampleLikelihoodMap.clear(); // not used in SNP model, sanity check to delete any older data
 
         final byte refBase = ref.getBase();
         final int indexOfRefBase = BaseUtils.simpleBaseToBaseIndex(refBase);
@@ -79,8 +80,8 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsC
         ArrayList<SampleGenotypeData> GLs = new ArrayList<SampleGenotypeData>(contexts.size());
         for ( Map.Entry<String, AlignmentContext> sample : contexts.entrySet() ) {
             ReadBackedPileup pileup = AlignmentContextUtils.stratify(sample.getValue(), contextType).getBasePileup();
-            if ( UAC.CONTAMINATION_PERCENTAGE > 0.0 )
-                pileup = createDecontaminatedPileup(pileup, UAC.CONTAMINATION_PERCENTAGE);
+            if ( UAC.CONTAMINATION_FRACTION > 0.0 )
+                pileup = perReadAlleleLikelihoodMap.createPerAlleleDownsampledBasePileup(pileup, UAC.CONTAMINATION_FRACTION, UAC.contaminationLog);
             if ( useBAQedPileup )
                 pileup = createBAQedPileup(pileup);
 
@@ -203,42 +204,6 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsC
         return allelesToUse;
     }
 
-    public ReadBackedPileup createDecontaminatedPileup(final ReadBackedPileup pileup, final double contaminationPercentage) {
-        // special case removal of all reads
-        if ( contaminationPercentage >= 1.0 )
-            return new ReadBackedPileupImpl(pileup.getLocation(), new ArrayList<PileupElement>());
-
-        // start by stratifying the reads by the alleles they represent at this position
-        for( final PileupElement pe : pileup ) {
-            final int baseIndex = BaseUtils.simpleBaseToBaseIndex(pe.getBase());
-            if ( baseIndex != -1 )
-                alleleStratifiedElements[baseIndex].add(pe);
-        }
-
-        // Down-sample *each* allele by the contamination fraction applied to the entire pileup.
-        // Unfortunately, we need to maintain the original pileup ordering of reads or FragmentUtils will complain later.
-        int numReadsToRemove = (int)Math.ceil((double)pileup.getNumberOfElements() * contaminationPercentage);
-        final TreeSet<PileupElement> elementsToKeep = new TreeSet<PileupElement>(new Comparator<PileupElement>() {
-            @Override
-            public int compare(PileupElement element1, PileupElement element2) {
-                final int difference = element1.getRead().getAlignmentStart() - element2.getRead().getAlignmentStart();
-                return difference != 0 ? difference : element1.getRead().getReadName().compareTo(element2.getRead().getReadName());
-            }
-        });
-
-        for ( int i = 0; i < 4; i++ ) {
-            final ArrayList<PileupElement> alleleList = alleleStratifiedElements[i];
-            if ( alleleList.size() > numReadsToRemove )
-                elementsToKeep.addAll(downsampleElements(alleleList, numReadsToRemove));
-        }
-
-        // clean up pointers so memory can be garbage collected if needed
-        for ( int i = 0; i < 4; i++ )
-            alleleStratifiedElements[i].clear();
-
-        return new ReadBackedPileupImpl(pileup.getLocation(), new ArrayList<PileupElement>(elementsToKeep));
-    }
-
     public ReadBackedPileup createBAQedPileup( final ReadBackedPileup pileup ) {
         final List<PileupElement> BAQedElements = new ArrayList<PileupElement>();
         for( final PileupElement PE : pileup ) {
@@ -257,22 +222,6 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsC
         public byte getQual( final int offset ) { return BAQ.calcBAQFromTag(getRead(), offset, true); }
     }
 
-    private List<PileupElement> downsampleElements(final ArrayList<PileupElement> elements, final int numElementsToRemove) {
-        final int pileupSize = elements.size();
-        final BitSet itemsToRemove = new BitSet(pileupSize);
-        for ( Integer selectedIndex : MathUtils.sampleIndicesWithoutReplacement(pileupSize, numElementsToRemove) ) {
-            itemsToRemove.set(selectedIndex);
-        }
-
-        ArrayList<PileupElement> elementsToKeep = new ArrayList<PileupElement>(pileupSize - numElementsToRemove);
-        for ( int i = 0; i < pileupSize; i++ ) {
-            if ( !itemsToRemove.get(i) )
-                elementsToKeep.add(elements.get(i));
-        }
-
-        return elementsToKeep;
-    }
-
     private static class SampleGenotypeData {
 
         public final String name;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java
index 4d9f37098..3c4a97ec1 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java
@@ -153,19 +153,19 @@ public class UnifiedGenotyperEngine {
     }
 
 
-        /**
-        * Compute full calls at a given locus. Entry point for engine calls from the UnifiedGenotyper.
-        *
-        * If allSamples != null, then the output variantCallContext is guarenteed to contain a genotype
-        * for every sample in allSamples.  If it's null there's no such guarentee.  Providing this
-        * argument is critical when the resulting calls will be written to a VCF file.
-        *
-        * @param tracker    the meta data tracker
-        * @param refContext the reference base
-        * @param rawContext contextual information around the locus
-        * @param allSamples set of all sample names that we might call (i.e., those in the VCF header)
-        * @return the VariantCallContext object
-        */
+    /**
+     * Compute full calls at a given locus. Entry point for engine calls from the UnifiedGenotyper.
+     *
+     * If allSamples != null, then the output variantCallContext is guarenteed to contain a genotype
+     * for every sample in allSamples.  If it's null there's no such guarentee.  Providing this
+     * argument is critical when the resulting calls will be written to a VCF file.
+     *
+     * @param tracker    the meta data tracker
+     * @param refContext the reference base
+     * @param rawContext contextual information around the locus
+     * @param allSamples set of all sample names that we might call (i.e., those in the VCF header)
+     * @return the VariantCallContext object
+     */
     public List<VariantCallContext> calculateLikelihoodsAndGenotypes(final RefMetaDataTracker tracker,
                                                                      final ReferenceContext refContext,
                                                                      final AlignmentContext rawContext,
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java
index 04a26672c..79962a3e4 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java
@@ -41,6 +41,7 @@ import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.sam.ReadUtils;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
+import java.io.PrintStream;
 import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -187,11 +188,14 @@ public class PairHMMIndelErrorModel {
                                                                         final LinkedHashMap<Allele, Haplotype> haplotypeMap,
                                                                         final ReferenceContext ref,
                                                                         final int eventLength,
-                                                                        final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap){
+                                                                        final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap,
+                                                                        final double downsamplingFraction,
+                                                                        final PrintStream downsamplingLog) {
         final int numHaplotypes = haplotypeMap.size();
 
         final int readCounts[] = new int[pileup.getNumberOfElements()];
         final double[][] readLikelihoods = computeGeneralReadHaplotypeLikelihoods(pileup, haplotypeMap, ref, eventLength, perReadAlleleLikelihoodMap, readCounts);
+        perReadAlleleLikelihoodMap.performPerAlleleDownsampling(downsamplingFraction, downsamplingLog);
         return getDiploidHaplotypeLikelihoods(numHaplotypes, readCounts, readLikelihoods);
         
     }
diff --git a/public/java/src/org/broadinstitute/sting/utils/genotyper/PerReadAlleleLikelihoodMap.java b/public/java/src/org/broadinstitute/sting/utils/genotyper/PerReadAlleleLikelihoodMap.java
index bae38283d..22d249240 100644
--- a/public/java/src/org/broadinstitute/sting/utils/genotyper/PerReadAlleleLikelihoodMap.java
+++ b/public/java/src/org/broadinstitute/sting/utils/genotyper/PerReadAlleleLikelihoodMap.java
@@ -28,9 +28,11 @@ package org.broadinstitute.sting.utils.genotyper;
 import org.broadinstitute.sting.utils.classloader.GATKLiteUtils;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
+import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
+import java.io.PrintStream;
 import java.lang.reflect.Constructor;
 import java.util.*;
 
@@ -41,6 +43,9 @@ public abstract class PerReadAlleleLikelihoodMap {
     protected List<Allele> alleles;
     protected Map<GATKSAMRecord,Map<Allele,Double>> likelihoodReadMap;
 
+    public abstract void performPerAlleleDownsampling(final double downsamplingFraction, final PrintStream log);
+    public abstract ReadBackedPileup createPerAlleleDownsampledBasePileup(final ReadBackedPileup pileup, final double downsamplingFraction, final PrintStream log);
+
     public void add(GATKSAMRecord read, Allele a, Double likelihood) {
         Map<Allele,Double> likelihoodMap;
         if (likelihoodReadMap.containsKey(read)){
@@ -118,8 +123,6 @@ public abstract class PerReadAlleleLikelihoodMap {
         return (maxLike - prevMaxLike > INDEL_LIKELIHOOD_THRESH ? mostLikelyAllele : Allele.NO_CALL );
     }
 
-    public abstract PerReadAlleleLikelihoodMap createPerAlleleDownsampledMap(final double downsamplingFraction);
-
     public static PerReadAlleleLikelihoodMap getBestAvailablePerReadAlleleLikelihoodMap() {
         final Class PerReadAlleleLikelihoodMapClass = GATKLiteUtils.getProtectedClassIfAvailable(PerReadAlleleLikelihoodMap.class);
         try {
diff --git a/public/java/src/org/broadinstitute/sting/utils/genotyper/StandardPerReadAlleleLikelihoodMap.java b/public/java/src/org/broadinstitute/sting/utils/genotyper/StandardPerReadAlleleLikelihoodMap.java
index 7ac757783..7db818592 100644
--- a/public/java/src/org/broadinstitute/sting/utils/genotyper/StandardPerReadAlleleLikelihoodMap.java
+++ b/public/java/src/org/broadinstitute/sting/utils/genotyper/StandardPerReadAlleleLikelihoodMap.java
@@ -26,10 +26,11 @@ package org.broadinstitute.sting.utils.genotyper;
 
 
 import org.broadinstitute.sting.utils.classloader.PublicPackageSource;
-import org.broadinstitute.sting.utils.pileup.PileupElement;
+import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
+import java.io.PrintStream;
 import java.util.*;
 
 public class StandardPerReadAlleleLikelihoodMap extends PerReadAlleleLikelihoodMap implements PublicPackageSource {
@@ -40,5 +41,6 @@ public class StandardPerReadAlleleLikelihoodMap extends PerReadAlleleLikelihoodM
     }
 
     // not implemented in the standard version
-    public PerReadAlleleLikelihoodMap createPerAlleleDownsampledMap(final double downsamplingFraction) { return this; }
- }
+    public void performPerAlleleDownsampling(final double downsamplingFraction, final PrintStream log) {}
+    public ReadBackedPileup createPerAlleleDownsampledBasePileup(final ReadBackedPileup pileup, final double downsamplingFraction, final PrintStream log) { return pileup; }
+}

From 72714ee43eb06cb96a49d739da5955a52bf8be86 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Thu, 25 Oct 2012 02:47:42 -0400
Subject: [PATCH 20/40] Minor patches to get the contamination down-sampling
 working for indels.  Adding @Hidden logging output for easy debugging.

---
 .../AdvancedPerReadAlleleLikelihoodMap.java   |  3 +-
 .../StandardCallerArgumentCollection.java     |  2 +-
 .../walkers/genotyper/UnifiedGenotyper.java   | 39 ++++++-------------
 3 files changed, 15 insertions(+), 29 deletions(-)

diff --git a/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java b/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
index 80618cfdb..d6cddcf84 100644
--- a/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
+++ b/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
@@ -56,7 +56,8 @@ public class AdvancedPerReadAlleleLikelihoodMap extends StandardPerReadAlleleLik
 
         for ( Map.Entry<GATKSAMRecord, Map<Allele, Double>> entry : likelihoodReadMap.entrySet() ) {
             final Allele bestAllele = getMostLikelyAllele(entry.getValue());
-            alleleReadMap.get(bestAllele).add(entry.getKey());
+            if ( bestAllele != Allele.NO_CALL )
+                alleleReadMap.get(bestAllele).add(entry.getKey());
         }
 
         // compute the reads to remove and actually remove them
diff --git a/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
index 36456ca09..2f6d9d16f 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
@@ -79,7 +79,7 @@ public class StandardCallerArgumentCollection {
     public double CONTAMINATION_FRACTION = 0.0;
 
     @Hidden
-    @Argument(shortName = "logRemovedReadsFromContaminationFiltering", doc="contaminationLog", required=false)
+    @Argument(fullName = "logRemovedReadsFromContaminationFiltering", shortName="contaminationLog", required=false)
     public PrintStream contaminationLog = null;
 
     @Hidden
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
index 3116d3a7d..a17c2aa1a 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
@@ -234,36 +234,21 @@ public class UnifiedGenotyper extends LocusWalker<List<VariantCallContext>, Unif
             if (UAC.samplePloidy != VariantContextUtils.DEFAULT_PLOIDY ||
                     UAC.referenceSampleName != null ||
                     UAC.referenceSampleRod.isBound())  {
-                throw new UserException.NotSupportedInGATKLite("Usage of ploidy values different than 2 not supported in this GATK version");
+                throw new UserException.NotSupportedInGATKLite("you cannot enable usage of ploidy values other than 2");
             }
+
+            if ( UAC.CONTAMINATION_FRACTION > 0.0 ) {
+                throw new UserException.NotSupportedInGATKLite("you cannot enable usage of contamination down-sampling");
+            }
+        }
+
+        if ( UAC.TREAT_ALL_READS_AS_SINGLE_POOL ) {
+            samples.add(GenotypeLikelihoodsCalculationModel.DUMMY_SAMPLE_NAME);
+        } else {
             // get all of the unique sample names
             samples = SampleUtils.getSAMFileSamples(getToolkit().getSAMFileHeader());
-
-        } else {
-            // in full mode: check for consistency in ploidy/pool calling arguments
-            // check for correct calculation models
-/*            if (UAC.samplePloidy != VariantContextUtils.DEFAULT_PLOIDY) {
-                // polyploidy requires POOL GL and AF calculation models to be specified right now
-                if (UAC.GLmodel != GenotypeLikelihoodsCalculationModel.Model.POOLSNP && UAC.GLmodel != GenotypeLikelihoodsCalculationModel.Model.POOLINDEL
-                        && UAC.GLmodel != GenotypeLikelihoodsCalculationModel.Model.POOLBOTH)   {
-                    throw new UserException("Incorrect genotype calculation model chosen. Only [POOLSNP|POOLINDEL|POOLBOTH] supported with this walker if sample ploidy != 2");
-                }
-
-                if (UAC.AFmodel != AFCalc.Model.POOL)
-                    throw new UserException("Incorrect AF Calculation model. Only POOL model supported if sample ploidy != 2");
-
-            }
-  */
-            // get all of the unique sample names
-            if (UAC.TREAT_ALL_READS_AS_SINGLE_POOL) {
-                samples.clear();
-                samples.add(GenotypeLikelihoodsCalculationModel.DUMMY_SAMPLE_NAME);
-            } else {
-                samples = SampleUtils.getSAMFileSamples(getToolkit().getSAMFileHeader());
-                if (UAC.referenceSampleName != null )
-                    samples.remove(UAC.referenceSampleName);
-            }
-
+            if ( UAC.referenceSampleName != null )
+                samples.remove(UAC.referenceSampleName);
         }
 
         // check for a bad max alleles value

From e6652f7777b16f1bdd6266bd0c031d15a51f339f Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Thu, 25 Oct 2012 09:36:05 -0400
Subject: [PATCH 21/40] Added integration test for contamination down-sampling

---
 .../UnifiedGenotyperIntegrationTest.java      | 96 +++++++++++--------
 1 file changed, 56 insertions(+), 40 deletions(-)

diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index dee54b9f8..9e50a8556 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -26,7 +26,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing normal calling
     //
     // --------------------------------------------------------------------------------------------------------------
-    @Test
+    //@Test
     public void testMultiSamplePilot1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000", 1,
@@ -34,7 +34,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot1", spec);
     }
 
-    @Test
+    //@Test
     public void testWithAllelesPassedIn1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
@@ -42,7 +42,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 with alleles passed in", spec1);
     }
 
-    @Test
+    //@Test
     public void testWithAllelesPassedIn2() {
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
@@ -50,7 +50,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 with alleles passed in and emitting all sites", spec2);
     }
 
-    @Test
+    //@Test
     public void testSingleSamplePilot2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
@@ -58,7 +58,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test SingleSample Pilot2", spec);
     }
 
-    @Test
+    //@Test
     public void testMultipleSNPAlleles() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + privateTestDir + "multiallelic.snps.bam -o %s -L " + privateTestDir + "multiallelic.snps.intervals", 1,
@@ -66,7 +66,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test Multiple SNP alleles", spec);
     }
 
-    @Test
+    //@Test
     public void testBadRead() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm BOTH -I " + privateTestDir + "badRead.test.bam -o %s -L 1:22753424-22753464", 1,
@@ -74,7 +74,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test bad read", spec);
     }
 
-    @Test
+    //@Test
     public void testReverseTrim() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm INDEL -I " + validationDataLocation + "CEUTrio.HiSeq.b37.chr20.10_11mb.bam -o %s -L 20:10289124 -L 20:10090289", 1,
@@ -82,7 +82,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test reverse trim", spec);
     }
 
-    @Test
+    //@Test
     public void testMismatchedPLs() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm INDEL -I " + privateTestDir + "mismatchedPLs.bam -o %s -L 1:24020341", 1,
@@ -98,7 +98,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
     private final static String COMPRESSED_OUTPUT_MD5 = "fd236bd635d514e4214d364f45ec4d10";
 
-    @Test
+    //@Test
     public void testCompressedOutput() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
@@ -112,7 +112,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testParallelization() {
 
         // Note that we need to turn off any randomization for this to work, so no downsampling and no annotations
@@ -145,7 +145,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testMinBaseQualityScore() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --min_base_quality_score 26", 1,
@@ -153,7 +153,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test min_base_quality_score 26", spec);
     }
 
-    @Test
+    //@Test
     public void testSLOD() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH --dbsnp " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -161,7 +161,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test SLOD", spec);
     }
 
-    @Test
+    //@Test
     public void testNDA() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --annotateNDA -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -169,7 +169,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test NDA", spec);
     }
 
-    @Test
+    //@Test
     public void testCompTrack() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH -comp:FOO " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -177,7 +177,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test using comp track", spec);
     }
 
-    @Test
+    //@Test
     public void testNoCmdLineHeaderStdout() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandNoCmdLineHeaderStdout + " -glm INDEL -L 1:67,225,396-67,288,518", 0,
@@ -185,17 +185,17 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("testNoCmdLineHeaderStdout", spec);
     }
 
-    @Test
+    //@Test
     public void testOutputParameterSitesOnly() {
         testOutputParameters("-sites_only", "97ba874eafc9884a4de027a84c036311");
     }
 
-    @Test
+    //@Test
     public void testOutputParameterAllConfident() {
         testOutputParameters("--output_mode EMIT_ALL_CONFIDENT_SITES", "f9ea04d96eeef29e71d37e60518c2579");
     }
 
-    @Test
+    //@Test
     public void testOutputParameterAllSites() {
         testOutputParameters("--output_mode EMIT_ALL_SITES", "41c046d38ea328421df924e37e017645");
     }
@@ -207,7 +207,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest(String.format("testParameter[%s]", args), spec);
     }
 
-    @Test
+    //@Test
     public void testConfidence() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 -stand_call_conf 10 ", 1,
@@ -215,7 +215,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test confidence 1", spec1);
     }
 
-    @Test
+    //@Test
     public void testConfidence2() {
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 -stand_emit_conf 10 ", 1,
@@ -228,12 +228,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing heterozygosity
     //
     // --------------------------------------------------------------------------------------------------------------
-    @Test
+    //@Test
     public void testHeterozyosity1() {
         testHeterozosity( 0.01, "986923de51c71635d47e3d06fe3794a1" );
     }
 
-    @Test
+    //@Test
     public void testHeterozyosity2() {
         testHeterozosity( 1.0 / 1850, "fb12b1553f813004a394a391a8540873" );
     }
@@ -251,7 +251,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing calls with SLX, 454, and SOLID data
     //
     // --------------------------------------------------------------------------------------------------------------
-    @Test
+    //@Test
     public void testMultiTechnologies() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand +
@@ -269,7 +269,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing calls with BAQ
     //
     // --------------------------------------------------------------------------------------------------------------
-    @Test
+    //@Test
     public void testCallingWithBAQ() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand +
@@ -289,7 +289,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
     // Basic indel testing with SLX data
-    @Test
+    //@Test
     public void testSimpleIndels() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels +
@@ -303,7 +303,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     }
 
     // Basic indel testing with SLX data
-    @Test
+    //@Test
     public void testIndelsWithLowMinAlleleCnt() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels +
@@ -317,7 +317,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest(String.format("test indel caller in SLX with low min allele count"), spec);
     }
 
-    @Test
+    //@Test
     public void testMultiTechnologyIndels() {
          WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                  baseCommandIndels +
@@ -330,7 +330,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
          executeTest(String.format("test indel calling, multiple technologies"), spec);
      }
 
-    @Test
+    //@Test
     public void testWithIndelAllelesPassedIn1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "indelAllelesForUG.vcf -I " + validationDataLocation +
@@ -339,7 +339,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 indels with alleles passed in", spec);
     }
 
-    @Test
+    //@Test
     public void testWithIndelAllelesPassedIn2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles "
@@ -349,7 +349,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 indels with alleles passed in and emitting all sites", spec);
     }
 
-    @Test
+    //@Test
     public void testMultiSampleIndels1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10450700-10551000", 1,
@@ -363,7 +363,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot1 CEU indels using GENOTYPE_GIVEN_ALLELES", spec2);
     }
 
-    @Test
+    //@Test
     public void testGGAwithNoEvidenceInReads() {
         final String vcf = "small.indel.test.vcf";
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
@@ -373,7 +373,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test GENOTYPE_GIVEN_ALLELES with no evidence in reads", spec);
     }
 
-    @Test
+    //@Test
     public void testBaseIndelQualityScores() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndelsb37 +
@@ -392,7 +392,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testSnpEffAnnotationRequestedWithoutRodBinding() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000 " +
@@ -411,7 +411,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     final static String assessMinIndelFraction = baseCommandIndelsb37 + " -I " + validationDataLocation
             + "978604.bam -L 1:978,586-978,626 -o %s --sites_only -rf Sample -goodSM 7377 -goodSM 22-0022 -goodSM 134 -goodSM 344029-53 -goodSM 14030";
 
-    @Test
+    //@Test
     public void testMinIndelFraction0() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.0", 1,
@@ -419,7 +419,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test minIndelFraction 0.0", spec);
     }
 
-    @Test
+    //@Test
     public void testMinIndelFraction25() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.25", 1,
@@ -427,7 +427,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test minIndelFraction 0.25", spec);
     }
 
-    @Test
+    //@Test
     public void testMinIndelFraction100() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 1", 1,
@@ -441,7 +441,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testNsInCigar() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -I " + validationDataLocation + "testWithNs.bam -o %s -L 8:141799600-141814700", 1,
@@ -455,7 +455,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testReducedBam() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -I " + privateTestDir + "bamExample.ReducedRead.ADAnnotation.bam -o %s -L 1:67,225,396-67,288,518", 1,
@@ -463,12 +463,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test calling on a ReducedRead BAM", spec);
     }
 
-    @Test
+    //@Test
     public void testReducedBamSNPs() {
         testReducedCalling("SNP", "1d4a826b144723ff0766c36aa0239287");
     }
 
-    @Test
+    //@Test
     public void testReducedBamINDELs() {
         testReducedCalling("INDEL", "68ef51d5c98480e0c0192e0eecb95bca");
     }
@@ -480,4 +480,20 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                 Arrays.asList(md5));
         executeTest("test calling on a ReducedRead BAM with " + model, spec);
     }
+
+    // --------------------------------------------------------------------------------------------------------------
+    //
+    // testing contamination down-sampling
+    //
+    // --------------------------------------------------------------------------------------------------------------
+
+    @Test
+    public void testContaminationDownsampling() {
+        WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
+                baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --contamination_percentage_to_filter 0.20", 1,
+                Arrays.asList("27dd04159e06d9524fb8a4eef41f96ae"));
+        executeTest("test contamination_percentage_to_filter 0.20", spec);
+    }
+
+
 }

From c53c55da12ebf95141f3645ab3197d67e1ca0fb5 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Thu, 25 Oct 2012 09:37:08 -0400
Subject: [PATCH 22/40] Re-enable tests

---
 .../UnifiedGenotyperIntegrationTest.java      | 80 +++++++++----------
 1 file changed, 40 insertions(+), 40 deletions(-)

diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index 9e50a8556..8eb1c4657 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -26,7 +26,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing normal calling
     //
     // --------------------------------------------------------------------------------------------------------------
-    //@Test
+    @Test
     public void testMultiSamplePilot1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000", 1,
@@ -34,7 +34,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot1", spec);
     }
 
-    //@Test
+    @Test
     public void testWithAllelesPassedIn1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
@@ -42,7 +42,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 with alleles passed in", spec1);
     }
 
-    //@Test
+    @Test
     public void testWithAllelesPassedIn2() {
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
@@ -50,7 +50,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 with alleles passed in and emitting all sites", spec2);
     }
 
-    //@Test
+    @Test
     public void testSingleSamplePilot2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
@@ -58,7 +58,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test SingleSample Pilot2", spec);
     }
 
-    //@Test
+    @Test
     public void testMultipleSNPAlleles() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + privateTestDir + "multiallelic.snps.bam -o %s -L " + privateTestDir + "multiallelic.snps.intervals", 1,
@@ -66,7 +66,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test Multiple SNP alleles", spec);
     }
 
-    //@Test
+    @Test
     public void testBadRead() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm BOTH -I " + privateTestDir + "badRead.test.bam -o %s -L 1:22753424-22753464", 1,
@@ -74,7 +74,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test bad read", spec);
     }
 
-    //@Test
+    @Test
     public void testReverseTrim() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm INDEL -I " + validationDataLocation + "CEUTrio.HiSeq.b37.chr20.10_11mb.bam -o %s -L 20:10289124 -L 20:10090289", 1,
@@ -82,7 +82,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test reverse trim", spec);
     }
 
-    //@Test
+    @Test
     public void testMismatchedPLs() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm INDEL -I " + privateTestDir + "mismatchedPLs.bam -o %s -L 1:24020341", 1,
@@ -98,7 +98,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
     private final static String COMPRESSED_OUTPUT_MD5 = "fd236bd635d514e4214d364f45ec4d10";
 
-    //@Test
+    @Test
     public void testCompressedOutput() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
@@ -112,7 +112,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testParallelization() {
 
         // Note that we need to turn off any randomization for this to work, so no downsampling and no annotations
@@ -145,7 +145,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testMinBaseQualityScore() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --min_base_quality_score 26", 1,
@@ -153,7 +153,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test min_base_quality_score 26", spec);
     }
 
-    //@Test
+    @Test
     public void testSLOD() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH --dbsnp " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -161,7 +161,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test SLOD", spec);
     }
 
-    //@Test
+    @Test
     public void testNDA() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --annotateNDA -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -169,7 +169,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test NDA", spec);
     }
 
-    //@Test
+    @Test
     public void testCompTrack() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH -comp:FOO " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -177,7 +177,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test using comp track", spec);
     }
 
-    //@Test
+    @Test
     public void testNoCmdLineHeaderStdout() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandNoCmdLineHeaderStdout + " -glm INDEL -L 1:67,225,396-67,288,518", 0,
@@ -185,17 +185,17 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("testNoCmdLineHeaderStdout", spec);
     }
 
-    //@Test
+    @Test
     public void testOutputParameterSitesOnly() {
         testOutputParameters("-sites_only", "97ba874eafc9884a4de027a84c036311");
     }
 
-    //@Test
+    @Test
     public void testOutputParameterAllConfident() {
         testOutputParameters("--output_mode EMIT_ALL_CONFIDENT_SITES", "f9ea04d96eeef29e71d37e60518c2579");
     }
 
-    //@Test
+    @Test
     public void testOutputParameterAllSites() {
         testOutputParameters("--output_mode EMIT_ALL_SITES", "41c046d38ea328421df924e37e017645");
     }
@@ -207,7 +207,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest(String.format("testParameter[%s]", args), spec);
     }
 
-    //@Test
+    @Test
     public void testConfidence() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 -stand_call_conf 10 ", 1,
@@ -215,7 +215,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test confidence 1", spec1);
     }
 
-    //@Test
+    @Test
     public void testConfidence2() {
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 -stand_emit_conf 10 ", 1,
@@ -228,12 +228,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing heterozygosity
     //
     // --------------------------------------------------------------------------------------------------------------
-    //@Test
+    @Test
     public void testHeterozyosity1() {
         testHeterozosity( 0.01, "986923de51c71635d47e3d06fe3794a1" );
     }
 
-    //@Test
+    @Test
     public void testHeterozyosity2() {
         testHeterozosity( 1.0 / 1850, "fb12b1553f813004a394a391a8540873" );
     }
@@ -251,7 +251,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing calls with SLX, 454, and SOLID data
     //
     // --------------------------------------------------------------------------------------------------------------
-    //@Test
+    @Test
     public void testMultiTechnologies() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand +
@@ -269,7 +269,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing calls with BAQ
     //
     // --------------------------------------------------------------------------------------------------------------
-    //@Test
+    @Test
     public void testCallingWithBAQ() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand +
@@ -289,7 +289,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
     // Basic indel testing with SLX data
-    //@Test
+    @Test
     public void testSimpleIndels() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels +
@@ -303,7 +303,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     }
 
     // Basic indel testing with SLX data
-    //@Test
+    @Test
     public void testIndelsWithLowMinAlleleCnt() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels +
@@ -317,7 +317,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest(String.format("test indel caller in SLX with low min allele count"), spec);
     }
 
-    //@Test
+    @Test
     public void testMultiTechnologyIndels() {
          WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                  baseCommandIndels +
@@ -330,7 +330,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
          executeTest(String.format("test indel calling, multiple technologies"), spec);
      }
 
-    //@Test
+    @Test
     public void testWithIndelAllelesPassedIn1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "indelAllelesForUG.vcf -I " + validationDataLocation +
@@ -339,7 +339,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 indels with alleles passed in", spec);
     }
 
-    //@Test
+    @Test
     public void testWithIndelAllelesPassedIn2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles "
@@ -349,7 +349,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 indels with alleles passed in and emitting all sites", spec);
     }
 
-    //@Test
+    @Test
     public void testMultiSampleIndels1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10450700-10551000", 1,
@@ -363,7 +363,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot1 CEU indels using GENOTYPE_GIVEN_ALLELES", spec2);
     }
 
-    //@Test
+    @Test
     public void testGGAwithNoEvidenceInReads() {
         final String vcf = "small.indel.test.vcf";
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
@@ -373,7 +373,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test GENOTYPE_GIVEN_ALLELES with no evidence in reads", spec);
     }
 
-    //@Test
+    @Test
     public void testBaseIndelQualityScores() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndelsb37 +
@@ -392,7 +392,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testSnpEffAnnotationRequestedWithoutRodBinding() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000 " +
@@ -411,7 +411,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     final static String assessMinIndelFraction = baseCommandIndelsb37 + " -I " + validationDataLocation
             + "978604.bam -L 1:978,586-978,626 -o %s --sites_only -rf Sample -goodSM 7377 -goodSM 22-0022 -goodSM 134 -goodSM 344029-53 -goodSM 14030";
 
-    //@Test
+    @Test
     public void testMinIndelFraction0() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.0", 1,
@@ -419,7 +419,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test minIndelFraction 0.0", spec);
     }
 
-    //@Test
+    @Test
     public void testMinIndelFraction25() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.25", 1,
@@ -427,7 +427,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test minIndelFraction 0.25", spec);
     }
 
-    //@Test
+    @Test
     public void testMinIndelFraction100() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 1", 1,
@@ -441,7 +441,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testNsInCigar() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -I " + validationDataLocation + "testWithNs.bam -o %s -L 8:141799600-141814700", 1,
@@ -455,7 +455,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testReducedBam() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -I " + privateTestDir + "bamExample.ReducedRead.ADAnnotation.bam -o %s -L 1:67,225,396-67,288,518", 1,
@@ -463,12 +463,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test calling on a ReducedRead BAM", spec);
     }
 
-    //@Test
+    @Test
     public void testReducedBamSNPs() {
         testReducedCalling("SNP", "1d4a826b144723ff0766c36aa0239287");
     }
 
-    //@Test
+    @Test
     public void testReducedBamINDELs() {
         testReducedCalling("INDEL", "68ef51d5c98480e0c0192e0eecb95bca");
     }

From 6dc7d872ece8610b0f08a90bc12452984aa26c52 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Thu, 25 Oct 2012 10:06:13 -0400
Subject: [PATCH 23/40] Fix GenotypeAndValidate to handle SNPs and indels as
 reported on the forum.  Recent changes to the UnifiedArgumentCollection made
 this stop working.  Adding in JIRA to create integration tests for this tool.

---
 .../sting/gatk/walkers/validation/GenotypeAndValidate.java   | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/GenotypeAndValidate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/GenotypeAndValidate.java
index 832c0ede3..79b6e17f6 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/GenotypeAndValidate.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/GenotypeAndValidate.java
@@ -333,8 +333,9 @@ public class GenotypeAndValidate extends RodWalker<GenotypeAndValidate.CountedDa
         snpEngine = new UnifiedGenotyperEngine(getToolkit(), uac);
 
         // Adding the INDEL calling arguments for UG
-        uac.GLmodel = GenotypeLikelihoodsCalculationModel.Model.INDEL;
-        indelEngine = new UnifiedGenotyperEngine(getToolkit(), uac);
+        UnifiedArgumentCollection uac_indel = new UnifiedArgumentCollection(uac);
+        uac_indel.GLmodel = GenotypeLikelihoodsCalculationModel.Model.INDEL;
+        indelEngine = new UnifiedGenotyperEngine(getToolkit(), uac_indel);
 
         // make sure we have callConf set to the threshold set by the UAC so we can use it later.
         callConf = uac.STANDARD_CONFIDENCE_FOR_CALLING;

From e93ff3ea6ea864984e0a132239a7b34534b11629 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Thu, 25 Oct 2012 12:45:23 -0400
Subject: [PATCH 24/40] Let's go back to having the SB/SLOD NOT computed by
 default.  If you recall, it was only enabled by default because we thought we
 were going to use it when we made VQSR use random forests.  But since we
 decided not to change VQSR, there's no reason to triple the computation for
 every variant site anymore.

---
 .../genotyper/UnifiedArgumentCollection.java   |  6 +++---
 .../walkers/genotyper/UnifiedGenotyper.java    |  2 +-
 .../genotyper/UnifiedGenotyperEngine.java      |  2 +-
 .../walkers/variantutils/SelectVariants.java   |  1 -
 .../UnifiedGenotyperIntegrationTest.java       | 18 +++++++++---------
 5 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java
index e50f25fe8..5f6ddf0f1 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java
@@ -47,8 +47,8 @@ public class UnifiedArgumentCollection extends StandardCallerArgumentCollection
     /**
      * Note that calculating the SLOD increases the runtime by an appreciable amount.
      */
-    @Argument(fullName = "noSLOD", shortName = "nosl", doc = "If provided, we will not calculate the SLOD", required = false)
-    public boolean NO_SLOD = false;
+    @Argument(fullName = "computeSLOD", shortName = "slod", doc = "If provided, we will calculate the SLOD (SB annotation)", required = false)
+    public boolean COMPUTE_SLOD = false;
 
     /**
      * Depending on the value of the --max_alternate_alleles argument, we may genotype only a fraction of the alleles being sent on for genotyping.
@@ -204,7 +204,7 @@ public class UnifiedArgumentCollection extends StandardCallerArgumentCollection
         this.GLmodel = uac.GLmodel;
         this.AFmodel = uac.AFmodel;
         this.PCR_error = uac.PCR_error;
-        this.NO_SLOD = uac.NO_SLOD;
+        this.COMPUTE_SLOD = uac.COMPUTE_SLOD;
         this.ANNOTATE_NUMBER_OF_ALLELES_DISCOVERED = uac.ANNOTATE_NUMBER_OF_ALLELES_DISCOVERED;
         this.MIN_BASE_QUALTY_SCORE = uac.MIN_BASE_QUALTY_SCORE;
         this.MAX_DELETION_FRACTION = uac.MAX_DELETION_FRACTION;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
index a17c2aa1a..ef56eaa6d 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
@@ -290,7 +290,7 @@ public class UnifiedGenotyper extends LocusWalker<List<VariantCallContext>, Unif
             headerInfo.addAll(annotationEngine.getVCFAnnotationDescriptions());
 
         // annotation (INFO) fields from UnifiedGenotyper
-        if ( !UAC.NO_SLOD )
+        if ( UAC.COMPUTE_SLOD )
             VCFStandardHeaderLines.addStandardInfoLines(headerInfo, true, VCFConstants.STRAND_BIAS_KEY);
 
         if ( UAC.ANNOTATE_NUMBER_OF_ALLELES_DISCOVERED )
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java
index 3c4a97ec1..97254c478 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java
@@ -451,7 +451,7 @@ public class UnifiedGenotyperEngine {
             attributes.put(VCFConstants.MLE_ALLELE_FREQUENCY_KEY, MLEfrequencies);
         }
 
-        if ( !UAC.NO_SLOD && !limitedContext && !bestGuessIsRef ) {
+        if ( UAC.COMPUTE_SLOD && !limitedContext && !bestGuessIsRef ) {
             //final boolean DEBUG_SLOD = false;
 
             // the overall lod
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java
index c7b1d0fc7..d1b7cb96f 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java
@@ -459,7 +459,6 @@ public class SelectVariants extends RodWalker<Integer, Integer> implements TreeR
             UAC.GLmodel = GenotypeLikelihoodsCalculationModel.Model.BOTH;
             UAC.OutputMode = UnifiedGenotyperEngine.OUTPUT_MODE.EMIT_ALL_SITES;
             UAC.GenotypingMode = GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES;
-            UAC.NO_SLOD = true;
             UG_engine = new UnifiedGenotyperEngine(getToolkit(), UAC, logger, null, null, samples, VariantContextUtils.DEFAULT_PLOIDY);
             headerLines.addAll(UnifiedGenotyper.getHeaderInfo(UAC, null, null));
         }
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index 8eb1c4657..446f8cbe8 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -16,9 +16,9 @@ import java.util.List;
 
 public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
-    private final static String baseCommand = "-T UnifiedGenotyper -R " + b36KGReference + " -nosl --no_cmdline_in_header -glm BOTH -minIndelFrac 0.0 --dbsnp " + b36dbSNP129;
-    private final static String baseCommandIndels = "-T UnifiedGenotyper -R " + b36KGReference + " -nosl --no_cmdline_in_header -glm INDEL -mbq 20 -minIndelFrac 0.0 --dbsnp " + b36dbSNP129;
-    private final static String baseCommandIndelsb37 = "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm INDEL -mbq 20 --dbsnp " + b37dbSNP132;
+    private final static String baseCommand = "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH -minIndelFrac 0.0 --dbsnp " + b36dbSNP129;
+    private final static String baseCommandIndels = "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm INDEL -mbq 20 -minIndelFrac 0.0 --dbsnp " + b36dbSNP129;
+    private final static String baseCommandIndelsb37 = "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm INDEL -mbq 20 --dbsnp " + b37dbSNP132;
     private final static String baseCommandNoCmdLineHeaderStdout = "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -I " + privateTestDir + "bamExample.ReducedRead.ADAnnotation.bam";
 
     // --------------------------------------------------------------------------------------------------------------
@@ -61,7 +61,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testMultipleSNPAlleles() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-                "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + privateTestDir + "multiallelic.snps.bam -o %s -L " + privateTestDir + "multiallelic.snps.intervals", 1,
+                "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + privateTestDir + "multiallelic.snps.bam -o %s -L " + privateTestDir + "multiallelic.snps.intervals", 1,
                 Arrays.asList("543f68e42034bf44cfb24da8c9204320"));
         executeTest("test Multiple SNP alleles", spec);
     }
@@ -69,7 +69,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testBadRead() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-                "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm BOTH -I " + privateTestDir + "badRead.test.bam -o %s -L 1:22753424-22753464", 1,
+                "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm BOTH -I " + privateTestDir + "badRead.test.bam -o %s -L 1:22753424-22753464", 1,
                 Arrays.asList("d915535c1458733f09f82670092fcab6"));
         executeTest("test bad read", spec);
     }
@@ -77,7 +77,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testReverseTrim() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-                "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm INDEL -I " + validationDataLocation + "CEUTrio.HiSeq.b37.chr20.10_11mb.bam -o %s -L 20:10289124 -L 20:10090289", 1,
+                "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm INDEL -I " + validationDataLocation + "CEUTrio.HiSeq.b37.chr20.10_11mb.bam -o %s -L 20:10289124 -L 20:10090289", 1,
                 Arrays.asList("5ce03dd9ca2d9324c1d4a9d64389beb5"));
         executeTest("test reverse trim", spec);
     }
@@ -85,7 +85,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testMismatchedPLs() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-                "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm INDEL -I " + privateTestDir + "mismatchedPLs.bam -o %s -L 1:24020341", 1,
+                "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm INDEL -I " + privateTestDir + "mismatchedPLs.bam -o %s -L 1:24020341", 1,
                 Arrays.asList("3c006b06b17bbe8e787d64eff6a63a19"));
         executeTest("test mismatched PLs", spec);
     }
@@ -156,7 +156,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testSLOD() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-                "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH --dbsnp " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
+                "-T UnifiedGenotyper -R " + b36KGReference + " --computeSLOD --no_cmdline_in_header -glm BOTH --dbsnp " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
                 Arrays.asList("c7429e670ba477bf9a6bbee2fb41c5a9"));
         executeTest("test SLOD", spec);
     }
@@ -173,7 +173,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testCompTrack() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH -comp:FOO " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
-                Arrays.asList("8a9b424e00cdbe6b5e73d517335b2186"));
+                Arrays.asList("a6de72c5197f152be98a6f9029091a9e"));
         executeTest("test using comp track", spec);
     }
 

From cc8c12b9548cc24d13dbcc9f798e3f3dab4ded98 Mon Sep 17 00:00:00 2001
From: Mark DePristo <depristo@broadinstitute.org>
Date: Thu, 25 Oct 2012 14:46:09 -0400
Subject: [PATCH 25/40] Committing a broken version of BaseRecalibration

-- I'm committing because there's some kind of fundamental problem with the ReadCovariates cache, in that historical data isn't being cleared / computed properly, and I'd rather it fail for a while than leave it in JIRA.
-- The integration tests test the -nct with PrintReads to get 1, 2, 4 and the 4 fails.  But that's because of this incorrect calculation
-- Updating GATKPerformanceOverTime with the new @ClassType annotation
---
 .../walkers/bqsr/BQSRIntegrationTest.java     | 22 +++++++++++++------
 .../recalibration/BaseRecalibration.java      |  9 +++++---
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
index 90d67d393..a50219f48 100644
--- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
+++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
@@ -5,7 +5,9 @@ import org.broadinstitute.sting.utils.exceptions.UserException;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 
 /**
  * @author ebanks
@@ -118,20 +120,26 @@ public class BQSRIntegrationTest extends WalkerTest {
 
     @DataProvider(name = "PRTest")
     public Object[][] createPRTestData() {
-        return new Object[][]{
-                {new PRTest("", "ab2f209ab98ad3432e208cbd524a4c4a")},
-                {new PRTest(" -qq -1", "5226c06237b213b9e9b25a32ed92d09a")},
-                {new PRTest(" -qq 6", "b592a5c62b952a012e18adb898ea9c33")},
-                {new PRTest(" -DIQ", "8977bea0c57b808e65e9505eb648cdf7")}
-        };
+        List<Object[]> tests = new ArrayList<Object[]>();
+
+        tests.add(new Object[]{1, new PRTest(" -qq -1", "a1d87da5dcbde35170d6ba6bc3ee2812")});
+        tests.add(new Object[]{1, new PRTest(" -qq 6", "a0fecae6d0e5ab9917862fa306186d10")});
+        tests.add(new Object[]{1, new PRTest(" -DIQ", "8977bea0c57b808e65e9505eb648cdf7")});
+
+        for ( final int nct : Arrays.asList(1, 2, 4) ) {
+            tests.add(new Object[]{nct, new PRTest("", "d1bbb4ce6aa93e866f106f8b11d888ed")});
+        }
+
+        return tests.toArray(new Object[][]{});
     }
 
     @Test(dataProvider = "PRTest")
-    public void testPR(PRTest params) {
+    public void testPR(final int nct, PRTest params) {
         WalkerTestSpec spec = new WalkerTestSpec(
                 "-T PrintReads" +
                         " -R " + hg18Reference +
                         " -I " + privateTestDir + "HiSeq.1mb.1RG.bam" +
+                        " -nct " + nct +
                         " -BQSR " + privateTestDir + "HiSeq.20mb.1RG.table" +
                         params.args +
                         " -o %s",
diff --git a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java
index 7ad9302a8..2f9387acb 100644
--- a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java
+++ b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java
@@ -116,9 +116,12 @@ public class BaseRecalibration {
         // Compute all covariates for the read
         // TODO -- the need to clear here suggests there's an error in the indexing / assumption code
         // TODO -- for BI and DI. Perhaps due to the indel buffer size on the ends of the reads?
-        // TODO -- the output varies with -nt 1 and -nt 2 if you don't call clear here
-        // TODO -- needs to be fixed.
-        final ReadCovariates readCovariates = readCovariatesCache.get().clear();
+        // TODO -- the output varies depending on whether we clear or not
+        //final ReadCovariates readCovariates = readCovariatesCache.get().clear();
+
+        // the original code -- doesn't do any clearing
+        final ReadCovariates readCovariates = readCovariatesCache.get();
+
         RecalUtils.computeCovariates(read, requestedCovariates, readCovariates);
 
         for (final EventType errorModel : EventType.values()) { // recalibrate all three quality strings

From 884d031e72881f658ecd7df9803bdd930c64163b Mon Sep 17 00:00:00 2001
From: David Roazen <droazen@broadinstitute.org>
Date: Thu, 25 Oct 2012 13:30:49 -0400
Subject: [PATCH 26/40] NestedIntegerArray: Pre-allocate only the first two
 dimensions

It turns out that pre-allocating the entire tree was too expensive in
terms of memory when using large values for the -mcs and -ics parameters.

Pre-allocating the first two dimensions prevents us from ever locking the
root node during a put(). Contention between threads over lower levels
of the tree should be minimal given that puts are rare compared to gets.

Also output dimensions and pre-allocation info at startup. If pre-allocation
takes longer than usual this gives the user a sense of what is causing the
delay.
---
 .../utils/collections/NestedIntegerArray.java | 51 ++++++++++++++-----
 1 file changed, 37 insertions(+), 14 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java b/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java
index c83de8ec7..050ed52ac 100755
--- a/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java
+++ b/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java
@@ -25,9 +25,11 @@
 
 package org.broadinstitute.sting.utils.collections;
 
+import org.apache.log4j.Logger;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 /**
@@ -38,39 +40,50 @@ import java.util.List;
 
 public class NestedIntegerArray<T> {
 
+    private static Logger logger = Logger.getLogger(NestedIntegerArray.class);
+
     protected final Object[] data;
 
     protected final int numDimensions;
     protected final int[] dimensions;
 
+    // Preallocate the first two dimensions to limit contention during tree traversals in put()
+    private static final int NUM_DIMENSIONS_TO_PREALLOCATE = 2;
+
     public NestedIntegerArray(final int... dimensions) {
         numDimensions = dimensions.length;
         if ( numDimensions == 0 )
             throw new ReviewedStingException("There must be at least one dimension to an NestedIntegerArray");
         this.dimensions = dimensions.clone();
 
+        int dimensionsToPreallocate = Math.min(dimensions.length, NUM_DIMENSIONS_TO_PREALLOCATE);
+
+        logger.info(String.format("Creating NestedIntegerArray with dimensions %s", Arrays.toString(dimensions)));
+        logger.info(String.format("Pre-allocating first %d dimensions", dimensionsToPreallocate));
+
         data = new Object[dimensions[0]];
-        prepopulateArray(data, 0);
+        preallocateArray(data, 0, dimensionsToPreallocate);
+
+        logger.info(String.format("Done pre-allocating first %d dimensions", dimensionsToPreallocate));
     }
 
     /**
-     * Recursively allocate the entire tree of arrays in all its dimensions.
+     * Recursively allocate the first dimensionsToPreallocate dimensions of the tree
      *
-     * Doing this upfront uses more memory initially, but saves time over the course of the run
-     * and (crucially) avoids having to make threads wait while traversing the tree to check
-     * whether branches exist or not.
+     * Pre-allocating the first few dimensions helps limit contention during tree traversals in put()
      *
      * @param subarray current node in the tree
      * @param dimension current level in the tree
+     * @param dimensionsToPreallocate preallocate only this many dimensions (starting from the first)
      */
-    private void prepopulateArray( Object[] subarray, int dimension ) {
-        if ( dimension >= numDimensions - 1 ) {
+    private void preallocateArray( Object[] subarray, int dimension, int dimensionsToPreallocate ) {
+        if ( dimension >= dimensionsToPreallocate - 1 ) {
             return;
         }
 
         for ( int i = 0; i < subarray.length; i++ ) {
             subarray[i] = new Object[dimensions[dimension + 1]];
-            prepopulateArray((Object[])subarray[i], dimension + 1);
+            preallocateArray((Object[])subarray[i], dimension + 1, dimensionsToPreallocate);
         }
     }
 
@@ -82,8 +95,9 @@ public class NestedIntegerArray<T> {
             if ( keys[i] >= dimensions[i] )
                 return null;
 
-            myData = (Object[])myData[keys[i]];  // interior nodes in the tree will never be null, so we can safely traverse
-                                                 // down to the leaves
+            myData = (Object[])myData[keys[i]];
+            if ( myData == null )
+                return null;
         }
 
         return (T)myData[keys[numNestedDimensions]];
@@ -92,8 +106,8 @@ public class NestedIntegerArray<T> {
     /**
      * Insert a value at the position specified by the given keys.
      *
-     * This method is THREAD-SAFE despite not being synchronized, however the caller MUST
-     * check the return value to see if the put succeeded. This method RETURNS FALSE if
+     * This method is thread-safe, however the caller MUST check the
+     * return value to see if the put succeeded. This method RETURNS FALSE if
      * the value could not be inserted because there already was a value present
      * at the specified location. In this case the caller should do a get() to get
      * the already-existing value and (potentially) update it.
@@ -113,8 +127,17 @@ public class NestedIntegerArray<T> {
             if ( keys[i] >= dimensions[i] )
                 throw new ReviewedStingException("Key " + keys[i] + " is too large for dimension " + i + " (max is " + (dimensions[i]-1) + ")");
 
-            myData = (Object[])myData[keys[i]];  // interior nodes in the tree will never be null, so we can safely traverse
-                                                 // down to the leaves
+            // If we're at or beyond the last dimension that was pre-allocated, we need to do a synchronized
+            // check to see if the next branch exists, and if it doesn't, create it
+            if ( i >= NUM_DIMENSIONS_TO_PREALLOCATE - 1 ) {
+                synchronized ( myData ) {
+                    if ( myData[keys[i]] == null ) {
+                        myData[keys[i]] = new Object[dimensions[i + 1]];
+                    }
+                }
+            }
+
+            myData = (Object[])myData[keys[i]];
         }
 
         synchronized ( myData ) {   // lock the bottom row while we examine and (potentially) update it

From dde3060bb84ecbb83878daf834b5d922eb4ef8d4 Mon Sep 17 00:00:00 2001
From: Ami Levy Moonshine <ami@broadinstitute.org>
Date: Thu, 25 Oct 2012 15:36:17 -0400
Subject: [PATCH 27/40] add the CEUtrio best practices results (UG + PBT) to
 the bundle

---
 .../gatk/walkers/qc/AssessReducedQuals.java   | 123 +++++++++++++-----
 .../queue/qscripts/GATKResourcesBundle.scala  |   9 +-
 2 files changed, 96 insertions(+), 36 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java
index c8ec33364..e74684c53 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java
@@ -1,5 +1,6 @@
 package org.broadinstitute.sting.gatk.walkers.qc;
 
+import org.broadinstitute.sting.commandline.Argument;
 import org.broadinstitute.sting.commandline.Output;
 import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
@@ -7,73 +8,125 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.LocusWalker;
 import org.broadinstitute.sting.gatk.walkers.TreeReducible;
 import org.broadinstitute.sting.utils.GenomeLoc;
+import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 
 import java.io.PrintStream;
+import java.util.List;
 
 /**
- * Created with IntelliJ IDEA.
- * User: ami
- * Date: 10/19/12
- * Time: 9:09 AM
- * To change this template use File | Settings | File Templates.
+ * Emits intervals in which the differences between the original and reduced bam quals are bigger epsilon (unless the quals of
+ * the reduced bam are above sufficient threshold)
+ *
+ * <h2>Input</h2>
+ * <p>
+ * The original and reduced BAM files.
+ * </p>
+ *
+ * <h2>Output</h2>
+ * <p>
+ * A list of intervals in which the differences between the original and reduced bam quals are bigger epsilon.
+ * </p>
+ *
+ * <h2>Examples</h2>
+ * <pre>
+ * java -Xmx2g -jar GenomeAnalysisTK.jar \
+ *   -I:original original.bam \
+ *   -I:reduced reduced.bam \
+ *   -R ref.fasta \
+ *   -T AssessReducedQuals \
+ *   -o output.intervals
+ * </pre>
+ *
+ * @author ami
  */
+
 public class AssessReducedQuals extends LocusWalker<GenomeLoc, GenomeLoc> implements TreeReducible<GenomeLoc> {
 
-    private static final String original = "original";
     private static final String reduced = "reduced";
+    private static final String original = "original";
+    private static final int originalQualsIndex = 0;
+    private static final int reducedQualsIndex = 1;
+
+    @Argument(fullName = "sufficientQualSum", shortName = "sufficientQualSum", doc = "When a reduced bam qual sum is above this threshold, it passes even without comparing to the non-reduced bam ", required = false)
+    public int sufficientQualSum = 600;
+
+    @Argument(fullName = "qual_epsilon", shortName = "epsilon", doc = "when |Quals_reduced_bam - Quals_original_bam| > epsilon*Quals_original_bam we output this interval", required = false)
+    public int qual_epsilon = 0;
+
+    @Argument(fullName = "debugLevel", shortName = "debug", doc = "debug mode on")
+    public int debugLevel = 0;
 
     @Output
     protected PrintStream out;
 
+    public void initialize() {
+        if (debugLevel != 0)
+            out.println("  Debug mode" +
+                        "Debug:\tsufficientQualSum: "+sufficientQualSum+ "\n " +
+                        "Debug:\tqual_epsilon: "+qual_epsilon);
+    }
+
     @Override
     public boolean includeReadsWithDeletionAtLoci() { return true; }
 
-    public void initialize() {}    //todo: why we need that?
-
     @Override
     public GenomeLoc map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
         if ( tracker == null )
             return null;
 
-        int originalQualsIndex = 0;
-        int reducedQualsIndex = 1;
-        double epsilon = 0;
-        double[] quals = getPileupQuals(context.getBasePileup());
-        return (quals[originalQualsIndex] - quals[reducedQualsIndex] >= epsilon || quals[originalQualsIndex] - quals[reducedQualsIndex] <= -1*epsilon) ? ref.getLocus() : null;
+        boolean reportLocus;
+        final int[] quals = getPileupQuals(context.getBasePileup());
+        if (debugLevel != 0)
+            out.println("Debug:\tLocus Quals\t"+ref.getLocus()+"\toriginal\t"+quals[originalQualsIndex]+"\treduced\t"+quals[reducedQualsIndex]);
+        final int epsilon = MathUtils.fastRound(quals[originalQualsIndex]*qual_epsilon);
+        final int calcOriginalQuals = Math.min(quals[originalQualsIndex],sufficientQualSum);
+        final int calcReducedQuals = Math.min(quals[reducedQualsIndex],sufficientQualSum);
+        final int OriginalReducedQualDiff = calcOriginalQuals - calcReducedQuals;
+        reportLocus = OriginalReducedQualDiff > epsilon || OriginalReducedQualDiff < -1*epsilon;
+        if(debugLevel != 0 && reportLocus)
+            out.println("Debug:\tEmited Locus\t"+ref.getLocus()+"\toriginal\t"+quals[originalQualsIndex]+"\treduced\t"+quals[reducedQualsIndex]+"\tepsilon\t"+epsilon+"\tdiff\t"+OriginalReducedQualDiff);
 
+        return reportLocus ? ref.getLocus() : null;
     }
 
-    private double[] getPileupQuals(final ReadBackedPileup readPileup) {
+    private final int[] getPileupQuals(final ReadBackedPileup readPileup) {
 
-        int originalQualsIndex = 0;
-        int reducedQualsIndex = 1;
-        double[] quals = new double[2];
+        final int[] quals = new int[2];
+        String[] printPileup = {"Debug 2:\toriginal pileup:\t"+readPileup.getLocation()+"\nDebug 2:----------------------------------\n",
+                                "Debug 2:\t reduced pileup:\t"+readPileup.getLocation()+"\nDebug 2:----------------------------------\n"};
 
-        for( PileupElement p : readPileup){
-            if ( (int)p.getQual() > 2 && p.getMappingQual() > 0 && !p.isDeletion() ){
-                if (p.getRead().isReducedRead()){
-                    double tempQual = (double)(p.getQual()) * p.getRepresentativeCount();
-                    quals[reducedQualsIndex] += tempQual;
-                }
-                else
-                {
-                    double tempQual = (double)(p.getQual());
-                    quals[originalQualsIndex] += tempQual;
-                }
+        for( PileupElement p : readPileup ){
+            final List<String> tags = getToolkit().getReaderIDForRead(p.getRead()).getTags().getPositionalTags();
+            if ( isGoodRead(p,tags) ){
+                final int tempQual = (int)(p.getQual()) * p.getRepresentativeCount();
+                final int tagIndex = getTagIndex(tags);
+                quals[tagIndex] += tempQual;
+                if(debugLevel == 2)
+                    printPileup[tagIndex] += "\tDebug 2: ("+p+")\tMQ="+p.getMappingQual()+":QU="+p.getQual()+":RC="+p.getRepresentativeCount()+":OS="+p.getOffset()+"\n";
             }
-
         }
-
-
+        if(debugLevel == 2){
+            out.println(printPileup[originalQualsIndex]);
+            out.println(printPileup[reducedQualsIndex]);
+        }
         return quals;
     }
 
-    //public void onTraversalDone(GenomeLoc sum) {
-    //    if ( sum != null )
-    //        out.println(sum);
-    //}
+    private final boolean isGoodRead(PileupElement p, List<String> tags){
+        return !p.isDeletion() && (tags.contains(reduced) || (tags.contains(original) && (int)p.getQual() >= 20 && p.getMappingQual() >= 20));
+    }
+
+    private final int getTagIndex(List<String> tags){
+        return tags.contains(reduced) ? 1 : 0;
+    }
+
+    @Override
+    public void onTraversalDone(GenomeLoc sum) {
+        if ( sum != null )
+            out.println(sum);
+    }
 
     @Override
     public GenomeLoc treeReduce(GenomeLoc lhs, GenomeLoc rhs) {
diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala
index 5e66520ca..9e8815762 100755
--- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala
+++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala
@@ -136,7 +136,14 @@ class GATKResourcesBundle extends QScript {
 
     addResource(new Resource("/humgen/gsa-hpprojects/GATK/data/Comparisons/Unvalidated/GoldStandardIndel/gold.standard.indel.MillsAnd1000G.b37.vcf",
       "Mills_and_1000G_gold_standard.indels", b37, true, false))
-    
+
+    //
+    // CEU trio (NA12878,NA12891,NA12892) best practices results (including PBT)
+    //
+
+    addResource(new Resource("/humgen/gsa-hpprojects/NA12878Collection/callsets/CEUtrio_BestPractices/current/CEUTrio.HiSeq.WGS.b37.UG.snps_and_indels.recalibrated.filtered.phaseByTransmission.vcf",
+      "CEUTrio.HiSeq.WGS.b37.UG.bestPractices.phaseByTransmission",b37,true,false))
+
     //
     // example call set for wiki tutorial
     //

From 422e16c62e7b99c1928de1655dfa359dc9945428 Mon Sep 17 00:00:00 2001
From: David Roazen <droazen@broadinstitute.org>
Date: Thu, 25 Oct 2012 16:16:00 -0400
Subject: [PATCH 28/40] BaseRecalibration: don't cache instances of
 ReadCovariates across reads

Caching and reusing ReadCovariates instances across reads sounds good in theory, but:

-it doesn't work unless you zero out the internal arrays before each read
-the internal arrays must be sized proportionally to the maximum POSSIBLE
recalibrated read length (5000!!!), instead of the ACTUAL read lengths

By contrast, creating a new instance per read is basically equivalent to doing an
efficient low-level memset-style clear on a much smaller array (since we use the actual
rather than the maximum read length to create it). So this should be faster than caching
instances and calling clear() but slower than caching instances and not calling clear().

Credit to Ryan to proposing this approach.
---
 .../recalibration/BaseRecalibration.java      | 22 +------------------
 1 file changed, 1 insertion(+), 21 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java
index 2f9387acb..5d4020a07 100644
--- a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java
+++ b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java
@@ -61,17 +61,6 @@ public class BaseRecalibration {
 //            qualityScoreByFullCovariateKey[i] = new NestedHashMap();
 //    }
 
-    /**
-     * Thread local cache to allow multi-threaded use of this class
-     */
-    private ThreadLocal<ReadCovariates> readCovariatesCache;
-    {
-        readCovariatesCache = new ThreadLocal<ReadCovariates> () {
-            @Override protected ReadCovariates initialValue() {
-                return new ReadCovariates(MAXIMUM_RECALIBRATED_READ_LENGTH, requestedCovariates.length);
-            }
-        };
-    }
 
     /**
      * Constructor using a GATK Report file
@@ -113,16 +102,7 @@ public class BaseRecalibration {
             }
         }
 
-        // Compute all covariates for the read
-        // TODO -- the need to clear here suggests there's an error in the indexing / assumption code
-        // TODO -- for BI and DI. Perhaps due to the indel buffer size on the ends of the reads?
-        // TODO -- the output varies depending on whether we clear or not
-        //final ReadCovariates readCovariates = readCovariatesCache.get().clear();
-
-        // the original code -- doesn't do any clearing
-        final ReadCovariates readCovariates = readCovariatesCache.get();
-
-        RecalUtils.computeCovariates(read, requestedCovariates, readCovariates);
+        final ReadCovariates readCovariates = RecalUtils.computeCovariates(read, requestedCovariates);
 
         for (final EventType errorModel : EventType.values()) { // recalibrate all three quality strings
             if (disableIndelQuals && errorModel != EventType.BASE_SUBSTITUTION) {

From 6b8b7df6517647d05bedf3acc9411c599e5a95cd Mon Sep 17 00:00:00 2001
From: Mark DePristo <depristo@broadinstitute.org>
Date: Thu, 25 Oct 2012 17:26:29 -0400
Subject: [PATCH 29/40] Queue now understands -nct and requests the appropriate
 number of cores from LSF, SGE, etc

-- NCT wasn't previously recognized by Queue as needing more processors per machine.  This commit fixes this.  Also a potential cause of poor GATKPerformanceOverTime, in that runs with -nct could flood a node and cause it to have hundreds of cores in contention.
---
 .../sting/queue/extensions/gatk/ArgumentDefinitionField.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java
index 368cb3d11..2226c6458 100644
--- a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java
+++ b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java
@@ -346,7 +346,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
 
         @Override
         protected String getFreezeFields() {
-            return String.format("if (num_threads.isDefined) nCoresRequest = num_threads%n");
+            return String.format("if (num_threads.isDefined) nCoresRequest = num_threads%nif (num_cpu_threads_per_data_thread.isDefined) nCoresRequest = Some(nCoresRequest.getOrElse(1) * num_cpu_threads_per_data_thread.getOrElse(1))%n");
         }
     }
 

From 91f2c847a33dd5966deacaae7ff9d5481f20f9db Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Fri, 26 Oct 2012 00:57:40 -0400
Subject: [PATCH 31/40] Fixing problem reported on forum for VF: DP couldn't be
 filtered from the FORMAT field, only from the INFO field.  Fixed and added
 integration test.

---
 .../variantcontext/VariantJEXLContext.java     |  2 ++
 .../VariantFiltrationIntegrationTest.java      | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantJEXLContext.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantJEXLContext.java
index 913615a84..abe85e383 100644
--- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantJEXLContext.java
+++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantJEXLContext.java
@@ -193,6 +193,8 @@ class JEXLMap implements Map<VariantContextUtils.JexlVCMatchExp, Boolean> {
                 infoMap.put("isHet", g.isHet() ? "1" : "0");
                 infoMap.put("isHomVar", g.isHomVar() ? "1" : "0");
                 infoMap.put(VCFConstants.GENOTYPE_QUALITY_KEY, g.getGQ());
+                if ( g.hasDP() )
+                    infoMap.put(VCFConstants.DEPTH_KEY, g.getDP());
                 for ( Map.Entry<String, Object> e : g.getExtendedAttributes().entrySet() ) {
                     if ( e.getValue() != null && !e.getValue().equals(VCFConstants.MISSING_VALUE_v4) )
                         infoMap.put(e.getKey(), e.getValue());
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/filters/VariantFiltrationIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/filters/VariantFiltrationIntegrationTest.java
index 97b985a29..27e5f3d46 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/filters/VariantFiltrationIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/filters/VariantFiltrationIntegrationTest.java
@@ -108,4 +108,22 @@ public class VariantFiltrationIntegrationTest extends WalkerTest {
                 Arrays.asList("8ed32a2272bab8043a255362335395ef"));
         executeTest("testUnfilteredBecomesFilteredAndPass", spec);
     }
+
+    @Test
+    public void testFilteringDPfromINFO() {
+        WalkerTestSpec spec = new WalkerTestSpec(
+                "-T VariantFiltration -o %s --no_cmdline_in_header -R " + b37KGReference
+                        + " --filterExpression 'DP < 8' --filterName lowDP -V " + privateTestDir + "filteringDepthInFormat.vcf", 1,
+                Arrays.asList("a01f7cce53ea556c9741aa60b6124c41"));
+        executeTest("testFilteringDPfromINFO", spec);
+    }
+
+    @Test
+    public void testFilteringDPfromFORMAT() {
+        WalkerTestSpec spec = new WalkerTestSpec(
+                "-T VariantFiltration -o %s --no_cmdline_in_header -R " + b37KGReference
+                        + " --genotypeFilterExpression 'DP < 8' --genotypeFilterName lowDP -V " + privateTestDir + "filteringDepthInFormat.vcf", 1,
+                Arrays.asList("e10485c7c33d9211d0c1294fd7858476"));
+        executeTest("testFilteringDPfromFORMAT", spec);
+    }
 }

From bf3d61ce825caa65c17a61429d7fb8939b36af9d Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Fri, 26 Oct 2012 01:04:51 -0400
Subject: [PATCH 32/40] The default value for
 --contamination_fraction_to_filter is now 0.05 (5%) in both UG and HC.  Users
 of GATK-lite get pushed down to 0% by default (since it's not enabled) or get
 a user error if they try to set it.

---
 ...GenotyperGeneralPloidyIntegrationTest.java | 14 +++---
 .../HaplotypeCallerIntegrationTest.java       | 18 ++++----
 .../StandardCallerArgumentCollection.java     |  5 +-
 .../walkers/genotyper/UnifiedGenotyper.java   |  8 +++-
 .../UnifiedGenotyperIntegrationTest.java      | 46 ++++++++-----------
 5 files changed, 45 insertions(+), 46 deletions(-)

diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperGeneralPloidyIntegrationTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperGeneralPloidyIntegrationTest.java
index ae3befe66..73bc8fba6 100644
--- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperGeneralPloidyIntegrationTest.java
+++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperGeneralPloidyIntegrationTest.java
@@ -55,36 +55,36 @@ public class UnifiedGenotyperGeneralPloidyIntegrationTest extends WalkerTest {
 
     @Test(enabled = true)
     public void testSNP_ACS_Pools() {
-        PC_LSV_Test_short(" -maxAltAlleles 1 -ploidy 6 -out_mode EMIT_ALL_CONFIDENT_SITES","LSV_SNP_ACS","SNP","ec19f0b7c7d57493cecfff988a4815c8");
+        PC_LSV_Test_short(" -maxAltAlleles 1 -ploidy 6 -out_mode EMIT_ALL_CONFIDENT_SITES","LSV_SNP_ACS","SNP","df0e67c975ef74d593f1c704daab1705");
     }
 
     @Test(enabled = true)
     public void testBOTH_GGA_Pools() {
-        PC_LSV_Test(String.format(" -maxAltAlleles 2 -ploidy 24 -gt_mode GENOTYPE_GIVEN_ALLELES -out_mode EMIT_ALL_SITES -alleles %s",LSV_ALLELES),"LSV_BOTH_GGA","BOTH","9ce24f4ff787aed9d3754519a60ef49f");
+        PC_LSV_Test(String.format(" -maxAltAlleles 2 -ploidy 24 -gt_mode GENOTYPE_GIVEN_ALLELES -out_mode EMIT_ALL_SITES -alleles %s",LSV_ALLELES),"LSV_BOTH_GGA","BOTH","7e5b28c9e21cc7e45c58c41177d8a0fc");
     }
 
     @Test(enabled = true)
     public void testINDEL_GGA_Pools() {
-        PC_LSV_Test(String.format(" -maxAltAlleles 1 -ploidy 24 -gt_mode GENOTYPE_GIVEN_ALLELES  -out_mode EMIT_ALL_SITES -alleles %s",LSV_ALLELES),"LSV_INDEL_GGA","INDEL","492c8ba9a80a902097ff15bbeb031592");
+        PC_LSV_Test(String.format(" -maxAltAlleles 1 -ploidy 24 -gt_mode GENOTYPE_GIVEN_ALLELES  -out_mode EMIT_ALL_SITES -alleles %s",LSV_ALLELES),"LSV_INDEL_GGA","INDEL","ae6c276cc46785a794acff6f7d10ecf7");
     }
 
     @Test(enabled = true)
     public void testINDEL_maxAltAlleles2_ploidy3_Pools_noRef() {
-        PC_LSV_Test_NoRef(" -maxAltAlleles 2 -ploidy 3","LSV_INDEL_DISC_NOREF_p3","INDEL","848e1092b5cd57b0da5f1187e67134e7");
+        PC_LSV_Test_NoRef(" -maxAltAlleles 2 -ploidy 3","LSV_INDEL_DISC_NOREF_p3","INDEL","481452ad7d6378cffb5cd834cc621d55");
     }
 
     @Test(enabled = true)
     public void testINDEL_maxAltAlleles2_ploidy1_Pools_noRef() {
-        PC_LSV_Test_NoRef(" -maxAltAlleles 2 -ploidy 1","LSV_INDEL_DISC_NOREF_p1","INDEL","51a7b51d82a341adec0e6510f5dfadd8");
+        PC_LSV_Test_NoRef(" -maxAltAlleles 2 -ploidy 1","LSV_INDEL_DISC_NOREF_p1","INDEL","812957e51277aca9925c1a7bb4d9a118");
     }
 
     @Test(enabled = true)
     public void testMT_SNP_DISCOVERY_sp4() {
-         PC_MT_Test(CEUTRIO_BAM, " -maxAltAlleles 1 -ploidy 8", "MT_SNP_DISCOVERY_sp4","0a8c3b06243040b743dd90d497bb3f83");
+         PC_MT_Test(CEUTRIO_BAM, " -maxAltAlleles 1 -ploidy 8", "MT_SNP_DISCOVERY_sp4","dd568dc30be90135a3a8957a45a7321c");
     }
 
     @Test(enabled = true)
     public void testMT_SNP_GGA_sp10() {
-        PC_MT_Test(CEUTRIO_BAM, String.format(" -maxAltAlleles 1 -ploidy 20 -gt_mode GENOTYPE_GIVEN_ALLELES  -out_mode EMIT_ALL_SITES -alleles %s",NA12891_CALLS), "MT_SNP_GGA_sp10", "f8ea18ec6a717a77fdf8c5f2482d8d8d");
+        PC_MT_Test(CEUTRIO_BAM, String.format(" -maxAltAlleles 1 -ploidy 20 -gt_mode GENOTYPE_GIVEN_ALLELES  -out_mode EMIT_ALL_SITES -alleles %s",NA12891_CALLS), "MT_SNP_GGA_sp10", "bf793c43b635a931207170be8035b288");
     }
 }
diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
index 71f03ea00..86f3748ce 100644
--- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
+++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
@@ -21,17 +21,17 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
 
     @Test
     public void testHaplotypeCallerMultiSample() {
-        HCTest(CEUTRIO_BAM, "", "1d826f852ec1457efbfa7ee828c5783a");
+        HCTest(CEUTRIO_BAM, "", "aa1df35d6e64d7ca93feb4d2dd15dd0e");
     }
 
     @Test
     public void testHaplotypeCallerSingleSample() {
-        HCTest(NA12878_BAM, "", "01367428c26d3eaf9297c58bf8677dd3");
+        HCTest(NA12878_BAM, "", "186c7f322978283c01249c6de2829215");
     }
 
     @Test
     public void testHaplotypeCallerMultiSampleGGA() {
-        HCTest(CEUTRIO_BAM, "--max_alternate_alleles 3 -gt_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "combined.phase1.chr20.raw.indels.sites.vcf", "53caa950535749f99d3c5b9bb61c7b60");
+        HCTest(CEUTRIO_BAM, "--max_alternate_alleles 3 -gt_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "combined.phase1.chr20.raw.indels.sites.vcf", "de9e78a52207fe62144dba5337965469");
     }
 
     private void HCTestComplexVariants(String bam, String args, String md5) {
@@ -42,7 +42,7 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
 
     @Test
     public void testHaplotypeCallerMultiSampleComplex() {
-        HCTestComplexVariants(privateTestDir + "AFR.complex.variants.bam", "", "966d338f423c86a390d685aa6336ec69");
+        HCTestComplexVariants(privateTestDir + "AFR.complex.variants.bam", "", "000dbb1b48f94d017cfec127c6cabe8f");
     }
 
     private void HCTestSymbolicVariants(String bam, String args, String md5) {
@@ -53,7 +53,7 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
 
     @Test
     public void testHaplotypeCallerSingleSampleSymbolic() {
-        HCTestSymbolicVariants(NA12878_CHR20_BAM, "", "b4ea70a446e4782bd3700ca14dd726ff");
+        HCTestSymbolicVariants(NA12878_CHR20_BAM, "", "16013a9203367c3d1c4ce1dcdc81ef4a");
     }
 
     private void HCTestIndelQualityScores(String bam, String args, String md5) {
@@ -64,20 +64,20 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
 
     @Test
     public void testHaplotypeCallerSingleSampleIndelQualityScores() {
-        HCTestIndelQualityScores(NA12878_RECALIBRATED_BAM, "", "2581e760279291a3901a506d060bfac8");
+        HCTestIndelQualityScores(NA12878_RECALIBRATED_BAM, "", "b369c2a6cb5c99a424551b33bae16f3b");
     }
 
     @Test
     public void HCTestProblematicReadsModifiedInActiveRegions() {
         final String base = String.format("-T HaplotypeCaller -R %s -I %s", REF, privateTestDir + "haplotype-problem-4.bam") + " --no_cmdline_in_header -o %s -minPruning 3";
-        final WalkerTestSpec spec = new WalkerTestSpec(base, Arrays.asList("c54c0c9411054bf629bfd98b616e53fc"));
+        final WalkerTestSpec spec = new WalkerTestSpec(base, Arrays.asList("c306140ad28515ee06c603c225217939"));
         executeTest("HCTestProblematicReadsModifiedInActiveRegions: ", spec);
     }
 
     @Test
     public void HCTestStructuralIndels() {
         final String base = String.format("-T HaplotypeCaller -R %s -I %s", REF, privateTestDir + "AFR.structural.indels.bam") + " --no_cmdline_in_header -o %s -minPruning 6 -L 20:8187565-8187800 -L 20:18670537-18670730";
-        final WalkerTestSpec spec = new WalkerTestSpec(base, Arrays.asList("c642dcd93771f6f084d55de31f180d1b"));
+        final WalkerTestSpec spec = new WalkerTestSpec(base, Arrays.asList("b6c67ee8e99cc8f53a6587bb26028047"));
         executeTest("HCTestStructuralIndels: ", spec);
     }
 
@@ -91,7 +91,7 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
     public void HCTestReducedBam() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T HaplotypeCaller -R " + b37KGReference + " --no_cmdline_in_header -I " + privateTestDir + "bamExample.ReducedRead.ADAnnotation.bam -o %s -L 1:67,225,396-67,288,518", 1,
-                Arrays.asList("79af83432dc4a1768b3ebffffc4d2b8f"));
+                Arrays.asList("4beb9f87ab3f316a9384c3d0dca6ebe9"));
         executeTest("HC calling on a ReducedRead BAM", spec);
     }
 }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
index 2f6d9d16f..547f375bb 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
@@ -75,8 +75,9 @@ public class StandardCallerArgumentCollection {
      * Basically, it will ignore the contamination fraction of reads for each alternate allele.  So if the pileup contains N total bases, then we
      * will try to remove (N * contamination fraction) bases for each alternate allele.
      */
-    @Argument(fullName = "contamination_percentage_to_filter", shortName = "contamination", doc = "Fraction of contamination in sequencing data (for all samples) to aggressively remove", required = false)
-    public double CONTAMINATION_FRACTION = 0.0;
+    @Argument(fullName = "contamination_fraction_to_filter", shortName = "contamination", doc = "Fraction of contamination in sequencing data (for all samples) to aggressively remove", required = false)
+    public double CONTAMINATION_FRACTION = DEFAULT_CONTAMINATION_FRACTION;
+    public static final double DEFAULT_CONTAMINATION_FRACTION = 0.05;
 
     @Hidden
     @Argument(fullName = "logRemovedReadsFromContaminationFiltering", shortName="contaminationLog", required=false)
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
index ef56eaa6d..36be2e7c6 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
@@ -28,6 +28,7 @@ package org.broadinstitute.sting.gatk.walkers.genotyper;
 import org.broadinstitute.sting.commandline.*;
 import org.broadinstitute.sting.gatk.CommandLineGATK;
 import org.broadinstitute.sting.gatk.arguments.DbsnpArgumentCollection;
+import org.broadinstitute.sting.gatk.arguments.StandardCallerArgumentCollection;
 import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.downsampling.DownsampleType;
@@ -238,7 +239,12 @@ public class UnifiedGenotyper extends LocusWalker<List<VariantCallContext>, Unif
             }
 
             if ( UAC.CONTAMINATION_FRACTION > 0.0 ) {
-                throw new UserException.NotSupportedInGATKLite("you cannot enable usage of contamination down-sampling");
+                if ( UAC.CONTAMINATION_FRACTION == StandardCallerArgumentCollection.DEFAULT_CONTAMINATION_FRACTION ) {
+                    UAC.CONTAMINATION_FRACTION = 0.0;
+                    logger.warn("setting contamination down-sampling fraction to 0.0 because it is not enabled in GATK-lite");
+                } else {
+                    throw new UserException.NotSupportedInGATKLite("you cannot enable usage of contamination down-sampling");
+                }
             }
         }
 
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index 446f8cbe8..a5192ffff 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -30,7 +30,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testMultiSamplePilot1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000", 1,
-                Arrays.asList("847605f4efafef89529fe0e496315edd"));
+                Arrays.asList("cdec335abc9ad8e59335e39a73e0e95a"));
         executeTest("test MultiSample Pilot1", spec);
     }
 
@@ -62,7 +62,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testMultipleSNPAlleles() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + privateTestDir + "multiallelic.snps.bam -o %s -L " + privateTestDir + "multiallelic.snps.intervals", 1,
-                Arrays.asList("543f68e42034bf44cfb24da8c9204320"));
+                Arrays.asList("94dc17d76d841f1d3a36160767ffa034"));
         executeTest("test Multiple SNP alleles", spec);
     }
 
@@ -86,7 +86,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testMismatchedPLs() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm INDEL -I " + privateTestDir + "mismatchedPLs.bam -o %s -L 1:24020341", 1,
-                Arrays.asList("3c006b06b17bbe8e787d64eff6a63a19"));
+                Arrays.asList("d847acf841ba8ba653f996ce4869f439"));
         executeTest("test mismatched PLs", spec);
     }
 
@@ -96,7 +96,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    private final static String COMPRESSED_OUTPUT_MD5 = "fd236bd635d514e4214d364f45ec4d10";
+    private final static String COMPRESSED_OUTPUT_MD5 = "6792419c482e767a3deb28913ed2b1ad";
 
     @Test
     public void testCompressedOutput() {
@@ -149,7 +149,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testMinBaseQualityScore() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --min_base_quality_score 26", 1,
-                Arrays.asList("839ecd30d354a36b5dfa2b5e99859765"));
+                Arrays.asList("56157d930da6ccd224bce1ca93f11e41"));
         executeTest("test min_base_quality_score 26", spec);
     }
 
@@ -165,7 +165,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testNDA() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --annotateNDA -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
-                Arrays.asList("abd8e33e649cc11b55e200d3940cc7e2"));
+                Arrays.asList("480437dd6e2760f4ab3194431519f331"));
         executeTest("test NDA", spec);
     }
 
@@ -173,7 +173,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testCompTrack() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH -comp:FOO " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
-                Arrays.asList("a6de72c5197f152be98a6f9029091a9e"));
+                Arrays.asList("22c039412fd387dde6125b07c9a74a25"));
         executeTest("test using comp track", spec);
     }
 
@@ -211,18 +211,10 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testConfidence() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 -stand_call_conf 10 ", 1,
-                Arrays.asList("9addd225a985178339a0c49dc5fdc220"));
+                Arrays.asList("df524e98903d96ab9353bee7c16a69de"));
         executeTest("test confidence 1", spec1);
     }
 
-    @Test
-    public void testConfidence2() {
-        WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
-                baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 -stand_emit_conf 10 ", 1,
-                Arrays.asList("9addd225a985178339a0c49dc5fdc220"));
-        executeTest("test confidence 2", spec2);
-    }
-
     // --------------------------------------------------------------------------------------------------------------
     //
     // testing heterozygosity
@@ -230,12 +222,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // --------------------------------------------------------------------------------------------------------------
     @Test
     public void testHeterozyosity1() {
-        testHeterozosity( 0.01, "986923de51c71635d47e3d06fe3794a1" );
+        testHeterozosity( 0.01, "8e61498ca03a8d805372a64c466b3b42" );
     }
 
     @Test
     public void testHeterozyosity2() {
-        testHeterozosity( 1.0 / 1850, "fb12b1553f813004a394a391a8540873" );
+        testHeterozosity( 1.0 / 1850, "668d06b5173cf3b97d052726988e1d7b" );
     }
 
     private void testHeterozosity(final double arg, final String md5) {
@@ -259,7 +251,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                         " -o %s" +
                         " -L 1:10,000,000-10,100,000",
                 1,
-                Arrays.asList("04a87b87ee4323eba853c78f25551d1a"));
+                Arrays.asList("908eb5e21fa39e7fb377cf4a9c4c7835"));
 
         executeTest(String.format("test multiple technologies"), spec);
     }
@@ -278,7 +270,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                         " -L 1:10,000,000-10,100,000" +
                         " -baq CALCULATE_AS_NECESSARY",
                 1,
-                Arrays.asList("98058fc913b61c22d44875da1f5ea89c"));
+                Arrays.asList("c814558bb0ed2e19b12e1a2bf4465d52"));
 
         executeTest(String.format("test calling with BAQ"), spec);
     }
@@ -312,7 +304,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                         " -minIndelCnt 1" +
                         " -L 1:10,000,000-10,100,000",
                 1,
-                Arrays.asList("6a0c2a3a7bcc56ad01428c71408055aa"));
+                Arrays.asList("8b486a098029d5a106b0a37eff541c15"));
 
         executeTest(String.format("test indel caller in SLX with low min allele count"), spec);
     }
@@ -325,7 +317,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                          " -o %s" +
                          " -L 1:10,000,000-10,500,000",
                  1,
-                 Arrays.asList("5f2721c3323de5390d2d47446139f32b"));
+                 Arrays.asList("18efedc50cae2aacaba372265e38310b"));
 
          executeTest(String.format("test indel calling, multiple technologies"), spec);
      }
@@ -353,13 +345,13 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testMultiSampleIndels1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10450700-10551000", 1,
-                Arrays.asList("a4761d7f25e7a62f34494801c98a0da7"));
+                Arrays.asList("f7d0d0aee603df25c1f0525bb8df189e"));
         List<File> result = executeTest("test MultiSample Pilot1 CEU indels", spec1).getFirst();
 
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + result.get(0).getAbsolutePath() + " -I " + validationDataLocation +
                         "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10450700-10551000", 1,
-                Arrays.asList("c526c234947482d1cd2ffc5102083a08"));
+                Arrays.asList("fc91d457a16b4ca994959c2b5f3f0352"));
         executeTest("test MultiSample Pilot1 CEU indels using GENOTYPE_GIVEN_ALLELES", spec2);
     }
 
@@ -415,7 +407,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testMinIndelFraction0() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.0", 1,
-                Arrays.asList("ba4fafec383fb988f20c8cf53dd3e9a0"));
+                Arrays.asList("857b8e5df444463ac27f665c4f67fbe2"));
         executeTest("test minIndelFraction 0.0", spec);
     }
 
@@ -423,7 +415,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testMinIndelFraction25() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.25", 1,
-                Arrays.asList("4c57a88de275105156aaafc6f9041365"));
+                Arrays.asList("81d4c7d9010fd6733b2997bc378e7471"));
         executeTest("test minIndelFraction 0.25", spec);
     }
 
@@ -490,7 +482,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testContaminationDownsampling() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-                baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --contamination_percentage_to_filter 0.20", 1,
+                baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --contamination_fraction_to_filter 0.20", 1,
                 Arrays.asList("27dd04159e06d9524fb8a4eef41f96ae"));
         executeTest("test contamination_percentage_to_filter 0.20", spec);
     }

From a53e03d52577e09cad20738b507260cb8e8f0980 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Fri, 26 Oct 2012 02:13:04 -0400
Subject: [PATCH 33/40] Do not let reduced reads get removed in the
 contamination down-sampling

---
 .../AlleleBiasedDownsamplingUtils.java        |  4 ++
 .../AdvancedPerReadAlleleLikelihoodMap.java   |  9 +++--
 .../UnifiedGenotyperIntegrationTest.java      | 40 +++++++++----------
 3 files changed, 30 insertions(+), 23 deletions(-)

diff --git a/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java b/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
index e224a9c57..830fd111b 100755
--- a/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
@@ -60,6 +60,10 @@ public class AlleleBiasedDownsamplingUtils {
 
         // start by stratifying the reads by the alleles they represent at this position
         for( final PileupElement pe : pileup ) {
+            // abort if we have a reduced read - we do not want to remove it!
+            if ( pe.getRead().isReducedRead() )
+                return pileup;
+
             final int baseIndex = BaseUtils.simpleBaseToBaseIndex(pe.getBase());
             if ( baseIndex != -1 )
                 alleleStratifiedElements[baseIndex].add(pe);
diff --git a/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java b/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
index d6cddcf84..77a7c3bd9 100644
--- a/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
+++ b/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
@@ -55,9 +55,12 @@ public class AdvancedPerReadAlleleLikelihoodMap extends StandardPerReadAlleleLik
             alleleReadMap.put(allele, new ArrayList<GATKSAMRecord>());
 
         for ( Map.Entry<GATKSAMRecord, Map<Allele, Double>> entry : likelihoodReadMap.entrySet() ) {
-            final Allele bestAllele = getMostLikelyAllele(entry.getValue());
-            if ( bestAllele != Allele.NO_CALL )
-                alleleReadMap.get(bestAllele).add(entry.getKey());
+            // do not remove reduced reads!
+            if ( !entry.getKey().isReducedRead() ) {
+                final Allele bestAllele = getMostLikelyAllele(entry.getValue());
+                if ( bestAllele != Allele.NO_CALL )
+                    alleleReadMap.get(bestAllele).add(entry.getKey());
+            }
         }
 
         // compute the reads to remove and actually remove them
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index a5192ffff..93fe7e80d 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -38,7 +38,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testWithAllelesPassedIn1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
-                Arrays.asList("bc15123620e1134f799005d71d1180fe"));
+                Arrays.asList("efddb5e258f97fd4f6661cff9eaa57de"));
         executeTest("test MultiSample Pilot2 with alleles passed in", spec1);
     }
 
@@ -46,7 +46,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testWithAllelesPassedIn2() {
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
-                Arrays.asList("1ba7afccc8552f20d72d0b62237558e3"));
+                Arrays.asList("24532eb381724cd74e99370da28d49ed"));
         executeTest("test MultiSample Pilot2 with alleles passed in and emitting all sites", spec2);
     }
 
@@ -54,7 +54,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testSingleSamplePilot2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
-                Arrays.asList("afb8768f31ab57eb43f75c1115eadc99"));
+                Arrays.asList("062a946160eec1d0fc135d58ca654ff4"));
         executeTest("test SingleSample Pilot2", spec);
     }
 
@@ -78,7 +78,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testReverseTrim() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm INDEL -I " + validationDataLocation + "CEUTrio.HiSeq.b37.chr20.10_11mb.bam -o %s -L 20:10289124 -L 20:10090289", 1,
-                Arrays.asList("5ce03dd9ca2d9324c1d4a9d64389beb5"));
+                Arrays.asList("9106d01ca0d0a8fedd068e72d509f380"));
         executeTest("test reverse trim", spec);
     }
 
@@ -117,7 +117,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
         // Note that we need to turn off any randomization for this to work, so no downsampling and no annotations
 
-        String md5 = "d408b4661b820ed86272415b8ea08780";
+        String md5 = "3d25ea660275a455ca443a786bff3d32";
 
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -dt NONE -G none -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,075,000", 1,
@@ -157,7 +157,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testSLOD() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --computeSLOD --no_cmdline_in_header -glm BOTH --dbsnp " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
-                Arrays.asList("c7429e670ba477bf9a6bbee2fb41c5a9"));
+                Arrays.asList("9f95cfe14d53a697c58247833bfd72a6"));
         executeTest("test SLOD", spec);
     }
 
@@ -177,7 +177,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test using comp track", spec);
     }
 
-    @Test
+    //@Test
     public void testNoCmdLineHeaderStdout() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandNoCmdLineHeaderStdout + " -glm INDEL -L 1:67,225,396-67,288,518", 0,
@@ -187,17 +187,17 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
     @Test
     public void testOutputParameterSitesOnly() {
-        testOutputParameters("-sites_only", "97ba874eafc9884a4de027a84c036311");
+        testOutputParameters("-sites_only", "40aeb4c9e31fe7046b72afc58e7599cb");
     }
 
     @Test
     public void testOutputParameterAllConfident() {
-        testOutputParameters("--output_mode EMIT_ALL_CONFIDENT_SITES", "f9ea04d96eeef29e71d37e60518c2579");
+        testOutputParameters("--output_mode EMIT_ALL_CONFIDENT_SITES", "c706ca93b25ff83613cb4e95dcac567c");
     }
 
     @Test
     public void testOutputParameterAllSites() {
-        testOutputParameters("--output_mode EMIT_ALL_SITES", "41c046d38ea328421df924e37e017645");
+        testOutputParameters("--output_mode EMIT_ALL_SITES", "8a263fd0a94463ce1de9990f2b8ec841");
     }
 
     private void testOutputParameters(final String args, final String md5) {
@@ -289,7 +289,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                         " -o %s" +
                         " -L 1:10,000,000-10,500,000",
                 1,
-                Arrays.asList("650c53774afacfc07a595675e8cdde17"));
+                Arrays.asList("3593495aab5f6204c65de0b073a6ff65"));
 
         executeTest(String.format("test indel caller in SLX"), spec);
     }
@@ -327,7 +327,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "indelAllelesForUG.vcf -I " + validationDataLocation +
                         "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,100,000", 1,
-                Arrays.asList("7e3f67bf371112be5dbadb4fe6faa52a"));
+                Arrays.asList("3ff8c7c80a518aa3eb8671a21479de5f"));
         executeTest("test MultiSample Pilot2 indels with alleles passed in", spec);
     }
 
@@ -337,7 +337,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                 baseCommandIndels + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles "
                         + privateTestDir + "indelAllelesForUG.vcf -I " + validationDataLocation +
                         "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,100,000", 1,
-                Arrays.asList("bc31c4977cb7e563ddf9c8dea27f3f4f"));
+                Arrays.asList("578c0540f4f2052a634a829bcb9cc27d"));
         executeTest("test MultiSample Pilot2 indels with alleles passed in and emitting all sites", spec);
     }
 
@@ -436,8 +436,8 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testNsInCigar() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-                "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -I " + validationDataLocation + "testWithNs.bam -o %s -L 8:141799600-141814700", 1,
-                Arrays.asList("e8ebfaac0804b782f22ab8ea35152735"));
+                "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -I " + validationDataLocation + "testWithNs.bam -o %s -L 8:141799600-141814700", 1,
+                Arrays.asList("bd7984a374f0ae5d277bd5fc5065f64f"));
         executeTest("test calling on reads with Ns in CIGAR", spec);
     }
 
@@ -450,25 +450,25 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testReducedBam() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-                "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -I " + privateTestDir + "bamExample.ReducedRead.ADAnnotation.bam -o %s -L 1:67,225,396-67,288,518", 1,
-                Arrays.asList("da9c05f87bd6415e97f90c49cf68ed19"));
+                "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -I " + privateTestDir + "bamExample.ReducedRead.ADAnnotation.bam -o %s -L 1:67,225,396-67,288,518", 1,
+                Arrays.asList("1b711c0966a2f76eb21801e73b76b758"));
         executeTest("test calling on a ReducedRead BAM", spec);
     }
 
     @Test
     public void testReducedBamSNPs() {
-        testReducedCalling("SNP", "1d4a826b144723ff0766c36aa0239287");
+        testReducedCalling("SNP", "9ba4867cadb366746ee63e7a4afdb95e");
     }
 
     @Test
     public void testReducedBamINDELs() {
-        testReducedCalling("INDEL", "68ef51d5c98480e0c0192e0eecb95bca");
+        testReducedCalling("INDEL", "132a4e0ccf9230b5bb4b56c649e2bdd5");
     }
 
 
     private void testReducedCalling(final String model, final String md5) {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-                "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -I " + privateTestDir + "NA12878.HiSeq.b37.chr20.10_11mb.reduced.bam -o %s -L 20:10,000,000-11,000,000 -glm " + model, 1,
+                "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -I " + privateTestDir + "NA12878.HiSeq.b37.chr20.10_11mb.reduced.bam -o %s -L 20:10,000,000-11,000,000 -glm " + model, 1,
                 Arrays.asList(md5));
         executeTest("test calling on a ReducedRead BAM with " + model, spec);
     }

From ebebec7fdb74e180e4bc019e18b13261227a93c3 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Fri, 26 Oct 2012 02:15:32 -0400
Subject: [PATCH 34/40] Accidentally left one test disabled

---
 .../gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index 93fe7e80d..1b2fbb848 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -177,7 +177,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test using comp track", spec);
     }
 
-    //@Test
+    @Test
     public void testNoCmdLineHeaderStdout() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandNoCmdLineHeaderStdout + " -glm INDEL -L 1:67,225,396-67,288,518", 0,

From 7a706ed345cdae1fe7f825e1aec13aeed825e0fb Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Fri, 26 Oct 2012 11:23:44 -0400
Subject: [PATCH 36/40] Fix some of the broken integration tests

---
 .../AlleleBiasedDownsamplingUtils.java        | 10 +--
 .../UnifiedGenotyperIntegrationTest.java      | 78 +++++++++----------
 .../NanoSchedulerIntegrationTest.java         |  3 +-
 3 files changed, 45 insertions(+), 46 deletions(-)

diff --git a/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java b/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
index 830fd111b..b84cde58e 100755
--- a/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
@@ -37,12 +37,6 @@ import java.util.*;
 
 public class AlleleBiasedDownsamplingUtils {
 
-    private static final ArrayList<PileupElement>[] alleleStratifiedElements = new ArrayList[4];
-    static {
-        for ( int i = 0; i < 4; i++ )
-            alleleStratifiedElements[i] = new ArrayList<PileupElement>();
-    }
-
     /**
      * Computes an allele biased version of the given pileup
      *
@@ -58,6 +52,10 @@ public class AlleleBiasedDownsamplingUtils {
         if ( downsamplingFraction >= 1.0 )
             return new ReadBackedPileupImpl(pileup.getLocation(), new ArrayList<PileupElement>());
 
+        final ArrayList<PileupElement>[] alleleStratifiedElements = new ArrayList[4];
+        for ( int i = 0; i < 4; i++ )
+            alleleStratifiedElements[i] = new ArrayList<PileupElement>();
+
         // start by stratifying the reads by the alleles they represent at this position
         for( final PileupElement pe : pileup ) {
             // abort if we have a reduced read - we do not want to remove it!
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index 1b2fbb848..ea2f0fb97 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -26,7 +26,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing normal calling
     //
     // --------------------------------------------------------------------------------------------------------------
-    @Test
+    //@Test
     public void testMultiSamplePilot1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000", 1,
@@ -34,7 +34,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot1", spec);
     }
 
-    @Test
+    //@Test
     public void testWithAllelesPassedIn1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
@@ -42,7 +42,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 with alleles passed in", spec1);
     }
 
-    @Test
+    //@Test
     public void testWithAllelesPassedIn2() {
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
@@ -50,7 +50,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 with alleles passed in and emitting all sites", spec2);
     }
 
-    @Test
+    //@Test
     public void testSingleSamplePilot2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
@@ -58,7 +58,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test SingleSample Pilot2", spec);
     }
 
-    @Test
+    //@Test
     public void testMultipleSNPAlleles() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + privateTestDir + "multiallelic.snps.bam -o %s -L " + privateTestDir + "multiallelic.snps.intervals", 1,
@@ -66,7 +66,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test Multiple SNP alleles", spec);
     }
 
-    @Test
+    //@Test
     public void testBadRead() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm BOTH -I " + privateTestDir + "badRead.test.bam -o %s -L 1:22753424-22753464", 1,
@@ -74,7 +74,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test bad read", spec);
     }
 
-    @Test
+    //@Test
     public void testReverseTrim() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm INDEL -I " + validationDataLocation + "CEUTrio.HiSeq.b37.chr20.10_11mb.bam -o %s -L 20:10289124 -L 20:10090289", 1,
@@ -82,7 +82,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test reverse trim", spec);
     }
 
-    @Test
+    //@Test
     public void testMismatchedPLs() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm INDEL -I " + privateTestDir + "mismatchedPLs.bam -o %s -L 1:24020341", 1,
@@ -98,7 +98,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
     private final static String COMPRESSED_OUTPUT_MD5 = "6792419c482e767a3deb28913ed2b1ad";
 
-    @Test
+    //@Test
     public void testCompressedOutput() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
@@ -145,7 +145,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testMinBaseQualityScore() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --min_base_quality_score 26", 1,
@@ -153,7 +153,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test min_base_quality_score 26", spec);
     }
 
-    @Test
+    //@Test
     public void testSLOD() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --computeSLOD --no_cmdline_in_header -glm BOTH --dbsnp " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -161,7 +161,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test SLOD", spec);
     }
 
-    @Test
+    //@Test
     public void testNDA() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --annotateNDA -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -169,7 +169,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test NDA", spec);
     }
 
-    @Test
+    //@Test
     public void testCompTrack() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH -comp:FOO " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -177,7 +177,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test using comp track", spec);
     }
 
-    @Test
+    //@Test
     public void testNoCmdLineHeaderStdout() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandNoCmdLineHeaderStdout + " -glm INDEL -L 1:67,225,396-67,288,518", 0,
@@ -185,17 +185,17 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("testNoCmdLineHeaderStdout", spec);
     }
 
-    @Test
+    //@Test
     public void testOutputParameterSitesOnly() {
         testOutputParameters("-sites_only", "40aeb4c9e31fe7046b72afc58e7599cb");
     }
 
-    @Test
+    //@Test
     public void testOutputParameterAllConfident() {
         testOutputParameters("--output_mode EMIT_ALL_CONFIDENT_SITES", "c706ca93b25ff83613cb4e95dcac567c");
     }
 
-    @Test
+    //@Test
     public void testOutputParameterAllSites() {
         testOutputParameters("--output_mode EMIT_ALL_SITES", "8a263fd0a94463ce1de9990f2b8ec841");
     }
@@ -207,7 +207,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest(String.format("testParameter[%s]", args), spec);
     }
 
-    @Test
+    //@Test
     public void testConfidence() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 -stand_call_conf 10 ", 1,
@@ -220,12 +220,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing heterozygosity
     //
     // --------------------------------------------------------------------------------------------------------------
-    @Test
+    //@Test
     public void testHeterozyosity1() {
         testHeterozosity( 0.01, "8e61498ca03a8d805372a64c466b3b42" );
     }
 
-    @Test
+    //@Test
     public void testHeterozyosity2() {
         testHeterozosity( 1.0 / 1850, "668d06b5173cf3b97d052726988e1d7b" );
     }
@@ -243,7 +243,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing calls with SLX, 454, and SOLID data
     //
     // --------------------------------------------------------------------------------------------------------------
-    @Test
+    //@Test
     public void testMultiTechnologies() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand +
@@ -261,7 +261,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing calls with BAQ
     //
     // --------------------------------------------------------------------------------------------------------------
-    @Test
+    //@Test
     public void testCallingWithBAQ() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand +
@@ -281,7 +281,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
     // Basic indel testing with SLX data
-    @Test
+    //@Test
     public void testSimpleIndels() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels +
@@ -295,7 +295,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     }
 
     // Basic indel testing with SLX data
-    @Test
+    //@Test
     public void testIndelsWithLowMinAlleleCnt() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels +
@@ -309,7 +309,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest(String.format("test indel caller in SLX with low min allele count"), spec);
     }
 
-    @Test
+    //@Test
     public void testMultiTechnologyIndels() {
          WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                  baseCommandIndels +
@@ -322,7 +322,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
          executeTest(String.format("test indel calling, multiple technologies"), spec);
      }
 
-    @Test
+    //@Test
     public void testWithIndelAllelesPassedIn1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "indelAllelesForUG.vcf -I " + validationDataLocation +
@@ -331,7 +331,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 indels with alleles passed in", spec);
     }
 
-    @Test
+    //@Test
     public void testWithIndelAllelesPassedIn2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles "
@@ -341,7 +341,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 indels with alleles passed in and emitting all sites", spec);
     }
 
-    @Test
+    //@Test
     public void testMultiSampleIndels1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10450700-10551000", 1,
@@ -355,7 +355,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot1 CEU indels using GENOTYPE_GIVEN_ALLELES", spec2);
     }
 
-    @Test
+    //@Test
     public void testGGAwithNoEvidenceInReads() {
         final String vcf = "small.indel.test.vcf";
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
@@ -365,7 +365,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test GENOTYPE_GIVEN_ALLELES with no evidence in reads", spec);
     }
 
-    @Test
+    //@Test
     public void testBaseIndelQualityScores() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndelsb37 +
@@ -384,7 +384,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testSnpEffAnnotationRequestedWithoutRodBinding() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000 " +
@@ -403,7 +403,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     final static String assessMinIndelFraction = baseCommandIndelsb37 + " -I " + validationDataLocation
             + "978604.bam -L 1:978,586-978,626 -o %s --sites_only -rf Sample -goodSM 7377 -goodSM 22-0022 -goodSM 134 -goodSM 344029-53 -goodSM 14030";
 
-    @Test
+    //@Test
     public void testMinIndelFraction0() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.0", 1,
@@ -411,7 +411,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test minIndelFraction 0.0", spec);
     }
 
-    @Test
+    //@Test
     public void testMinIndelFraction25() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.25", 1,
@@ -419,7 +419,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test minIndelFraction 0.25", spec);
     }
 
-    @Test
+    //@Test
     public void testMinIndelFraction100() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 1", 1,
@@ -433,7 +433,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testNsInCigar() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -I " + validationDataLocation + "testWithNs.bam -o %s -L 8:141799600-141814700", 1,
@@ -447,7 +447,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testReducedBam() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -I " + privateTestDir + "bamExample.ReducedRead.ADAnnotation.bam -o %s -L 1:67,225,396-67,288,518", 1,
@@ -455,12 +455,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test calling on a ReducedRead BAM", spec);
     }
 
-    @Test
+    //@Test
     public void testReducedBamSNPs() {
         testReducedCalling("SNP", "9ba4867cadb366746ee63e7a4afdb95e");
     }
 
-    @Test
+    //@Test
     public void testReducedBamINDELs() {
         testReducedCalling("INDEL", "132a4e0ccf9230b5bb4b56c649e2bdd5");
     }
@@ -479,7 +479,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testContaminationDownsampling() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --contamination_fraction_to_filter 0.20", 1,
diff --git a/public/java/test/org/broadinstitute/sting/utils/nanoScheduler/NanoSchedulerIntegrationTest.java b/public/java/test/org/broadinstitute/sting/utils/nanoScheduler/NanoSchedulerIntegrationTest.java
index 37614f15f..80b0b4ee2 100755
--- a/public/java/test/org/broadinstitute/sting/utils/nanoScheduler/NanoSchedulerIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/utils/nanoScheduler/NanoSchedulerIntegrationTest.java
@@ -32,11 +32,12 @@ public class NanoSchedulerIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
                 buildCommandLine(
                         "-T UnifiedGenotyper -R " + b37KGReference,
-                        "-nosl --no_cmdline_in_header -G",
+                        "--no_cmdline_in_header -G",
                         //"--dbsnp " + b37dbSNP132,
                         "-I " + privateTestDir + "NA12878.HiSeq.b37.chr20.10_11mb.bam",
                         "-L 20:10,000,000-10,100,000",
                         "-glm " + glm,
+                        "--contamination_fraction_to_filter 0.0",
                         "-nt " + nt,
                         "-nct " + nct,
                         "-o %s"

From ed11b7dab25c87c34942698ba10c6cb77ed2da4f Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Fri, 26 Oct 2012 12:10:44 -0400
Subject: [PATCH 37/40] Fix UG parallelization test

---
 .../AlleleBiasedDownsamplingUtils.java        |  2 +-
 .../UnifiedGenotyperIntegrationTest.java      | 86 +++++++++----------
 2 files changed, 44 insertions(+), 44 deletions(-)

diff --git a/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java b/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
index b84cde58e..59357e1c4 100755
--- a/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
@@ -45,7 +45,7 @@ public class AlleleBiasedDownsamplingUtils {
      * @param log                       logging output
      * @return allele biased pileup
      */
-    public synchronized static ReadBackedPileup createAlleleBiasedBasePileup(final ReadBackedPileup pileup, final double downsamplingFraction, final PrintStream log) {
+    public static ReadBackedPileup createAlleleBiasedBasePileup(final ReadBackedPileup pileup, final double downsamplingFraction, final PrintStream log) {
         // special case removal of all or no reads
         if ( downsamplingFraction <= 0.0 )
             return pileup;
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index ea2f0fb97..2a17435aa 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -26,7 +26,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing normal calling
     //
     // --------------------------------------------------------------------------------------------------------------
-    //@Test
+    @Test
     public void testMultiSamplePilot1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000", 1,
@@ -34,7 +34,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot1", spec);
     }
 
-    //@Test
+    @Test
     public void testWithAllelesPassedIn1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
@@ -42,7 +42,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 with alleles passed in", spec1);
     }
 
-    //@Test
+    @Test
     public void testWithAllelesPassedIn2() {
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
@@ -50,7 +50,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 with alleles passed in and emitting all sites", spec2);
     }
 
-    //@Test
+    @Test
     public void testSingleSamplePilot2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
@@ -58,7 +58,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test SingleSample Pilot2", spec);
     }
 
-    //@Test
+    @Test
     public void testMultipleSNPAlleles() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + privateTestDir + "multiallelic.snps.bam -o %s -L " + privateTestDir + "multiallelic.snps.intervals", 1,
@@ -66,7 +66,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test Multiple SNP alleles", spec);
     }
 
-    //@Test
+    @Test
     public void testBadRead() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm BOTH -I " + privateTestDir + "badRead.test.bam -o %s -L 1:22753424-22753464", 1,
@@ -74,7 +74,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test bad read", spec);
     }
 
-    //@Test
+    @Test
     public void testReverseTrim() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm INDEL -I " + validationDataLocation + "CEUTrio.HiSeq.b37.chr20.10_11mb.bam -o %s -L 20:10289124 -L 20:10090289", 1,
@@ -82,7 +82,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test reverse trim", spec);
     }
 
-    //@Test
+    @Test
     public void testMismatchedPLs() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm INDEL -I " + privateTestDir + "mismatchedPLs.bam -o %s -L 1:24020341", 1,
@@ -98,7 +98,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
     private final static String COMPRESSED_OUTPUT_MD5 = "6792419c482e767a3deb28913ed2b1ad";
 
-    //@Test
+    @Test
     public void testCompressedOutput() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
@@ -117,24 +117,24 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
         // Note that we need to turn off any randomization for this to work, so no downsampling and no annotations
 
-        String md5 = "3d25ea660275a455ca443a786bff3d32";
+        String md5 = "d408b4661b820ed86272415b8ea08780";
 
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
-                baseCommand + " -dt NONE -G none -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,075,000", 1,
+                baseCommand + " -dt NONE -G none --contamination_fraction_to_filter 0.0 -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,075,000", 1,
                 Arrays.asList(md5));
         executeTest("test parallelization (single thread)", spec1);
 
         GenomeAnalysisEngine.resetRandomGenerator();
 
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
-                baseCommand + " -dt NONE -G none -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,075,000 -nt 2", 1,
+                baseCommand + " -dt NONE -G none --contamination_fraction_to_filter 0.0 -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,075,000 -nt 2", 1,
                 Arrays.asList(md5));
         executeTest("test parallelization (2 threads)", spec2);
 
         GenomeAnalysisEngine.resetRandomGenerator();
 
         WalkerTest.WalkerTestSpec spec3 = new WalkerTest.WalkerTestSpec(
-                baseCommand + " -dt NONE -G none -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,075,000 -nt 4", 1,
+                baseCommand + " -dt NONE -G none --contamination_fraction_to_filter 0.0 -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,075,000 -nt 4", 1,
                 Arrays.asList(md5));
         executeTest("test parallelization (4 threads)", spec3);
     }
@@ -145,7 +145,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testMinBaseQualityScore() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --min_base_quality_score 26", 1,
@@ -153,7 +153,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test min_base_quality_score 26", spec);
     }
 
-    //@Test
+    @Test
     public void testSLOD() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --computeSLOD --no_cmdline_in_header -glm BOTH --dbsnp " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -161,7 +161,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test SLOD", spec);
     }
 
-    //@Test
+    @Test
     public void testNDA() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --annotateNDA -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -169,7 +169,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test NDA", spec);
     }
 
-    //@Test
+    @Test
     public void testCompTrack() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH -comp:FOO " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -177,7 +177,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test using comp track", spec);
     }
 
-    //@Test
+    @Test
     public void testNoCmdLineHeaderStdout() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandNoCmdLineHeaderStdout + " -glm INDEL -L 1:67,225,396-67,288,518", 0,
@@ -185,17 +185,17 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("testNoCmdLineHeaderStdout", spec);
     }
 
-    //@Test
+    @Test
     public void testOutputParameterSitesOnly() {
         testOutputParameters("-sites_only", "40aeb4c9e31fe7046b72afc58e7599cb");
     }
 
-    //@Test
+    @Test
     public void testOutputParameterAllConfident() {
         testOutputParameters("--output_mode EMIT_ALL_CONFIDENT_SITES", "c706ca93b25ff83613cb4e95dcac567c");
     }
 
-    //@Test
+    @Test
     public void testOutputParameterAllSites() {
         testOutputParameters("--output_mode EMIT_ALL_SITES", "8a263fd0a94463ce1de9990f2b8ec841");
     }
@@ -207,7 +207,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest(String.format("testParameter[%s]", args), spec);
     }
 
-    //@Test
+    @Test
     public void testConfidence() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 -stand_call_conf 10 ", 1,
@@ -220,12 +220,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing heterozygosity
     //
     // --------------------------------------------------------------------------------------------------------------
-    //@Test
+    @Test
     public void testHeterozyosity1() {
         testHeterozosity( 0.01, "8e61498ca03a8d805372a64c466b3b42" );
     }
 
-    //@Test
+    @Test
     public void testHeterozyosity2() {
         testHeterozosity( 1.0 / 1850, "668d06b5173cf3b97d052726988e1d7b" );
     }
@@ -243,7 +243,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing calls with SLX, 454, and SOLID data
     //
     // --------------------------------------------------------------------------------------------------------------
-    //@Test
+    @Test
     public void testMultiTechnologies() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand +
@@ -261,7 +261,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing calls with BAQ
     //
     // --------------------------------------------------------------------------------------------------------------
-    //@Test
+    @Test
     public void testCallingWithBAQ() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand +
@@ -281,7 +281,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
     // Basic indel testing with SLX data
-    //@Test
+    @Test
     public void testSimpleIndels() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels +
@@ -295,7 +295,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     }
 
     // Basic indel testing with SLX data
-    //@Test
+    @Test
     public void testIndelsWithLowMinAlleleCnt() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels +
@@ -309,7 +309,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest(String.format("test indel caller in SLX with low min allele count"), spec);
     }
 
-    //@Test
+    @Test
     public void testMultiTechnologyIndels() {
          WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                  baseCommandIndels +
@@ -322,7 +322,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
          executeTest(String.format("test indel calling, multiple technologies"), spec);
      }
 
-    //@Test
+    @Test
     public void testWithIndelAllelesPassedIn1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "indelAllelesForUG.vcf -I " + validationDataLocation +
@@ -331,7 +331,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 indels with alleles passed in", spec);
     }
 
-    //@Test
+    @Test
     public void testWithIndelAllelesPassedIn2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles "
@@ -341,7 +341,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 indels with alleles passed in and emitting all sites", spec);
     }
 
-    //@Test
+    @Test
     public void testMultiSampleIndels1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10450700-10551000", 1,
@@ -355,7 +355,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot1 CEU indels using GENOTYPE_GIVEN_ALLELES", spec2);
     }
 
-    //@Test
+    @Test
     public void testGGAwithNoEvidenceInReads() {
         final String vcf = "small.indel.test.vcf";
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
@@ -365,7 +365,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test GENOTYPE_GIVEN_ALLELES with no evidence in reads", spec);
     }
 
-    //@Test
+    @Test
     public void testBaseIndelQualityScores() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndelsb37 +
@@ -384,7 +384,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testSnpEffAnnotationRequestedWithoutRodBinding() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000 " +
@@ -403,7 +403,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     final static String assessMinIndelFraction = baseCommandIndelsb37 + " -I " + validationDataLocation
             + "978604.bam -L 1:978,586-978,626 -o %s --sites_only -rf Sample -goodSM 7377 -goodSM 22-0022 -goodSM 134 -goodSM 344029-53 -goodSM 14030";
 
-    //@Test
+    @Test
     public void testMinIndelFraction0() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.0", 1,
@@ -411,7 +411,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test minIndelFraction 0.0", spec);
     }
 
-    //@Test
+    @Test
     public void testMinIndelFraction25() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.25", 1,
@@ -419,7 +419,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test minIndelFraction 0.25", spec);
     }
 
-    //@Test
+    @Test
     public void testMinIndelFraction100() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 1", 1,
@@ -433,7 +433,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testNsInCigar() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -I " + validationDataLocation + "testWithNs.bam -o %s -L 8:141799600-141814700", 1,
@@ -447,7 +447,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testReducedBam() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -I " + privateTestDir + "bamExample.ReducedRead.ADAnnotation.bam -o %s -L 1:67,225,396-67,288,518", 1,
@@ -455,12 +455,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test calling on a ReducedRead BAM", spec);
     }
 
-    //@Test
+    @Test
     public void testReducedBamSNPs() {
         testReducedCalling("SNP", "9ba4867cadb366746ee63e7a4afdb95e");
     }
 
-    //@Test
+    @Test
     public void testReducedBamINDELs() {
         testReducedCalling("INDEL", "132a4e0ccf9230b5bb4b56c649e2bdd5");
     }
@@ -479,7 +479,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testContaminationDownsampling() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --contamination_fraction_to_filter 0.20", 1,

From 251983b8fb829c084dfb29b8d60e9bc4e636491e Mon Sep 17 00:00:00 2001
From: Mark DePristo <depristo@broadinstitute.org>
Date: Fri, 26 Oct 2012 13:18:18 -0400
Subject: [PATCH 38/40] Add GATK-wide command line argument to control the
 maximum runtime allowed for the GATK

-- Providing this optional argument -maxRuntime (in -maxRuntimeUnits units) causes the GATK to exit gracefully when the max. runtime has been exceeded.  By cleanly I mean that the engine simply stops at the next available cycle in the walker as through the end of processing had been reached.  This means that all output files are closed properly, etc.
-- Emits an info message that looks like "INFO  10:36:52,723 MicroScheduler - Aborting execution (cleanly) because the runtime has exceeded the requested maximum 10.0000 s".  Otherwise there's currently no way to differentiate a truly completed run from a timelimit exceeded run, which may be a useful thing for a future update
-- Resolves GSA-630 / GATK max runtime to deal with bad LSA calling?
-- Added new JIRA entry for Ami to restart chr1 macarthur with this argument set to -maxRuntime 1 -maxRuntimeUnits DAYS to see if we can do all of chr1 in one weekend.
---
 .../sting/gatk/GenomeAnalysisEngine.java      | 31 ++++++-
 .../arguments/GATKArgumentCollection.java     | 10 ++-
 .../executive/HierarchicalMicroScheduler.java |  2 +-
 .../gatk/executive/LinearMicroScheduler.java  |  2 +-
 .../sting/gatk/executive/MicroScheduler.java  | 22 +++++
 .../sting/utils/AutoFormattingTime.java       | 31 ++++++-
 .../utils/progressmeter/ProgressMeter.java    |  9 ++
 .../sting/gatk/MaxRuntimeIntegrationTest.java | 86 +++++++++++++++++++
 8 files changed, 185 insertions(+), 8 deletions(-)
 create mode 100644 public/java/test/org/broadinstitute/sting/gatk/MaxRuntimeIntegrationTest.java

diff --git a/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java b/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java
index 67e5ad95b..b7000e0ee 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java
@@ -64,6 +64,7 @@ import org.broadinstitute.sting.utils.threading.ThreadEfficiencyMonitor;
 
 import java.io.File;
 import java.util.*;
+import java.util.concurrent.TimeUnit;
 
 /**
  * A GenomeAnalysisEngine that runs a specified walker.
@@ -73,6 +74,7 @@ public class GenomeAnalysisEngine {
      * our log, which we want to capture anything from this class
      */
     private static Logger logger = Logger.getLogger(GenomeAnalysisEngine.class);
+    public static final long NO_RUNTIME_LIMIT = -1;
 
     /**
      * The GATK command-line argument parsing code.
@@ -1090,6 +1092,33 @@ public class GenomeAnalysisEngine {
     public String createApproximateCommandLineArgumentString(Object... argumentProviders) {
         return CommandLineUtils.createApproximateCommandLineArgumentString(parsingEngine,argumentProviders);
     }
-    
 
+    /**
+     * Does the current runtime in unit exceed the runtime limit, if one has been provided?
+     *
+     * @param runtime the runtime of this GATK instance in minutes
+     * @param unit the time unit of runtime
+     * @return false if not limit was requested or if runtime <= the limit, true otherwise
+     */
+    public boolean exceedsRuntimeLimit(final long runtime, final TimeUnit unit) {
+        if ( runtime < 0 ) throw new IllegalArgumentException("runtime must be >= 0 but got " + runtime);
+
+        if ( getArguments().maxRuntime == NO_RUNTIME_LIMIT )
+            return false;
+        else {
+            final long actualRuntimeNano = TimeUnit.NANOSECONDS.convert(runtime, unit);
+            final long maxRuntimeNano = getRuntimeLimitInNanoseconds();
+            return actualRuntimeNano > maxRuntimeNano;
+        }
+    }
+
+    /**
+     * @return the runtime limit in nanoseconds, or -1 if no limit was specified
+     */
+    public long getRuntimeLimitInNanoseconds() {
+        if ( getArguments().maxRuntime == NO_RUNTIME_LIMIT )
+            return -1;
+        else
+            return TimeUnit.NANOSECONDS.convert(getArguments().maxRuntime, getArguments().maxRuntimeUnits);
+    }
 }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java
index 7875ced5a..e2b943582 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java
@@ -31,6 +31,7 @@ import org.broadinstitute.sting.commandline.Argument;
 import org.broadinstitute.sting.commandline.Hidden;
 import org.broadinstitute.sting.commandline.Input;
 import org.broadinstitute.sting.commandline.IntervalBinding;
+import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
 import org.broadinstitute.sting.gatk.downsampling.DownsampleType;
 import org.broadinstitute.sting.gatk.downsampling.DownsamplingMethod;
 import org.broadinstitute.sting.gatk.phonehome.GATKRunReport;
@@ -44,6 +45,7 @@ import java.io.File;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 /**
  * @author aaron
@@ -91,7 +93,7 @@ public class GATKArgumentCollection {
 
     // --------------------------------------------------------------------------------------------------------------
     //
-    // XXX
+    // General features
     //
     // --------------------------------------------------------------------------------------------------------------
 
@@ -143,6 +145,12 @@ public class GATKArgumentCollection {
     @Argument(fullName = "disableRandomization",doc="Completely eliminates randomization from nondeterministic methods. To be used mostly in the testing framework where dynamic parallelism can result in differing numbers of calls to the generator.")
     public boolean disableRandomization = false;
 
+    @Argument(fullName = "maxRuntime", shortName = "maxRuntime", doc="If provided, that GATK will stop execution cleanly as soon after maxRuntime has been exceeded, truncating the run but not exiting with a failure.  By default the value is interpreted in minutes, but this can be changed by maxRuntimeUnits", required = false)
+    public long maxRuntime = GenomeAnalysisEngine.NO_RUNTIME_LIMIT;
+
+    @Argument(fullName = "maxRuntimeUnits", shortName = "maxRuntimeUnits", doc="The TimeUnit for maxRuntime", required = false)
+    public TimeUnit maxRuntimeUnits = TimeUnit.MINUTES;
+
     // --------------------------------------------------------------------------------------------------------------
     //
     // Downsampling Arguments
diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java
index 31f2a469c..cc0161b2d 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java
@@ -123,7 +123,7 @@ public class HierarchicalMicroScheduler extends MicroScheduler implements Hierar
         final ReduceTree reduceTree = new ReduceTree(this);
         initializeWalker(walker);
 
-        while (isShardTraversePending() || isTreeReducePending()) {
+        while (! abortExecution() && (isShardTraversePending() || isTreeReducePending())) {
             // Check for errors during execution.
             errorTracker.throwErrorIfPending();
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java
index 5b94e0767..f3c1ae91c 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java
@@ -63,7 +63,7 @@ public class LinearMicroScheduler extends MicroScheduler {
 
         final TraversalEngine traversalEngine = borrowTraversalEngine(this);
         for (Shard shard : shardStrategy ) {
-            if ( done || shard == null ) // we ran out of shards that aren't owned
+            if ( abortExecution() || done || shard == null ) // we ran out of shards that aren't owned
                 break;
 
             if(shard.getShardType() == Shard.ShardType.LOCUS) {
diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java
index df4ed9ef8..38170040a 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java
@@ -39,6 +39,7 @@ import org.broadinstitute.sting.gatk.iterators.StingSAMIterator;
 import org.broadinstitute.sting.gatk.resourcemanagement.ThreadAllocation;
 import org.broadinstitute.sting.gatk.traversals.*;
 import org.broadinstitute.sting.gatk.walkers.*;
+import org.broadinstitute.sting.utils.AutoFormattingTime;
 import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.broadinstitute.sting.utils.exceptions.UserException;
@@ -52,6 +53,7 @@ import javax.management.ObjectName;
 import java.io.File;
 import java.lang.management.ManagementFactory;
 import java.util.*;
+import java.util.concurrent.TimeUnit;
 
 
 /**
@@ -269,6 +271,26 @@ public abstract class MicroScheduler implements MicroSchedulerMBean {
         this.threadEfficiencyMonitor = threadEfficiencyMonitor;
     }
 
+    /**
+     * Should we stop all execution work and exit gracefully?
+     *
+     * Returns true in the case where some external signal or time limit has been received, indicating
+     * that this GATK shouldn't continue executing.  This isn't a kill signal, it is really a "shutdown
+     * gracefully at the next opportunity" signal.  Concrete implementations of the MicroScheduler
+     * examine this value as often as reasonable and, if it returns true, stop what they are doing
+     * at the next available opportunity, shutdown their resources, call notify done, and return.
+     *
+     * @return true if we should abort execution, or false otherwise
+     */
+    protected boolean abortExecution() {
+        final boolean abort = engine.exceedsRuntimeLimit(progressMeter.getRuntimeInNanoseconds(), TimeUnit.NANOSECONDS);
+        if ( abort ) {
+            final AutoFormattingTime aft = new AutoFormattingTime(TimeUnit.SECONDS.convert(engine.getRuntimeLimitInNanoseconds(), TimeUnit.NANOSECONDS), 1, 4);
+            logger.info("Aborting execution (cleanly) because the runtime has exceeded the requested maximum " + aft);
+        }
+        return abort;
+    }
+
     /**
      * Walks a walker over the given list of intervals.
      *
diff --git a/public/java/src/org/broadinstitute/sting/utils/AutoFormattingTime.java b/public/java/src/org/broadinstitute/sting/utils/AutoFormattingTime.java
index 8964c16cb..4455666e8 100644
--- a/public/java/src/org/broadinstitute/sting/utils/AutoFormattingTime.java
+++ b/public/java/src/org/broadinstitute/sting/utils/AutoFormattingTime.java
@@ -4,12 +4,21 @@ package org.broadinstitute.sting.utils;
  * Simple utility class that makes it convenient to print unit adjusted times
  */
 public class AutoFormattingTime {
-    double timeInSeconds;           // in Seconds
-    int precision;      // for format
+    private final int width; // for format
+    private final int precision;      // for format
 
-    public AutoFormattingTime(double timeInSeconds, int precision) {
+    double timeInSeconds;           // in Seconds
+    private final String formatString;
+
+    public AutoFormattingTime(double timeInSeconds, final int width, int precision) {
+        this.width = width;
         this.timeInSeconds = timeInSeconds;
         this.precision = precision;
+        this.formatString = "%" + width + "." + precision + "f %s";
+    }
+
+    public AutoFormattingTime(double timeInSeconds, int precision) {
+        this(timeInSeconds, 6, precision);
     }
 
     public AutoFormattingTime(double timeInSeconds) {
@@ -20,6 +29,20 @@ public class AutoFormattingTime {
         return timeInSeconds;
     }
 
+    /**
+     * @return the precision (a la format's %WIDTH.PERCISIONf)
+     */
+    public int getWidth() {
+        return width;
+    }
+
+    /**
+     * @return the precision (a la format's %WIDTH.PERCISIONf)
+     */
+    public int getPrecision() {
+        return precision;
+    }
+
     /**
      * Instead of 10000 s, returns 2.8 hours
      * @return
@@ -48,6 +71,6 @@ public class AutoFormattingTime {
             }
         }
 
-        return String.format("%6."+precision+"f %s", unitTime, unit);
+        return String.format(formatString, unitTime, unit);
     }
 }
diff --git a/public/java/src/org/broadinstitute/sting/utils/progressmeter/ProgressMeter.java b/public/java/src/org/broadinstitute/sting/utils/progressmeter/ProgressMeter.java
index 69cf52fc2..a8715e242 100755
--- a/public/java/src/org/broadinstitute/sting/utils/progressmeter/ProgressMeter.java
+++ b/public/java/src/org/broadinstitute/sting/utils/progressmeter/ProgressMeter.java
@@ -24,6 +24,7 @@
 
 package org.broadinstitute.sting.utils.progressmeter;
 
+import com.google.java.contract.Ensures;
 import com.google.java.contract.Invariant;
 import org.apache.log4j.Logger;
 import org.broadinstitute.sting.utils.*;
@@ -200,6 +201,14 @@ public class ProgressMeter {
                 "Location", processingUnitName, processingUnitName));
     }
 
+    /**
+     * @return the current runtime in nanoseconds
+     */
+    @Ensures("result >= 0")
+    public long getRuntimeInNanoseconds() {
+        return timer.getElapsedTimeNano();
+    }
+
     /**
      * Utility routine that prints out process information (including timing) every N records or
      * every M seconds, for N and M set in global variables.
diff --git a/public/java/test/org/broadinstitute/sting/gatk/MaxRuntimeIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/MaxRuntimeIntegrationTest.java
new file mode 100644
index 000000000..6cfd7bf46
--- /dev/null
+++ b/public/java/test/org/broadinstitute/sting/gatk/MaxRuntimeIntegrationTest.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.gatk;
+
+import org.broadinstitute.sting.WalkerTest;
+import org.broadinstitute.sting.utils.SimpleTimer;
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ */
+public class MaxRuntimeIntegrationTest extends WalkerTest {
+    private static final long STARTUP_TIME = TimeUnit.NANOSECONDS.convert(20, TimeUnit.SECONDS);
+
+    private class MaxRuntimeTestProvider extends TestDataProvider {
+        final long maxRuntime;
+        final TimeUnit unit;
+
+        public MaxRuntimeTestProvider(final long maxRuntime, final TimeUnit unit) {
+            super(MaxRuntimeTestProvider.class);
+            this.maxRuntime = maxRuntime;
+            this.unit = unit;
+            setName(String.format("Max runtime test : %d of %s", maxRuntime, unit));
+        }
+
+        public long expectedMaxRuntimeNano() {
+            return TimeUnit.NANOSECONDS.convert(maxRuntime, unit) + STARTUP_TIME;
+        }
+    }
+
+    @DataProvider(name = "MaxRuntimeProvider")
+    public Object[][] makeMaxRuntimeProvider() {
+        for ( final TimeUnit requestedUnits : Arrays.asList(TimeUnit.NANOSECONDS, TimeUnit.MILLISECONDS, TimeUnit.SECONDS, TimeUnit.MINUTES) )
+            new MaxRuntimeTestProvider(requestedUnits.convert(30, TimeUnit.SECONDS), requestedUnits);
+
+        return MaxRuntimeTestProvider.getTests(MaxRuntimeTestProvider.class);
+    }
+
+    //
+    // Loop over errors to throw, make sure they are the errors we get back from the engine, regardless of NT type
+    //
+    @Test(enabled = true, dataProvider = "MaxRuntimeProvider", timeOut = 60 * 1000)
+    public void testMaxRuntime(final MaxRuntimeTestProvider cfg) {
+        WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
+                "-T UnifiedGenotyper -R " + hg18Reference
+                        + " -I " + validationDataLocation + "NA12878.WEx.downsampled20x.bam -o /dev/null"
+                        + " -maxRuntime " + cfg.maxRuntime + " -maxRuntimeUnits " + cfg.unit, 0,
+                Collections.<String>emptyList());
+        final SimpleTimer timer = new SimpleTimer().start();
+        executeTest("Max runtime " + cfg, spec);
+        final long actualRuntimeNano = timer.getElapsedTimeNano();
+
+        Assert.assertTrue(actualRuntimeNano < cfg.expectedMaxRuntimeNano(),
+                "Actual runtime " + TimeUnit.SECONDS.convert(actualRuntimeNano, TimeUnit.NANOSECONDS)
+                        + " exceeded max. tolerated runtime " + TimeUnit.SECONDS.convert(cfg.expectedMaxRuntimeNano(), TimeUnit.NANOSECONDS)
+                        + " given requested runtime " + cfg.maxRuntime + " " + cfg.unit);
+    }
+}
\ No newline at end of file

From a8704ca73f1e9c94430bfeef5541d87df425a573 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Fri, 26 Oct 2012 13:20:27 -0400
Subject: [PATCH 39/40] Adding TODO notes for Ami

---
 .../sting/gatk/walkers/qc/AssessReducedQuals.java         | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java
index e74684c53..78bcf1228 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java
@@ -55,8 +55,8 @@ public class AssessReducedQuals extends LocusWalker<GenomeLoc, GenomeLoc> implem
     @Argument(fullName = "qual_epsilon", shortName = "epsilon", doc = "when |Quals_reduced_bam - Quals_original_bam| > epsilon*Quals_original_bam we output this interval", required = false)
     public int qual_epsilon = 0;
 
-    @Argument(fullName = "debugLevel", shortName = "debug", doc = "debug mode on")
-    public int debugLevel = 0;
+    @Argument(fullName = "debugLevel", shortName = "debug", doc = "debug mode on")   // TODO -- best to make this optional
+    public int debugLevel = 0;   // TODO -- best to make this an enum or boolean
 
     @Output
     protected PrintStream out;
@@ -114,7 +114,11 @@ public class AssessReducedQuals extends LocusWalker<GenomeLoc, GenomeLoc> implem
         return quals;
     }
 
+    // TODO -- arguments/variables should be final, not method declaration
     private final boolean isGoodRead(PileupElement p, List<String> tags){
+        // TODO -- this isn't quite right.  You don't need the tags here.  Instead, you want to check whether the read itself (which
+        // TODO --  you can get from the PileupElement) is a reduced read (not all reads from the reduced bam are reduced) and only
+        // TODO --  for them do you want to ignore that min mapping quality cutoff (but you *do* still want the min base cutoff).
         return !p.isDeletion() && (tags.contains(reduced) || (tags.contains(original) && (int)p.getQual() >= 20 && p.getMappingQual() >= 20));
     }
 

From 35483a7eefc3cf5bd68276a2a6daa8cd7a6f2e6b Mon Sep 17 00:00:00 2001
From: David Roazen <droazen@broadinstitute.org>
Date: Fri, 26 Oct 2012 14:15:16 -0400
Subject: [PATCH 40/40] Update MD5s for PrintReads with BQSR Integration Test

The MD5s for these tests were changed in commit 87435f1074615b2cd016f042980109fd53962c8d
to match the output of a broken version of BaseRecalibration. With the patch in
commit c397102ecc1fd1d2cd8f209a8f358ab4a60b50a7, the output once again matches the
*original* MD5s for these tests, and does not vary as you increase -nct.

Final resolution to GSA-632
---
 .../sting/gatk/walkers/bqsr/BQSRIntegrationTest.java        | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
index a50219f48..780ef9e0b 100644
--- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
+++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
@@ -122,12 +122,12 @@ public class BQSRIntegrationTest extends WalkerTest {
     public Object[][] createPRTestData() {
         List<Object[]> tests = new ArrayList<Object[]>();
 
-        tests.add(new Object[]{1, new PRTest(" -qq -1", "a1d87da5dcbde35170d6ba6bc3ee2812")});
-        tests.add(new Object[]{1, new PRTest(" -qq 6", "a0fecae6d0e5ab9917862fa306186d10")});
+        tests.add(new Object[]{1, new PRTest(" -qq -1", "5226c06237b213b9e9b25a32ed92d09a")});
+        tests.add(new Object[]{1, new PRTest(" -qq 6", "b592a5c62b952a012e18adb898ea9c33")});
         tests.add(new Object[]{1, new PRTest(" -DIQ", "8977bea0c57b808e65e9505eb648cdf7")});
 
         for ( final int nct : Arrays.asList(1, 2, 4) ) {
-            tests.add(new Object[]{nct, new PRTest("", "d1bbb4ce6aa93e866f106f8b11d888ed")});
+            tests.add(new Object[]{nct, new PRTest("", "ab2f209ab98ad3432e208cbd524a4c4a")});
         }
 
         return tests.toArray(new Object[][]{});