From 1da4ad9607a887bf520ceb1d896ca6512166b6bf Mon Sep 17 00:00:00 2001
From: Ami Levy Moonshine <ami@broadinstitute.org>
Date: Mon, 22 Oct 2012 10:45:53 -0400
Subject: [PATCH 01/50] new class to test the quals of reduced bam vs
 non-reduced bam

---
 .../gatk/walkers/qc/AssessReducedQuals.java   | 116 ++++++++++++++++++
 1 file changed, 116 insertions(+)
 create mode 100644 public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java
new file mode 100644
index 000000000..c8ec33364
--- /dev/null
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java
@@ -0,0 +1,116 @@
+package org.broadinstitute.sting.gatk.walkers.qc;
+
+import org.broadinstitute.sting.commandline.Output;
+import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
+import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
+import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
+import org.broadinstitute.sting.gatk.walkers.LocusWalker;
+import org.broadinstitute.sting.gatk.walkers.TreeReducible;
+import org.broadinstitute.sting.utils.GenomeLoc;
+import org.broadinstitute.sting.utils.pileup.PileupElement;
+import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
+
+import java.io.PrintStream;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: ami
+ * Date: 10/19/12
+ * Time: 9:09 AM
+ * To change this template use File | Settings | File Templates.
+ */
+public class AssessReducedQuals extends LocusWalker<GenomeLoc, GenomeLoc> implements TreeReducible<GenomeLoc> {
+
+    private static final String original = "original";
+    private static final String reduced = "reduced";
+
+    @Output
+    protected PrintStream out;
+
+    @Override
+    public boolean includeReadsWithDeletionAtLoci() { return true; }
+
+    public void initialize() {}    //todo: why we need that?
+
+    @Override
+    public GenomeLoc map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
+        if ( tracker == null )
+            return null;
+
+        int originalQualsIndex = 0;
+        int reducedQualsIndex = 1;
+        double epsilon = 0;
+        double[] quals = getPileupQuals(context.getBasePileup());
+        return (quals[originalQualsIndex] - quals[reducedQualsIndex] >= epsilon || quals[originalQualsIndex] - quals[reducedQualsIndex] <= -1*epsilon) ? ref.getLocus() : null;
+
+    }
+
+    private double[] getPileupQuals(final ReadBackedPileup readPileup) {
+
+        int originalQualsIndex = 0;
+        int reducedQualsIndex = 1;
+        double[] quals = new double[2];
+
+        for( PileupElement p : readPileup){
+            if ( (int)p.getQual() > 2 && p.getMappingQual() > 0 && !p.isDeletion() ){
+                if (p.getRead().isReducedRead()){
+                    double tempQual = (double)(p.getQual()) * p.getRepresentativeCount();
+                    quals[reducedQualsIndex] += tempQual;
+                }
+                else
+                {
+                    double tempQual = (double)(p.getQual());
+                    quals[originalQualsIndex] += tempQual;
+                }
+            }
+
+        }
+
+
+        return quals;
+    }
+
+    //public void onTraversalDone(GenomeLoc sum) {
+    //    if ( sum != null )
+    //        out.println(sum);
+    //}
+
+    @Override
+    public GenomeLoc treeReduce(GenomeLoc lhs, GenomeLoc rhs) {
+        if ( lhs == null )
+            return rhs;
+
+        if ( rhs == null )
+            return lhs;
+
+        // if contiguous, just merge them
+        if ( lhs.contiguousP(rhs) )
+            return getToolkit().getGenomeLocParser().createGenomeLoc(lhs.getContig(), lhs.getStart(), rhs.getStop());
+
+        // otherwise, print the lhs and start over with the rhs
+        out.println(lhs);
+        return rhs;
+    }
+
+    @Override
+    public GenomeLoc reduceInit() {
+        return null;
+    }
+
+    @Override
+    public GenomeLoc reduce(GenomeLoc value, GenomeLoc sum) {
+        if ( value == null )
+            return sum;
+
+        if ( sum == null )
+            return value;
+
+        // if contiguous, just merge them
+        if ( sum.contiguousP(value) )
+            return getToolkit().getGenomeLocParser().createGenomeLoc(sum.getContig(), sum.getStart(), value.getStop());
+
+        // otherwise, print the sum and start over with the value
+        out.println(sum);
+        return value;
+    }
+}

From 6e421a72d642b1e2477af3528a2248859332cd0f Mon Sep 17 00:00:00 2001
From: Mark DePristo <depristo@broadinstitute.org>
Date: Tue, 23 Oct 2012 20:10:39 -0400
Subject: [PATCH 02/50] Add more exhaustive unit tests for input errors to
 NanoScheduler

-- Resolves issue GSA-515 / Nanoscheduler GSA-605 / Seems that -nct may deadlock as not reproducible
-- It seems that it's not an input error problem (or at least cannot be provoked with unit tests)
-- I'll keep an eye on this later
---
 .../nanoScheduler/NanoSchedulerUnitTest.java  | 100 +++++++++++++-----
 1 file changed, 73 insertions(+), 27 deletions(-)

diff --git a/public/java/test/org/broadinstitute/sting/utils/nanoScheduler/NanoSchedulerUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/nanoScheduler/NanoSchedulerUnitTest.java
index 2ba2da734..af2e18ad9 100644
--- a/public/java/test/org/broadinstitute/sting/utils/nanoScheduler/NanoSchedulerUnitTest.java
+++ b/public/java/test/org/broadinstitute/sting/utils/nanoScheduler/NanoSchedulerUnitTest.java
@@ -23,6 +23,7 @@ import java.util.List;
  * To change this template use File | Settings | File Templates.
  */
 public class NanoSchedulerUnitTest extends BaseTest {
+    private final static boolean DEBUG = false;
     private final static boolean debug = false;
     public static final int NANO_SCHEDULE_MAX_RUNTIME = 30000;
 
@@ -30,19 +31,22 @@ public class NanoSchedulerUnitTest extends BaseTest {
         @Override public Integer apply(Integer input) { return input * 2; }
     }
 
+    private static void maybeDelayMe(final int input) {
+        try {
+            if ( input % 7 == 0 ) {
+                final int milliToSleep = (input % 10);
+                //System.out.printf("Sleeping %d millseconds%n", milliToSleep);
+                Thread.sleep(milliToSleep);
+            }
+        } catch ( InterruptedException ex ) {
+            throw new RuntimeException(ex);
+        }
+    }
+
     private static class Map2xWithDelays extends Map2x {
         @Override public Integer apply(Integer input) {
-            try {
-                if ( input % 7 == 0 ) {
-                    final int milliToSleep = (input % 10);
-                    //System.out.printf("Sleeping %d millseconds%n", milliToSleep);
-                    Thread.sleep(milliToSleep);
-                }
-
-                return input * 2;
-            } catch ( InterruptedException ex ) {
-                throw new RuntimeException(ex);
-            }
+            maybeDelayMe(input);
+            return input * 2;
         }
     }
 
@@ -117,10 +121,12 @@ public class NanoSchedulerUnitTest extends BaseTest {
     }
 
     static NanoSchedulerBasicTest exampleTest = null;
+    static NanoSchedulerBasicTest exampleTestWithDelays = null;
 
     @BeforeSuite
     public void setUp() throws Exception {
         exampleTest = new NanoSchedulerBasicTest(10, 2, 1, 10, false);
+        exampleTestWithDelays = new NanoSchedulerBasicTest(10, 2, 1, 10, true);
     }
 
     @DataProvider(name = "NanoSchedulerBasicTest")
@@ -151,14 +157,14 @@ public class NanoSchedulerUnitTest extends BaseTest {
         return NanoSchedulerBasicTest.getTests(NanoSchedulerBasicTest.class);
     }
 
-    @Test(enabled = true, dataProvider = "NanoSchedulerBasicTest", timeOut = NANO_SCHEDULE_MAX_RUNTIME)
+    @Test(enabled = true && ! DEBUG, dataProvider = "NanoSchedulerBasicTest", timeOut = NANO_SCHEDULE_MAX_RUNTIME)
     public void testSingleThreadedNanoScheduler(final NanoSchedulerBasicTest test) throws InterruptedException {
         logger.warn("Running " + test);
         if ( test.nThreads == 1 )
             testNanoScheduler(test);
     }
 
-    @Test(enabled = true, dataProvider = "NanoSchedulerBasicTest", timeOut = NANO_SCHEDULE_MAX_RUNTIME, dependsOnMethods = "testSingleThreadedNanoScheduler")
+    @Test(enabled = true && ! DEBUG, dataProvider = "NanoSchedulerBasicTest", timeOut = NANO_SCHEDULE_MAX_RUNTIME, dependsOnMethods = "testSingleThreadedNanoScheduler")
     public void testMultiThreadedNanoScheduler(final NanoSchedulerBasicTest test) throws InterruptedException {
         logger.warn("Running " + test);
         if ( test.nThreads >= 1 )
@@ -195,7 +201,7 @@ public class NanoSchedulerUnitTest extends BaseTest {
         }
     }
 
-    @Test(enabled = true, dataProvider = "NanoSchedulerBasicTest", dependsOnMethods = "testMultiThreadedNanoScheduler", timeOut = NANO_SCHEDULE_MAX_RUNTIME)
+    @Test(enabled = true && ! DEBUG, dataProvider = "NanoSchedulerBasicTest", dependsOnMethods = "testMultiThreadedNanoScheduler", timeOut = NANO_SCHEDULE_MAX_RUNTIME)
     public void testNanoSchedulerInLoop(final NanoSchedulerBasicTest test) throws InterruptedException {
         if ( test.bufferSize > 1) {
             logger.warn("Running " + test);
@@ -213,7 +219,7 @@ public class NanoSchedulerUnitTest extends BaseTest {
         }
     }
 
-    @Test(timeOut = NANO_SCHEDULE_MAX_RUNTIME)
+    @Test(enabled = true && ! DEBUG, timeOut = NANO_SCHEDULE_MAX_RUNTIME)
     public void testShutdown() throws InterruptedException {
         final NanoScheduler<Integer, Integer, Integer> nanoScheduler = new NanoScheduler<Integer, Integer, Integer>(1, 2);
         Assert.assertFalse(nanoScheduler.isShutdown(), "scheduler should be alive");
@@ -221,38 +227,78 @@ public class NanoSchedulerUnitTest extends BaseTest {
         Assert.assertTrue(nanoScheduler.isShutdown(), "scheduler should be dead");
     }
 
-    @Test(expectedExceptions = IllegalStateException.class, timeOut = NANO_SCHEDULE_MAX_RUNTIME)
+    @Test(enabled = true && ! DEBUG, expectedExceptions = IllegalStateException.class, timeOut = NANO_SCHEDULE_MAX_RUNTIME)
     public void testShutdownExecuteFailure() throws InterruptedException {
         final NanoScheduler<Integer, Integer, Integer> nanoScheduler = new NanoScheduler<Integer, Integer, Integer>(1, 2);
         nanoScheduler.shutdown();
         nanoScheduler.execute(exampleTest.makeReader(), exampleTest.makeMap(), exampleTest.initReduce(), exampleTest.makeReduce());
     }
 
-    @Test(expectedExceptions = NullPointerException.class, timeOut = 10000, invocationCount = 50)
+    @DataProvider(name = "NanoSchedulerInputExceptionTest")
+    public Object[][] createNanoSchedulerInputExceptionTest() {
+        List<Object[]> tests = new ArrayList<Object[]>();
+
+
+        for ( final int bufSize : Arrays.asList(100) ) {
+            for ( final int nThreads : Arrays.asList(8) ) {
+                for ( final boolean addDelays : Arrays.asList(true, false) ) {
+                    final NanoSchedulerBasicTest test = new NanoSchedulerBasicTest(bufSize, nThreads, 1, 1000000, false);
+                    final int maxN = addDelays ? 10000 : 100000;
+                    for ( int nElementsBeforeError = 0; nElementsBeforeError < maxN; nElementsBeforeError += Math.max(nElementsBeforeError / 10, 1) ) {
+                        tests.add(new Object[]{nElementsBeforeError, test, addDelays});
+                    }
+                }
+            }
+        }
+
+        return tests.toArray(new Object[][]{});
+    }
+
+    @Test(enabled = true, expectedExceptions = NullPointerException.class, timeOut = 10000)
     public void testInputErrorIsThrown_NPE() throws InterruptedException {
-        executeTestErrorThrowingInput(new NullPointerException());
+        executeTestErrorThrowingInput(10, new NullPointerException(), exampleTest, false);
     }
 
-    @Test(expectedExceptions = ReviewedStingException.class, timeOut = 10000, invocationCount = 50)
+    @Test(enabled = true, expectedExceptions = ReviewedStingException.class, timeOut = 10000)
     public void testInputErrorIsThrown_RSE() throws InterruptedException {
-        executeTestErrorThrowingInput(new ReviewedStingException("test"));
+        executeTestErrorThrowingInput(10, new ReviewedStingException("test"), exampleTest, false);
     }
 
-    private void executeTestErrorThrowingInput(final RuntimeException ex) {
-        final NanoScheduler<Integer, Integer, Integer> nanoScheduler = new NanoScheduler<Integer, Integer, Integer>(1, 2);
-        nanoScheduler.execute(new ErrorThrowingIterator(ex), exampleTest.makeMap(), exampleTest.initReduce(), exampleTest.makeReduce());
+    @Test(enabled = true, expectedExceptions = NullPointerException.class, dataProvider = "NanoSchedulerInputExceptionTest", timeOut = 10000, invocationCount = 1)
+    public void testInputErrorDoesntDeadlock(final int nElementsBeforeError, final NanoSchedulerBasicTest test, final boolean addDelays ) throws InterruptedException {
+        executeTestErrorThrowingInput(nElementsBeforeError, new NullPointerException(), test, addDelays);
+    }
+
+    private void executeTestErrorThrowingInput(final int nElementsBeforeError, final RuntimeException ex, final NanoSchedulerBasicTest test, final boolean addDelays) {
+        logger.warn("executeTestErrorThrowingInput " + nElementsBeforeError + " ex=" + ex + " test=" + test + " addInputDelays=" + addDelays);
+        final NanoScheduler<Integer, Integer, Integer> nanoScheduler = test.makeScheduler();
+        nanoScheduler.execute(new ErrorThrowingIterator(nElementsBeforeError, ex, addDelays), test.makeMap(), test.initReduce(), test.makeReduce());
     }
 
     private static class ErrorThrowingIterator implements Iterator<Integer> {
+        final int nElementsBeforeError;
+        final boolean addDelays;
+        int i = 0;
         final RuntimeException ex;
 
-        private ErrorThrowingIterator(RuntimeException ex) {
+        private ErrorThrowingIterator(final int nElementsBeforeError, RuntimeException ex, boolean addDelays) {
+            this.nElementsBeforeError = nElementsBeforeError;
             this.ex = ex;
+            this.addDelays = addDelays;
         }
 
-        @Override public boolean hasNext() { throw ex; }
-        @Override public Integer next() { throw ex; }
-        @Override public void remove() { throw ex; }
+        @Override public boolean hasNext() { return true; }
+        @Override public Integer next() {
+            if ( i++ > nElementsBeforeError ) {
+                throw ex;
+            } else if ( addDelays ) {
+                maybeDelayMe(i);
+                return i;
+            } else {
+                return i;
+            }
+        }
+        @Override public void remove() { throw new UnsupportedOperationException("x"); }
     }
 
     public static void main(String [ ] args) {

From 5b7b42356b803e691cd864a178f3306bb9de551b Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Wed, 24 Oct 2012 14:02:50 -0400
Subject: [PATCH 03/50] Fix bug in GenotypeAndValidate where it doesn't check
 vc.hasAttribute() before using vc.getAttribute().

---
 .../validation/GenotypeAndValidate.java       | 37 ++++++++++---------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/GenotypeAndValidate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/GenotypeAndValidate.java
index 5beeeb3e6..832c0ede3 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/GenotypeAndValidate.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/GenotypeAndValidate.java
@@ -245,24 +245,21 @@ public class GenotypeAndValidate extends RodWalker<GenotypeAndValidate.CountedDa
     @Argument(fullName="condition_on_depth", shortName="depth", doc="Condition validation on a minimum depth of coverage by the reads", required=false)
     private int minDepth = -1;
 
-    /**
-     * If your VCF or BAM file has more than one sample and you only want to validate one, use this parameter to choose it.
-     */
-    @Hidden
-    @Argument(fullName ="sample", shortName ="sn", doc="Name of the sample to validate (in case your VCF/BAM has more than one sample)", required=false)
-    private String sample = "";
-
-    /**
+   /**
      * Print out discordance sites to standard out.
      */
     @Hidden
     @Argument(fullName ="print_interesting_sites", shortName ="print_interesting", doc="Print out interesting sites to standard out", required=false)
-    private boolean printInterestingSites;
+    private boolean printInterestingSites = false;
 
     private UnifiedGenotyperEngine snpEngine;
     private UnifiedGenotyperEngine indelEngine;
     private Set<String> samples;
 
+    private enum GVstatus {
+        T, F, NONE
+    }
+
     public static class CountedData {
         private long nAltCalledAlt = 0L;
         private long nAltCalledRef = 0L;
@@ -368,9 +365,10 @@ public class GenotypeAndValidate extends RodWalker<GenotypeAndValidate.CountedDa
         // Do not operate on variants that are not covered to the optional minimum depth
         if (!context.hasReads() || (minDepth > 0 && context.getBasePileup().getBases().length < minDepth)) {
             counter.nUncovered = 1L;
-            if (vcComp.getAttribute("GV").equals("T"))
+            final GVstatus status = getGVstatus(vcComp);
+            if ( status == GVstatus.T )
                 counter.nAltNotCalled = 1L;
-            else if (vcComp.getAttribute("GV").equals("F"))
+            else if ( status == GVstatus.F )
                 counter.nRefNotCalled = 1L;
             else
                 counter.nNoStatusNotCalled = 1L;
@@ -427,10 +425,11 @@ public class GenotypeAndValidate extends RodWalker<GenotypeAndValidate.CountedDa
 //            if (!vcComp.hasExtendedAttribute("GV"))
 //                throw new UserException.BadInput("Variant has no GV annotation in the INFO field. " + vcComp.getChr() + ":" + vcComp.getStart());
 
+            final GVstatus status = getGVstatus(vcComp);
             if (call.isCalledAlt(callConf)) {
-                if (vcComp.getAttribute("GV").equals("T"))
+                if ( status == GVstatus.T )
                     counter.nAltCalledAlt = 1L;
-                else if (vcComp.getAttribute("GV").equals("F")) {
+                else if ( status == GVstatus.F ) {
                     counter.nRefCalledAlt = 1L;
                     if ( printInterestingSites )
                         System.out.println("Truth=REF Call=ALT at " + call.getChr() + ":" + call.getStart());
@@ -439,12 +438,12 @@ public class GenotypeAndValidate extends RodWalker<GenotypeAndValidate.CountedDa
                     counter.nNoStatusCalledAlt = 1L;
             }
             else if (call.isCalledRef(callConf)) {
-                if (vcComp.getAttribute("GV").equals("T")) {
+                if ( status == GVstatus.T ) {
                     counter.nAltCalledRef = 1L;
                     if ( printInterestingSites )
                         System.out.println("Truth=ALT Call=REF at " + call.getChr() + ":" + call.getStart());
                 }
-                else if (vcComp.getAttribute("GV").equals("F"))
+                else if ( status == GVstatus.F )
                     counter.nRefCalledRef = 1L;
 
                 else
@@ -452,9 +451,9 @@ public class GenotypeAndValidate extends RodWalker<GenotypeAndValidate.CountedDa
             }
             else {
                 counter.nNotConfidentCalls = 1L;
-                if (vcComp.getAttribute("GV").equals("T"))
+                if ( status == GVstatus.T )
                     counter.nAltNotCalled = 1L;
-                else if (vcComp.getAttribute("GV").equals("F"))
+                else if ( status == GVstatus.F )
                     counter.nRefNotCalled = 1L;
                 else
                     counter.nNoStatusNotCalled = 1L;
@@ -475,6 +474,10 @@ public class GenotypeAndValidate extends RodWalker<GenotypeAndValidate.CountedDa
         return counter;
     }
 
+    private GVstatus getGVstatus(final VariantContext vc) {
+        return ( !vc.hasAttribute("GV") ) ? GVstatus.NONE : (vc.getAttribute("GV").equals("T") ? GVstatus.T : GVstatus.F);
+    }
+
     //---------------------------------------------------------------------------------------------------------------
     //
     // reduce

From 094db7bf2468352811764d4e98f38d637516a577 Mon Sep 17 00:00:00 2001
From: Ryan Poplin <rpoplin@broadinstitute.org>
Date: Wed, 24 Oct 2012 14:07:36 -0400
Subject: [PATCH 04/50] We now require at least 10 samples to merge variants
 into complex events in the HC. Added a new population based bam for the
 complex event integration test.

---
 .../sting/gatk/walkers/haplotypecaller/GenotypingEngine.java  | 2 +-
 .../walkers/haplotypecaller/LikelihoodCalculationEngine.java  | 2 +-
 .../haplotypecaller/HaplotypeCallerIntegrationTest.java       | 4 +---
 3 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/GenotypingEngine.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/GenotypingEngine.java
index 8738def50..d91df82e2 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/GenotypingEngine.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/GenotypingEngine.java
@@ -217,7 +217,7 @@ public class GenotypingEngine {
         }
 
         cleanUpSymbolicUnassembledEvents( haplotypes );
-        if( activeAllelesToGenotype.isEmpty() && haplotypes.get(0).getSampleKeySet().size() >= 3 ) { // if not in GGA mode and have at least 3 samples try to create MNP and complex events by looking at LD structure
+        if( activeAllelesToGenotype.isEmpty() && haplotypes.get(0).getSampleKeySet().size() >= 10 ) { // if not in GGA mode and have at least 10 samples try to create MNP and complex events by looking at LD structure
             mergeConsecutiveEventsBasedOnLD( haplotypes, startPosKeySet, ref, refLoc );
         }
         if( !activeAllelesToGenotype.isEmpty() ) { // we are in GGA mode!
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java
index 62554c4ab..e90d89a8c 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java
@@ -139,7 +139,7 @@ public class LikelihoodCalculationEngine {
     }
 
     private static int computeFirstDifferingPosition( final byte[] b1, final byte[] b2 ) {
-        for( int iii = 0; iii < b1.length && iii < b2.length; iii++ ){
+        for( int iii = 0; iii < b1.length && iii < b2.length; iii++ ) {
             if( b1[iii] != b2[iii] ) {
                 return iii;
             }
diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
index 1d4fad19e..a0849d0c1 100644
--- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
+++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
@@ -42,7 +42,7 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
 
     @Test
     public void testHaplotypeCallerMultiSampleComplex() {
-        HCTestComplexVariants(CEUTRIO_BAM, "", "30598abeeb0b0ae5816ffdbf0c4044fd");
+        HCTestComplexVariants(privateTestDir + "AFR.complex.variants.bam", "", "966d338f423c86a390d685aa6336ec69");
     }
 
     private void HCTestSymbolicVariants(String bam, String args, String md5) {
@@ -94,6 +94,4 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
                 Arrays.asList("79af83432dc4a1768b3ebffffc4d2b8f"));
         executeTest("HC calling on a ReducedRead BAM", spec);
     }
-
-
 }

From a27ee26481e94b182003212944d75001e6a1a4c0 Mon Sep 17 00:00:00 2001
From: Ryan Poplin <rpoplin@broadinstitute.org>
Date: Wed, 24 Oct 2012 14:08:39 -0400
Subject: [PATCH 05/50] updating HC integration test.

---
 .../walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
index a0849d0c1..71f03ea00 100644
--- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
+++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
@@ -21,7 +21,7 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
 
     @Test
     public void testHaplotypeCallerMultiSample() {
-        HCTest(CEUTRIO_BAM, "", "ee866a8694a6f6c77242041275350ab9");
+        HCTest(CEUTRIO_BAM, "", "1d826f852ec1457efbfa7ee828c5783a");
     }
 
     @Test

From 991658acf485c3c7634387eea0dab46018aa6894 Mon Sep 17 00:00:00 2001
From: David Roazen <droazen@broadinstitute.org>
Date: Mon, 22 Oct 2012 13:36:07 -0400
Subject: [PATCH 06/50] BQSR: use more granular locking for concurrency control

-With this change, BQSR performance scales properly by thread rather
 than gaining nothing from additional threads.
-Benefits are seen when using either -nt (HierarchicalMicroScheduler) or -nct
 (NanoScheduler)
-Removes high-level locks in the recalibration engines and NestedIntegerArray
 in favor of maximally-granular locks on and around manipulation of the leaf
 nodes of the NestedIntegerArray.
-NestedIntegerArray now creates all interior nodes upfront rather than on
 the fly to avoid the need for locking during tree traversals. This uses
 more memory in the initial part of BQSR runs, but the BQSR would eventually
 converge to use this memory anyway over the course of a typical run.

IMPORTANT NOTE: This does not mean it's safe to run the old BaseRecalibrator
walker with multiple threads. The BaseRecalibrator walker is and will never be
thread-safe, as it's a LocusWalker that uses read attributes to track
state information. ONLY the newer DelocalizedBaseRecalibrator can be made
thread-safe (and will hopefully be made so in my subsequent commits). This
commit addresses performance, not correctness.
---
 .../bqsr/AdvancedRecalibrationEngine.java     |  71 +++-----
 .../bqsr/StandardRecalibrationEngine.java     | 157 +++++++++++++++---
 .../LoggingNestedIntegerArray.java            |   6 +-
 .../utils/collections/NestedIntegerArray.java |  56 +++++--
 .../utils/threading/ThreadLocalArray.java     |  64 +++++++
 5 files changed, 270 insertions(+), 84 deletions(-)
 create mode 100644 public/java/src/org/broadinstitute/sting/utils/threading/ThreadLocalArray.java

diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/AdvancedRecalibrationEngine.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/AdvancedRecalibrationEngine.java
index b89f68e24..be8357679 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/AdvancedRecalibrationEngine.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/AdvancedRecalibrationEngine.java
@@ -28,26 +28,22 @@ package org.broadinstitute.sting.gatk.walkers.bqsr;
 import org.broadinstitute.sting.utils.recalibration.covariates.Covariate;
 import org.broadinstitute.sting.utils.BaseUtils;
 import org.broadinstitute.sting.utils.classloader.ProtectedPackageSource;
-import org.broadinstitute.sting.utils.collections.NestedIntegerArray;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.recalibration.EventType;
 import org.broadinstitute.sting.utils.recalibration.ReadCovariates;
-import org.broadinstitute.sting.utils.recalibration.RecalDatum;
 import org.broadinstitute.sting.utils.recalibration.RecalibrationTables;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
+import org.broadinstitute.sting.utils.threading.ThreadLocalArray;
 
 public class AdvancedRecalibrationEngine extends StandardRecalibrationEngine implements ProtectedPackageSource {
 
-    // optimizations: don't reallocate an array each time
-    private byte[] tempQualArray;
-    private boolean[] tempErrorArray;
-    private double[] tempFractionalErrorArray;
+    // optimization: only allocate temp arrays once per thread
+    private final ThreadLocal<byte[]> threadLocalTempQualArray = new ThreadLocalArray<byte[]>(EventType.values().length, byte.class);
+    private final ThreadLocal<boolean[]> threadLocalTempErrorArray = new ThreadLocalArray<boolean[]>(EventType.values().length, boolean.class);
+    private final ThreadLocal<double[]> threadLocalTempFractionalErrorArray = new ThreadLocalArray<double[]>(EventType.values().length, double.class);
 
     public void initialize(final Covariate[] covariates, final RecalibrationTables recalibrationTables) {
         super.initialize(covariates, recalibrationTables);
-        tempQualArray = new byte[EventType.values().length];
-        tempErrorArray = new boolean[EventType.values().length];
-        tempFractionalErrorArray = new double[EventType.values().length];
     }
 
     /**
@@ -60,10 +56,13 @@ public class AdvancedRecalibrationEngine extends StandardRecalibrationEngine imp
      * @param refBase       The reference base at this locus
      */
     @Override
-    public synchronized void updateDataForPileupElement(final PileupElement pileupElement, final byte refBase) {
+    public void updateDataForPileupElement(final PileupElement pileupElement, final byte refBase) {
         final int offset = pileupElement.getOffset();
         final ReadCovariates readCovariates = covariateKeySetFrom(pileupElement.getRead());
 
+        byte[] tempQualArray = threadLocalTempQualArray.get();
+        boolean[] tempErrorArray = threadLocalTempErrorArray.get();
+
         tempQualArray[EventType.BASE_SUBSTITUTION.index] = pileupElement.getQual();
         tempErrorArray[EventType.BASE_SUBSTITUTION.index] = !BaseUtils.basesAreEqual(pileupElement.getBase(), refBase);
         tempQualArray[EventType.BASE_INSERTION.index] = pileupElement.getBaseInsertionQual();
@@ -77,40 +76,29 @@ public class AdvancedRecalibrationEngine extends StandardRecalibrationEngine imp
             final byte qual = tempQualArray[eventIndex];
             final boolean isError = tempErrorArray[eventIndex];
 
-            final NestedIntegerArray<RecalDatum> rgRecalTable = recalibrationTables.getReadGroupTable();
-            final RecalDatum rgPreviousDatum = rgRecalTable.get(keys[0], eventIndex);
-            final RecalDatum rgThisDatum = createDatumObject(qual, isError);
-            if (rgPreviousDatum == null) // key doesn't exist yet in the map so make a new bucket and add it
-                rgRecalTable.put(rgThisDatum, keys[0], eventIndex);
-            else
-                rgPreviousDatum.combine(rgThisDatum);
+            // TODO: should this really be combine rather than increment?
+            combineDatumOrPutIfNecessary(recalibrationTables.getReadGroupTable(), qual, isError, keys[0], eventIndex);
 
-            final NestedIntegerArray<RecalDatum> qualRecalTable = recalibrationTables.getQualityScoreTable();
-            final RecalDatum qualPreviousDatum = qualRecalTable.get(keys[0], keys[1], eventIndex);
-            if (qualPreviousDatum == null)
-                qualRecalTable.put(createDatumObject(qual, isError), keys[0], keys[1], eventIndex);
-            else
-                qualPreviousDatum.increment(isError);
+            incrementDatumOrPutIfNecessary(recalibrationTables.getQualityScoreTable(), qual, isError, keys[0], keys[1], eventIndex);
 
             for (int i = 2; i < covariates.length; i++) {
                 if (keys[i] < 0)
                     continue;
-                final NestedIntegerArray<RecalDatum> covRecalTable = recalibrationTables.getTable(i);
-                final RecalDatum covPreviousDatum = covRecalTable.get(keys[0], keys[1], keys[i], eventIndex);
-                if (covPreviousDatum == null)
-                    covRecalTable.put(createDatumObject(qual, isError), keys[0], keys[1], keys[i], eventIndex);
-                else
-                    covPreviousDatum.increment(isError);
+
+                incrementDatumOrPutIfNecessary(recalibrationTables.getTable(i), qual, isError, keys[0], keys[1], keys[i], eventIndex);
             }
         }
     }
 
     @Override
-    public synchronized void updateDataForRead(final GATKSAMRecord read, final boolean[] skip, final double[] snpErrors, final double[] insertionErrors, final double[] deletionErrors ) {
+    public void updateDataForRead(final GATKSAMRecord read, final boolean[] skip, final double[] snpErrors, final double[] insertionErrors, final double[] deletionErrors ) {
         for( int offset = 0; offset < read.getReadBases().length; offset++ ) {
             if( !skip[offset] ) {
                 final ReadCovariates readCovariates = covariateKeySetFrom(read);
 
+                byte[] tempQualArray = threadLocalTempQualArray.get();
+                double[] tempFractionalErrorArray = threadLocalTempFractionalErrorArray.get();
+
                 tempQualArray[EventType.BASE_SUBSTITUTION.index] = read.getBaseQualities()[offset];
                 tempFractionalErrorArray[EventType.BASE_SUBSTITUTION.index] = snpErrors[offset];
                 tempQualArray[EventType.BASE_INSERTION.index] = read.getBaseInsertionQualities()[offset];
@@ -124,30 +112,15 @@ public class AdvancedRecalibrationEngine extends StandardRecalibrationEngine imp
                     final byte qual = tempQualArray[eventIndex];
                     final double isError = tempFractionalErrorArray[eventIndex];
 
-                    final NestedIntegerArray<RecalDatum> rgRecalTable = recalibrationTables.getReadGroupTable();
-                    final RecalDatum rgPreviousDatum = rgRecalTable.get(keys[0], eventIndex);
-                    final RecalDatum rgThisDatum = createDatumObject(qual, isError);
-                    if (rgPreviousDatum == null) // key doesn't exist yet in the map so make a new bucket and add it
-                        rgRecalTable.put(rgThisDatum, keys[0], eventIndex);
-                    else
-                        rgPreviousDatum.combine(rgThisDatum);
+                    combineDatumOrPutIfNecessary(recalibrationTables.getReadGroupTable(), qual, isError, keys[0], eventIndex);
 
-                    final NestedIntegerArray<RecalDatum> qualRecalTable = recalibrationTables.getQualityScoreTable();
-                    final RecalDatum qualPreviousDatum = qualRecalTable.get(keys[0], keys[1], eventIndex);
-                    if (qualPreviousDatum == null)
-                        qualRecalTable.put(createDatumObject(qual, isError), keys[0], keys[1], eventIndex);
-                    else
-                        qualPreviousDatum.increment(1.0, isError);
+                    incrementDatumOrPutIfNecessary(recalibrationTables.getQualityScoreTable(), qual, isError, keys[0], keys[1], eventIndex);
 
                     for (int i = 2; i < covariates.length; i++) {
                         if (keys[i] < 0)
                             continue;
-                        final NestedIntegerArray<RecalDatum> covRecalTable = recalibrationTables.getTable(i);
-                        final RecalDatum covPreviousDatum = covRecalTable.get(keys[0], keys[1], keys[i], eventIndex);
-                        if (covPreviousDatum == null)
-                            covRecalTable.put(createDatumObject(qual, isError), keys[0], keys[1], keys[i], eventIndex);
-                        else
-                            covPreviousDatum.increment(1.0, isError);
+
+                        incrementDatumOrPutIfNecessary(recalibrationTables.getTable(i), qual, isError, keys[0], keys[1], keys[i], eventIndex);
                     }
                 }
             }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/StandardRecalibrationEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/StandardRecalibrationEngine.java
index 4fe9c5323..1f7fbbf42 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/StandardRecalibrationEngine.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/StandardRecalibrationEngine.java
@@ -55,7 +55,7 @@ public class StandardRecalibrationEngine implements RecalibrationEngine, PublicP
      * @param refBase       The reference base at this locus
      */
     @Override
-    public synchronized void updateDataForPileupElement(final PileupElement pileupElement, final byte refBase) {
+    public void updateDataForPileupElement(final PileupElement pileupElement, final byte refBase) {
         final int offset = pileupElement.getOffset();
         final ReadCovariates readCovariates = covariateKeySetFrom(pileupElement.getRead());
 
@@ -65,35 +65,21 @@ public class StandardRecalibrationEngine implements RecalibrationEngine, PublicP
         final int[] keys = readCovariates.getKeySet(offset, EventType.BASE_SUBSTITUTION);
         final int eventIndex = EventType.BASE_SUBSTITUTION.index;
 
-        final NestedIntegerArray<RecalDatum> rgRecalTable = recalibrationTables.getReadGroupTable();
-        final RecalDatum rgPreviousDatum = rgRecalTable.get(keys[0], eventIndex);
-        final RecalDatum rgThisDatum = createDatumObject(qual, isError);
-        if (rgPreviousDatum == null) // key doesn't exist yet in the map so make a new bucket and add it
-            rgRecalTable.put(rgThisDatum, keys[0], eventIndex);
-        else
-            rgPreviousDatum.combine(rgThisDatum);
+        // TODO: should this really be combine rather than increment?
+        combineDatumOrPutIfNecessary(recalibrationTables.getReadGroupTable(), qual, isError, keys[0], eventIndex);
 
-        final NestedIntegerArray<RecalDatum> qualRecalTable = recalibrationTables.getQualityScoreTable();
-        final RecalDatum qualPreviousDatum = qualRecalTable.get(keys[0], keys[1], eventIndex);
-        if (qualPreviousDatum == null)
-            qualRecalTable.put(createDatumObject(qual, isError), keys[0], keys[1], eventIndex);
-        else
-            qualPreviousDatum.increment(isError);
+        incrementDatumOrPutIfNecessary(recalibrationTables.getQualityScoreTable(), qual, isError, keys[0], keys[1], eventIndex);
 
         for (int i = 2; i < covariates.length; i++) {
             if (keys[i] < 0)
                 continue;
-            final NestedIntegerArray<RecalDatum> covRecalTable = recalibrationTables.getTable(i);
-            final RecalDatum covPreviousDatum = covRecalTable.get(keys[0], keys[1], keys[i], eventIndex);
-            if (covPreviousDatum == null)
-                covRecalTable.put(createDatumObject(qual, isError), keys[0], keys[1], keys[i], eventIndex);
-            else
-                covPreviousDatum.increment(isError);
+
+            incrementDatumOrPutIfNecessary(recalibrationTables.getTable(i), qual, isError, keys[0], keys[1], keys[i], eventIndex);
         }
     }
 
     @Override
-    public synchronized void updateDataForRead( final GATKSAMRecord read, final boolean[] skip, final double[] snpErrors, final double[] insertionErrors, final double[] deletionErrors ) {
+    public void updateDataForRead( final GATKSAMRecord read, final boolean[] skip, final double[] snpErrors, final double[] insertionErrors, final double[] deletionErrors ) {
         throw new UnsupportedOperationException("Delocalized BQSR is not available in the GATK-lite version");
     }
 
@@ -121,4 +107,133 @@ public class StandardRecalibrationEngine implements RecalibrationEngine, PublicP
     protected ReadCovariates covariateKeySetFrom(GATKSAMRecord read) {
         return (ReadCovariates) read.getTemporaryAttribute(BaseRecalibrator.COVARS_ATTRIBUTE);
     }
+
+    /**
+     * Increments the RecalDatum at the specified position in the specified table, or put a new item there
+     * if there isn't already one.
+     *
+     * Does this in a thread-safe way WITHOUT being synchronized: relies on the behavior of NestedIntegerArray.put()
+     * to return false if another thread inserts a new item at our position in the middle of our put operation.
+     *
+     * @param table the table that holds/will hold our item
+     * @param qual qual for this event
+     * @param isError error status for this event
+     * @param keys location in table of our item
+     */
+    protected void incrementDatumOrPutIfNecessary( final NestedIntegerArray<RecalDatum> table, final byte qual, final boolean isError, final int... keys ) {
+        final RecalDatum existingDatum = table.get(keys);
+
+        // There are three cases here:
+        //
+        // 1. The original get succeeded (existingDatum != null) because there already was an item in this position.
+        //    In this case we can just increment the existing item.
+        //
+        // 2. The original get failed (existingDatum == null), and we successfully put a new item in this position
+        //    and are done.
+        //
+        // 3. The original get failed (existingDatum == null), AND the put fails because another thread comes along
+        //    in the interim and puts an item in this position. In this case we need to do another get after the
+        //    failed put to get the item the other thread put here and increment it.
+        if ( existingDatum == null ) {
+            // No existing item, try to put a new one
+            if ( ! table.put(createDatumObject(qual, isError), keys) ) {
+                // Failed to put a new item because another thread came along and put an item here first.
+                // Get the newly-put item and increment it (item is guaranteed to exist at this point)
+                table.get(keys).increment(isError);
+            }
+        }
+        else {
+            // Easy case: already an item here, so increment it
+            existingDatum.increment(isError);
+        }
+    }
+
+    /**
+     * Increments the RecalDatum at the specified position in the specified table, or put a new item there
+     * if there isn't already one.
+     *
+     * Does this in a thread-safe way WITHOUT being synchronized: relies on the behavior of NestedIntegerArray.put()
+     * to return false if another thread inserts a new item at our position in the middle of our put operation.
+     *
+     * @param table the table that holds/will hold our item
+     * @param qual qual for this event
+     * @param isError error value for this event
+     * @param keys location in table of our item
+     */
+    protected void incrementDatumOrPutIfNecessary( final NestedIntegerArray<RecalDatum> table, final byte qual, final double isError, final int... keys ) {
+        final RecalDatum existingDatum = table.get(keys);
+
+        if ( existingDatum == null ) {
+            // No existing item, try to put a new one
+            if ( ! table.put(createDatumObject(qual, isError), keys) ) {
+                // Failed to put a new item because another thread came along and put an item here first.
+                // Get the newly-put item and increment it (item is guaranteed to exist at this point)
+                table.get(keys).increment(1.0, isError);
+            }
+        }
+        else {
+            // Easy case: already an item here, so increment it
+            existingDatum.increment(1.0, isError);
+        }
+    }
+
+    /**
+     * Combines the RecalDatum at the specified position in the specified table with a new RecalDatum, or put a
+     * new item there if there isn't already one.
+     *
+     * Does this in a thread-safe way WITHOUT being synchronized: relies on the behavior of NestedIntegerArray.put()
+     * to return false if another thread inserts a new item at our position in the middle of our put operation.
+     *
+     * @param table the table that holds/will hold our item
+     * @param qual qual for this event
+     * @param isError error status for this event
+     * @param keys location in table of our item
+     */
+    protected void combineDatumOrPutIfNecessary( final NestedIntegerArray<RecalDatum> table, final byte qual, final boolean isError, final int... keys ) {
+        final RecalDatum existingDatum = table.get(keys);
+        final RecalDatum newDatum = createDatumObject(qual, isError);
+
+        if ( existingDatum == null ) {
+            // No existing item, try to put a new one
+            if ( ! table.put(newDatum, keys) ) {
+                // Failed to put a new item because another thread came along and put an item here first.
+                // Get the newly-put item and combine it with our item (item is guaranteed to exist at this point)
+                table.get(keys).combine(newDatum);
+            }
+        }
+        else {
+            // Easy case: already an item here, so combine it with our item
+            existingDatum.combine(newDatum);
+        }
+    }
+
+    /**
+     * Combines the RecalDatum at the specified position in the specified table with a new RecalDatum, or put a
+     * new item there if there isn't already one.
+     *
+     * Does this in a thread-safe way WITHOUT being synchronized: relies on the behavior of NestedIntegerArray.put()
+     * to return false if another thread inserts a new item at our position in the middle of our put operation.
+     *
+     * @param table the table that holds/will hold our item
+     * @param qual qual for this event
+     * @param isError error value for this event
+     * @param keys location in table of our item
+     */
+    protected void combineDatumOrPutIfNecessary( final NestedIntegerArray<RecalDatum> table, final byte qual, final double isError, final int... keys ) {
+        final RecalDatum existingDatum = table.get(keys);
+        final RecalDatum newDatum = createDatumObject(qual, isError);
+
+        if ( existingDatum == null ) {
+            // No existing item, try to put a new one
+            if ( ! table.put(newDatum, keys) ) {
+                // Failed to put a new item because another thread came along and put an item here first.
+                // Get the newly-put item and combine it with our item (item is guaranteed to exist at this point)
+                table.get(keys).combine(newDatum);
+            }
+        }
+        else {
+            // Easy case: already an item here, so combine it with our item
+            existingDatum.combine(newDatum);
+        }
+    }
 }
diff --git a/public/java/src/org/broadinstitute/sting/utils/collections/LoggingNestedIntegerArray.java b/public/java/src/org/broadinstitute/sting/utils/collections/LoggingNestedIntegerArray.java
index 6fda0245b..63927ac84 100644
--- a/public/java/src/org/broadinstitute/sting/utils/collections/LoggingNestedIntegerArray.java
+++ b/public/java/src/org/broadinstitute/sting/utils/collections/LoggingNestedIntegerArray.java
@@ -99,9 +99,7 @@ public class LoggingNestedIntegerArray<T> extends NestedIntegerArray<T> {
     }
 
     @Override
-    public void put( final T value, final int... keys ) {
-        super.put(value, keys);
-
+    public boolean put( final T value, final int... keys ) {
         StringBuilder logEntry = new StringBuilder();
 
         logEntry.append(logEntryLabel);
@@ -116,5 +114,7 @@ public class LoggingNestedIntegerArray<T> extends NestedIntegerArray<T> {
 
         // PrintStream methods all use synchronized blocks internally, so our logging is thread-safe
         log.println(logEntry.toString());
+
+        return super.put(value, keys);
     }
 }
diff --git a/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java b/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java
index 31d316555..9f1eb546d 100755
--- a/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java
+++ b/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java
@@ -50,6 +50,28 @@ public class NestedIntegerArray<T> {
         this.dimensions = dimensions.clone();
 
         data = new Object[dimensions[0]];
+        prepopulateArray(data, 0);
+    }
+
+    /**
+     * Recursively allocate the entire tree of arrays in all its dimensions.
+     *
+     * Doing this upfront uses more memory initially, but saves time over the course of the run
+     * and (crucially) avoids having to make threads wait while traversing the tree to check
+     * whether branches exist or not.
+     *
+     * @param subarray current node in the tree
+     * @param dimension current level in the tree
+     */
+    private void prepopulateArray( Object[] subarray, int dimension ) {
+        if ( dimension >= numDimensions - 1 ) {
+            return;
+        }
+
+        for ( int i = 0; i < subarray.length; i++ ) {
+            subarray[i] = new Object[dimensions[dimension + 1]];
+            prepopulateArray((Object[])subarray[i], dimension + 1);
+        }
     }
 
     public T get(final int... keys) {
@@ -59,14 +81,15 @@ public class NestedIntegerArray<T> {
         for( int i = 0; i < numNestedDimensions; i++ ) {
             if ( keys[i] >= dimensions[i] )
                 return null;
-            myData = (Object[])myData[keys[i]];
-            if ( myData == null )
-                return null;
+
+            myData = (Object[])myData[keys[i]];  // interior nodes in the tree will never be null, so we can safely traverse
+                                                 // down to the leaves
         }
+
         return (T)myData[keys[numNestedDimensions]];
     }
 
-    public synchronized void put(final T value, final int... keys) { // WARNING! value comes before the keys!
+    public boolean put(final T value, final int... keys) { // WARNING! value comes before the keys!
         if ( keys.length != numDimensions )
             throw new ReviewedStingException("Exactly " + numDimensions + " keys should be passed to this NestedIntegerArray but " + keys.length + " were provided");
 
@@ -75,15 +98,26 @@ public class NestedIntegerArray<T> {
         for ( int i = 0; i < numNestedDimensions; i++ ) {
             if ( keys[i] >= dimensions[i] )
                 throw new ReviewedStingException("Key " + keys[i] + " is too large for dimension " + i + " (max is " + (dimensions[i]-1) + ")");
-            Object[] temp = (Object[])myData[keys[i]];
-            if ( temp == null ) {
-                temp = new Object[dimensions[i+1]];
-                myData[keys[i]] = temp;
-            }
-            myData = temp;
+
+            myData = (Object[])myData[keys[i]];  // interior nodes in the tree will never be null, so we can safely traverse
+                                                 // down to the leaves
         }
 
-        myData[keys[numNestedDimensions]] = value;
+        synchronized ( myData ) {   // lock the bottom row while we examine and (potentially) update it
+
+            // Insert the new value only if there still isn't any existing value in this position
+            if ( myData[keys[numNestedDimensions]] == null ) {
+                myData[keys[numNestedDimensions]] = value;
+            }
+            else {
+                // Already have a value for this leaf (perhaps another thread came along and inserted one
+                // while we traversed the tree), so return false to notify the caller that we didn't put
+                // the item
+                return false;
+            }
+        }
+
+        return true;
     }
 
     public List<T> getAllValues() {
diff --git a/public/java/src/org/broadinstitute/sting/utils/threading/ThreadLocalArray.java b/public/java/src/org/broadinstitute/sting/utils/threading/ThreadLocalArray.java
new file mode 100644
index 000000000..cc50152ac
--- /dev/null
+++ b/public/java/src/org/broadinstitute/sting/utils/threading/ThreadLocalArray.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2012, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.threading;
+
+import java.lang.reflect.Array;
+
+/**
+ * ThreadLocal implementation for arrays
+ *
+ * Example usage:
+ *
+ * private ThreadLocal<byte[]> threadLocalByteArray = new ThreadLocalArray<byte[]>(length, byte.class);
+ * ....
+ * byte[] byteArray = threadLocalByteArray.get();
+ *
+ * @param <T> the type of the array itself (eg., int[], double[], etc.)
+ *
+ * @author David Roazen
+ */
+public class ThreadLocalArray<T> extends ThreadLocal<T> {
+    private int arraySize;
+    private Class arrayElementType;
+
+    /**
+     * Create a new ThreadLocalArray
+     *
+     * @param arraySize desired length of the array
+     * @param arrayElementType type of the elements within the array (eg., Byte.class, Integer.class, etc.)
+     */
+    public ThreadLocalArray( int arraySize, Class arrayElementType ) {
+        super();
+
+        this.arraySize = arraySize;
+        this.arrayElementType = arrayElementType;
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    protected T initialValue() {
+        return (T)Array.newInstance(arrayElementType, arraySize);
+    }
+}

From 32a6d7000a4dab5f208c9d5e78f35b314edbaec3 Mon Sep 17 00:00:00 2001
From: David Roazen <droazen@broadinstitute.org>
Date: Wed, 24 Oct 2012 13:06:34 -0400
Subject: [PATCH 07/50] Thread-safe ReadGroupCovariate

The ReadGroupCovariate class was not thread-safe. This led to horrible race conditions
in multithreaded runs of the BQSR where (for example) the same read group could get
inserted into the reverse lookup table twice with different IDs.

Should fix the intermittent crash reported in GSA-492.
---
 .../covariates/ReadGroupCovariate.java        | 29 +++++++++++++++----
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/utils/recalibration/covariates/ReadGroupCovariate.java b/public/java/src/org/broadinstitute/sting/utils/recalibration/covariates/ReadGroupCovariate.java
index 85568dac9..29c15adf7 100755
--- a/public/java/src/org/broadinstitute/sting/utils/recalibration/covariates/ReadGroupCovariate.java
+++ b/public/java/src/org/broadinstitute/sting/utils/recalibration/covariates/ReadGroupCovariate.java
@@ -66,7 +66,9 @@ public class ReadGroupCovariate implements RequiredCovariate {
     }
 
     @Override
-    public String formatKey(final int key) {
+    public synchronized String formatKey(final int key) {
+        // This method is synchronized so that we don't attempt to do a get()
+        // from the reverse lookup table while that table is being updated
         return readGroupReverseLookupTable.get(key);
     }
 
@@ -76,17 +78,32 @@ public class ReadGroupCovariate implements RequiredCovariate {
     }
 
     private int keyForReadGroup(final String readGroupId) {
-        if (!readGroupLookupTable.containsKey(readGroupId)) {
-            readGroupLookupTable.put(readGroupId, nextId);
-            readGroupReverseLookupTable.put(nextId, readGroupId);
-            nextId++;
+        // Rather than synchronize this entire method (which would be VERY expensive for walkers like the BQSR),
+        // synchronize only the table updates.
+
+        // Before entering the synchronized block, check to see if this read group is not in our tables.
+        // If it's not, either we will have to insert it, OR another thread will insert it first.
+        // This preliminary check avoids doing any synchronization most of the time.
+        if ( ! readGroupLookupTable.containsKey(readGroupId) ) {
+
+            synchronized ( this ) {
+
+                // Now we need to make sure the key is STILL not there, since another thread may have come along
+                // and inserted it while we were waiting to enter this synchronized block!
+                if ( ! readGroupLookupTable.containsKey(readGroupId) ) {
+                    readGroupLookupTable.put(readGroupId, nextId);
+                    readGroupReverseLookupTable.put(nextId, readGroupId);
+                    nextId++;
+                }
+            }
         }
 
         return readGroupLookupTable.get(readGroupId);
     }
 
     @Override
-    public int maximumKeyValue() {
+    public synchronized int maximumKeyValue() {
+        // Synchronized so that we don't query table size while the tables are being updated
         return readGroupLookupTable.size() - 1;
     }
 

From 02018ca76495b1405a3b1ec4086afdb470ef0b95 Mon Sep 17 00:00:00 2001
From: David Roazen <droazen@broadinstitute.org>
Date: Wed, 24 Oct 2012 15:06:13 -0400
Subject: [PATCH 08/50] Legacy BaseRecalibrator walker is neither TreeReducible
 nor NanoSchedulable

The old BaseRecalibrator walker is and never will be thread-safe, since it's a
LocusWalker that uses read attributes to track state.

ONLY the newer DelocalizedBaseRecalibrator is believed likely to be thread-safe
at this point. It is safe to run the DelocalizedBaseRecalibrator with -nct > 1
for testing purposes, but wait for further testing to be done before using it
for production purposes in multithreaded mode.
---
 .../sting/gatk/walkers/bqsr/BQSRIntegrationTest.java       | 5 -----
 .../sting/gatk/walkers/bqsr/BaseRecalibrator.java          | 7 +------
 2 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
index 26ee78484..90d67d393 100644
--- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
+++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
@@ -73,11 +73,6 @@ public class BQSRIntegrationTest extends WalkerTest {
                 params.getCommandLine(),
                 Arrays.asList(params.md5));
         executeTest("testBQSR-"+params.args, spec).getFirst();
-
-        WalkerTestSpec specNT2 = new WalkerTestSpec(
-                params.getCommandLine() + " -nt 2",
-                Arrays.asList(params.md5));
-        executeTest("testBQSR-nt2-"+params.args, specNT2).getFirst();
     }
 
     @Test
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/BaseRecalibrator.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/BaseRecalibrator.java
index 2c7a12a36..245c8e3e8 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/BaseRecalibrator.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/BaseRecalibrator.java
@@ -109,7 +109,7 @@ import java.util.ArrayList;
 @ReadFilters({MappingQualityZeroFilter.class, MappingQualityUnavailableFilter.class}) // only look at covered loci, not every loci of the reference file
 @Requires({DataSource.READS, DataSource.REFERENCE}) // filter out all reads with zero or unavailable mapping quality
 @PartitionBy(PartitionType.LOCUS) // this walker requires both -I input.bam and -R reference.fasta
-public class BaseRecalibrator extends LocusWalker<Long, Long> implements TreeReducible<Long>, NanoSchedulable {
+public class BaseRecalibrator extends LocusWalker<Long, Long> {
 
     @ArgumentCollection
     private final RecalibrationArgumentCollection RAC = new RecalibrationArgumentCollection(); // all the command line arguments for BQSR and it's covariates
@@ -277,11 +277,6 @@ public class BaseRecalibrator extends LocusWalker<Long, Long> implements TreeRed
         return sum;
     }
 
-    public Long treeReduce(Long sum1, Long sum2) {
-        sum1 += sum2;
-        return sum1;
-    }
-
     @Override
     public void onTraversalDone(Long result) {
         logger.info("Calculating quantized quality scores...");

From d9aa9855f874f80ddc629c3c7179c00108991c9e Mon Sep 17 00:00:00 2001
From: David Roazen <droazen@broadinstitute.org>
Date: Wed, 24 Oct 2012 15:29:13 -0400
Subject: [PATCH 09/50] Better comments in NestedIntegerArray

---
 .../utils/collections/NestedIntegerArray.java      | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java b/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java
index 9f1eb546d..c83de8ec7 100755
--- a/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java
+++ b/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java
@@ -89,6 +89,20 @@ public class NestedIntegerArray<T> {
         return (T)myData[keys[numNestedDimensions]];
     }
 
+    /**
+     * Insert a value at the position specified by the given keys.
+     *
+     * This method is THREAD-SAFE despite not being synchronized, however the caller MUST
+     * check the return value to see if the put succeeded. This method RETURNS FALSE if
+     * the value could not be inserted because there already was a value present
+     * at the specified location. In this case the caller should do a get() to get
+     * the already-existing value and (potentially) update it.
+     *
+     * @param value value to insert
+     * @param keys keys specifying the location of the value in the tree
+     * @return true if the value was inserted, false if it could not be inserted because there was already
+     *         a value at the specified position
+     */
     public boolean put(final T value, final int... keys) { // WARNING! value comes before the keys!
         if ( keys.length != numDimensions )
             throw new ReviewedStingException("Exactly " + numDimensions + " keys should be passed to this NestedIntegerArray but " + keys.length + " were provided");

From 9da7bbf689dd63ed6d3aed4a10b7eea1f48ace63 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Wed, 24 Oct 2012 15:49:07 -0400
Subject: [PATCH 10/50] Refactoring the PerReadAlleleLikelihoodMap in
 preparation for adding contntamination downsampling into protected only.

---
 .../gatk/walkers/genotyper/ErrorModel.java    |  3 +-
 ...dyGenotypeLikelihoodsCalculationModel.java |  6 +--
 ...GeneralPloidyIndelGenotypeLikelihoods.java |  4 +-
 ...elGenotypeLikelihoodsCalculationModel.java |  2 +-
 ...NPGenotypeLikelihoodsCalculationModel.java |  2 +-
 .../haplotypecaller/HaplotypeCaller.java      |  2 +-
 .../LikelihoodCalculationEngine.java          |  4 +-
 .../AdvancedPerReadAlleleLikelihoodMap.java   | 40 +++++++++++++++++
 .../gatk/walkers/annotator/AlleleBalance.java |  2 +-
 .../annotator/AlleleBalanceBySample.java      |  2 +-
 .../gatk/walkers/annotator/BaseCounts.java    |  2 +-
 .../annotator/BaseQualityRankSumTest.java     |  5 +--
 .../walkers/annotator/ChromosomeCounts.java   |  4 +-
 .../annotator/ClippingRankSumTest.java        |  2 +-
 .../walkers/annotator/DepthOfCoverage.java    |  2 +-
 .../annotator/DepthPerAlleleBySample.java     |  2 +-
 .../gatk/walkers/annotator/FisherStrand.java  |  2 +-
 .../gatk/walkers/annotator/GCContent.java     |  2 +-
 .../walkers/annotator/HaplotypeScore.java     |  3 +-
 .../gatk/walkers/annotator/HardyWeinberg.java |  2 +-
 .../walkers/annotator/HomopolymerRun.java     |  2 +-
 .../walkers/annotator/InbreedingCoeff.java    |  4 +-
 .../gatk/walkers/annotator/IndelType.java     |  2 +-
 .../sting/gatk/walkers/annotator/LowMQ.java   |  2 +-
 .../walkers/annotator/MVLikelihoodRatio.java  |  2 +-
 .../annotator/MappingQualityRankSumTest.java  |  5 +--
 .../walkers/annotator/MappingQualityZero.java |  3 +-
 .../annotator/MappingQualityZeroBySample.java |  2 +-
 .../annotator/MappingQualityZeroFraction.java |  2 +-
 .../gatk/walkers/annotator/NBaseCount.java    |  2 +-
 .../gatk/walkers/annotator/QualByDepth.java   |  4 +-
 .../walkers/annotator/RMSMappingQuality.java  |  4 +-
 .../gatk/walkers/annotator/RankSumTest.java   |  4 +-
 .../walkers/annotator/ReadPosRankSumTest.java |  2 +-
 .../gatk/walkers/annotator/SampleList.java    |  2 +-
 .../sting/gatk/walkers/annotator/SnpEff.java  |  2 +-
 .../walkers/annotator/SpanningDeletions.java  |  2 +-
 .../annotator/TandemRepeatAnnotator.java      |  2 +-
 .../annotator/TechnologyComposition.java      |  2 +-
 .../TransmissionDisequilibriumTest.java       |  2 +-
 .../annotator/VariantAnnotatorEngine.java     |  2 +-
 .../ActiveRegionBasedAnnotation.java          |  4 +-
 .../interfaces/GenotypeAnnotation.java        |  2 +-
 .../interfaces/InfoFieldAnnotation.java       |  2 +-
 .../GenotypeLikelihoodsCalculationModel.java  |  2 +-
 ...elGenotypeLikelihoodsCalculationModel.java |  4 +-
 ...NPGenotypeLikelihoodsCalculationModel.java |  2 +-
 .../genotyper/UnifiedGenotyperEngine.java     | 12 ++---
 .../indels/PairHMMIndelErrorModel.java        |  3 +-
 .../genotyper/PerReadAlleleLikelihoodMap.java | 44 ++++++++++---------
 .../StandardPerReadAlleleLikelihoodMap.java   | 44 +++++++++++++++++++
 51 files changed, 165 insertions(+), 101 deletions(-)
 create mode 100644 protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
 rename public/java/src/org/broadinstitute/sting/{gatk/walkers => utils}/genotyper/PerReadAlleleLikelihoodMap.java (76%)
 create mode 100644 public/java/src/org/broadinstitute/sting/utils/genotyper/StandardPerReadAlleleLikelihoodMap.java

diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ErrorModel.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ErrorModel.java
index 8042c15d8..a01973ad5 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ErrorModel.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ErrorModel.java
@@ -1,7 +1,6 @@
 package org.broadinstitute.sting.gatk.walkers.genotyper;
 
 import com.google.java.contract.Requires;
-import org.apache.commons.lang.ArrayUtils;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.walkers.indels.PairHMMIndelErrorModel;
 import org.broadinstitute.sting.utils.Haplotype;
@@ -60,7 +59,7 @@ public class ErrorModel  {
 
         boolean hasCalledAlleles = false;
 
-        final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap = new PerReadAlleleLikelihoodMap();
+        final org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap = org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap.getBestAvailablePerReadAlleleLikelihoodMap();
         if (refSampleVC != null) {
 
             for (Allele allele : refSampleVC.getAlleles()) {
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyGenotypeLikelihoodsCalculationModel.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyGenotypeLikelihoodsCalculationModel.java
index 2522fc16e..f6ad445c7 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyGenotypeLikelihoodsCalculationModel.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyGenotypeLikelihoodsCalculationModel.java
@@ -195,7 +195,7 @@ public abstract class GeneralPloidyGenotypeLikelihoodsCalculationModel extends G
                                          final List<Allele> allAllelesToUse,
                                          final boolean useBAQedPileup,
                                          final GenomeLocParser locParser,
-                                         final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+                                         final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
 
         HashMap<String, ErrorModel> perLaneErrorModels = getPerLaneErrorModels(tracker, ref, contexts);
         if (perLaneErrorModels == null && UAC.referenceSampleName != null)
@@ -231,7 +231,7 @@ public abstract class GeneralPloidyGenotypeLikelihoodsCalculationModel extends G
             ReadBackedPileup pileup = AlignmentContextUtils.stratify(sample.getValue(), contextType).getBasePileup();
             if (!perReadAlleleLikelihoodMap.containsKey(sample.getKey())){
                 // no likelihoods have been computed for this sample at this site
-                perReadAlleleLikelihoodMap.put(sample.getKey(), new PerReadAlleleLikelihoodMap());
+                perReadAlleleLikelihoodMap.put(sample.getKey(), org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap.getBestAvailablePerReadAlleleLikelihoodMap());
             }
 
             // create the GenotypeLikelihoods object
@@ -333,7 +333,7 @@ public abstract class GeneralPloidyGenotypeLikelihoodsCalculationModel extends G
                                                                                final boolean useBQAedPileup,
                                                                                final ReferenceContext ref,
                                                                                final boolean ignoreLaneInformation,
-                                                                               final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap);
+                                                                               final org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap);
 
     protected abstract List<Allele> getInitialAllelesToUse(final RefMetaDataTracker tracker,
                                                            final ReferenceContext ref,
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyIndelGenotypeLikelihoods.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyIndelGenotypeLikelihoods.java
index afbd49a08..4bcaa5ff9 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyIndelGenotypeLikelihoods.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyIndelGenotypeLikelihoods.java
@@ -27,7 +27,7 @@ public class GeneralPloidyIndelGenotypeLikelihoods extends GeneralPloidyGenotype
     double[][] readHaplotypeLikelihoods;
 
     final byte refBase;
-    final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap;
+    final org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap;
 
     public GeneralPloidyIndelGenotypeLikelihoods(final List<Allele> alleles,
                                                  final double[] logLikelihoods,
@@ -37,7 +37,7 @@ public class GeneralPloidyIndelGenotypeLikelihoods extends GeneralPloidyGenotype
                                                  final PairHMMIndelErrorModel pairModel,
                                                  final LinkedHashMap<Allele, Haplotype> haplotypeMap,
                                                  final ReferenceContext referenceContext,
-                                                 final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap) {
+                                                 final org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap) {
         super(alleles, logLikelihoods, ploidy, perLaneErrorModels, ignoreLaneInformation);
         this.pairModel = pairModel;
         this.haplotypeMap = haplotypeMap;
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyIndelGenotypeLikelihoodsCalculationModel.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyIndelGenotypeLikelihoodsCalculationModel.java
index f09a1ea3e..eb4cf1839 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyIndelGenotypeLikelihoodsCalculationModel.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidyIndelGenotypeLikelihoodsCalculationModel.java
@@ -74,7 +74,7 @@ public class GeneralPloidyIndelGenotypeLikelihoodsCalculationModel extends Gener
                                                                                final boolean useBQAedPileup,
                                                                                final ReferenceContext ref,
                                                                                final boolean ignoreLaneInformation,
-                                                                               final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap){
+                                                                               final org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap){
         return new GeneralPloidyIndelGenotypeLikelihoods(alleles, logLikelihoods, ploidy,perLaneErrorModels,ignoreLaneInformation, pairModel, haplotypeMap, ref, perReadAlleleLikelihoodMap);
     }
 
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidySNPGenotypeLikelihoodsCalculationModel.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidySNPGenotypeLikelihoodsCalculationModel.java
index 4376ec601..9f2fdc096 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidySNPGenotypeLikelihoodsCalculationModel.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GeneralPloidySNPGenotypeLikelihoodsCalculationModel.java
@@ -50,7 +50,7 @@ public class GeneralPloidySNPGenotypeLikelihoodsCalculationModel extends General
                                                                                final boolean useBQAedPileup,
                                                                                final ReferenceContext ref,
                                                                                final boolean ignoreLaneInformation,
-                                                                               final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap){
+                                                                               final org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap){
         return new GeneralPloidySNPGenotypeLikelihoods(alleles, null, UAC.samplePloidy, perLaneErrorModels, useBQAedPileup, UAC.IGNORE_LANE_INFO);
     }
 
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java
index a08614deb..3293ac817 100755
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java
@@ -424,7 +424,7 @@ public class HaplotypeCaller extends ActiveRegionWalker<Integer, Integer> implem
                   : genotypingEngine.assignGenotypeLikelihoodsAndCallIndependentEvents( UG_engine, bestHaplotypes, fullReferenceWithPadding, getPaddedLoc(activeRegion), activeRegion.getLocation(), getToolkit().getGenomeLocParser(), activeAllelesToGenotype ) ) ) {
             if( DEBUG ) { System.out.println(callResult.getFirst().toStringWithoutGenotypes()); }
 
-            final Map<String, PerReadAlleleLikelihoodMap> stratifiedReadMap = LikelihoodCalculationEngine.partitionReadsBasedOnLikelihoods( getToolkit().getGenomeLocParser(), perSampleReadList, perSampleFilteredReadList, callResult );
+            final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> stratifiedReadMap = LikelihoodCalculationEngine.partitionReadsBasedOnLikelihoods( getToolkit().getGenomeLocParser(), perSampleReadList, perSampleFilteredReadList, callResult );
             final VariantContext annotatedCall = annotationEngine.annotateContext(stratifiedReadMap, callResult.getFirst());
             final Map<String, Object> myAttributes = new LinkedHashMap<String, Object>(annotatedCall.getAttributes());
 
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java
index 62554c4ab..a05e20bd3 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java
@@ -27,7 +27,7 @@ package org.broadinstitute.sting.gatk.walkers.haplotypecaller;
 
 import com.google.java.contract.Ensures;
 import com.google.java.contract.Requires;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.*;
 import org.broadinstitute.sting.utils.collections.Pair;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
@@ -350,7 +350,7 @@ public class LikelihoodCalculationEngine {
         final Map<String, PerReadAlleleLikelihoodMap> returnMap = new HashMap<String, PerReadAlleleLikelihoodMap>();
         final GenomeLoc callLoc = parser.createGenomeLoc(call.getFirst());
         for( final Map.Entry<String, ArrayList<GATKSAMRecord>> sample : perSampleReadList.entrySet() ) {
-            final PerReadAlleleLikelihoodMap likelihoodMap = new PerReadAlleleLikelihoodMap();
+            final PerReadAlleleLikelihoodMap likelihoodMap = PerReadAlleleLikelihoodMap.getBestAvailablePerReadAlleleLikelihoodMap();
 
             final ArrayList<GATKSAMRecord> readsForThisSample = sample.getValue();
             for( int iii = 0; iii < readsForThisSample.size(); iii++ ) {
diff --git a/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java b/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
new file mode 100644
index 000000000..db1bf8762
--- /dev/null
+++ b/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2011 The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+ * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package org.broadinstitute.sting.utils.genotyper;
+
+
+import org.broadinstitute.sting.utils.classloader.ProtectedPackageSource;
+import org.broadinstitute.sting.utils.pileup.PileupElement;
+import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
+import org.broadinstitute.sting.utils.variantcontext.Allele;
+
+import java.util.*;
+
+public class AdvancedPerReadAlleleLikelihoodMap extends StandardPerReadAlleleLikelihoodMap implements ProtectedPackageSource {
+
+    public PerReadAlleleLikelihoodMap createPerAlleleDownsampledMap(final double downsamplingFraction) {
+        return this;
+    }
+ }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalance.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalance.java
index a68f0df21..18bdb02ed 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalance.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalance.java
@@ -30,7 +30,7 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalanceBySample.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalanceBySample.java
index 1e1f65333..4d79c4112 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalanceBySample.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalanceBySample.java
@@ -6,7 +6,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ExperimentalAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.GenotypeAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFFormatHeaderLine;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseCounts.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseCounts.java
index 3cbca4f52..aef3e49cf 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseCounts.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseCounts.java
@@ -36,7 +36,7 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.BaseUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java
index 577b1cfdc..e59fc827d 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java
@@ -1,14 +1,11 @@
 package org.broadinstitute.sting.gatk.walkers.annotator;
 
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.IndelGenotypeLikelihoodsCalculationModel;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
-import org.broadinstitute.sting.utils.BaseUtils;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
-import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
 import java.util.*;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ChromosomeCounts.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ChromosomeCounts.java
index 4ae1a0bba..0c78c0204 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ChromosomeCounts.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ChromosomeCounts.java
@@ -34,13 +34,11 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ActiveRegionBa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLine;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFStandardHeaderLines;
-import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
-import org.broadinstitute.sting.utils.variantcontext.Allele;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 import org.broadinstitute.sting.utils.variantcontext.VariantContextUtils;
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ClippingRankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ClippingRankSumTest.java
index c74f98ca3..1dff4d1a3 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ClippingRankSumTest.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ClippingRankSumTest.java
@@ -1,6 +1,6 @@
 package org.broadinstitute.sting.gatk.walkers.annotator;
 
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthOfCoverage.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthOfCoverage.java
index 1cc88fc24..c9481f244 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthOfCoverage.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthOfCoverage.java
@@ -7,7 +7,7 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ActiveRegionBa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFStandardHeaderLines;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthPerAlleleBySample.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthPerAlleleBySample.java
index d7790969e..89a239e54 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthPerAlleleBySample.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthPerAlleleBySample.java
@@ -6,7 +6,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.GenotypeAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFFormatHeaderLine;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFStandardHeaderLines;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java
index ec0393cdc..bdf7baec9 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java
@@ -32,7 +32,7 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ActiveRegionBa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.QualityUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/GCContent.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/GCContent.java
index 3fe5c5837..07391c78c 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/GCContent.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/GCContent.java
@@ -7,7 +7,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ExperimentalAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.BaseUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java
index 6487eac50..ca7180510 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java
@@ -32,8 +32,7 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ActiveRegionBa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.IndelGenotypeLikelihoodsCalculationModel;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.BaseUtils;
 import org.broadinstitute.sting.utils.Haplotype;
 import org.broadinstitute.sting.utils.MathUtils;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HardyWeinberg.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HardyWeinberg.java
index 06fa04526..0340f457c 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HardyWeinberg.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HardyWeinberg.java
@@ -7,7 +7,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.WorkInProgressAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.QualityUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HomopolymerRun.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HomopolymerRun.java
index 5891cbc69..037b357ae 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HomopolymerRun.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HomopolymerRun.java
@@ -5,7 +5,7 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.GenomeLoc;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/InbreedingCoeff.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/InbreedingCoeff.java
index 9a4de3c36..dd058b469 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/InbreedingCoeff.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/InbreedingCoeff.java
@@ -8,12 +8,10 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ActiveRegionBa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
-import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
-import org.broadinstitute.sting.utils.variantcontext.Allele;
 import org.broadinstitute.sting.utils.variantcontext.Genotype;
 import org.broadinstitute.sting.utils.variantcontext.GenotypesContext;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/IndelType.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/IndelType.java
index 5f405cb46..c67d829c2 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/IndelType.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/IndelType.java
@@ -6,7 +6,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ExperimentalAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.IndelUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/LowMQ.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/LowMQ.java
index 4be601bc8..c9a4d0ee6 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/LowMQ.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/LowMQ.java
@@ -5,7 +5,7 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MVLikelihoodRatio.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MVLikelihoodRatio.java
index 85f61c91c..c9d5ca261 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MVLikelihoodRatio.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MVLikelihoodRatio.java
@@ -9,7 +9,7 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ExperimentalAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.RodRequiringAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.MendelianViolation;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityRankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityRankSumTest.java
index 787c9b29b..82596a501 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityRankSumTest.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityRankSumTest.java
@@ -1,14 +1,11 @@
 package org.broadinstitute.sting.gatk.walkers.annotator;
 
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.IndelGenotypeLikelihoodsCalculationModel;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
-import org.broadinstitute.sting.utils.QualityUtils;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
-import org.broadinstitute.sting.utils.sam.AlignmentUtils;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZero.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZero.java
index 74f9c0d0c..364bbdbb9 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZero.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZero.java
@@ -7,14 +7,13 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ActiveRegionBa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFStandardHeaderLines;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
-import org.broadinstitute.sting.utils.variantcontext.Allele;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
 import java.util.Arrays;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroBySample.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroBySample.java
index 44657a7e7..afb4ceb60 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroBySample.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroBySample.java
@@ -30,7 +30,7 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.GenotypeAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFFormatHeaderLine;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroFraction.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroFraction.java
index 21ee66ea2..5f9f3416d 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroFraction.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroFraction.java
@@ -6,7 +6,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ExperimentalAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/NBaseCount.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/NBaseCount.java
index 8e4edaf0e..3e6aa62a2 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/NBaseCount.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/NBaseCount.java
@@ -5,7 +5,7 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.BaseUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/QualByDepth.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/QualByDepth.java
index a48d4a678..d75947879 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/QualByDepth.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/QualByDepth.java
@@ -7,11 +7,9 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ActiveRegionBa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
-import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
-import org.broadinstitute.sting.utils.variantcontext.Allele;
 import org.broadinstitute.sting.utils.variantcontext.Genotype;
 import org.broadinstitute.sting.utils.variantcontext.GenotypesContext;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RMSMappingQuality.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RMSMappingQuality.java
index 680478da0..474b6b150 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RMSMappingQuality.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RMSMappingQuality.java
@@ -7,16 +7,14 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ActiveRegionBa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.QualityUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFStandardHeaderLines;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
-import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
-import org.broadinstitute.sting.utils.variantcontext.Allele;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
 import java.util.*;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java
index 7c7391812..0df7aff71 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java
@@ -7,15 +7,13 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ActiveRegionBasedAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.IndelGenotypeLikelihoodsCalculationModel;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.MannWhitneyU;
 import org.broadinstitute.sting.utils.QualityUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLine;
 import org.broadinstitute.sting.utils.collections.Pair;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
-import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 import org.broadinstitute.sting.utils.variantcontext.Genotype;
 import org.broadinstitute.sting.utils.variantcontext.GenotypesContext;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java
index de0ce2ce2..d01233bb2 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java
@@ -5,7 +5,7 @@ import net.sf.samtools.CigarElement;
 import net.sf.samtools.CigarOperator;
 import net.sf.samtools.SAMRecord;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.gatk.walkers.indels.PairHMMIndelErrorModel;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SampleList.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SampleList.java
index abe55de5a..33e895187 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SampleList.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SampleList.java
@@ -30,7 +30,7 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineCount;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SnpEff.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SnpEff.java
index f0bd7ecd9..b3b0be153 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SnpEff.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SnpEff.java
@@ -33,7 +33,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.RodRequiringAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.Utils;
 import org.broadinstitute.sting.utils.codecs.vcf.*;
 import org.broadinstitute.sting.utils.exceptions.UserException;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SpanningDeletions.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SpanningDeletions.java
index f6bb4e747..8e1140af1 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SpanningDeletions.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SpanningDeletions.java
@@ -6,7 +6,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TandemRepeatAnnotator.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TandemRepeatAnnotator.java
index 3d62f2d96..c72ba1c5f 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TandemRepeatAnnotator.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TandemRepeatAnnotator.java
@@ -30,7 +30,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineCount;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TechnologyComposition.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TechnologyComposition.java
index 43ef188a8..57b50c6e2 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TechnologyComposition.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TechnologyComposition.java
@@ -7,7 +7,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatible;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ExperimentalAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java
index c3e98c20f..be7288a7e 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java
@@ -8,7 +8,7 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompa
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ExperimentalAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.RodRequiringAnnotation;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineCount;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java
index eae13e1b5..ee4f77752 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java
@@ -31,7 +31,7 @@ import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.*;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.*;
 import org.broadinstitute.sting.utils.exceptions.UserException;
 import org.broadinstitute.sting.utils.variantcontext.*;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/ActiveRegionBasedAnnotation.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/ActiveRegionBasedAnnotation.java
index 9c5710872..03fcba760 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/ActiveRegionBasedAnnotation.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/ActiveRegionBasedAnnotation.java
@@ -1,9 +1,7 @@
 package org.broadinstitute.sting.gatk.walkers.annotator.interfaces;
 
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
-import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
-import org.broadinstitute.sting.utils.variantcontext.Allele;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
 import java.util.List;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/GenotypeAnnotation.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/GenotypeAnnotation.java
index 6c55c1c00..6970908b5 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/GenotypeAnnotation.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/GenotypeAnnotation.java
@@ -3,7 +3,7 @@ package org.broadinstitute.sting.gatk.walkers.annotator.interfaces;
 import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFFormatHeaderLine;
 import org.broadinstitute.sting.utils.variantcontext.Genotype;
 import org.broadinstitute.sting.utils.variantcontext.GenotypeBuilder;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/InfoFieldAnnotation.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/InfoFieldAnnotation.java
index 738be9883..5b2dc310d 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/InfoFieldAnnotation.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/InfoFieldAnnotation.java
@@ -3,7 +3,7 @@ package org.broadinstitute.sting.gatk.walkers.annotator.interfaces;
 import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java
index 77da35c41..ae9b01f2d 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java
@@ -104,7 +104,7 @@ public abstract class GenotypeLikelihoodsCalculationModel implements Cloneable {
                                                    final List<Allele> allAllelesToUse,
                                                    final boolean useBAQedPileup,
                                                    final GenomeLocParser locParser,
-                                                   final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap);
+                                                   final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap);
 
 
     protected int getFilteredDepth(ReadBackedPileup pileup) {
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java
index 1761f9cec..e1092454a 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java
@@ -81,7 +81,7 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood
                                          final List<Allele> allAllelesToUse,
                                          final boolean useBAQedPileup,
                                          final GenomeLocParser locParser,
-                                         final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+                                         final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
 
         GenomeLoc loc = ref.getLocus();
 //        if (!ref.getLocus().equals(lastSiteVisited)) {
@@ -118,7 +118,7 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood
 
             if (!perReadAlleleLikelihoodMap.containsKey(sample.getKey())){
                 // no likelihoods have been computed for this sample at this site
-                perReadAlleleLikelihoodMap.put(sample.getKey(), new PerReadAlleleLikelihoodMap());
+                perReadAlleleLikelihoodMap.put(sample.getKey(), org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap.getBestAvailablePerReadAlleleLikelihoodMap());
             }
             final ReadBackedPileup pileup = context.getBasePileup();
             if (pileup != null) {
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java
index d053e13a8..54303e12c 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java
@@ -64,7 +64,7 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsC
                                          final List<Allele> allAllelesToUse,
                                          final boolean useBAQedPileup,
                                          final GenomeLocParser locParser,
-                                         final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+                                         final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
 
         perReadAlleleLikelihoodMap.clear(); // not used in SNP model, sanity check to delete any older data
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java
index a52b5dfe6..4d9f37098 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java
@@ -174,7 +174,7 @@ public class UnifiedGenotyperEngine {
 
         final List<GenotypeLikelihoodsCalculationModel.Model> models = getGLModelsToUse(tracker, refContext, rawContext);
 
-        final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap = new HashMap<String,PerReadAlleleLikelihoodMap>();
+        final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap = new HashMap<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap>();
 
         if ( models.isEmpty() ) {
             results.add(UAC.OutputMode == OUTPUT_MODE.EMIT_ALL_SITES && UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES ? generateEmptyContext(tracker, refContext, null, rawContext) : null);
@@ -209,7 +209,7 @@ public class UnifiedGenotyperEngine {
     public VariantContext calculateLikelihoods(final RefMetaDataTracker tracker,
                                                final ReferenceContext refContext,
                                                final AlignmentContext rawContext,
-                                               final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+                                               final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
         final List<GenotypeLikelihoodsCalculationModel.Model> models = getGLModelsToUse(tracker, refContext, rawContext);
         if ( models.isEmpty() ) {
             return null;
@@ -275,7 +275,7 @@ public class UnifiedGenotyperEngine {
                                                 final List<Allele> alternateAllelesToUse,
                                                 final boolean useBAQedPileup,
                                                 final GenotypeLikelihoodsCalculationModel.Model model,
-                                                final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+                                                final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
 
         // initialize the data for this thread if that hasn't been done yet
         if ( glcm.get() == null ) {
@@ -313,7 +313,7 @@ public class UnifiedGenotyperEngine {
         return new VariantCallContext(vc, false);
     }
 
-    public VariantCallContext calculateGenotypes(final VariantContext vc, final GenotypeLikelihoodsCalculationModel.Model model, final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+    public VariantCallContext calculateGenotypes(final VariantContext vc, final GenotypeLikelihoodsCalculationModel.Model model, final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
         return calculateGenotypes(null, null, null, null, vc, model, perReadAlleleLikelihoodMap);
     }
 
@@ -327,7 +327,7 @@ public class UnifiedGenotyperEngine {
                                                  final Map<String, AlignmentContext> stratifiedContexts,
                                                  final VariantContext vc,
                                                  final GenotypeLikelihoodsCalculationModel.Model model,
-                                                 final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+                                                 final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
         return calculateGenotypes(tracker, refContext, rawContext, stratifiedContexts, vc, model, false,perReadAlleleLikelihoodMap);
     }
 
@@ -346,7 +346,7 @@ public class UnifiedGenotyperEngine {
                                                  final AlignmentContext rawContext, Map<String, AlignmentContext> stratifiedContexts,
                                                  final VariantContext vc, final GenotypeLikelihoodsCalculationModel.Model model,
                                                  final boolean inheritAttributesFromInputVC,
-                                                 final Map<String,PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+                                                 final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
 
         boolean limitedContext = tracker == null || refContext == null || rawContext == null || stratifiedContexts == null;
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java
index 3d287057c..04a26672c 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java
@@ -27,7 +27,7 @@ package org.broadinstitute.sting.gatk.walkers.indels;
 
 import com.google.java.contract.Ensures;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
-import org.broadinstitute.sting.gatk.walkers.genotyper.PerReadAlleleLikelihoodMap;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.Haplotype;
 import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.clipping.ReadClipper;
@@ -42,7 +42,6 @@ import org.broadinstitute.sting.utils.sam.ReadUtils;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/PerReadAlleleLikelihoodMap.java b/public/java/src/org/broadinstitute/sting/utils/genotyper/PerReadAlleleLikelihoodMap.java
similarity index 76%
rename from public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/PerReadAlleleLikelihoodMap.java
rename to public/java/src/org/broadinstitute/sting/utils/genotyper/PerReadAlleleLikelihoodMap.java
index a83adc275..bae38283d 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/PerReadAlleleLikelihoodMap.java
+++ b/public/java/src/org/broadinstitute/sting/utils/genotyper/PerReadAlleleLikelihoodMap.java
@@ -22,26 +22,24 @@
  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
  * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
-package org.broadinstitute.sting.gatk.walkers.genotyper;
+package org.broadinstitute.sting.utils.genotyper;
 
 
-//import org.broadinstitute.sting.gatk.walkers.Requires;
-import com.google.java.contract.Requires;
+import org.broadinstitute.sting.utils.classloader.GATKLiteUtils;
+import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
+import java.lang.reflect.Constructor;
 import java.util.*;
 
-public class PerReadAlleleLikelihoodMap {
+public abstract class PerReadAlleleLikelihoodMap {
+
     public static final double INDEL_LIKELIHOOD_THRESH = 0.1;
 
-    private List<Allele> alleles;
-    private Map<GATKSAMRecord,Map<Allele,Double>> likelihoodReadMap;
-    public PerReadAlleleLikelihoodMap() {
-        likelihoodReadMap = new LinkedHashMap<GATKSAMRecord,Map<Allele,Double>>();
-        alleles = new ArrayList<Allele>();
-    }
+    protected List<Allele> alleles;
+    protected Map<GATKSAMRecord,Map<Allele,Double>> likelihoodReadMap;
 
     public void add(GATKSAMRecord read, Allele a, Double likelihood) {
         Map<Allele,Double> likelihoodMap;
@@ -95,16 +93,6 @@ public class PerReadAlleleLikelihoodMap {
     public int getNumberOfStoredElements() {
         return likelihoodReadMap.size();
     }
-    /**
-     * Returns list of reads greedily associated with a particular allele.
-     * Needs to loop for each read, and assign to each allele
-     * @param a                          Desired allele
-     * @return
-     */
-    @Requires("a!=null")
-    public List<GATKSAMRecord> getReadsAssociatedWithAllele(Allele a) {
-        return null;
-    }
 
     public Map<Allele,Double> getLikelihoodsAssociatedWithPileupElement(PileupElement p) {
         if (!likelihoodReadMap.containsKey(p.getRead()))
@@ -129,4 +117,18 @@ public class PerReadAlleleLikelihoodMap {
         }
         return (maxLike - prevMaxLike > INDEL_LIKELIHOOD_THRESH ? mostLikelyAllele : Allele.NO_CALL );
     }
- }
+
+    public abstract PerReadAlleleLikelihoodMap createPerAlleleDownsampledMap(final double downsamplingFraction);
+
+    public static PerReadAlleleLikelihoodMap getBestAvailablePerReadAlleleLikelihoodMap() {
+        final Class PerReadAlleleLikelihoodMapClass = GATKLiteUtils.getProtectedClassIfAvailable(PerReadAlleleLikelihoodMap.class);
+        try {
+            Constructor constructor = PerReadAlleleLikelihoodMapClass.getDeclaredConstructor((Class[])null);
+            constructor.setAccessible(true);
+            return (PerReadAlleleLikelihoodMap)constructor.newInstance();
+        }
+        catch (Exception e) {
+            throw new ReviewedStingException("Unable to create RecalibrationEngine class instance " + PerReadAlleleLikelihoodMapClass.getSimpleName());
+        }
+    }
+}
diff --git a/public/java/src/org/broadinstitute/sting/utils/genotyper/StandardPerReadAlleleLikelihoodMap.java b/public/java/src/org/broadinstitute/sting/utils/genotyper/StandardPerReadAlleleLikelihoodMap.java
new file mode 100644
index 000000000..7ac757783
--- /dev/null
+++ b/public/java/src/org/broadinstitute/sting/utils/genotyper/StandardPerReadAlleleLikelihoodMap.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2011 The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+ * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package org.broadinstitute.sting.utils.genotyper;
+
+
+import org.broadinstitute.sting.utils.classloader.PublicPackageSource;
+import org.broadinstitute.sting.utils.pileup.PileupElement;
+import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
+import org.broadinstitute.sting.utils.variantcontext.Allele;
+
+import java.util.*;
+
+public class StandardPerReadAlleleLikelihoodMap extends PerReadAlleleLikelihoodMap implements PublicPackageSource {
+
+    public StandardPerReadAlleleLikelihoodMap() {
+        likelihoodReadMap = new LinkedHashMap<GATKSAMRecord,Map<Allele,Double>>();
+        alleles = new ArrayList<Allele>();
+    }
+
+    // not implemented in the standard version
+    public PerReadAlleleLikelihoodMap createPerAlleleDownsampledMap(final double downsamplingFraction) { return this; }
+ }

From c6b57fffdaf2a86a252f08a91881621663e04bf6 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Wed, 24 Oct 2012 22:52:25 -0400
Subject: [PATCH 12/50] Added allele biased down-sampling capabilities to the
 PerReadAlleleLikelihoodMap object, which means that both the UG and HC can
 use this functionality.  Note that it's only available in protected, so
 GATK-lite users won't be allowed to enable it.  Needs more testing.

---
 .../AlleleBiasedDownsamplingUtils.java        | 195 ++++++++++++++++++
 .../gatk/walkers/genotyper/ErrorModel.java    |   3 +-
 .../haplotypecaller/HaplotypeCaller.java      |   3 +-
 .../LikelihoodCalculationEngine.java          |   8 +-
 .../AdvancedPerReadAlleleLikelihoodMap.java   |  35 +++-
 .../StandardCallerArgumentCollection.java     |  33 ++-
 ...elGenotypeLikelihoodsCalculationModel.java |   7 +-
 ...NPGenotypeLikelihoodsCalculationModel.java |  67 +-----
 .../genotyper/UnifiedGenotyperEngine.java     |  26 +--
 .../indels/PairHMMIndelErrorModel.java        |   6 +-
 .../genotyper/PerReadAlleleLikelihoodMap.java |   7 +-
 .../StandardPerReadAlleleLikelihoodMap.java   |   8 +-
 12 files changed, 289 insertions(+), 109 deletions(-)
 create mode 100755 protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java

diff --git a/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java b/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
new file mode 100755
index 000000000..e224a9c57
--- /dev/null
+++ b/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
@@ -0,0 +1,195 @@
+/*
+ * Copyright (c) 2010.
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+ * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.gatk.downsampling;
+
+import net.sf.samtools.SAMReadGroupRecord;
+import net.sf.samtools.SAMRecord;
+import org.broadinstitute.sting.utils.*;
+import org.broadinstitute.sting.utils.pileup.*;
+import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
+import org.broadinstitute.sting.utils.variantcontext.Allele;
+
+import java.io.PrintStream;
+import java.util.*;
+
+public class AlleleBiasedDownsamplingUtils {
+
+    private static final ArrayList<PileupElement>[] alleleStratifiedElements = new ArrayList[4];
+    static {
+        for ( int i = 0; i < 4; i++ )
+            alleleStratifiedElements[i] = new ArrayList<PileupElement>();
+    }
+
+    /**
+     * Computes an allele biased version of the given pileup
+     *
+     * @param pileup                    the original pileup
+     * @param downsamplingFraction      the fraction of total reads to remove per allele
+     * @param log                       logging output
+     * @return allele biased pileup
+     */
+    public synchronized static ReadBackedPileup createAlleleBiasedBasePileup(final ReadBackedPileup pileup, final double downsamplingFraction, final PrintStream log) {
+        // special case removal of all or no reads
+        if ( downsamplingFraction <= 0.0 )
+            return pileup;
+        if ( downsamplingFraction >= 1.0 )
+            return new ReadBackedPileupImpl(pileup.getLocation(), new ArrayList<PileupElement>());
+
+        // start by stratifying the reads by the alleles they represent at this position
+        for( final PileupElement pe : pileup ) {
+            final int baseIndex = BaseUtils.simpleBaseToBaseIndex(pe.getBase());
+            if ( baseIndex != -1 )
+                alleleStratifiedElements[baseIndex].add(pe);
+        }
+
+        // Down-sample *each* allele by the contamination fraction applied to the entire pileup.
+        // Unfortunately, we need to maintain the original pileup ordering of reads or FragmentUtils will complain later.
+        int numReadsToRemove = (int)(pileup.getNumberOfElements() * downsamplingFraction); // floor
+        final TreeSet<PileupElement> elementsToKeep = new TreeSet<PileupElement>(new Comparator<PileupElement>() {
+            @Override
+            public int compare(PileupElement element1, PileupElement element2) {
+                final int difference = element1.getRead().getAlignmentStart() - element2.getRead().getAlignmentStart();
+                return difference != 0 ? difference : element1.getRead().getReadName().compareTo(element2.getRead().getReadName());
+            }
+        });
+
+        for ( int i = 0; i < 4; i++ ) {
+            final ArrayList<PileupElement> alleleList = alleleStratifiedElements[i];
+            if ( alleleList.size() <= numReadsToRemove )
+                logAllElements(alleleList, log);
+            else
+                elementsToKeep.addAll(downsampleElements(alleleList, numReadsToRemove, log));
+        }
+
+        // clean up pointers so memory can be garbage collected if needed
+        for ( int i = 0; i < 4; i++ )
+            alleleStratifiedElements[i].clear();
+
+        return new ReadBackedPileupImpl(pileup.getLocation(), new ArrayList<PileupElement>(elementsToKeep));
+    }
+
+    /**
+     * Performs allele biased down-sampling on a pileup and computes the list of elements to keep
+     *
+     * @param elements                  original list of records
+     * @param numElementsToRemove       the number of records to remove
+     * @param log                       logging output
+     * @return the list of pileup elements TO KEEP
+     */
+    private static List<PileupElement> downsampleElements(final ArrayList<PileupElement> elements, final int numElementsToRemove, final PrintStream log) {
+        final int pileupSize = elements.size();
+        final BitSet itemsToRemove = new BitSet(pileupSize);
+        for ( Integer selectedIndex : MathUtils.sampleIndicesWithoutReplacement(pileupSize, numElementsToRemove) ) {
+            itemsToRemove.set(selectedIndex);
+        }
+
+        ArrayList<PileupElement> elementsToKeep = new ArrayList<PileupElement>(pileupSize - numElementsToRemove);
+        for ( int i = 0; i < pileupSize; i++ ) {
+            if ( itemsToRemove.get(i) )
+                logRead(elements.get(i).getRead(), log);
+            else
+                elementsToKeep.add(elements.get(i));
+        }
+
+        return elementsToKeep;
+    }
+
+    /**
+     * Computes reads to remove based on an allele biased down-sampling
+     *
+     * @param alleleReadMap             original list of records per allele
+     * @param downsamplingFraction      the fraction of total reads to remove per allele
+     * @param log                       logging output
+     * @return list of reads TO REMOVE from allele biased down-sampling
+     */
+    public static List<GATKSAMRecord> selectAlleleBiasedReads(final Map<Allele, List<GATKSAMRecord>> alleleReadMap, final double downsamplingFraction, final PrintStream log) {
+        int totalReads = 0;
+        for ( final List<GATKSAMRecord> reads : alleleReadMap.values() )
+            totalReads += reads.size();
+
+        // Down-sample *each* allele by the contamination fraction applied to the entire pileup.
+        int numReadsToRemove = (int)(totalReads * downsamplingFraction);
+        final List<GATKSAMRecord> readsToRemove = new ArrayList<GATKSAMRecord>(numReadsToRemove * alleleReadMap.size());
+        for ( final List<GATKSAMRecord> reads : alleleReadMap.values() ) {
+            if ( reads.size() <= numReadsToRemove ) {
+                readsToRemove.addAll(reads);
+                logAllReads(reads, log);
+            } else {
+                readsToRemove.addAll(downsampleReads(reads, numReadsToRemove, log));
+            }
+        }
+
+        return readsToRemove;
+    }
+
+    /**
+     * Performs allele biased down-sampling on a pileup and computes the list of elements to remove
+     *
+     * @param reads                     original list of records
+     * @param numElementsToRemove       the number of records to remove
+     * @param log                       logging output
+     * @return the list of pileup elements TO REMOVE
+     */
+    private static List<GATKSAMRecord> downsampleReads(final List<GATKSAMRecord> reads, final int numElementsToRemove, final PrintStream log) {
+        final int pileupSize = reads.size();
+        final BitSet itemsToRemove = new BitSet(pileupSize);
+        for ( Integer selectedIndex : MathUtils.sampleIndicesWithoutReplacement(pileupSize, numElementsToRemove) ) {
+            itemsToRemove.set(selectedIndex);
+        }
+
+        ArrayList<GATKSAMRecord> readsToRemove = new ArrayList<GATKSAMRecord>(pileupSize - numElementsToRemove);
+        for ( int i = 0; i < pileupSize; i++ ) {
+            if ( itemsToRemove.get(i) ) {
+                final GATKSAMRecord read = reads.get(i);
+                readsToRemove.add(read);
+                logRead(read, log);
+            }
+        }
+
+        return readsToRemove;
+    }
+
+    private static void logAllElements(final List<PileupElement> elements, final PrintStream log) {
+        if ( log != null ) {
+            for ( final PileupElement p : elements )
+                logRead(p.getRead(), log);
+        }
+    }
+
+    private static void logAllReads(final List<GATKSAMRecord> reads, final PrintStream log) {
+        if ( log != null ) {
+            for ( final GATKSAMRecord read : reads )
+                logRead(read, log);
+        }
+    }
+
+    private static void logRead(final SAMRecord read, final PrintStream log) {
+        if ( log != null ) {
+            final SAMReadGroupRecord readGroup = read.getReadGroup();
+            log.println(String.format("%s\t%s\t%s\t%s", read.getReadName(), readGroup.getSample(), readGroup.getLibrary(), readGroup.getPlatformUnit()));
+        }
+    }
+}
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ErrorModel.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ErrorModel.java
index a01973ad5..fc6d23382 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ErrorModel.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ErrorModel.java
@@ -5,6 +5,7 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.walkers.indels.PairHMMIndelErrorModel;
 import org.broadinstitute.sting.utils.Haplotype;
 import org.broadinstitute.sting.utils.MathUtils;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
@@ -59,7 +60,7 @@ public class ErrorModel  {
 
         boolean hasCalledAlleles = false;
 
-        final org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap = org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap.getBestAvailablePerReadAlleleLikelihoodMap();
+        final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap = PerReadAlleleLikelihoodMap.getBestAvailablePerReadAlleleLikelihoodMap();
         if (refSampleVC != null) {
 
             for (Allele allele : refSampleVC.getAlleles()) {
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java
index 3293ac817..5aba23faa 100755
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java
@@ -51,6 +51,7 @@ import org.broadinstitute.sting.utils.exceptions.UserException;
 import org.broadinstitute.sting.utils.fasta.CachingIndexedFastaSequenceFile;
 import org.broadinstitute.sting.utils.fragments.FragmentCollection;
 import org.broadinstitute.sting.utils.fragments.FragmentUtils;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.help.DocumentedGATKFeature;
 import org.broadinstitute.sting.utils.pairhmm.PairHMM;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
@@ -424,7 +425,7 @@ public class HaplotypeCaller extends ActiveRegionWalker<Integer, Integer> implem
                   : genotypingEngine.assignGenotypeLikelihoodsAndCallIndependentEvents( UG_engine, bestHaplotypes, fullReferenceWithPadding, getPaddedLoc(activeRegion), activeRegion.getLocation(), getToolkit().getGenomeLocParser(), activeAllelesToGenotype ) ) ) {
             if( DEBUG ) { System.out.println(callResult.getFirst().toStringWithoutGenotypes()); }
 
-            final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> stratifiedReadMap = LikelihoodCalculationEngine.partitionReadsBasedOnLikelihoods( getToolkit().getGenomeLocParser(), perSampleReadList, perSampleFilteredReadList, callResult );
+            final Map<String, PerReadAlleleLikelihoodMap> stratifiedReadMap = LikelihoodCalculationEngine.partitionReadsBasedOnLikelihoods( getToolkit().getGenomeLocParser(), perSampleReadList, perSampleFilteredReadList, callResult, UG_engine.getUAC().CONTAMINATION_FRACTION, UG_engine.getUAC().contaminationLog );
             final VariantContext annotatedCall = annotationEngine.annotateContext(stratifiedReadMap, callResult.getFirst());
             final Map<String, Object> myAttributes = new LinkedHashMap<String, Object>(annotatedCall.getAttributes());
 
diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java
index a05e20bd3..e1a9ef7c8 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java
@@ -38,6 +38,7 @@ import org.broadinstitute.sting.utils.sam.ReadUtils;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
+import java.io.PrintStream;
 import java.util.*;
 
 public class LikelihoodCalculationEngine {
@@ -346,7 +347,9 @@ public class LikelihoodCalculationEngine {
     public static Map<String, PerReadAlleleLikelihoodMap> partitionReadsBasedOnLikelihoods( final GenomeLocParser parser,
                                                                                             final HashMap<String, ArrayList<GATKSAMRecord>> perSampleReadList,
                                                                                             final HashMap<String, ArrayList<GATKSAMRecord>> perSampleFilteredReadList,
-                                                                                            final Pair<VariantContext, HashMap<Allele,ArrayList<Haplotype>>> call) {
+                                                                                            final Pair<VariantContext, HashMap<Allele,ArrayList<Haplotype>>> call,
+                                                                                            final double downsamplingFraction,
+                                                                                            final PrintStream downsamplingLog ) {
         final Map<String, PerReadAlleleLikelihoodMap> returnMap = new HashMap<String, PerReadAlleleLikelihoodMap>();
         final GenomeLoc callLoc = parser.createGenomeLoc(call.getFirst());
         for( final Map.Entry<String, ArrayList<GATKSAMRecord>> sample : perSampleReadList.entrySet() ) {
@@ -370,6 +373,9 @@ public class LikelihoodCalculationEngine {
                 }
             }
 
+            // down-sample before adding filtered reads
+            likelihoodMap.performPerAlleleDownsampling(downsamplingFraction, downsamplingLog);
+
             // add all filtered reads to the NO_CALL list because they weren't given any likelihoods
             for( final GATKSAMRecord read : perSampleFilteredReadList.get(sample.getKey()) ) {
                 // only count the read if it overlaps the event, otherwise it is not added to the output read list at all
diff --git a/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java b/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
index db1bf8762..80618cfdb 100644
--- a/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
+++ b/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
@@ -25,16 +25,43 @@
 package org.broadinstitute.sting.utils.genotyper;
 
 
+import org.broadinstitute.sting.gatk.downsampling.AlleleBiasedDownsamplingUtils;
 import org.broadinstitute.sting.utils.classloader.ProtectedPackageSource;
-import org.broadinstitute.sting.utils.pileup.PileupElement;
+import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
+import java.io.PrintStream;
 import java.util.*;
 
 public class AdvancedPerReadAlleleLikelihoodMap extends StandardPerReadAlleleLikelihoodMap implements ProtectedPackageSource {
 
-    public PerReadAlleleLikelihoodMap createPerAlleleDownsampledMap(final double downsamplingFraction) {
-        return this;
+    public ReadBackedPileup createPerAlleleDownsampledBasePileup(final ReadBackedPileup pileup, final double downsamplingFraction, final PrintStream log) {
+        return AlleleBiasedDownsamplingUtils.createAlleleBiasedBasePileup(pileup, downsamplingFraction, log);
     }
- }
+
+    public void performPerAlleleDownsampling(final double downsamplingFraction, final PrintStream log) {
+        // special case removal of all or no reads
+        if ( downsamplingFraction <= 0.0 )
+            return;
+        if ( downsamplingFraction >= 1.0 ) {
+            likelihoodReadMap.clear();
+            return;
+        }
+
+        // start by stratifying the reads by the alleles they represent at this position
+        final Map<Allele, List<GATKSAMRecord>> alleleReadMap = new HashMap<Allele, List<GATKSAMRecord>>(alleles.size());
+        for ( Allele allele : alleles )
+            alleleReadMap.put(allele, new ArrayList<GATKSAMRecord>());
+
+        for ( Map.Entry<GATKSAMRecord, Map<Allele, Double>> entry : likelihoodReadMap.entrySet() ) {
+            final Allele bestAllele = getMostLikelyAllele(entry.getValue());
+            alleleReadMap.get(bestAllele).add(entry.getKey());
+        }
+
+        // compute the reads to remove and actually remove them
+        final List<GATKSAMRecord> readsToRemove = AlleleBiasedDownsamplingUtils.selectAlleleBiasedReads(alleleReadMap, downsamplingFraction, log);
+        for ( final GATKSAMRecord read : readsToRemove )
+            likelihoodReadMap.remove(read);
+    }
+}
diff --git a/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
index aa3a1e6ac..36456ca09 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
@@ -7,6 +7,7 @@ import org.broadinstitute.sting.gatk.walkers.genotyper.afcalc.AFCalcFactory;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
 import java.io.File;
+import java.io.PrintStream;
 
 /**
  * Created with IntelliJ IDEA.
@@ -63,26 +64,23 @@ public class StandardCallerArgumentCollection {
     public int MAX_ALTERNATE_ALLELES = 6;
 
     /**
-     * If there are more than this number of alternate alleles presented to the genotyper (either through discovery or GENOTYPE_GIVEN ALLELES),
-     * then only this many alleles will be used.  Note that genotyping sites with many alternate alleles is both CPU and memory intensive and it
-     * scales exponentially based on the number of alternate alleles.  Unless there is a good reason to change the default value, we highly recommend
-     * that you not play around with this parameter.
-     *
-     * This argument has been retired in GATK 2.2.  Please specify just maxAltAlleles from now on
+     * Controls the model used to calculate the probability that a site is variant plus the various sample genotypes in the data at a given locus.
      */
-    @Deprecated
-    @Hidden
-    @Argument(fullName = "max_alternate_alleles_for_indels", shortName = "maxAltAllelesForIndels", doc = "This argument has been retired in GATK 2.2.  Please specify just maxAltAlleles from now on, which will apply to any variant, regardless of type", required = false)
-    public int MAX_ALTERNATE_ALLELES_FOR_INDELS = -1;
+    @Advanced
+    @Argument(fullName = "p_nonref_model", shortName = "pnrm", doc = "Non-reference probability calculation model to employ", required = false)
+    public AFCalcFactory.Calculation AFmodel = AFCalcFactory.Calculation.getDefaultModel();
 
     /**
      * If this fraction is greater is than zero, the caller will aggressively attempt to remove contamination through biased down-sampling of reads.
      * Basically, it will ignore the contamination fraction of reads for each alternate allele.  So if the pileup contains N total bases, then we
      * will try to remove (N * contamination fraction) bases for each alternate allele.
      */
-    @Hidden
     @Argument(fullName = "contamination_percentage_to_filter", shortName = "contamination", doc = "Fraction of contamination in sequencing data (for all samples) to aggressively remove", required = false)
-    public double CONTAMINATION_PERCENTAGE = 0.0;
+    public double CONTAMINATION_FRACTION = 0.0;
+
+    @Hidden
+    @Argument(shortName = "logRemovedReadsFromContaminationFiltering", doc="contaminationLog", required=false)
+    public PrintStream contaminationLog = null;
 
     @Hidden
     @Argument(shortName = "logExactCalls", doc="x", required=false)
@@ -96,19 +94,12 @@ public class StandardCallerArgumentCollection {
         this.GenotypingMode = SCAC.GenotypingMode;
         this.heterozygosity = SCAC.heterozygosity;
         this.MAX_ALTERNATE_ALLELES = SCAC.MAX_ALTERNATE_ALLELES;
-        this.MAX_ALTERNATE_ALLELES_FOR_INDELS = SCAC.MAX_ALTERNATE_ALLELES_FOR_INDELS;
         this.OutputMode = SCAC.OutputMode;
         this.STANDARD_CONFIDENCE_FOR_CALLING = SCAC.STANDARD_CONFIDENCE_FOR_CALLING;
         this.STANDARD_CONFIDENCE_FOR_EMITTING = SCAC.STANDARD_CONFIDENCE_FOR_EMITTING;
-        this.CONTAMINATION_PERCENTAGE = SCAC.CONTAMINATION_PERCENTAGE;
+        this.CONTAMINATION_FRACTION = SCAC.CONTAMINATION_FRACTION;
+        this.contaminationLog = SCAC.contaminationLog;
         this.exactCallsLog = SCAC.exactCallsLog;
         this.AFmodel = SCAC.AFmodel;
     }
-
-    /**
-     * Controls the model used to calculate the probability that a site is variant plus the various sample genotypes in the data at a given locus.
-     */
-    @Advanced
-    @Argument(fullName = "p_nonref_model", shortName = "pnrm", doc = "Non-reference probability calculation model to employ", required = false)
-    public AFCalcFactory.Calculation AFmodel = AFCalcFactory.Calculation.getDefaultModel();
 }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java
index e1092454a..0d9f443e2 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java
@@ -35,6 +35,7 @@ import org.broadinstitute.sting.utils.BaseUtils;
 import org.broadinstitute.sting.utils.GenomeLoc;
 import org.broadinstitute.sting.utils.GenomeLocParser;
 import org.broadinstitute.sting.utils.Haplotype;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.variantcontext.*;
@@ -81,7 +82,7 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood
                                          final List<Allele> allAllelesToUse,
                                          final boolean useBAQedPileup,
                                          final GenomeLocParser locParser,
-                                         final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+                                         final Map<String, PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
 
         GenomeLoc loc = ref.getLocus();
 //        if (!ref.getLocus().equals(lastSiteVisited)) {
@@ -118,12 +119,12 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood
 
             if (!perReadAlleleLikelihoodMap.containsKey(sample.getKey())){
                 // no likelihoods have been computed for this sample at this site
-                perReadAlleleLikelihoodMap.put(sample.getKey(), org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap.getBestAvailablePerReadAlleleLikelihoodMap());
+                perReadAlleleLikelihoodMap.put(sample.getKey(), PerReadAlleleLikelihoodMap.getBestAvailablePerReadAlleleLikelihoodMap());
             }
             final ReadBackedPileup pileup = context.getBasePileup();
             if (pileup != null) {
                 final GenotypeBuilder b = new GenotypeBuilder(sample.getKey());
-                final double[] genotypeLikelihoods = pairModel.computeDiploidReadHaplotypeLikelihoods(pileup, haplotypeMap, ref, eventLength, perReadAlleleLikelihoodMap.get(sample.getKey()));
+                final double[] genotypeLikelihoods = pairModel.computeDiploidReadHaplotypeLikelihoods(pileup, haplotypeMap, ref, eventLength, perReadAlleleLikelihoodMap.get(sample.getKey()), UAC.CONTAMINATION_FRACTION, UAC.contaminationLog);
                 b.PL(genotypeLikelihoods);
                 b.DP(getFilteredDepth(pileup));
                 genotypes.add(b.make());
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java
index 54303e12c..791cdc325 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java
@@ -36,6 +36,7 @@ import org.broadinstitute.sting.utils.GenomeLocParser;
 import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.baq.BAQ;
 import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileupImpl;
@@ -48,13 +49,13 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsC
     private final boolean useAlleleFromVCF;
 
     private final double[] likelihoodSums = new double[4];
-    private final ArrayList<PileupElement>[] alleleStratifiedElements = new ArrayList[4];
+
+    private final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap;
 
     protected SNPGenotypeLikelihoodsCalculationModel(UnifiedArgumentCollection UAC, Logger logger) {
         super(UAC, logger);
         useAlleleFromVCF = UAC.GenotypingMode == GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES;
-        for ( int i = 0; i < 4; i++ )
-            alleleStratifiedElements[i] = new ArrayList<PileupElement>();
+        perReadAlleleLikelihoodMap = PerReadAlleleLikelihoodMap.getBestAvailablePerReadAlleleLikelihoodMap();
     }
 
     public VariantContext getLikelihoods(final RefMetaDataTracker tracker,
@@ -64,9 +65,9 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsC
                                          final List<Allele> allAllelesToUse,
                                          final boolean useBAQedPileup,
                                          final GenomeLocParser locParser,
-                                         final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap> perReadAlleleLikelihoodMap) {
+                                         final Map<String, PerReadAlleleLikelihoodMap> sampleLikelihoodMap) {
 
-        perReadAlleleLikelihoodMap.clear(); // not used in SNP model, sanity check to delete any older data
+        sampleLikelihoodMap.clear(); // not used in SNP model, sanity check to delete any older data
 
         final byte refBase = ref.getBase();
         final int indexOfRefBase = BaseUtils.simpleBaseToBaseIndex(refBase);
@@ -79,8 +80,8 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsC
         ArrayList<SampleGenotypeData> GLs = new ArrayList<SampleGenotypeData>(contexts.size());
         for ( Map.Entry<String, AlignmentContext> sample : contexts.entrySet() ) {
             ReadBackedPileup pileup = AlignmentContextUtils.stratify(sample.getValue(), contextType).getBasePileup();
-            if ( UAC.CONTAMINATION_PERCENTAGE > 0.0 )
-                pileup = createDecontaminatedPileup(pileup, UAC.CONTAMINATION_PERCENTAGE);
+            if ( UAC.CONTAMINATION_FRACTION > 0.0 )
+                pileup = perReadAlleleLikelihoodMap.createPerAlleleDownsampledBasePileup(pileup, UAC.CONTAMINATION_FRACTION, UAC.contaminationLog);
             if ( useBAQedPileup )
                 pileup = createBAQedPileup(pileup);
 
@@ -203,42 +204,6 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsC
         return allelesToUse;
     }
 
-    public ReadBackedPileup createDecontaminatedPileup(final ReadBackedPileup pileup, final double contaminationPercentage) {
-        // special case removal of all reads
-        if ( contaminationPercentage >= 1.0 )
-            return new ReadBackedPileupImpl(pileup.getLocation(), new ArrayList<PileupElement>());
-
-        // start by stratifying the reads by the alleles they represent at this position
-        for( final PileupElement pe : pileup ) {
-            final int baseIndex = BaseUtils.simpleBaseToBaseIndex(pe.getBase());
-            if ( baseIndex != -1 )
-                alleleStratifiedElements[baseIndex].add(pe);
-        }
-
-        // Down-sample *each* allele by the contamination fraction applied to the entire pileup.
-        // Unfortunately, we need to maintain the original pileup ordering of reads or FragmentUtils will complain later.
-        int numReadsToRemove = (int)Math.ceil((double)pileup.getNumberOfElements() * contaminationPercentage);
-        final TreeSet<PileupElement> elementsToKeep = new TreeSet<PileupElement>(new Comparator<PileupElement>() {
-            @Override
-            public int compare(PileupElement element1, PileupElement element2) {
-                final int difference = element1.getRead().getAlignmentStart() - element2.getRead().getAlignmentStart();
-                return difference != 0 ? difference : element1.getRead().getReadName().compareTo(element2.getRead().getReadName());
-            }
-        });
-
-        for ( int i = 0; i < 4; i++ ) {
-            final ArrayList<PileupElement> alleleList = alleleStratifiedElements[i];
-            if ( alleleList.size() > numReadsToRemove )
-                elementsToKeep.addAll(downsampleElements(alleleList, numReadsToRemove));
-        }
-
-        // clean up pointers so memory can be garbage collected if needed
-        for ( int i = 0; i < 4; i++ )
-            alleleStratifiedElements[i].clear();
-
-        return new ReadBackedPileupImpl(pileup.getLocation(), new ArrayList<PileupElement>(elementsToKeep));
-    }
-
     public ReadBackedPileup createBAQedPileup( final ReadBackedPileup pileup ) {
         final List<PileupElement> BAQedElements = new ArrayList<PileupElement>();
         for( final PileupElement PE : pileup ) {
@@ -257,22 +222,6 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsC
         public byte getQual( final int offset ) { return BAQ.calcBAQFromTag(getRead(), offset, true); }
     }
 
-    private List<PileupElement> downsampleElements(final ArrayList<PileupElement> elements, final int numElementsToRemove) {
-        final int pileupSize = elements.size();
-        final BitSet itemsToRemove = new BitSet(pileupSize);
-        for ( Integer selectedIndex : MathUtils.sampleIndicesWithoutReplacement(pileupSize, numElementsToRemove) ) {
-            itemsToRemove.set(selectedIndex);
-        }
-
-        ArrayList<PileupElement> elementsToKeep = new ArrayList<PileupElement>(pileupSize - numElementsToRemove);
-        for ( int i = 0; i < pileupSize; i++ ) {
-            if ( !itemsToRemove.get(i) )
-                elementsToKeep.add(elements.get(i));
-        }
-
-        return elementsToKeep;
-    }
-
     private static class SampleGenotypeData {
 
         public final String name;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java
index 4d9f37098..3c4a97ec1 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java
@@ -153,19 +153,19 @@ public class UnifiedGenotyperEngine {
     }
 
 
-        /**
-        * Compute full calls at a given locus. Entry point for engine calls from the UnifiedGenotyper.
-        *
-        * If allSamples != null, then the output variantCallContext is guarenteed to contain a genotype
-        * for every sample in allSamples.  If it's null there's no such guarentee.  Providing this
-        * argument is critical when the resulting calls will be written to a VCF file.
-        *
-        * @param tracker    the meta data tracker
-        * @param refContext the reference base
-        * @param rawContext contextual information around the locus
-        * @param allSamples set of all sample names that we might call (i.e., those in the VCF header)
-        * @return the VariantCallContext object
-        */
+    /**
+     * Compute full calls at a given locus. Entry point for engine calls from the UnifiedGenotyper.
+     *
+     * If allSamples != null, then the output variantCallContext is guarenteed to contain a genotype
+     * for every sample in allSamples.  If it's null there's no such guarentee.  Providing this
+     * argument is critical when the resulting calls will be written to a VCF file.
+     *
+     * @param tracker    the meta data tracker
+     * @param refContext the reference base
+     * @param rawContext contextual information around the locus
+     * @param allSamples set of all sample names that we might call (i.e., those in the VCF header)
+     * @return the VariantCallContext object
+     */
     public List<VariantCallContext> calculateLikelihoodsAndGenotypes(final RefMetaDataTracker tracker,
                                                                      final ReferenceContext refContext,
                                                                      final AlignmentContext rawContext,
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java
index 04a26672c..79962a3e4 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java
@@ -41,6 +41,7 @@ import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.sam.ReadUtils;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
+import java.io.PrintStream;
 import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -187,11 +188,14 @@ public class PairHMMIndelErrorModel {
                                                                         final LinkedHashMap<Allele, Haplotype> haplotypeMap,
                                                                         final ReferenceContext ref,
                                                                         final int eventLength,
-                                                                        final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap){
+                                                                        final PerReadAlleleLikelihoodMap perReadAlleleLikelihoodMap,
+                                                                        final double downsamplingFraction,
+                                                                        final PrintStream downsamplingLog) {
         final int numHaplotypes = haplotypeMap.size();
 
         final int readCounts[] = new int[pileup.getNumberOfElements()];
         final double[][] readLikelihoods = computeGeneralReadHaplotypeLikelihoods(pileup, haplotypeMap, ref, eventLength, perReadAlleleLikelihoodMap, readCounts);
+        perReadAlleleLikelihoodMap.performPerAlleleDownsampling(downsamplingFraction, downsamplingLog);
         return getDiploidHaplotypeLikelihoods(numHaplotypes, readCounts, readLikelihoods);
         
     }
diff --git a/public/java/src/org/broadinstitute/sting/utils/genotyper/PerReadAlleleLikelihoodMap.java b/public/java/src/org/broadinstitute/sting/utils/genotyper/PerReadAlleleLikelihoodMap.java
index bae38283d..22d249240 100644
--- a/public/java/src/org/broadinstitute/sting/utils/genotyper/PerReadAlleleLikelihoodMap.java
+++ b/public/java/src/org/broadinstitute/sting/utils/genotyper/PerReadAlleleLikelihoodMap.java
@@ -28,9 +28,11 @@ package org.broadinstitute.sting.utils.genotyper;
 import org.broadinstitute.sting.utils.classloader.GATKLiteUtils;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
+import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
+import java.io.PrintStream;
 import java.lang.reflect.Constructor;
 import java.util.*;
 
@@ -41,6 +43,9 @@ public abstract class PerReadAlleleLikelihoodMap {
     protected List<Allele> alleles;
     protected Map<GATKSAMRecord,Map<Allele,Double>> likelihoodReadMap;
 
+    public abstract void performPerAlleleDownsampling(final double downsamplingFraction, final PrintStream log);
+    public abstract ReadBackedPileup createPerAlleleDownsampledBasePileup(final ReadBackedPileup pileup, final double downsamplingFraction, final PrintStream log);
+
     public void add(GATKSAMRecord read, Allele a, Double likelihood) {
         Map<Allele,Double> likelihoodMap;
         if (likelihoodReadMap.containsKey(read)){
@@ -118,8 +123,6 @@ public abstract class PerReadAlleleLikelihoodMap {
         return (maxLike - prevMaxLike > INDEL_LIKELIHOOD_THRESH ? mostLikelyAllele : Allele.NO_CALL );
     }
 
-    public abstract PerReadAlleleLikelihoodMap createPerAlleleDownsampledMap(final double downsamplingFraction);
-
     public static PerReadAlleleLikelihoodMap getBestAvailablePerReadAlleleLikelihoodMap() {
         final Class PerReadAlleleLikelihoodMapClass = GATKLiteUtils.getProtectedClassIfAvailable(PerReadAlleleLikelihoodMap.class);
         try {
diff --git a/public/java/src/org/broadinstitute/sting/utils/genotyper/StandardPerReadAlleleLikelihoodMap.java b/public/java/src/org/broadinstitute/sting/utils/genotyper/StandardPerReadAlleleLikelihoodMap.java
index 7ac757783..7db818592 100644
--- a/public/java/src/org/broadinstitute/sting/utils/genotyper/StandardPerReadAlleleLikelihoodMap.java
+++ b/public/java/src/org/broadinstitute/sting/utils/genotyper/StandardPerReadAlleleLikelihoodMap.java
@@ -26,10 +26,11 @@ package org.broadinstitute.sting.utils.genotyper;
 
 
 import org.broadinstitute.sting.utils.classloader.PublicPackageSource;
-import org.broadinstitute.sting.utils.pileup.PileupElement;
+import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
+import java.io.PrintStream;
 import java.util.*;
 
 public class StandardPerReadAlleleLikelihoodMap extends PerReadAlleleLikelihoodMap implements PublicPackageSource {
@@ -40,5 +41,6 @@ public class StandardPerReadAlleleLikelihoodMap extends PerReadAlleleLikelihoodM
     }
 
     // not implemented in the standard version
-    public PerReadAlleleLikelihoodMap createPerAlleleDownsampledMap(final double downsamplingFraction) { return this; }
- }
+    public void performPerAlleleDownsampling(final double downsamplingFraction, final PrintStream log) {}
+    public ReadBackedPileup createPerAlleleDownsampledBasePileup(final ReadBackedPileup pileup, final double downsamplingFraction, final PrintStream log) { return pileup; }
+}

From 72714ee43eb06cb96a49d739da5955a52bf8be86 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Thu, 25 Oct 2012 02:47:42 -0400
Subject: [PATCH 13/50] Minor patches to get the contamination down-sampling
 working for indels.  Adding @Hidden logging output for easy debugging.

---
 .../AdvancedPerReadAlleleLikelihoodMap.java   |  3 +-
 .../StandardCallerArgumentCollection.java     |  2 +-
 .../walkers/genotyper/UnifiedGenotyper.java   | 39 ++++++-------------
 3 files changed, 15 insertions(+), 29 deletions(-)

diff --git a/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java b/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
index 80618cfdb..d6cddcf84 100644
--- a/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
+++ b/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
@@ -56,7 +56,8 @@ public class AdvancedPerReadAlleleLikelihoodMap extends StandardPerReadAlleleLik
 
         for ( Map.Entry<GATKSAMRecord, Map<Allele, Double>> entry : likelihoodReadMap.entrySet() ) {
             final Allele bestAllele = getMostLikelyAllele(entry.getValue());
-            alleleReadMap.get(bestAllele).add(entry.getKey());
+            if ( bestAllele != Allele.NO_CALL )
+                alleleReadMap.get(bestAllele).add(entry.getKey());
         }
 
         // compute the reads to remove and actually remove them
diff --git a/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
index 36456ca09..2f6d9d16f 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
@@ -79,7 +79,7 @@ public class StandardCallerArgumentCollection {
     public double CONTAMINATION_FRACTION = 0.0;
 
     @Hidden
-    @Argument(shortName = "logRemovedReadsFromContaminationFiltering", doc="contaminationLog", required=false)
+    @Argument(fullName = "logRemovedReadsFromContaminationFiltering", shortName="contaminationLog", required=false)
     public PrintStream contaminationLog = null;
 
     @Hidden
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
index 3116d3a7d..a17c2aa1a 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
@@ -234,36 +234,21 @@ public class UnifiedGenotyper extends LocusWalker<List<VariantCallContext>, Unif
             if (UAC.samplePloidy != VariantContextUtils.DEFAULT_PLOIDY ||
                     UAC.referenceSampleName != null ||
                     UAC.referenceSampleRod.isBound())  {
-                throw new UserException.NotSupportedInGATKLite("Usage of ploidy values different than 2 not supported in this GATK version");
+                throw new UserException.NotSupportedInGATKLite("you cannot enable usage of ploidy values other than 2");
             }
+
+            if ( UAC.CONTAMINATION_FRACTION > 0.0 ) {
+                throw new UserException.NotSupportedInGATKLite("you cannot enable usage of contamination down-sampling");
+            }
+        }
+
+        if ( UAC.TREAT_ALL_READS_AS_SINGLE_POOL ) {
+            samples.add(GenotypeLikelihoodsCalculationModel.DUMMY_SAMPLE_NAME);
+        } else {
             // get all of the unique sample names
             samples = SampleUtils.getSAMFileSamples(getToolkit().getSAMFileHeader());
-
-        } else {
-            // in full mode: check for consistency in ploidy/pool calling arguments
-            // check for correct calculation models
-/*            if (UAC.samplePloidy != VariantContextUtils.DEFAULT_PLOIDY) {
-                // polyploidy requires POOL GL and AF calculation models to be specified right now
-                if (UAC.GLmodel != GenotypeLikelihoodsCalculationModel.Model.POOLSNP && UAC.GLmodel != GenotypeLikelihoodsCalculationModel.Model.POOLINDEL
-                        && UAC.GLmodel != GenotypeLikelihoodsCalculationModel.Model.POOLBOTH)   {
-                    throw new UserException("Incorrect genotype calculation model chosen. Only [POOLSNP|POOLINDEL|POOLBOTH] supported with this walker if sample ploidy != 2");
-                }
-
-                if (UAC.AFmodel != AFCalc.Model.POOL)
-                    throw new UserException("Incorrect AF Calculation model. Only POOL model supported if sample ploidy != 2");
-
-            }
-  */
-            // get all of the unique sample names
-            if (UAC.TREAT_ALL_READS_AS_SINGLE_POOL) {
-                samples.clear();
-                samples.add(GenotypeLikelihoodsCalculationModel.DUMMY_SAMPLE_NAME);
-            } else {
-                samples = SampleUtils.getSAMFileSamples(getToolkit().getSAMFileHeader());
-                if (UAC.referenceSampleName != null )
-                    samples.remove(UAC.referenceSampleName);
-            }
-
+            if ( UAC.referenceSampleName != null )
+                samples.remove(UAC.referenceSampleName);
         }
 
         // check for a bad max alleles value

From e6652f7777b16f1bdd6266bd0c031d15a51f339f Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Thu, 25 Oct 2012 09:36:05 -0400
Subject: [PATCH 14/50] Added integration test for contamination down-sampling

---
 .../UnifiedGenotyperIntegrationTest.java      | 96 +++++++++++--------
 1 file changed, 56 insertions(+), 40 deletions(-)

diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index dee54b9f8..9e50a8556 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -26,7 +26,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing normal calling
     //
     // --------------------------------------------------------------------------------------------------------------
-    @Test
+    //@Test
     public void testMultiSamplePilot1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000", 1,
@@ -34,7 +34,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot1", spec);
     }
 
-    @Test
+    //@Test
     public void testWithAllelesPassedIn1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
@@ -42,7 +42,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 with alleles passed in", spec1);
     }
 
-    @Test
+    //@Test
     public void testWithAllelesPassedIn2() {
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
@@ -50,7 +50,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 with alleles passed in and emitting all sites", spec2);
     }
 
-    @Test
+    //@Test
     public void testSingleSamplePilot2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
@@ -58,7 +58,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test SingleSample Pilot2", spec);
     }
 
-    @Test
+    //@Test
     public void testMultipleSNPAlleles() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + privateTestDir + "multiallelic.snps.bam -o %s -L " + privateTestDir + "multiallelic.snps.intervals", 1,
@@ -66,7 +66,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test Multiple SNP alleles", spec);
     }
 
-    @Test
+    //@Test
     public void testBadRead() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm BOTH -I " + privateTestDir + "badRead.test.bam -o %s -L 1:22753424-22753464", 1,
@@ -74,7 +74,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test bad read", spec);
     }
 
-    @Test
+    //@Test
     public void testReverseTrim() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm INDEL -I " + validationDataLocation + "CEUTrio.HiSeq.b37.chr20.10_11mb.bam -o %s -L 20:10289124 -L 20:10090289", 1,
@@ -82,7 +82,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test reverse trim", spec);
     }
 
-    @Test
+    //@Test
     public void testMismatchedPLs() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm INDEL -I " + privateTestDir + "mismatchedPLs.bam -o %s -L 1:24020341", 1,
@@ -98,7 +98,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
     private final static String COMPRESSED_OUTPUT_MD5 = "fd236bd635d514e4214d364f45ec4d10";
 
-    @Test
+    //@Test
     public void testCompressedOutput() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
@@ -112,7 +112,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testParallelization() {
 
         // Note that we need to turn off any randomization for this to work, so no downsampling and no annotations
@@ -145,7 +145,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testMinBaseQualityScore() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --min_base_quality_score 26", 1,
@@ -153,7 +153,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test min_base_quality_score 26", spec);
     }
 
-    @Test
+    //@Test
     public void testSLOD() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH --dbsnp " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -161,7 +161,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test SLOD", spec);
     }
 
-    @Test
+    //@Test
     public void testNDA() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --annotateNDA -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -169,7 +169,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test NDA", spec);
     }
 
-    @Test
+    //@Test
     public void testCompTrack() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH -comp:FOO " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -177,7 +177,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test using comp track", spec);
     }
 
-    @Test
+    //@Test
     public void testNoCmdLineHeaderStdout() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandNoCmdLineHeaderStdout + " -glm INDEL -L 1:67,225,396-67,288,518", 0,
@@ -185,17 +185,17 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("testNoCmdLineHeaderStdout", spec);
     }
 
-    @Test
+    //@Test
     public void testOutputParameterSitesOnly() {
         testOutputParameters("-sites_only", "97ba874eafc9884a4de027a84c036311");
     }
 
-    @Test
+    //@Test
     public void testOutputParameterAllConfident() {
         testOutputParameters("--output_mode EMIT_ALL_CONFIDENT_SITES", "f9ea04d96eeef29e71d37e60518c2579");
     }
 
-    @Test
+    //@Test
     public void testOutputParameterAllSites() {
         testOutputParameters("--output_mode EMIT_ALL_SITES", "41c046d38ea328421df924e37e017645");
     }
@@ -207,7 +207,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest(String.format("testParameter[%s]", args), spec);
     }
 
-    @Test
+    //@Test
     public void testConfidence() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 -stand_call_conf 10 ", 1,
@@ -215,7 +215,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test confidence 1", spec1);
     }
 
-    @Test
+    //@Test
     public void testConfidence2() {
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 -stand_emit_conf 10 ", 1,
@@ -228,12 +228,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing heterozygosity
     //
     // --------------------------------------------------------------------------------------------------------------
-    @Test
+    //@Test
     public void testHeterozyosity1() {
         testHeterozosity( 0.01, "986923de51c71635d47e3d06fe3794a1" );
     }
 
-    @Test
+    //@Test
     public void testHeterozyosity2() {
         testHeterozosity( 1.0 / 1850, "fb12b1553f813004a394a391a8540873" );
     }
@@ -251,7 +251,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing calls with SLX, 454, and SOLID data
     //
     // --------------------------------------------------------------------------------------------------------------
-    @Test
+    //@Test
     public void testMultiTechnologies() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand +
@@ -269,7 +269,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing calls with BAQ
     //
     // --------------------------------------------------------------------------------------------------------------
-    @Test
+    //@Test
     public void testCallingWithBAQ() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand +
@@ -289,7 +289,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
     // Basic indel testing with SLX data
-    @Test
+    //@Test
     public void testSimpleIndels() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels +
@@ -303,7 +303,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     }
 
     // Basic indel testing with SLX data
-    @Test
+    //@Test
     public void testIndelsWithLowMinAlleleCnt() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels +
@@ -317,7 +317,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest(String.format("test indel caller in SLX with low min allele count"), spec);
     }
 
-    @Test
+    //@Test
     public void testMultiTechnologyIndels() {
          WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                  baseCommandIndels +
@@ -330,7 +330,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
          executeTest(String.format("test indel calling, multiple technologies"), spec);
      }
 
-    @Test
+    //@Test
     public void testWithIndelAllelesPassedIn1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "indelAllelesForUG.vcf -I " + validationDataLocation +
@@ -339,7 +339,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 indels with alleles passed in", spec);
     }
 
-    @Test
+    //@Test
     public void testWithIndelAllelesPassedIn2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles "
@@ -349,7 +349,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 indels with alleles passed in and emitting all sites", spec);
     }
 
-    @Test
+    //@Test
     public void testMultiSampleIndels1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10450700-10551000", 1,
@@ -363,7 +363,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot1 CEU indels using GENOTYPE_GIVEN_ALLELES", spec2);
     }
 
-    @Test
+    //@Test
     public void testGGAwithNoEvidenceInReads() {
         final String vcf = "small.indel.test.vcf";
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
@@ -373,7 +373,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test GENOTYPE_GIVEN_ALLELES with no evidence in reads", spec);
     }
 
-    @Test
+    //@Test
     public void testBaseIndelQualityScores() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndelsb37 +
@@ -392,7 +392,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testSnpEffAnnotationRequestedWithoutRodBinding() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000 " +
@@ -411,7 +411,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     final static String assessMinIndelFraction = baseCommandIndelsb37 + " -I " + validationDataLocation
             + "978604.bam -L 1:978,586-978,626 -o %s --sites_only -rf Sample -goodSM 7377 -goodSM 22-0022 -goodSM 134 -goodSM 344029-53 -goodSM 14030";
 
-    @Test
+    //@Test
     public void testMinIndelFraction0() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.0", 1,
@@ -419,7 +419,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test minIndelFraction 0.0", spec);
     }
 
-    @Test
+    //@Test
     public void testMinIndelFraction25() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.25", 1,
@@ -427,7 +427,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test minIndelFraction 0.25", spec);
     }
 
-    @Test
+    //@Test
     public void testMinIndelFraction100() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 1", 1,
@@ -441,7 +441,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testNsInCigar() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -I " + validationDataLocation + "testWithNs.bam -o %s -L 8:141799600-141814700", 1,
@@ -455,7 +455,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testReducedBam() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -I " + privateTestDir + "bamExample.ReducedRead.ADAnnotation.bam -o %s -L 1:67,225,396-67,288,518", 1,
@@ -463,12 +463,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test calling on a ReducedRead BAM", spec);
     }
 
-    @Test
+    //@Test
     public void testReducedBamSNPs() {
         testReducedCalling("SNP", "1d4a826b144723ff0766c36aa0239287");
     }
 
-    @Test
+    //@Test
     public void testReducedBamINDELs() {
         testReducedCalling("INDEL", "68ef51d5c98480e0c0192e0eecb95bca");
     }
@@ -480,4 +480,20 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                 Arrays.asList(md5));
         executeTest("test calling on a ReducedRead BAM with " + model, spec);
     }
+
+    // --------------------------------------------------------------------------------------------------------------
+    //
+    // testing contamination down-sampling
+    //
+    // --------------------------------------------------------------------------------------------------------------
+
+    @Test
+    public void testContaminationDownsampling() {
+        WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
+                baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --contamination_percentage_to_filter 0.20", 1,
+                Arrays.asList("27dd04159e06d9524fb8a4eef41f96ae"));
+        executeTest("test contamination_percentage_to_filter 0.20", spec);
+    }
+
+
 }

From c53c55da12ebf95141f3645ab3197d67e1ca0fb5 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Thu, 25 Oct 2012 09:37:08 -0400
Subject: [PATCH 15/50] Re-enable tests

---
 .../UnifiedGenotyperIntegrationTest.java      | 80 +++++++++----------
 1 file changed, 40 insertions(+), 40 deletions(-)

diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index 9e50a8556..8eb1c4657 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -26,7 +26,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing normal calling
     //
     // --------------------------------------------------------------------------------------------------------------
-    //@Test
+    @Test
     public void testMultiSamplePilot1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000", 1,
@@ -34,7 +34,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot1", spec);
     }
 
-    //@Test
+    @Test
     public void testWithAllelesPassedIn1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
@@ -42,7 +42,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 with alleles passed in", spec1);
     }
 
-    //@Test
+    @Test
     public void testWithAllelesPassedIn2() {
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
@@ -50,7 +50,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 with alleles passed in and emitting all sites", spec2);
     }
 
-    //@Test
+    @Test
     public void testSingleSamplePilot2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
@@ -58,7 +58,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test SingleSample Pilot2", spec);
     }
 
-    //@Test
+    @Test
     public void testMultipleSNPAlleles() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + privateTestDir + "multiallelic.snps.bam -o %s -L " + privateTestDir + "multiallelic.snps.intervals", 1,
@@ -66,7 +66,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test Multiple SNP alleles", spec);
     }
 
-    //@Test
+    @Test
     public void testBadRead() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm BOTH -I " + privateTestDir + "badRead.test.bam -o %s -L 1:22753424-22753464", 1,
@@ -74,7 +74,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test bad read", spec);
     }
 
-    //@Test
+    @Test
     public void testReverseTrim() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm INDEL -I " + validationDataLocation + "CEUTrio.HiSeq.b37.chr20.10_11mb.bam -o %s -L 20:10289124 -L 20:10090289", 1,
@@ -82,7 +82,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test reverse trim", spec);
     }
 
-    //@Test
+    @Test
     public void testMismatchedPLs() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm INDEL -I " + privateTestDir + "mismatchedPLs.bam -o %s -L 1:24020341", 1,
@@ -98,7 +98,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
     private final static String COMPRESSED_OUTPUT_MD5 = "fd236bd635d514e4214d364f45ec4d10";
 
-    //@Test
+    @Test
     public void testCompressedOutput() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
@@ -112,7 +112,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testParallelization() {
 
         // Note that we need to turn off any randomization for this to work, so no downsampling and no annotations
@@ -145,7 +145,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testMinBaseQualityScore() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --min_base_quality_score 26", 1,
@@ -153,7 +153,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test min_base_quality_score 26", spec);
     }
 
-    //@Test
+    @Test
     public void testSLOD() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH --dbsnp " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -161,7 +161,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test SLOD", spec);
     }
 
-    //@Test
+    @Test
     public void testNDA() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --annotateNDA -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -169,7 +169,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test NDA", spec);
     }
 
-    //@Test
+    @Test
     public void testCompTrack() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH -comp:FOO " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -177,7 +177,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test using comp track", spec);
     }
 
-    //@Test
+    @Test
     public void testNoCmdLineHeaderStdout() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandNoCmdLineHeaderStdout + " -glm INDEL -L 1:67,225,396-67,288,518", 0,
@@ -185,17 +185,17 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("testNoCmdLineHeaderStdout", spec);
     }
 
-    //@Test
+    @Test
     public void testOutputParameterSitesOnly() {
         testOutputParameters("-sites_only", "97ba874eafc9884a4de027a84c036311");
     }
 
-    //@Test
+    @Test
     public void testOutputParameterAllConfident() {
         testOutputParameters("--output_mode EMIT_ALL_CONFIDENT_SITES", "f9ea04d96eeef29e71d37e60518c2579");
     }
 
-    //@Test
+    @Test
     public void testOutputParameterAllSites() {
         testOutputParameters("--output_mode EMIT_ALL_SITES", "41c046d38ea328421df924e37e017645");
     }
@@ -207,7 +207,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest(String.format("testParameter[%s]", args), spec);
     }
 
-    //@Test
+    @Test
     public void testConfidence() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 -stand_call_conf 10 ", 1,
@@ -215,7 +215,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test confidence 1", spec1);
     }
 
-    //@Test
+    @Test
     public void testConfidence2() {
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 -stand_emit_conf 10 ", 1,
@@ -228,12 +228,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing heterozygosity
     //
     // --------------------------------------------------------------------------------------------------------------
-    //@Test
+    @Test
     public void testHeterozyosity1() {
         testHeterozosity( 0.01, "986923de51c71635d47e3d06fe3794a1" );
     }
 
-    //@Test
+    @Test
     public void testHeterozyosity2() {
         testHeterozosity( 1.0 / 1850, "fb12b1553f813004a394a391a8540873" );
     }
@@ -251,7 +251,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing calls with SLX, 454, and SOLID data
     //
     // --------------------------------------------------------------------------------------------------------------
-    //@Test
+    @Test
     public void testMultiTechnologies() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand +
@@ -269,7 +269,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing calls with BAQ
     //
     // --------------------------------------------------------------------------------------------------------------
-    //@Test
+    @Test
     public void testCallingWithBAQ() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand +
@@ -289,7 +289,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
     // Basic indel testing with SLX data
-    //@Test
+    @Test
     public void testSimpleIndels() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels +
@@ -303,7 +303,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     }
 
     // Basic indel testing with SLX data
-    //@Test
+    @Test
     public void testIndelsWithLowMinAlleleCnt() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels +
@@ -317,7 +317,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest(String.format("test indel caller in SLX with low min allele count"), spec);
     }
 
-    //@Test
+    @Test
     public void testMultiTechnologyIndels() {
          WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                  baseCommandIndels +
@@ -330,7 +330,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
          executeTest(String.format("test indel calling, multiple technologies"), spec);
      }
 
-    //@Test
+    @Test
     public void testWithIndelAllelesPassedIn1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "indelAllelesForUG.vcf -I " + validationDataLocation +
@@ -339,7 +339,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 indels with alleles passed in", spec);
     }
 
-    //@Test
+    @Test
     public void testWithIndelAllelesPassedIn2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles "
@@ -349,7 +349,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 indels with alleles passed in and emitting all sites", spec);
     }
 
-    //@Test
+    @Test
     public void testMultiSampleIndels1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10450700-10551000", 1,
@@ -363,7 +363,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot1 CEU indels using GENOTYPE_GIVEN_ALLELES", spec2);
     }
 
-    //@Test
+    @Test
     public void testGGAwithNoEvidenceInReads() {
         final String vcf = "small.indel.test.vcf";
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
@@ -373,7 +373,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test GENOTYPE_GIVEN_ALLELES with no evidence in reads", spec);
     }
 
-    //@Test
+    @Test
     public void testBaseIndelQualityScores() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndelsb37 +
@@ -392,7 +392,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testSnpEffAnnotationRequestedWithoutRodBinding() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000 " +
@@ -411,7 +411,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     final static String assessMinIndelFraction = baseCommandIndelsb37 + " -I " + validationDataLocation
             + "978604.bam -L 1:978,586-978,626 -o %s --sites_only -rf Sample -goodSM 7377 -goodSM 22-0022 -goodSM 134 -goodSM 344029-53 -goodSM 14030";
 
-    //@Test
+    @Test
     public void testMinIndelFraction0() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.0", 1,
@@ -419,7 +419,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test minIndelFraction 0.0", spec);
     }
 
-    //@Test
+    @Test
     public void testMinIndelFraction25() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.25", 1,
@@ -427,7 +427,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test minIndelFraction 0.25", spec);
     }
 
-    //@Test
+    @Test
     public void testMinIndelFraction100() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 1", 1,
@@ -441,7 +441,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testNsInCigar() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -I " + validationDataLocation + "testWithNs.bam -o %s -L 8:141799600-141814700", 1,
@@ -455,7 +455,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testReducedBam() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -I " + privateTestDir + "bamExample.ReducedRead.ADAnnotation.bam -o %s -L 1:67,225,396-67,288,518", 1,
@@ -463,12 +463,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test calling on a ReducedRead BAM", spec);
     }
 
-    //@Test
+    @Test
     public void testReducedBamSNPs() {
         testReducedCalling("SNP", "1d4a826b144723ff0766c36aa0239287");
     }
 
-    //@Test
+    @Test
     public void testReducedBamINDELs() {
         testReducedCalling("INDEL", "68ef51d5c98480e0c0192e0eecb95bca");
     }

From 6dc7d872ece8610b0f08a90bc12452984aa26c52 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Thu, 25 Oct 2012 10:06:13 -0400
Subject: [PATCH 16/50] Fix GenotypeAndValidate to handle SNPs and indels as
 reported on the forum.  Recent changes to the UnifiedArgumentCollection made
 this stop working.  Adding in JIRA to create integration tests for this tool.

---
 .../sting/gatk/walkers/validation/GenotypeAndValidate.java   | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/GenotypeAndValidate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/GenotypeAndValidate.java
index 832c0ede3..79b6e17f6 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/GenotypeAndValidate.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/GenotypeAndValidate.java
@@ -333,8 +333,9 @@ public class GenotypeAndValidate extends RodWalker<GenotypeAndValidate.CountedDa
         snpEngine = new UnifiedGenotyperEngine(getToolkit(), uac);
 
         // Adding the INDEL calling arguments for UG
-        uac.GLmodel = GenotypeLikelihoodsCalculationModel.Model.INDEL;
-        indelEngine = new UnifiedGenotyperEngine(getToolkit(), uac);
+        UnifiedArgumentCollection uac_indel = new UnifiedArgumentCollection(uac);
+        uac_indel.GLmodel = GenotypeLikelihoodsCalculationModel.Model.INDEL;
+        indelEngine = new UnifiedGenotyperEngine(getToolkit(), uac_indel);
 
         // make sure we have callConf set to the threshold set by the UAC so we can use it later.
         callConf = uac.STANDARD_CONFIDENCE_FOR_CALLING;

From e93ff3ea6ea864984e0a132239a7b34534b11629 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Thu, 25 Oct 2012 12:45:23 -0400
Subject: [PATCH 17/50] Let's go back to having the SB/SLOD NOT computed by
 default.  If you recall, it was only enabled by default because we thought we
 were going to use it when we made VQSR use random forests.  But since we
 decided not to change VQSR, there's no reason to triple the computation for
 every variant site anymore.

---
 .../genotyper/UnifiedArgumentCollection.java   |  6 +++---
 .../walkers/genotyper/UnifiedGenotyper.java    |  2 +-
 .../genotyper/UnifiedGenotyperEngine.java      |  2 +-
 .../walkers/variantutils/SelectVariants.java   |  1 -
 .../UnifiedGenotyperIntegrationTest.java       | 18 +++++++++---------
 5 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java
index e50f25fe8..5f6ddf0f1 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java
@@ -47,8 +47,8 @@ public class UnifiedArgumentCollection extends StandardCallerArgumentCollection
     /**
      * Note that calculating the SLOD increases the runtime by an appreciable amount.
      */
-    @Argument(fullName = "noSLOD", shortName = "nosl", doc = "If provided, we will not calculate the SLOD", required = false)
-    public boolean NO_SLOD = false;
+    @Argument(fullName = "computeSLOD", shortName = "slod", doc = "If provided, we will calculate the SLOD (SB annotation)", required = false)
+    public boolean COMPUTE_SLOD = false;
 
     /**
      * Depending on the value of the --max_alternate_alleles argument, we may genotype only a fraction of the alleles being sent on for genotyping.
@@ -204,7 +204,7 @@ public class UnifiedArgumentCollection extends StandardCallerArgumentCollection
         this.GLmodel = uac.GLmodel;
         this.AFmodel = uac.AFmodel;
         this.PCR_error = uac.PCR_error;
-        this.NO_SLOD = uac.NO_SLOD;
+        this.COMPUTE_SLOD = uac.COMPUTE_SLOD;
         this.ANNOTATE_NUMBER_OF_ALLELES_DISCOVERED = uac.ANNOTATE_NUMBER_OF_ALLELES_DISCOVERED;
         this.MIN_BASE_QUALTY_SCORE = uac.MIN_BASE_QUALTY_SCORE;
         this.MAX_DELETION_FRACTION = uac.MAX_DELETION_FRACTION;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
index a17c2aa1a..ef56eaa6d 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
@@ -290,7 +290,7 @@ public class UnifiedGenotyper extends LocusWalker<List<VariantCallContext>, Unif
             headerInfo.addAll(annotationEngine.getVCFAnnotationDescriptions());
 
         // annotation (INFO) fields from UnifiedGenotyper
-        if ( !UAC.NO_SLOD )
+        if ( UAC.COMPUTE_SLOD )
             VCFStandardHeaderLines.addStandardInfoLines(headerInfo, true, VCFConstants.STRAND_BIAS_KEY);
 
         if ( UAC.ANNOTATE_NUMBER_OF_ALLELES_DISCOVERED )
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java
index 3c4a97ec1..97254c478 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java
@@ -451,7 +451,7 @@ public class UnifiedGenotyperEngine {
             attributes.put(VCFConstants.MLE_ALLELE_FREQUENCY_KEY, MLEfrequencies);
         }
 
-        if ( !UAC.NO_SLOD && !limitedContext && !bestGuessIsRef ) {
+        if ( UAC.COMPUTE_SLOD && !limitedContext && !bestGuessIsRef ) {
             //final boolean DEBUG_SLOD = false;
 
             // the overall lod
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java
index c7b1d0fc7..d1b7cb96f 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java
@@ -459,7 +459,6 @@ public class SelectVariants extends RodWalker<Integer, Integer> implements TreeR
             UAC.GLmodel = GenotypeLikelihoodsCalculationModel.Model.BOTH;
             UAC.OutputMode = UnifiedGenotyperEngine.OUTPUT_MODE.EMIT_ALL_SITES;
             UAC.GenotypingMode = GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES;
-            UAC.NO_SLOD = true;
             UG_engine = new UnifiedGenotyperEngine(getToolkit(), UAC, logger, null, null, samples, VariantContextUtils.DEFAULT_PLOIDY);
             headerLines.addAll(UnifiedGenotyper.getHeaderInfo(UAC, null, null));
         }
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index 8eb1c4657..446f8cbe8 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -16,9 +16,9 @@ import java.util.List;
 
 public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
-    private final static String baseCommand = "-T UnifiedGenotyper -R " + b36KGReference + " -nosl --no_cmdline_in_header -glm BOTH -minIndelFrac 0.0 --dbsnp " + b36dbSNP129;
-    private final static String baseCommandIndels = "-T UnifiedGenotyper -R " + b36KGReference + " -nosl --no_cmdline_in_header -glm INDEL -mbq 20 -minIndelFrac 0.0 --dbsnp " + b36dbSNP129;
-    private final static String baseCommandIndelsb37 = "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm INDEL -mbq 20 --dbsnp " + b37dbSNP132;
+    private final static String baseCommand = "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH -minIndelFrac 0.0 --dbsnp " + b36dbSNP129;
+    private final static String baseCommandIndels = "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm INDEL -mbq 20 -minIndelFrac 0.0 --dbsnp " + b36dbSNP129;
+    private final static String baseCommandIndelsb37 = "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm INDEL -mbq 20 --dbsnp " + b37dbSNP132;
     private final static String baseCommandNoCmdLineHeaderStdout = "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -I " + privateTestDir + "bamExample.ReducedRead.ADAnnotation.bam";
 
     // --------------------------------------------------------------------------------------------------------------
@@ -61,7 +61,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testMultipleSNPAlleles() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-                "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + privateTestDir + "multiallelic.snps.bam -o %s -L " + privateTestDir + "multiallelic.snps.intervals", 1,
+                "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + privateTestDir + "multiallelic.snps.bam -o %s -L " + privateTestDir + "multiallelic.snps.intervals", 1,
                 Arrays.asList("543f68e42034bf44cfb24da8c9204320"));
         executeTest("test Multiple SNP alleles", spec);
     }
@@ -69,7 +69,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testBadRead() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-                "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm BOTH -I " + privateTestDir + "badRead.test.bam -o %s -L 1:22753424-22753464", 1,
+                "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm BOTH -I " + privateTestDir + "badRead.test.bam -o %s -L 1:22753424-22753464", 1,
                 Arrays.asList("d915535c1458733f09f82670092fcab6"));
         executeTest("test bad read", spec);
     }
@@ -77,7 +77,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testReverseTrim() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-                "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm INDEL -I " + validationDataLocation + "CEUTrio.HiSeq.b37.chr20.10_11mb.bam -o %s -L 20:10289124 -L 20:10090289", 1,
+                "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm INDEL -I " + validationDataLocation + "CEUTrio.HiSeq.b37.chr20.10_11mb.bam -o %s -L 20:10289124 -L 20:10090289", 1,
                 Arrays.asList("5ce03dd9ca2d9324c1d4a9d64389beb5"));
         executeTest("test reverse trim", spec);
     }
@@ -85,7 +85,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testMismatchedPLs() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-                "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm INDEL -I " + privateTestDir + "mismatchedPLs.bam -o %s -L 1:24020341", 1,
+                "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm INDEL -I " + privateTestDir + "mismatchedPLs.bam -o %s -L 1:24020341", 1,
                 Arrays.asList("3c006b06b17bbe8e787d64eff6a63a19"));
         executeTest("test mismatched PLs", spec);
     }
@@ -156,7 +156,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testSLOD() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-                "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH --dbsnp " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
+                "-T UnifiedGenotyper -R " + b36KGReference + " --computeSLOD --no_cmdline_in_header -glm BOTH --dbsnp " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
                 Arrays.asList("c7429e670ba477bf9a6bbee2fb41c5a9"));
         executeTest("test SLOD", spec);
     }
@@ -173,7 +173,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testCompTrack() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH -comp:FOO " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
-                Arrays.asList("8a9b424e00cdbe6b5e73d517335b2186"));
+                Arrays.asList("a6de72c5197f152be98a6f9029091a9e"));
         executeTest("test using comp track", spec);
     }
 

From cc8c12b9548cc24d13dbcc9f798e3f3dab4ded98 Mon Sep 17 00:00:00 2001
From: Mark DePristo <depristo@broadinstitute.org>
Date: Thu, 25 Oct 2012 14:46:09 -0400
Subject: [PATCH 18/50] Committing a broken version of BaseRecalibration

-- I'm committing because there's some kind of fundamental problem with the ReadCovariates cache, in that historical data isn't being cleared / computed properly, and I'd rather it fail for a while than leave it in JIRA.
-- The integration tests test the -nct with PrintReads to get 1, 2, 4 and the 4 fails.  But that's because of this incorrect calculation
-- Updating GATKPerformanceOverTime with the new @ClassType annotation
---
 .../walkers/bqsr/BQSRIntegrationTest.java     | 22 +++++++++++++------
 .../recalibration/BaseRecalibration.java      |  9 +++++---
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
index 90d67d393..a50219f48 100644
--- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
+++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
@@ -5,7 +5,9 @@ import org.broadinstitute.sting.utils.exceptions.UserException;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 
 /**
  * @author ebanks
@@ -118,20 +120,26 @@ public class BQSRIntegrationTest extends WalkerTest {
 
     @DataProvider(name = "PRTest")
     public Object[][] createPRTestData() {
-        return new Object[][]{
-                {new PRTest("", "ab2f209ab98ad3432e208cbd524a4c4a")},
-                {new PRTest(" -qq -1", "5226c06237b213b9e9b25a32ed92d09a")},
-                {new PRTest(" -qq 6", "b592a5c62b952a012e18adb898ea9c33")},
-                {new PRTest(" -DIQ", "8977bea0c57b808e65e9505eb648cdf7")}
-        };
+        List<Object[]> tests = new ArrayList<Object[]>();
+
+        tests.add(new Object[]{1, new PRTest(" -qq -1", "a1d87da5dcbde35170d6ba6bc3ee2812")});
+        tests.add(new Object[]{1, new PRTest(" -qq 6", "a0fecae6d0e5ab9917862fa306186d10")});
+        tests.add(new Object[]{1, new PRTest(" -DIQ", "8977bea0c57b808e65e9505eb648cdf7")});
+
+        for ( final int nct : Arrays.asList(1, 2, 4) ) {
+            tests.add(new Object[]{nct, new PRTest("", "d1bbb4ce6aa93e866f106f8b11d888ed")});
+        }
+
+        return tests.toArray(new Object[][]{});
     }
 
     @Test(dataProvider = "PRTest")
-    public void testPR(PRTest params) {
+    public void testPR(final int nct, PRTest params) {
         WalkerTestSpec spec = new WalkerTestSpec(
                 "-T PrintReads" +
                         " -R " + hg18Reference +
                         " -I " + privateTestDir + "HiSeq.1mb.1RG.bam" +
+                        " -nct " + nct +
                         " -BQSR " + privateTestDir + "HiSeq.20mb.1RG.table" +
                         params.args +
                         " -o %s",
diff --git a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java
index 7ad9302a8..2f9387acb 100644
--- a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java
+++ b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java
@@ -116,9 +116,12 @@ public class BaseRecalibration {
         // Compute all covariates for the read
         // TODO -- the need to clear here suggests there's an error in the indexing / assumption code
         // TODO -- for BI and DI. Perhaps due to the indel buffer size on the ends of the reads?
-        // TODO -- the output varies with -nt 1 and -nt 2 if you don't call clear here
-        // TODO -- needs to be fixed.
-        final ReadCovariates readCovariates = readCovariatesCache.get().clear();
+        // TODO -- the output varies depending on whether we clear or not
+        //final ReadCovariates readCovariates = readCovariatesCache.get().clear();
+
+        // the original code -- doesn't do any clearing
+        final ReadCovariates readCovariates = readCovariatesCache.get();
+
         RecalUtils.computeCovariates(read, requestedCovariates, readCovariates);
 
         for (final EventType errorModel : EventType.values()) { // recalibrate all three quality strings

From 884d031e72881f658ecd7df9803bdd930c64163b Mon Sep 17 00:00:00 2001
From: David Roazen <droazen@broadinstitute.org>
Date: Thu, 25 Oct 2012 13:30:49 -0400
Subject: [PATCH 19/50] NestedIntegerArray: Pre-allocate only the first two
 dimensions

It turns out that pre-allocating the entire tree was too expensive in
terms of memory when using large values for the -mcs and -ics parameters.

Pre-allocating the first two dimensions prevents us from ever locking the
root node during a put(). Contention between threads over lower levels
of the tree should be minimal given that puts are rare compared to gets.

Also output dimensions and pre-allocation info at startup. If pre-allocation
takes longer than usual this gives the user a sense of what is causing the
delay.
---
 .../utils/collections/NestedIntegerArray.java | 51 ++++++++++++++-----
 1 file changed, 37 insertions(+), 14 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java b/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java
index c83de8ec7..050ed52ac 100755
--- a/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java
+++ b/public/java/src/org/broadinstitute/sting/utils/collections/NestedIntegerArray.java
@@ -25,9 +25,11 @@
 
 package org.broadinstitute.sting.utils.collections;
 
+import org.apache.log4j.Logger;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 /**
@@ -38,39 +40,50 @@ import java.util.List;
 
 public class NestedIntegerArray<T> {
 
+    private static Logger logger = Logger.getLogger(NestedIntegerArray.class);
+
     protected final Object[] data;
 
     protected final int numDimensions;
     protected final int[] dimensions;
 
+    // Preallocate the first two dimensions to limit contention during tree traversals in put()
+    private static final int NUM_DIMENSIONS_TO_PREALLOCATE = 2;
+
     public NestedIntegerArray(final int... dimensions) {
         numDimensions = dimensions.length;
         if ( numDimensions == 0 )
             throw new ReviewedStingException("There must be at least one dimension to an NestedIntegerArray");
         this.dimensions = dimensions.clone();
 
+        int dimensionsToPreallocate = Math.min(dimensions.length, NUM_DIMENSIONS_TO_PREALLOCATE);
+
+        logger.info(String.format("Creating NestedIntegerArray with dimensions %s", Arrays.toString(dimensions)));
+        logger.info(String.format("Pre-allocating first %d dimensions", dimensionsToPreallocate));
+
         data = new Object[dimensions[0]];
-        prepopulateArray(data, 0);
+        preallocateArray(data, 0, dimensionsToPreallocate);
+
+        logger.info(String.format("Done pre-allocating first %d dimensions", dimensionsToPreallocate));
     }
 
     /**
-     * Recursively allocate the entire tree of arrays in all its dimensions.
+     * Recursively allocate the first dimensionsToPreallocate dimensions of the tree
      *
-     * Doing this upfront uses more memory initially, but saves time over the course of the run
-     * and (crucially) avoids having to make threads wait while traversing the tree to check
-     * whether branches exist or not.
+     * Pre-allocating the first few dimensions helps limit contention during tree traversals in put()
      *
      * @param subarray current node in the tree
      * @param dimension current level in the tree
+     * @param dimensionsToPreallocate preallocate only this many dimensions (starting from the first)
      */
-    private void prepopulateArray( Object[] subarray, int dimension ) {
-        if ( dimension >= numDimensions - 1 ) {
+    private void preallocateArray( Object[] subarray, int dimension, int dimensionsToPreallocate ) {
+        if ( dimension >= dimensionsToPreallocate - 1 ) {
             return;
         }
 
         for ( int i = 0; i < subarray.length; i++ ) {
             subarray[i] = new Object[dimensions[dimension + 1]];
-            prepopulateArray((Object[])subarray[i], dimension + 1);
+            preallocateArray((Object[])subarray[i], dimension + 1, dimensionsToPreallocate);
         }
     }
 
@@ -82,8 +95,9 @@ public class NestedIntegerArray<T> {
             if ( keys[i] >= dimensions[i] )
                 return null;
 
-            myData = (Object[])myData[keys[i]];  // interior nodes in the tree will never be null, so we can safely traverse
-                                                 // down to the leaves
+            myData = (Object[])myData[keys[i]];
+            if ( myData == null )
+                return null;
         }
 
         return (T)myData[keys[numNestedDimensions]];
@@ -92,8 +106,8 @@ public class NestedIntegerArray<T> {
     /**
      * Insert a value at the position specified by the given keys.
      *
-     * This method is THREAD-SAFE despite not being synchronized, however the caller MUST
-     * check the return value to see if the put succeeded. This method RETURNS FALSE if
+     * This method is thread-safe, however the caller MUST check the
+     * return value to see if the put succeeded. This method RETURNS FALSE if
      * the value could not be inserted because there already was a value present
      * at the specified location. In this case the caller should do a get() to get
      * the already-existing value and (potentially) update it.
@@ -113,8 +127,17 @@ public class NestedIntegerArray<T> {
             if ( keys[i] >= dimensions[i] )
                 throw new ReviewedStingException("Key " + keys[i] + " is too large for dimension " + i + " (max is " + (dimensions[i]-1) + ")");
 
-            myData = (Object[])myData[keys[i]];  // interior nodes in the tree will never be null, so we can safely traverse
-                                                 // down to the leaves
+            // If we're at or beyond the last dimension that was pre-allocated, we need to do a synchronized
+            // check to see if the next branch exists, and if it doesn't, create it
+            if ( i >= NUM_DIMENSIONS_TO_PREALLOCATE - 1 ) {
+                synchronized ( myData ) {
+                    if ( myData[keys[i]] == null ) {
+                        myData[keys[i]] = new Object[dimensions[i + 1]];
+                    }
+                }
+            }
+
+            myData = (Object[])myData[keys[i]];
         }
 
         synchronized ( myData ) {   // lock the bottom row while we examine and (potentially) update it

From dde3060bb84ecbb83878daf834b5d922eb4ef8d4 Mon Sep 17 00:00:00 2001
From: Ami Levy Moonshine <ami@broadinstitute.org>
Date: Thu, 25 Oct 2012 15:36:17 -0400
Subject: [PATCH 20/50] add the CEUtrio best practices results (UG + PBT) to
 the bundle

---
 .../gatk/walkers/qc/AssessReducedQuals.java   | 123 +++++++++++++-----
 .../queue/qscripts/GATKResourcesBundle.scala  |   9 +-
 2 files changed, 96 insertions(+), 36 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java
index c8ec33364..e74684c53 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java
@@ -1,5 +1,6 @@
 package org.broadinstitute.sting.gatk.walkers.qc;
 
+import org.broadinstitute.sting.commandline.Argument;
 import org.broadinstitute.sting.commandline.Output;
 import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
@@ -7,73 +8,125 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.LocusWalker;
 import org.broadinstitute.sting.gatk.walkers.TreeReducible;
 import org.broadinstitute.sting.utils.GenomeLoc;
+import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 
 import java.io.PrintStream;
+import java.util.List;
 
 /**
- * Created with IntelliJ IDEA.
- * User: ami
- * Date: 10/19/12
- * Time: 9:09 AM
- * To change this template use File | Settings | File Templates.
+ * Emits intervals in which the differences between the original and reduced bam quals are bigger epsilon (unless the quals of
+ * the reduced bam are above sufficient threshold)
+ *
+ * <h2>Input</h2>
+ * <p>
+ * The original and reduced BAM files.
+ * </p>
+ *
+ * <h2>Output</h2>
+ * <p>
+ * A list of intervals in which the differences between the original and reduced bam quals are bigger epsilon.
+ * </p>
+ *
+ * <h2>Examples</h2>
+ * <pre>
+ * java -Xmx2g -jar GenomeAnalysisTK.jar \
+ *   -I:original original.bam \
+ *   -I:reduced reduced.bam \
+ *   -R ref.fasta \
+ *   -T AssessReducedQuals \
+ *   -o output.intervals
+ * </pre>
+ *
+ * @author ami
  */
+
 public class AssessReducedQuals extends LocusWalker<GenomeLoc, GenomeLoc> implements TreeReducible<GenomeLoc> {
 
-    private static final String original = "original";
     private static final String reduced = "reduced";
+    private static final String original = "original";
+    private static final int originalQualsIndex = 0;
+    private static final int reducedQualsIndex = 1;
+
+    @Argument(fullName = "sufficientQualSum", shortName = "sufficientQualSum", doc = "When a reduced bam qual sum is above this threshold, it passes even without comparing to the non-reduced bam ", required = false)
+    public int sufficientQualSum = 600;
+
+    @Argument(fullName = "qual_epsilon", shortName = "epsilon", doc = "when |Quals_reduced_bam - Quals_original_bam| > epsilon*Quals_original_bam we output this interval", required = false)
+    public int qual_epsilon = 0;
+
+    @Argument(fullName = "debugLevel", shortName = "debug", doc = "debug mode on")
+    public int debugLevel = 0;
 
     @Output
     protected PrintStream out;
 
+    public void initialize() {
+        if (debugLevel != 0)
+            out.println("  Debug mode" +
+                        "Debug:\tsufficientQualSum: "+sufficientQualSum+ "\n " +
+                        "Debug:\tqual_epsilon: "+qual_epsilon);
+    }
+
     @Override
     public boolean includeReadsWithDeletionAtLoci() { return true; }
 
-    public void initialize() {}    //todo: why we need that?
-
     @Override
     public GenomeLoc map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
         if ( tracker == null )
             return null;
 
-        int originalQualsIndex = 0;
-        int reducedQualsIndex = 1;
-        double epsilon = 0;
-        double[] quals = getPileupQuals(context.getBasePileup());
-        return (quals[originalQualsIndex] - quals[reducedQualsIndex] >= epsilon || quals[originalQualsIndex] - quals[reducedQualsIndex] <= -1*epsilon) ? ref.getLocus() : null;
+        boolean reportLocus;
+        final int[] quals = getPileupQuals(context.getBasePileup());
+        if (debugLevel != 0)
+            out.println("Debug:\tLocus Quals\t"+ref.getLocus()+"\toriginal\t"+quals[originalQualsIndex]+"\treduced\t"+quals[reducedQualsIndex]);
+        final int epsilon = MathUtils.fastRound(quals[originalQualsIndex]*qual_epsilon);
+        final int calcOriginalQuals = Math.min(quals[originalQualsIndex],sufficientQualSum);
+        final int calcReducedQuals = Math.min(quals[reducedQualsIndex],sufficientQualSum);
+        final int OriginalReducedQualDiff = calcOriginalQuals - calcReducedQuals;
+        reportLocus = OriginalReducedQualDiff > epsilon || OriginalReducedQualDiff < -1*epsilon;
+        if(debugLevel != 0 && reportLocus)
+            out.println("Debug:\tEmited Locus\t"+ref.getLocus()+"\toriginal\t"+quals[originalQualsIndex]+"\treduced\t"+quals[reducedQualsIndex]+"\tepsilon\t"+epsilon+"\tdiff\t"+OriginalReducedQualDiff);
 
+        return reportLocus ? ref.getLocus() : null;
     }
 
-    private double[] getPileupQuals(final ReadBackedPileup readPileup) {
+    private final int[] getPileupQuals(final ReadBackedPileup readPileup) {
 
-        int originalQualsIndex = 0;
-        int reducedQualsIndex = 1;
-        double[] quals = new double[2];
+        final int[] quals = new int[2];
+        String[] printPileup = {"Debug 2:\toriginal pileup:\t"+readPileup.getLocation()+"\nDebug 2:----------------------------------\n",
+                                "Debug 2:\t reduced pileup:\t"+readPileup.getLocation()+"\nDebug 2:----------------------------------\n"};
 
-        for( PileupElement p : readPileup){
-            if ( (int)p.getQual() > 2 && p.getMappingQual() > 0 && !p.isDeletion() ){
-                if (p.getRead().isReducedRead()){
-                    double tempQual = (double)(p.getQual()) * p.getRepresentativeCount();
-                    quals[reducedQualsIndex] += tempQual;
-                }
-                else
-                {
-                    double tempQual = (double)(p.getQual());
-                    quals[originalQualsIndex] += tempQual;
-                }
+        for( PileupElement p : readPileup ){
+            final List<String> tags = getToolkit().getReaderIDForRead(p.getRead()).getTags().getPositionalTags();
+            if ( isGoodRead(p,tags) ){
+                final int tempQual = (int)(p.getQual()) * p.getRepresentativeCount();
+                final int tagIndex = getTagIndex(tags);
+                quals[tagIndex] += tempQual;
+                if(debugLevel == 2)
+                    printPileup[tagIndex] += "\tDebug 2: ("+p+")\tMQ="+p.getMappingQual()+":QU="+p.getQual()+":RC="+p.getRepresentativeCount()+":OS="+p.getOffset()+"\n";
             }
-
         }
-
-
+        if(debugLevel == 2){
+            out.println(printPileup[originalQualsIndex]);
+            out.println(printPileup[reducedQualsIndex]);
+        }
         return quals;
     }
 
-    //public void onTraversalDone(GenomeLoc sum) {
-    //    if ( sum != null )
-    //        out.println(sum);
-    //}
+    private final boolean isGoodRead(PileupElement p, List<String> tags){
+        return !p.isDeletion() && (tags.contains(reduced) || (tags.contains(original) && (int)p.getQual() >= 20 && p.getMappingQual() >= 20));
+    }
+
+    private final int getTagIndex(List<String> tags){
+        return tags.contains(reduced) ? 1 : 0;
+    }
+
+    @Override
+    public void onTraversalDone(GenomeLoc sum) {
+        if ( sum != null )
+            out.println(sum);
+    }
 
     @Override
     public GenomeLoc treeReduce(GenomeLoc lhs, GenomeLoc rhs) {
diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala
index 5e66520ca..9e8815762 100755
--- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala
+++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala
@@ -136,7 +136,14 @@ class GATKResourcesBundle extends QScript {
 
     addResource(new Resource("/humgen/gsa-hpprojects/GATK/data/Comparisons/Unvalidated/GoldStandardIndel/gold.standard.indel.MillsAnd1000G.b37.vcf",
       "Mills_and_1000G_gold_standard.indels", b37, true, false))
-    
+
+    //
+    // CEU trio (NA12878,NA12891,NA12892) best practices results (including PBT)
+    //
+
+    addResource(new Resource("/humgen/gsa-hpprojects/NA12878Collection/callsets/CEUtrio_BestPractices/current/CEUTrio.HiSeq.WGS.b37.UG.snps_and_indels.recalibrated.filtered.phaseByTransmission.vcf",
+      "CEUTrio.HiSeq.WGS.b37.UG.bestPractices.phaseByTransmission",b37,true,false))
+
     //
     // example call set for wiki tutorial
     //

From 422e16c62e7b99c1928de1655dfa359dc9945428 Mon Sep 17 00:00:00 2001
From: David Roazen <droazen@broadinstitute.org>
Date: Thu, 25 Oct 2012 16:16:00 -0400
Subject: [PATCH 21/50] BaseRecalibration: don't cache instances of
 ReadCovariates across reads

Caching and reusing ReadCovariates instances across reads sounds good in theory, but:

-it doesn't work unless you zero out the internal arrays before each read
-the internal arrays must be sized proportionally to the maximum POSSIBLE
recalibrated read length (5000!!!), instead of the ACTUAL read lengths

By contrast, creating a new instance per read is basically equivalent to doing an
efficient low-level memset-style clear on a much smaller array (since we use the actual
rather than the maximum read length to create it). So this should be faster than caching
instances and calling clear() but slower than caching instances and not calling clear().

Credit to Ryan to proposing this approach.
---
 .../recalibration/BaseRecalibration.java      | 22 +------------------
 1 file changed, 1 insertion(+), 21 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java
index 2f9387acb..5d4020a07 100644
--- a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java
+++ b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java
@@ -61,17 +61,6 @@ public class BaseRecalibration {
 //            qualityScoreByFullCovariateKey[i] = new NestedHashMap();
 //    }
 
-    /**
-     * Thread local cache to allow multi-threaded use of this class
-     */
-    private ThreadLocal<ReadCovariates> readCovariatesCache;
-    {
-        readCovariatesCache = new ThreadLocal<ReadCovariates> () {
-            @Override protected ReadCovariates initialValue() {
-                return new ReadCovariates(MAXIMUM_RECALIBRATED_READ_LENGTH, requestedCovariates.length);
-            }
-        };
-    }
 
     /**
      * Constructor using a GATK Report file
@@ -113,16 +102,7 @@ public class BaseRecalibration {
             }
         }
 
-        // Compute all covariates for the read
-        // TODO -- the need to clear here suggests there's an error in the indexing / assumption code
-        // TODO -- for BI and DI. Perhaps due to the indel buffer size on the ends of the reads?
-        // TODO -- the output varies depending on whether we clear or not
-        //final ReadCovariates readCovariates = readCovariatesCache.get().clear();
-
-        // the original code -- doesn't do any clearing
-        final ReadCovariates readCovariates = readCovariatesCache.get();
-
-        RecalUtils.computeCovariates(read, requestedCovariates, readCovariates);
+        final ReadCovariates readCovariates = RecalUtils.computeCovariates(read, requestedCovariates);
 
         for (final EventType errorModel : EventType.values()) { // recalibrate all three quality strings
             if (disableIndelQuals && errorModel != EventType.BASE_SUBSTITUTION) {

From 6b8b7df6517647d05bedf3acc9411c599e5a95cd Mon Sep 17 00:00:00 2001
From: Mark DePristo <depristo@broadinstitute.org>
Date: Thu, 25 Oct 2012 17:26:29 -0400
Subject: [PATCH 22/50] Queue now understands -nct and requests the appropriate
 number of cores from LSF, SGE, etc

-- NCT wasn't previously recognized by Queue as needing more processors per machine.  This commit fixes this.  Also a potential cause of poor GATKPerformanceOverTime, in that runs with -nct could flood a node and cause it to have hundreds of cores in contention.
---
 .../sting/queue/extensions/gatk/ArgumentDefinitionField.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java
index 368cb3d11..2226c6458 100644
--- a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java
+++ b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java
@@ -346,7 +346,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
 
         @Override
         protected String getFreezeFields() {
-            return String.format("if (num_threads.isDefined) nCoresRequest = num_threads%n");
+            return String.format("if (num_threads.isDefined) nCoresRequest = num_threads%nif (num_cpu_threads_per_data_thread.isDefined) nCoresRequest = Some(nCoresRequest.getOrElse(1) * num_cpu_threads_per_data_thread.getOrElse(1))%n");
         }
     }
 

From 91f2c847a33dd5966deacaae7ff9d5481f20f9db Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Fri, 26 Oct 2012 00:57:40 -0400
Subject: [PATCH 24/50] Fixing problem reported on forum for VF: DP couldn't be
 filtered from the FORMAT field, only from the INFO field.  Fixed and added
 integration test.

---
 .../variantcontext/VariantJEXLContext.java     |  2 ++
 .../VariantFiltrationIntegrationTest.java      | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantJEXLContext.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantJEXLContext.java
index 913615a84..abe85e383 100644
--- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantJEXLContext.java
+++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantJEXLContext.java
@@ -193,6 +193,8 @@ class JEXLMap implements Map<VariantContextUtils.JexlVCMatchExp, Boolean> {
                 infoMap.put("isHet", g.isHet() ? "1" : "0");
                 infoMap.put("isHomVar", g.isHomVar() ? "1" : "0");
                 infoMap.put(VCFConstants.GENOTYPE_QUALITY_KEY, g.getGQ());
+                if ( g.hasDP() )
+                    infoMap.put(VCFConstants.DEPTH_KEY, g.getDP());
                 for ( Map.Entry<String, Object> e : g.getExtendedAttributes().entrySet() ) {
                     if ( e.getValue() != null && !e.getValue().equals(VCFConstants.MISSING_VALUE_v4) )
                         infoMap.put(e.getKey(), e.getValue());
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/filters/VariantFiltrationIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/filters/VariantFiltrationIntegrationTest.java
index 97b985a29..27e5f3d46 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/filters/VariantFiltrationIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/filters/VariantFiltrationIntegrationTest.java
@@ -108,4 +108,22 @@ public class VariantFiltrationIntegrationTest extends WalkerTest {
                 Arrays.asList("8ed32a2272bab8043a255362335395ef"));
         executeTest("testUnfilteredBecomesFilteredAndPass", spec);
     }
+
+    @Test
+    public void testFilteringDPfromINFO() {
+        WalkerTestSpec spec = new WalkerTestSpec(
+                "-T VariantFiltration -o %s --no_cmdline_in_header -R " + b37KGReference
+                        + " --filterExpression 'DP < 8' --filterName lowDP -V " + privateTestDir + "filteringDepthInFormat.vcf", 1,
+                Arrays.asList("a01f7cce53ea556c9741aa60b6124c41"));
+        executeTest("testFilteringDPfromINFO", spec);
+    }
+
+    @Test
+    public void testFilteringDPfromFORMAT() {
+        WalkerTestSpec spec = new WalkerTestSpec(
+                "-T VariantFiltration -o %s --no_cmdline_in_header -R " + b37KGReference
+                        + " --genotypeFilterExpression 'DP < 8' --genotypeFilterName lowDP -V " + privateTestDir + "filteringDepthInFormat.vcf", 1,
+                Arrays.asList("e10485c7c33d9211d0c1294fd7858476"));
+        executeTest("testFilteringDPfromFORMAT", spec);
+    }
 }

From bf3d61ce825caa65c17a61429d7fb8939b36af9d Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Fri, 26 Oct 2012 01:04:51 -0400
Subject: [PATCH 25/50] The default value for
 --contamination_fraction_to_filter is now 0.05 (5%) in both UG and HC.  Users
 of GATK-lite get pushed down to 0% by default (since it's not enabled) or get
 a user error if they try to set it.

---
 ...GenotyperGeneralPloidyIntegrationTest.java | 14 +++---
 .../HaplotypeCallerIntegrationTest.java       | 18 ++++----
 .../StandardCallerArgumentCollection.java     |  5 +-
 .../walkers/genotyper/UnifiedGenotyper.java   |  8 +++-
 .../UnifiedGenotyperIntegrationTest.java      | 46 ++++++++-----------
 5 files changed, 45 insertions(+), 46 deletions(-)

diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperGeneralPloidyIntegrationTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperGeneralPloidyIntegrationTest.java
index ae3befe66..73bc8fba6 100644
--- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperGeneralPloidyIntegrationTest.java
+++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperGeneralPloidyIntegrationTest.java
@@ -55,36 +55,36 @@ public class UnifiedGenotyperGeneralPloidyIntegrationTest extends WalkerTest {
 
     @Test(enabled = true)
     public void testSNP_ACS_Pools() {
-        PC_LSV_Test_short(" -maxAltAlleles 1 -ploidy 6 -out_mode EMIT_ALL_CONFIDENT_SITES","LSV_SNP_ACS","SNP","ec19f0b7c7d57493cecfff988a4815c8");
+        PC_LSV_Test_short(" -maxAltAlleles 1 -ploidy 6 -out_mode EMIT_ALL_CONFIDENT_SITES","LSV_SNP_ACS","SNP","df0e67c975ef74d593f1c704daab1705");
     }
 
     @Test(enabled = true)
     public void testBOTH_GGA_Pools() {
-        PC_LSV_Test(String.format(" -maxAltAlleles 2 -ploidy 24 -gt_mode GENOTYPE_GIVEN_ALLELES -out_mode EMIT_ALL_SITES -alleles %s",LSV_ALLELES),"LSV_BOTH_GGA","BOTH","9ce24f4ff787aed9d3754519a60ef49f");
+        PC_LSV_Test(String.format(" -maxAltAlleles 2 -ploidy 24 -gt_mode GENOTYPE_GIVEN_ALLELES -out_mode EMIT_ALL_SITES -alleles %s",LSV_ALLELES),"LSV_BOTH_GGA","BOTH","7e5b28c9e21cc7e45c58c41177d8a0fc");
     }
 
     @Test(enabled = true)
     public void testINDEL_GGA_Pools() {
-        PC_LSV_Test(String.format(" -maxAltAlleles 1 -ploidy 24 -gt_mode GENOTYPE_GIVEN_ALLELES  -out_mode EMIT_ALL_SITES -alleles %s",LSV_ALLELES),"LSV_INDEL_GGA","INDEL","492c8ba9a80a902097ff15bbeb031592");
+        PC_LSV_Test(String.format(" -maxAltAlleles 1 -ploidy 24 -gt_mode GENOTYPE_GIVEN_ALLELES  -out_mode EMIT_ALL_SITES -alleles %s",LSV_ALLELES),"LSV_INDEL_GGA","INDEL","ae6c276cc46785a794acff6f7d10ecf7");
     }
 
     @Test(enabled = true)
     public void testINDEL_maxAltAlleles2_ploidy3_Pools_noRef() {
-        PC_LSV_Test_NoRef(" -maxAltAlleles 2 -ploidy 3","LSV_INDEL_DISC_NOREF_p3","INDEL","848e1092b5cd57b0da5f1187e67134e7");
+        PC_LSV_Test_NoRef(" -maxAltAlleles 2 -ploidy 3","LSV_INDEL_DISC_NOREF_p3","INDEL","481452ad7d6378cffb5cd834cc621d55");
     }
 
     @Test(enabled = true)
     public void testINDEL_maxAltAlleles2_ploidy1_Pools_noRef() {
-        PC_LSV_Test_NoRef(" -maxAltAlleles 2 -ploidy 1","LSV_INDEL_DISC_NOREF_p1","INDEL","51a7b51d82a341adec0e6510f5dfadd8");
+        PC_LSV_Test_NoRef(" -maxAltAlleles 2 -ploidy 1","LSV_INDEL_DISC_NOREF_p1","INDEL","812957e51277aca9925c1a7bb4d9a118");
     }
 
     @Test(enabled = true)
     public void testMT_SNP_DISCOVERY_sp4() {
-         PC_MT_Test(CEUTRIO_BAM, " -maxAltAlleles 1 -ploidy 8", "MT_SNP_DISCOVERY_sp4","0a8c3b06243040b743dd90d497bb3f83");
+         PC_MT_Test(CEUTRIO_BAM, " -maxAltAlleles 1 -ploidy 8", "MT_SNP_DISCOVERY_sp4","dd568dc30be90135a3a8957a45a7321c");
     }
 
     @Test(enabled = true)
     public void testMT_SNP_GGA_sp10() {
-        PC_MT_Test(CEUTRIO_BAM, String.format(" -maxAltAlleles 1 -ploidy 20 -gt_mode GENOTYPE_GIVEN_ALLELES  -out_mode EMIT_ALL_SITES -alleles %s",NA12891_CALLS), "MT_SNP_GGA_sp10", "f8ea18ec6a717a77fdf8c5f2482d8d8d");
+        PC_MT_Test(CEUTRIO_BAM, String.format(" -maxAltAlleles 1 -ploidy 20 -gt_mode GENOTYPE_GIVEN_ALLELES  -out_mode EMIT_ALL_SITES -alleles %s",NA12891_CALLS), "MT_SNP_GGA_sp10", "bf793c43b635a931207170be8035b288");
     }
 }
diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
index 71f03ea00..86f3748ce 100644
--- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
+++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
@@ -21,17 +21,17 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
 
     @Test
     public void testHaplotypeCallerMultiSample() {
-        HCTest(CEUTRIO_BAM, "", "1d826f852ec1457efbfa7ee828c5783a");
+        HCTest(CEUTRIO_BAM, "", "aa1df35d6e64d7ca93feb4d2dd15dd0e");
     }
 
     @Test
     public void testHaplotypeCallerSingleSample() {
-        HCTest(NA12878_BAM, "", "01367428c26d3eaf9297c58bf8677dd3");
+        HCTest(NA12878_BAM, "", "186c7f322978283c01249c6de2829215");
     }
 
     @Test
     public void testHaplotypeCallerMultiSampleGGA() {
-        HCTest(CEUTRIO_BAM, "--max_alternate_alleles 3 -gt_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "combined.phase1.chr20.raw.indels.sites.vcf", "53caa950535749f99d3c5b9bb61c7b60");
+        HCTest(CEUTRIO_BAM, "--max_alternate_alleles 3 -gt_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "combined.phase1.chr20.raw.indels.sites.vcf", "de9e78a52207fe62144dba5337965469");
     }
 
     private void HCTestComplexVariants(String bam, String args, String md5) {
@@ -42,7 +42,7 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
 
     @Test
     public void testHaplotypeCallerMultiSampleComplex() {
-        HCTestComplexVariants(privateTestDir + "AFR.complex.variants.bam", "", "966d338f423c86a390d685aa6336ec69");
+        HCTestComplexVariants(privateTestDir + "AFR.complex.variants.bam", "", "000dbb1b48f94d017cfec127c6cabe8f");
     }
 
     private void HCTestSymbolicVariants(String bam, String args, String md5) {
@@ -53,7 +53,7 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
 
     @Test
     public void testHaplotypeCallerSingleSampleSymbolic() {
-        HCTestSymbolicVariants(NA12878_CHR20_BAM, "", "b4ea70a446e4782bd3700ca14dd726ff");
+        HCTestSymbolicVariants(NA12878_CHR20_BAM, "", "16013a9203367c3d1c4ce1dcdc81ef4a");
     }
 
     private void HCTestIndelQualityScores(String bam, String args, String md5) {
@@ -64,20 +64,20 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
 
     @Test
     public void testHaplotypeCallerSingleSampleIndelQualityScores() {
-        HCTestIndelQualityScores(NA12878_RECALIBRATED_BAM, "", "2581e760279291a3901a506d060bfac8");
+        HCTestIndelQualityScores(NA12878_RECALIBRATED_BAM, "", "b369c2a6cb5c99a424551b33bae16f3b");
     }
 
     @Test
     public void HCTestProblematicReadsModifiedInActiveRegions() {
         final String base = String.format("-T HaplotypeCaller -R %s -I %s", REF, privateTestDir + "haplotype-problem-4.bam") + " --no_cmdline_in_header -o %s -minPruning 3";
-        final WalkerTestSpec spec = new WalkerTestSpec(base, Arrays.asList("c54c0c9411054bf629bfd98b616e53fc"));
+        final WalkerTestSpec spec = new WalkerTestSpec(base, Arrays.asList("c306140ad28515ee06c603c225217939"));
         executeTest("HCTestProblematicReadsModifiedInActiveRegions: ", spec);
     }
 
     @Test
     public void HCTestStructuralIndels() {
         final String base = String.format("-T HaplotypeCaller -R %s -I %s", REF, privateTestDir + "AFR.structural.indels.bam") + " --no_cmdline_in_header -o %s -minPruning 6 -L 20:8187565-8187800 -L 20:18670537-18670730";
-        final WalkerTestSpec spec = new WalkerTestSpec(base, Arrays.asList("c642dcd93771f6f084d55de31f180d1b"));
+        final WalkerTestSpec spec = new WalkerTestSpec(base, Arrays.asList("b6c67ee8e99cc8f53a6587bb26028047"));
         executeTest("HCTestStructuralIndels: ", spec);
     }
 
@@ -91,7 +91,7 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
     public void HCTestReducedBam() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T HaplotypeCaller -R " + b37KGReference + " --no_cmdline_in_header -I " + privateTestDir + "bamExample.ReducedRead.ADAnnotation.bam -o %s -L 1:67,225,396-67,288,518", 1,
-                Arrays.asList("79af83432dc4a1768b3ebffffc4d2b8f"));
+                Arrays.asList("4beb9f87ab3f316a9384c3d0dca6ebe9"));
         executeTest("HC calling on a ReducedRead BAM", spec);
     }
 }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
index 2f6d9d16f..547f375bb 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/arguments/StandardCallerArgumentCollection.java
@@ -75,8 +75,9 @@ public class StandardCallerArgumentCollection {
      * Basically, it will ignore the contamination fraction of reads for each alternate allele.  So if the pileup contains N total bases, then we
      * will try to remove (N * contamination fraction) bases for each alternate allele.
      */
-    @Argument(fullName = "contamination_percentage_to_filter", shortName = "contamination", doc = "Fraction of contamination in sequencing data (for all samples) to aggressively remove", required = false)
-    public double CONTAMINATION_FRACTION = 0.0;
+    @Argument(fullName = "contamination_fraction_to_filter", shortName = "contamination", doc = "Fraction of contamination in sequencing data (for all samples) to aggressively remove", required = false)
+    public double CONTAMINATION_FRACTION = DEFAULT_CONTAMINATION_FRACTION;
+    public static final double DEFAULT_CONTAMINATION_FRACTION = 0.05;
 
     @Hidden
     @Argument(fullName = "logRemovedReadsFromContaminationFiltering", shortName="contaminationLog", required=false)
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
index ef56eaa6d..36be2e7c6 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
@@ -28,6 +28,7 @@ package org.broadinstitute.sting.gatk.walkers.genotyper;
 import org.broadinstitute.sting.commandline.*;
 import org.broadinstitute.sting.gatk.CommandLineGATK;
 import org.broadinstitute.sting.gatk.arguments.DbsnpArgumentCollection;
+import org.broadinstitute.sting.gatk.arguments.StandardCallerArgumentCollection;
 import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.downsampling.DownsampleType;
@@ -238,7 +239,12 @@ public class UnifiedGenotyper extends LocusWalker<List<VariantCallContext>, Unif
             }
 
             if ( UAC.CONTAMINATION_FRACTION > 0.0 ) {
-                throw new UserException.NotSupportedInGATKLite("you cannot enable usage of contamination down-sampling");
+                if ( UAC.CONTAMINATION_FRACTION == StandardCallerArgumentCollection.DEFAULT_CONTAMINATION_FRACTION ) {
+                    UAC.CONTAMINATION_FRACTION = 0.0;
+                    logger.warn("setting contamination down-sampling fraction to 0.0 because it is not enabled in GATK-lite");
+                } else {
+                    throw new UserException.NotSupportedInGATKLite("you cannot enable usage of contamination down-sampling");
+                }
             }
         }
 
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index 446f8cbe8..a5192ffff 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -30,7 +30,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testMultiSamplePilot1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000", 1,
-                Arrays.asList("847605f4efafef89529fe0e496315edd"));
+                Arrays.asList("cdec335abc9ad8e59335e39a73e0e95a"));
         executeTest("test MultiSample Pilot1", spec);
     }
 
@@ -62,7 +62,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testMultipleSNPAlleles() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + privateTestDir + "multiallelic.snps.bam -o %s -L " + privateTestDir + "multiallelic.snps.intervals", 1,
-                Arrays.asList("543f68e42034bf44cfb24da8c9204320"));
+                Arrays.asList("94dc17d76d841f1d3a36160767ffa034"));
         executeTest("test Multiple SNP alleles", spec);
     }
 
@@ -86,7 +86,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testMismatchedPLs() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm INDEL -I " + privateTestDir + "mismatchedPLs.bam -o %s -L 1:24020341", 1,
-                Arrays.asList("3c006b06b17bbe8e787d64eff6a63a19"));
+                Arrays.asList("d847acf841ba8ba653f996ce4869f439"));
         executeTest("test mismatched PLs", spec);
     }
 
@@ -96,7 +96,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    private final static String COMPRESSED_OUTPUT_MD5 = "fd236bd635d514e4214d364f45ec4d10";
+    private final static String COMPRESSED_OUTPUT_MD5 = "6792419c482e767a3deb28913ed2b1ad";
 
     @Test
     public void testCompressedOutput() {
@@ -149,7 +149,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testMinBaseQualityScore() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --min_base_quality_score 26", 1,
-                Arrays.asList("839ecd30d354a36b5dfa2b5e99859765"));
+                Arrays.asList("56157d930da6ccd224bce1ca93f11e41"));
         executeTest("test min_base_quality_score 26", spec);
     }
 
@@ -165,7 +165,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testNDA() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --annotateNDA -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
-                Arrays.asList("abd8e33e649cc11b55e200d3940cc7e2"));
+                Arrays.asList("480437dd6e2760f4ab3194431519f331"));
         executeTest("test NDA", spec);
     }
 
@@ -173,7 +173,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testCompTrack() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH -comp:FOO " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
-                Arrays.asList("a6de72c5197f152be98a6f9029091a9e"));
+                Arrays.asList("22c039412fd387dde6125b07c9a74a25"));
         executeTest("test using comp track", spec);
     }
 
@@ -211,18 +211,10 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testConfidence() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 -stand_call_conf 10 ", 1,
-                Arrays.asList("9addd225a985178339a0c49dc5fdc220"));
+                Arrays.asList("df524e98903d96ab9353bee7c16a69de"));
         executeTest("test confidence 1", spec1);
     }
 
-    @Test
-    public void testConfidence2() {
-        WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
-                baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 -stand_emit_conf 10 ", 1,
-                Arrays.asList("9addd225a985178339a0c49dc5fdc220"));
-        executeTest("test confidence 2", spec2);
-    }
-
     // --------------------------------------------------------------------------------------------------------------
     //
     // testing heterozygosity
@@ -230,12 +222,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // --------------------------------------------------------------------------------------------------------------
     @Test
     public void testHeterozyosity1() {
-        testHeterozosity( 0.01, "986923de51c71635d47e3d06fe3794a1" );
+        testHeterozosity( 0.01, "8e61498ca03a8d805372a64c466b3b42" );
     }
 
     @Test
     public void testHeterozyosity2() {
-        testHeterozosity( 1.0 / 1850, "fb12b1553f813004a394a391a8540873" );
+        testHeterozosity( 1.0 / 1850, "668d06b5173cf3b97d052726988e1d7b" );
     }
 
     private void testHeterozosity(final double arg, final String md5) {
@@ -259,7 +251,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                         " -o %s" +
                         " -L 1:10,000,000-10,100,000",
                 1,
-                Arrays.asList("04a87b87ee4323eba853c78f25551d1a"));
+                Arrays.asList("908eb5e21fa39e7fb377cf4a9c4c7835"));
 
         executeTest(String.format("test multiple technologies"), spec);
     }
@@ -278,7 +270,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                         " -L 1:10,000,000-10,100,000" +
                         " -baq CALCULATE_AS_NECESSARY",
                 1,
-                Arrays.asList("98058fc913b61c22d44875da1f5ea89c"));
+                Arrays.asList("c814558bb0ed2e19b12e1a2bf4465d52"));
 
         executeTest(String.format("test calling with BAQ"), spec);
     }
@@ -312,7 +304,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                         " -minIndelCnt 1" +
                         " -L 1:10,000,000-10,100,000",
                 1,
-                Arrays.asList("6a0c2a3a7bcc56ad01428c71408055aa"));
+                Arrays.asList("8b486a098029d5a106b0a37eff541c15"));
 
         executeTest(String.format("test indel caller in SLX with low min allele count"), spec);
     }
@@ -325,7 +317,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                          " -o %s" +
                          " -L 1:10,000,000-10,500,000",
                  1,
-                 Arrays.asList("5f2721c3323de5390d2d47446139f32b"));
+                 Arrays.asList("18efedc50cae2aacaba372265e38310b"));
 
          executeTest(String.format("test indel calling, multiple technologies"), spec);
      }
@@ -353,13 +345,13 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testMultiSampleIndels1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10450700-10551000", 1,
-                Arrays.asList("a4761d7f25e7a62f34494801c98a0da7"));
+                Arrays.asList("f7d0d0aee603df25c1f0525bb8df189e"));
         List<File> result = executeTest("test MultiSample Pilot1 CEU indels", spec1).getFirst();
 
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + result.get(0).getAbsolutePath() + " -I " + validationDataLocation +
                         "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10450700-10551000", 1,
-                Arrays.asList("c526c234947482d1cd2ffc5102083a08"));
+                Arrays.asList("fc91d457a16b4ca994959c2b5f3f0352"));
         executeTest("test MultiSample Pilot1 CEU indels using GENOTYPE_GIVEN_ALLELES", spec2);
     }
 
@@ -415,7 +407,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testMinIndelFraction0() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.0", 1,
-                Arrays.asList("ba4fafec383fb988f20c8cf53dd3e9a0"));
+                Arrays.asList("857b8e5df444463ac27f665c4f67fbe2"));
         executeTest("test minIndelFraction 0.0", spec);
     }
 
@@ -423,7 +415,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testMinIndelFraction25() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.25", 1,
-                Arrays.asList("4c57a88de275105156aaafc6f9041365"));
+                Arrays.asList("81d4c7d9010fd6733b2997bc378e7471"));
         executeTest("test minIndelFraction 0.25", spec);
     }
 
@@ -490,7 +482,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testContaminationDownsampling() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-                baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --contamination_percentage_to_filter 0.20", 1,
+                baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --contamination_fraction_to_filter 0.20", 1,
                 Arrays.asList("27dd04159e06d9524fb8a4eef41f96ae"));
         executeTest("test contamination_percentage_to_filter 0.20", spec);
     }

From a53e03d52577e09cad20738b507260cb8e8f0980 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Fri, 26 Oct 2012 02:13:04 -0400
Subject: [PATCH 26/50] Do not let reduced reads get removed in the
 contamination down-sampling

---
 .../AlleleBiasedDownsamplingUtils.java        |  4 ++
 .../AdvancedPerReadAlleleLikelihoodMap.java   |  9 +++--
 .../UnifiedGenotyperIntegrationTest.java      | 40 +++++++++----------
 3 files changed, 30 insertions(+), 23 deletions(-)

diff --git a/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java b/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
index e224a9c57..830fd111b 100755
--- a/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
@@ -60,6 +60,10 @@ public class AlleleBiasedDownsamplingUtils {
 
         // start by stratifying the reads by the alleles they represent at this position
         for( final PileupElement pe : pileup ) {
+            // abort if we have a reduced read - we do not want to remove it!
+            if ( pe.getRead().isReducedRead() )
+                return pileup;
+
             final int baseIndex = BaseUtils.simpleBaseToBaseIndex(pe.getBase());
             if ( baseIndex != -1 )
                 alleleStratifiedElements[baseIndex].add(pe);
diff --git a/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java b/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
index d6cddcf84..77a7c3bd9 100644
--- a/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
+++ b/protected/java/src/org/broadinstitute/sting/utils/genotyper/AdvancedPerReadAlleleLikelihoodMap.java
@@ -55,9 +55,12 @@ public class AdvancedPerReadAlleleLikelihoodMap extends StandardPerReadAlleleLik
             alleleReadMap.put(allele, new ArrayList<GATKSAMRecord>());
 
         for ( Map.Entry<GATKSAMRecord, Map<Allele, Double>> entry : likelihoodReadMap.entrySet() ) {
-            final Allele bestAllele = getMostLikelyAllele(entry.getValue());
-            if ( bestAllele != Allele.NO_CALL )
-                alleleReadMap.get(bestAllele).add(entry.getKey());
+            // do not remove reduced reads!
+            if ( !entry.getKey().isReducedRead() ) {
+                final Allele bestAllele = getMostLikelyAllele(entry.getValue());
+                if ( bestAllele != Allele.NO_CALL )
+                    alleleReadMap.get(bestAllele).add(entry.getKey());
+            }
         }
 
         // compute the reads to remove and actually remove them
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index a5192ffff..93fe7e80d 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -38,7 +38,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testWithAllelesPassedIn1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
-                Arrays.asList("bc15123620e1134f799005d71d1180fe"));
+                Arrays.asList("efddb5e258f97fd4f6661cff9eaa57de"));
         executeTest("test MultiSample Pilot2 with alleles passed in", spec1);
     }
 
@@ -46,7 +46,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testWithAllelesPassedIn2() {
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
-                Arrays.asList("1ba7afccc8552f20d72d0b62237558e3"));
+                Arrays.asList("24532eb381724cd74e99370da28d49ed"));
         executeTest("test MultiSample Pilot2 with alleles passed in and emitting all sites", spec2);
     }
 
@@ -54,7 +54,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testSingleSamplePilot2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
-                Arrays.asList("afb8768f31ab57eb43f75c1115eadc99"));
+                Arrays.asList("062a946160eec1d0fc135d58ca654ff4"));
         executeTest("test SingleSample Pilot2", spec);
     }
 
@@ -78,7 +78,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testReverseTrim() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm INDEL -I " + validationDataLocation + "CEUTrio.HiSeq.b37.chr20.10_11mb.bam -o %s -L 20:10289124 -L 20:10090289", 1,
-                Arrays.asList("5ce03dd9ca2d9324c1d4a9d64389beb5"));
+                Arrays.asList("9106d01ca0d0a8fedd068e72d509f380"));
         executeTest("test reverse trim", spec);
     }
 
@@ -117,7 +117,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
         // Note that we need to turn off any randomization for this to work, so no downsampling and no annotations
 
-        String md5 = "d408b4661b820ed86272415b8ea08780";
+        String md5 = "3d25ea660275a455ca443a786bff3d32";
 
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -dt NONE -G none -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,075,000", 1,
@@ -157,7 +157,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testSLOD() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --computeSLOD --no_cmdline_in_header -glm BOTH --dbsnp " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
-                Arrays.asList("c7429e670ba477bf9a6bbee2fb41c5a9"));
+                Arrays.asList("9f95cfe14d53a697c58247833bfd72a6"));
         executeTest("test SLOD", spec);
     }
 
@@ -177,7 +177,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test using comp track", spec);
     }
 
-    @Test
+    //@Test
     public void testNoCmdLineHeaderStdout() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandNoCmdLineHeaderStdout + " -glm INDEL -L 1:67,225,396-67,288,518", 0,
@@ -187,17 +187,17 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
     @Test
     public void testOutputParameterSitesOnly() {
-        testOutputParameters("-sites_only", "97ba874eafc9884a4de027a84c036311");
+        testOutputParameters("-sites_only", "40aeb4c9e31fe7046b72afc58e7599cb");
     }
 
     @Test
     public void testOutputParameterAllConfident() {
-        testOutputParameters("--output_mode EMIT_ALL_CONFIDENT_SITES", "f9ea04d96eeef29e71d37e60518c2579");
+        testOutputParameters("--output_mode EMIT_ALL_CONFIDENT_SITES", "c706ca93b25ff83613cb4e95dcac567c");
     }
 
     @Test
     public void testOutputParameterAllSites() {
-        testOutputParameters("--output_mode EMIT_ALL_SITES", "41c046d38ea328421df924e37e017645");
+        testOutputParameters("--output_mode EMIT_ALL_SITES", "8a263fd0a94463ce1de9990f2b8ec841");
     }
 
     private void testOutputParameters(final String args, final String md5) {
@@ -289,7 +289,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                         " -o %s" +
                         " -L 1:10,000,000-10,500,000",
                 1,
-                Arrays.asList("650c53774afacfc07a595675e8cdde17"));
+                Arrays.asList("3593495aab5f6204c65de0b073a6ff65"));
 
         executeTest(String.format("test indel caller in SLX"), spec);
     }
@@ -327,7 +327,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "indelAllelesForUG.vcf -I " + validationDataLocation +
                         "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,100,000", 1,
-                Arrays.asList("7e3f67bf371112be5dbadb4fe6faa52a"));
+                Arrays.asList("3ff8c7c80a518aa3eb8671a21479de5f"));
         executeTest("test MultiSample Pilot2 indels with alleles passed in", spec);
     }
 
@@ -337,7 +337,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                 baseCommandIndels + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles "
                         + privateTestDir + "indelAllelesForUG.vcf -I " + validationDataLocation +
                         "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,100,000", 1,
-                Arrays.asList("bc31c4977cb7e563ddf9c8dea27f3f4f"));
+                Arrays.asList("578c0540f4f2052a634a829bcb9cc27d"));
         executeTest("test MultiSample Pilot2 indels with alleles passed in and emitting all sites", spec);
     }
 
@@ -436,8 +436,8 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testNsInCigar() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-                "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -I " + validationDataLocation + "testWithNs.bam -o %s -L 8:141799600-141814700", 1,
-                Arrays.asList("e8ebfaac0804b782f22ab8ea35152735"));
+                "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -I " + validationDataLocation + "testWithNs.bam -o %s -L 8:141799600-141814700", 1,
+                Arrays.asList("bd7984a374f0ae5d277bd5fc5065f64f"));
         executeTest("test calling on reads with Ns in CIGAR", spec);
     }
 
@@ -450,25 +450,25 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testReducedBam() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-                "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -I " + privateTestDir + "bamExample.ReducedRead.ADAnnotation.bam -o %s -L 1:67,225,396-67,288,518", 1,
-                Arrays.asList("da9c05f87bd6415e97f90c49cf68ed19"));
+                "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -I " + privateTestDir + "bamExample.ReducedRead.ADAnnotation.bam -o %s -L 1:67,225,396-67,288,518", 1,
+                Arrays.asList("1b711c0966a2f76eb21801e73b76b758"));
         executeTest("test calling on a ReducedRead BAM", spec);
     }
 
     @Test
     public void testReducedBamSNPs() {
-        testReducedCalling("SNP", "1d4a826b144723ff0766c36aa0239287");
+        testReducedCalling("SNP", "9ba4867cadb366746ee63e7a4afdb95e");
     }
 
     @Test
     public void testReducedBamINDELs() {
-        testReducedCalling("INDEL", "68ef51d5c98480e0c0192e0eecb95bca");
+        testReducedCalling("INDEL", "132a4e0ccf9230b5bb4b56c649e2bdd5");
     }
 
 
     private void testReducedCalling(final String model, final String md5) {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-                "-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -I " + privateTestDir + "NA12878.HiSeq.b37.chr20.10_11mb.reduced.bam -o %s -L 20:10,000,000-11,000,000 -glm " + model, 1,
+                "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -I " + privateTestDir + "NA12878.HiSeq.b37.chr20.10_11mb.reduced.bam -o %s -L 20:10,000,000-11,000,000 -glm " + model, 1,
                 Arrays.asList(md5));
         executeTest("test calling on a ReducedRead BAM with " + model, spec);
     }

From ebebec7fdb74e180e4bc019e18b13261227a93c3 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Fri, 26 Oct 2012 02:15:32 -0400
Subject: [PATCH 27/50] Accidentally left one test disabled

---
 .../gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index 93fe7e80d..1b2fbb848 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -177,7 +177,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test using comp track", spec);
     }
 
-    //@Test
+    @Test
     public void testNoCmdLineHeaderStdout() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandNoCmdLineHeaderStdout + " -glm INDEL -L 1:67,225,396-67,288,518", 0,

From 7a706ed345cdae1fe7f825e1aec13aeed825e0fb Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Fri, 26 Oct 2012 11:23:44 -0400
Subject: [PATCH 29/50] Fix some of the broken integration tests

---
 .../AlleleBiasedDownsamplingUtils.java        | 10 +--
 .../UnifiedGenotyperIntegrationTest.java      | 78 +++++++++----------
 .../NanoSchedulerIntegrationTest.java         |  3 +-
 3 files changed, 45 insertions(+), 46 deletions(-)

diff --git a/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java b/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
index 830fd111b..b84cde58e 100755
--- a/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
@@ -37,12 +37,6 @@ import java.util.*;
 
 public class AlleleBiasedDownsamplingUtils {
 
-    private static final ArrayList<PileupElement>[] alleleStratifiedElements = new ArrayList[4];
-    static {
-        for ( int i = 0; i < 4; i++ )
-            alleleStratifiedElements[i] = new ArrayList<PileupElement>();
-    }
-
     /**
      * Computes an allele biased version of the given pileup
      *
@@ -58,6 +52,10 @@ public class AlleleBiasedDownsamplingUtils {
         if ( downsamplingFraction >= 1.0 )
             return new ReadBackedPileupImpl(pileup.getLocation(), new ArrayList<PileupElement>());
 
+        final ArrayList<PileupElement>[] alleleStratifiedElements = new ArrayList[4];
+        for ( int i = 0; i < 4; i++ )
+            alleleStratifiedElements[i] = new ArrayList<PileupElement>();
+
         // start by stratifying the reads by the alleles they represent at this position
         for( final PileupElement pe : pileup ) {
             // abort if we have a reduced read - we do not want to remove it!
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index 1b2fbb848..ea2f0fb97 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -26,7 +26,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing normal calling
     //
     // --------------------------------------------------------------------------------------------------------------
-    @Test
+    //@Test
     public void testMultiSamplePilot1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000", 1,
@@ -34,7 +34,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot1", spec);
     }
 
-    @Test
+    //@Test
     public void testWithAllelesPassedIn1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
@@ -42,7 +42,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 with alleles passed in", spec1);
     }
 
-    @Test
+    //@Test
     public void testWithAllelesPassedIn2() {
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
@@ -50,7 +50,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 with alleles passed in and emitting all sites", spec2);
     }
 
-    @Test
+    //@Test
     public void testSingleSamplePilot2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
@@ -58,7 +58,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test SingleSample Pilot2", spec);
     }
 
-    @Test
+    //@Test
     public void testMultipleSNPAlleles() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + privateTestDir + "multiallelic.snps.bam -o %s -L " + privateTestDir + "multiallelic.snps.intervals", 1,
@@ -66,7 +66,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test Multiple SNP alleles", spec);
     }
 
-    @Test
+    //@Test
     public void testBadRead() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm BOTH -I " + privateTestDir + "badRead.test.bam -o %s -L 1:22753424-22753464", 1,
@@ -74,7 +74,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test bad read", spec);
     }
 
-    @Test
+    //@Test
     public void testReverseTrim() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm INDEL -I " + validationDataLocation + "CEUTrio.HiSeq.b37.chr20.10_11mb.bam -o %s -L 20:10289124 -L 20:10090289", 1,
@@ -82,7 +82,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test reverse trim", spec);
     }
 
-    @Test
+    //@Test
     public void testMismatchedPLs() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm INDEL -I " + privateTestDir + "mismatchedPLs.bam -o %s -L 1:24020341", 1,
@@ -98,7 +98,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
     private final static String COMPRESSED_OUTPUT_MD5 = "6792419c482e767a3deb28913ed2b1ad";
 
-    @Test
+    //@Test
     public void testCompressedOutput() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
@@ -145,7 +145,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testMinBaseQualityScore() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --min_base_quality_score 26", 1,
@@ -153,7 +153,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test min_base_quality_score 26", spec);
     }
 
-    @Test
+    //@Test
     public void testSLOD() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --computeSLOD --no_cmdline_in_header -glm BOTH --dbsnp " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -161,7 +161,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test SLOD", spec);
     }
 
-    @Test
+    //@Test
     public void testNDA() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --annotateNDA -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -169,7 +169,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test NDA", spec);
     }
 
-    @Test
+    //@Test
     public void testCompTrack() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH -comp:FOO " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -177,7 +177,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test using comp track", spec);
     }
 
-    @Test
+    //@Test
     public void testNoCmdLineHeaderStdout() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandNoCmdLineHeaderStdout + " -glm INDEL -L 1:67,225,396-67,288,518", 0,
@@ -185,17 +185,17 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("testNoCmdLineHeaderStdout", spec);
     }
 
-    @Test
+    //@Test
     public void testOutputParameterSitesOnly() {
         testOutputParameters("-sites_only", "40aeb4c9e31fe7046b72afc58e7599cb");
     }
 
-    @Test
+    //@Test
     public void testOutputParameterAllConfident() {
         testOutputParameters("--output_mode EMIT_ALL_CONFIDENT_SITES", "c706ca93b25ff83613cb4e95dcac567c");
     }
 
-    @Test
+    //@Test
     public void testOutputParameterAllSites() {
         testOutputParameters("--output_mode EMIT_ALL_SITES", "8a263fd0a94463ce1de9990f2b8ec841");
     }
@@ -207,7 +207,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest(String.format("testParameter[%s]", args), spec);
     }
 
-    @Test
+    //@Test
     public void testConfidence() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 -stand_call_conf 10 ", 1,
@@ -220,12 +220,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing heterozygosity
     //
     // --------------------------------------------------------------------------------------------------------------
-    @Test
+    //@Test
     public void testHeterozyosity1() {
         testHeterozosity( 0.01, "8e61498ca03a8d805372a64c466b3b42" );
     }
 
-    @Test
+    //@Test
     public void testHeterozyosity2() {
         testHeterozosity( 1.0 / 1850, "668d06b5173cf3b97d052726988e1d7b" );
     }
@@ -243,7 +243,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing calls with SLX, 454, and SOLID data
     //
     // --------------------------------------------------------------------------------------------------------------
-    @Test
+    //@Test
     public void testMultiTechnologies() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand +
@@ -261,7 +261,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing calls with BAQ
     //
     // --------------------------------------------------------------------------------------------------------------
-    @Test
+    //@Test
     public void testCallingWithBAQ() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand +
@@ -281,7 +281,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
     // Basic indel testing with SLX data
-    @Test
+    //@Test
     public void testSimpleIndels() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels +
@@ -295,7 +295,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     }
 
     // Basic indel testing with SLX data
-    @Test
+    //@Test
     public void testIndelsWithLowMinAlleleCnt() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels +
@@ -309,7 +309,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest(String.format("test indel caller in SLX with low min allele count"), spec);
     }
 
-    @Test
+    //@Test
     public void testMultiTechnologyIndels() {
          WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                  baseCommandIndels +
@@ -322,7 +322,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
          executeTest(String.format("test indel calling, multiple technologies"), spec);
      }
 
-    @Test
+    //@Test
     public void testWithIndelAllelesPassedIn1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "indelAllelesForUG.vcf -I " + validationDataLocation +
@@ -331,7 +331,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 indels with alleles passed in", spec);
     }
 
-    @Test
+    //@Test
     public void testWithIndelAllelesPassedIn2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles "
@@ -341,7 +341,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 indels with alleles passed in and emitting all sites", spec);
     }
 
-    @Test
+    //@Test
     public void testMultiSampleIndels1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10450700-10551000", 1,
@@ -355,7 +355,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot1 CEU indels using GENOTYPE_GIVEN_ALLELES", spec2);
     }
 
-    @Test
+    //@Test
     public void testGGAwithNoEvidenceInReads() {
         final String vcf = "small.indel.test.vcf";
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
@@ -365,7 +365,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test GENOTYPE_GIVEN_ALLELES with no evidence in reads", spec);
     }
 
-    @Test
+    //@Test
     public void testBaseIndelQualityScores() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndelsb37 +
@@ -384,7 +384,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testSnpEffAnnotationRequestedWithoutRodBinding() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000 " +
@@ -403,7 +403,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     final static String assessMinIndelFraction = baseCommandIndelsb37 + " -I " + validationDataLocation
             + "978604.bam -L 1:978,586-978,626 -o %s --sites_only -rf Sample -goodSM 7377 -goodSM 22-0022 -goodSM 134 -goodSM 344029-53 -goodSM 14030";
 
-    @Test
+    //@Test
     public void testMinIndelFraction0() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.0", 1,
@@ -411,7 +411,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test minIndelFraction 0.0", spec);
     }
 
-    @Test
+    //@Test
     public void testMinIndelFraction25() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.25", 1,
@@ -419,7 +419,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test minIndelFraction 0.25", spec);
     }
 
-    @Test
+    //@Test
     public void testMinIndelFraction100() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 1", 1,
@@ -433,7 +433,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testNsInCigar() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -I " + validationDataLocation + "testWithNs.bam -o %s -L 8:141799600-141814700", 1,
@@ -447,7 +447,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testReducedBam() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -I " + privateTestDir + "bamExample.ReducedRead.ADAnnotation.bam -o %s -L 1:67,225,396-67,288,518", 1,
@@ -455,12 +455,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test calling on a ReducedRead BAM", spec);
     }
 
-    @Test
+    //@Test
     public void testReducedBamSNPs() {
         testReducedCalling("SNP", "9ba4867cadb366746ee63e7a4afdb95e");
     }
 
-    @Test
+    //@Test
     public void testReducedBamINDELs() {
         testReducedCalling("INDEL", "132a4e0ccf9230b5bb4b56c649e2bdd5");
     }
@@ -479,7 +479,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    @Test
+    //@Test
     public void testContaminationDownsampling() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --contamination_fraction_to_filter 0.20", 1,
diff --git a/public/java/test/org/broadinstitute/sting/utils/nanoScheduler/NanoSchedulerIntegrationTest.java b/public/java/test/org/broadinstitute/sting/utils/nanoScheduler/NanoSchedulerIntegrationTest.java
index 37614f15f..80b0b4ee2 100755
--- a/public/java/test/org/broadinstitute/sting/utils/nanoScheduler/NanoSchedulerIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/utils/nanoScheduler/NanoSchedulerIntegrationTest.java
@@ -32,11 +32,12 @@ public class NanoSchedulerIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
                 buildCommandLine(
                         "-T UnifiedGenotyper -R " + b37KGReference,
-                        "-nosl --no_cmdline_in_header -G",
+                        "--no_cmdline_in_header -G",
                         //"--dbsnp " + b37dbSNP132,
                         "-I " + privateTestDir + "NA12878.HiSeq.b37.chr20.10_11mb.bam",
                         "-L 20:10,000,000-10,100,000",
                         "-glm " + glm,
+                        "--contamination_fraction_to_filter 0.0",
                         "-nt " + nt,
                         "-nct " + nct,
                         "-o %s"

From ed11b7dab25c87c34942698ba10c6cb77ed2da4f Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Fri, 26 Oct 2012 12:10:44 -0400
Subject: [PATCH 30/50] Fix UG parallelization test

---
 .../AlleleBiasedDownsamplingUtils.java        |  2 +-
 .../UnifiedGenotyperIntegrationTest.java      | 86 +++++++++----------
 2 files changed, 44 insertions(+), 44 deletions(-)

diff --git a/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java b/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
index b84cde58e..59357e1c4 100755
--- a/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/downsampling/AlleleBiasedDownsamplingUtils.java
@@ -45,7 +45,7 @@ public class AlleleBiasedDownsamplingUtils {
      * @param log                       logging output
      * @return allele biased pileup
      */
-    public synchronized static ReadBackedPileup createAlleleBiasedBasePileup(final ReadBackedPileup pileup, final double downsamplingFraction, final PrintStream log) {
+    public static ReadBackedPileup createAlleleBiasedBasePileup(final ReadBackedPileup pileup, final double downsamplingFraction, final PrintStream log) {
         // special case removal of all or no reads
         if ( downsamplingFraction <= 0.0 )
             return pileup;
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index ea2f0fb97..2a17435aa 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -26,7 +26,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing normal calling
     //
     // --------------------------------------------------------------------------------------------------------------
-    //@Test
+    @Test
     public void testMultiSamplePilot1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000", 1,
@@ -34,7 +34,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot1", spec);
     }
 
-    //@Test
+    @Test
     public void testWithAllelesPassedIn1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
@@ -42,7 +42,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 with alleles passed in", spec1);
     }
 
-    //@Test
+    @Test
     public void testWithAllelesPassedIn2() {
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
@@ -50,7 +50,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 with alleles passed in and emitting all sites", spec2);
     }
 
-    //@Test
+    @Test
     public void testSingleSamplePilot2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
@@ -58,7 +58,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test SingleSample Pilot2", spec);
     }
 
-    //@Test
+    @Test
     public void testMultipleSNPAlleles() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + privateTestDir + "multiallelic.snps.bam -o %s -L " + privateTestDir + "multiallelic.snps.intervals", 1,
@@ -66,7 +66,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test Multiple SNP alleles", spec);
     }
 
-    //@Test
+    @Test
     public void testBadRead() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm BOTH -I " + privateTestDir + "badRead.test.bam -o %s -L 1:22753424-22753464", 1,
@@ -74,7 +74,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test bad read", spec);
     }
 
-    //@Test
+    @Test
     public void testReverseTrim() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm INDEL -I " + validationDataLocation + "CEUTrio.HiSeq.b37.chr20.10_11mb.bam -o %s -L 20:10289124 -L 20:10090289", 1,
@@ -82,7 +82,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test reverse trim", spec);
     }
 
-    //@Test
+    @Test
     public void testMismatchedPLs() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -glm INDEL -I " + privateTestDir + "mismatchedPLs.bam -o %s -L 1:24020341", 1,
@@ -98,7 +98,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
     private final static String COMPRESSED_OUTPUT_MD5 = "6792419c482e767a3deb28913ed2b1ad";
 
-    //@Test
+    @Test
     public void testCompressedOutput() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
@@ -117,24 +117,24 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
         // Note that we need to turn off any randomization for this to work, so no downsampling and no annotations
 
-        String md5 = "3d25ea660275a455ca443a786bff3d32";
+        String md5 = "d408b4661b820ed86272415b8ea08780";
 
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
-                baseCommand + " -dt NONE -G none -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,075,000", 1,
+                baseCommand + " -dt NONE -G none --contamination_fraction_to_filter 0.0 -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,075,000", 1,
                 Arrays.asList(md5));
         executeTest("test parallelization (single thread)", spec1);
 
         GenomeAnalysisEngine.resetRandomGenerator();
 
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
-                baseCommand + " -dt NONE -G none -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,075,000 -nt 2", 1,
+                baseCommand + " -dt NONE -G none --contamination_fraction_to_filter 0.0 -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,075,000 -nt 2", 1,
                 Arrays.asList(md5));
         executeTest("test parallelization (2 threads)", spec2);
 
         GenomeAnalysisEngine.resetRandomGenerator();
 
         WalkerTest.WalkerTestSpec spec3 = new WalkerTest.WalkerTestSpec(
-                baseCommand + " -dt NONE -G none -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,075,000 -nt 4", 1,
+                baseCommand + " -dt NONE -G none --contamination_fraction_to_filter 0.0 -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,075,000 -nt 4", 1,
                 Arrays.asList(md5));
         executeTest("test parallelization (4 threads)", spec3);
     }
@@ -145,7 +145,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testMinBaseQualityScore() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --min_base_quality_score 26", 1,
@@ -153,7 +153,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test min_base_quality_score 26", spec);
     }
 
-    //@Test
+    @Test
     public void testSLOD() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --computeSLOD --no_cmdline_in_header -glm BOTH --dbsnp " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -161,7 +161,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test SLOD", spec);
     }
 
-    //@Test
+    @Test
     public void testNDA() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --annotateNDA -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -169,7 +169,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test NDA", spec);
     }
 
-    //@Test
+    @Test
     public void testCompTrack() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --no_cmdline_in_header -glm BOTH -comp:FOO " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
@@ -177,7 +177,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test using comp track", spec);
     }
 
-    //@Test
+    @Test
     public void testNoCmdLineHeaderStdout() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandNoCmdLineHeaderStdout + " -glm INDEL -L 1:67,225,396-67,288,518", 0,
@@ -185,17 +185,17 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("testNoCmdLineHeaderStdout", spec);
     }
 
-    //@Test
+    @Test
     public void testOutputParameterSitesOnly() {
         testOutputParameters("-sites_only", "40aeb4c9e31fe7046b72afc58e7599cb");
     }
 
-    //@Test
+    @Test
     public void testOutputParameterAllConfident() {
         testOutputParameters("--output_mode EMIT_ALL_CONFIDENT_SITES", "c706ca93b25ff83613cb4e95dcac567c");
     }
 
-    //@Test
+    @Test
     public void testOutputParameterAllSites() {
         testOutputParameters("--output_mode EMIT_ALL_SITES", "8a263fd0a94463ce1de9990f2b8ec841");
     }
@@ -207,7 +207,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest(String.format("testParameter[%s]", args), spec);
     }
 
-    //@Test
+    @Test
     public void testConfidence() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 -stand_call_conf 10 ", 1,
@@ -220,12 +220,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing heterozygosity
     //
     // --------------------------------------------------------------------------------------------------------------
-    //@Test
+    @Test
     public void testHeterozyosity1() {
         testHeterozosity( 0.01, "8e61498ca03a8d805372a64c466b3b42" );
     }
 
-    //@Test
+    @Test
     public void testHeterozyosity2() {
         testHeterozosity( 1.0 / 1850, "668d06b5173cf3b97d052726988e1d7b" );
     }
@@ -243,7 +243,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing calls with SLX, 454, and SOLID data
     //
     // --------------------------------------------------------------------------------------------------------------
-    //@Test
+    @Test
     public void testMultiTechnologies() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand +
@@ -261,7 +261,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     // testing calls with BAQ
     //
     // --------------------------------------------------------------------------------------------------------------
-    //@Test
+    @Test
     public void testCallingWithBAQ() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand +
@@ -281,7 +281,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
     // Basic indel testing with SLX data
-    //@Test
+    @Test
     public void testSimpleIndels() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels +
@@ -295,7 +295,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     }
 
     // Basic indel testing with SLX data
-    //@Test
+    @Test
     public void testIndelsWithLowMinAlleleCnt() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels +
@@ -309,7 +309,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest(String.format("test indel caller in SLX with low min allele count"), spec);
     }
 
-    //@Test
+    @Test
     public void testMultiTechnologyIndels() {
          WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                  baseCommandIndels +
@@ -322,7 +322,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
          executeTest(String.format("test indel calling, multiple technologies"), spec);
      }
 
-    //@Test
+    @Test
     public void testWithIndelAllelesPassedIn1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + privateTestDir + "indelAllelesForUG.vcf -I " + validationDataLocation +
@@ -331,7 +331,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 indels with alleles passed in", spec);
     }
 
-    //@Test
+    @Test
     public void testWithIndelAllelesPassedIn2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles "
@@ -341,7 +341,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot2 indels with alleles passed in and emitting all sites", spec);
     }
 
-    //@Test
+    @Test
     public void testMultiSampleIndels1() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10450700-10551000", 1,
@@ -355,7 +355,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test MultiSample Pilot1 CEU indels using GENOTYPE_GIVEN_ALLELES", spec2);
     }
 
-    //@Test
+    @Test
     public void testGGAwithNoEvidenceInReads() {
         final String vcf = "small.indel.test.vcf";
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
@@ -365,7 +365,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test GENOTYPE_GIVEN_ALLELES with no evidence in reads", spec);
     }
 
-    //@Test
+    @Test
     public void testBaseIndelQualityScores() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndelsb37 +
@@ -384,7 +384,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testSnpEffAnnotationRequestedWithoutRodBinding() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000 " +
@@ -403,7 +403,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     final static String assessMinIndelFraction = baseCommandIndelsb37 + " -I " + validationDataLocation
             + "978604.bam -L 1:978,586-978,626 -o %s --sites_only -rf Sample -goodSM 7377 -goodSM 22-0022 -goodSM 134 -goodSM 344029-53 -goodSM 14030";
 
-    //@Test
+    @Test
     public void testMinIndelFraction0() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.0", 1,
@@ -411,7 +411,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test minIndelFraction 0.0", spec);
     }
 
-    //@Test
+    @Test
     public void testMinIndelFraction25() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 0.25", 1,
@@ -419,7 +419,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test minIndelFraction 0.25", spec);
     }
 
-    //@Test
+    @Test
     public void testMinIndelFraction100() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 assessMinIndelFraction + " -minIndelFrac 1", 1,
@@ -433,7 +433,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testNsInCigar() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -I " + validationDataLocation + "testWithNs.bam -o %s -L 8:141799600-141814700", 1,
@@ -447,7 +447,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testReducedBam() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -I " + privateTestDir + "bamExample.ReducedRead.ADAnnotation.bam -o %s -L 1:67,225,396-67,288,518", 1,
@@ -455,12 +455,12 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test calling on a ReducedRead BAM", spec);
     }
 
-    //@Test
+    @Test
     public void testReducedBamSNPs() {
         testReducedCalling("SNP", "9ba4867cadb366746ee63e7a4afdb95e");
     }
 
-    //@Test
+    @Test
     public void testReducedBamINDELs() {
         testReducedCalling("INDEL", "132a4e0ccf9230b5bb4b56c649e2bdd5");
     }
@@ -479,7 +479,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    //@Test
+    @Test
     public void testContaminationDownsampling() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000 --contamination_fraction_to_filter 0.20", 1,

From 251983b8fb829c084dfb29b8d60e9bc4e636491e Mon Sep 17 00:00:00 2001
From: Mark DePristo <depristo@broadinstitute.org>
Date: Fri, 26 Oct 2012 13:18:18 -0400
Subject: [PATCH 31/50] Add GATK-wide command line argument to control the
 maximum runtime allowed for the GATK

-- Providing this optional argument -maxRuntime (in -maxRuntimeUnits units) causes the GATK to exit gracefully when the max. runtime has been exceeded.  By cleanly I mean that the engine simply stops at the next available cycle in the walker as through the end of processing had been reached.  This means that all output files are closed properly, etc.
-- Emits an info message that looks like "INFO  10:36:52,723 MicroScheduler - Aborting execution (cleanly) because the runtime has exceeded the requested maximum 10.0000 s".  Otherwise there's currently no way to differentiate a truly completed run from a timelimit exceeded run, which may be a useful thing for a future update
-- Resolves GSA-630 / GATK max runtime to deal with bad LSA calling?
-- Added new JIRA entry for Ami to restart chr1 macarthur with this argument set to -maxRuntime 1 -maxRuntimeUnits DAYS to see if we can do all of chr1 in one weekend.
---
 .../sting/gatk/GenomeAnalysisEngine.java      | 31 ++++++-
 .../arguments/GATKArgumentCollection.java     | 10 ++-
 .../executive/HierarchicalMicroScheduler.java |  2 +-
 .../gatk/executive/LinearMicroScheduler.java  |  2 +-
 .../sting/gatk/executive/MicroScheduler.java  | 22 +++++
 .../sting/utils/AutoFormattingTime.java       | 31 ++++++-
 .../utils/progressmeter/ProgressMeter.java    |  9 ++
 .../sting/gatk/MaxRuntimeIntegrationTest.java | 86 +++++++++++++++++++
 8 files changed, 185 insertions(+), 8 deletions(-)
 create mode 100644 public/java/test/org/broadinstitute/sting/gatk/MaxRuntimeIntegrationTest.java

diff --git a/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java b/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java
index 67e5ad95b..b7000e0ee 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java
@@ -64,6 +64,7 @@ import org.broadinstitute.sting.utils.threading.ThreadEfficiencyMonitor;
 
 import java.io.File;
 import java.util.*;
+import java.util.concurrent.TimeUnit;
 
 /**
  * A GenomeAnalysisEngine that runs a specified walker.
@@ -73,6 +74,7 @@ public class GenomeAnalysisEngine {
      * our log, which we want to capture anything from this class
      */
     private static Logger logger = Logger.getLogger(GenomeAnalysisEngine.class);
+    public static final long NO_RUNTIME_LIMIT = -1;
 
     /**
      * The GATK command-line argument parsing code.
@@ -1090,6 +1092,33 @@ public class GenomeAnalysisEngine {
     public String createApproximateCommandLineArgumentString(Object... argumentProviders) {
         return CommandLineUtils.createApproximateCommandLineArgumentString(parsingEngine,argumentProviders);
     }
-    
 
+    /**
+     * Does the current runtime in unit exceed the runtime limit, if one has been provided?
+     *
+     * @param runtime the runtime of this GATK instance in minutes
+     * @param unit the time unit of runtime
+     * @return false if not limit was requested or if runtime <= the limit, true otherwise
+     */
+    public boolean exceedsRuntimeLimit(final long runtime, final TimeUnit unit) {
+        if ( runtime < 0 ) throw new IllegalArgumentException("runtime must be >= 0 but got " + runtime);
+
+        if ( getArguments().maxRuntime == NO_RUNTIME_LIMIT )
+            return false;
+        else {
+            final long actualRuntimeNano = TimeUnit.NANOSECONDS.convert(runtime, unit);
+            final long maxRuntimeNano = getRuntimeLimitInNanoseconds();
+            return actualRuntimeNano > maxRuntimeNano;
+        }
+    }
+
+    /**
+     * @return the runtime limit in nanoseconds, or -1 if no limit was specified
+     */
+    public long getRuntimeLimitInNanoseconds() {
+        if ( getArguments().maxRuntime == NO_RUNTIME_LIMIT )
+            return -1;
+        else
+            return TimeUnit.NANOSECONDS.convert(getArguments().maxRuntime, getArguments().maxRuntimeUnits);
+    }
 }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java
index 7875ced5a..e2b943582 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java
@@ -31,6 +31,7 @@ import org.broadinstitute.sting.commandline.Argument;
 import org.broadinstitute.sting.commandline.Hidden;
 import org.broadinstitute.sting.commandline.Input;
 import org.broadinstitute.sting.commandline.IntervalBinding;
+import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
 import org.broadinstitute.sting.gatk.downsampling.DownsampleType;
 import org.broadinstitute.sting.gatk.downsampling.DownsamplingMethod;
 import org.broadinstitute.sting.gatk.phonehome.GATKRunReport;
@@ -44,6 +45,7 @@ import java.io.File;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 /**
  * @author aaron
@@ -91,7 +93,7 @@ public class GATKArgumentCollection {
 
     // --------------------------------------------------------------------------------------------------------------
     //
-    // XXX
+    // General features
     //
     // --------------------------------------------------------------------------------------------------------------
 
@@ -143,6 +145,12 @@ public class GATKArgumentCollection {
     @Argument(fullName = "disableRandomization",doc="Completely eliminates randomization from nondeterministic methods. To be used mostly in the testing framework where dynamic parallelism can result in differing numbers of calls to the generator.")
     public boolean disableRandomization = false;
 
+    @Argument(fullName = "maxRuntime", shortName = "maxRuntime", doc="If provided, that GATK will stop execution cleanly as soon after maxRuntime has been exceeded, truncating the run but not exiting with a failure.  By default the value is interpreted in minutes, but this can be changed by maxRuntimeUnits", required = false)
+    public long maxRuntime = GenomeAnalysisEngine.NO_RUNTIME_LIMIT;
+
+    @Argument(fullName = "maxRuntimeUnits", shortName = "maxRuntimeUnits", doc="The TimeUnit for maxRuntime", required = false)
+    public TimeUnit maxRuntimeUnits = TimeUnit.MINUTES;
+
     // --------------------------------------------------------------------------------------------------------------
     //
     // Downsampling Arguments
diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java
index 31f2a469c..cc0161b2d 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java
@@ -123,7 +123,7 @@ public class HierarchicalMicroScheduler extends MicroScheduler implements Hierar
         final ReduceTree reduceTree = new ReduceTree(this);
         initializeWalker(walker);
 
-        while (isShardTraversePending() || isTreeReducePending()) {
+        while (! abortExecution() && (isShardTraversePending() || isTreeReducePending())) {
             // Check for errors during execution.
             errorTracker.throwErrorIfPending();
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java
index 5b94e0767..f3c1ae91c 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java
@@ -63,7 +63,7 @@ public class LinearMicroScheduler extends MicroScheduler {
 
         final TraversalEngine traversalEngine = borrowTraversalEngine(this);
         for (Shard shard : shardStrategy ) {
-            if ( done || shard == null ) // we ran out of shards that aren't owned
+            if ( abortExecution() || done || shard == null ) // we ran out of shards that aren't owned
                 break;
 
             if(shard.getShardType() == Shard.ShardType.LOCUS) {
diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java
index df4ed9ef8..38170040a 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java
@@ -39,6 +39,7 @@ import org.broadinstitute.sting.gatk.iterators.StingSAMIterator;
 import org.broadinstitute.sting.gatk.resourcemanagement.ThreadAllocation;
 import org.broadinstitute.sting.gatk.traversals.*;
 import org.broadinstitute.sting.gatk.walkers.*;
+import org.broadinstitute.sting.utils.AutoFormattingTime;
 import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.broadinstitute.sting.utils.exceptions.UserException;
@@ -52,6 +53,7 @@ import javax.management.ObjectName;
 import java.io.File;
 import java.lang.management.ManagementFactory;
 import java.util.*;
+import java.util.concurrent.TimeUnit;
 
 
 /**
@@ -269,6 +271,26 @@ public abstract class MicroScheduler implements MicroSchedulerMBean {
         this.threadEfficiencyMonitor = threadEfficiencyMonitor;
     }
 
+    /**
+     * Should we stop all execution work and exit gracefully?
+     *
+     * Returns true in the case where some external signal or time limit has been received, indicating
+     * that this GATK shouldn't continue executing.  This isn't a kill signal, it is really a "shutdown
+     * gracefully at the next opportunity" signal.  Concrete implementations of the MicroScheduler
+     * examine this value as often as reasonable and, if it returns true, stop what they are doing
+     * at the next available opportunity, shutdown their resources, call notify done, and return.
+     *
+     * @return true if we should abort execution, or false otherwise
+     */
+    protected boolean abortExecution() {
+        final boolean abort = engine.exceedsRuntimeLimit(progressMeter.getRuntimeInNanoseconds(), TimeUnit.NANOSECONDS);
+        if ( abort ) {
+            final AutoFormattingTime aft = new AutoFormattingTime(TimeUnit.SECONDS.convert(engine.getRuntimeLimitInNanoseconds(), TimeUnit.NANOSECONDS), 1, 4);
+            logger.info("Aborting execution (cleanly) because the runtime has exceeded the requested maximum " + aft);
+        }
+        return abort;
+    }
+
     /**
      * Walks a walker over the given list of intervals.
      *
diff --git a/public/java/src/org/broadinstitute/sting/utils/AutoFormattingTime.java b/public/java/src/org/broadinstitute/sting/utils/AutoFormattingTime.java
index 8964c16cb..4455666e8 100644
--- a/public/java/src/org/broadinstitute/sting/utils/AutoFormattingTime.java
+++ b/public/java/src/org/broadinstitute/sting/utils/AutoFormattingTime.java
@@ -4,12 +4,21 @@ package org.broadinstitute.sting.utils;
  * Simple utility class that makes it convenient to print unit adjusted times
  */
 public class AutoFormattingTime {
-    double timeInSeconds;           // in Seconds
-    int precision;      // for format
+    private final int width; // for format
+    private final int precision;      // for format
 
-    public AutoFormattingTime(double timeInSeconds, int precision) {
+    double timeInSeconds;           // in Seconds
+    private final String formatString;
+
+    public AutoFormattingTime(double timeInSeconds, final int width, int precision) {
+        this.width = width;
         this.timeInSeconds = timeInSeconds;
         this.precision = precision;
+        this.formatString = "%" + width + "." + precision + "f %s";
+    }
+
+    public AutoFormattingTime(double timeInSeconds, int precision) {
+        this(timeInSeconds, 6, precision);
     }
 
     public AutoFormattingTime(double timeInSeconds) {
@@ -20,6 +29,20 @@ public class AutoFormattingTime {
         return timeInSeconds;
     }
 
+    /**
+     * @return the precision (a la format's %WIDTH.PERCISIONf)
+     */
+    public int getWidth() {
+        return width;
+    }
+
+    /**
+     * @return the precision (a la format's %WIDTH.PERCISIONf)
+     */
+    public int getPrecision() {
+        return precision;
+    }
+
     /**
      * Instead of 10000 s, returns 2.8 hours
      * @return
@@ -48,6 +71,6 @@ public class AutoFormattingTime {
             }
         }
 
-        return String.format("%6."+precision+"f %s", unitTime, unit);
+        return String.format(formatString, unitTime, unit);
     }
 }
diff --git a/public/java/src/org/broadinstitute/sting/utils/progressmeter/ProgressMeter.java b/public/java/src/org/broadinstitute/sting/utils/progressmeter/ProgressMeter.java
index 69cf52fc2..a8715e242 100755
--- a/public/java/src/org/broadinstitute/sting/utils/progressmeter/ProgressMeter.java
+++ b/public/java/src/org/broadinstitute/sting/utils/progressmeter/ProgressMeter.java
@@ -24,6 +24,7 @@
 
 package org.broadinstitute.sting.utils.progressmeter;
 
+import com.google.java.contract.Ensures;
 import com.google.java.contract.Invariant;
 import org.apache.log4j.Logger;
 import org.broadinstitute.sting.utils.*;
@@ -200,6 +201,14 @@ public class ProgressMeter {
                 "Location", processingUnitName, processingUnitName));
     }
 
+    /**
+     * @return the current runtime in nanoseconds
+     */
+    @Ensures("result >= 0")
+    public long getRuntimeInNanoseconds() {
+        return timer.getElapsedTimeNano();
+    }
+
     /**
      * Utility routine that prints out process information (including timing) every N records or
      * every M seconds, for N and M set in global variables.
diff --git a/public/java/test/org/broadinstitute/sting/gatk/MaxRuntimeIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/MaxRuntimeIntegrationTest.java
new file mode 100644
index 000000000..6cfd7bf46
--- /dev/null
+++ b/public/java/test/org/broadinstitute/sting/gatk/MaxRuntimeIntegrationTest.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.gatk;
+
+import org.broadinstitute.sting.WalkerTest;
+import org.broadinstitute.sting.utils.SimpleTimer;
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ */
+public class MaxRuntimeIntegrationTest extends WalkerTest {
+    private static final long STARTUP_TIME = TimeUnit.NANOSECONDS.convert(20, TimeUnit.SECONDS);
+
+    private class MaxRuntimeTestProvider extends TestDataProvider {
+        final long maxRuntime;
+        final TimeUnit unit;
+
+        public MaxRuntimeTestProvider(final long maxRuntime, final TimeUnit unit) {
+            super(MaxRuntimeTestProvider.class);
+            this.maxRuntime = maxRuntime;
+            this.unit = unit;
+            setName(String.format("Max runtime test : %d of %s", maxRuntime, unit));
+        }
+
+        public long expectedMaxRuntimeNano() {
+            return TimeUnit.NANOSECONDS.convert(maxRuntime, unit) + STARTUP_TIME;
+        }
+    }
+
+    @DataProvider(name = "MaxRuntimeProvider")
+    public Object[][] makeMaxRuntimeProvider() {
+        for ( final TimeUnit requestedUnits : Arrays.asList(TimeUnit.NANOSECONDS, TimeUnit.MILLISECONDS, TimeUnit.SECONDS, TimeUnit.MINUTES) )
+            new MaxRuntimeTestProvider(requestedUnits.convert(30, TimeUnit.SECONDS), requestedUnits);
+
+        return MaxRuntimeTestProvider.getTests(MaxRuntimeTestProvider.class);
+    }
+
+    //
+    // Loop over errors to throw, make sure they are the errors we get back from the engine, regardless of NT type
+    //
+    @Test(enabled = true, dataProvider = "MaxRuntimeProvider", timeOut = 60 * 1000)
+    public void testMaxRuntime(final MaxRuntimeTestProvider cfg) {
+        WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
+                "-T UnifiedGenotyper -R " + hg18Reference
+                        + " -I " + validationDataLocation + "NA12878.WEx.downsampled20x.bam -o /dev/null"
+                        + " -maxRuntime " + cfg.maxRuntime + " -maxRuntimeUnits " + cfg.unit, 0,
+                Collections.<String>emptyList());
+        final SimpleTimer timer = new SimpleTimer().start();
+        executeTest("Max runtime " + cfg, spec);
+        final long actualRuntimeNano = timer.getElapsedTimeNano();
+
+        Assert.assertTrue(actualRuntimeNano < cfg.expectedMaxRuntimeNano(),
+                "Actual runtime " + TimeUnit.SECONDS.convert(actualRuntimeNano, TimeUnit.NANOSECONDS)
+                        + " exceeded max. tolerated runtime " + TimeUnit.SECONDS.convert(cfg.expectedMaxRuntimeNano(), TimeUnit.NANOSECONDS)
+                        + " given requested runtime " + cfg.maxRuntime + " " + cfg.unit);
+    }
+}
\ No newline at end of file

From a8704ca73f1e9c94430bfeef5541d87df425a573 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Fri, 26 Oct 2012 13:20:27 -0400
Subject: [PATCH 32/50] Adding TODO notes for Ami

---
 .../sting/gatk/walkers/qc/AssessReducedQuals.java         | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java
index e74684c53..78bcf1228 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/AssessReducedQuals.java
@@ -55,8 +55,8 @@ public class AssessReducedQuals extends LocusWalker<GenomeLoc, GenomeLoc> implem
     @Argument(fullName = "qual_epsilon", shortName = "epsilon", doc = "when |Quals_reduced_bam - Quals_original_bam| > epsilon*Quals_original_bam we output this interval", required = false)
     public int qual_epsilon = 0;
 
-    @Argument(fullName = "debugLevel", shortName = "debug", doc = "debug mode on")
-    public int debugLevel = 0;
+    @Argument(fullName = "debugLevel", shortName = "debug", doc = "debug mode on")   // TODO -- best to make this optional
+    public int debugLevel = 0;   // TODO -- best to make this an enum or boolean
 
     @Output
     protected PrintStream out;
@@ -114,7 +114,11 @@ public class AssessReducedQuals extends LocusWalker<GenomeLoc, GenomeLoc> implem
         return quals;
     }
 
+    // TODO -- arguments/variables should be final, not method declaration
     private final boolean isGoodRead(PileupElement p, List<String> tags){
+        // TODO -- this isn't quite right.  You don't need the tags here.  Instead, you want to check whether the read itself (which
+        // TODO --  you can get from the PileupElement) is a reduced read (not all reads from the reduced bam are reduced) and only
+        // TODO --  for them do you want to ignore that min mapping quality cutoff (but you *do* still want the min base cutoff).
         return !p.isDeletion() && (tags.contains(reduced) || (tags.contains(original) && (int)p.getQual() >= 20 && p.getMappingQual() >= 20));
     }
 

From 35483a7eefc3cf5bd68276a2a6daa8cd7a6f2e6b Mon Sep 17 00:00:00 2001
From: David Roazen <droazen@broadinstitute.org>
Date: Fri, 26 Oct 2012 14:15:16 -0400
Subject: [PATCH 33/50] Update MD5s for PrintReads with BQSR Integration Test

The MD5s for these tests were changed in commit 87435f1074615b2cd016f042980109fd53962c8d
to match the output of a broken version of BaseRecalibration. With the patch in
commit c397102ecc1fd1d2cd8f209a8f358ab4a60b50a7, the output once again matches the
*original* MD5s for these tests, and does not vary as you increase -nct.

Final resolution to GSA-632
---
 .../sting/gatk/walkers/bqsr/BQSRIntegrationTest.java        | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
index a50219f48..780ef9e0b 100644
--- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
+++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
@@ -122,12 +122,12 @@ public class BQSRIntegrationTest extends WalkerTest {
     public Object[][] createPRTestData() {
         List<Object[]> tests = new ArrayList<Object[]>();
 
-        tests.add(new Object[]{1, new PRTest(" -qq -1", "a1d87da5dcbde35170d6ba6bc3ee2812")});
-        tests.add(new Object[]{1, new PRTest(" -qq 6", "a0fecae6d0e5ab9917862fa306186d10")});
+        tests.add(new Object[]{1, new PRTest(" -qq -1", "5226c06237b213b9e9b25a32ed92d09a")});
+        tests.add(new Object[]{1, new PRTest(" -qq 6", "b592a5c62b952a012e18adb898ea9c33")});
         tests.add(new Object[]{1, new PRTest(" -DIQ", "8977bea0c57b808e65e9505eb648cdf7")});
 
         for ( final int nct : Arrays.asList(1, 2, 4) ) {
-            tests.add(new Object[]{nct, new PRTest("", "d1bbb4ce6aa93e866f106f8b11d888ed")});
+            tests.add(new Object[]{nct, new PRTest("", "ab2f209ab98ad3432e208cbd524a4c4a")});
         }
 
         return tests.toArray(new Object[][]{});

From 682a72faf72b61ca24f2d87b675291372c48aa12 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Fri, 26 Oct 2012 16:10:12 -0400
Subject: [PATCH 35/50] Hmm, thought I got all the md5s last time.  Apparently
 not.

---
 .../walkers/genotyper/UnifiedGenotyperIntegrationTest.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index 2a17435aa..49651100a 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -157,7 +157,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testSLOD() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b36KGReference + " --computeSLOD --no_cmdline_in_header -glm BOTH --dbsnp " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1,
-                Arrays.asList("9f95cfe14d53a697c58247833bfd72a6"));
+                Arrays.asList("6ccb9bd88934e4272d0ce362dd35e603"));
         executeTest("test SLOD", spec);
     }
 
@@ -361,7 +361,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndelsb37 + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -out_mode EMIT_ALL_SITES -alleles " + privateTestDir + vcf + " -I " + validationDataLocation +
                         "NA12878.HiSeq.WGS.bwa.cleaned.recal.hg19.20.bam -o %s -L " + validationDataLocation + vcf, 1,
-                Arrays.asList("d76eacc4021b78ccc0a9026162e814a7"));
+                Arrays.asList("9a7cd58b9e3d5b72608c0d529321deba"));
         executeTest("test GENOTYPE_GIVEN_ALLELES with no evidence in reads", spec);
     }
 
@@ -457,7 +457,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
     @Test
     public void testReducedBamSNPs() {
-        testReducedCalling("SNP", "9ba4867cadb366746ee63e7a4afdb95e");
+        testReducedCalling("SNP", "e7fc11baf208a1bca7b462d3148c936e");
     }
 
     @Test

From fa9b2a91d05c1e5a5ecfb4db8028835e6881e264 Mon Sep 17 00:00:00 2001
From: Mark DePristo <depristo@broadinstitute.org>
Date: Fri, 26 Oct 2012 16:25:22 -0400
Subject: [PATCH 36/50] Bugfix for GSA-552

-- https://jira.broadinstitute.org/browse/GSA-552
-- User reports a null exception while using VariantsToVCF:
   http://gatkforums.broadinstitute.org/discussion/1461/nullpointerexception-converting-vcf3-to-vcf-using-variantstovcf
   The problem is that he left out an input VCF file for the --variant argument and the command-line argument parsing code didn't catch this, so we NPE out later on.
---
 .../sting/commandline/ArgumentTypeDescriptor.java               | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java b/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java
index 54ade61f6..5d7eba16c 100644
--- a/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java
+++ b/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java
@@ -406,7 +406,7 @@ public abstract class ArgumentTypeDescriptor {
             else
                 throw new UserException.CommandLineException(
                         String.format("Failed to parse value %s for argument %s. Message: %s",
-                                value.asString(), fieldName, e.getMessage()));
+                                value, fieldName, e.getMessage()));
         }
     }
 }

From ac5e58a265e7563a866ad8b5f4fd756af8fa9579 Mon Sep 17 00:00:00 2001
From: Mark DePristo <depristo@broadinstitute.org>
Date: Fri, 26 Oct 2012 16:34:07 -0400
Subject: [PATCH 37/50] Bugfix for GSA-540 / Update metadata maps when adding
 lines to VCFHeader

-- https://jira.broadinstitute.org/browse/GSA-540
-- http://gatkforums.broadinstitute.org/discussion/1433/possible-bug-and-fix-in-java-code-of-vcfheader-org-broadinstitute-sting-utils-codecs-vcf-vcfheader
---
 .../org/broadinstitute/sting/utils/codecs/vcf/VCFHeader.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFHeader.java b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFHeader.java
index 2663e848f..44a3e9af3 100755
--- a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFHeader.java
+++ b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFHeader.java
@@ -159,6 +159,7 @@ public class VCFHeader {
      */
     public void addMetaDataLine(VCFHeaderLine headerLine) {
         mMetaData.add(headerLine);
+        loadMetaDataMaps();
     }
 
     /**
@@ -236,7 +237,6 @@ public class VCFHeader {
                     + VCFConstants.GENOTYPE_PL_KEY + " field.  As the GATK now only manages PL fields internally"
                     + " automatically adding a corresponding PL field to your VCF header");
             addMetaDataLine(new VCFFormatHeaderLine(VCFConstants.GENOTYPE_PL_KEY, VCFHeaderLineCount.G, VCFHeaderLineType.Integer, "Normalized, Phred-scaled likelihoods for genotypes as defined in the VCF specification"));
-            loadMetaDataMaps();
         }
     }
 

From b4fbf6280ac0c2469b51852b28f270664de1abb9 Mon Sep 17 00:00:00 2001
From: Andrey Sivachenko <asivache@broadinstitute.org>
Date: Fri, 26 Oct 2012 23:48:40 -0400
Subject: [PATCH 38/50] fixing missing sample genotype bug, missing AD/DP bug,
 and putting annotations in more natural order (Ref/Alt)

---
 .../walkers/indels/SomaticIndelDetector.java  | 103 ++++++++++++++----
 1 file changed, 81 insertions(+), 22 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetector.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetector.java
index 7c73f59e9..be668b9c3 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetector.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetector.java
@@ -438,8 +438,6 @@ public class SomaticIndelDetector extends ReadWalker<Integer,Integer> {
 
 		location = getToolkit().getGenomeLocParser().createGenomeLoc(getToolkit().getSAMFileHeader().getSequence(0).getSequenceName(),1);
 
-        normalSamples = SampleUtils.getSAMFileSamples(getToolkit().getSAMFileHeaders().get(0));
-
         try {
             // we already checked that bedOutput and output_file are not set simultaneously
             if ( bedOutput != null ) bedWriter = new FileWriter(bedOutput);
@@ -1152,8 +1150,9 @@ public class SomaticIndelDetector extends ReadWalker<Integer,Integer> {
 
         GenotypesContext genotypes = GenotypesContext.create();
         for ( String sample : normalSamples ) {
-            final GenotypeBuilder gb = new GenotypeBuilder(sample);
-            gb.attributes(call.makeStatsAttributes(null));
+            GenotypeBuilder gb = new GenotypeBuilder(sample);
+
+            gb=call.addStatsAttributes(gb);
             gb.alleles(! discard_event
                 ? alleles             // we made a call - put actual het genotype here:
                 : homref_alleles);    // no call: genotype is ref/ref (but alleles still contain the alt if we observed anything at all)
@@ -1200,8 +1199,11 @@ public class SomaticIndelDetector extends ReadWalker<Integer,Integer> {
         if ( start == 0 )
             return;
 
-        Map<String,Object> attrsNormal = nCall.makeStatsAttributes(null);
-        Map<String,Object> attrsTumor = tCall.makeStatsAttributes(null);
+        GenotypeBuilder nGB = new GenotypeBuilder();
+        GenotypeBuilder tGB = new GenotypeBuilder();
+
+        nCall.addStatsAttributes(nGB);
+        tCall.addStatsAttributes(tGB);
 
         Map<String,Object> attrs = new HashMap();
 
@@ -1242,11 +1244,11 @@ public class SomaticIndelDetector extends ReadWalker<Integer,Integer> {
         GenotypesContext genotypes = GenotypesContext.create();
 
         for ( String sample : normalSamples ) {
-            genotypes.add(GenotypeBuilder.create(sample, homRefN ? homRefAlleles : alleles, attrsNormal));
+            genotypes.add(nGB.name(sample).alleles(homRefN ? homRefAlleles : alleles).make());
         }
 
         for ( String sample : tumorSamples ) {
-            genotypes.add(GenotypeBuilder.create(sample, homRefT ? homRefAlleles : alleles, attrsTumor));
+            genotypes.add(tGB.name(sample).alleles(homRefT ? homRefAlleles : alleles).make());
         }
 
         Set<String> filters = null;
@@ -1254,14 +1256,6 @@ public class SomaticIndelDetector extends ReadWalker<Integer,Integer> {
             filters = new HashSet<String>();
             filters.add("NoCall");
         }
-        if ( nCall.getCoverage() < minNormalCoverage ) {
-            if ( filters == null ) filters = new HashSet<String>();
-            filters.add("NCov");
-        }
-        if ( tCall.getCoverage() < minCoverage ) {
-            if ( filters == null ) filters = new HashSet<String>();
-            filters.add("TCov");
-        }
 
         VariantContext vc = new VariantContextBuilder("IGv2_Indel_call", refName, start, stop, alleles)
                 .genotypes(genotypes).filters(filters).attributes(attrs).make();
@@ -1844,6 +1838,38 @@ public class SomaticIndelDetector extends ReadWalker<Integer,Integer> {
              VCFIndelAttributes.recordStrandCounts(strand_cons.first,strand_cons.second,strand_ref.first,strand_ref.second,attr);
              return attr;
          }
+
+
+    /**
+      * Adds alignment statistics directly into the genotype builder object.
+      *
+      * @param gb
+      * @return
+      */
+     public GenotypeBuilder addStatsAttributes(GenotypeBuilder gb) {
+          if ( gb == null ) gb = new GenotypeBuilder();
+
+          gb = VCFIndelAttributes.recordDepth(getConsensusVariantCount(),getAllVariantCount(),getCoverage(),gb);
+
+          gb = VCFIndelAttributes.recordAvMM(getAvConsensusMismatches(),getAvRefMismatches(),gb);
+
+          gb = VCFIndelAttributes.recordAvMapQ(getAvConsensusMapq(),getAvRefMapq(),gb);
+
+          gb = VCFIndelAttributes.recordNQSMMRate(getNQSConsensusMMRate(),getNQSRefMMRate(),gb);
+
+          gb = VCFIndelAttributes.recordNQSAvQ(getNQSConsensusAvQual(),getNQSRefAvQual(),gb);
+
+          gb = VCFIndelAttributes.recordOffsetFromStart(from_start_median,from_start_mad,gb);
+
+          gb = VCFIndelAttributes.recordOffsetFromEnd(from_end_median,from_end_mad,gb);
+
+          PrimitivePair.Int strand_cons = getConsensusStrandCounts();
+          PrimitivePair.Int strand_ref = getRefStrandCounts();
+
+          gb = VCFIndelAttributes.recordStrandCounts(strand_cons.first,strand_cons.second,strand_ref.first,strand_ref.second,gb);
+          return gb;
+      }
+
     }
 
     interface IndelListener {
@@ -2181,7 +2207,7 @@ class VCFIndelAttributes {
 
         lines.add(new VCFFormatHeaderLine(NQS_AVQ_KEY, 2, VCFHeaderLineType.Float, "Within NQS window: average quality of bases from consensus indel-supporting reads/from reference-supporting reads"));
 
-        lines.add(new VCFFormatHeaderLine(STRAND_COUNT_KEY, 4, VCFHeaderLineType.Integer, "Strandness: counts of forward-/reverse-aligned indel-supporting reads / forward-/reverse-aligned reference supporting reads"));
+        lines.add(new VCFFormatHeaderLine(STRAND_COUNT_KEY, 4, VCFHeaderLineType.Integer, "Strandness: counts of forward-/reverse-aligned reference and indel-supporting reads (FwdRef,RevRef,FwdIndel,RevIndel)"));
 
         lines.add(new VCFFormatHeaderLine(RSTART_OFFSET_KEY, 2, VCFHeaderLineType.Integer, "Median/mad of indel offsets from the starts of the reads"));
         lines.add(new VCFFormatHeaderLine(REND_OFFSET_KEY, 2, VCFHeaderLineType.Integer, "Median/mad of indel offsets from the ends of the reads"));
@@ -2194,39 +2220,72 @@ class VCFIndelAttributes {
         return attrs;
     }
 
+    public static GenotypeBuilder recordStrandCounts(int cnt_cons_fwd, int cnt_cons_rev, int cnt_ref_fwd, int cnt_ref_rev,
+                                                     GenotypeBuilder gb) {
+        return gb.attribute(STRAND_COUNT_KEY, new Integer[] {cnt_ref_fwd, cnt_ref_rev,cnt_cons_fwd, cnt_cons_rev } );
+    }
+
     public static Map<String,Object> recordDepth(int cnt_cons, int cnt_indel, int cnt_total, Map<String,Object> attrs) {
-        attrs.put(ALLELIC_DEPTH_KEY, new Integer[] {cnt_cons, cnt_indel} );
+        attrs.put(ALLELIC_DEPTH_KEY, new Integer[] {cnt_total-cnt_indel, cnt_cons} );
         attrs.put(DEPTH_TOTAL_KEY, cnt_total);
         return attrs;
     }
 
+    public static GenotypeBuilder recordDepth(int cnt_cons, int cnt_indel, int cnt_total, GenotypeBuilder gb) {
+        return gb.AD(new int[] {cnt_total-cnt_indel,cnt_cons} ).DP(cnt_total);
+    }
+
     public static Map<String,Object> recordAvMapQ(double cons, double ref, Map<String,Object> attrs) {
-        attrs.put(MAPQ_KEY, new Float[] {(float)cons, (float)ref} );
+        attrs.put(MAPQ_KEY, new Float[] {(float)ref, (float)cons} );
         return attrs;
     }
 
+    public static GenotypeBuilder recordAvMapQ(double cons, double ref, GenotypeBuilder gb) {
+        return gb.attribute(MAPQ_KEY,new float[] {(float)ref, (float)cons} );
+    }
+
     public static Map<String,Object> recordAvMM(double cons, double ref, Map<String,Object> attrs) {
-        attrs.put(MM_KEY, new Float[] {(float)cons, (float)ref} );
+        attrs.put(MM_KEY, new Float[] {(float)ref, (float)cons} );
         return attrs;
     }
 
+    public static GenotypeBuilder recordAvMM(double cons, double ref, GenotypeBuilder gb) {
+        return gb.attribute(MM_KEY, new float[] {(float)ref, (float)cons} );
+    }
+
     public static Map<String,Object> recordNQSMMRate(double cons, double ref, Map<String,Object> attrs) {
-        attrs.put(NQS_MMRATE_KEY, new Float[] {(float)cons, (float)ref} );
+        attrs.put(NQS_MMRATE_KEY, new Float[] {(float)ref, (float)cons} );
         return attrs;
     }
 
+    public static GenotypeBuilder recordNQSMMRate(double cons, double ref, GenotypeBuilder gb) {
+        return gb.attribute(NQS_MMRATE_KEY, new float[] {(float)ref, (float)cons} );
+    }
+
     public static Map<String,Object> recordNQSAvQ(double cons, double ref, Map<String,Object> attrs) {
-        attrs.put(NQS_AVQ_KEY, new Float[] {(float)cons, (float)ref} );
+        attrs.put(NQS_AVQ_KEY, new float[] {(float)ref, (float)cons} );
         return attrs;
     }
 
+    public static GenotypeBuilder recordNQSAvQ(double cons, double ref, GenotypeBuilder gb) {
+        return gb.attribute(NQS_AVQ_KEY, new float[] {(float)ref, (float)cons} );
+    }
+
     public static Map<String,Object> recordOffsetFromStart(int median, int mad, Map<String,Object> attrs) {
         attrs.put(RSTART_OFFSET_KEY, new Integer[] {median, mad} );
         return attrs;
     }
 
+    public static GenotypeBuilder recordOffsetFromStart(int median, int mad, GenotypeBuilder gb) {
+        return gb.attribute(RSTART_OFFSET_KEY, new int[] {median, mad} );
+    }
+
     public static Map<String,Object> recordOffsetFromEnd(int median, int mad, Map<String,Object> attrs) {
         attrs.put(REND_OFFSET_KEY, new Integer[] {median, mad} );
         return attrs;
     }
+
+    public static GenotypeBuilder recordOffsetFromEnd(int median, int mad, GenotypeBuilder gb) {
+        return gb.attribute(REND_OFFSET_KEY, new int[] {median, mad} );
+    }
 }

From f3ac5d404dc50ba89229cb07dba118f42f6956a6 Mon Sep 17 00:00:00 2001
From: Andrey Sivachenko <asivache@broadinstitute.org>
Date: Fri, 26 Oct 2012 23:52:21 -0400
Subject: [PATCH 39/50] updating vcf header attribute descriptions in order to
 reflect correctly what's actually being written...

---
 .../gatk/walkers/indels/SomaticIndelDetector.java      | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetector.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetector.java
index be668b9c3..0165c6cf3 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetector.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetector.java
@@ -2196,16 +2196,16 @@ class VCFIndelAttributes {
     public static Set<? extends VCFHeaderLine> getAttributeHeaderLines() {
         Set<VCFHeaderLine> lines = new HashSet<VCFHeaderLine>();
 
-        lines.add(new VCFFormatHeaderLine(ALLELIC_DEPTH_KEY, 2, VCFHeaderLineType.Integer, "# of reads supporting consensus indel/reference at the site"));
+        lines.add(new VCFFormatHeaderLine(ALLELIC_DEPTH_KEY, 2, VCFHeaderLineType.Integer, "# of reads supporting consensus reference/indel at the site"));
         lines.add(new VCFFormatHeaderLine(DEPTH_TOTAL_KEY, 1, VCFHeaderLineType.Integer, "Total coverage at the site"));
 
-        lines.add(new VCFFormatHeaderLine(MAPQ_KEY, 2, VCFHeaderLineType.Float, "Average mapping qualities of consensus indel-supporting reads/reference-supporting reads"));
+        lines.add(new VCFFormatHeaderLine(MAPQ_KEY, 2, VCFHeaderLineType.Float, "Average mapping qualities of ref-/consensus indel-supporting reads"));
 
-        lines.add(new VCFFormatHeaderLine(MM_KEY, 2, VCFHeaderLineType.Float, "Average # of mismatches per consensus indel-supporting read/per reference-supporting read"));
+        lines.add(new VCFFormatHeaderLine(MM_KEY, 2, VCFHeaderLineType.Float, "Average # of mismatches per ref-/consensus indel-supporting read"));
 
-        lines.add(new VCFFormatHeaderLine(NQS_MMRATE_KEY, 2, VCFHeaderLineType.Float, "Within NQS window: fraction of mismatching bases in consensus indel-supporting reads/in reference-supporting reads"));
+        lines.add(new VCFFormatHeaderLine(NQS_MMRATE_KEY, 2, VCFHeaderLineType.Float, "Within NQS window: fraction of mismatching bases in ref/consensus indel-supporting reads"));
 
-        lines.add(new VCFFormatHeaderLine(NQS_AVQ_KEY, 2, VCFHeaderLineType.Float, "Within NQS window: average quality of bases from consensus indel-supporting reads/from reference-supporting reads"));
+        lines.add(new VCFFormatHeaderLine(NQS_AVQ_KEY, 2, VCFHeaderLineType.Float, "Within NQS window: average quality of bases in ref-/consensus indel-supporting reads"));
 
         lines.add(new VCFFormatHeaderLine(STRAND_COUNT_KEY, 4, VCFHeaderLineType.Integer, "Strandness: counts of forward-/reverse-aligned reference and indel-supporting reads (FwdRef,RevRef,FwdIndel,RevIndel)"));
 

From 43625f652e7bb70e4c2507f05a0c9997aeaf655c Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Sat, 27 Oct 2012 19:43:46 -0400
Subject: [PATCH 40/50] Shoot, mixed up the md5s last time.

---
 .../walkers/genotyper/UnifiedGenotyperIntegrationTest.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index 49651100a..9212d0e53 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -361,7 +361,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommandIndelsb37 + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -out_mode EMIT_ALL_SITES -alleles " + privateTestDir + vcf + " -I " + validationDataLocation +
                         "NA12878.HiSeq.WGS.bwa.cleaned.recal.hg19.20.bam -o %s -L " + validationDataLocation + vcf, 1,
-                Arrays.asList("9a7cd58b9e3d5b72608c0d529321deba"));
+                Arrays.asList("d76eacc4021b78ccc0a9026162e814a7"));
         executeTest("test GENOTYPE_GIVEN_ALLELES with no evidence in reads", spec);
     }
 
@@ -451,7 +451,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testReducedBam() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-T UnifiedGenotyper -R " + b37KGReference + " --no_cmdline_in_header -I " + privateTestDir + "bamExample.ReducedRead.ADAnnotation.bam -o %s -L 1:67,225,396-67,288,518", 1,
-                Arrays.asList("1b711c0966a2f76eb21801e73b76b758"));
+                Arrays.asList("9a7cd58b9e3d5b72608c0d529321deba"));
         executeTest("test calling on a ReducedRead BAM", spec);
     }
 

From ac99437eec6da551b0de03b02c0a5ce5500e44d0 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Mon, 29 Oct 2012 01:45:33 -0400
Subject: [PATCH 41/50] Bug fixes to hapmap conversion in VariantsToVCF

---
 .../sting/gatk/refdata/VariantContextAdaptors.java              | 2 +-
 .../sting/gatk/walkers/variantutils/VariantsToVCF.java          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/gatk/refdata/VariantContextAdaptors.java b/public/java/src/org/broadinstitute/sting/gatk/refdata/VariantContextAdaptors.java
index 2b46414a8..5c7da82d0 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/refdata/VariantContextAdaptors.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/refdata/VariantContextAdaptors.java
@@ -364,7 +364,7 @@ public class VariantContextAdaptors {
 
             long end = hapmap.getEnd();
             if ( deletionLength > 0 )
-                end += deletionLength;
+                end += (deletionLength - 1);
             VariantContext vc = new VariantContextBuilder(name, hapmap.getChr(), hapmap.getStart(), end, alleles).id(hapmap.getName()).genotypes(genotypes).make();
             return vc;
        }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToVCF.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToVCF.java
index 5f80f77a4..059e9c5fb 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToVCF.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToVCF.java
@@ -160,7 +160,7 @@ public class VariantsToVCF extends RodWalker<Integer, Integer> {
 
                         Map<String, Allele> alleleMap = new HashMap<String, Allele>(2);
                         alleleMap.put(RawHapMapFeature.DELETION, Allele.create(ref.getBase(), dbsnpVC.isSimpleInsertion()));
-                        alleleMap.put(RawHapMapFeature.INSERTION, Allele.create(ref.getBase() + ((RawHapMapFeature)record).getAlleles()[1], !dbsnpVC.isSimpleInsertion()));
+                        alleleMap.put(RawHapMapFeature.INSERTION, Allele.create((char)ref.getBase() + ((RawHapMapFeature)record).getAlleles()[1], !dbsnpVC.isSimpleInsertion()));
                         hapmap.setActualAlleles(alleleMap);
 
                         // also, use the correct positioning for insertions

From 4e661847b21fb8322aa4d95035a9776695f0ac7f Mon Sep 17 00:00:00 2001
From: Ryan Poplin <rpoplin@broadinstitute.org>
Date: Mon, 29 Oct 2012 12:53:39 -0400
Subject: [PATCH 42/50] DelocalizedBaseRecalibrator becomes the
 BaseRecalibrator.

---
 .../bqsr/AdvancedRecalibrationEngine.java     |  45 ---
 .../walkers/bqsr/BQSRIntegrationTest.java     |  30 +-
 .../gatk/walkers/bqsr/BaseRecalibrator.java   | 294 ++++++++++++++----
 .../walkers/bqsr/RecalibrationEngine.java     |   2 -
 .../bqsr/StandardRecalibrationEngine.java     | 129 ++------
 5 files changed, 265 insertions(+), 235 deletions(-)

diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/AdvancedRecalibrationEngine.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/AdvancedRecalibrationEngine.java
index be8357679..d0bcd0eb3 100644
--- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/AdvancedRecalibrationEngine.java
+++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/AdvancedRecalibrationEngine.java
@@ -39,57 +39,12 @@ public class AdvancedRecalibrationEngine extends StandardRecalibrationEngine imp
 
     // optimization: only allocate temp arrays once per thread
     private final ThreadLocal<byte[]> threadLocalTempQualArray = new ThreadLocalArray<byte[]>(EventType.values().length, byte.class);
-    private final ThreadLocal<boolean[]> threadLocalTempErrorArray = new ThreadLocalArray<boolean[]>(EventType.values().length, boolean.class);
     private final ThreadLocal<double[]> threadLocalTempFractionalErrorArray = new ThreadLocalArray<double[]>(EventType.values().length, double.class);
 
     public void initialize(final Covariate[] covariates, final RecalibrationTables recalibrationTables) {
         super.initialize(covariates, recalibrationTables);
     }
 
-    /**
-     * Loop through the list of requested covariates and pick out the value from the read, offset, and reference
-     * Using the list of covariate values as a key, pick out the RecalDatum and increment,
-     * adding one to the number of observations and potentially one to the number of mismatches for all three
-     * categories (mismatches, insertions and deletions).
-     *
-     * @param pileupElement The pileup element to update
-     * @param refBase       The reference base at this locus
-     */
-    @Override
-    public void updateDataForPileupElement(final PileupElement pileupElement, final byte refBase) {
-        final int offset = pileupElement.getOffset();
-        final ReadCovariates readCovariates = covariateKeySetFrom(pileupElement.getRead());
-
-        byte[] tempQualArray = threadLocalTempQualArray.get();
-        boolean[] tempErrorArray = threadLocalTempErrorArray.get();
-
-        tempQualArray[EventType.BASE_SUBSTITUTION.index] = pileupElement.getQual();
-        tempErrorArray[EventType.BASE_SUBSTITUTION.index] = !BaseUtils.basesAreEqual(pileupElement.getBase(), refBase);
-        tempQualArray[EventType.BASE_INSERTION.index] = pileupElement.getBaseInsertionQual();
-        tempErrorArray[EventType.BASE_INSERTION.index] = (pileupElement.getRead().getReadNegativeStrandFlag()) ? pileupElement.isAfterInsertion() : pileupElement.isBeforeInsertion();
-        tempQualArray[EventType.BASE_DELETION.index] = pileupElement.getBaseDeletionQual();
-        tempErrorArray[EventType.BASE_DELETION.index] = (pileupElement.getRead().getReadNegativeStrandFlag()) ? pileupElement.isAfterDeletedBase() : pileupElement.isBeforeDeletedBase();
-
-        for (final EventType eventType : EventType.values()) {
-            final int[] keys = readCovariates.getKeySet(offset, eventType);
-            final int eventIndex = eventType.index;
-            final byte qual = tempQualArray[eventIndex];
-            final boolean isError = tempErrorArray[eventIndex];
-
-            // TODO: should this really be combine rather than increment?
-            combineDatumOrPutIfNecessary(recalibrationTables.getReadGroupTable(), qual, isError, keys[0], eventIndex);
-
-            incrementDatumOrPutIfNecessary(recalibrationTables.getQualityScoreTable(), qual, isError, keys[0], keys[1], eventIndex);
-
-            for (int i = 2; i < covariates.length; i++) {
-                if (keys[i] < 0)
-                    continue;
-
-                incrementDatumOrPutIfNecessary(recalibrationTables.getTable(i), qual, isError, keys[0], keys[1], keys[i], eventIndex);
-            }
-        }
-    }
-
     @Override
     public void updateDataForRead(final GATKSAMRecord read, final boolean[] skip, final double[] snpErrors, final double[] insertionErrors, final double[] deletionErrors ) {
         for( int offset = 0; offset < read.getReadBases().length; offset++ ) {
diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
index 780ef9e0b..b839382dc 100644
--- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
+++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/BQSRIntegrationTest.java
@@ -51,21 +51,21 @@ public class BQSRIntegrationTest extends WalkerTest {
         String HiSeqBam = privateTestDir + "HiSeq.1mb.1RG.bam";
         String HiSeqInterval = "chr1:10,000,000-10,100,000";
         return new Object[][]{
-                {new BQSRTest(hg18Reference, HiSeqBam, HiSeqInterval, "", "55a46d8f5d2f9acfa2d7659e18b6df43")},
-                {new BQSRTest(hg18Reference, HiSeqBam, HiSeqInterval, " --no_standard_covs -cov ContextCovariate", "8e930f56a8905a5999af7d6ba8a92f91")},
-                {new BQSRTest(hg18Reference, HiSeqBam, HiSeqInterval, " --no_standard_covs -cov CycleCovariate", "8e87bee4bd6531b405082c4da785f1f5")},
-                {new BQSRTest(hg18Reference, HiSeqBam, HiSeqInterval, " --indels_context_size 4", "b309a5f57b861d7f31cb76cdac4ff8a7")},
-                {new BQSRTest(hg18Reference, HiSeqBam, HiSeqInterval, " --low_quality_tail 5", "4c75d47ed2cf93b499be8fbb29b24dfd")},
-                {new BQSRTest(hg18Reference, HiSeqBam, HiSeqInterval, " --quantizing_levels 6", "43b06e5568a89e4ce1dd9146ce580c89")},
-                {new BQSRTest(hg18Reference, HiSeqBam, HiSeqInterval, " --mismatches_context_size 4", "25f4f48dba27475b0cd7c06ef0239aba")},
-                {new BQSRTest(b36KGReference, validationDataLocation + "NA12892.SLX.SRP000031.2009_06.selected.1Mb.1RG.bam", "1:10,000,000-10,200,000", "", "dfcba9acc32b4a1dfeceea135b48615a")},
-                {new BQSRTest(b36KGReference, validationDataLocation + "NA19240.chr1.BFAST.SOLID.bam", "1:10,000,000-10,200,000", "", "e8077b721f2e6f51c1945b6f6236835c")},
-                {new BQSRTest(b36KGReference, validationDataLocation + "NA12873.454.SRP000031.2009_06.chr1.10_20mb.1RG.bam", "1:10,000,000-10,200,000", "", "fbdc8d0fd312e3a7f49063c580cf5d92")},
-                {new BQSRTest(b36KGReference, validationDataLocation + "originalQuals.1kg.chr1.1-1K.1RG.bam", "1:1-1,000", " -OQ", "4f47415628201a4f3c33e48ec066677b")},
-                {new BQSRTest(b36KGReference, validationDataLocation + "NA19240.chr1.BFAST.SOLID.bam", "1:10,000,000-20,000,000", " --solid_recal_mode REMOVE_REF_BIAS", "1e89d2b88f4218363b9322b38e9536f2")},
-                {new BQSRTest(b36KGReference, privateTestDir + "NA19240.chr1.BFAST.SOLID.hasCSNoCall.bam", "1:50,000-80,000", " --solid_nocall_strategy LEAVE_READ_UNRECALIBRATED", "a7beb0b16756257a274eecf73474ed90")},
-                {new BQSRTest(b36KGReference, validationDataLocation + "NA12892.SLX.SRP000031.2009_06.selected.1Mb.1RG.bam", "1:10,000,000-10,200,000", " -knownSites:anyNameABCD,VCF " + privateTestDir + "vcfexample3.vcf", "dfcba9acc32b4a1dfeceea135b48615a")},
-                {new BQSRTest(b36KGReference, validationDataLocation + "NA12892.SLX.SRP000031.2009_06.selected.1Mb.1RG.bam", "1:10,000,000-10,200,000", " -knownSites:bed " + validationDataLocation + "bqsrKnownTest.bed", "2082c70e08f1c14290c3812021832f83")},
+                {new BQSRTest(hg18Reference, HiSeqBam, HiSeqInterval, "", "387b41dc2221a1a4a782958944662b25")},
+                {new BQSRTest(hg18Reference, HiSeqBam, HiSeqInterval, " --no_standard_covs -cov ContextCovariate", "b5e26902e76abbd59f94f65c70d18165")},
+                {new BQSRTest(hg18Reference, HiSeqBam, HiSeqInterval, " --no_standard_covs -cov CycleCovariate", "a8a9c3f83269911cb61c5fe8fb98dc4a")},
+                {new BQSRTest(hg18Reference, HiSeqBam, HiSeqInterval, " --indels_context_size 4", "f43a0473101c63ae93444c300d843e81")},
+                {new BQSRTest(hg18Reference, HiSeqBam, HiSeqInterval, " --low_quality_tail 5", "9e05e63339d4716584bfc717cab6bd0f")},
+                {new BQSRTest(hg18Reference, HiSeqBam, HiSeqInterval, " --quantizing_levels 6", "1cf9b9c9c64617dc0f3d2f203f918dbe")},
+                {new BQSRTest(hg18Reference, HiSeqBam, HiSeqInterval, " --mismatches_context_size 4", "aa1949a77bc3066fee551a217c970c0d")},
+                {new BQSRTest(b36KGReference, validationDataLocation + "NA12892.SLX.SRP000031.2009_06.selected.1Mb.1RG.bam", "1:10,000,000-10,200,000", "", "f70d8b5358bc2f76696f14b7a807ede0")},
+                {new BQSRTest(b36KGReference, validationDataLocation + "NA19240.chr1.BFAST.SOLID.bam", "1:10,000,000-10,200,000", "", "4c0f63e06830681560a1e9f9aad9fe98")},
+                {new BQSRTest(b36KGReference, validationDataLocation + "NA12873.454.SRP000031.2009_06.chr1.10_20mb.1RG.bam", "1:10,000,000-10,200,000", "", "be2812cd3dae3c326cf35ae3f1c8ad9e")},
+                {new BQSRTest(b36KGReference, validationDataLocation + "originalQuals.1kg.chr1.1-1K.1RG.bam", "1:1-1,000", " -OQ", "03c29a0c1d21f72b12daf51cec111599")},
+                {new BQSRTest(b36KGReference, validationDataLocation + "NA19240.chr1.BFAST.SOLID.bam", "1:10,000,000-20,000,000", " --solid_recal_mode REMOVE_REF_BIAS", "7080b2cad02ec6e67ebc766b2dccebf8")},
+                {new BQSRTest(b36KGReference, privateTestDir + "NA19240.chr1.BFAST.SOLID.hasCSNoCall.bam", "1:50,000-80,000", " --solid_nocall_strategy LEAVE_READ_UNRECALIBRATED", "30e76055c16843b6e33e5b9bd8ced57c")},
+                {new BQSRTest(b36KGReference, validationDataLocation + "NA12892.SLX.SRP000031.2009_06.selected.1Mb.1RG.bam", "1:10,000,000-10,200,000", " -knownSites:anyNameABCD,VCF " + privateTestDir + "vcfexample3.vcf", "f70d8b5358bc2f76696f14b7a807ede0")},
+                {new BQSRTest(b36KGReference, validationDataLocation + "NA12892.SLX.SRP000031.2009_06.selected.1Mb.1RG.bam", "1:10,000,000-10,200,000", " -knownSites:bed " + validationDataLocation + "bqsrKnownTest.bed", "5e657fd6a44dcdc7674b6e5a2de5dc83")},
         };
     }
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/BaseRecalibrator.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/BaseRecalibrator.java
index 245c8e3e8..9506510a9 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/BaseRecalibrator.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/BaseRecalibrator.java
@@ -25,35 +25,41 @@
 
 package org.broadinstitute.sting.gatk.walkers.bqsr;
 
+import net.sf.picard.reference.IndexedFastaSequenceFile;
+import net.sf.samtools.CigarElement;
 import net.sf.samtools.SAMFileHeader;
+import org.broad.tribble.Feature;
+import org.broadinstitute.sting.commandline.Advanced;
+import org.broadinstitute.sting.commandline.Argument;
 import org.broadinstitute.sting.commandline.ArgumentCollection;
 import org.broadinstitute.sting.gatk.CommandLineGATK;
-import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
-import org.broadinstitute.sting.gatk.filters.MappingQualityUnavailableFilter;
-import org.broadinstitute.sting.gatk.filters.MappingQualityZeroFilter;
+import org.broadinstitute.sting.gatk.filters.*;
 import org.broadinstitute.sting.gatk.iterators.ReadTransformer;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.*;
+import org.broadinstitute.sting.utils.BaseUtils;
+import org.broadinstitute.sting.utils.baq.BAQ;
 import org.broadinstitute.sting.utils.classloader.GATKLiteUtils;
+import org.broadinstitute.sting.utils.clipping.ReadClipper;
 import org.broadinstitute.sting.utils.collections.Pair;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.broadinstitute.sting.utils.fasta.CachingIndexedFastaSequenceFile;
 import org.broadinstitute.sting.utils.help.DocumentedGATKFeature;
-import org.broadinstitute.sting.utils.pileup.PileupElement;
-import org.broadinstitute.sting.utils.recalibration.QuantizationInfo;
-import org.broadinstitute.sting.utils.recalibration.RecalUtils;
-import org.broadinstitute.sting.utils.recalibration.RecalibrationReport;
-import org.broadinstitute.sting.utils.recalibration.RecalibrationTables;
+import org.broadinstitute.sting.utils.recalibration.*;
 import org.broadinstitute.sting.utils.recalibration.covariates.Covariate;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.sam.ReadUtils;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.lang.reflect.Constructor;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 
 /**
  * First pass of the base quality score recalibration -- Generates recalibration table based on various user-specified covariates (such as read group, reported quality score, machine cycle, and nucleotide context).
@@ -103,19 +109,20 @@ import java.util.ArrayList;
  * </pre>
  */
 
-@DocumentedGATKFeature( groupName = "BAM Processing and Analysis Tools", extraDocs = {CommandLineGATK.class} )
+@DocumentedGATKFeature(groupName = "BAM Processing and Analysis Tools", extraDocs = {CommandLineGATK.class})
 @BAQMode(ApplicationTime = ReadTransformer.ApplicationTime.FORBIDDEN)
-@By(DataSource.READS)
-@ReadFilters({MappingQualityZeroFilter.class, MappingQualityUnavailableFilter.class}) // only look at covered loci, not every loci of the reference file
-@Requires({DataSource.READS, DataSource.REFERENCE}) // filter out all reads with zero or unavailable mapping quality
-@PartitionBy(PartitionType.LOCUS) // this walker requires both -I input.bam and -R reference.fasta
-public class BaseRecalibrator extends LocusWalker<Long, Long> {
-
+@ReadFilters({MappingQualityZeroFilter.class, MappingQualityUnavailableFilter.class, UnmappedReadFilter.class, NotPrimaryAlignmentFilter.class, DuplicateReadFilter.class, FailsVendorQualityCheckFilter.class})
+@PartitionBy(PartitionType.READ)
+public class BaseRecalibrator extends ReadWalker<Long, Long> implements NanoSchedulable {
     @ArgumentCollection
     private final RecalibrationArgumentCollection RAC = new RecalibrationArgumentCollection(); // all the command line arguments for BQSR and it's covariates
 
+    @Advanced
+    @Argument(fullName = "bqsrBAQGapOpenPenalty", shortName="bqsrBAQGOP", doc="BQSR BAQ gap open penalty (Phred Scaled).  Default value is 40.  30 is perhaps better for whole genome call sets", required = false)
+    public double BAQGOP = BAQ.DEFAULT_GOP;
+
     private QuantizationInfo quantizationInfo; // an object that keeps track of the information necessary for quality score quantization
-    
+
     private RecalibrationTables recalibrationTables;
 
     private Covariate[] requestedCovariates; // list to hold the all the covariate objects that were requested (required + standard + experimental)
@@ -124,12 +131,12 @@ public class BaseRecalibrator extends LocusWalker<Long, Long> {
 
     private int minimumQToUse;
 
-    protected static final String SKIP_RECORD_ATTRIBUTE = "SKIP"; // used to label reads that should be skipped.
-    protected static final String SEEN_ATTRIBUTE = "SEEN"; // used to label reads as processed.
     protected static final String COVARS_ATTRIBUTE = "COVARS"; // used to store covariates array as a temporary attribute inside GATKSAMRecord.\
 
     private static final String NO_DBSNP_EXCEPTION = "This calculation is critically dependent on being able to skip over known variant sites. Please provide a VCF file containing known sites of genetic variation.";
 
+    private BAQ baq; // BAQ the reads on the fly to generate the alignment uncertainty vector
+    private IndexedFastaSequenceFile referenceReader; // fasta reference reader for use with BAQ calculation
 
     /**
      * Parse the -cov arguments and create a list of covariates to be used here
@@ -137,6 +144,8 @@ public class BaseRecalibrator extends LocusWalker<Long, Long> {
      */
     public void initialize() {
 
+        baq = new BAQ(BAQGOP); // setup the BAQ object with the provided gap open penalty
+
         // check for unsupported access
         if (getToolkit().isGATKLite() && !getToolkit().getArguments().disableIndelQuals)
             throw new UserException.NotSupportedInGATKLite("base insertion/deletion recalibration is not supported, please use the --disable_indel_quals argument");
@@ -185,13 +194,21 @@ public class BaseRecalibrator extends LocusWalker<Long, Long> {
         recalibrationEngine.initialize(requestedCovariates, recalibrationTables);
 
         minimumQToUse = getToolkit().getArguments().PRESERVE_QSCORES_LESS_THAN;
+
+        try {
+            // fasta reference reader for use with BAQ calculation
+            referenceReader = new CachingIndexedFastaSequenceFile(getToolkit().getArguments().referenceFile);
+        } catch( FileNotFoundException e ) {
+            throw new UserException.CouldNotReadInputFile(getToolkit().getArguments().referenceFile, e);
+        }
+
     }
 
     private RecalibrationEngine initializeRecalibrationEngine() {
 
         final Class recalibrationEngineClass = GATKLiteUtils.getProtectedClassIfAvailable(RecalibrationEngine.class);
         try {
-            Constructor constructor = recalibrationEngineClass.getDeclaredConstructor((Class[])null);
+            final Constructor constructor = recalibrationEngineClass.getDeclaredConstructor((Class[])null);
             constructor.setAccessible(true);
             return (RecalibrationEngine)constructor.newInstance();
         }
@@ -200,60 +217,207 @@ public class BaseRecalibrator extends LocusWalker<Long, Long> {
         }
     }
 
-    private boolean readHasBeenSkipped( final GATKSAMRecord read ) {
-        return read.containsTemporaryAttribute(SKIP_RECORD_ATTRIBUTE);
-    }
-
-    private boolean isLowQualityBase( final PileupElement p ) {
-        return p.getQual() < minimumQToUse;
-    }
-
-    private boolean readNotSeen( final GATKSAMRecord read ) {
-        return !read.containsTemporaryAttribute(SEEN_ATTRIBUTE);
+    private boolean isLowQualityBase( final GATKSAMRecord read, final int offset ) {
+        return read.getBaseQualities()[offset] < minimumQToUse;
     }
 
     /**
      * For each read at this locus get the various covariate values and increment that location in the map based on
      * whether or not the base matches the reference at this particular location
-     *
-     * @param tracker the reference metadata tracker
-     * @param ref     the reference context
-     * @param context the alignment context
-     * @return returns 1, but this value isn't used in the reduce step
      */
-    public Long map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
-        long countedSites = 0L;
-        // Only analyze sites not present in the provided known sites
-        if (tracker.getValues(RAC.knownSites).size() == 0) {
-            for (final PileupElement p : context.getBasePileup()) {
-                final GATKSAMRecord read = p.getRead();
-                final int offset = p.getOffset();
+    public Long map( final ReferenceContext ref, final GATKSAMRecord originalRead, final RefMetaDataTracker metaDataTracker ) {
 
-                // This read has been marked to be skipped or base is low quality (we don't recalibrate low quality bases)
-                if (readHasBeenSkipped(read) || p.isInsertionAtBeginningOfRead() || isLowQualityBase(p) )
-                    continue;
+        final GATKSAMRecord read = ReadClipper.hardClipAdaptorSequence(originalRead);
+        if( read.isEmpty() ) { return 0L; } // the whole read was inside the adaptor so skip it
 
-                if (readNotSeen(read)) {
-                    read.setTemporaryAttribute(SEEN_ATTRIBUTE, true);
-                    RecalUtils.parsePlatformForRead(read, RAC);
-                    if (!RecalUtils.isColorSpaceConsistent(RAC.SOLID_NOCALL_STRATEGY, read)) {
-                        read.setTemporaryAttribute(SKIP_RECORD_ATTRIBUTE, true);
-                        continue;
+        RecalUtils.parsePlatformForRead(read, RAC);
+        if (!RecalUtils.isColorSpaceConsistent(RAC.SOLID_NOCALL_STRATEGY, read)) { // parse the solid color space and check for color no-calls
+            return 0L; // skip this read completely
+        }
+        read.setTemporaryAttribute(COVARS_ATTRIBUTE, RecalUtils.computeCovariates(read, requestedCovariates));
+
+        final boolean[] skip = calculateSkipArray(read, metaDataTracker); // skip known sites of variation as well as low quality and non-regular bases
+        final int[] isSNP = calculateIsSNP(read, ref, originalRead);
+        final int[] isInsertion = calculateIsIndel(read, EventType.BASE_INSERTION);
+        final int[] isDeletion = calculateIsIndel(read, EventType.BASE_DELETION);
+        final byte[] baqArray = calculateBAQArray(read);
+
+        if( baqArray != null ) { // some reads just can't be BAQ'ed
+            final double[] snpErrors = calculateFractionalErrorArray(isSNP, baqArray);
+            final double[] insertionErrors = calculateFractionalErrorArray(isInsertion, baqArray);
+            final double[] deletionErrors = calculateFractionalErrorArray(isDeletion, baqArray);
+            recalibrationEngine.updateDataForRead(read, skip, snpErrors, insertionErrors, deletionErrors);
+            return 1L;
+        } else {
+            return 0L;
+        }
+    }
+
+    protected boolean[] calculateSkipArray( final GATKSAMRecord read, final RefMetaDataTracker metaDataTracker ) {
+        final byte[] bases = read.getReadBases();
+        final boolean[] skip = new boolean[bases.length];
+        final boolean[] knownSites = calculateKnownSites(read, metaDataTracker.getValues(RAC.knownSites));
+        for( int iii = 0; iii < bases.length; iii++ ) {
+            skip[iii] = !BaseUtils.isRegularBase(bases[iii]) || isLowQualityBase(read, iii) || knownSites[iii] || badSolidOffset(read, iii);
+        }
+        return skip;
+    }
+
+    protected boolean badSolidOffset( final GATKSAMRecord read, final int offset ) {
+        return ReadUtils.isSOLiDRead(read) && RAC.SOLID_RECAL_MODE != RecalUtils.SOLID_RECAL_MODE.DO_NOTHING && !RecalUtils.isColorSpaceConsistent(read, offset);
+    }
+
+    protected boolean[] calculateKnownSites( final GATKSAMRecord read, final List<Feature> features ) {
+        final int BUFFER_SIZE = 0;
+        final int readLength = read.getReadBases().length;
+        final boolean[] knownSites = new boolean[readLength];
+        Arrays.fill(knownSites, false);
+        for( final Feature f : features ) {
+            int featureStartOnRead = ReadUtils.getReadCoordinateForReferenceCoordinate(read.getSoftStart(), read.getCigar(), f.getStart(), ReadUtils.ClippingTail.LEFT_TAIL, true); // BUGBUG: should I use LEFT_TAIL here?
+            if( featureStartOnRead == ReadUtils.CLIPPING_GOAL_NOT_REACHED ) { featureStartOnRead = 0; }
+            int featureEndOnRead = ReadUtils.getReadCoordinateForReferenceCoordinate(read.getSoftStart(), read.getCigar(), f.getEnd(), ReadUtils.ClippingTail.LEFT_TAIL, true);
+            if( featureEndOnRead == ReadUtils.CLIPPING_GOAL_NOT_REACHED ) { featureEndOnRead = readLength; }
+            Arrays.fill(knownSites, Math.max(0, featureStartOnRead - BUFFER_SIZE), Math.min(readLength, featureEndOnRead + 1 + BUFFER_SIZE), true);
+        }
+        return knownSites;
+    }
+
+    // BUGBUG: can be merged with calculateIsIndel
+    protected static int[] calculateIsSNP( final GATKSAMRecord read, final ReferenceContext ref, final GATKSAMRecord originalRead ) {
+        final byte[] readBases = read.getReadBases();
+        final byte[] refBases = Arrays.copyOfRange(ref.getBases(), read.getAlignmentStart() - originalRead.getAlignmentStart(), ref.getBases().length + read.getAlignmentEnd() - originalRead.getAlignmentEnd());
+        final int[] snp = new int[readBases.length];
+        int readPos = 0;
+        int refPos = 0;
+        for ( final CigarElement ce : read.getCigar().getCigarElements() ) {
+            final int elementLength = ce.getLength();
+            switch (ce.getOperator()) {
+                case M:
+                case EQ:
+                case X:
+                    for( int iii = 0; iii < elementLength; iii++ ) {
+                        snp[readPos] = ( BaseUtils.basesAreEqual(readBases[readPos], refBases[refPos]) ? 0 : 1 );
+                        readPos++;
+                        refPos++;
                     }
-                    read.setTemporaryAttribute(COVARS_ATTRIBUTE, RecalUtils.computeCovariates(read, requestedCovariates));
-                }
-
-                // SOLID bams have inserted the reference base into the read if the color space in inconsistent with the read base so skip it
-                if (!ReadUtils.isSOLiDRead(read) ||
-                    RAC.SOLID_RECAL_MODE == RecalUtils.SOLID_RECAL_MODE.DO_NOTHING ||
-                        RecalUtils.isColorSpaceConsistent(read, offset))
-                    // This base finally passed all the checks for a good base, so add it to the big data hashmap
-                    recalibrationEngine.updateDataForPileupElement(p, ref.getBase());
+                    break;
+                case D:
+                case N:
+                    refPos += elementLength;
+                    break;
+                case I:
+                case S: // ReferenceContext doesn't have the soft clipped bases!
+                    readPos += elementLength;
+                    break;
+                case H:
+                case P:
+                    break;
+                default:
+                    throw new ReviewedStingException("Unsupported cigar operator: " + ce.getOperator());
             }
-            countedSites++;
+        }
+        return snp;
+    }
+
+    protected static int[] calculateIsIndel( final GATKSAMRecord read, final EventType mode ) {
+        final byte[] readBases = read.getReadBases();
+        final int[] indel = new int[readBases.length];
+        Arrays.fill(indel, 0);
+        int readPos = 0;
+        for ( final CigarElement ce : read.getCigar().getCigarElements() ) {
+            final int elementLength = ce.getLength();
+            switch (ce.getOperator()) {
+                case M:
+                case EQ:
+                case X:
+                case S:
+                {
+                    readPos += elementLength;
+                    break;
+                }
+                case D:
+                {
+                    final int index = ( read.getReadNegativeStrandFlag() ? readPos : ( readPos > 0 ? readPos - 1 : readPos ) );
+                    indel[index] = ( mode.equals(EventType.BASE_DELETION) ? 1 : 0 );
+                    break;
+                }
+                case I:
+                {
+                    final boolean forwardStrandRead = !read.getReadNegativeStrandFlag();
+                    if( forwardStrandRead ) {
+                        indel[(readPos > 0 ? readPos - 1 : readPos)] = ( mode.equals(EventType.BASE_INSERTION) ? 1 : 0 );
+                    }
+                    for (int iii = 0; iii < elementLength; iii++) {
+                        readPos++;
+                    }
+                    if( !forwardStrandRead ) {
+                        indel[(readPos < indel.length ? readPos : readPos - 1)] = ( mode.equals(EventType.BASE_INSERTION) ? 1 : 0 );
+                    }
+                    break;
+                }
+                case N:
+                case H:
+                case P:
+                    break;
+                default:
+                    throw new ReviewedStingException("Unsupported cigar operator: " + ce.getOperator());
+            }
+        }
+        return indel;
+    }
+
+    protected static double[] calculateFractionalErrorArray( final int[] errorArray, final byte[] baqArray ) {
+        if(errorArray.length != baqArray.length ) {
+            throw new ReviewedStingException("Array length mismatch detected. Malformed read?");
         }
 
-        return countedSites;
+        final byte NO_BAQ_UNCERTAINTY = (byte)'@';
+        final int BLOCK_START_UNSET = -1;
+
+        final double[] fractionalErrors = new double[baqArray.length];
+        Arrays.fill(fractionalErrors, 0.0);
+        boolean inBlock = false;
+        int blockStartIndex = BLOCK_START_UNSET;
+        int iii;
+        for( iii = 0; iii < fractionalErrors.length; iii++ ) {
+            if( baqArray[iii] == NO_BAQ_UNCERTAINTY ) {
+                if( !inBlock ) {
+                    fractionalErrors[iii] = (double) errorArray[iii];
+                } else {
+                    calculateAndStoreErrorsInBlock(iii, blockStartIndex, errorArray, fractionalErrors);
+                    inBlock = false; // reset state variables
+                    blockStartIndex = BLOCK_START_UNSET; // reset state variables
+                }
+            } else {
+                inBlock = true;
+                if( blockStartIndex == BLOCK_START_UNSET ) { blockStartIndex = iii; }
+            }
+        }
+        if( inBlock ) {
+            calculateAndStoreErrorsInBlock(iii-1, blockStartIndex, errorArray, fractionalErrors);
+        }
+        if( fractionalErrors.length != errorArray.length ) {
+            throw new ReviewedStingException("Output array length mismatch detected. Malformed read?");
+        }
+        return fractionalErrors;
+    }
+
+    private static void calculateAndStoreErrorsInBlock( final int iii,
+                                                        final int blockStartIndex,
+                                                        final int[] errorArray,
+                                                        final double[] fractionalErrors ) {
+        int totalErrors = 0;
+        for( int jjj = Math.max(0,blockStartIndex-1); jjj <= iii; jjj++ ) {
+            totalErrors += errorArray[jjj];
+        }
+        for( int jjj = Math.max(0, blockStartIndex-1); jjj <= iii; jjj++ ) {
+            fractionalErrors[jjj] = ((double) totalErrors) / ((double)(iii - Math.max(0,blockStartIndex-1) + 1));
+        }
+    }
+
+    private byte[] calculateBAQArray( final GATKSAMRecord read ) {
+        baq.baqRead(read, referenceReader, BAQ.CalculationMode.RECALCULATE, BAQ.QualityMode.ADD_TAG);
+        return BAQ.getBAQTag(read);
     }
 
     /**
@@ -286,12 +450,12 @@ public class BaseRecalibrator extends LocusWalker<Long, Long> {
         generateReport();
         logger.info("...done!");
 
-        if (RAC.RECAL_PDF_FILE != null) {
+        if ( RAC.RECAL_PDF_FILE != null ) {
             logger.info("Generating recalibration plots...");
             generatePlots();
         }
 
-        logger.info("Processed: " + result + " sites");
+        logger.info("Processed: " + result + " reads");
     }
 
     private void generatePlots() {
@@ -304,7 +468,6 @@ public class BaseRecalibrator extends LocusWalker<Long, Long> {
             RecalUtils.generateRecalibrationPlot(RAC, recalibrationTables, requestedCovariates);
     }
 
-
     /**
      * go through the quality score table and use the # observations and the empirical quality score
      * to build a quality score histogram for quantization. Then use the QuantizeQual algorithm to
@@ -317,5 +480,4 @@ public class BaseRecalibrator extends LocusWalker<Long, Long> {
     private void generateReport() {
         RecalUtils.outputRecalibrationReport(RAC, quantizationInfo, recalibrationTables, requestedCovariates);
     }
-}
-
+}
\ No newline at end of file
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalibrationEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalibrationEngine.java
index ce60f5a3a..962d62d5e 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalibrationEngine.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalibrationEngine.java
@@ -33,7 +33,5 @@ public interface RecalibrationEngine {
 
     public void initialize(final Covariate[] covariates, final RecalibrationTables recalibrationTables);
 
-    public void updateDataForPileupElement(final PileupElement pileupElement, final byte refBase);
-
     public void updateDataForRead(final GATKSAMRecord read, final boolean[] skip, final double[] snpErrors, final double[] insertionErrors, final double[] deletionErrors);
 }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/StandardRecalibrationEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/StandardRecalibrationEngine.java
index 1f7fbbf42..6031aa955 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/StandardRecalibrationEngine.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/StandardRecalibrationEngine.java
@@ -46,41 +46,30 @@ public class StandardRecalibrationEngine implements RecalibrationEngine, PublicP
         this.recalibrationTables = recalibrationTables;
     }
 
-    /**
-     * Loop through the list of requested covariates and pick out the value from the read, offset, and reference
-     * Using the list of covariate values as a key, pick out the RecalDatum and increment,
-     * adding one to the number of observations and potentially one to the number of mismatches for mismatches only.
-     *
-     * @param pileupElement The pileup element to update
-     * @param refBase       The reference base at this locus
-     */
-    @Override
-    public void updateDataForPileupElement(final PileupElement pileupElement, final byte refBase) {
-        final int offset = pileupElement.getOffset();
-        final ReadCovariates readCovariates = covariateKeySetFrom(pileupElement.getRead());
-
-        final byte qual = pileupElement.getQual();
-        final boolean isError = !BaseUtils.basesAreEqual(pileupElement.getBase(), refBase);
-
-        final int[] keys = readCovariates.getKeySet(offset, EventType.BASE_SUBSTITUTION);
-        final int eventIndex = EventType.BASE_SUBSTITUTION.index;
-
-        // TODO: should this really be combine rather than increment?
-        combineDatumOrPutIfNecessary(recalibrationTables.getReadGroupTable(), qual, isError, keys[0], eventIndex);
-
-        incrementDatumOrPutIfNecessary(recalibrationTables.getQualityScoreTable(), qual, isError, keys[0], keys[1], eventIndex);
-
-        for (int i = 2; i < covariates.length; i++) {
-            if (keys[i] < 0)
-                continue;
-
-            incrementDatumOrPutIfNecessary(recalibrationTables.getTable(i), qual, isError, keys[0], keys[1], keys[i], eventIndex);
-        }
-    }
-
     @Override
     public void updateDataForRead( final GATKSAMRecord read, final boolean[] skip, final double[] snpErrors, final double[] insertionErrors, final double[] deletionErrors ) {
-        throw new UnsupportedOperationException("Delocalized BQSR is not available in the GATK-lite version");
+        for( int offset = 0; offset < read.getReadBases().length; offset++ ) {
+            if( !skip[offset] ) {
+                final ReadCovariates readCovariates = covariateKeySetFrom(read);
+
+                final byte qual = read.getBaseQualities()[offset];
+                final double isError = snpErrors[offset];
+
+                final int[] keys = readCovariates.getKeySet(offset, EventType.BASE_SUBSTITUTION);
+                final int eventIndex = EventType.BASE_SUBSTITUTION.index;
+
+                combineDatumOrPutIfNecessary(recalibrationTables.getReadGroupTable(), qual, isError, keys[0], eventIndex);
+
+                incrementDatumOrPutIfNecessary(recalibrationTables.getQualityScoreTable(), qual, isError, keys[0], keys[1], eventIndex);
+
+                for (int i = 2; i < covariates.length; i++) {
+                    if (keys[i] < 0)
+                        continue;
+
+                    incrementDatumOrPutIfNecessary(recalibrationTables.getTable(i), qual, isError, keys[0], keys[1], keys[i], eventIndex);
+                }
+            }
+        }
     }
 
     /**
@@ -90,10 +79,6 @@ public class StandardRecalibrationEngine implements RecalibrationEngine, PublicP
      * @param isError       whether or not the observation is an error
      * @return a new RecalDatum object with the observation and the error
      */
-    protected RecalDatum createDatumObject(final byte reportedQual, final boolean isError) {
-        return new RecalDatum(1, isError ? 1:0, reportedQual);
-    }
-
     protected RecalDatum createDatumObject(final byte reportedQual, final double isError) {
         return new RecalDatum(1, isError, reportedQual);
     }
@@ -108,46 +93,6 @@ public class StandardRecalibrationEngine implements RecalibrationEngine, PublicP
         return (ReadCovariates) read.getTemporaryAttribute(BaseRecalibrator.COVARS_ATTRIBUTE);
     }
 
-    /**
-     * Increments the RecalDatum at the specified position in the specified table, or put a new item there
-     * if there isn't already one.
-     *
-     * Does this in a thread-safe way WITHOUT being synchronized: relies on the behavior of NestedIntegerArray.put()
-     * to return false if another thread inserts a new item at our position in the middle of our put operation.
-     *
-     * @param table the table that holds/will hold our item
-     * @param qual qual for this event
-     * @param isError error status for this event
-     * @param keys location in table of our item
-     */
-    protected void incrementDatumOrPutIfNecessary( final NestedIntegerArray<RecalDatum> table, final byte qual, final boolean isError, final int... keys ) {
-        final RecalDatum existingDatum = table.get(keys);
-
-        // There are three cases here:
-        //
-        // 1. The original get succeeded (existingDatum != null) because there already was an item in this position.
-        //    In this case we can just increment the existing item.
-        //
-        // 2. The original get failed (existingDatum == null), and we successfully put a new item in this position
-        //    and are done.
-        //
-        // 3. The original get failed (existingDatum == null), AND the put fails because another thread comes along
-        //    in the interim and puts an item in this position. In this case we need to do another get after the
-        //    failed put to get the item the other thread put here and increment it.
-        if ( existingDatum == null ) {
-            // No existing item, try to put a new one
-            if ( ! table.put(createDatumObject(qual, isError), keys) ) {
-                // Failed to put a new item because another thread came along and put an item here first.
-                // Get the newly-put item and increment it (item is guaranteed to exist at this point)
-                table.get(keys).increment(isError);
-            }
-        }
-        else {
-            // Easy case: already an item here, so increment it
-            existingDatum.increment(isError);
-        }
-    }
-
     /**
      * Increments the RecalDatum at the specified position in the specified table, or put a new item there
      * if there isn't already one.
@@ -177,36 +122,6 @@ public class StandardRecalibrationEngine implements RecalibrationEngine, PublicP
         }
     }
 
-    /**
-     * Combines the RecalDatum at the specified position in the specified table with a new RecalDatum, or put a
-     * new item there if there isn't already one.
-     *
-     * Does this in a thread-safe way WITHOUT being synchronized: relies on the behavior of NestedIntegerArray.put()
-     * to return false if another thread inserts a new item at our position in the middle of our put operation.
-     *
-     * @param table the table that holds/will hold our item
-     * @param qual qual for this event
-     * @param isError error status for this event
-     * @param keys location in table of our item
-     */
-    protected void combineDatumOrPutIfNecessary( final NestedIntegerArray<RecalDatum> table, final byte qual, final boolean isError, final int... keys ) {
-        final RecalDatum existingDatum = table.get(keys);
-        final RecalDatum newDatum = createDatumObject(qual, isError);
-
-        if ( existingDatum == null ) {
-            // No existing item, try to put a new one
-            if ( ! table.put(newDatum, keys) ) {
-                // Failed to put a new item because another thread came along and put an item here first.
-                // Get the newly-put item and combine it with our item (item is guaranteed to exist at this point)
-                table.get(keys).combine(newDatum);
-            }
-        }
-        else {
-            // Easy case: already an item here, so combine it with our item
-            existingDatum.combine(newDatum);
-        }
-    }
-
     /**
      * Combines the RecalDatum at the specified position in the specified table with a new RecalDatum, or put a
      * new item there if there isn't already one.

From be902375aceb6cd6f70943b8a20609ff2d2c746a Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Mon, 29 Oct 2012 16:29:27 -0400
Subject: [PATCH 43/50] 'Bug' fix: fix the error message from the vcf validator
 so people realize that the file fails strict validation but still adheres to
 the spec.

---
 .../sting/gatk/walkers/variantutils/ValidateVariants.java   | 2 +-
 .../sting/utils/exceptions/UserException.java               | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariants.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariants.java
index c92551a73..2f4d24312 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariants.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariants.java
@@ -172,7 +172,7 @@ public class ValidateVariants extends RodWalker<Integer, Integer> {
                 numErrors++;
                 logger.warn("***** " + e.getMessage() + " *****");
             } else {
-                throw new UserException.MalformedFile(file, e.getMessage());
+                throw new UserException.FailsStrictValidation(file, e.getMessage());
             }
         }
     }
diff --git a/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java b/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java
index c1f408bc7..a49a12292 100755
--- a/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java
+++ b/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java
@@ -283,6 +283,12 @@ public class UserException extends ReviewedStingException {
         }
     }
 
+    public static class FailsStrictValidation extends UserException {
+        public FailsStrictValidation(File f, String message) {
+            super(String.format("File %s fails strict validation: %s", f.getAbsolutePath(), message));
+        }
+    }
+
     public static class MalformedFile extends UserException {
         public MalformedFile(String message) {
             super(String.format("Unknown file is malformed: %s", message));

From 5ee2feb2a3e8d6c5ae85e12af3fe1fd41e1cca52 Mon Sep 17 00:00:00 2001
From: Ryan Poplin <rpoplin@broadinstitute.org>
Date: Mon, 29 Oct 2012 18:53:27 -0400
Subject: [PATCH 44/50] updating pipeline test md5s

---
 .../sting/queue/pipeline/DataProcessingPipelineTest.scala       | 2 +-
 .../sting/queue/pipeline/PacbioProcessingPipelineTest.scala     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/DataProcessingPipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/DataProcessingPipelineTest.scala
index 19f00ac62..944ef7977 100644
--- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/DataProcessingPipelineTest.scala
+++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/DataProcessingPipelineTest.scala
@@ -60,7 +60,7 @@ class DataProcessingPipelineTest {
       " -bwa /home/unix/carneiro/bin/bwa",
       " -bwape ",
       " -p " + projectName).mkString
-    spec.fileMD5s += testOut -> "6e70efbe6bafc3fedd60bd406bd201db"
+    spec.fileMD5s += testOut -> "9fca827ecc8436465b831bb6f879357a"
     PipelineTest.executeTest(spec)
   }
 
diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/PacbioProcessingPipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/PacbioProcessingPipelineTest.scala
index 74e947377..3e9af3e68 100644
--- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/PacbioProcessingPipelineTest.scala
+++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/PacbioProcessingPipelineTest.scala
@@ -40,7 +40,7 @@ class PacbioProcessingPipelineTest {
       " -blasr ",
       " -test ",
       " -D " + BaseTest.publicTestDir + "exampleDBSNP.vcf").mkString
-    spec.fileMD5s += testOut -> "61b06e8b78a93e6644657e6d38851084"
+    spec.fileMD5s += testOut -> "b84f9c45e045685067ded681d5e6224c"
     PipelineTest.executeTest(spec)
   }
 }

From b6a1967f12a92daad681916a238699c0ab4b4b7e Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Mon, 29 Oct 2012 21:47:09 -0400
Subject: [PATCH 45/50] Better documentation for ValidateVariants so that
 people realize it's used for strict validation of the VCF file.  Added an
 option to turn off strict validation and an integration test to cover it.

---
 .../variantutils/ValidateVariants.java        | 15 +++++----
 .../ValidateVariantsIntegrationTest.java      | 31 +++++++++++++------
 2 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariants.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariants.java
index 2f4d24312..3e6ab050a 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariants.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariants.java
@@ -46,16 +46,20 @@ import java.util.Set;
 
 
 /**
- * Strictly validates a variants file.
+ * Validates a VCF file with an extra strict set of criteria.
  *
  * <p>
  * ValidateVariants is a GATK tool that takes a VCF file and validates much of the information inside it.
- * Checks include the correctness of the reference base(s), accuracy of AC & AN values, tests against rsIDs
- * when a dbSNP file is provided, and that all alternate alleles are present in at least one sample.
+ * In addition to standard adherence to the VCF specification, this tool performs extra checks to make ensure
+ * the information contained within the file is correct.  Checks include the correctness of the reference base(s),
+ * accuracy of AC & AN values, tests against rsIDs when a dbSNP file is provided, and that all alternate alleles
+ * are present in at least one sample.
+ *
+ * If you are looking simply to test the adherence to the VCF specification, use --validationType NONE.
  *
  * <h2>Input</h2>
  * <p>
- * A variant set to filter.
+ * A variant set to validate.
  * </p>
  *
  * <h2>Examples</h2>
@@ -79,10 +83,9 @@ public class ValidateVariants extends RodWalker<Integer, Integer> {
     protected DbsnpArgumentCollection dbsnp = new DbsnpArgumentCollection();
 
     public enum ValidationType {
-        ALL, REF, IDS, ALLELES, CHR_COUNTS
+        ALL, REF, IDS, ALLELES, CHR_COUNTS, NONE
     }
 
-    @Hidden
     @Argument(fullName = "validationType", shortName = "type", doc = "which validation type to run", required = false)
     protected ValidationType type = ValidationType.ALL;
 
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariantsIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariantsIntegrationTest.java
index 6a3d755d7..67d47997b 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariantsIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariantsIntegrationTest.java
@@ -33,6 +33,8 @@ import java.util.Arrays;
 
 public class ValidateVariantsIntegrationTest extends WalkerTest {
 
+    protected static final String emptyMd5 = "d41d8cd98f00b204e9800998ecf8427e";
+
     public static String baseTestString(String file, String type) {
         return "-T ValidateVariants -R " + b36KGReference + " -L 1:10001292-10001303 --variant:vcf " + privateTestDir + file + " --validationType " + type;
     }
@@ -42,7 +44,7 @@ public class ValidateVariantsIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
                 baseTestString("validationExampleGood.vcf", "ALL"),
                 0,
-                Arrays.asList("d41d8cd98f00b204e9800998ecf8427e")
+                Arrays.asList(emptyMd5)
         );
 
         executeTest("test good file", spec);
@@ -53,7 +55,7 @@ public class ValidateVariantsIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
                 baseTestString("validationExampleBad.vcf", "REF"),
                 0,
-                UserException.MalformedFile.class
+                UserException.FailsStrictValidation.class
         );
 
         executeTest("test bad ref base #1", spec);
@@ -64,7 +66,7 @@ public class ValidateVariantsIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
                 baseTestString("validationExampleBad2.vcf", "REF"),
                 0,
-                UserException.MalformedFile.class
+                UserException.FailsStrictValidation.class
         );
 
         executeTest("test bad ref base #2", spec);
@@ -75,7 +77,7 @@ public class ValidateVariantsIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
                 baseTestString("validationExampleBad.vcf", "CHR_COUNTS"),
                 0,
-                UserException.MalformedFile.class
+                UserException.FailsStrictValidation.class
         );
 
         executeTest("test bad chr counts #1", spec);
@@ -86,7 +88,7 @@ public class ValidateVariantsIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
                 baseTestString("validationExampleBad2.vcf", "CHR_COUNTS"),
                 0,
-                UserException.MalformedFile.class
+                UserException.FailsStrictValidation.class
         );
 
         executeTest("test bad chr counts #2", spec);
@@ -97,7 +99,7 @@ public class ValidateVariantsIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
                 baseTestString("validationExampleBad.vcf", "IDS") + " --dbsnp " + b36dbSNP129,
                 0,
-                UserException.MalformedFile.class
+                UserException.FailsStrictValidation.class
         );
 
         executeTest("test bad RS ID", spec);
@@ -108,7 +110,7 @@ public class ValidateVariantsIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
             baseTestString("validationExampleBad.vcf", "ALLELES"),
             0,
-            UserException.MalformedFile.class
+            UserException.FailsStrictValidation.class
         );
 
         executeTest("test bad alt allele", spec);
@@ -119,18 +121,29 @@ public class ValidateVariantsIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
             baseTestString("validationExampleBad3.vcf", "REF"),
             0,
-            UserException.MalformedFile.class
+            UserException.FailsStrictValidation.class
         );
 
         executeTest("test bad ref allele in deletion", spec);
     }
 
+    @Test
+    public void testNoValidation() {
+        WalkerTestSpec spec = new WalkerTestSpec(
+                baseTestString("validationExampleBad.vcf", "NONE"),
+                0,
+                Arrays.asList(emptyMd5)
+        );
+
+        executeTest("test no validation", spec);
+    }
+
     @Test
     public void testComplexEvents() {
         WalkerTestSpec spec = new WalkerTestSpec(
                 baseTestString("complexEvents.vcf", "ALL"),
                 0,
-                Arrays.asList("d41d8cd98f00b204e9800998ecf8427e")
+                Arrays.asList(emptyMd5)
         );
 
         executeTest("test validating complex events", spec);

From c95e89392071d72ff5344197c33e7f722efd043e Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Mon, 29 Oct 2012 21:51:35 -0400
Subject: [PATCH 46/50] Better error message for unused ALT alleles

---
 .../sting/utils/variantcontext/VariantContext.java            | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java
index e453e2f8a..b1aa4e759 100755
--- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java
+++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java
@@ -1073,13 +1073,13 @@ public class VariantContext implements Feature { // to enable tribble integratio
         }
 
         if ( reportedAlleles.size() != observedAlleles.size() )
-            throw new TribbleException.InternalCodecException(String.format("the ALT allele(s) for the record at position %s:%d do not match what is observed in the per-sample genotypes", getChr(), getStart()));
+            throw new TribbleException.InternalCodecException(String.format("one or more of the ALT allele(s) for the record at position %s:%d are not observed at all in the sample genotypes", getChr(), getStart()));
 
         int originalSize = reportedAlleles.size();
         // take the intersection and see if things change
         observedAlleles.retainAll(reportedAlleles);
         if ( observedAlleles.size() != originalSize )
-            throw new TribbleException.InternalCodecException(String.format("the ALT allele(s) for the record at position %s:%d do not match what is observed in the per-sample genotypes", getChr(), getStart()));
+            throw new TribbleException.InternalCodecException(String.format("one or more of the ALT allele(s) for the record at position %s:%d are not observed at all in the sample genotypes", getChr(), getStart()));
     }
 
     public void validateChromosomeCounts() {

From 8a402024c237292ffef81093f99ea3d74d3721e3 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Tue, 30 Oct 2012 09:11:56 -0400
Subject: [PATCH 47/50] Updating bundle script to handle new naming convention
 of CEU trio best practices callset

---
 .../sting/queue/qscripts/GATKResourcesBundle.scala            | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala
index 9e8815762..951e380c3 100755
--- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala
+++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala
@@ -141,8 +141,8 @@ class GATKResourcesBundle extends QScript {
     // CEU trio (NA12878,NA12891,NA12892) best practices results (including PBT)
     //
 
-    addResource(new Resource("/humgen/gsa-hpprojects/NA12878Collection/callsets/CEUtrio_BestPractices/current/CEUTrio.HiSeq.WGS.b37.UG.snps_and_indels.recalibrated.filtered.phaseByTransmission.vcf",
-      "CEUTrio.HiSeq.WGS.b37.UG.bestPractices.phaseByTransmission",b37,true,false))
+    addResource(new Resource("/humgen/gsa-hpprojects/NA12878Collection/callsets/CEUtrio_BestPractices/CEUTrio.HiSeq.WGS.b37.snps_and_indels.recalibrated.filtered.phased.CURRENT.vcf",
+      "CEUTrio.HiSeq.WGS.b37.bestPractices.phased",b37,true,false))
 
     //
     // example call set for wiki tutorial

From 2aa28abe0a4948e3308248f5ec57b24d1a6d90b2 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Tue, 30 Oct 2012 14:27:10 -0400
Subject: [PATCH 48/50] Fixing md5s to reflect the new HapMap file

---
 .../varianteval/VariantEvalIntegrationTest.java      |  2 +-
 .../VariantRecalibrationWalkersIntegrationTest.java  | 12 ++++++------
 .../variantutils/SelectVariantsIntegrationTest.java  |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalIntegrationTest.java
index c92d6d4cf..7a3a20bdd 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalIntegrationTest.java
@@ -354,7 +354,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
     @Test
     public void testCompOverlap() {
         String extraArgs = "-T VariantEval -R " + b37KGReference + " -L " + variantEvalTestDataRoot + "pacbio.hg19.intervals --comp:comphapmap " + comparisonDataLocation + "Validated/HapMap/3.3/genotypes_r27_nr.b37_fwd.vcf --eval " + variantEvalTestDataRoot + "pacbio.ts.recalibrated.vcf -noEV -EV CompOverlap -sn NA12878 -noST -ST Novelty -o %s";
-        WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("59ad39e03678011b5f62492fa83ede04"));
+        WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("d0d9208060e69e157dac3bf01bdd83b0"));
         executeTestParallel("testCompOverlap",spec);
     }
 
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrationWalkersIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrationWalkersIntegrationTest.java
index aec087f2c..0c4924229 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrationWalkersIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrationWalkersIntegrationTest.java
@@ -26,9 +26,9 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest {
     }
 
     VRTest lowPass = new VRTest(validationDataLocation + "phase1.projectConsensus.chr20.raw.snps.vcf",
-            "f360ce3eb2b0b887301be917a9843e2b",  // tranches
-            "287fea5ea066bf3fdd71f5ce9b58eab3",  // recal file
-            "afa297c743437551cc2bd36ddd6d6d75"); // cut VCF
+            "4d08c8eee61dd1bdea8c5765f34e41f0",  // tranches
+            "ce396fe4045e020b61471f6737dff36e",  // recal file
+            "4f59bd61be900b25c6ecedaa68b9c8de"); // cut VCF
 
     @DataProvider(name = "VRTest")
     public Object[][] createData1() {
@@ -75,9 +75,9 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest {
     }
 
     VRTest bcfTest = new VRTest(privateTestDir + "vqsr.bcf_test.snps.unfiltered.bcf",
-            "a8ce3cd3dccafdf7d580bcce7d660a9a",  // tranches
-            "74c10fc15f9739a938b7138909fbde04",  // recal file
-            "c30d163871a37f2bbf8ee7f761e870b4"); // cut VCF
+            "6a1eef4d02857dbb117a15420b5c0ce9",  // tranches
+            "238366af66b05b6d21749e799c25353d",  // recal file
+            "3928d6bc5007becf52312ade70f14c42"); // cut VCF
 
     @DataProvider(name = "VRBCFTest")
     public Object[][] createVRBCFTest() {
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariantsIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariantsIntegrationTest.java
index f29ac8cad..a1d673b56 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariantsIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariantsIntegrationTest.java
@@ -20,7 +20,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
                         + b37hapmapGenotypes + " -disc " + testFile
                         + " -o %s --no_cmdline_in_header -U LENIENT_VCF_PROCESSING",
                 1,
-                Arrays.asList("d88bdae45ae0e74e8d8fd196627e612c")
+                Arrays.asList("954415f84996d27b07d00855e96d33a2")
         );
         spec.disableShadowBCF();
 
@@ -49,7 +49,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
                         + b37hapmapGenotypes + " -disc " + testFile
                         + " -o %s --no_cmdline_in_header -U LENIENT_VCF_PROCESSING",
                 1,
-                Arrays.asList("c0b937edb6a8b6392d477511d4f1ebcf")
+                Arrays.asList("ca1b5226eaeaffb78d4abd9d2ee10c43")
         );
         spec.disableShadowBCF();
 

From e1e480a0b96d1a8715bbe339923a9de7cb832d12 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Tue, 30 Oct 2012 14:54:29 -0400
Subject: [PATCH 49/50] Bug fix: don't add no-call alleles to the list of ALT
 alleles being validated.

---
 .../sting/utils/variantcontext/VariantContext.java              | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java
index b1aa4e759..27a5b0c24 100755
--- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java
+++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java
@@ -1071,6 +1071,8 @@ public class VariantContext implements Feature { // to enable tribble integratio
             if ( g.isCalled() )
                 observedAlleles.addAll(g.getAlleles());
         }
+        if ( observedAlleles.contains(Allele.NO_CALL) )
+            observedAlleles.remove(Allele.NO_CALL);
 
         if ( reportedAlleles.size() != observedAlleles.size() )
             throw new TribbleException.InternalCodecException(String.format("one or more of the ALT allele(s) for the record at position %s:%d are not observed at all in the sample genotypes", getChr(), getStart()));

From eccb76c304c0699c352d69fa27ec8e586151d989 Mon Sep 17 00:00:00 2001
From: Eric Banks <ebanks@broadinstitute.org>
Date: Tue, 30 Oct 2012 15:09:46 -0400
Subject: [PATCH 50/50] Only run UG in the bundle for chr20

---
 .../sting/queue/qscripts/GATKResourcesBundle.scala               | 1 +
 1 file changed, 1 insertion(+)

diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala
index 951e380c3..24ab50451 100755
--- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala
+++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala
@@ -317,6 +317,7 @@ class GATKResourcesBundle extends QScript {
   class UG(@Input bam: File, @Input ref: File, @Input outVCF: File) extends UnifiedGenotyper with UNIVERSAL_GATK_ARGS {
     this.input_file = List(bam)
     this.reference_sequence = ref
+    this.intervalsString ++= List("20");
     this.out = outVCF
   }