Merge branch 'master' of ssh://gsa1/humgen/gsa-scr1/gsa-engineering/git/unstable

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthOfCoverage.java

@@ -41,7 +41,7 @@ public class DepthOfCoverage extends InfoFieldAnnotation implements StandardAnno
 
         int depth = 0;
         for ( Map.Entry<String, AlignmentContext> sample : stratifiedContexts.entrySet() )
-            depth += sample.getValue().getBasePileup().depthOfCoverage();
+            depth += sample.getValue().hasBasePileup() ? sample.getValue().getBasePileup().depthOfCoverage() : sample.getValue().getExtendedEventPileup().depthOfCoverage();
         Map<String, Object> map = new HashMap<String, Object>();
         map.put(getKeyNames().get(0), String.format("%d", depth));
         return map;

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java

@@ -205,7 +205,7 @@ public class FisherStrand extends InfoFieldAnnotation implements StandardAnnotat
 
         for ( Map.Entry<String, AlignmentContext> sample : stratifiedContexts.entrySet() ) {
             for (PileupElement p : sample.getValue().getBasePileup()) {
-                if ( p.isDeletion() ) // ignore deletions
+                if ( p.isDeletion() || p.isReducedRead() ) // ignore deletions and reduced reads
                     continue;
 
                 if ( p.getRead().getMappingQuality() < 20 || p.getQual() < 20 )
@@ -258,6 +258,8 @@ public class FisherStrand extends InfoFieldAnnotation implements StandardAnnotat
                  continue;
 
             for (final PileupElement p: pileup) {
+                if ( p.isReducedRead() ) // ignore reduced reads
+                    continue;
                 if ( p.getRead().getMappingQuality() < 20)
                     continue;
                 if (indelLikelihoodMap.containsKey(p)) {

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/QualByDepth.java

@@ -43,7 +43,7 @@ public class QualByDepth extends InfoFieldAnnotation implements StandardAnnotati
             if ( context == null )
                 continue;
 
-            depth += context.getBasePileup().depthOfCoverage();
+            depth += context.hasBasePileup() ? context.getBasePileup().depthOfCoverage() : context.getExtendedEventPileup().depthOfCoverage();
         }
 
         if ( depth == 0 )

public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java

@@ -31,8 +31,6 @@ import org.broadinstitute.sting.commandline.Input;
 import org.broadinstitute.sting.commandline.RodBinding;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
-import java.io.File;
-
 
 public class UnifiedArgumentCollection {
 
@@ -103,18 +101,13 @@ public class UnifiedArgumentCollection {
     @Argument(fullName = "assume_single_sample_reads", shortName = "single_sample", doc = "The single sample that we should assume is represented in the input bam (and therefore associate with all reads regardless of whether they have read groups)", required = false)
     public String ASSUME_SINGLE_SAMPLE = null;
 
-    // TODO -- delete me
+    /**
-    @Hidden
+     * The minimum confidence needed in a given base for it to be used in variant calling.  Note that the base quality of a base
-    @Argument(fullName = "abort_at_too_much_coverage", doc = "Don't call a site if the downsampled coverage is greater than this value", required = false)
+     * is capped by the mapping quality so that bases on reads with low mapping quality may get filtered out depending on this value.
-    public int COVERAGE_AT_WHICH_TO_ABORT = -1;
+     */
-
-    // control the various parameters to be used
     @Argument(fullName = "min_base_quality_score", shortName = "mbq", doc = "Minimum base quality required to consider a base for calling", required = false)
     public int MIN_BASE_QUALTY_SCORE = 17;
 
-    @Argument(fullName = "min_mapping_quality_score", shortName = "mmq", doc = "Minimum read mapping quality required to consider a read for calling", required = false)
-    public int MIN_MAPPING_QUALTY_SCORE = 20;
-
     @Argument(fullName = "max_deletion_fraction", shortName = "deletions", doc = "Maximum fraction of reads with deletions spanning this locus for it to be callable [to disable, set to < 0 or > 1; default:0.05]", required = false)
     public Double MAX_DELETION_FRACTION = 0.05;
 
@@ -165,14 +158,6 @@ public class UnifiedArgumentCollection {
     @Argument(fullName = "ignoreSNPAlleles", shortName = "ignoreSNPAlleles", doc = "expt", required = false)
     public boolean IGNORE_SNP_ALLELES = false;
 
-    @Deprecated
-    @Argument(fullName="output_all_callable_bases", shortName="all_bases", doc="Please use --output_mode EMIT_ALL_SITES instead" ,required=false)
-    private Boolean ALL_BASES_DEPRECATED = false;   
-
-    @Deprecated
-    @Argument(fullName="genotype", shortName="genotype", doc="Please use --output_mode EMIT_ALL_CONFIDENT_SITES instead" ,required=false)
-    private Boolean GENOTYPE_DEPRECATED = false;
-
 
     // Developers must remember to add any newly added arguments to the list here as well otherwise they won't get changed from their default value!
     public UnifiedArgumentCollection clone() {
@@ -189,7 +174,6 @@ public class UnifiedArgumentCollection {
         uac.STANDARD_CONFIDENCE_FOR_CALLING = STANDARD_CONFIDENCE_FOR_CALLING;
         uac.STANDARD_CONFIDENCE_FOR_EMITTING = STANDARD_CONFIDENCE_FOR_EMITTING;
         uac.MIN_BASE_QUALTY_SCORE = MIN_BASE_QUALTY_SCORE;
-        uac.MIN_MAPPING_QUALTY_SCORE = MIN_MAPPING_QUALTY_SCORE;
         uac.MAX_DELETION_FRACTION = MAX_DELETION_FRACTION;
         uac.MIN_INDEL_COUNT_FOR_GENOTYPING = MIN_INDEL_COUNT_FOR_GENOTYPING;
         uac.INDEL_HETEROZYGOSITY = INDEL_HETEROZYGOSITY;
@@ -200,7 +184,6 @@ public class UnifiedArgumentCollection {
         uac.alleles = alleles;
 
         // todo- arguments to remove
-        uac.COVERAGE_AT_WHICH_TO_ABORT = COVERAGE_AT_WHICH_TO_ABORT;
         uac.IGNORE_SNP_ALLELES = IGNORE_SNP_ALLELES;
         uac.BANDED_INDEL_COMPUTATION = BANDED_INDEL_COMPUTATION;
         return uac;

public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java

@@ -120,7 +120,6 @@ public class UnifiedGenotyperEngine {
         this.samples = new TreeSet<String>(samples);
         // note that, because we cap the base quality by the mapping quality, minMQ cannot be less than minBQ
         this.UAC = UAC.clone();
-        this.UAC.MIN_MAPPING_QUALTY_SCORE = Math.max(UAC.MIN_MAPPING_QUALTY_SCORE, UAC.MIN_BASE_QUALTY_SCORE);
 
         this.logger = logger;
         this.verboseWriter = verboseWriter;
@@ -146,9 +145,6 @@ public class UnifiedGenotyperEngine {
      * @return the VariantCallContext object
      */
     public VariantCallContext calculateLikelihoodsAndGenotypes(RefMetaDataTracker tracker, ReferenceContext refContext, AlignmentContext rawContext) {
-        if ( UAC.COVERAGE_AT_WHICH_TO_ABORT > 0 && rawContext.size() > UAC.COVERAGE_AT_WHICH_TO_ABORT )
-            return null;
-
         final GenotypeLikelihoodsCalculationModel.Model model = getCurrentGLModel(tracker, refContext, rawContext );
         if( model == null ) {
             return (UAC.OutputMode == OUTPUT_MODE.EMIT_ALL_SITES && UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES ? generateEmptyContext(tracker, refContext, null, rawContext) : null);
@@ -566,7 +562,7 @@ public class UnifiedGenotyperEngine {
 
             if (UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES) {
                 // regular pileup in this case
-                ReadBackedPileup pileup = rawContext.getBasePileup() .getMappingFilteredPileup(UAC.MIN_MAPPING_QUALTY_SCORE);
+                ReadBackedPileup pileup = rawContext.getBasePileup() .getMappingFilteredPileup(UAC.MIN_BASE_QUALTY_SCORE);
 
                 // don't call when there is no coverage
                 if ( pileup.getNumberOfElements() == 0 && UAC.OutputMode != OUTPUT_MODE.EMIT_ALL_SITES  )
@@ -583,7 +579,7 @@ public class UnifiedGenotyperEngine {
                 ReadBackedExtendedEventPileup rawPileup = rawContext.getExtendedEventPileup();
 
                 // filter the context based on min mapping quality
-                ReadBackedExtendedEventPileup pileup = rawPileup.getMappingFilteredPileup(UAC.MIN_MAPPING_QUALTY_SCORE);
+                ReadBackedExtendedEventPileup pileup = rawPileup.getMappingFilteredPileup(UAC.MIN_BASE_QUALTY_SCORE);
 
                 // don't call when there is no coverage
                 if ( pileup.getNumberOfElements() == 0 && UAC.OutputMode != OUTPUT_MODE.EMIT_ALL_SITES  )

public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java

@@ -724,38 +724,6 @@ public class ReadUtils {
         return true;
     }
 
-    /**
-     * Looks for a read coordinate that corresponds to the reference coordinate in the soft clipped region before
-     * the alignment start of the read.
-     *
-     * @param read
-     * @param refCoord
-     * @return the corresponding read coordinate or -1 if it failed to find it (it has been hard clipped before)
-     */
-    @Requires({"refCoord >= read.getUnclippedStart()", "refCoord < read.getAlignmentStart()"})
-    private static int getReadCoordinateForReferenceCoordinateBeforeAlignmentStart(SAMRecord read, int refCoord) {
-        if (getRefCoordSoftUnclippedStart(read) <= refCoord)
-            return refCoord - getRefCoordSoftUnclippedStart(read) + 1;
-        return -1;
-    }
-
-
-    /**
-     * Looks for a read coordinate that corresponds to the reference coordinate in the soft clipped region after
-     * the alignment end of the read.
-     *
-     * @param read
-     * @param refCoord
-     * @return the corresponding read coordinate or -1 if it failed to find it (it has been hard clipped before)
-     */
-    @Requires({"refCoord <= read.getUnclippedEnd()", "refCoord > read.getAlignmentEnd()"})
-    private static int getReadCoordinateForReferenceCoordinateBeforeAlignmentEnd(SAMRecord read, int refCoord) {
-        if (getRefCoordSoftUnclippedEnd(read) >= refCoord)
-            return refCoord - getRefCoordSoftUnclippedStart(read) + 1;
-        return -1;
-    }
-
-
     public enum ClippingTail {
         LEFT_TAIL,
         RIGHT_TAIL
@@ -800,25 +768,14 @@ public class ReadUtils {
      * @param refCoord
      * @return the read coordinate corresponding to the requested reference coordinate. (see warning!)
      */
-    @Requires({"refCoord >= read.getUnclippedStart()", "refCoord <= read.getUnclippedEnd()"})
+    @Requires({"refCoord >= getRefCoordSoftUnclippedStart(read)", "refCoord <= getRefCoordSoftUnclippedEnd(read)"})
     @Ensures({"result.getFirst() >= 0", "result.getFirst() < read.getReadLength()"})
     public static Pair<Integer, Boolean> getReadCoordinateForReferenceCoordinate(SAMRecord read, int refCoord) {
         int readBases = 0;
         int refBases = 0;
         boolean fallsInsideDeletion = false;
 
-        if (refCoord < read.getAlignmentStart()) {
+        int goal = refCoord - getRefCoordSoftUnclippedStart(read);  // The goal is to move this many reference bases
-            readBases = getReadCoordinateForReferenceCoordinateBeforeAlignmentStart(read, refCoord);
-            if (readBases < 0)
-                throw new ReviewedStingException("Requested a coordinate in a hard clipped area of the read. No equivalent read coordinate.");
-        }
-        else if (refCoord > read.getAlignmentEnd()) {
-            readBases = getReadCoordinateForReferenceCoordinateBeforeAlignmentEnd(read, refCoord);
-            if (readBases < 0)
-                throw new ReviewedStingException("Requested a coordinate in a hard clipped area of the read. No equivalent read coordinate.");
-        }
-        else {
-            int goal = refCoord - read.getAlignmentStart();  // The goal is to move this many reference bases
         boolean goalReached = refBases == goal;
 
         Iterator<CigarElement> cigarElementIterator = read.getCigar().getCigarElements().iterator();
@@ -826,7 +783,7 @@ public class ReadUtils {
             CigarElement cigarElement = cigarElementIterator.next();
             int shift = 0;
 
-                if (cigarElement.getOperator().consumesReferenceBases()) {
+            if (cigarElement.getOperator().consumesReferenceBases() || cigarElement.getOperator() == CigarOperator.SOFT_CLIP) {
                 if (refBases + cigarElement.getLength() < goal)
                     shift = cigarElement.getLength();
                 else
@@ -889,7 +846,7 @@ public class ReadUtils {
 
             if (!goalReached)
                 throw new ReviewedStingException("Somehow the requested coordinate is not covered by the read. Too many deletions?");
-        }
+        
 
         return new Pair<Integer, Boolean>(readBases, fallsInsideDeletion);
     }

public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java

@@ -18,8 +18,8 @@ import java.util.Map;
 public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
     private final static String baseCommand = "-T UnifiedGenotyper -R " + b36KGReference + " -NO_HEADER -glm BOTH --dbsnp " + b36dbSNP129;
-    private final static String baseCommandIndels = "-T UnifiedGenotyper -R " + b36KGReference + " -NO_HEADER -glm INDEL --dbsnp " + b36dbSNP129;
+    private final static String baseCommandIndels = "-T UnifiedGenotyper -R " + b36KGReference + " -NO_HEADER -glm INDEL -mbq 20 --dbsnp " + b36dbSNP129;
-    private final static String baseCommandIndelsb37 = "-T UnifiedGenotyper -R " + b37KGReference + " -NO_HEADER -glm INDEL --dbsnp " + b37dbSNP132;
+    private final static String baseCommandIndelsb37 = "-T UnifiedGenotyper -R " + b37KGReference + " -NO_HEADER -glm INDEL -mbq 20 --dbsnp " + b37dbSNP132;
 
     // --------------------------------------------------------------------------------------------------------------
     //
@@ -51,7 +51,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testSingleSamplePilot2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
-                Arrays.asList("d1cbd1fb9f3f7323941a95bc2def7e5a"));
+                Arrays.asList("6762b72ae60155ad71738d7c76b80e4b"));
         executeTest("test SingleSample Pilot2", spec);
     }
 
@@ -61,7 +61,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    private final static String COMPRESSED_OUTPUT_MD5 = "2732b169cdccb21eb3ea00429619de79";
+    private final static String COMPRESSED_OUTPUT_MD5 = "bc71dba7bbdb23e7d5cc60461fdd897b";
 
     @Test
     public void testCompressedOutput() {
@@ -82,7 +82,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
         // Note that we need to turn off any randomization for this to work, so no downsampling and no annotations
 
-        String md5 = "cbac3960bbcb9d6192c57549208c182c";
+        String md5 = "b9504e446b9313559c3ed97add7e8dc1";
 
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -dt NONE -G none -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,075,000", 1,
@@ -113,9 +113,8 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testCallingParameters() {
         HashMap<String, String> e = new HashMap<String, String>();
-        e.put( "--min_base_quality_score 26", "531966aee1cd5dced61c96c4fedb59a9" );
+        e.put( "--min_base_quality_score 26", "bb3f294eab3e2cf52c70e63b23aac5ee" );
-        e.put( "--min_mapping_quality_score 26", "c71ca370947739cb7d87b59452be7a07" );
+        e.put( "--computeSLOD", "eb34979efaadba1e34bd82bcacf5c722" );
-        e.put( "--computeSLOD", "1a5648f26c18ced27df4be031b44e72d" );
 
         for ( Map.Entry<String, String> entry : e.entrySet() ) {
             WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
@@ -161,8 +160,8 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testHeterozyosity() {
         HashMap<Double, String> e = new HashMap<Double, String>();
-        e.put( 0.01, "aed69402ddffe7f2ed5ca98563bfba02" );
+        e.put( 0.01, "f84da90c310367bd51f2ab6e346fa3d8" );
-        e.put( 1.0 / 1850, "fa94a059f08c1821b721335d93ed2ea5" );
+        e.put( 1.0 / 1850, "5791e7fef40d4412b6d8f84e0a809c6c" );
 
         for ( Map.Entry<Double, String> entry : e.entrySet() ) {
             WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
@@ -186,7 +185,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                         " -o %s" +
                         " -L 1:10,000,000-10,100,000",
                 1,
-                Arrays.asList("1c080e6596d4c830bb5d147b04e2a82c"));
+                Arrays.asList("9cc9538ac83770e12bd0830d285bfbd0"));
 
         executeTest(String.format("test multiple technologies"), spec);
     }
@@ -205,7 +204,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                         " -L 1:10,000,000-10,100,000" +
                         " -baq CALCULATE_AS_NECESSARY",
                 1,
-                Arrays.asList("9129ad748ca3be2d3b321d2d7e83ae5b"));
+                Arrays.asList("eaf8043edb46dfbe9f97ae03baa797ed"));
 
         executeTest(String.format("test calling with BAQ"), spec);
     }
@@ -241,7 +240,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                 1,
                 Arrays.asList("5fe98ee853586dc9db58f0bc97daea63"));
 
-        executeTest(String.format("test indel caller in SLX witn low min allele count"), spec);
+        executeTest(String.format("test indel caller in SLX with low min allele count"), spec);
     }
 
     @Test