Final fixes before BCF2 mark III push

-- Added MLEAC and MLEAF format lines to PoolCallerWalker
-- VariantFiltrationWalker now throws an error when JEXL variables cannot be found (XXX < 0.5) but passes through (albeit with a disgusting warning) when a variable is found but its value is a bad type (AF < 0.5) where AF == [0.04,0.00] at multi-allelic variation
-- Allow values to pass assertEquals in VariantContextTestProvider when one file contains X=[null, null] and the other has X missing

public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java

@@ -52,8 +52,8 @@ public class VariantContextUtils {
     private final static boolean ASSUME_MISSING_FIELDS_ARE_STRINGS = false;
 
     static {
-        engine.setSilent(true); // will throw errors now for selects that don't evaluate properly
-        engine.setLenient(true);
+        engine.setSilent(false); // will throw errors now for selects that don't evaluate properly
+        engine.setLenient(false);
         engine.setDebug(false);
     }
 

public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java

@@ -16,7 +16,7 @@ public class VariantAnnotatorIntegrationTest extends WalkerTest {
     public void testHasAnnotsNotAsking1() {
         WalkerTestSpec spec = new WalkerTestSpec(
                 baseTestString() + " --variant " + privateTestDir + "vcfexample2.vcf -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -L 1:10,020,000-10,021,000", 1,
-                Arrays.asList("bd6848e7dbf2f809ee2f690ee2cf8ef4"));
+                Arrays.asList("55785745fe13ad81a2c4a14373d091f0"));
         executeTest("test file has annotations, not asking for annotations, #1", spec);
     }
 

public/java/test/org/broadinstitute/sting/gatk/walkers/filters/VariantFiltrationIntegrationTest.java

@@ -16,7 +16,7 @@ public class VariantFiltrationIntegrationTest extends WalkerTest {
     public void testNoAction() {
         WalkerTestSpec spec = new WalkerTestSpec(
                 baseTestString() + " --variant " + privateTestDir + "vcfexample2.vcf -L 1:10,020,000-10,021,000", 1,
-                Arrays.asList("49471b44ac165929d3ff81f98ce19063"));
+                Arrays.asList("fbf88e25df30181ca5422a374c7b36fa"));
         executeTest("test no action", spec);
     }
 
@@ -24,7 +24,7 @@ public class VariantFiltrationIntegrationTest extends WalkerTest {
     public void testClusteredSnps() {
         WalkerTestSpec spec = new WalkerTestSpec(
                 baseTestString() + " -window 10 --variant " + privateTestDir + "vcfexample2.vcf -L 1:10,020,000-10,021,000", 1,
-                Arrays.asList("8b45895d7ae1f36b70e7fd26aa9451d3"));
+                Arrays.asList("bb69f49e9ef0054f0ccd6d38f5ffa46a"));
         executeTest("test clustered SNPs", spec);
     }
 
@@ -32,7 +32,7 @@ public class VariantFiltrationIntegrationTest extends WalkerTest {
     public void testMask1() {
         WalkerTestSpec spec1 = new WalkerTestSpec(
                 baseTestString() + " -maskName foo --mask " + privateTestDir + "vcfexample2.vcf --variant " + privateTestDir + "vcfexample2.vcf -L 1:10,020,000-10,021,000", 1,
-                Arrays.asList("06307029f5da87ae4edd9804063a98f9"));
+                Arrays.asList("7e3225a32fcd6066901247992b2c5ca8"));
         executeTest("test mask all", spec1);
     }
 
@@ -40,7 +40,7 @@ public class VariantFiltrationIntegrationTest extends WalkerTest {
     public void testMask2() {
         WalkerTestSpec spec2 = new WalkerTestSpec(
                 baseTestString() + " -maskName foo --mask:VCF " + privateTestDir + "vcfMask.vcf --variant " + privateTestDir + "vcfexample2.vcf -L 1:10,020,000-10,021,000", 1,
-                Arrays.asList("1fd06f6b2642685093ed36342f002b58"));
+                Arrays.asList("3485fe95e3f0864c3575baf05cef4bcc"));
         executeTest("test mask some", spec2);
     }
 
@@ -48,7 +48,7 @@ public class VariantFiltrationIntegrationTest extends WalkerTest {
     public void testMask3() {
         WalkerTestSpec spec3 = new WalkerTestSpec(
                 baseTestString() + " -maskName foo -maskExtend 10 --mask:VCF " + privateTestDir + "vcfMask.vcf --variant " + privateTestDir + "vcfexample2.vcf -L 1:10,020,000-10,021,000", 1,
-                Arrays.asList("d8c5206d5d13477a5929fb1ae5a6bfc4"));
+                Arrays.asList("367ab9c028a68e4eda2055e3bb8b486c"));
         executeTest("test mask extend", spec3);
     }
 
@@ -56,7 +56,7 @@ public class VariantFiltrationIntegrationTest extends WalkerTest {
     public void testFilter1() {
         WalkerTestSpec spec = new WalkerTestSpec(
                 baseTestString() + " -filter 'DoC < 20 || FisherStrand > 20.0' -filterName foo --variant " + privateTestDir + "vcfexample2.vcf -L 1:10,020,000-10,021,000", 1,
-                Arrays.asList("a3be095e8aa75d9ef4235b9487527307"));
+                Arrays.asList("5a10d969e50a58d8dfbf1da54bf293df"));
         executeTest("test filter #1", spec);
     }
 
@@ -64,7 +64,7 @@ public class VariantFiltrationIntegrationTest extends WalkerTest {
     public void testFilter2() {
         WalkerTestSpec spec = new WalkerTestSpec(
                 baseTestString() + " -filter 'AlleleBalance < 70.0 && FisherStrand == 1.4' -filterName bar --variant " + privateTestDir + "vcfexample2.vcf -L 1:10,020,000-10,021,000", 1,
-                Arrays.asList("bd1361ddc52d73b8cd7adeb9e5c47200"));
+                Arrays.asList("886dbbca2350083819ff67224f6efbd6"));
         executeTest("test filter #2", spec);
     }
 
@@ -72,7 +72,7 @@ public class VariantFiltrationIntegrationTest extends WalkerTest {
     public void testFilterWithSeparateNames() {
         WalkerTestSpec spec = new WalkerTestSpec(
                 baseTestString() + " --filterName ABF -filter 'AlleleBalance < 0.7' --filterName FSF -filter 'FisherStrand == 1.4' --variant " + privateTestDir + "vcfexample2.vcf -L 1:10,020,000-10,021,000", 1,
-                Arrays.asList("4a43ec0285433df426ab482f88cf7ca6"));
+                Arrays.asList("ee78c2e7128a8f9549233493c7cf6949"));
         executeTest("test filter with separate names #2", spec);
     }
 
@@ -80,15 +80,15 @@ public class VariantFiltrationIntegrationTest extends WalkerTest {
     public void testGenotypeFilters1() {
         WalkerTestSpec spec1 = new WalkerTestSpec(
                 baseTestString() + " -G_filter 'GQ == 0.60' -G_filterName foo --variant " + privateTestDir + "vcfexample2.vcf -L 1:10,020,000-10,021,000", 1,
-                Arrays.asList("5ee4485a022e163645c08b9691384f67"));
+                Arrays.asList("285dd348c47c8c1e85d2886f9b33559e"));
         executeTest("test genotype filter #1", spec1);
     }
 
     @Test
     public void testGenotypeFilters2() {
         WalkerTestSpec spec2 = new WalkerTestSpec(
-                baseTestString() + " -G_filter 'AF == 0.04 && isHomVar == 1' -G_filterName foo --variant " + privateTestDir + "vcfexample2.vcf -L 1:10,020,000-10,021,000", 1,
-                Arrays.asList("d0a068c8cfb0758d2a8d471383f39b68"));
+                baseTestString() + " -G_filter 'isHomVar == 1' -G_filterName foo --variant " + privateTestDir + "vcfexample2.vcf -L 1:10,020,000-10,021,000", 1,
+                Arrays.asList("a9c835a13eb72aa22d5e271894d8ac33"));
         executeTest("test genotype filter #2", spec2);
     }
 

public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java

@@ -341,7 +341,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + result.get(0).getAbsolutePath() + " -I " + validationDataLocation +
                         "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10450700-10551000", 1,
-                Arrays.asList("5c7db047ae9417d37c6bbda1d8ea6019"));
+                Arrays.asList("c48c8a1a8ec88c6f3c99187e08496ae0"));
         executeTest("test MultiSample Pilot1 CEU indels using GENOTYPE_GIVEN_ALLELES", spec2);
     }
 

public/java/test/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrationWalkersIntegrationTest.java

@@ -26,7 +26,7 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest {
 
     VRTest lowPass = new VRTest("phase1.projectConsensus.chr20.raw.snps.vcf",
             "0ddd1e0e483d2eaf56004615cea23ec7",  // tranches
-            "6e1f98bb819ccf03e17a2288742160d3",  // recal file
+            "b9709e4180e56abc691b208bd3e8626c",  // recal file
             "c58ff4140e8914f0b656ed625c7f73b9"); // cut VCF
 
     @DataProvider(name = "VRTest")
@@ -74,7 +74,7 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest {
 
     VRTest indel = new VRTest("combined.phase1.chr20.raw.indels.sites.vcf",
             "da4458d05f6396f5c4ab96f274e5ccdc",  // tranches
-            "8e2417336fa62e6c4d9f61b6deebdd82",  // recal file
+            "a04a9001f62eff43d363f4d63769f3ee",  // recal file
             "05e88052e0798f1c1e83f0a8938bce56"); // cut VCF
 
     @DataProvider(name = "VRIndelTest")
@@ -132,7 +132,7 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest {
                         " -o %s" +
                         " -tranchesFile " + privateTestDir + "VQSR.mixedTest.tranches" +
                         " -recalFile " + privateTestDir + "VQSR.mixedTest.recal",
-                Arrays.asList("beadf841bbf39c3f0d0bc7fb55462b37"));
+                Arrays.asList("d670c684f73e2744b6c01738a01d5ec4"));
         executeTest("testApplyRecalibrationSnpAndIndelTogether", spec);
     }
 }

public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/CombineVariantsIntegrationTest.java

@@ -125,7 +125,7 @@ public class CombineVariantsIntegrationTest extends WalkerTest {
                         " -priority NA19240_BGI,NA19240_ILLUMINA,NA19240_WUGSC,denovoInfo" +
                         " -genotypeMergeOptions UNIQUIFY -L 1"),
                 1,
-                Arrays.asList("c0d4d601aa5d2b29927c535868448d2a"));
+                Arrays.asList("3039cfff7abee6aa7fbbafec66a1b019"));
         cvExecuteTest("threeWayWithRefs", spec);
     }
 

public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/LiftoverVariantsIntegrationTest.java

@@ -40,7 +40,7 @@ public class LiftoverVariantsIntegrationTest extends WalkerTest {
          WalkerTestSpec spec = new WalkerTestSpec(
                  "-T LiftoverVariants -o %s -R " + b36KGReference + " --variant " + privateTestDir + "yri.trio.gatk_glftrio.intersection.annotated.filtered.chr1.500.noheader.vcf -chain " + validationDataLocation + "b36ToHg19.broad.over.chain -dict /seq/references/Homo_sapiens_assembly19/v0/Homo_sapiens_assembly19.dict",
                  1,
-                 Arrays.asList("b8f4171b0c39954b283dfed4afed87d7"));
+                 Arrays.asList("a139480c004859452d4095fe4859b42e"));
          executeTest("test b36 to hg19", spec);
     }
 
@@ -49,7 +49,7 @@ public class LiftoverVariantsIntegrationTest extends WalkerTest {
          WalkerTestSpec spec = new WalkerTestSpec(
                  "-T LiftoverVariants -o %s -R " + b36KGReference + " --variant " + privateTestDir + "yri.trio.gatk_glftrio.intersection.annotated.filtered.chr1.500.noheader.unsortedSamples.vcf -chain " + validationDataLocation + "b36ToHg19.broad.over.chain -dict /seq/references/Homo_sapiens_assembly19/v0/Homo_sapiens_assembly19.dict",
                  1,
-                 Arrays.asList("2fd072a88006e3d7cfea4cac5d7b0d98"));
+                 Arrays.asList("91344768f1e98c979364ec0d5d3aa9d6"));
          executeTest("test b36 to hg19, unsorted samples", spec);
     }
 

public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariantsIntegrationTest.java

@@ -20,7 +20,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
                         + b37hapmapGenotypes + " -disc " + testFile
                         + " -o %s --no_cmdline_in_header --allowMissingVCFHeaders --allowMissingVCFHeaders",
                 1,
-                Arrays.asList("18ca4d3e69874c5e55377ca2ad7c6014")
+                Arrays.asList("d88bdae45ae0e74e8d8fd196627e612c")
         );
         spec.disableShadowBCF();
 
@@ -34,7 +34,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
                 baseTestString(" -sn A -sn B -sn C --variant " + testfile),
                 1,
-                Arrays.asList("ddb082f5f15944b7545b45c20440a474")
+                Arrays.asList("337bb7fc23153cf67acc42a466834775")
         );
 
         executeTest("testRepeatedLineSelection--" + testfile, spec);
@@ -49,7 +49,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
                         + b37hapmapGenotypes + " -disc " + testFile
                         + " -o %s --no_cmdline_in_header --allowMissingVCFHeaders",
                 1,
-                Arrays.asList("2e0eaed94e2ab14349b967dc89a09823")
+                Arrays.asList("54289033d35d32b8ebbb38c51fbb614c")
         );
         spec.disableShadowBCF();
 
@@ -64,7 +64,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
                 baseTestString(" -sn A -se '[CDH]' -sf " + samplesFile + " -env -ef -select 'DP < 250' --variant " + testfile),
                 1,
-                Arrays.asList("88c142737080847e44dc8e8c982cfc74")
+                Arrays.asList("ad0514b723ee1479d861291622bd4311")
         );
         spec.disableShadowBCF();
         executeTest("testComplexSelection--" + testfile, spec);
@@ -78,7 +78,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
                 "-T SelectVariants -R " + b36KGReference + " -L 1:1-1000000 -o %s --no_cmdline_in_header -xl_sn A -xl_sf " + samplesFile + " --variant " + testfile,
                 1,
-                Arrays.asList("338bfc635667793b89c349446982a36d")
+                Arrays.asList("bc0e00d0629b2bc6799e7e9db0dc775c")
         );
         spec.disableShadowBCF();
 
@@ -95,7 +95,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
                         + b37hapmapGenotypes + " --variant " + testFile
                         + " -o %s --no_cmdline_in_header --allowMissingVCFHeaders",
                 1,
-                Arrays.asList("25b3a8fd7b62b035c9d7c090db3c682d")
+                Arrays.asList("946e7f2e0ae08dc0e931c1634360fc46")
         );
         spec.disableShadowBCF();
 
@@ -109,7 +109,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
                 "-T SelectVariants -R " + b36KGReference + " -restrictAllelesTo MULTIALLELIC -selectType MIXED --variant " + testFile + " -o %s --no_cmdline_in_header",
                 1,
-                Arrays.asList("f0509f4c2c6c9eff3bec8171f00762b3")
+                Arrays.asList("a111642779b05de33ad04073d6022c21")
         );
 
         executeTest("testVariantTypeSelection--" + testFile, spec);
@@ -122,7 +122,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
                 "-T SelectVariants -R " + b36KGReference + " -sn NA12892 --variant:dbsnp " + testFile + " -o %s --no_cmdline_in_header",
                 1,
-                Arrays.asList("2a0436eecc2bc29fe559e4d1b9e13580")
+                Arrays.asList("d12ae1617deb38f5ed712dc326935b9a")
         );
 
         executeTest("testUsingDbsnpName--" + testFile, spec);
@@ -135,7 +135,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
                 "-T SelectVariants -R " + b36KGReference + " -regenotype -sn NA12892 --variant " + testFile + " -o %s --no_cmdline_in_header",
                 1,
-                Arrays.asList("be38bdc7bd88f5d09cf1a9d55cfecb0b")
+                Arrays.asList("c22ad8864d9951403672a24c20d6c3c2")
         );
 
         executeTest("testRegenotype--" + testFile, spec);
@@ -148,7 +148,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
                 "-T SelectVariants -R " + b36KGReference + " -select 'KG_FREQ < 0.5' --variant " + testFile + " -o %s --no_cmdline_in_header",
                 1,
-                Arrays.asList("7dabe6910a40a9db5006c6af66b4617c")
+                Arrays.asList("44f7c47395ca5b2afef5313f592c8cea")
         );
 
         executeTest("testMultipleRecordsAtOnePosition--" + testFile, spec);
@@ -161,7 +161,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
                 "-T SelectVariants -R " + b37KGReference + " --variant " + testFile + " -o %s --no_cmdline_in_header",
                 1,
-                Arrays.asList("034965918283d5597bb443e65cd20cf8")
+                Arrays.asList("a0b7f77edc16df0992d2c1363136a17e")
         );
 
         executeTest("testNoGTs--" + testFile, spec);
@@ -176,7 +176,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
         spec = new WalkerTestSpec(
                 baseTestString(" -sn A -se '[CDH]' -sf " + samplesFile + " -env -ef -select 'DP < 250' --variant " + testfile + " -nt 2"),
                 1,
-                Arrays.asList("88c142737080847e44dc8e8c982cfc74")
+                Arrays.asList("ad0514b723ee1479d861291622bd4311")
         );
         spec.disableShadowBCF();
         executeTest("testParallelization (2 threads)--" + testfile, spec);
@@ -190,7 +190,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
         spec = new WalkerTestSpec(
                 baseTestString(" -sn A -se '[CDH]' -sf " + samplesFile + " -env -ef -select 'DP < 250' --variant " + testfile + " -nt 4"),
                 1,
-                Arrays.asList("88c142737080847e44dc8e8c982cfc74")
+                Arrays.asList("ad0514b723ee1479d861291622bd4311")
         );
         spec.disableShadowBCF();
 
@@ -204,7 +204,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
                 "-T SelectVariants -R " + b37KGReference + " -o %s --no_cmdline_in_header -sf " + samplesFile + " --excludeNonVariants --variant " + testfile,
                 1,
-                Arrays.asList("3cbabcd256070ea113ddc04fef3b6900")
+                Arrays.asList("9acd6effcc78bfb832bed5edfd6a1b5b")
         );
         executeTest("test select from multi allelic with excludeNonVariants --" + testfile, spec);
     }

public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/VCFStreamingIntegrationTest.java

@@ -60,7 +60,7 @@ public class VCFStreamingIntegrationTest extends WalkerTest {
                     " --no_cmdline_in_header " +
                     " -o %s",
             1,
-            Arrays.asList("6248c072f9a9eafe9d386d2834e94f03")
+            Arrays.asList("2cdcd9e140eb1b6da7e365e37dd7d859")
         );
 
         executeTest("testSimpleVCFStreaming", spec);

public/java/test/org/broadinstitute/sting/utils/codecs/vcf/VCFIntegrationTest.java

@@ -46,7 +46,7 @@ public class VCFIntegrationTest extends WalkerTest {
         String baseCommand = "-R " + b37KGReference + " --no_cmdline_in_header -o %s ";
 
         String test1 = baseCommand + "-T SelectVariants -V " + testVCF;
-        WalkerTestSpec spec1 = new WalkerTestSpec(test1, 1, Arrays.asList("cf3f09ef63feba6958d32f95d587af80"));
+        WalkerTestSpec spec1 = new WalkerTestSpec(test1, 1, Arrays.asList("0f82ac11852e7f958c1a0ce52398c2ae"));
         executeTest("Test reading and writing samtools vcf", spec1);
     }
 
@@ -55,7 +55,7 @@ public class VCFIntegrationTest extends WalkerTest {
         String testVCF = privateTestDir + "ex2.vcf";
         String baseCommand = "-R " + b36KGReference + " --no_cmdline_in_header -o %s ";
         String test1 = baseCommand + "-T SelectVariants -V " + testVCF;
-        WalkerTestSpec spec1 = new WalkerTestSpec(test1, 1, Arrays.asList("5839ed4972e2ccf3a7b190752ede6596"));
+        WalkerTestSpec spec1 = new WalkerTestSpec(test1, 1, Arrays.asList("9773d6a121cfcb18d090965bc520f120"));
         executeTest("Test reading and writing samtools WEx vcf/BCF example", spec1);
     }
 }

public/java/test/org/broadinstitute/sting/utils/variantcontext/VariantContextTestProvider.java

@@ -724,10 +724,23 @@ public class VariantContextTestProvider {
         // and they must all be null
         for ( final String missingExpected : expectedKeys ) {
             final Object value = expected.get(missingExpected);
-            Assert.assertTrue(value == null || value.equals(VCFConstants.MISSING_VALUE_v4));
+            Assert.assertTrue(isMissing(value));
         }
     }
 
+    private static final boolean isMissing(final Object value) {
+        if ( value == null ) return true;
+        else if ( value.equals(VCFConstants.MISSING_VALUE_v4) ) return true;
+        else if ( value instanceof List ) {
+            // handles the case where all elements are null or the list is empty
+            for ( final Object elt : (List)value)
+                if ( elt != null )
+                    return false;
+            return true;
+        } else
+            return false;
+    }
+
     private static void assertAttributesEquals(final Object actual, final Object expected) {
         if ( expected instanceof Double ) {
             // must be very tolerant because doubles are being rounded to 2 sig figs