Merge pull request #641 from broadinstitute/dr_post_test_cleanup

Improve test suite tmp file cleanup

protected/gatk-tools-protected/src/test/java/org/broadinstitute/gatk/tools/walkers/variantrecalibration/VariantRecalibrationWalkersIntegrationTest.java

@@ -126,7 +126,8 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest {
                         " -recalFile %s" +
                         " -tranchesFile %s",
                 Arrays.asList(params.recalMD5, params.tranchesMD5));
-        executeTest("testVariantRecalibrator-"+params.inVCF, spec).getFirst();
+        final List<File> outputFiles = executeTest("testVariantRecalibrator-"+params.inVCF, spec).getFirst();
+        setPDFsForDeletion(outputFiles);
     }
 
     @Test(dataProvider = "VRTest",dependsOnMethods="testVariantRecalibrator")
@@ -142,7 +143,8 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest {
                         " -recalFile " + getMd5DB().getMD5FilePath(params.recalMD5, null),
                 Arrays.asList(params.cutVCFMD5));
         spec.disableShadowBCF(); // TODO -- enable when we support symbolic alleles
-        executeTest("testApplyRecalibration-"+params.inVCF, spec);
+        final List<File> outputFiles = executeTest("testApplyRecalibration-"+params.inVCF, spec).getFirst();
+        setPDFsForDeletion(outputFiles);
     }
 
     @Test(dataProvider = "VRAggregateTest")
@@ -163,7 +165,8 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest {
                         " -recalFile %s" +
                         " -tranchesFile %s",
                 Arrays.asList(params.recalMD5, params.tranchesMD5));
-        executeTest("testVariantRecalibratorAggregate-"+params.inVCF, spec).getFirst();
+        final List<File> outputFiles = executeTest("testVariantRecalibratorAggregate-"+params.inVCF, spec).getFirst();
+        setPDFsForDeletion(outputFiles);
     }
 
     @Test(dataProvider = "VRAggregateTest",dependsOnMethods="testVariantRecalibratorAggregate")
@@ -179,7 +182,8 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest {
                         " -recalFile " + getMd5DB().getMD5FilePath(params.recalMD5, null),
                 Arrays.asList(params.cutVCFMD5));
         spec.disableShadowBCF(); // TODO -- enable when we support symbolic alleles
-        executeTest("testApplyRecalibrationAggregate-"+params.inVCF, spec);
+        final List<File> outputFiles = executeTest("testApplyRecalibrationAggregate-"+params.inVCF, spec).getFirst();
+        setPDFsForDeletion(outputFiles);
     }
 
     VRTest bcfTest = new VRTest(privateTestDir + "vqsr.bcf_test.snps.unfiltered.bcf",
@@ -212,7 +216,8 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest {
                 2,
                 Arrays.asList("bcf", "txt"),
                 Arrays.asList(params.recalMD5, params.tranchesMD5));
-        executeTest("testVariantRecalibrator-"+params.inVCF, spec).getFirst();
+        final List<File> outputFiles = executeTest("testVariantRecalibrator-"+params.inVCF, spec).getFirst();
+        setPDFsForDeletion(outputFiles);
     }
 
     @Test(dataProvider = "VRBCFTest", dependsOnMethods="testVariantRecalibratorWithBCF")
@@ -228,7 +233,8 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest {
                         " -recalFile " + getMd5DB().getMD5FilePath(params.recalMD5, null),
                 Arrays.asList(params.cutVCFMD5));
         spec.disableShadowBCF();
-        executeTest("testApplyRecalibration-"+params.inVCF, spec);
+        final List<File> outputFiles = executeTest("testApplyRecalibration-"+params.inVCF, spec).getFirst();
+        setPDFsForDeletion(outputFiles);
     }
 
 
@@ -266,7 +272,8 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest {
                         " -recalFile %s" +
                         " -tranchesFile %s",
                 Arrays.asList(params.recalMD5, params.tranchesMD5));
-        executeTest("testVariantRecalibratorIndel-"+params.inVCF, spec).getFirst();
+        final List<File> outputFiles = executeTest("testVariantRecalibratorIndel-"+params.inVCF, spec).getFirst();
+        setPDFsForDeletion(outputFiles);
     }
 
     @Test(dataProvider = "VRIndelTest",dependsOnMethods="testVariantRecalibratorIndel")
@@ -283,7 +290,8 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest {
                         " -recalFile " + getMd5DB().getMD5FilePath(params.recalMD5, null),
                 Arrays.asList(params.cutVCFMD5));
         spec.disableShadowBCF(); // has to be disabled because the input VCF is missing LowQual annotation
-        executeTest("testApplyRecalibrationIndel-" + params.inVCF, spec);
+        final List<File> outputFiles = executeTest("testApplyRecalibrationIndel-" + params.inVCF, spec).getFirst();
+        setPDFsForDeletion(outputFiles);
     }
 
     @Test
@@ -299,7 +307,8 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest {
                         " -tranchesFile " + privateTestDir + "VQSR.mixedTest.tranches" +
                         " -recalFile " + privateTestDir + "VQSR.mixedTest.recal",
                 Arrays.asList("03a0ed00af6aac76d39e569f90594a02"));
-        executeTest("testApplyRecalibrationSnpAndIndelTogether", spec);
+        final List<File> outputFiles = executeTest("testApplyRecalibrationSnpAndIndelTogether", spec).getFirst();
+        setPDFsForDeletion(outputFiles);
     }
 
     @Test(enabled = true)
@@ -317,13 +326,20 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest {
 
         final WalkerTestSpec spec = new WalkerTestSpec(base, 1, Arrays.asList(""));
         spec.disableShadowBCF();
-        final File VCF = executeTest("testApplyRecalibrationSnpAndIndelTogether", spec).first.get(0);
+        final List<File> outputFiles = executeTest("testApplyRecalibrationSnpAndIndelTogether", spec).getFirst();
-
+        setPDFsForDeletion(outputFiles);
+        final File VCF = outputFiles.get(0);
         for( final VariantContext VC : GATKVCFUtils.readAllVCs(VCF, new VCFCodec()).getSecond() ) {
             if( VC != null ) {
                 Assert.assertTrue(VC.isNotFiltered()); // there should only be unfiltered records in the output VCF file
             }
         }
     }
+
+    private void setPDFsForDeletion( final List<File> walkerOutputFiles ) {
+        for ( final File outputFile : walkerOutputFiles ) {
+            new File(outputFile.getAbsolutePath() + ".pdf").deleteOnExit();
+        }
+    }
 }
 

public/gatk-tools-public/src/test/java/org/broadinstitute/gatk/engine/walkers/WalkerTest.java

@@ -319,9 +319,12 @@ public class WalkerTest extends BaseTest {
             String ext = spec.exts == null ? ".tmp" : "." + spec.exts.get(i);
             File fl = createTempFile(String.format("walktest.tmp_param.%d", i), ext);
 
-            // Mark corresponding indices for deletion on exit as well just in case an index is created for the temp file:
+            // Cleanup any potential shadow BCFs on exit too, if we're generating them
-            new File(fl.getAbsolutePath() + Tribble.STANDARD_INDEX_EXTENSION).deleteOnExit();
+            if ( spec.includeShadowBCF && GENERATE_SHADOW_BCF ) {
-            new File(fl.getAbsolutePath() + TabixUtils.STANDARD_INDEX_EXTENSION).deleteOnExit();
+                final File potentalShadowBCFFile = BCF2Utils.shadowBCF(fl);
+                potentalShadowBCFFile.deleteOnExit();
+                new File(potentalShadowBCFFile.getAbsolutePath() + Tribble.STANDARD_INDEX_EXTENSION).deleteOnExit();
+            }
 
             tmpFiles.add(fl);
         }
@@ -336,10 +339,12 @@ public class WalkerTest extends BaseTest {
             List<String> md5s = new LinkedList<String>();
             md5s.addAll(spec.md5s);
 
-            // check to see if they included any auxillary files, if so add them to the list
+            // check to see if they included any auxillary files, if so add them to the list and set them to be deleted on exit
             for (String md5 : spec.auxillaryFiles.keySet()) {
                 md5s.add(md5);
-                tmpFiles.add(spec.auxillaryFiles.get(md5));
+                final File auxFile = spec.auxillaryFiles.get(md5);
+                auxFile.deleteOnExit();
+                tmpFiles.add(auxFile);
             }
             return executeTest(name, spec.getTestClassName(), spec.getOutputFileLocation(), md5s, tmpFiles, args, null);
         }

public/gatk-tools-public/src/test/java/org/broadinstitute/gatk/utils/BaseTest.java

@@ -25,6 +25,8 @@
 
 package org.broadinstitute.gatk.utils;
 
+import htsjdk.tribble.Tribble;
+import htsjdk.tribble.util.TabixUtils;
 import org.apache.log4j.AppenderSkeleton;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
@@ -291,6 +293,13 @@ public abstract class BaseTest {
         try {
             File file = File.createTempFile(name, extension);
             file.deleteOnExit();
+
+            // Mark corresponding indices for deletion on exit as well just in case an index is created for the temp file:
+            new File(file.getAbsolutePath() + Tribble.STANDARD_INDEX_EXTENSION).deleteOnExit();
+            new File(file.getAbsolutePath() + TabixUtils.STANDARD_INDEX_EXTENSION).deleteOnExit();
+            new File(file.getAbsolutePath() + ".bai").deleteOnExit();
+            new File(file.getAbsolutePath().replaceAll(extension + "$", ".bai")).deleteOnExit();
+
             return file;
         } catch (IOException ex) {
             throw new ReviewedGATKException("Cannot create temp file: " + ex.getMessage(), ex);