From 74b51c5c7a8835d94d09e0aa2d5ac565b2980ddf Mon Sep 17 00:00:00 2001 From: David Roazen Date: Tue, 27 May 2014 13:41:44 -0400 Subject: [PATCH] Improve test suite tmp file cleanup -Make BaseTest.createTempFile() mark any possible corresponding index files for deletion on exit -Make WalkerTest mark shadow BCF files and auxiliary for deletion on exit -Make VariantRecalibrationWalkersIntegrationTest mark PDF files for deletion on exit --- ...ntRecalibrationWalkersIntegrationTest.java | 38 +++++++++++++------ .../gatk/engine/walkers/WalkerTest.java | 15 +++++--- .../broadinstitute/gatk/utils/BaseTest.java | 9 +++++ 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/protected/gatk-tools-protected/src/test/java/org/broadinstitute/gatk/tools/walkers/variantrecalibration/VariantRecalibrationWalkersIntegrationTest.java b/protected/gatk-tools-protected/src/test/java/org/broadinstitute/gatk/tools/walkers/variantrecalibration/VariantRecalibrationWalkersIntegrationTest.java index 35c482c48..ddacc93af 100644 --- a/protected/gatk-tools-protected/src/test/java/org/broadinstitute/gatk/tools/walkers/variantrecalibration/VariantRecalibrationWalkersIntegrationTest.java +++ b/protected/gatk-tools-protected/src/test/java/org/broadinstitute/gatk/tools/walkers/variantrecalibration/VariantRecalibrationWalkersIntegrationTest.java @@ -126,7 +126,8 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest { " -recalFile %s" + " -tranchesFile %s", Arrays.asList(params.recalMD5, params.tranchesMD5)); - executeTest("testVariantRecalibrator-"+params.inVCF, spec).getFirst(); + final List outputFiles = executeTest("testVariantRecalibrator-"+params.inVCF, spec).getFirst(); + setPDFsForDeletion(outputFiles); } @Test(dataProvider = "VRTest",dependsOnMethods="testVariantRecalibrator") @@ -142,7 +143,8 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest { " -recalFile " + getMd5DB().getMD5FilePath(params.recalMD5, null), Arrays.asList(params.cutVCFMD5)); spec.disableShadowBCF(); // TODO -- enable when we support symbolic alleles - executeTest("testApplyRecalibration-"+params.inVCF, spec); + final List outputFiles = executeTest("testApplyRecalibration-"+params.inVCF, spec).getFirst(); + setPDFsForDeletion(outputFiles); } @Test(dataProvider = "VRAggregateTest") @@ -163,7 +165,8 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest { " -recalFile %s" + " -tranchesFile %s", Arrays.asList(params.recalMD5, params.tranchesMD5)); - executeTest("testVariantRecalibratorAggregate-"+params.inVCF, spec).getFirst(); + final List outputFiles = executeTest("testVariantRecalibratorAggregate-"+params.inVCF, spec).getFirst(); + setPDFsForDeletion(outputFiles); } @Test(dataProvider = "VRAggregateTest",dependsOnMethods="testVariantRecalibratorAggregate") @@ -179,7 +182,8 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest { " -recalFile " + getMd5DB().getMD5FilePath(params.recalMD5, null), Arrays.asList(params.cutVCFMD5)); spec.disableShadowBCF(); // TODO -- enable when we support symbolic alleles - executeTest("testApplyRecalibrationAggregate-"+params.inVCF, spec); + final List outputFiles = executeTest("testApplyRecalibrationAggregate-"+params.inVCF, spec).getFirst(); + setPDFsForDeletion(outputFiles); } VRTest bcfTest = new VRTest(privateTestDir + "vqsr.bcf_test.snps.unfiltered.bcf", @@ -212,7 +216,8 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest { 2, Arrays.asList("bcf", "txt"), Arrays.asList(params.recalMD5, params.tranchesMD5)); - executeTest("testVariantRecalibrator-"+params.inVCF, spec).getFirst(); + final List outputFiles = executeTest("testVariantRecalibrator-"+params.inVCF, spec).getFirst(); + setPDFsForDeletion(outputFiles); } @Test(dataProvider = "VRBCFTest", dependsOnMethods="testVariantRecalibratorWithBCF") @@ -228,7 +233,8 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest { " -recalFile " + getMd5DB().getMD5FilePath(params.recalMD5, null), Arrays.asList(params.cutVCFMD5)); spec.disableShadowBCF(); - executeTest("testApplyRecalibration-"+params.inVCF, spec); + final List outputFiles = executeTest("testApplyRecalibration-"+params.inVCF, spec).getFirst(); + setPDFsForDeletion(outputFiles); } @@ -266,7 +272,8 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest { " -recalFile %s" + " -tranchesFile %s", Arrays.asList(params.recalMD5, params.tranchesMD5)); - executeTest("testVariantRecalibratorIndel-"+params.inVCF, spec).getFirst(); + final List outputFiles = executeTest("testVariantRecalibratorIndel-"+params.inVCF, spec).getFirst(); + setPDFsForDeletion(outputFiles); } @Test(dataProvider = "VRIndelTest",dependsOnMethods="testVariantRecalibratorIndel") @@ -283,7 +290,8 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest { " -recalFile " + getMd5DB().getMD5FilePath(params.recalMD5, null), Arrays.asList(params.cutVCFMD5)); spec.disableShadowBCF(); // has to be disabled because the input VCF is missing LowQual annotation - executeTest("testApplyRecalibrationIndel-" + params.inVCF, spec); + final List outputFiles = executeTest("testApplyRecalibrationIndel-" + params.inVCF, spec).getFirst(); + setPDFsForDeletion(outputFiles); } @Test @@ -299,7 +307,8 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest { " -tranchesFile " + privateTestDir + "VQSR.mixedTest.tranches" + " -recalFile " + privateTestDir + "VQSR.mixedTest.recal", Arrays.asList("03a0ed00af6aac76d39e569f90594a02")); - executeTest("testApplyRecalibrationSnpAndIndelTogether", spec); + final List outputFiles = executeTest("testApplyRecalibrationSnpAndIndelTogether", spec).getFirst(); + setPDFsForDeletion(outputFiles); } @Test(enabled = true) @@ -317,13 +326,20 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest { final WalkerTestSpec spec = new WalkerTestSpec(base, 1, Arrays.asList("")); spec.disableShadowBCF(); - final File VCF = executeTest("testApplyRecalibrationSnpAndIndelTogether", spec).first.get(0); - + final List outputFiles = executeTest("testApplyRecalibrationSnpAndIndelTogether", spec).getFirst(); + setPDFsForDeletion(outputFiles); + final File VCF = outputFiles.get(0); for( final VariantContext VC : GATKVCFUtils.readAllVCs(VCF, new VCFCodec()).getSecond() ) { if( VC != null ) { Assert.assertTrue(VC.isNotFiltered()); // there should only be unfiltered records in the output VCF file } } } + + private void setPDFsForDeletion( final List walkerOutputFiles ) { + for ( final File outputFile : walkerOutputFiles ) { + new File(outputFile.getAbsolutePath() + ".pdf").deleteOnExit(); + } + } } diff --git a/public/gatk-tools-public/src/test/java/org/broadinstitute/gatk/engine/walkers/WalkerTest.java b/public/gatk-tools-public/src/test/java/org/broadinstitute/gatk/engine/walkers/WalkerTest.java index 3e5468002..eeb3e0de6 100644 --- a/public/gatk-tools-public/src/test/java/org/broadinstitute/gatk/engine/walkers/WalkerTest.java +++ b/public/gatk-tools-public/src/test/java/org/broadinstitute/gatk/engine/walkers/WalkerTest.java @@ -319,9 +319,12 @@ public class WalkerTest extends BaseTest { String ext = spec.exts == null ? ".tmp" : "." + spec.exts.get(i); File fl = createTempFile(String.format("walktest.tmp_param.%d", i), ext); - // Mark corresponding indices for deletion on exit as well just in case an index is created for the temp file: - new File(fl.getAbsolutePath() + Tribble.STANDARD_INDEX_EXTENSION).deleteOnExit(); - new File(fl.getAbsolutePath() + TabixUtils.STANDARD_INDEX_EXTENSION).deleteOnExit(); + // Cleanup any potential shadow BCFs on exit too, if we're generating them + if ( spec.includeShadowBCF && GENERATE_SHADOW_BCF ) { + final File potentalShadowBCFFile = BCF2Utils.shadowBCF(fl); + potentalShadowBCFFile.deleteOnExit(); + new File(potentalShadowBCFFile.getAbsolutePath() + Tribble.STANDARD_INDEX_EXTENSION).deleteOnExit(); + } tmpFiles.add(fl); } @@ -336,10 +339,12 @@ public class WalkerTest extends BaseTest { List md5s = new LinkedList(); md5s.addAll(spec.md5s); - // check to see if they included any auxillary files, if so add them to the list + // check to see if they included any auxillary files, if so add them to the list and set them to be deleted on exit for (String md5 : spec.auxillaryFiles.keySet()) { md5s.add(md5); - tmpFiles.add(spec.auxillaryFiles.get(md5)); + final File auxFile = spec.auxillaryFiles.get(md5); + auxFile.deleteOnExit(); + tmpFiles.add(auxFile); } return executeTest(name, spec.getTestClassName(), spec.getOutputFileLocation(), md5s, tmpFiles, args, null); } diff --git a/public/gatk-tools-public/src/test/java/org/broadinstitute/gatk/utils/BaseTest.java b/public/gatk-tools-public/src/test/java/org/broadinstitute/gatk/utils/BaseTest.java index bfc71bea1..4090fced9 100644 --- a/public/gatk-tools-public/src/test/java/org/broadinstitute/gatk/utils/BaseTest.java +++ b/public/gatk-tools-public/src/test/java/org/broadinstitute/gatk/utils/BaseTest.java @@ -25,6 +25,8 @@ package org.broadinstitute.gatk.utils; +import htsjdk.tribble.Tribble; +import htsjdk.tribble.util.TabixUtils; import org.apache.log4j.AppenderSkeleton; import org.apache.log4j.Level; import org.apache.log4j.Logger; @@ -291,6 +293,13 @@ public abstract class BaseTest { try { File file = File.createTempFile(name, extension); file.deleteOnExit(); + + // Mark corresponding indices for deletion on exit as well just in case an index is created for the temp file: + new File(file.getAbsolutePath() + Tribble.STANDARD_INDEX_EXTENSION).deleteOnExit(); + new File(file.getAbsolutePath() + TabixUtils.STANDARD_INDEX_EXTENSION).deleteOnExit(); + new File(file.getAbsolutePath() + ".bai").deleteOnExit(); + new File(file.getAbsolutePath().replaceAll(extension + "$", ".bai")).deleteOnExit(); + return file; } catch (IOException ex) { throw new ReviewedGATKException("Cannot create temp file: " + ex.getMessage(), ex);