From b69d210255324d80cfca3986849a716492c76d1e Mon Sep 17 00:00:00 2001
From: Mark DePristo <depristo@broadinstitute.org>
Date: Mon, 17 Jun 2013 10:50:07 -0400
Subject: [PATCH] Bugfix: allow gzip VCF output in multi-threaded GATK output

-- VariantContextWriterStorage was gzipping the intermediate files that would be merged in, but the mergeInto function couldn't read those outputs, and we'd throw a very strange error. Now tmp. VCFs aren't compressed, even if the final VCF is.  Added integrationtest to ensure this behavior works going forward.
-- [delivers #47399279]
---
 .../storage/VariantContextWriterStorage.java  | 24 +++++++++++++++----
 .../gatk/EngineFeaturesIntegrationTest.java   | 15 ++++++++++++
 2 files changed, 35 insertions(+), 4 deletions(-)

diff --git a/public/java/src/org/broadinstitute/sting/gatk/io/storage/VariantContextWriterStorage.java b/public/java/src/org/broadinstitute/sting/gatk/io/storage/VariantContextWriterStorage.java
index 84709d6d8..80841bae7 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/io/storage/VariantContextWriterStorage.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/io/storage/VariantContextWriterStorage.java
@@ -67,12 +67,16 @@ public class VariantContextWriterStorage implements Storage<VariantContextWriter
     /**
      * Constructs an object which will write directly into the output file provided by the stub.
      * Intentionally delaying the writing of the header -- this should be filled in by the walker.
+     *
+     * Respecs the isCompressed() request in stub, so if isCompressed() is true then this
+     * will create a storage output that dumps output to a BlockCompressedOutputStream.
+     *
      * @param stub Stub to use when constructing the output file.
      */
     public VariantContextWriterStorage(VariantContextWriterStub stub)  {
         if ( stub.getOutputFile() != null ) {
             this.file = stub.getOutputFile();
-            writer = vcfWriterToFile(stub,stub.getOutputFile(),true);
+            writer = vcfWriterToFile(stub,stub.getOutputFile(),true,true);
         }
         else if ( stub.getOutputStream() != null ) {
             this.file = null;
@@ -86,13 +90,17 @@ public class VariantContextWriterStorage implements Storage<VariantContextWriter
 
     /**
      * Constructs an object which will redirect into a different file.
+     *
+     * Note that this function does not respect the isCompressed() request from the stub, in order
+     * to ensure that tmp. files can be read back in by the Tribble system, and merged with the mergeInto function.
+     *
      * @param stub Stub to use when synthesizing file / header info.
      * @param tempFile File into which to direct the output data.
      */
     public VariantContextWriterStorage(VariantContextWriterStub stub, File tempFile) {
         //logger.debug("Creating temporary output file " + tempFile.getAbsolutePath() + " for VariantContext output.");
         this.file = tempFile;
-        this.writer = vcfWriterToFile(stub, file, false);
+        this.writer = vcfWriterToFile(stub, file, false, false);
         writer.writeHeader(stub.getVCFHeader());
     }
 
@@ -101,11 +109,19 @@ public class VariantContextWriterStorage implements Storage<VariantContextWriter
      * @param stub Stub to use when constructing the output file.
      * @param file Target file into which to write VCF records.
      * @param indexOnTheFly true to index the file on the fly.  NOTE: will be forced to false for compressed files.
+     * @param allowCompressed if false, we won't compress the output, even if the stub requests it.  Critical
+     *                        for creating temp. output files that will be subsequently merged, as these do not
+     *                        support compressed output
      * @return A VCF writer for use with this class
      */
-    private VariantContextWriter vcfWriterToFile(VariantContextWriterStub stub, File file, boolean indexOnTheFly) {
+    private VariantContextWriter vcfWriterToFile(final VariantContextWriterStub stub,
+                                                 final File file,
+                                                 final boolean indexOnTheFly,
+                                                 final boolean allowCompressed) {
         try {
-            if ( stub.isCompressed() )
+            // we cannot merge compressed outputs, so don't compress if allowCompressed is false,
+            // which is the case when we have a temporary output file for later merging
+            if ( allowCompressed && stub.isCompressed() )
                 stream = new BlockCompressedOutputStream(file);
             else
                 stream = new PrintStream(new BufferedOutputStream(new FileOutputStream(file), BUFFER_SIZE));
diff --git a/public/java/test/org/broadinstitute/sting/gatk/EngineFeaturesIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/EngineFeaturesIntegrationTest.java
index 226224199..736989418 100644
--- a/public/java/test/org/broadinstitute/sting/gatk/EngineFeaturesIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/EngineFeaturesIntegrationTest.java
@@ -25,6 +25,7 @@
 
 package org.broadinstitute.sting.gatk;
 
+import net.sf.samtools.util.BlockCompressedInputStream;
 import org.broad.tribble.readers.AsciiLineReader;
 import org.broadinstitute.sting.WalkerTest;
 import org.broadinstitute.sting.commandline.Output;
@@ -212,4 +213,18 @@ public class EngineFeaturesIntegrationTest extends WalkerTest {
         Assert.assertNotNull(versionLine);
         Assert.assertEquals(versionLine.getValue(), CommandLineGATK.getVersionNumber());
     }
+
+    @Test(enabled = true)
+    public void testCompressedVCFOutputWithNT() throws Exception {
+        WalkerTestSpec spec = new WalkerTestSpec("-T UnifiedGenotyper -R " + b37KGReference + " -I "
+                + privateTestDir + "PCRFree.2x250.Illumina.20_10_11.bam"
+                + " -o %s -L 20:10,000,000-10,100,000 -nt 4",
+                1, Arrays.asList("vcf.gz"), Arrays.asList(""));
+        final File vcf = executeTest("testCompressedVCFOutputWithNT", spec).first.get(0);
+        final AsciiLineReader reader = new AsciiLineReader(new BlockCompressedInputStream(vcf));
+        int nLines = 0;
+        while ( reader.readLine() != null )
+            nLines++;
+        Assert.assertTrue(nLines > 0);
+    }
 }
\ No newline at end of file