diff --git a/public/gatk-framework/src/main/java/org/broadinstitute/sting/gatk/io/storage/VariantContextWriterStorage.java b/public/gatk-framework/src/main/java/org/broadinstitute/sting/gatk/io/storage/VariantContextWriterStorage.java index de203e59f..52a85c358 100644 --- a/public/gatk-framework/src/main/java/org/broadinstitute/sting/gatk/io/storage/VariantContextWriterStorage.java +++ b/public/gatk-framework/src/main/java/org/broadinstitute/sting/gatk/io/storage/VariantContextWriterStorage.java @@ -131,7 +131,6 @@ public class VariantContextWriterStorage implements Storage options = stub.getWriterOptions(indexOnTheFly); VariantContextWriter writer = VariantContextWriterFactory.create(file, this.stream, stub.getMasterSequenceDictionary(), stub.getIndexCreator(), options); diff --git a/public/gatk-framework/src/main/java/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java b/public/gatk-framework/src/main/java/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java index 34831471a..dbb6a3d58 100644 --- a/public/gatk-framework/src/main/java/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java +++ b/public/gatk-framework/src/main/java/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java @@ -25,6 +25,7 @@ package org.broadinstitute.sting.gatk.io.stubs; +import org.broad.tribble.AbstractFeatureReader; import org.broadinstitute.sting.commandline.*; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.variant.variantcontext.writer.VariantContextWriter; @@ -49,8 +50,6 @@ public class VCFWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor { public static final String NO_HEADER_ARG_NAME = "no_cmdline_in_header"; public static final String SITES_ONLY_ARG_NAME = "sites_only"; public static final String FORCE_BCF = "bcf"; - public static final HashSet SUPPORTED_ZIPPED_SUFFIXES = - new HashSet<>(Arrays.asList(VariantContextWriterFactory.BLOCK_COMPRESSED_EXTENSIONS)); /** * The engine into which output stubs should be fed. @@ -223,19 +222,6 @@ public class VCFWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor { * @return true if the file will be compressed. */ public static boolean isCompressed(String writerFileName) { - return writerFileName != null && SUPPORTED_ZIPPED_SUFFIXES.contains(getFileSuffix(writerFileName)); + return writerFileName != null && AbstractFeatureReader.hasBlockCompressedExtension(writerFileName); } - - /** - * Returns a lower-cased version of the suffix of the provided file. - * @param fileName the file name. Must not be null. - * @return lower-cased version of the file suffix. Will not be null. - */ - private static String getFileSuffix(String fileName) { - int indexOfLastDot = fileName.lastIndexOf("."); - if ( indexOfLastDot == -1 ) - return ""; - return fileName.substring(indexOfLastDot).toLowerCase(); - } - } diff --git a/public/gatk-framework/src/main/java/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrackBuilder.java b/public/gatk-framework/src/main/java/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrackBuilder.java index 1fe0a8bed..ea420295a 100644 --- a/public/gatk-framework/src/main/java/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrackBuilder.java +++ b/public/gatk-framework/src/main/java/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrackBuilder.java @@ -37,6 +37,7 @@ import org.broad.tribble.util.LittleEndianOutputStream; import org.broadinstitute.sting.commandline.Tags; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.gatk.arguments.ValidationExclusion; +import org.broadinstitute.sting.gatk.io.stubs.VCFWriterArgumentTypeDescriptor; import org.broadinstitute.sting.gatk.refdata.utils.RMDTriplet; import org.broadinstitute.sting.gatk.refdata.utils.RMDTriplet.RMDStorageType; import org.broadinstitute.sting.utils.GenomeLocParser; @@ -141,7 +142,7 @@ public class RMDTrackBuilder { // extends PluginManager { // return a feature reader track Pair pair; - if (inputFile.getAbsolutePath().endsWith(".gz")) + if (VCFWriterArgumentTypeDescriptor.isCompressed(inputFile.toString())) pair = createTabixIndexedFeatureSource(descriptor, name, inputFile); else pair = getFeatureSource(descriptor, name, inputFile, fileDescriptor.getStorageType()); @@ -178,9 +179,8 @@ public class RMDTrackBuilder { // extends PluginManager { // we might not know the index type, try loading with the default reader constructor logger.debug("Attempting to load " + inputFile + " as a tabix indexed file without validating it"); try { - final File indexFile = null;//new File(inputFile.getAbsoluteFile() + TabixUtils.STANDARD_INDEX_EXTENSION); - final SAMSequenceDictionary dict = null; //TabixUtils.getSequenceDictionary(indexFile); - return new Pair<>(AbstractFeatureReader.getFeatureReader(inputFile.getAbsolutePath(), createCodec(descriptor, name, inputFile)), dict); + // getFeatureReader will detect that it's Tabix + return new Pair<>(AbstractFeatureReader.getFeatureReader(inputFile.getAbsolutePath(), createCodec(descriptor, name, inputFile)), null); } catch (TribbleException e) { throw new UserException(e.getMessage(), e); } diff --git a/public/gatk-framework/src/main/java/org/broadinstitute/sting/tools/CatVariants.java b/public/gatk-framework/src/main/java/org/broadinstitute/sting/tools/CatVariants.java index b1fa87807..e8ef39e3e 100644 --- a/public/gatk-framework/src/main/java/org/broadinstitute/sting/tools/CatVariants.java +++ b/public/gatk-framework/src/main/java/org/broadinstitute/sting/tools/CatVariants.java @@ -53,7 +53,6 @@ import org.broadinstitute.variant.variantcontext.writer.VariantContextWriterFact import java.io.*; import java.util.*; - /** * * Concatenates VCF files of non-overlapped genome intervals, all with the same set of samples @@ -73,13 +72,12 @@ import java.util.*; *

Input

*

* One or more variant sets to combine. They should be of non-overlapping genome intervals and with the same samples (in the same order). - * The input files should be 'name.vcf' or 'name.VCF' or 'name.bcf' or 'name.BCF'. * If the files are ordered according to the appearance of intervals in the ref genome, then one can use the -assumeSorted flag. *

* *

Output

*

- * A combined VCF. The output file should be 'name.vcf' or 'name.VCF'. + * A combined VCF or BCF. The output file should have the same extension as the input(s). * <\p> * *

Important note

@@ -113,17 +111,17 @@ public class CatVariants extends CommandLineProgram { * The VCF or BCF files to merge together * * CatVariants can take any number of -V arguments on the command line. Each -V argument - * will be included in the final merged output VCF. The order of arguments does not matter, but it runs more + * will be included in the final merged output VCF/BCF. The order of arguments does not matter, but it runs more * efficiently if they are sorted based on the intervals and the assumeSorted argument is used. * */ - @Input(fullName="variant", shortName="V", doc="Input VCF file/s named .vcf or .bcf", required = true) + @Input(fullName="variant", shortName="V", doc="Input VCF file/s", required = true) private List variant = null; - @Output(fullName = "outputFile", shortName = "out", doc = "output file name .vcf or .bcf", required = true) + @Output(fullName = "outputFile", shortName = "out", doc = "output file", required = true) private File outputFile = null; - @Argument(fullName = "assumeSorted", shortName = "assumeSorted", doc = "assumeSorted should be true if he input files are already sorted (based on the position of the variants", required = false) + @Argument(fullName = "assumeSorted", shortName = "assumeSorted", doc = "assumeSorted should be true if the input files are already sorted (based on the position of the variants)", required = false) private Boolean assumeSorted = false; @Argument(fullName = "variant_index_type", doc = "which type of IndexCreator to use for VCF/BCF indices", required = false) @@ -137,19 +135,69 @@ public class CatVariants extends CommandLineProgram { */ private static void printUsage() { System.err.println("Usage: java -cp target/GenomeAnalysisTK.jar org.broadinstitute.sting.tools.CatVariants --reference --variant --outputFile [--assumeSorted]"); - System.err.println(" The input file(s) can be of type: VCF (must end in .vcf or .VCF) or"); - System.err.println(" BCF2 (must end in .bcf or .BCF)."); - System.err.println(" Output file must be of type vcf or bcf (must end in .vcf or .bcf)."); + System.err.println(" The output file must be of the same type as all input files."); System.err.println(" If the input files are already sorted, then indicate that with --assumeSorted to improve performance."); } + private enum FileType { + VCF, + BCF, + BLOCK_COMPRESSED_VCF, + INVALID + } + + private FileType fileExtensionCheck(File inFile, File outFile) { + final String inFileName = inFile.toString().toLowerCase(); + final String outFileName = outFile.toString().toLowerCase(); + + FileType inFileType = FileType.INVALID; + + if (inFileName.endsWith(".vcf")) { + inFileType = FileType.VCF; + if (outFileName.endsWith(".vcf")) + return inFileType; + } + + if (inFileName.endsWith(".bcf")) { + inFileType = FileType.BCF; + if (outFileName.endsWith(".bcf")) + return inFileType; + } + + for (String extension : AbstractFeatureReader.BLOCK_COMPRESSED_EXTENSIONS) { + if (inFileName.endsWith(".vcf" + extension)) { + inFileType = FileType.BLOCK_COMPRESSED_VCF; + if (outFileName.endsWith(".vcf" + extension)) + return inFileType; + } + } + + if (inFileType == FileType.INVALID) + System.err.println(String.format("File extension for input file %s is not valid for CatVariants", inFile)); + else + System.err.println(String.format("File extension mismatch between input %s and output %s", inFile, outFile)); + + printUsage(); + return FileType.INVALID; + } + + private FeatureReader getFeatureReader(final FileType fileType, final File file) { + FeatureReader reader = null; + switch(fileType) { + case VCF: + case BLOCK_COMPRESSED_VCF: + // getFeatureReader will handle both block-compressed and plain text VCFs + reader = AbstractFeatureReader.getFeatureReader(file.getAbsolutePath(), new VCFCodec(), false); + break; + case BCF: + reader = AbstractFeatureReader.getFeatureReader(file.getAbsolutePath(), new BCF2Codec(), false); + break; + } + return reader; + } + @Override protected int execute() throws Exception { - //if(help){ - // printUsage(); - // return 1; - //} - BasicConfigurator.configure(); logger.setLevel(Level.INFO); @@ -162,37 +210,27 @@ public class CatVariants extends CommandLineProgram { Comparator> positionComparator = new PositionComparator(); - - //PriorityQueue>> queue = - // new PriorityQueue>>(2000, comparator); Queue> priorityQueue; - if(assumeSorted) - priorityQueue = new LinkedList>(); + if (assumeSorted) + priorityQueue = new LinkedList<>(); else - priorityQueue = new PriorityQueue>(10000, positionComparator); + priorityQueue = new PriorityQueue<>(10000, positionComparator); - Iterator files = variant.iterator(); - File file; - while (files.hasNext()) { - file = files.next(); - if (!(file.getName().endsWith(".vcf") || file.getName().endsWith(".VCF") || file.getName().endsWith(".bcf") || file.getName().endsWith(".BCF"))){ - System.err.println("File " + file.getAbsolutePath() + " should be .vcf or .bcf"); - printUsage(); + FileType fileType = FileType.INVALID; + for (File file : variant) { + // if it returns a valid type, it will be the same for all files + fileType = fileExtensionCheck(file, outputFile); + if (fileType == FileType.INVALID) return 1; - } + if (assumeSorted){ - priorityQueue.add(new Pair(0,file)); + priorityQueue.add(new Pair<>(0,file)); } else{ if (!file.exists()) { throw new UserException(String.format("File %s doesn't exist",file.getAbsolutePath())); } - FeatureReader reader; - boolean useVCF = (file.getName().endsWith(".vcf") || file.getName().endsWith(".VCF")); - if(useVCF) - reader = AbstractFeatureReader.getFeatureReader(file.getAbsolutePath(), new VCFCodec(), false); - else - reader = AbstractFeatureReader.getFeatureReader(file.getAbsolutePath(), new BCF2Codec(), false); + FeatureReader reader = getFeatureReader(fileType, file); Iterator it = reader.iterator(); if(!it.hasNext()){ System.err.println(String.format("File %s is empty. This file will be ignored",file.getAbsolutePath())); @@ -201,37 +239,25 @@ public class CatVariants extends CommandLineProgram { VariantContext vc = it.next(); int firstPosition = vc.getStart(); reader.close(); - //queue.add(new Pair>(firstPosition,reader)); - priorityQueue.add(new Pair(firstPosition,file)); + priorityQueue.add(new Pair<>(firstPosition,file)); } } - if (!(outputFile.getName().endsWith(".vcf") || outputFile.getName().endsWith(".VCF"))){ - throw new UserException(String.format("Output file %s should be .vcf", outputFile)); - } - FileOutputStream outputStream = new FileOutputStream(outputFile); EnumSet options = EnumSet.of(Options.INDEX_ON_THE_FLY); final IndexCreator idxCreator = GATKVCFUtils.getIndexCreator(variant_index_type, variant_index_parameter, outputFile, ref.getSequenceDictionary()); final VariantContextWriter outputWriter = VariantContextWriterFactory.create(outputFile, outputStream, ref.getSequenceDictionary(), idxCreator, options); boolean firstFile = true; - int count =0; - //while(!queue.isEmpty()){ + int count = 0; while(!priorityQueue.isEmpty() ){ count++; - //FeatureReader reader = queue.remove().getSecond(); - file = priorityQueue.remove().getSecond(); + File file = priorityQueue.remove().getSecond(); if (!file.exists()) { throw new UserException(String.format("File %s doesn't exist",file.getAbsolutePath())); } - FeatureReader reader; - boolean useVCF = (file.getName().endsWith(".vcf") || file.getName().endsWith(".VCF")); - if(useVCF) - reader = AbstractFeatureReader.getFeatureReader(file.getAbsolutePath(), new VCFCodec(), false); - else - reader = AbstractFeatureReader.getFeatureReader(file.getAbsolutePath(), new BCF2Codec(), false); + FeatureReader reader = getFeatureReader(fileType, file); if(count%10 ==0) System.out.print(count); @@ -255,13 +281,11 @@ public class CatVariants extends CommandLineProgram { } System.out.println(); - outputStream.close(); outputWriter.close(); return 0; } - public static void main(String[] args){ try { CatVariants instance = new CatVariants(); @@ -286,5 +310,4 @@ public class CatVariants extends CommandLineProgram { return startPositionP1 < startPositionP2 ? -1 : 1 ; } } - } diff --git a/public/gatk-framework/src/main/java/org/broadinstitute/sting/utils/runtime/ProcessController.java b/public/gatk-framework/src/main/java/org/broadinstitute/sting/utils/runtime/ProcessController.java index 097362044..da7de61fe 100644 --- a/public/gatk-framework/src/main/java/org/broadinstitute/sting/utils/runtime/ProcessController.java +++ b/public/gatk-framework/src/main/java/org/broadinstitute/sting/utils/runtime/ProcessController.java @@ -29,6 +29,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; +import org.broadinstitute.sting.utils.Utils; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; @@ -267,6 +268,25 @@ public class ProcessController { return new ProcessOutput(exitCode, stdout, stderr); } + /** + * Executes a command line program with the settings and waits for it to return, + * processing the output on a background thread. + * + * Throws an IOException if the ProcessOutput exit code is nonzero + * + * @param settings Settings to be run. + */ + public ProcessOutput execAndCheck(ProcessSettings settings) throws IOException { + ProcessOutput po = exec(settings); + if (po.getExitValue() != 0) { + String message = String.format("Process exited with %d\nCommand Line: %s", + po.getExitValue(), + Utils.join(" ", settings.getCommand())); + throw new IOException(message); + } + return po; + } + /** * @return The set of still running processes. */ diff --git a/public/gatk-framework/src/test/java/org/broadinstitute/sting/commandline/LoggingIntegrationTest.java b/public/gatk-framework/src/test/java/org/broadinstitute/sting/commandline/LoggingIntegrationTest.java index 6c8c9a473..fdf0ae0ea 100644 --- a/public/gatk-framework/src/test/java/org/broadinstitute/sting/commandline/LoggingIntegrationTest.java +++ b/public/gatk-framework/src/test/java/org/broadinstitute/sting/commandline/LoggingIntegrationTest.java @@ -40,7 +40,6 @@ import org.broadinstitute.sting.BaseTest; import org.broadinstitute.sting.MD5DB; import org.broadinstitute.sting.MD5Mismatch; import org.broadinstitute.sting.gatk.CommandLineGATK; -import org.broadinstitute.sting.utils.Utils; import org.broadinstitute.sting.utils.runtime.*; public class LoggingIntegrationTest { @@ -100,14 +99,14 @@ public class LoggingIntegrationTest { // output argument ProcessSettings ps = new ProcessSettings(cfg.getCmdLine(false).split("\\s+")); - execAndCheck(pc, ps); + pc.execAndCheck(ps); String output_argument_md5 = md5db.calculateFileMD5(cfg.argumentOutputFile); // pipe to stdout ps = new ProcessSettings(cfg.getCmdLine(true).split("\\s+")); ps.setStdoutSettings(new OutputStreamSettings(cfg.pipedOutputFile)); - execAndCheck(pc, ps); + pc.execAndCheck(ps); MD5DB.MD5Match result = md5db.testFileMD5("LoggingIntegrationTest", "LoggingIntegrationTest", cfg.pipedOutputFile, output_argument_md5, false); if(result.failed) { @@ -115,21 +114,4 @@ public class LoggingIntegrationTest { Assert.fail(failure.toString()); } } - - /** - * Execute a process, and throw an IOException if the exit code is nonzero - * - * @param pc - * @param ps - * @throws IOException - */ - private void execAndCheck(ProcessController pc, ProcessSettings ps) throws IOException { - ProcessOutput po = pc.exec(ps); - if (po.getExitValue() != 0) { - String message = String.format("Process exited with %d\nCommand Line: %s", - po.getExitValue(), - Utils.join(" ", ps.getCommand())); - throw new IOException(message); - } - } } \ No newline at end of file diff --git a/public/gatk-framework/src/test/java/org/broadinstitute/sting/tools/CatVariantsIntegrationTest.java b/public/gatk-framework/src/test/java/org/broadinstitute/sting/tools/CatVariantsIntegrationTest.java new file mode 100644 index 000000000..d99b194b8 --- /dev/null +++ b/public/gatk-framework/src/test/java/org/broadinstitute/sting/tools/CatVariantsIntegrationTest.java @@ -0,0 +1,129 @@ +/* +* Copyright (c) 2012 The Broad Institute +* +* Permission is hereby granted, free of charge, to any person +* obtaining a copy of this software and associated documentation +* files (the "Software"), to deal in the Software without +* restriction, including without limitation the rights to use, +* copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following +* conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +* THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +package org.broadinstitute.sting.tools; + +import org.apache.commons.lang.StringUtils; +import org.broad.tribble.AbstractFeatureReader; +import org.broadinstitute.sting.BaseTest; +import org.broadinstitute.sting.MD5DB; +import org.broadinstitute.sting.MD5Mismatch; +import org.broadinstitute.sting.utils.runtime.ProcessController; +import org.broadinstitute.sting.utils.runtime.ProcessSettings; +import org.broadinstitute.sting.utils.runtime.RuntimeUtils; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.io.File; +import java.io.IOException; + +public class CatVariantsIntegrationTest { + private final MD5DB md5db = new MD5DB(); + private final File CatVariantsDir = new File(BaseTest.privateTestDir, "CatVariants"); + + private class CatVariantsTestProvider extends BaseTest.TestDataProvider { + private final File file1; + private final File file2; + public final File outputFile; + public final String md5; + + private CatVariantsTestProvider(final String file1, final String file2, final File outputFile, final String md5) { + super(CatVariantsTestProvider.class); + + this.file1 = new File(CatVariantsDir, file1); + this.file2 = new File(CatVariantsDir, file2); + this.outputFile = outputFile; + this.md5 = md5; + } + + public final String getCmdLine() { + return String.format("java -cp %s %s -R %s -V %s -V %s -out %s", + StringUtils.join(RuntimeUtils.getAbsoluteClassPaths(), File.pathSeparatorChar), + CatVariants.class.getCanonicalName(), BaseTest.b37KGReference, file1, file2, outputFile); + } + + public String toString() { + return "CatVariantsTestProvider " + outputFile; + } + } + + @DataProvider(name = "ExtensionsTest") + public Object[][] makeExtensionsTestProvider() { + new CatVariantsTestProvider("CatVariantsTest1.vcf", "CatVariantsTest2.vcf", BaseTest.createTempFile("CatVariantsTest", ".vcf"), "d0d81eb7fd3905256c4ac7c0fc480094"); + new CatVariantsTestProvider("CatVariantsTest1.bcf", "CatVariantsTest2.bcf", BaseTest.createTempFile("CatVariantsTest", ".bcf"), "6a57fcbbf3cae490896d13a288670d83"); + + for (String extension : AbstractFeatureReader.BLOCK_COMPRESSED_EXTENSIONS) + new CatVariantsTestProvider("CatVariantsTest1.vcf" + extension, "CatVariantsTest2.vcf" + extension, BaseTest.createTempFile("CatVariantsTest", ".vcf" + extension), "33f728ac5c70ce2994f3619a27f47088"); + + return CatVariantsTestProvider.getTests(CatVariantsTestProvider.class); + } + + @Test(dataProvider = "ExtensionsTest") + public void testExtensions(final CatVariantsTestProvider cfg) throws IOException { + + ProcessController pc = ProcessController.getThreadLocal(); + ProcessSettings ps = new ProcessSettings(cfg.getCmdLine().split("\\s+")); + pc.execAndCheck(ps); + + MD5DB.MD5Match result = md5db.testFileMD5("testExtensions", "CatVariantsTestProvider", cfg.outputFile, cfg.md5, false); + if(result.failed) { + final MD5Mismatch failure = new MD5Mismatch(result.actualMD5, result.expectedMD5, result.diffEngineOutput); + Assert.fail(failure.toString()); + } + } + + @Test(expectedExceptions = IOException.class) + public void testMismatchedExtensions1() throws IOException { + + String cmdLine = String.format("java -cp %s %s -R %s -V %s -V %s -out %s", + StringUtils.join(RuntimeUtils.getAbsoluteClassPaths(), File.pathSeparatorChar), + CatVariants.class.getCanonicalName(), + BaseTest.b37KGReference, + new File(CatVariantsDir, "CatVariantsTest1.vcf"), + new File(CatVariantsDir, "CatVariantsTest2.vcf"), + BaseTest.createTempFile("CatVariantsTest", ".bcf")); + + ProcessController pc = ProcessController.getThreadLocal(); + ProcessSettings ps = new ProcessSettings(cmdLine.split("\\s+")); + pc.execAndCheck(ps); + } + + @Test(expectedExceptions = IOException.class) + public void testMismatchedExtensions2() throws IOException { + + String cmdLine = String.format("java -cp %s %s -R %s -V %s -V %s -out %s", + StringUtils.join(RuntimeUtils.getAbsoluteClassPaths(), File.pathSeparatorChar), + CatVariants.class.getCanonicalName(), + BaseTest.b37KGReference, + new File(CatVariantsDir, "CatVariantsTest1.vcf"), + new File(CatVariantsDir, "CatVariantsTest2.bcf"), + BaseTest.createTempFile("CatVariantsTest", ".vcf")); + + ProcessController pc = ProcessController.getThreadLocal(); + ProcessSettings ps = new ProcessSettings(cmdLine.split("\\s+")); + pc.execAndCheck(ps); + } +} \ No newline at end of file diff --git a/public/gatk-framework/src/test/java/org/broadinstitute/sting/utils/variant/VCFIntegrationTest.java b/public/gatk-framework/src/test/java/org/broadinstitute/sting/utils/variant/VCFIntegrationTest.java index 379a2295b..7bd266151 100644 --- a/public/gatk-framework/src/test/java/org/broadinstitute/sting/utils/variant/VCFIntegrationTest.java +++ b/public/gatk-framework/src/test/java/org/broadinstitute/sting/utils/variant/VCFIntegrationTest.java @@ -25,6 +25,7 @@ package org.broadinstitute.sting.utils.variant; +import org.broad.tribble.AbstractFeatureReader; import org.broad.tribble.Tribble; import org.broad.tribble.index.AbstractIndex; import org.broad.tribble.index.ChrIndex; @@ -34,8 +35,8 @@ import org.broad.tribble.index.interval.IntervalTreeIndex; import org.broad.tribble.index.linear.LinearIndex; import org.broad.tribble.index.tabix.TabixIndex; import org.broad.tribble.util.TabixUtils; +import org.broadinstitute.sting.BaseTest; import org.broadinstitute.sting.WalkerTest; -import org.broadinstitute.sting.gatk.io.stubs.VCFWriterArgumentTypeDescriptor; import org.broadinstitute.variant.vcf.VCFCodec; import org.testng.Assert; import org.testng.TestException; @@ -45,7 +46,6 @@ import org.testng.annotations.Test; import java.io.File; import java.lang.reflect.Field; import java.util.Arrays; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; @@ -299,8 +299,8 @@ public class VCFIntegrationTest extends WalkerTest { @DataProvider(name = "BlockCompressedIndexDataProvider") public Object[][] blockCompressedIndexCreatorData() { - for (String suffix : VCFWriterArgumentTypeDescriptor.SUPPORTED_ZIPPED_SUFFIXES) - new BlockCompressedIndexCreatorTest(".vcf" + suffix); + for (final String extension : AbstractFeatureReader.BLOCK_COMPRESSED_EXTENSIONS) + new BlockCompressedIndexCreatorTest(".vcf" + extension); return TestDataProvider.getTests(BlockCompressedIndexCreatorTest.class); } @@ -330,4 +330,48 @@ public class VCFIntegrationTest extends WalkerTest { Assert.assertTrue(actualIndex instanceof TabixIndex, "testBlockCompressedIndexCreation: Want Tabix index but index is not Tabix: " + outTabixIdx); } + // + // + // Block-Compressed Input Tests + // + // + + private class BlockCompressedInputTest extends TestDataProvider { + private final String extension; + + private BlockCompressedInputTest(String extension) { + super(BlockCompressedInputTest.class); + + this.extension = extension; + } + + public String toString() { + return String.format("File extension %s", extension); + } + } + + @DataProvider(name = "BlockCompressedInputDataProvider") + public Object[][] blockCompressedInputData() { + for (final String extension : AbstractFeatureReader.BLOCK_COMPRESSED_EXTENSIONS) + new BlockCompressedInputTest(".vcf" + extension); + + return TestDataProvider.getTests(BlockCompressedInputTest.class); + } + + @Test(dataProvider = "BlockCompressedInputDataProvider") + public void testBlockCompressedInput(BlockCompressedInputTest testSpec) { + + File inputFile = new File(BaseTest.privateTestDir, "block_compressed_input_test" + testSpec.extension); + final String commandLine = " -T SelectVariants" + + " -R " + b37KGReference + + " --no_cmdline_in_header" + + " -V " + inputFile + + " -o %s "; + final String name = "testBlockCompressedInput: " + testSpec.toString(); + + final WalkerTestSpec spec = new WalkerTestSpec(commandLine, 1, Arrays.asList("3b60668bd973e43783d0406de80d2ed2")); + + executeTest(name, spec); + } + } diff --git a/public/repo/net/sf/picard/1.111.1902/picard-1.111.1902.jar b/public/repo/net/sf/picard/1.111.1920/picard-1.111.1920.jar similarity index 94% rename from public/repo/net/sf/picard/1.111.1902/picard-1.111.1902.jar rename to public/repo/net/sf/picard/1.111.1920/picard-1.111.1920.jar index 94c96b136..4933ed477 100644 Binary files a/public/repo/net/sf/picard/1.111.1902/picard-1.111.1902.jar and b/public/repo/net/sf/picard/1.111.1920/picard-1.111.1920.jar differ diff --git a/public/repo/net/sf/picard/1.111.1902/picard-1.111.1902.pom b/public/repo/net/sf/picard/1.111.1920/picard-1.111.1920.pom similarity index 90% rename from public/repo/net/sf/picard/1.111.1902/picard-1.111.1902.pom rename to public/repo/net/sf/picard/1.111.1920/picard-1.111.1920.pom index 00b43094e..14e5c9c88 100644 --- a/public/repo/net/sf/picard/1.111.1902/picard-1.111.1902.pom +++ b/public/repo/net/sf/picard/1.111.1920/picard-1.111.1920.pom @@ -3,23 +3,23 @@ 4.0.0 net.sf picard - 1.111.1902 + 1.111.1920 picard net.sf sam - 1.111.1902 + 1.111.1920 org.broadinstitute variant - 1.111.1902 + 1.111.1920 org.broad tribble - 1.111.1902 + 1.111.1920 diff --git a/public/repo/net/sf/sam/1.111.1902/sam-1.111.1902.jar b/public/repo/net/sf/sam/1.111.1920/sam-1.111.1920.jar similarity index 94% rename from public/repo/net/sf/sam/1.111.1902/sam-1.111.1902.jar rename to public/repo/net/sf/sam/1.111.1920/sam-1.111.1920.jar index fa4fd4c24..7f6839e1b 100644 Binary files a/public/repo/net/sf/sam/1.111.1902/sam-1.111.1902.jar and b/public/repo/net/sf/sam/1.111.1920/sam-1.111.1920.jar differ diff --git a/public/repo/net/sf/sam/1.111.1902/sam-1.111.1902.pom b/public/repo/net/sf/sam/1.111.1920/sam-1.111.1920.pom similarity index 95% rename from public/repo/net/sf/sam/1.111.1902/sam-1.111.1902.pom rename to public/repo/net/sf/sam/1.111.1920/sam-1.111.1920.pom index b2a77fb13..266aa93dd 100644 --- a/public/repo/net/sf/sam/1.111.1902/sam-1.111.1902.pom +++ b/public/repo/net/sf/sam/1.111.1920/sam-1.111.1920.pom @@ -3,7 +3,7 @@ 4.0.0 net.sf sam - 1.111.1902 + 1.111.1920 sam-jdk diff --git a/public/repo/org/broad/tribble/1.111.1902/tribble-1.111.1902.jar b/public/repo/org/broad/tribble/1.111.1920/tribble-1.111.1920.jar similarity index 93% rename from public/repo/org/broad/tribble/1.111.1902/tribble-1.111.1902.jar rename to public/repo/org/broad/tribble/1.111.1920/tribble-1.111.1920.jar index 1f4c9c2ac..804db912a 100644 Binary files a/public/repo/org/broad/tribble/1.111.1902/tribble-1.111.1902.jar and b/public/repo/org/broad/tribble/1.111.1920/tribble-1.111.1920.jar differ diff --git a/public/repo/org/broad/tribble/1.111.1902/tribble-1.111.1902.pom b/public/repo/org/broad/tribble/1.111.1920/tribble-1.111.1920.pom similarity index 87% rename from public/repo/org/broad/tribble/1.111.1902/tribble-1.111.1902.pom rename to public/repo/org/broad/tribble/1.111.1920/tribble-1.111.1920.pom index 2c84f11a5..af3cf6095 100644 --- a/public/repo/org/broad/tribble/1.111.1902/tribble-1.111.1902.pom +++ b/public/repo/org/broad/tribble/1.111.1920/tribble-1.111.1920.pom @@ -3,13 +3,13 @@ 4.0.0 org.broad tribble - 1.111.1902 + 1.111.1920 tribble net.sf sam - 1.111.1902 + 1.111.1920 diff --git a/public/repo/org/broadinstitute/variant/1.111.1902/variant-1.111.1902.jar b/public/repo/org/broadinstitute/variant/1.111.1920/variant-1.111.1920.jar similarity index 94% rename from public/repo/org/broadinstitute/variant/1.111.1902/variant-1.111.1902.jar rename to public/repo/org/broadinstitute/variant/1.111.1920/variant-1.111.1920.jar index 4e1779cdd..b0447f3df 100644 Binary files a/public/repo/org/broadinstitute/variant/1.111.1902/variant-1.111.1902.jar and b/public/repo/org/broadinstitute/variant/1.111.1920/variant-1.111.1920.jar differ diff --git a/public/repo/org/broadinstitute/variant/1.111.1902/variant-1.111.1902.pom b/public/repo/org/broadinstitute/variant/1.111.1920/variant-1.111.1920.pom similarity index 90% rename from public/repo/org/broadinstitute/variant/1.111.1902/variant-1.111.1902.pom rename to public/repo/org/broadinstitute/variant/1.111.1920/variant-1.111.1920.pom index a8c37196f..ca20e97d9 100644 --- a/public/repo/org/broadinstitute/variant/1.111.1902/variant-1.111.1902.pom +++ b/public/repo/org/broadinstitute/variant/1.111.1920/variant-1.111.1920.pom @@ -3,18 +3,18 @@ 4.0.0 org.broadinstitute variant - 1.111.1902 + 1.111.1920 variant org.broad tribble - 1.111.1902 + 1.111.1920 net.sf sam - 1.111.1902 + 1.111.1920 org.apache.commons diff --git a/public/sting-root/pom.xml b/public/sting-root/pom.xml index 946147f54..dcae77b99 100644 --- a/public/sting-root/pom.xml +++ b/public/sting-root/pom.xml @@ -43,7 +43,7 @@ -Xmx${test.maxmemory} -XX:+UseParallelOldGC -XX:ParallelGCThreads=${java.gc.threads} -XX:GCTimeLimit=${java.gc.timeLimit} -XX:GCHeapFreeLimit=${java.gc.heapFreeLimit} - 1.111.1902 + 1.111.1920 ${picard.public.version} ${picard.public.version} ${picard.public.version}