From 1d36d9fd691e0309fd88d21ed9509cdeba0a847f Mon Sep 17 00:00:00 2001 From: Ron Levine Date: Thu, 16 Feb 2017 10:09:55 -0500 Subject: [PATCH 1/2] Move htsjdk to ver 2.9.0 and picard to ver 2.8.3 --- .../VariantsToVCFIntegrationTest.java | 36 --------- .../reads/utilities/UnzipSingleBlock.java | 25 ++----- public/gatk-root/pom.xml | 4 +- .../utils/refdata/VariantContextAdaptors.java | 74 +------------------ .../utils/codecs/hapmap/HapMapUnitTest.java | 8 +- 5 files changed, 17 insertions(+), 130 deletions(-) diff --git a/protected/gatk-tools-protected/src/test/java/org/broadinstitute/gatk/tools/walkers/variantutils/VariantsToVCFIntegrationTest.java b/protected/gatk-tools-protected/src/test/java/org/broadinstitute/gatk/tools/walkers/variantutils/VariantsToVCFIntegrationTest.java index 71da7cf0c..7e50f87f9 100644 --- a/protected/gatk-tools-protected/src/test/java/org/broadinstitute/gatk/tools/walkers/variantutils/VariantsToVCFIntegrationTest.java +++ b/protected/gatk-tools-protected/src/test/java/org/broadinstitute/gatk/tools/walkers/variantutils/VariantsToVCFIntegrationTest.java @@ -67,42 +67,6 @@ import java.util.ArrayList; */ public class VariantsToVCFIntegrationTest extends WalkerTest { - @Test - public void testVariantsToVCFUsingGeliInput() { - List md5 = new ArrayList(); - md5.add("c73bcc3658b4a4d8bd9d794069d41dbd"); - - WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec( - "-R " + b36KGReference + - " --variant:GeliText " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.lod5.variants.geli.calls" + - " -T VariantsToVCF" + - " -L 1:10,000,000-11,000,000" + - " -sample NA123AB" + - " -o %s" + - " --no_cmdline_in_header", - 1, // just one output file - md5); - executeTest("testVariantsToVCFUsingGeliInput - calls", spec).getFirst(); - } - - @Test - public void testGenotypesToVCFUsingGeliInput() { - List md5 = new ArrayList(); - md5.add("a1771924b58dd633620114ef0f462acb"); - - WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec( - "-R " + b36KGReference + - " --variant:GeliText " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.lod5.genotypes.geli.calls" + - " -T VariantsToVCF" + - " -L 1:10,100,000-10,200,000" + - " -sample NA123AB" + - " -o %s" + - " --no_cmdline_in_header", - 1, // just one output file - md5); - executeTest("testVariantsToVCFUsingGeliInput - genotypes", spec).getFirst(); - } - @Test public void testGenotypesToVCFUsingHapMapInput() { List md5 = new ArrayList(); diff --git a/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/datasources/reads/utilities/UnzipSingleBlock.java b/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/datasources/reads/utilities/UnzipSingleBlock.java index d9538cce3..c65a7e4ce 100644 --- a/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/datasources/reads/utilities/UnzipSingleBlock.java +++ b/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/datasources/reads/utilities/UnzipSingleBlock.java @@ -25,15 +25,12 @@ package org.broadinstitute.gatk.engine.datasources.reads.utilities; -import htsjdk.samtools.util.BlockGunzipper; +import htsjdk.samtools.util.BlockCompressedInputStream; import org.broadinstitute.gatk.utils.commandline.CommandLineProgram; import org.broadinstitute.gatk.utils.commandline.Input; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; /** * Test decompression of a single BGZF block. @@ -45,20 +42,12 @@ public class UnzipSingleBlock extends CommandLineProgram { @Input(fullName = "compressed_block_size", shortName = "cbs", doc = "size of compressed block", required = true) private int compressedBufferSize; - public int execute() throws IOException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { - byte[] compressedBuffer = new byte[(int)blockFile.length()]; - byte[] uncompressedBuffer = new byte[65536]; + public int execute() throws IOException { + final byte[] uncompressedBuffer = new byte[65536]; - FileInputStream fis = new FileInputStream(blockFile); - fis.read(compressedBuffer); - fis.close(); - - BlockGunzipper gunzipper = new BlockGunzipper(); + final BlockCompressedInputStream gunzipper = new BlockCompressedInputStream(blockFile); gunzipper.setCheckCrcs(true); - Method unzipBlock = BlockGunzipper.class.getDeclaredMethod("unzipBlock",byte[].class,byte[].class,Integer.TYPE); - unzipBlock.setAccessible(true); - - unzipBlock.invoke(gunzipper,uncompressedBuffer,compressedBuffer,compressedBufferSize); + gunzipper.read(uncompressedBuffer); System.out.printf("SUCCESS!%n"); @@ -70,10 +59,10 @@ public class UnzipSingleBlock extends CommandLineProgram { * @param argv Command-line argument text. * @throws Exception on error. */ - public static void main(String[] argv) throws Exception { + public static void main(final String[] argv) throws Exception { int returnCode = 0; try { - UnzipSingleBlock instance = new UnzipSingleBlock(); + final UnzipSingleBlock instance = new UnzipSingleBlock(); start(instance, argv); returnCode = 0; } diff --git a/public/gatk-root/pom.xml b/public/gatk-root/pom.xml index 843f0cd8a..6d6365c83 100644 --- a/public/gatk-root/pom.xml +++ b/public/gatk-root/pom.xml @@ -44,8 +44,8 @@ org.testng.reporters.FailedReporter,org.testng.reporters.JUnitXMLReporter,org.broadinstitute.gatk.utils.TestNGTestTransformer,org.broadinstitute.gatk.utils.GATKTextReporter,org.uncommons.reportng.HTMLReporter - 2.8.1 - 2.7.2 + 2.9.0 + 2.8.3 diff --git a/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/refdata/VariantContextAdaptors.java b/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/refdata/VariantContextAdaptors.java index 868106bc6..d697682ab 100644 --- a/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/refdata/VariantContextAdaptors.java +++ b/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/refdata/VariantContextAdaptors.java @@ -25,15 +25,10 @@ package org.broadinstitute.gatk.utils.refdata; -import htsjdk.samtools.util.SequenceUtil; import htsjdk.tribble.Feature; -import htsjdk.tribble.annotation.Strand; -import htsjdk.tribble.gelitext.GeliTextFeature; import org.broadinstitute.gatk.utils.contexts.ReferenceContext; -import org.broadinstitute.gatk.utils.GenomeLoc; import org.broadinstitute.gatk.utils.classloader.PluginManager; import org.broadinstitute.gatk.utils.codecs.hapmap.RawHapMapFeature; -import org.broadinstitute.gatk.utils.variant.GATKVariantContextUtils; import htsjdk.variant.variantcontext.*; import java.util.*; @@ -59,10 +54,10 @@ public class VariantContextAdaptors { // // -------------------------------------------------------------------------------------------------------------- - private static Map,VCAdaptor> adaptors = new HashMap,VCAdaptor>(); + private static Map,VCAdaptor> adaptors = new HashMap<>(); static { - PluginManager vcAdaptorManager = new PluginManager(VCAdaptor.class); + PluginManager vcAdaptorManager = new PluginManager<>(VCAdaptor.class); List adaptorInstances = vcAdaptorManager.createAllTypes(); for(VCAdaptor adaptor: adaptorInstances) adaptors.put(adaptor.getAdaptableFeatureType(),adaptor); @@ -110,67 +105,6 @@ public class VariantContextAdaptors { } } - - // -------------------------------------------------------------------------------------------------------------- - // - // GELI to VariantContext - // - // -------------------------------------------------------------------------------------------------------------- - - private static class GeliTextAdaptor implements VCAdaptor { - /** - * Converts Geli text records to VariantContext. - * @return GeliTextFeature. - */ - @Override - public Class getAdaptableFeatureType() { return GeliTextFeature.class; } - - /** - * convert to a Variant Context, given: - * @param name the name of the ROD - * @param input the Rod object, in this case a RodGeliText - * @param ref the reference context - * @return a VariantContext object - */ - @Override - public VariantContext convert(String name, Object input, ReferenceContext ref) { - GeliTextFeature geli = (GeliTextFeature)input; - if ( ! Allele.acceptableAlleleBases(String.valueOf(geli.getRefBase())) ) - return null; - Allele refAllele = Allele.create(String.valueOf(geli.getRefBase()), true); - - // make sure we can convert it - if ( geli.getGenotype().isHet() || !geli.getGenotype().containsBase(geli.getRefBase())) { - // add the reference allele - List alleles = new ArrayList(); - List genotypeAlleles = new ArrayList(); - // add all of the alt alleles - for ( char alt : geli.getGenotype().toString().toCharArray() ) { - if ( ! Allele.acceptableAlleleBases(String.valueOf(alt)) ) { - return null; - } - Allele allele = Allele.create(String.valueOf(alt), false); - if (!alleles.contains(allele) && !refAllele.basesMatch(allele.getBases())) alleles.add(allele); - - // add the allele, first checking if it's reference or not - if (!refAllele.basesMatch(allele.getBases())) genotypeAlleles.add(allele); - else genotypeAlleles.add(refAllele); - } - - Map attributes = new HashMap(); - Collection genotypes = new ArrayList(); - Genotype call = GenotypeBuilder.create(name, genotypeAlleles); - - // add the call to the genotype list, and then use this list to create a VariantContext - genotypes.add(call); - alleles.add(refAllele); - GenomeLoc loc = ref.getGenomeLocParser().createGenomeLoc(geli.getChr(),geli.getStart()); - return new VariantContextBuilder(name, loc.getContig(), loc.getStart(), loc.getStop(), alleles).genotypes(genotypes).log10PError(-1 * geli.getLODBestToReference()).attributes(attributes).make(); - } else - return null; // can't handle anything else - } - } - // -------------------------------------------------------------------------------------------------------------- // // HapMap to VariantContext @@ -203,7 +137,7 @@ public class VariantContextAdaptors { if ( index < 0 ) return null; // we weren't given enough reference context to create the VariantContext - HashSet alleles = new HashSet(); + HashSet alleles = new HashSet<>(); Allele refSNPAllele = Allele.create(ref.getBase(), true); int deletionLength = -1; @@ -231,7 +165,7 @@ public class VariantContextAdaptors { String a1 = genotypeStrings[i].substring(0,1); String a2 = genotypeStrings[i].substring(1); - ArrayList myAlleles = new ArrayList(2); + ArrayList myAlleles = new ArrayList<>(2); // use the mapping to actual alleles, if available if ( alleleMap != null ) { diff --git a/public/gatk-utils/src/test/java/org/broadinstitute/gatk/utils/codecs/hapmap/HapMapUnitTest.java b/public/gatk-utils/src/test/java/org/broadinstitute/gatk/utils/codecs/hapmap/HapMapUnitTest.java index f41f728ec..e79dfc22e 100644 --- a/public/gatk-utils/src/test/java/org/broadinstitute/gatk/utils/codecs/hapmap/HapMapUnitTest.java +++ b/public/gatk-utils/src/test/java/org/broadinstitute/gatk/utils/codecs/hapmap/HapMapUnitTest.java @@ -28,7 +28,7 @@ package org.broadinstitute.gatk.utils.codecs.hapmap; import htsjdk.tribble.annotation.Strand; import htsjdk.tribble.readers.LineIterator; import htsjdk.tribble.readers.LineIteratorImpl; -import htsjdk.tribble.readers.LineReaderUtil; +import htsjdk.tribble.readers.SynchronousLineReader; import htsjdk.tribble.readers.PositionalBufferedStream; import org.broadinstitute.gatk.utils.BaseTest; import org.testng.Assert; @@ -160,11 +160,11 @@ public class HapMapUnitTest extends BaseTest { Assert.assertFalse(codec.canDecode("filename." + RawHapMapCodec.FILE_EXT + EXTRA_CHAR)); Assert.assertFalse(codec.canDecode("filename" + RawHapMapCodec.FILE_EXT)); } - - + public LineIterator getLineIterator() { try { - return new LineIteratorImpl(LineReaderUtil.fromBufferedStream(new PositionalBufferedStream(new FileInputStream(hapMapFile)))); + + return new LineIteratorImpl(new SynchronousLineReader(new PositionalBufferedStream(new FileInputStream(hapMapFile)))); } catch (FileNotFoundException e) { Assert.fail("Unable to open hapmap file : " + hapMapFile); } From ab75b08ea7f9353e2b0fec0e52ff8dbbbee7db8c Mon Sep 17 00:00:00 2001 From: Ron Levine Date: Fri, 17 Feb 2017 12:21:51 -0500 Subject: [PATCH 2/2] Remove unused compressed_block_size and close BlockCompressedInputStream --- .../engine/datasources/reads/utilities/UnzipSingleBlock.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/datasources/reads/utilities/UnzipSingleBlock.java b/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/datasources/reads/utilities/UnzipSingleBlock.java index c65a7e4ce..662babdf2 100644 --- a/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/datasources/reads/utilities/UnzipSingleBlock.java +++ b/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/datasources/reads/utilities/UnzipSingleBlock.java @@ -39,15 +39,13 @@ public class UnzipSingleBlock extends CommandLineProgram { @Input(fullName = "block_file", shortName = "b", doc = "block file over which to test unzipping", required = true) private File blockFile; - @Input(fullName = "compressed_block_size", shortName = "cbs", doc = "size of compressed block", required = true) - private int compressedBufferSize; - public int execute() throws IOException { final byte[] uncompressedBuffer = new byte[65536]; final BlockCompressedInputStream gunzipper = new BlockCompressedInputStream(blockFile); gunzipper.setCheckCrcs(true); gunzipper.read(uncompressedBuffer); + gunzipper.close(); System.out.printf("SUCCESS!%n");