diff --git a/build.xml b/build.xml index ec4d091e7..446982a44 100644 --- a/build.xml +++ b/build.xml @@ -67,6 +67,8 @@ + + @@ -118,6 +120,16 @@ + + + + + + + + + + @@ -509,7 +521,7 @@ docletpathref="doclet.classpath" classpathref="external.dependencies" classpath="${java.classes}" - additionalparam="${gatkdocs.include.hidden.arg} -private -build-timestamp "${build.timestamp}" -absolute-version ${build.version} -quiet -J-Xdebug -J-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"> + additionalparam="${gatkdocs.include.hidden.arg} -private -build-timestamp "${build.timestamp}" -absolute-version ${build.version} -quiet"> @@ -829,10 +841,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -853,6 +909,7 @@ - - - - - - - - - - @@ -904,6 +951,10 @@ + + + + @@ -916,62 +967,81 @@ + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + @@ -1029,14 +1099,14 @@ - + - - + + @@ -1045,8 +1115,8 @@ - - + + @@ -1057,11 +1127,11 @@ - + - + @@ -1070,7 +1140,7 @@ - + @@ -1078,14 +1148,14 @@ - - - + + + - - - + + + @@ -1121,7 +1191,7 @@ - + diff --git a/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java b/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java index 5fff8f609..d1d9cf7fe 100644 --- a/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java +++ b/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java @@ -379,8 +379,10 @@ class RodBindingArgumentTypeDescriptor extends ArgumentTypeDescriptor { } if ( tribbleType == null ) - if ( ! file.canRead() | ! file.isFile() ) { - throw new UserException.BadArgumentValue(name, "Couldn't read file to determine type: " + file); + if ( ! file.exists() ) { + throw new UserException.CouldNotReadInputFile(file, "file does not exist"); + } else if ( ! file.canRead() || ! file.isFile() ) { + throw new UserException.CouldNotReadInputFile(file, "file could not be read"); } else { throw new UserException.CommandLineException( String.format("No tribble type was provided on the command line and the type of the file could not be determined dynamically. " + diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java index 81d0c36ac..5dbd3f5fd 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java @@ -27,6 +27,7 @@ package org.broadinstitute.sting.gatk.walkers.variantutils; import org.broadinstitute.sting.commandline.*; import org.broadinstitute.sting.gatk.arguments.StandardVariantContextInputArgumentCollection; import org.broadinstitute.sting.utils.MathUtils; +import org.broadinstitute.sting.utils.variantcontext.Allele; import org.broadinstitute.sting.utils.variantcontext.VariantContext; import org.broadinstitute.sting.gatk.contexts.AlignmentContext; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; @@ -133,7 +134,7 @@ public class VariantsToTable extends RodWalker { /** * By default, this tool throws a UserException when it encounters a field without a value in some record. This - * is generally useful when you mistype -F CHROM, so that you get a friendly warning about CHRMO not being + * is generally useful when you mistype -F CHROM, so that you get a friendly warning about CHROM not being * found before the tool runs through 40M 1000G records. However, in some cases you genuinely want to allow such * fields (e.g., AC not being calculated for filtered records, if included). When provided, this argument * will cause VariantsToTable to write out NA values for missing fields instead of throwing an error. @@ -294,6 +295,14 @@ public class VariantsToTable extends RodWalker { return x.toString(); } }); + getters.put("EVENTLENGTH", new Getter() { public String get(VariantContext vc) { + int maxLength = 0; + for ( final Allele a : vc.getAlternateAlleles() ) { + final int length = a.length() - vc.getReference().length(); + if( Math.abs(length) > Math.abs(maxLength) ) { maxLength = length; } + } + return Integer.toString(maxLength); + }}); getters.put("QUAL", new Getter() { public String get(VariantContext vc) { return Double.toString(vc.getPhredScaledQual()); } }); getters.put("TRANSITION", new Getter() { public String get(VariantContext vc) { if ( vc.isSNP() && vc.isBiallelic() ) @@ -304,7 +313,7 @@ public class VariantsToTable extends RodWalker { getters.put("FILTER", new Getter() { public String get(VariantContext vc) { return vc.isNotFiltered() ? "PASS" : Utils.join(",", vc.getFilters()); } }); - + getters.put("ID", new Getter() { public String get(VariantContext vc) { return vc.hasID() ? vc.getID() : "."; } }); getters.put("HET", new Getter() { public String get(VariantContext vc) { return Integer.toString(vc.getHetCount()); } }); getters.put("HOM-REF", new Getter() { public String get(VariantContext vc) { return Integer.toString(vc.getHomRefCount()); } }); getters.put("HOM-VAR", new Getter() { public String get(VariantContext vc) { return Integer.toString(vc.getHomVarCount()); } }); diff --git a/public/java/src/org/broadinstitute/sting/utils/SimpleTimer.java b/public/java/src/org/broadinstitute/sting/utils/SimpleTimer.java index 342087b41..a5ac10250 100644 --- a/public/java/src/org/broadinstitute/sting/utils/SimpleTimer.java +++ b/public/java/src/org/broadinstitute/sting/utils/SimpleTimer.java @@ -46,7 +46,7 @@ public class SimpleTimer { * @return the name associated with this timer */ @Ensures("result != null") - public String getName() { + public synchronized String getName() { return name; } @@ -58,7 +58,7 @@ public class SimpleTimer { */ @Requires("running == false") @Ensures({"result != null", "elapsed == 0l"}) - public SimpleTimer start() { + public synchronized SimpleTimer start() { elapsed = 0l; restart(); return this; @@ -73,7 +73,7 @@ public class SimpleTimer { */ @Requires("running == false") @Ensures("result != null") - public SimpleTimer restart() { + public synchronized SimpleTimer restart() { running = true; startTime = currentTime(); return this; @@ -82,14 +82,14 @@ public class SimpleTimer { /** * @return is this timer running? */ - public boolean isRunning() { + public synchronized boolean isRunning() { return running; } /** * @return A convenience function to obtain the current time in milliseconds from this timer */ - public long currentTime() { + public synchronized long currentTime() { return System.currentTimeMillis(); } @@ -101,7 +101,7 @@ public class SimpleTimer { */ @Requires("running == true") @Ensures({"result != null", "elapsed >= old(elapsed)", "running == false"}) - public SimpleTimer stop() { + public synchronized SimpleTimer stop() { running = false; elapsed += currentTime() - startTime; return this; @@ -116,12 +116,7 @@ public class SimpleTimer { @Ensures({ "result >= (elapsed/1000.0)", "result >= 0"}) - public double getElapsedTime() { + public synchronized double getElapsedTime() { return (running ? (currentTime() - startTime + elapsed) : elapsed) / 1000.0; } - - - public void printElapsedTime(PrintStream out) { - out.printf("SimpleTimer %s: %.2f%n", name, getElapsedTime()); - } } diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java index c86b91b79..6b101ca74 100755 --- a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java +++ b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java @@ -381,7 +381,8 @@ public abstract class AbstractVCFCodec implements FeatureCodec, NameAwareCodec, } } - attributes.put(VariantContext.ID_KEY, id); + if ( ! id.equals(VCFConstants.EMPTY_ID_FIELD) ) + attributes.put(VariantContext.ID_KEY, id); return attributes; } diff --git a/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java b/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java index 1dea726ae..a1f75be41 100755 --- a/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java +++ b/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java @@ -111,6 +111,10 @@ public class UserException extends ReviewedStingException { super(String.format("Couldn't read file because %s caused by %s", message, e.getMessage())); } + public CouldNotReadInputFile(File file) { + super(String.format("Couldn't read file %s", file.getAbsolutePath())); + } + public CouldNotReadInputFile(File file, String message) { super(String.format("Couldn't read file %s because %s", file.getAbsolutePath(), message)); } diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/AlignmentUtils.java b/public/java/src/org/broadinstitute/sting/utils/sam/AlignmentUtils.java index 344eccb83..b8e892101 100644 --- a/public/java/src/org/broadinstitute/sting/utils/sam/AlignmentUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/AlignmentUtils.java @@ -353,7 +353,7 @@ public class AlignmentUtils { break; case D: case N: - refPos++; + refPos += elementLength; break; case M: for ( int jjj = 0; jjj < elementLength; jjj++ ) { diff --git a/public/java/test/org/broadinstitute/sting/MD5DB.java b/public/java/test/org/broadinstitute/sting/MD5DB.java index 374a9f8da..c9f53c581 100644 --- a/public/java/test/org/broadinstitute/sting/MD5DB.java +++ b/public/java/test/org/broadinstitute/sting/MD5DB.java @@ -239,7 +239,7 @@ public class MD5DB { if ( ! expectedMD5.equals(filemd5sum) ) { // we are going to fail for real in assertEquals (so we are counted by the testing framework). // prepare ourselves for the comparison - System.out.printf("##### Test %s is going fail #####%n", name); + System.out.printf("##### Test %s is going to fail #####%n", name); String pathToExpectedMD5File = getMD5FilePath(expectedMD5, "[No DB file found]"); String pathToFileMD5File = getMD5FilePath(filemd5sum, "[No DB file found]"); System.out.printf("##### Path to expected file (MD5=%s): %s%n", expectedMD5, pathToExpectedMD5File); diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/qc/TestVariantContextWalker.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/qc/TestVariantContextWalker.java new file mode 100755 index 000000000..7607049db --- /dev/null +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/qc/TestVariantContextWalker.java @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2010 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR + * THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.walkers.qc; + +import org.broadinstitute.sting.commandline.Argument; +import org.broadinstitute.sting.commandline.ArgumentCollection; +import org.broadinstitute.sting.commandline.Output; +import org.broadinstitute.sting.gatk.arguments.DbsnpArgumentCollection; +import org.broadinstitute.sting.gatk.arguments.StandardVariantContextInputArgumentCollection; +import org.broadinstitute.sting.gatk.contexts.AlignmentContext; +import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; +import org.broadinstitute.sting.gatk.refdata.VariantContextAdaptors; +import org.broadinstitute.sting.gatk.walkers.Reference; +import org.broadinstitute.sting.gatk.walkers.RodWalker; +import org.broadinstitute.sting.gatk.walkers.Window; +import org.broadinstitute.sting.utils.codecs.vcf.VCFWriter; +import org.broadinstitute.sting.utils.variantcontext.VariantContext; + +import java.io.PrintStream; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.List; + +/** + * Test routine for new VariantContext object + */ +@Reference(window=@Window(start=-20,stop=1)) +public class TestVariantContextWalker extends RodWalker { + @Output + PrintStream out; + + @ArgumentCollection + protected StandardVariantContextInputArgumentCollection variantCollection = new StandardVariantContextInputArgumentCollection(); + + @Argument(fullName="takeFirstOnly", doc="Only take the first second at a locus, as opposed to all", required=false) + boolean takeFirstOnly = false; + + @Argument(fullName="onlyContextsOfType", doc="Only take variant contexts of this type", required=false) + VariantContext.Type onlyOfThisType = null; + + @Argument(fullName="onlyContextsStartinAtCurrentPosition", doc="Only take variant contexts at actually start at the current position, excluding those at span to the current location but start earlier", required=false) + boolean onlyContextsStartinAtCurrentPosition = false; + + @Argument(fullName="printPerLocus", doc="If true, we'll print the variant contexts, in addition to counts", required=false) + boolean printContexts = false; + + @Argument(fullName="outputVCF", doc="If provided, we'll convert the first input context into a VCF", required=false) + VCFWriter writer = null; + + private boolean wroteHeader = false; + + public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) { + if ( ref == null ) + return 0; + else { + EnumSet allowedTypes = onlyOfThisType == null ? null : EnumSet.of(onlyOfThisType); + + int n = 0; + List contexts; + if ( onlyContextsStartinAtCurrentPosition ) + contexts = tracker.getValues(variantCollection.variants, context.getLocation()); + else // ! onlyContextsStartinAtCurrentPosition + contexts = tracker.getValues(variantCollection.variants); + + for ( VariantContext vc : contexts ) { + if ( allowedTypes == null || allowedTypes.contains(vc.getType()) ) { + // we need to trigger decoding of the genotype string to pass integration tests + vc.getGenotypes(); + + if ( writer != null && n == 0 ) { + if ( ! wroteHeader ) { + writer.writeHeader(VariantContextAdaptors.createVCFHeader(null, vc)); + wroteHeader = true; + } + + writer.add(vc); + } + + n++; + if ( printContexts ) out.printf(" %s%n", vc); + if ( takeFirstOnly ) break; + } + } + + if ( n > 0 && printContexts ) { + out.printf("%s => had %d variant context objects%n", context.getLocation(), n); + out.printf("---------------------------------------------%n"); + } + + return n; + } + } + + public Integer reduceInit() { + return 0; + } + + public Integer reduce(Integer point, Integer sum) { + return point + sum; + } + + @Override + public void onTraversalDone(Integer result) { + // Double check traversal result to make count is the same. + // TODO: Is this check necessary? + out.println("[REDUCE RESULT] Traversal result is: " + result); + } +} diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationWalkersIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationWalkersIntegrationTest.java index b2833b935..cbcd5835f 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationWalkersIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationWalkersIntegrationTest.java @@ -89,7 +89,7 @@ public class RecalibrationWalkersIntegrationTest extends WalkerTest { @DataProvider(name = "trtestdata") public Object[][] createTRTestData() { new TRTest( validationDataLocation + "NA12892.SLX.SRP000031.2009_06.selected.bam", "2864f231fab7030377f3c8826796e48f" ); - new TRTest( validationDataLocation + "NA19240.chr1.BFAST.SOLID.bam", "c164dd635721ba6df3f06dac1877c32d"); + new TRTest( validationDataLocation + "NA19240.chr1.BFAST.SOLID.bam", "d04cf1f6df486e45226ebfbf93a188a5"); new TRTest( validationDataLocation + "NA12873.454.SRP000031.2009_06.chr1.10_20mb.bam", "74314e5562c1a65547bb0edaacffe602" ); new TRTest( validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.allTechs.bam", "2a37c6001826bfabf87063b1dfcf594f" ); return TRTest.getTests(TRTest.class); @@ -200,7 +200,7 @@ public class RecalibrationWalkersIntegrationTest extends WalkerTest { @Test public void testTableRecalibratorSolidIndelsRemoveRefBias() { HashMap e = new HashMap(); - e.put( validationDataLocation + "NA19240.chr1.BFAST.SOLID.bam", "7d5edb75b176e4151de225f699719ee4" ); + e.put( validationDataLocation + "NA19240.chr1.BFAST.SOLID.bam", "2ad4c17ac3ed380071137e4e53a398a5" ); for ( Map.Entry entry : e.entrySet() ) { String bam = entry.getKey(); diff --git a/public/java/test/org/broadinstitute/sting/utils/R/RScriptExecutorUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/R/RScriptExecutorUnitTest.java index 1bbf74db9..836a4473f 100644 --- a/public/java/test/org/broadinstitute/sting/utils/R/RScriptExecutorUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/R/RScriptExecutorUnitTest.java @@ -26,21 +26,12 @@ package org.broadinstitute.sting.utils.R; import org.apache.commons.io.FileUtils; import org.broadinstitute.sting.BaseTest; -import org.broadinstitute.sting.gatk.walkers.diffengine.DiffElement; -import org.broadinstitute.sting.gatk.walkers.diffengine.DiffEngine; -import org.broadinstitute.sting.gatk.walkers.diffengine.DiffNode; -import org.broadinstitute.sting.gatk.walkers.diffengine.Difference; import org.broadinstitute.sting.utils.exceptions.UserException; -import org.testng.Assert; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.File; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.List; /** @@ -49,7 +40,6 @@ import java.util.List; public class RScriptExecutorUnitTest extends BaseTest { final static String testrscript = "print(\"hello, world\")\n"; final static String publicRScript = "plot_Tranches.R"; - final static String privateRScript = "variantCallQC.R"; // -------------------------------------------------------------------------------- // @@ -74,17 +64,6 @@ public class RScriptExecutorUnitTest extends BaseTest { @Test public void testPublic() { testOne(publicRScript, null, null, true); } - @Test - public void testPrivate() { testOne(privateRScript, null, null, true); } - - // make sure we don't break finding something in private by adding another directory - @Test - public void testPrivateWithAdditionalPath1() { testOne(privateRScript, null, "dist", true); } - - // make sure we don't break finding something in private by adding another directory - @Test - public void testPrivateWithAdditionalPath2() { testOne(privateRScript, null, "doesNotExist", true); } - @Test(expectedExceptions = UserException.class) public void testNonExistantScriptException() { testOne("does_not_exist.R", null, null, true); } diff --git a/public/packages/GATKEngine.xml b/public/packages/GATKEngine.xml index 4364988e7..e6056408e 100644 --- a/public/packages/GATKEngine.xml +++ b/public/packages/GATKEngine.xml @@ -26,14 +26,10 @@ - - - - - - + + - + diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/VcfGatherFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/VcfGatherFunction.scala index d70022147..70046c913 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/VcfGatherFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/VcfGatherFunction.scala @@ -36,8 +36,6 @@ class VcfGatherFunction extends CombineVariants with GatherFunction { private lazy val originalGATK = this.originalFunction.asInstanceOf[CommandLineGATK] override def freezeFieldValues { - this.memoryLimit = Some(1) - this.jarFile = this.originalGATK.jarFile this.reference_sequence = this.originalGATK.reference_sequence this.intervals = this.originalGATK.intervals