Merge branch 'master' of ssh://gsa1/humgen/gsa-scr1/gsa-engineering/git/stable

build.xml

@@ -67,6 +67,8 @@
 
     <!-- If running the 'package' task, this property controls the name of the xml file to package -->
     <property name="package.xml.dir" value="${public.dir}/packages" />
+    <property name="package.output.dir" value="${dist.dir}/packages" />
+    <property name="staging.dir" value="staging" />
     <property name="executable" value="GenomeAnalysisTK" />
 
     <property environment="env"/>
@@ -118,6 +120,16 @@
       </fileset>
     </path>
 
+    <path id="testng.dependencies">
+        <fileset dir="${lib.dir}">
+            <include name="jcommander*.jar"/>
+            <include name="guice*.jar"/>
+            <include name="junit*.jar"/>
+            <include name="bsh*.jar"/>
+            <include name="snakeyaml*.jar"/>
+        </fileset>
+    </path>
+
     <path id="build.results">
       <fileset dir="dist">
         <patternset refid="dependency.mask" />
@@ -507,7 +519,7 @@
                  docletpathref="doclet.classpath"
                  classpathref="external.dependencies"
                  classpath="${java.classes}"
-                 additionalparam="${gatkdocs.include.hidden.arg} -private -build-timestamp &quot;${build.timestamp}&quot; -absolute-version ${build.version} -quiet -J-Xdebug -J-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"> <!-- -test to only do DocumentationTest walker -->
+                 additionalparam="${gatkdocs.include.hidden.arg} -private -build-timestamp &quot;${build.timestamp}&quot; -absolute-version ${build.version} -quiet"> <!-- -test to only do DocumentationTest walker -->
             <sourcefiles>
                 <union>
                     <fileset refid="all.java.source.files"/>
@@ -827,10 +839,54 @@
 
     <target name="test.compile" depends="init.usecontracts,test.java.compile,test.scala.compile" />
 
+
+    <path id="testng.default.classpath">
+        <path refid="external.dependencies" />
+        <pathelement location="${java.classes}" />
+        <pathelement location="${scala.classes}" />
+        <pathelement location="${java.contracts}" />
+        <pathelement location="${java.public.test.classes}" />
+        <pathelement location="${java.private.test.classes}" />
+        <pathelement location="${scala.public.test.classes}" />
+        <pathelement location="${scala.private.test.classes}" />
+    </path>
+
+    <path id="testng.gatk.releasetest.classpath">
+        <path refid="testng.dependencies" />
+        <path>
+            <fileset dir="${package.output.dir}">
+                <include name="**/${executable}.jar" />
+            </fileset>
+        </path>
+        <pathelement location="${java.contracts}" />
+        <pathelement location="${java.public.test.classes}" />
+        <pathelement location="${scala.public.test.classes}" />
+        <pathelement location="${scala.classes}" />
+        <path>
+            <fileset dir="${lib.dir}">
+                <include name="scala*.jar"/>
+                <include name="jna*.jar"/>
+            </fileset>
+        </path>
+    </path>
+
+    <path id="testng.queue.releasetest.classpath">
+        <path refid="testng.dependencies" />
+        <path>
+            <fileset dir="${package.output.dir}">
+                <include name="**/${executable}.jar" />
+            </fileset>
+        </path>
+        <pathelement location="${java.contracts}" />
+        <pathelement location="${java.public.test.classes}" />
+        <pathelement location="${scala.public.test.classes}" />
+    </path>
+
     <!-- TEST -->
     <macrodef name="run-test">
         <attribute name="testtype"/>
         <attribute name="outputdir"/>
+        <attribute name="classpath"/>
         <attribute name="runfailed"/>
 
         <sequential>
@@ -851,6 +907,7 @@
             <echo message="Sting: Running @{testtype} test cases!"/>
             <taskdef resource="testngtasks" classpath="${testng.jar}"/>
             <testng outputDir="@{outputdir}"
+                    classpathref="@{classpath}"
                     haltOnFailure="false" failureProperty="test.failure"
                     verbose="2"
                     workingDir="${basedir}"
@@ -863,16 +920,6 @@
                 <jvmarg line="${cofoja.jvm.args}"/>
 <!--                 <jvmarg value="-Xdebug"/> -->
 <!--                 <jvmarg value="-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005"/> -->
-                <classpath>
-                    <path refid="external.dependencies" />
-                    <pathelement location="${java.classes}" />
-                    <pathelement location="${scala.classes}" />
-                    <pathelement location="${java.contracts}" />
-                    <pathelement location="${java.public.test.classes}" />
-                    <pathelement location="${java.private.test.classes}" />
-                    <pathelement location="${scala.public.test.classes}" />
-                    <pathelement location="${scala.private.test.classes}" />
-                </classpath>
 
                 <classfileset dir="${java.public.test.classes}" includes="**/@{testtype}.class"/>
                 <classfileset dir="${java.private.test.classes}" erroronmissingdir="false">
@@ -902,6 +949,10 @@
         </sequential>
     </macrodef>
 
+    <target name="test.init">
+        <property name="testng.classpath" value="testng.default.classpath" />
+    </target>
+
     <target name="alltests">
         <antcall target="test" inheritAll="false"/>
         <antcall target="integrationtest" inheritAll="false"/>
@@ -914,62 +965,81 @@
         <antcall target="pipelinetest.public" inheritAll="false"/>
     </target>
 
+    <target name="alltests.gatk.packagejar" depends="init.buildpublic,package">
+        <antcall target="test.public" inheritAll="false">
+            <param name="testng.classpath" value="testng.gatk.releasetest.classpath" />
+        </antcall>
+        <antcall target="integrationtest.public" inheritAll="false">
+            <param name="testng.classpath" value="testng.gatk.releasetest.classpath" />
+        </antcall>
+    </target>
+
+    <target name="alltests.queue.packagejar" depends="init.buildpublic,queue">
+        <antcall target="package" inheritAll="false">
+            <param name="executable" value="Queue" />
+        </antcall>
+        <antcall target="pipelinetest.public" inheritAll="false">
+            <param name="executable" value="Queue" />
+            <param name="testng.classpath" value="testng.queue.releasetest.classpath" />
+        </antcall>
+    </target>
+
     <!-- Our four different test conditions: Test, IntegrationTest, PerformanceTest, PipelineTest -->
-    <target name="test" depends="init.buildall,test.compile,vcf.jar" description="Run unit tests">
+    <target name="test" depends="init.buildall,test.compile,vcf.jar,test.init" description="Run unit tests">
         <condition property="ttype" value="*UnitTest" else="${single}">
             <not><isset property="single"/></not>
         </condition>
-        <run-test testtype="${ttype}" outputdir="${report}/${ttype}" runfailed="false"/>
+        <run-test testtype="${ttype}" outputdir="${report}/${ttype}" classpath="${testng.classpath}" runfailed="false"/>
     </target>
     <target name="test.public" depends="init.buildpublic,test"/>
 
-    <target name="integrationtest" depends="init.buildall,test.compile" description="Run integration tests">
+    <target name="integrationtest" depends="init.buildall,test.compile,test.init" description="Run integration tests">
         <condition property="itype" value="*IntegrationTest" else="${single}">
             <not><isset property="single"/></not>
         </condition>
-        <run-test testtype="${itype}" outputdir="${report}/${itype}" runfailed="false"/>
+        <run-test testtype="${itype}" outputdir="${report}/${itype}" classpath="${testng.classpath}" runfailed="false"/>
     </target>
     <target name="integrationtest.public" depends="init.buildpublic,integrationtest"/>
 
-    <target name="performancetest" depends="init.buildall,test.compile" description="Run performance tests">
+    <target name="performancetest" depends="init.buildall,test.compile,test.init" description="Run performance tests">
        <condition property="ptype" value="*PerformanceTest" else="${single}">
             <not><isset property="single"/></not>
         </condition>
-        <run-test testtype="${ptype}" outputdir="${report}/${ptype}" runfailed="false"/>
+        <run-test testtype="${ptype}" outputdir="${report}/${ptype}" classpath="${testng.classpath}" runfailed="false"/>
     </target>
     <target name="performancetest.public" depends="init.buildpublic,performancetest" />
 
-    <target name="pipelinetest" depends="init.buildall,test.compile" description="Run pipeline tests">
+    <target name="pipelinetest" depends="init.buildall,test.compile,test.init" description="Run pipeline tests">
         <condition property="pipetype" value="*PipelineTest" else="${single}">
             <not><isset property="single"/></not>
         </condition>
-        <run-test testtype="${pipetype}" outputdir="${report}/${pipetype}" runfailed="false"/>
+        <run-test testtype="${pipetype}" outputdir="${report}/${pipetype}" classpath="${testng.classpath}" runfailed="false"/>
     </target>
     <target name="pipelinetest.public" depends="init.buildpublic,pipelinetest" />
 
-    <target name="pipelinetestrun" depends="init.buildall,test.compile" description="Run pipeline tests">
+    <target name="pipelinetestrun" depends="init.buildall,test.compile,test.init" description="Run pipeline tests">
         <property name="pipeline.run" value="run"/>
         <condition property="pipetype" value="*PipelineTest" else="${single}">
             <not><isset property="single"/></not>
         </condition>
-        <run-test testtype="${pipetype}" outputdir="${report}/${pipetype}" runfailed="false"/>
+        <run-test testtype="${pipetype}" outputdir="${report}/${pipetype}" classpath="${testng.classpath}" runfailed="false"/>
     </target>
     <target name="pipelinetestrun.public" depends="init.buildpublic,pipelinetestrun" />
 
-    <target name="failed-test" depends="init.buildall,test.compile">
+    <target name="failed-test" depends="init.buildall,test.compile,test.init">
-        <run-test testtype="${report}/*UnitTest/testng-failed.xml" outputdir="${report}/failed_rerun" runfailed="true"/>
+        <run-test testtype="${report}/*UnitTest/testng-failed.xml" outputdir="${report}/failed_rerun" classpath="${testng.classpath}" runfailed="true"/>
     </target>
 
-    <target name="failed-integration" depends="init.buildall,test.compile">
+    <target name="failed-integration" depends="init.buildall,test.compile,test.init">
-        <run-test testtype="${report}/*IntegrationTest/testng-failed.xml" outputdir="${report}/failed_rerun" runfailed="true"/>
+        <run-test testtype="${report}/*IntegrationTest/testng-failed.xml" outputdir="${report}/failed_rerun" classpath="${testng.classpath}" runfailed="true"/>
     </target>
 
-    <target name="failed-performance" depends="init.buildall,test.compile">
+    <target name="failed-performance" depends="init.buildall,test.compile,test.init">
-        <run-test testtype="${report}/*PerformanceTest/testng-failed.xml" outputdir="${report}/failed_rerun" runfailed="true"/>
+        <run-test testtype="${report}/*PerformanceTest/testng-failed.xml" outputdir="${report}/failed_rerun" classpath="${testng.classpath}" runfailed="true"/>
     </target>
 
-    <target name="failed-pipeline" depends="init.buildall,test.compile">
+    <target name="failed-pipeline" depends="init.buildall,test.compile,test.init">
-        <run-test testtype="${report}/*PipelineTest/testng-failed.xml" outputdir="${report}/failed_rerun" runfailed="true"/>
+        <run-test testtype="${report}/*PipelineTest/testng-failed.xml" outputdir="${report}/failed_rerun" classpath="${testng.classpath}" runfailed="true"/>
     </target>
 
     <!-- ******************************************************************************** -->
@@ -1027,14 +1097,14 @@
 
     <!-- Unzip all classes from their current locations and assemble them in a staging directory -->
     <target name="stage" description="stage files for distribution">
-        <mkdir dir="staging"/>
+        <mkdir dir="${staging.dir}"/>
         <!--
             HACK: Create the edu directory before EDU on case-insensitive mac filesystems.
             The ivy dependency colt -> concurrent contains an EDU.oswego package which
             BCEL doesn't even pull in but messes up edu.mit.broad.
           -->
-        <mkdir dir="staging/edu"/>
+        <mkdir dir="${staging.dir}/edu"/>
-        <unjar dest="staging" overwrite="false">
+        <unjar dest="${staging.dir}" overwrite="false">
           <fileset dir="${dist.dir}">
             <patternset refid="dependency.mask" />
           </fileset>
@@ -1043,8 +1113,8 @@
 
     <!-- Build a package consisting of all supporting files -->
     <target name="package" depends="dist,stage" description="bundle up an executable for distribution">
-      <mkdir dir="dist/packages" />
+      <mkdir dir="${package.output.dir}" />
-      <xslt destdir="dist/packages" style="${package.xml.dir}/CreatePackager.xsl" useImplicitFileset="false">
+      <xslt destdir="${package.output.dir}" style="${package.xml.dir}/CreatePackager.xsl" useImplicitFileset="false">
           <flattenmapper/>
           <fileset dir="${package.xml.dir}">
               <include name="${executable}.xml" />
@@ -1055,11 +1125,11 @@
           </fileset>
 
       </xslt>
-      <ant antfile="${dist.dir}/packages/${executable}.xml" target="package" />
+      <ant antfile="${package.output.dir}/${executable}.xml" target="package" />
     </target>
 
     <target name="release" depends="package" description="release a build, putting each file in a location specified by the package">
-      <ant antfile="${dist.dir}/packages/${executable}.xml" target="release" />      
+      <ant antfile="${package.output.dir}/${executable}.xml" target="release" />
     </target>
 
     <!-- Build a subset of picard with only those classes we need by completely abusing the packaging system -->
@@ -1068,7 +1138,7 @@
       <!-- Build out a classpath -->
       <pathconvert property="required.picard.jars" pathsep=":">
         <fileset dir="${basedir}">
-	     <include name="staging" />
+	     <include name="${staging.dir}" />
 	     <include name="${lib.dir}/picard-*.*.*.jar" />
 	     <include name="${lib.dir}/sam-*.jar" />
 	   </fileset>
@@ -1076,14 +1146,14 @@
       <echo message="required.picard.jars=${required.picard.jars}" />
 
       <!-- Stage picard-private -->
-      <delete dir="staging" />
+      <delete dir="${staging.dir}" />
-      <mkdir dir="staging" />
+      <mkdir dir="${staging.dir}" />
-      <unjar src="${picard.dist.dir}/picard-private.jar" dest="staging" overwrite="true" />
+      <unjar src="${picard.dist.dir}/picard-private.jar" dest="${staging.dir}" overwrite="true" />
 
       <!-- Use the packaging system to extract parts of picard-private we need -->
-      <mkdir dir="${dist.dir}/packages" />
+      <mkdir dir="${package.output.dir}" />
-      <xslt in="${package.xml.dir}/PicardPrivate.xml" out="${dist.dir}/packages/BuildPicardPrivate.xml" style="${package.xml.dir}/CreatePackager.xsl" />
+      <xslt in="${package.xml.dir}/PicardPrivate.xml" out="${package.output.dir}/BuildPicardPrivate.xml" style="${package.xml.dir}/CreatePackager.xsl" />
-      <ant antfile="${dist.dir}/packages/BuildPicardPrivate.xml">
+      <ant antfile="${package.output.dir}/BuildPicardPrivate.xml">
 	    <property name="additional.jars" value="${required.picard.jars}" />
       </ant>
     </target>
@@ -1119,7 +1189,7 @@
         <delete dir="${build.dir}"/>
         <delete dir="${lib.dir}"/>
         <delete dir="dump"/>
-        <delete dir="staging"/>
+        <delete dir="${staging.dir}"/>
         <delete dir="${dist.dir}"/>
         <delete dir="pipelinetests"/>
     </target>

public/java/test/org/broadinstitute/sting/gatk/walkers/qc/TestVariantContextWalker.java

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2010 The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+ * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.gatk.walkers.qc;
+
+import org.broadinstitute.sting.commandline.Argument;
+import org.broadinstitute.sting.commandline.ArgumentCollection;
+import org.broadinstitute.sting.commandline.Output;
+import org.broadinstitute.sting.gatk.arguments.DbsnpArgumentCollection;
+import org.broadinstitute.sting.gatk.arguments.StandardVariantContextInputArgumentCollection;
+import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
+import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
+import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
+import org.broadinstitute.sting.gatk.refdata.VariantContextAdaptors;
+import org.broadinstitute.sting.gatk.walkers.Reference;
+import org.broadinstitute.sting.gatk.walkers.RodWalker;
+import org.broadinstitute.sting.gatk.walkers.Window;
+import org.broadinstitute.sting.utils.codecs.vcf.VCFWriter;
+import org.broadinstitute.sting.utils.variantcontext.VariantContext;
+
+import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.List;
+
+/**
+ * Test routine for new VariantContext object
+ */
+@Reference(window=@Window(start=-20,stop=1))
+public class TestVariantContextWalker extends RodWalker<Integer, Integer> {
+    @Output
+    PrintStream out;
+
+    @ArgumentCollection
+    protected StandardVariantContextInputArgumentCollection variantCollection = new StandardVariantContextInputArgumentCollection();
+
+    @Argument(fullName="takeFirstOnly", doc="Only take the first second at a locus, as opposed to all", required=false)
+    boolean takeFirstOnly = false;
+
+    @Argument(fullName="onlyContextsOfType", doc="Only take variant contexts of this type", required=false)
+    VariantContext.Type onlyOfThisType = null;
+
+    @Argument(fullName="onlyContextsStartinAtCurrentPosition", doc="Only take variant contexts at actually start at the current position, excluding those at span to the current location but start earlier", required=false)
+    boolean onlyContextsStartinAtCurrentPosition = false;
+
+    @Argument(fullName="printPerLocus", doc="If true, we'll print the variant contexts, in addition to counts", required=false)
+    boolean printContexts = false;
+
+    @Argument(fullName="outputVCF", doc="If provided, we'll convert the first input context into a VCF", required=false)
+    VCFWriter writer = null;
+
+    private boolean wroteHeader = false;
+
+    public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
+        if ( ref == null )
+            return 0;
+        else {
+            EnumSet<VariantContext.Type> allowedTypes = onlyOfThisType == null ? null : EnumSet.of(onlyOfThisType);
+
+            int n = 0;
+            List<VariantContext> contexts;
+            if ( onlyContextsStartinAtCurrentPosition )
+                contexts = tracker.getValues(variantCollection.variants, context.getLocation());
+            else // ! onlyContextsStartinAtCurrentPosition
+                contexts = tracker.getValues(variantCollection.variants);
+
+            for ( VariantContext vc : contexts ) {
+                if ( allowedTypes == null || allowedTypes.contains(vc.getType()) ) {
+                    // we need to trigger decoding of the genotype string to pass integration tests
+                    vc.getGenotypes();
+
+                    if ( writer != null && n == 0 ) {
+                        if ( ! wroteHeader ) {
+                            writer.writeHeader(VariantContextAdaptors.createVCFHeader(null, vc));
+                            wroteHeader = true;
+                        }
+
+                        writer.add(vc);
+                    }
+
+                    n++;
+                    if ( printContexts ) out.printf("       %s%n", vc);
+                    if ( takeFirstOnly ) break;
+                }
+            }
+
+            if ( n > 0 && printContexts ) {
+                out.printf("%s => had %d variant context objects%n", context.getLocation(), n);
+                out.printf("---------------------------------------------%n");
+            }
+            
+            return n;
+        }
+    }
+
+    public Integer reduceInit() {
+        return 0;
+    }
+
+    public Integer reduce(Integer point, Integer sum) {
+        return point + sum;
+    }
+
+    @Override
+    public void onTraversalDone(Integer result) {
+        // Double check traversal result to make count is the same.
+        // TODO: Is this check necessary?
+        out.println("[REDUCE RESULT] Traversal result is: " + result);
+    }
+}

public/java/test/org/broadinstitute/sting/utils/R/RScriptExecutorUnitTest.java

@@ -26,21 +26,12 @@ package org.broadinstitute.sting.utils.R;
 
 import org.apache.commons.io.FileUtils;
 import org.broadinstitute.sting.BaseTest;
-import org.broadinstitute.sting.gatk.walkers.diffengine.DiffElement;
-import org.broadinstitute.sting.gatk.walkers.diffengine.DiffEngine;
-import org.broadinstitute.sting.gatk.walkers.diffengine.DiffNode;
-import org.broadinstitute.sting.gatk.walkers.diffengine.Difference;
 import org.broadinstitute.sting.utils.exceptions.UserException;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 
 /**
@@ -49,7 +40,6 @@ import java.util.List;
 public class RScriptExecutorUnitTest extends BaseTest {
     final static String testrscript = "print(\"hello, world\")\n";
     final static String publicRScript = "plot_Tranches.R";
-    final static String privateRScript = "variantCallQC.R";
 
     // --------------------------------------------------------------------------------
     //
@@ -74,17 +64,6 @@ public class RScriptExecutorUnitTest extends BaseTest {
     @Test
     public void testPublic() { testOne(publicRScript, null, null, true); }
 
-    @Test
-    public void testPrivate() { testOne(privateRScript, null, null, true); }
-
-    // make sure we don't break finding something in private by adding another directory
-    @Test
-    public void testPrivateWithAdditionalPath1() { testOne(privateRScript, null, "dist", true); }
-
-    // make sure we don't break finding something in private by adding another directory
-    @Test
-    public void testPrivateWithAdditionalPath2() { testOne(privateRScript, null, "doesNotExist", true); }
-
     @Test(expectedExceptions = UserException.class)
     public void testNonExistantScriptException() { testOne("does_not_exist.R", null, null, true); }
 

public/packages/GATKEngine.xml

@@ -26,14 +26,10 @@
 <package name="GATKEngine">
   <executable name="GATKEngine">
     <dependencies>
-      <!-- Core walkers -->
+      <!-- Package everything in org.broadinstitute.sting -->
-      <package name="org.broadinstitute.sting.gatk.walkers.**" />
+      <package name="org.broadinstitute.sting.**" />
-      <!-- All non-oneoff GATK-specific RODs -->
-      <package name="org.broadinstitute.sting.utils.codecs.**" />
-      <!-- Filters -->
-      <package name="org.broadinstitute.sting.gatk.filters" />
       <!-- Tribble codecs -->
-      <package name="org.broad.tribble.*" />
+      <package name="org.broad.tribble.**" />
       <!-- Workaround - depend on the logger impl required by JEXL -->
       <package name="org.apache.commons.logging.impl" />
     </dependencies>