Merge branch 'master' of ssh://gsa1/humgen/gsa-scr1/gsa-engineering/git/stable

This commit is contained in:
Mark DePristo 2011-10-17 13:38:00 -04:00
commit 09a09cacef
4 changed files with 249 additions and 72 deletions

158
build.xml
View File

@ -67,6 +67,8 @@
<!-- If running the 'package' task, this property controls the name of the xml file to package -->
<property name="package.xml.dir" value="${public.dir}/packages" />
<property name="package.output.dir" value="${dist.dir}/packages" />
<property name="staging.dir" value="staging" />
<property name="executable" value="GenomeAnalysisTK" />
<property environment="env"/>
@ -118,6 +120,16 @@
</fileset>
</path>
<path id="testng.dependencies">
<fileset dir="${lib.dir}">
<include name="jcommander*.jar"/>
<include name="guice*.jar"/>
<include name="junit*.jar"/>
<include name="bsh*.jar"/>
<include name="snakeyaml*.jar"/>
</fileset>
</path>
<path id="build.results">
<fileset dir="dist">
<patternset refid="dependency.mask" />
@ -507,7 +519,7 @@
docletpathref="doclet.classpath"
classpathref="external.dependencies"
classpath="${java.classes}"
additionalparam="${gatkdocs.include.hidden.arg} -private -build-timestamp &quot;${build.timestamp}&quot; -absolute-version ${build.version} -quiet -J-Xdebug -J-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"> <!-- -test to only do DocumentationTest walker -->
additionalparam="${gatkdocs.include.hidden.arg} -private -build-timestamp &quot;${build.timestamp}&quot; -absolute-version ${build.version} -quiet"> <!-- -test to only do DocumentationTest walker -->
<sourcefiles>
<union>
<fileset refid="all.java.source.files"/>
@ -827,10 +839,54 @@
<target name="test.compile" depends="init.usecontracts,test.java.compile,test.scala.compile" />
<path id="testng.default.classpath">
<path refid="external.dependencies" />
<pathelement location="${java.classes}" />
<pathelement location="${scala.classes}" />
<pathelement location="${java.contracts}" />
<pathelement location="${java.public.test.classes}" />
<pathelement location="${java.private.test.classes}" />
<pathelement location="${scala.public.test.classes}" />
<pathelement location="${scala.private.test.classes}" />
</path>
<path id="testng.gatk.releasetest.classpath">
<path refid="testng.dependencies" />
<path>
<fileset dir="${package.output.dir}">
<include name="**/${executable}.jar" />
</fileset>
</path>
<pathelement location="${java.contracts}" />
<pathelement location="${java.public.test.classes}" />
<pathelement location="${scala.public.test.classes}" />
<pathelement location="${scala.classes}" />
<path>
<fileset dir="${lib.dir}">
<include name="scala*.jar"/>
<include name="jna*.jar"/>
</fileset>
</path>
</path>
<path id="testng.queue.releasetest.classpath">
<path refid="testng.dependencies" />
<path>
<fileset dir="${package.output.dir}">
<include name="**/${executable}.jar" />
</fileset>
</path>
<pathelement location="${java.contracts}" />
<pathelement location="${java.public.test.classes}" />
<pathelement location="${scala.public.test.classes}" />
</path>
<!-- TEST -->
<macrodef name="run-test">
<attribute name="testtype"/>
<attribute name="outputdir"/>
<attribute name="classpath"/>
<attribute name="runfailed"/>
<sequential>
@ -851,6 +907,7 @@
<echo message="Sting: Running @{testtype} test cases!"/>
<taskdef resource="testngtasks" classpath="${testng.jar}"/>
<testng outputDir="@{outputdir}"
classpathref="@{classpath}"
haltOnFailure="false" failureProperty="test.failure"
verbose="2"
workingDir="${basedir}"
@ -863,16 +920,6 @@
<jvmarg line="${cofoja.jvm.args}"/>
<!-- <jvmarg value="-Xdebug"/> -->
<!-- <jvmarg value="-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005"/> -->
<classpath>
<path refid="external.dependencies" />
<pathelement location="${java.classes}" />
<pathelement location="${scala.classes}" />
<pathelement location="${java.contracts}" />
<pathelement location="${java.public.test.classes}" />
<pathelement location="${java.private.test.classes}" />
<pathelement location="${scala.public.test.classes}" />
<pathelement location="${scala.private.test.classes}" />
</classpath>
<classfileset dir="${java.public.test.classes}" includes="**/@{testtype}.class"/>
<classfileset dir="${java.private.test.classes}" erroronmissingdir="false">
@ -902,6 +949,10 @@
</sequential>
</macrodef>
<target name="test.init">
<property name="testng.classpath" value="testng.default.classpath" />
</target>
<target name="alltests">
<antcall target="test" inheritAll="false"/>
<antcall target="integrationtest" inheritAll="false"/>
@ -914,62 +965,81 @@
<antcall target="pipelinetest.public" inheritAll="false"/>
</target>
<target name="alltests.gatk.packagejar" depends="init.buildpublic,package">
<antcall target="test.public" inheritAll="false">
<param name="testng.classpath" value="testng.gatk.releasetest.classpath" />
</antcall>
<antcall target="integrationtest.public" inheritAll="false">
<param name="testng.classpath" value="testng.gatk.releasetest.classpath" />
</antcall>
</target>
<target name="alltests.queue.packagejar" depends="init.buildpublic,queue">
<antcall target="package" inheritAll="false">
<param name="executable" value="Queue" />
</antcall>
<antcall target="pipelinetest.public" inheritAll="false">
<param name="executable" value="Queue" />
<param name="testng.classpath" value="testng.queue.releasetest.classpath" />
</antcall>
</target>
<!-- Our four different test conditions: Test, IntegrationTest, PerformanceTest, PipelineTest -->
<target name="test" depends="init.buildall,test.compile,vcf.jar" description="Run unit tests">
<target name="test" depends="init.buildall,test.compile,vcf.jar,test.init" description="Run unit tests">
<condition property="ttype" value="*UnitTest" else="${single}">
<not><isset property="single"/></not>
</condition>
<run-test testtype="${ttype}" outputdir="${report}/${ttype}" runfailed="false"/>
<run-test testtype="${ttype}" outputdir="${report}/${ttype}" classpath="${testng.classpath}" runfailed="false"/>
</target>
<target name="test.public" depends="init.buildpublic,test"/>
<target name="integrationtest" depends="init.buildall,test.compile" description="Run integration tests">
<target name="integrationtest" depends="init.buildall,test.compile,test.init" description="Run integration tests">
<condition property="itype" value="*IntegrationTest" else="${single}">
<not><isset property="single"/></not>
</condition>
<run-test testtype="${itype}" outputdir="${report}/${itype}" runfailed="false"/>
<run-test testtype="${itype}" outputdir="${report}/${itype}" classpath="${testng.classpath}" runfailed="false"/>
</target>
<target name="integrationtest.public" depends="init.buildpublic,integrationtest"/>
<target name="performancetest" depends="init.buildall,test.compile" description="Run performance tests">
<target name="performancetest" depends="init.buildall,test.compile,test.init" description="Run performance tests">
<condition property="ptype" value="*PerformanceTest" else="${single}">
<not><isset property="single"/></not>
</condition>
<run-test testtype="${ptype}" outputdir="${report}/${ptype}" runfailed="false"/>
<run-test testtype="${ptype}" outputdir="${report}/${ptype}" classpath="${testng.classpath}" runfailed="false"/>
</target>
<target name="performancetest.public" depends="init.buildpublic,performancetest" />
<target name="pipelinetest" depends="init.buildall,test.compile" description="Run pipeline tests">
<target name="pipelinetest" depends="init.buildall,test.compile,test.init" description="Run pipeline tests">
<condition property="pipetype" value="*PipelineTest" else="${single}">
<not><isset property="single"/></not>
</condition>
<run-test testtype="${pipetype}" outputdir="${report}/${pipetype}" runfailed="false"/>
<run-test testtype="${pipetype}" outputdir="${report}/${pipetype}" classpath="${testng.classpath}" runfailed="false"/>
</target>
<target name="pipelinetest.public" depends="init.buildpublic,pipelinetest" />
<target name="pipelinetestrun" depends="init.buildall,test.compile" description="Run pipeline tests">
<target name="pipelinetestrun" depends="init.buildall,test.compile,test.init" description="Run pipeline tests">
<property name="pipeline.run" value="run"/>
<condition property="pipetype" value="*PipelineTest" else="${single}">
<not><isset property="single"/></not>
</condition>
<run-test testtype="${pipetype}" outputdir="${report}/${pipetype}" runfailed="false"/>
<run-test testtype="${pipetype}" outputdir="${report}/${pipetype}" classpath="${testng.classpath}" runfailed="false"/>
</target>
<target name="pipelinetestrun.public" depends="init.buildpublic,pipelinetestrun" />
<target name="failed-test" depends="init.buildall,test.compile">
<run-test testtype="${report}/*UnitTest/testng-failed.xml" outputdir="${report}/failed_rerun" runfailed="true"/>
<target name="failed-test" depends="init.buildall,test.compile,test.init">
<run-test testtype="${report}/*UnitTest/testng-failed.xml" outputdir="${report}/failed_rerun" classpath="${testng.classpath}" runfailed="true"/>
</target>
<target name="failed-integration" depends="init.buildall,test.compile">
<run-test testtype="${report}/*IntegrationTest/testng-failed.xml" outputdir="${report}/failed_rerun" runfailed="true"/>
<target name="failed-integration" depends="init.buildall,test.compile,test.init">
<run-test testtype="${report}/*IntegrationTest/testng-failed.xml" outputdir="${report}/failed_rerun" classpath="${testng.classpath}" runfailed="true"/>
</target>
<target name="failed-performance" depends="init.buildall,test.compile">
<run-test testtype="${report}/*PerformanceTest/testng-failed.xml" outputdir="${report}/failed_rerun" runfailed="true"/>
<target name="failed-performance" depends="init.buildall,test.compile,test.init">
<run-test testtype="${report}/*PerformanceTest/testng-failed.xml" outputdir="${report}/failed_rerun" classpath="${testng.classpath}" runfailed="true"/>
</target>
<target name="failed-pipeline" depends="init.buildall,test.compile">
<run-test testtype="${report}/*PipelineTest/testng-failed.xml" outputdir="${report}/failed_rerun" runfailed="true"/>
<target name="failed-pipeline" depends="init.buildall,test.compile,test.init">
<run-test testtype="${report}/*PipelineTest/testng-failed.xml" outputdir="${report}/failed_rerun" classpath="${testng.classpath}" runfailed="true"/>
</target>
<!-- ******************************************************************************** -->
@ -1027,14 +1097,14 @@
<!-- Unzip all classes from their current locations and assemble them in a staging directory -->
<target name="stage" description="stage files for distribution">
<mkdir dir="staging"/>
<mkdir dir="${staging.dir}"/>
<!--
HACK: Create the edu directory before EDU on case-insensitive mac filesystems.
The ivy dependency colt -> concurrent contains an EDU.oswego package which
BCEL doesn't even pull in but messes up edu.mit.broad.
-->
<mkdir dir="staging/edu"/>
<unjar dest="staging" overwrite="false">
<mkdir dir="${staging.dir}/edu"/>
<unjar dest="${staging.dir}" overwrite="false">
<fileset dir="${dist.dir}">
<patternset refid="dependency.mask" />
</fileset>
@ -1043,8 +1113,8 @@
<!-- Build a package consisting of all supporting files -->
<target name="package" depends="dist,stage" description="bundle up an executable for distribution">
<mkdir dir="dist/packages" />
<xslt destdir="dist/packages" style="${package.xml.dir}/CreatePackager.xsl" useImplicitFileset="false">
<mkdir dir="${package.output.dir}" />
<xslt destdir="${package.output.dir}" style="${package.xml.dir}/CreatePackager.xsl" useImplicitFileset="false">
<flattenmapper/>
<fileset dir="${package.xml.dir}">
<include name="${executable}.xml" />
@ -1055,11 +1125,11 @@
</fileset>
</xslt>
<ant antfile="${dist.dir}/packages/${executable}.xml" target="package" />
<ant antfile="${package.output.dir}/${executable}.xml" target="package" />
</target>
<target name="release" depends="package" description="release a build, putting each file in a location specified by the package">
<ant antfile="${dist.dir}/packages/${executable}.xml" target="release" />
<ant antfile="${package.output.dir}/${executable}.xml" target="release" />
</target>
<!-- Build a subset of picard with only those classes we need by completely abusing the packaging system -->
@ -1068,7 +1138,7 @@
<!-- Build out a classpath -->
<pathconvert property="required.picard.jars" pathsep=":">
<fileset dir="${basedir}">
<include name="staging" />
<include name="${staging.dir}" />
<include name="${lib.dir}/picard-*.*.*.jar" />
<include name="${lib.dir}/sam-*.jar" />
</fileset>
@ -1076,14 +1146,14 @@
<echo message="required.picard.jars=${required.picard.jars}" />
<!-- Stage picard-private -->
<delete dir="staging" />
<mkdir dir="staging" />
<unjar src="${picard.dist.dir}/picard-private.jar" dest="staging" overwrite="true" />
<delete dir="${staging.dir}" />
<mkdir dir="${staging.dir}" />
<unjar src="${picard.dist.dir}/picard-private.jar" dest="${staging.dir}" overwrite="true" />
<!-- Use the packaging system to extract parts of picard-private we need -->
<mkdir dir="${dist.dir}/packages" />
<xslt in="${package.xml.dir}/PicardPrivate.xml" out="${dist.dir}/packages/BuildPicardPrivate.xml" style="${package.xml.dir}/CreatePackager.xsl" />
<ant antfile="${dist.dir}/packages/BuildPicardPrivate.xml">
<mkdir dir="${package.output.dir}" />
<xslt in="${package.xml.dir}/PicardPrivate.xml" out="${package.output.dir}/BuildPicardPrivate.xml" style="${package.xml.dir}/CreatePackager.xsl" />
<ant antfile="${package.output.dir}/BuildPicardPrivate.xml">
<property name="additional.jars" value="${required.picard.jars}" />
</ant>
</target>
@ -1119,7 +1189,7 @@
<delete dir="${build.dir}"/>
<delete dir="${lib.dir}"/>
<delete dir="dump"/>
<delete dir="staging"/>
<delete dir="${staging.dir}"/>
<delete dir="${dist.dir}"/>
<delete dir="pipelinetests"/>
</target>

View File

@ -0,0 +1,132 @@
/*
* Copyright (c) 2010 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
* THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.walkers.qc;
import org.broadinstitute.sting.commandline.Argument;
import org.broadinstitute.sting.commandline.ArgumentCollection;
import org.broadinstitute.sting.commandline.Output;
import org.broadinstitute.sting.gatk.arguments.DbsnpArgumentCollection;
import org.broadinstitute.sting.gatk.arguments.StandardVariantContextInputArgumentCollection;
import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
import org.broadinstitute.sting.gatk.refdata.VariantContextAdaptors;
import org.broadinstitute.sting.gatk.walkers.Reference;
import org.broadinstitute.sting.gatk.walkers.RodWalker;
import org.broadinstitute.sting.gatk.walkers.Window;
import org.broadinstitute.sting.utils.codecs.vcf.VCFWriter;
import org.broadinstitute.sting.utils.variantcontext.VariantContext;
import java.io.PrintStream;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
/**
* Test routine for new VariantContext object
*/
@Reference(window=@Window(start=-20,stop=1))
public class TestVariantContextWalker extends RodWalker<Integer, Integer> {
@Output
PrintStream out;
@ArgumentCollection
protected StandardVariantContextInputArgumentCollection variantCollection = new StandardVariantContextInputArgumentCollection();
@Argument(fullName="takeFirstOnly", doc="Only take the first second at a locus, as opposed to all", required=false)
boolean takeFirstOnly = false;
@Argument(fullName="onlyContextsOfType", doc="Only take variant contexts of this type", required=false)
VariantContext.Type onlyOfThisType = null;
@Argument(fullName="onlyContextsStartinAtCurrentPosition", doc="Only take variant contexts at actually start at the current position, excluding those at span to the current location but start earlier", required=false)
boolean onlyContextsStartinAtCurrentPosition = false;
@Argument(fullName="printPerLocus", doc="If true, we'll print the variant contexts, in addition to counts", required=false)
boolean printContexts = false;
@Argument(fullName="outputVCF", doc="If provided, we'll convert the first input context into a VCF", required=false)
VCFWriter writer = null;
private boolean wroteHeader = false;
public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
if ( ref == null )
return 0;
else {
EnumSet<VariantContext.Type> allowedTypes = onlyOfThisType == null ? null : EnumSet.of(onlyOfThisType);
int n = 0;
List<VariantContext> contexts;
if ( onlyContextsStartinAtCurrentPosition )
contexts = tracker.getValues(variantCollection.variants, context.getLocation());
else // ! onlyContextsStartinAtCurrentPosition
contexts = tracker.getValues(variantCollection.variants);
for ( VariantContext vc : contexts ) {
if ( allowedTypes == null || allowedTypes.contains(vc.getType()) ) {
// we need to trigger decoding of the genotype string to pass integration tests
vc.getGenotypes();
if ( writer != null && n == 0 ) {
if ( ! wroteHeader ) {
writer.writeHeader(VariantContextAdaptors.createVCFHeader(null, vc));
wroteHeader = true;
}
writer.add(vc);
}
n++;
if ( printContexts ) out.printf(" %s%n", vc);
if ( takeFirstOnly ) break;
}
}
if ( n > 0 && printContexts ) {
out.printf("%s => had %d variant context objects%n", context.getLocation(), n);
out.printf("---------------------------------------------%n");
}
return n;
}
}
public Integer reduceInit() {
return 0;
}
public Integer reduce(Integer point, Integer sum) {
return point + sum;
}
@Override
public void onTraversalDone(Integer result) {
// Double check traversal result to make count is the same.
// TODO: Is this check necessary?
out.println("[REDUCE RESULT] Traversal result is: " + result);
}
}

View File

@ -26,21 +26,12 @@ package org.broadinstitute.sting.utils.R;
import org.apache.commons.io.FileUtils;
import org.broadinstitute.sting.BaseTest;
import org.broadinstitute.sting.gatk.walkers.diffengine.DiffElement;
import org.broadinstitute.sting.gatk.walkers.diffengine.DiffEngine;
import org.broadinstitute.sting.gatk.walkers.diffengine.DiffNode;
import org.broadinstitute.sting.gatk.walkers.diffengine.Difference;
import org.broadinstitute.sting.utils.exceptions.UserException;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
/**
@ -49,7 +40,6 @@ import java.util.List;
public class RScriptExecutorUnitTest extends BaseTest {
final static String testrscript = "print(\"hello, world\")\n";
final static String publicRScript = "plot_Tranches.R";
final static String privateRScript = "variantCallQC.R";
// --------------------------------------------------------------------------------
//
@ -74,17 +64,6 @@ public class RScriptExecutorUnitTest extends BaseTest {
@Test
public void testPublic() { testOne(publicRScript, null, null, true); }
@Test
public void testPrivate() { testOne(privateRScript, null, null, true); }
// make sure we don't break finding something in private by adding another directory
@Test
public void testPrivateWithAdditionalPath1() { testOne(privateRScript, null, "dist", true); }
// make sure we don't break finding something in private by adding another directory
@Test
public void testPrivateWithAdditionalPath2() { testOne(privateRScript, null, "doesNotExist", true); }
@Test(expectedExceptions = UserException.class)
public void testNonExistantScriptException() { testOne("does_not_exist.R", null, null, true); }

View File

@ -26,14 +26,10 @@
<package name="GATKEngine">
<executable name="GATKEngine">
<dependencies>
<!-- Core walkers -->
<package name="org.broadinstitute.sting.gatk.walkers.**" />
<!-- All non-oneoff GATK-specific RODs -->
<package name="org.broadinstitute.sting.utils.codecs.**" />
<!-- Filters -->
<package name="org.broadinstitute.sting.gatk.filters" />
<!-- Package everything in org.broadinstitute.sting -->
<package name="org.broadinstitute.sting.**" />
<!-- Tribble codecs -->
<package name="org.broad.tribble.*" />
<package name="org.broad.tribble.**" />
<!-- Workaround - depend on the logger impl required by JEXL -->
<package name="org.apache.commons.logging.impl" />
</dependencies>