zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added a performance test build option (for the upcoming performance test suite), and added a sample performance test for VariantEval. 

IMPORTANT: it was really redundant that we had -Dsingle and -Dsingleintegration to run single unit tests and integration tests, now you can just use -Dsingle to run a single test for performance, unit, and integration tests.

git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@3136 348d0f76-0448-11de-a6fe-93d51630548a
This commit is contained in:
aaron 2010-04-08 15:37:15 +00:00
parent 4014a8a674
commit 9f6377f7fb
3 changed files with 277 additions and 225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
build.xml
View File

@ -8,9 +8,6 @@
<property name="dist.dir" value="dist" />
<property name="resource.file" value="StingText.properties" />
<property name="single" value="*Test" />
<property name="singleintegration" value="*IntegrationTest" />
<!-- do we want to halt on failure of a junit test? default to yes (Bamboo uses 'no') -->
<property name="halt" value="yes" />
@ -297,7 +294,9 @@
</jar>
</target>
<!-- ***************************************************************************** -->
<!-- *********** Tests and associated tasks ********* -->
<!-- ***************************************************************************** -->
<!-- where to put reports and tests-->
<property name="report" value="${build.dir}/report"/>
<property name="test.classes" value="${build.dir}/testclasses"/>
@ -307,67 +306,63 @@
<property name="test.maxmemory" value="2g"/>
<!-- TEST -->
<target name="test" depends="test.compile,agent" description="Run unit tests">
<mkdir dir="${report}"/>
<echo message="Sting: Running test cases!"/>
<junit printsummary="yes" showoutput="yes" maxmemory="${test.maxmemory}" clonevm="yes" haltonfailure="${halt}" failureProperty="test.failure">
<jvmarg value="-javaagent:${build.dir}/TestAgent.jar"/>
<formatter type="${testOutput}" usefile="${usefile}" />
<classpath>
<pathelement location="build"/>
<path refid="runtime.dependencies"/>
<pathelement location="${test.classes}"/>
<pathelement location="lib/junit-4.4.jar"/>
</classpath>
<macrodef name="run-test">
<attribute name="testtype"/>
<sequential>
<mkdir dir="${report}"/>
<echo message="Sting: Running test cases!"/>
<junit printsummary="yes" showoutput="yes" maxmemory="${test.maxmemory}" clonevm="yes" haltonfailure="${halt}" failureProperty="test.failure">
<formatter type="${testOutput}" usefile="${usefile}" />
<classpath>
<pathelement location="build"/>
<path refid="runtime.dependencies"/>
<pathelement location="${test.classes}"/>
<pathelement location="lib/junit-4.4.jar"/>
</classpath>
<batchtest fork="yes" todir="${report}">
<fileset dir="${test.classes}">
<include name="**/${single}.class"/>
<exclude name="**/BaseTest.class"/>
<exclude name="**/*IntegrationTest.class"/>
</fileset>
</batchtest>
</junit>
<fail message="test failed" if="test.failure" />
<batchtest fork="yes" todir="${report}">
<fileset dir="${test.classes}">
<include name="**/${single}@{testtype}.class"/>
<exclude name="**/BaseTest.class"/>
</fileset>
</batchtest>
</junit>
<fail message="test failed" if="test.failure" />
</sequential>
</macrodef>
<!-- our three different test conditions: Test, IntegrationTest, PerformanceTest -->
<target name="test" depends="test.compile" description="Run unit tests">
<condition property="ttype" value="*UnitTest" else="">
<not><isset property="single"/></not>
</condition>
<condition property="single" value="">
<not><isset property="single"/></not>
</condition>
<run-test testtype="${ttype}"/>
</target>
<!-- Our long tests, tests that are more software integration and test -->
<target name="integrationtest" depends="test.compile"
description="Run the long integration unit tests">
<mkdir dir="${report}"/>
<echo message="Sting: Running integration test cases!"/>
<junit printsummary="yes" showoutput="yes" maxmemory="${test.maxmemory}" clonevm="yes" haltonfailure="${halt}" failureProperty="test.failure">
<formatter type="${testOutput}" usefile="${usefile}" />
<classpath>
<pathelement location="${dist.dir}/GenomeAnalysisTK.jar"/>
<path refid="runtime.dependencies"/>
<pathelement location="${test.classes}"/>
<pathelement location="lib/junit-4.4.jar"/>
</classpath>
<batchtest fork="yes" todir="${report}">
<fileset dir="${test.classes}">
<include name="**/${singleintegration}.class"/>
</fileset>
</batchtest>
</junit>
<fail message="test failed" if="test.failure" />
<target name="integrationtest" depends="test.compile" description="Run unit tests">
<condition property="ttype" value="*IntegrationTest" else="">
<not><isset property="single"/></not>
</condition>
<condition property="single" value="">
<not><isset property="single"/></not>
</condition>
<run-test testtype="${ttype}"/>
</target>
<!-- this builds the instrumentation library we're going to use for test cases -->
<target name="agent" description="build the agent">
<echo message="Sting: Creating test agent!"/>
<jar jarfile="${build.dir}/TestAgent.jar">
<fileset dir="build">
<include name="**/TrivialInstrumenter.class" />
</fileset>
<manifest>
<attribute name="Class-Path" value="${jar.classpath}"/>
<attribute name="Premain-Class" value="org.broadinstitute.sting.TrivialInstrumenter"/>
</manifest>
</jar>
<target name="performancetest" depends="test.compile" description="Run unit tests">
<condition property="ttype" value="*PerformanceTest" else="">
<not><isset property="single"/></not>
</condition>
<condition property="single" value="">
<not><isset property="single"/></not>
</condition>
<run-test testtype="${ttype}"/>
</target>
<!-- ***************************************************************************** -->
<target name="javadoc" depends="init,resolve" description="generates javadoc">
<mkdir dir="javadoc"/>
<javadoc destdir="javadoc"

165
java/test/org/broadinstitute/sting/TrivialInstrumenter.java
View File

@ -1,165 +0,0 @@
package org.broadinstitute.sting;
import javassist.*;
import org.junit.Ignore;
import java.io.IOException;
import java.lang.instrument.ClassFileTransformer;
import java.lang.instrument.IllegalClassFormatException;
import java.lang.instrument.Instrumentation;
/*
* Copyright (c) 2009 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
/**
* @author aaron
* @version 1.0
* @date May 5, 2009
* <p/>
* Class TrivalInstrumenter
* <p/>
* A simple instrumentation class, that adds information to our test cases.
*/
/** A trivial example program that basically just says hello! */
@Ignore
public class TrivialInstrumenter implements ClassFileTransformer {
public static void premain(String options, Instrumentation ins) {
if (options != null) {
System.out.printf(" I've been called with options: \"%s\"\n", options);
} else
ins.addTransformer(new TrivialInstrumenter());
}
public byte[] transform(ClassLoader loader,
String className,
Class cBR, java.security.ProtectionDomain pD,
byte[] classfileBuffer)
throws IllegalClassFormatException {
int size = classfileBuffer.length;
if (className.contains("broadinstitute") &&
className.endsWith("Test") &&
!(className.endsWith("BaseTest"))) {
ClassPool pool = ClassPool.getDefault();
CtClass cl = null;
try {
cl = pool.makeClass(new java.io.ByteArrayInputStream(classfileBuffer));
if (cl.isInterface() == false) {
for (CtBehavior meth : cl.getDeclaredMethods()) {
if (meth.isEmpty() == false) {
Object anns[] = meth.getAvailableAnnotations();
boolean weAreAJunitTest = false;
for (Object obj : anns) {
if (obj instanceof org.junit.Test) {
weAreAJunitTest = true;
}
}
if (weAreAJunitTest) {
addAnnouncement(meth, cl);
}
}
}
classfileBuffer = cl.toBytecode();
return classfileBuffer;
}
// baseTearDown
} catch (NotFoundException e) {
e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
} catch (IOException e) {
e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
} catch (CannotCompileException e) {
e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
} finally {
if (cl != null) {
cl.detach();
}
}
}
return null;
}
private static void addTiming(CtClass clas, String mname)
throws NotFoundException, CannotCompileException {
// get the method information (throws exception if method with
// given name is not declared directly by this class, returns
// arbitrary choice if more than one with the given name)
CtMethod mold = clas.getDeclaredMethod(mname);
// rename old method to synthetic name, then duplicate the
// method with original name for use as interceptor
String nname = mname + "$impl";
mold.setName(nname);
CtMethod mnew = CtNewMethod.copy(mold, mname, clas, null);
// start the body text generation by saving the start time
// to a local variable, then call the timed method; the
// actual code generated needs to depend on whether the
// timed method returns a value
String type = mold.getReturnType().getName();
StringBuffer body = new StringBuffer();
body.append("{\nlong start = System.currentTimeMillis();\n");
if (!"void".equals(type)) {
body.append(type + " result = ");
}
body.append(nname + "($$);\n");
// finish body text generation with call to print the timing
// information, and return saved value (if not void)
body.append("System.out.println(\"Call to method " + mname +
" took \" +\n (System.currentTimeMillis()-start) + " +
"\" ms.\");\n");
if (!"void".equals(type)) {
body.append("return result;\n");
}
body.append("}");
// replace the body of the interceptor method with generated
// code block and add it to class
mnew.setBody(body.toString());
clas.addMethod(mnew);
// print the generated code block just to show what was done
//System.out.println("Interceptor method body:");
//System.out.println(body.toString());
}
private void addAnnouncement(CtBehavior method, CtClass cl)
throws NotFoundException, CannotCompileException {
String name = method.getName();
method.insertAfter("logger.warn(\"\");");
}
}

222
java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalPerformanceTest.java 100644
View File

@ -0,0 +1,222 @@
package org.broadinstitute.sting.gatk.walkers.varianteval;
import org.broadinstitute.sting.WalkerTest;
import org.junit.Test;
import java.io.File;
import java.util.*;
/**
*
* @author aaron
*
* Class VariantEvalPerformanceTest
*
* TODO: remove me Eric
*/
public class VariantEvalPerformanceTest extends WalkerTest {
@Test
public void testEvalVariantROD() {
HashMap<String, String> md5 = new HashMap<String, String>();
md5.put("", "9cfda40f521d75a3e8bafc44a663c14a");
md5.put("-A", "8fea7cc25f551ce170636fc35c5ae0fe");
/**
* the above MD5 was calculated from running the following command:
*
* java -jar ./dist/GenomeAnalysisTK.jar \
* -R /broad/1KG/reference/human_b36_both.fasta \
* -T VariantEval \
* --DBSNP /humgen/gsa-scr1/GATK_Data/dbsnp_129_b36.rod \
* -L 1:10,000,000-11,000,000 \
* --outerr myVariantEval \
* --supressDateInformation \
* --rodBind eval,Variants,/humgen/gsa-scr1/GATK_Data/Validation_Data/NA12878.1kg.p2.chr1_10mb_11_mb.SLX.lod5.variants.geli.calls
*
*/
for ( Map.Entry<String, String> e : md5.entrySet() ) {
WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
"-R " + oneKGLocation + "reference/human_b36_both.fasta" +
" --rodBind eval,Variants," + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.lod5.variants.geli.calls" +
" -T VariantEval" +
" --DBSNP /humgen/gsa-scr1/GATK_Data/dbsnp_129_b36.rod" +
" -L 1:10,000,000-11,000,000" +
" --outerr %s" +
" --supressDateInformation " + e.getKey(),
1, // just one output file
Arrays.asList(e.getValue()));
List<File> result = executeTest("testEvalVariantROD", spec).getFirst();
}
}
@Test
public void testEvalVariantRODConfSix() {
List<String> md5 = new ArrayList<String>();
md5.add("11d636d105f902680c46b9f2e330d922");
/**
* the above MD5 was calculated from running the following command:
*
* java -jar ./dist/GenomeAnalysisTK.jar \
* -R /broad/1KG/reference/human_b36_both.fasta \
* -T VariantEval \
* --DBSNP /humgen/gsa-scr1/GATK_Data/dbsnp_129_b36.rod \
* -L 1:10,000,000-11,000,000 \
* --outerr myVariantEval \
* --supressDateInformation \
* --rodBind eval,Variants,/humgen/gsa-scr1/GATK_Data/Validation_Data/NA12878.1kg.p2.chr1_10mb_11_mb.SLX.lod5.variants.geli.calls \
* -minConfidenceScore 6
*/
WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
"-R " + oneKGLocation + "reference/human_b36_both.fasta" +
" --rodBind eval,Variants," + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.lod5.variants.geli.calls" +
" -T VariantEval" +
" --DBSNP /humgen/gsa-scr1/GATK_Data/dbsnp_129_b36.rod" +
" -L 1:10,000,000-11,000,000" +
" --outerr %s" +
" --supressDateInformation" +
" -minPhredConfidenceScore 60",
1, // just one output file
md5);
List<File> result = executeTest("testEvalVariantRODConfSixty", spec).getFirst();
}
@Test
public void testEvalVariantRODOutputViolations() {
List<String> md5 = new ArrayList<String>();
md5.add("12ecd457d62329e9d4e593de904a457d");
/**
* the above MD5 was calculated from running the following command:
*
* java -jar ./dist/GenomeAnalysisTK.jar \
* -R /broad/1KG/reference/human_b36_both.fasta \
* -T VariantEval \
* --DBSNP /humgen/gsa-scr1/GATK_Data/dbsnp_129_b36.rod \
* -L 1:10,000,000-11,000,000 \
* --outerr myVariantEval \
* --supressDateInformation \
* --rodBind eval,Variants,/humgen/gsa-scr1/GATK_Data/Validation_Data/NA12878.1kg.p2.chr1_10mb_11_mb.SLX.lod5.variants.geli.calls \
* --includeViolations
*/
WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
"-R " + oneKGLocation + "reference/human_b36_both.fasta" +
" --rodBind eval,Variants," + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.lod5.variants.geli.calls" +
" -T VariantEval" +
" --DBSNP /humgen/gsa-scr1/GATK_Data/dbsnp_129_b36.rod" +
" -L 1:10,000,000-11,000,000" +
" --outerr %s" +
" --supressDateInformation" +
" --includeViolations",
1, // just one output file
md5);
List<File> result = executeTest("testEvalVariantRODOutputViolations", spec).getFirst();
}
@Test
public void testEvalGenotypeROD() {
List<String> md5 = new ArrayList<String>();
md5.add("6ed44fd586c89dafd40cb8e0194dc456");
/**
* the above MD5 was calculated after running the following command:
*
* java -jar ./dist/GenomeAnalysisTK.jar \
* -R /broad/1KG/reference/human_b36_both.fasta \
* -T VariantEval \
* --DBSNP /humgen/gsa-scr1/GATK_Data/dbsnp_129_b36.rod \
* -L 1:10,000,000-11,000,000 \
* --outerr myVariantEval \
* --supressDateInformation \
* --evalContainsGenotypes \
* --rodBind eval,Variants,/humgen/gsa-scr1/GATK_Data/Validation_Data/NA12878.1kg.p2.chr1_10mb_11_mb.SLX.lod5.genotypes.geli.calls \
* --rodBind hapmap-chip,GFF,/humgen/gsa-scr1/GATK_Data/1KG_gffs/NA12878.1kg.gff
*/
WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
"-R " + oneKGLocation + "reference/human_b36_both.fasta" +
" --rodBind eval,Variants," + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.lod5.genotypes.geli.calls" +
" -T VariantEval" +
" --DBSNP /humgen/gsa-scr1/GATK_Data/dbsnp_129_b36.rod" +
" -L 1:10,000,000-11,000,000" +
" --outerr %s" +
" --supressDateInformation" +
" --evalContainsGenotypes" +
" --rodBind hapmap-chip,GFF,/humgen/gsa-scr1/GATK_Data/1KG_gffs/NA12878.1kg.gff",
1, // just one output file
md5);
List<File> result = executeTest("testEvalGenotypeROD", spec).getFirst();
}
@Test
public void testEvalMarksGenotypingExample() {
List<String> md5 = new ArrayList<String>();
md5.add("c0396cfe89a63948aebbbae0a0e06678");
/**
* Run with the following commands:
*
* java -Xmx2048m -jar /humgen/gsa-hphome1/depristo/dev/GenomeAnalysisTK/trunk/dist/GenomeAnalysisTK.jar
* -T VariantEval -R /broad/1KG/reference/human_b36_both.fasta -l INFO
* -B eval,Variants,/humgen/gsa-scr1/ebanks/concordanceForMark/UMichVsBroad.venn.set1Only.calls
* -D /humgen/gsa-scr1/GATK_Data/dbsnp_129_b36.rod -hc /humgen/gsa-scr1/GATK_Data/1KG_gffs/NA12878.1kg.gff
* -G -L 1 -o /humgen/gsa-scr1/ebanks/concordanceForMark/UMichVsBroad.venn.set1Only.calls.eval
*/
WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
"-T VariantEval -R " + oneKGLocation + "reference/human_b36_both.fasta " +
"-B eval,Variants," + validationDataLocation + "UMichVsBroad.venn.set1Only.calls " +
"-D /humgen/gsa-scr1/GATK_Data/dbsnp_129_b36.rod -hc /humgen/gsa-scr1/GATK_Data/1KG_gffs/NA12878.1kg.gff " +
"-G " +
"--supressDateInformation " +
"-L 1:1-10,000,000 " +
"--outerr %s",
1, // just one output file
md5);
List<File> result = executeTest("testEvalMarksGenotypingExample", spec).getFirst();
}
@Test
public void testEvalRuntimeWithLotsOfIntervals() {
List<String> md5 = new ArrayList<String>();
md5.add("6a90341517fc3c5026529301d9970c7b");
WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
"-T VariantEval -R " + oneKGLocation + "reference/human_b36_both.fasta " +
"-B eval,Variants," + validationDataLocation + "NA12878.pilot_3.all.geli.calls " +
"-D /humgen/gsa-scr1/GATK_Data/dbsnp_129_b36.rod " +
"--supressDateInformation " +
"-L /humgen/gsa-scr1/GATK_Data/thousand_genomes_alpha_redesign.targets.b36.interval_list " +
"--outerr %s",
1, // just one output file
md5);
List<File> result = executeTest("testEvalRuntimeWithLotsOfIntervals", spec).getFirst();
}
@Test
public void testVCFVariantEvals() {
HashMap<String, String> md5 = new HashMap<String, String>();
md5.put("", "ee6b096169d6c5e2ce49d394fbec799b");
md5.put("-A", "a443193c0810363f85278b1cfaed2fff");
md5.put("-A --includeFilteredRecords", "812d7f2ecac28b1be7e7028af17df9c0");
md5.put("-A --sampleName NA12878", "a443193c0810363f85278b1cfaed2fff");
md5.put("-A -vcfInfoSelector AF=0.50", "afed4bf0c9f11b86f6e5356012f9cf2d");
for ( Map.Entry<String, String> e : md5.entrySet() ) {
WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
"-R " + oneKGLocation + "reference/human_b36_both.fasta" +
" --rodBind eval,VCF," + validationDataLocation + "NA12878.example1.vcf" +
" -T VariantEval" +
" --DBSNP /humgen/gsa-scr1/GATK_Data/dbsnp_129_b36.rod" +
" -hc /humgen/gsa-scr1/GATK_Data/1KG_gffs/NA12878.1kg.gff" +
" -G" +
" -L 1:1-10,000" +
" --outerr %s" +
" --supressDateInformation " + e.getKey(),
1, // just one output file
Arrays.asList(e.getValue()));
List<File> result = executeTest("testVCFVariantEvals", spec).getFirst();
}
}
}