zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #553 from broadinstitute/dr_rename_pipeline_tests 

Rename existing PipelineTests to QueueTests to prepare for upcoming push of new pipeline tests
This commit is contained in:
droazen 2014-03-10 21:36:45 -04:00
parent d0501d0083 78562c14bb
commit 8b53567dc7
24 changed files with 153 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
ant-bridge.sh
View File

@ -103,14 +103,14 @@ for arg in "${@}" ; do
mvn_args="${mvn_args} -Dsting.packagetests.enabled=true"
mvn_args="${mvn_args} -Dsting.packagecommittests.skipped=false"
# TODO: This runs only the pipeline tests (full, non-dry run), but not the commit tests for Queue.
# TODO: This runs only the queue tests (full, non-dry run), but not the commit tests for Queue.
elif [[ "${arg}" == "queuefull.binary.release.tests" ]] ; then
local_repo="sitetemprepo"
mvn_args="install -Dmaven.repo.local=${local_repo} && mvn verify"
mvn_args="${mvn_args} -Dmaven.repo.local=${local_repo}"
mvn_args="${mvn_args} -Dsting.packagetests.enabled=true"
mvn_args="${mvn_args} -Dsting.packagepipelinetests.skipped=false"
mvn_args="${mvn_args} -Dsting.pipelinetests.run=true"
mvn_args="${mvn_args} -Dsting.packagequeuetests.skipped=false"
mvn_args="${mvn_args} -Dsting.queuetests.run=true"
elif [[ "${arg}" == "committests" ]] ; then
mvn_args="verify -Dsting.committests.skipped=false"
@ -130,11 +130,11 @@ for arg in "${@}" ; do
elif [[ "${arg}" == "knowledgebasetest" ]] ; then
mvn_args="verify -Dsting.knowledgebasetests.skipped=false"
elif [[ "${arg}" == "pipelinetest" ]] ; then
mvn_args="verify -Dsting.pipelinetests.skipped=false"
elif [[ "${arg}" == "queuetest" ]] ; then
mvn_args="verify -Dsting.queuetests.skipped=false"
elif [[ "${arg}" == "pipelinetestrun" ]] ; then
mvn_args="verify -Dsting.pipelinetests.skipped=false -Dsting.pipelinetests.run=true"
elif [[ "${arg}" == "queuetestrun" ]] ; then
mvn_args="verify -Dsting.queuetests.skipped=false -Dsting.queuetests.run=true"
elif [[ "${arg}" == "fasttest" ]] ; then
mvn_args="verify -Dsting.committests.skipped=false -pl private/gatk-private -am -Dresource.bundle.skip=true"

14
pom.xml
View File

@ -48,7 +48,7 @@
<sting.packagecommittests.skipped>true</sting.packagecommittests.skipped>
<sting.packageunittests.skipped>${sting.packagecommittests.skipped}</sting.packageunittests.skipped>
<sting.packageintegrationtests.skipped>${sting.packagecommittests.skipped}</sting.packageintegrationtests.skipped>
<sting.packagepipelinetests.skipped>${sting.packagecommittests.skipped}</sting.packagepipelinetests.skipped>
<sting.packagequeuetests.skipped>${sting.packagecommittests.skipped}</sting.packagequeuetests.skipped>
<sting.packagelargescaletests.skipped>true</sting.packagelargescaletests.skipped>
<sting.packageknowledgebasetests.skipped>true</sting.packageknowledgebasetests.skipped>
@ -62,7 +62,7 @@
<sting.serialcommittests.skipped>true</sting.serialcommittests.skipped>
<sting.serialunittests.skipped>${sting.serialcommittests.skipped}</sting.serialunittests.skipped>
<sting.serialintegrationtests.skipped>${sting.serialcommittests.skipped}</sting.serialintegrationtests.skipped>
<sting.serialpipelinetests.skipped>${sting.serialcommittests.skipped}</sting.serialpipelinetests.skipped>
<sting.serialqueuetests.skipped>${sting.serialcommittests.skipped}</sting.serialqueuetests.skipped>
<sting.seriallargescaletests.skipped>true</sting.seriallargescaletests.skipped>
<sting.serialknowledgebasetests.skipped>true</sting.serialknowledgebasetests.skipped>
</properties>
@ -561,7 +561,7 @@
<sting.packagetests.artifactId>${sting.packagetests.artifactId}</sting.packagetests.artifactId>
<sting.packagetests.testClasses>${project.build.testOutputDirectory}</sting.packagetests.testClasses>
<sting.packagetests.basedir>${project.basedir}</sting.packagetests.basedir>
<sting.pipelinetests.run>${sting.pipelinetests.run}</sting.pipelinetests.run>
<sting.queuetests.run>${sting.queuetests.run}</sting.queuetests.run>
<maven.surefire.debug>${maven.surefire.debug}</maven.surefire.debug>
<maven.failsafe.debug>${maven.failsafe.debug}</maven.failsafe.debug>
</properties>
@ -613,7 +613,7 @@
</configuration>
</execution>
<execution>
<id>package-pipelinetests</id>
<id>package-queuetests</id>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
@ -622,11 +622,11 @@
<goals>
<goal>verify</goal>
</goals>
<reportsDirectory>${project.build.directory}/invoker-reports/pipeline/${it.test}</reportsDirectory>
<skipInvocation>${sting.packagepipelinetests.skipped}</skipInvocation>
<reportsDirectory>${project.build.directory}/invoker-reports/queuetest/${it.test}</reportsDirectory>
<skipInvocation>${sting.packagequeuetests.skipped}</skipInvocation>
<properties>
<integrationtests.profile.enabled>true</integrationtests.profile.enabled>
<sting.packagepipelinetests.skipped>${sting.packagepipelinetests.skipped}</sting.packagepipelinetests.skipped>
<sting.packagequeuetests.skipped>${sting.packagequeuetests.skipped}</sting.packagequeuetests.skipped>
</properties>
</configuration>
</execution>

2
protected/gatk-protected/pom.xml
View File

@ -99,7 +99,7 @@
<id>package-knowledgebasetests</id>
</execution>
<execution>
<id>package-pipelinetests</id>
<id>package-queuetests</id>
</execution>
</executions>
</plugin>

2
public/gatk-framework/pom.xml
View File

@ -86,7 +86,7 @@
<id>package-knowledgebasetests</id>
</execution>
<execution>
<id>package-pipelinetests</id>
<id>package-queuetests</id>
</execution>
</executions>
</plugin>

2
public/gatk-framework/src/test/java/org/broadinstitute/sting/BaseTest.java
View File

@ -135,7 +135,7 @@ public abstract class BaseTest {
public final static String NA12878_PCRFREE = privateTestDir + "PCRFree.2x250.Illumina.20_10_11.bam";
public final static String NA12878_WEx = privateTestDir + "CEUTrio.HiSeq.WEx.b37_decoy.NA12878.20_10_11mb.bam";
public static final boolean pipelineTestRunModeIsSet = System.getProperty("pipeline.run", "").equals("run");
public static final boolean queueTestRunModeIsSet = System.getProperty("queuetest.run", "").equals("true");
/** before the class starts up */
static {

10
public/gatk-framework/src/test/java/org/broadinstitute/sting/jna/drmaa/v1_0/JnaSessionPipelineTest.java → public/gatk-framework/src/test/java/org/broadinstitute/sting/jna/drmaa/v1_0/JnaSessionQueueTest.java
View File

@ -35,7 +35,7 @@ import org.testng.annotations.Test;
import java.io.File;
import java.util.*;
public class JnaSessionPipelineTest extends BaseTest {
public class JnaSessionQueueTest extends BaseTest {
private String implementation = null;
private static final SessionFactory factory = new JnaSessionFactory();
@ -52,15 +52,15 @@ public class JnaSessionPipelineTest extends BaseTest {
@Test(dependsOnMethods = { "testDrmaa" })
public void testSubmitEcho() throws Exception {
if ( ! pipelineTestRunModeIsSet ) {
throw new SkipException("Skipping testSubmitEcho because we are in pipeline test dry run mode");
if ( ! queueTestRunModeIsSet ) {
throw new SkipException("Skipping testSubmitEcho because we are in queue test dry run mode");
}
if (implementation.contains("LSF")) {
System.err.println(" ***********************************************************");
System.err.println(" *************************************************************");
System.err.println(" **** ****");
System.err.println(" **** Skipping JnaSessionPipelineTest.testSubmitEcho() ****");
System.err.println(" **** Skipping JnaSessionQueueTest.testSubmitEcho() ****");
System.err.println(" **** Are you using the dotkit .combined_LSF_SGE? ****");
System.err.println(" **** ****");
System.err.println(" *************************************************************");
@ -68,7 +68,7 @@ public class JnaSessionPipelineTest extends BaseTest {
throw new SkipException("Skipping testSubmitEcho because correct DRMAA implementation not found");
}
File outFile = tryCreateNetworkTempFile("JnaSessionPipelineTest.out");
File outFile = tryCreateNetworkTempFile("JnaSessionQueueTest.out");
Session session = factory.getSession();
session.init(null);
try {

8
public/gatk-framework/src/test/java/org/broadinstitute/sting/jna/drmaa/v1_0/LibDrmaaPipelineTest.java → public/gatk-framework/src/test/java/org/broadinstitute/sting/jna/drmaa/v1_0/LibDrmaaQueueTest.java
View File

@ -41,7 +41,7 @@ import java.io.File;
import java.util.Arrays;
import java.util.List;
public class LibDrmaaPipelineTest extends BaseTest {
public class LibDrmaaQueueTest extends BaseTest {
private String implementation = null;
@Test
@ -88,7 +88,7 @@ public class LibDrmaaPipelineTest extends BaseTest {
@Test(dependsOnMethods = { "testDrmaa" })
public void testSubmitEcho() throws Exception {
if ( ! pipelineTestRunModeIsSet ) {
if ( ! queueTestRunModeIsSet ) {
throw new SkipException("Skipping testSubmitEcho because we are in pipeline test dry run mode");
}
@ -96,7 +96,7 @@ public class LibDrmaaPipelineTest extends BaseTest {
System.err.println(" *********************************************************");
System.err.println(" ***********************************************************");
System.err.println(" **** ****");
System.err.println(" **** Skipping LibDrmaaPipelineTest.testSubmitEcho() ****");
System.err.println(" **** Skipping LibDrmaaQueueTest.testSubmitEcho() ****");
System.err.println(" **** Are you using the dotkit .combined_LSF_SGE? ****");
System.err.println(" **** ****");
System.err.println(" ***********************************************************");
@ -107,7 +107,7 @@ public class LibDrmaaPipelineTest extends BaseTest {
Memory error = new Memory(LibDrmaa.DRMAA_ERROR_STRING_BUFFER);
int errnum;
File outFile = tryCreateNetworkTempFile("LibDrmaaPipelineTest.out");
File outFile = tryCreateNetworkTempFile("LibDrmaaQueueTest.out");
errnum = LibDrmaa.drmaa_init(null, error, LibDrmaa.DRMAA_ERROR_STRING_BUFFER_LEN);

14
public/gatk-framework/src/test/java/org/broadinstitute/sting/jna/lsf/v7_0_6/LibBatPipelineTest.java → public/gatk-framework/src/test/java/org/broadinstitute/sting/jna/lsf/v7_0_6/LibBatQueueTest.java
View File

@ -39,12 +39,12 @@ import org.broadinstitute.sting.jna.lsf.v7_0_6.LibBat.*;
import java.io.File;
/**
* Really unit tests, but these test will only run on systems with LSF setup.
* Really unit tests, but these tests will only run on systems with LSF set up.
*/
public class LibBatPipelineTest extends BaseTest {
public class LibBatQueueTest extends BaseTest {
@BeforeClass
public void initLibBat() {
Assert.assertFalse(LibBat.lsb_init("LibBatPipelineTest") < 0, LibBat.lsb_sperror("lsb_init() failed"));
Assert.assertFalse(LibBat.lsb_init("LibBatQueueTest") < 0, LibBat.lsb_sperror("lsb_init() failed"));
}
@Test
@ -94,19 +94,19 @@ public class LibBatPipelineTest extends BaseTest {
@Test
public void testSubmitEcho() throws Exception {
if ( ! pipelineTestRunModeIsSet ) {
throw new SkipException("Skipping testSubmitEcho because we are in pipeline test dry run mode");
if ( ! queueTestRunModeIsSet ) {
throw new SkipException("Skipping testSubmitEcho because we are in queue test dry run mode");
}
String queue = "hour";
File outFile = tryCreateNetworkTempFile("LibBatPipelineTest.out");
File outFile = tryCreateNetworkTempFile("LibBatQueueTest.out");
submit req = new submit();
for (int i = 0; i < LibLsf.LSF_RLIM_NLIMITS; i++)
req.rLimits[i] = LibLsf.DEFAULT_RLIMIT;
req.projectName = "LibBatPipelineTest";
req.projectName = "LibBatQueueTest";
req.options |= LibBat.SUB_PROJECT_NAME;
req.queue = queue;

4
public/gatk-package/pom.xml
View File

@ -105,9 +105,9 @@
</configuration>
</execution>
<execution>
<id>pipeline-tests</id>
<id>queue-tests</id>
<configuration>
<skip>${sting.serialpipelinetests.skipped}</skip>
<skip>${sting.serialqueuetests.skipped}</skip>
<dependenciesToScan>
<scan>org.broadinstitute.sting:.*</scan>
</dependenciesToScan>

8
public/package-tests/pom.xml
View File

@ -21,7 +21,7 @@
<sting.basedir>${project.basedir}/../..</sting.basedir>
<sting.packageunittests.skipped>true</sting.packageunittests.skipped>
<sting.packageintegrationtests.skipped>true</sting.packageintegrationtests.skipped>
<sting.packagepipelinetests.skipped>true</sting.packagepipelinetests.skipped>
<sting.packagequeuetests.skipped>true</sting.packagequeuetests.skipped>
<sting.packagelargescaletests.skipped>true</sting.packagelargescaletests.skipped>
<sting.packageknowledgebasetests.skipped>true</sting.packageknowledgebasetests.skipped>
</properties>
@ -161,13 +161,13 @@
</configuration>
</execution>
<execution>
<id>pipeline-tests</id>
<id>queue-tests</id>
<goals>
<goal>verify</goal>
</goals>
<!-- run pipeline dry run tests -->
<!-- run queue dry run tests -->
<configuration>
<skip>${sting.packagepipelinetests.skipped}</skip>
<skip>${sting.packagequeuetests.skipped}</skip>
</configuration>
</execution>
<execution>

2
public/pom.xml
View File

@ -19,7 +19,7 @@
<module>sting-utils</module>
<module>gatk-framework</module>
<module>gatk-package</module>
<!-- <module>external-example</module> -->
<!-- <module>external-example</module> -->
<!-- queue optionally enabled as profiles -->
</modules>

2
public/queue-framework/pom.xml
View File

@ -198,7 +198,7 @@
<id>package-knowledgebasetests</id>
</execution>
<execution>
<id>package-pipelinetests</id>
<id>package-queuetests</id>
</execution>
</executions>
</plugin>

50
public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala → public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/QueueTest.scala
View File

@ -39,13 +39,13 @@ import org.broadinstitute.sting.gatk.report.GATKReport
import org.apache.commons.io.FileUtils
import org.apache.commons.io.filefilter.WildcardFileFilter
object PipelineTest extends BaseTest with Logging {
object QueueTest extends BaseTest with Logging {
private val validationReportsDataLocation = "/humgen/gsa-hpprojects/GATK/validationreports/submitted/"
private val md5DB = new MD5DB()
/**
* All the job runners configured to run PipelineTests at The Broad.
* All the job runners configured to run QueueTests at The Broad.
*/
final val allJobRunners = Seq("Lsf706", "GridEngine", "Shell")
@ -56,15 +56,15 @@ object PipelineTest extends BaseTest with Logging {
/**
* Returns the top level output path to this test.
* @param testName The name of the test passed to PipelineTest.executeTest()
* @param testName The name of the test passed to QueueTest.executeTest()
* @param jobRunner The name of the job manager to run the jobs.
* @return the top level output path to this test.
*/
def testDir(testName: String, jobRunner: String) = "pipelinetests/%s/%s/".format(testName, jobRunner)
def testDir(testName: String, jobRunner: String) = "queuetests/%s/%s/".format(testName, jobRunner)
/**
* Returns the directory where relative output files will be written for this test.
* @param testName The name of the test passed to PipelineTest.executeTest()
* @param testName The name of the test passed to QueueTest.executeTest()
* @param jobRunner The name of the job manager to run the jobs.
* @return the directory where relative output files will be written for this test.
*/
@ -72,44 +72,44 @@ object PipelineTest extends BaseTest with Logging {
/**
* Returns the directory where temp files will be written for this test.
* @param testName The name of the test passed to PipelineTest.executeTest()
* @param testName The name of the test passed to QueueTest.executeTest()
* @param jobRunner The name of the job manager to run the jobs.
* @return the directory where temp files will be written for this test.
*/
private def tempDir(testName: String, jobRunner: String) = testDir(testName, jobRunner) + "temp/"
/**
* Runs the pipelineTest.
* @param pipelineTest test to run.
* Runs the queueTest.
* @param queueTest test to run.
*/
def executeTest(pipelineTest: PipelineTestSpec) {
var jobRunners = pipelineTest.jobRunners
def executeTest(queueTest: QueueTestSpec) {
var jobRunners = queueTest.jobRunners
if (jobRunners == null)
jobRunners = defaultJobRunners
jobRunners.foreach(executeTest(pipelineTest, _))
jobRunners.foreach(executeTest(queueTest, _))
}
/**
* Runs the pipelineTest.
* @param pipelineTest test to run.
* Runs the queueTest.
* @param queueTest test to run.
* @param jobRunner The name of the job manager to run the jobs.
*/
def executeTest(pipelineTest: PipelineTestSpec, jobRunner: String) {
def executeTest(queueTest: QueueTestSpec, jobRunner: String) {
// Reset the order of functions added to the graph.
QScript.resetAddOrder()
val name = pipelineTest.name
val name = queueTest.name
if (name == null)
Assert.fail("PipelineTestSpec.name is null")
Assert.fail("QueueTestSpec.name is null")
println(Utils.dupString('-', 80))
executeTest(name, pipelineTest.args, pipelineTest.jobQueue, pipelineTest.expectedException, jobRunner)
if (BaseTest.pipelineTestRunModeIsSet) {
assertMatchingMD5s(name, pipelineTest.fileMD5s.map{case (file, md5) => new File(runDir(name, jobRunner), file) -> md5}, pipelineTest.parameterize)
if (pipelineTest.evalSpec != null)
validateEval(name, pipelineTest.evalSpec, jobRunner)
for (path <- pipelineTest.expectedFilePaths)
executeTest(name, queueTest.args, queueTest.jobQueue, queueTest.expectedException, jobRunner)
if (BaseTest.queueTestRunModeIsSet) {
assertMatchingMD5s(name, queueTest.fileMD5s.map{case (file, md5) => new File(runDir(name, jobRunner), file) -> md5}, queueTest.parameterize)
if (queueTest.evalSpec != null)
validateEval(name, queueTest.evalSpec, jobRunner)
for (path <- queueTest.expectedFilePaths)
assertPathExists(runDir(name, jobRunner), path)
for (path <- pipelineTest.unexpectedFilePaths)
for (path <- queueTest.unexpectedFilePaths)
assertPathDoesNotExist(runDir(name, jobRunner), path)
println(" => %s PASSED (%s)".format(name, jobRunner))
}
@ -128,7 +128,7 @@ object PipelineTest extends BaseTest with Logging {
Assert.fail("%d of %d MD5s did not match".format(failed, fileMD5s.size))
}
private def validateEval(name: String, evalSpec: PipelineTestEvalSpec, jobRunner: String) {
private def validateEval(name: String, evalSpec: QueueTestEvalSpec, jobRunner: String) {
// write the report to the shared validation data location
val formatter = new SimpleDateFormat("yyyy.MM.dd.HH.mm.ss")
val reportLocation = "%s%s/%s/validation.%s.eval".format(validationReportsDataLocation, jobRunner, name, formatter.format(new Date))
@ -176,7 +176,7 @@ object PipelineTest extends BaseTest with Logging {
if (jobQueue != null)
command = Utils.appendArray(command, "-jobQueue", jobQueue)
if (BaseTest.pipelineTestRunModeIsSet)
if (BaseTest.queueTestRunModeIsSet)
command = Utils.appendArray(command, "-run")
// run the executable

2
public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/PipelineTestEvalSpec.scala → public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/QueueTestEvalSpec.scala
View File

@ -28,7 +28,7 @@ package org.broadinstitute.sting.queue.pipeline
/**
* Data validations to evaluate on a GATKReport.
*/
class PipelineTestEvalSpec {
class QueueTestEvalSpec {
/** Eval modules to output. */
var evalReport: String = _

4
public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/PipelineTestSpec.scala → public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/QueueTestSpec.scala
View File

@ -25,7 +25,7 @@
package org.broadinstitute.sting.queue.pipeline
class PipelineTestSpec(var name: String = null) {
class QueueTestSpec(var name: String = null) {
/** The arguments to pass to the Queue test, ex: "-S scala/qscript/examples/HelloWorld.scala" */
var args: String = _
@ -40,7 +40,7 @@ class PipelineTestSpec(var name: String = null) {
var fileMD5s = Map.empty[String, String]
/** VariantEval validations to run on a VCF after the pipeline has completed. */
var evalSpec: PipelineTestEvalSpec = _
var evalSpec: QueueTestEvalSpec = _
/** Expected exception from the test. */
var expectedException: Class[_ <: Exception] = null

8
public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/examples/ExampleCountLociPipelineTest.scala → public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/examples/ExampleCountLociQueueTest.scala
View File

@ -26,14 +26,14 @@
package org.broadinstitute.sting.queue.pipeline.examples
import org.testng.annotations.Test
import org.broadinstitute.sting.queue.pipeline.{PipelineTest, PipelineTestSpec}
import org.broadinstitute.sting.queue.pipeline.{QueueTest, QueueTestSpec}
import org.broadinstitute.sting.BaseTest
class ExampleCountLociPipelineTest {
class ExampleCountLociQueueTest {
@Test(timeOut=36000000)
def testCountLoci() {
val testOut = "count.out"
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "countloci"
spec.args = Array(
" -S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleCountLoci.scala",
@ -41,6 +41,6 @@ class ExampleCountLociPipelineTest {
" -I " + BaseTest.publicTestDir + "exampleBAM.bam",
" -o " + testOut).mkString
spec.fileMD5s += testOut -> "ade93df31a6150321c1067e749cae9be"
PipelineTest.executeTest(spec)
QueueTest.executeTest(spec)
}
}

8
public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/examples/ExampleCountReadsPipelineTest.scala → public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/examples/ExampleCountReadsQueueTest.scala
View File

@ -26,18 +26,18 @@
package org.broadinstitute.sting.queue.pipeline.examples
import org.testng.annotations.Test
import org.broadinstitute.sting.queue.pipeline.{PipelineTest, PipelineTestSpec}
import org.broadinstitute.sting.queue.pipeline.{QueueTest, QueueTestSpec}
import org.broadinstitute.sting.BaseTest
class ExampleCountReadsPipelineTest {
class ExampleCountReadsQueueTest {
@Test(timeOut=36000000)
def testCountReads() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "countreads"
spec.args = Array(
" -S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleCountReads.scala",
" -R " + BaseTest.publicTestDir + "exampleFASTA.fasta",
" -I " + BaseTest.publicTestDir + "exampleBAM.bam").mkString
PipelineTest.executeTest(spec)
QueueTest.executeTest(spec)
}
}

16
public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/examples/ExamplePrintReadsPipelineTest.scala → public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/examples/ExamplePrintReadsQueueTest.scala
View File

@ -50,34 +50,34 @@ package org.broadinstitute.sting.queue.pipeline.examples
*/
import org.testng.annotations.Test
import org.broadinstitute.sting.queue.pipeline.{PipelineTest, PipelineTestSpec}
import org.broadinstitute.sting.queue.pipeline.{QueueTest, QueueTestSpec}
import org.broadinstitute.sting.BaseTest
class ExamplePrintReadsPipelineTest {
class ExamplePrintReadsQueueTest {
@Test(timeOut=36000000)
def testDevNullOutput() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "devnulloutput"
spec.args = Array(
" -S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExamplePrintReads.scala",
" -R " + BaseTest.publicTestDir + "exampleFASTA.fasta",
" -I " + BaseTest.publicTestDir + "exampleBAM.bam",
" -out /dev/null").mkString
spec.jobRunners = PipelineTest.allJobRunners
PipelineTest.executeTest(spec)
spec.jobRunners = QueueTest.allJobRunners
QueueTest.executeTest(spec)
}
@Test(timeOut=36000000)
def testCleanupBai() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "cleanupbai"
spec.args = Array(
" -S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExamplePrintReads.scala",
" -R " + BaseTest.publicTestDir + "exampleFASTA.fasta",
" -I " + BaseTest.publicTestDir + "exampleBAM.bam",
" -out exampleOut.bam").mkString
spec.jobRunners = PipelineTest.allJobRunners
spec.jobRunners = QueueTest.allJobRunners
spec.unexpectedFilePaths :+= ".queue/scatterGather/ExamplePrintReads-1-sg/temp_1_of_1/exampleOut.bai"
PipelineTest.executeTest(spec)
QueueTest.executeTest(spec)
}
}

8
public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/examples/ExampleReadFilterPipelineTest.scala → public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/examples/ExampleReadFilterQueueTest.scala
View File

@ -74,18 +74,18 @@ package org.broadinstitute.sting.queue.pipeline.examples
*/
import org.testng.annotations.Test
import org.broadinstitute.sting.queue.pipeline.{PipelineTest, PipelineTestSpec}
import org.broadinstitute.sting.queue.pipeline.{QueueTest, QueueTestSpec}
import org.broadinstitute.sting.BaseTest
class ExampleReadFilterPipelineTest {
class ExampleReadFilterQueueTest {
@Test(timeOut=36000000)
def testExampleReadFilter() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "examplereadfilter"
spec.args = Array(
" -S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleReadFilter.scala",
" -R " + BaseTest.publicTestDir + "exampleFASTA.fasta",
" -I " + BaseTest.publicTestDir + "exampleBAM.bam").mkString
PipelineTest.executeTest(spec)
QueueTest.executeTest(spec)
}
}

10
public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/examples/ExampleRetryMemoryLimitPipelineTest.scala → public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/examples/ExampleRetryMemoryLimitQueueTest.scala
View File

@ -26,22 +26,22 @@
package org.broadinstitute.sting.queue.pipeline.examples
import org.testng.annotations.Test
import org.broadinstitute.sting.queue.pipeline.{PipelineTest, PipelineTestSpec}
import org.broadinstitute.sting.queue.pipeline.{QueueTest, QueueTestSpec}
import org.broadinstitute.sting.BaseTest
class ExampleRetryMemoryLimitPipelineTest {
class ExampleRetryMemoryLimitQueueTest {
// This test is currently disabled due to unexplained intermittent failures (see GSA-943)
@Test(timeOut=36000000,enabled = false)
def testRetryMemoryLimit() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "RetryMemoryLimit"
spec.args = Array(
" -S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleRetryMemoryLimit.scala",
" -R " + BaseTest.publicTestDir + "exampleFASTA.fasta",
" -I " + BaseTest.publicTestDir + "exampleBAM.bam",
" -retry 1").mkString
spec.jobRunners = PipelineTest.allJobRunners
PipelineTest.executeTest(spec)
spec.jobRunners = QueueTest.allJobRunners
QueueTest.executeTest(spec)
}
}

30
public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/examples/ExampleUnifiedGenotyperPipelineTest.scala → public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/examples/ExampleUnifiedGenotyperQueueTest.scala
View File

@ -26,13 +26,13 @@
package org.broadinstitute.sting.queue.pipeline.examples
import org.testng.annotations.{DataProvider, Test}
import org.broadinstitute.sting.queue.pipeline.{PipelineTest, PipelineTestSpec}
import org.broadinstitute.sting.queue.pipeline.{QueueTest, QueueTestSpec}
import org.broadinstitute.sting.BaseTest
class ExampleUnifiedGenotyperPipelineTest {
class ExampleUnifiedGenotyperQueueTest {
@Test(timeOut=36000000)
def testUnifiedGenotyper() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "unifiedgenotyper"
spec.args = Array(
" -S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleUnifiedGenotyper.scala",
@ -40,8 +40,8 @@ class ExampleUnifiedGenotyperPipelineTest {
" -I " + BaseTest.publicTestDir + "exampleBAM.bam",
" -filter QD",
" -filterExpression 'QD < 2.0'").mkString
spec.jobRunners = PipelineTest.allJobRunners
PipelineTest.executeTest(spec)
spec.jobRunners = QueueTest.allJobRunners
QueueTest.executeTest(spec)
}
@DataProvider(name = "ugIntervals")
@ -54,7 +54,7 @@ class ExampleUnifiedGenotyperPipelineTest {
@Test(dataProvider = "ugIntervals", timeOut=36000000)
def testUnifiedGenotyperWithIntervals(intervalsName: String, intervalsPath: String) {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "unifiedgenotyper_with_" + intervalsName
spec.args = Array(
" -S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleUnifiedGenotyper.scala",
@ -62,20 +62,20 @@ class ExampleUnifiedGenotyperPipelineTest {
" -R " + BaseTest.hg18Reference,
" -L " + intervalsPath).mkString
spec.jobRunners = Seq("Lsf706")
PipelineTest.executeTest(spec)
QueueTest.executeTest(spec)
}
@Test(timeOut=36000000)
def testUnifiedGenotyperNoGCOpt() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "unifiedgenotyper_no_gc_opt"
spec.args = Array(
" -S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleUnifiedGenotyper.scala",
" -R " + BaseTest.publicTestDir + "exampleFASTA.fasta",
" -I " + BaseTest.publicTestDir + "exampleBAM.bam",
" -noGCOpt").mkString
spec.jobRunners = PipelineTest.allJobRunners
PipelineTest.executeTest(spec)
spec.jobRunners = QueueTest.allJobRunners
QueueTest.executeTest(spec)
}
@DataProvider(name="resMemReqParams")
@ -83,7 +83,7 @@ class ExampleUnifiedGenotyperPipelineTest {
@Test(dataProvider = "resMemReqParams", timeOut=36000000)
def testUnifiedGenotyperResMemReqParam(reqParam: String) {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "unifiedgenotyper_" + reqParam
spec.args = Array(
" -S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleUnifiedGenotyper.scala",
@ -91,21 +91,21 @@ class ExampleUnifiedGenotyperPipelineTest {
" -I " + BaseTest.publicTestDir + "exampleBAM.bam",
" -resMemReqParam " + reqParam).mkString
spec.jobRunners = Seq("GridEngine")
PipelineTest.executeTest(spec)
QueueTest.executeTest(spec)
}
@Test(timeOut=36000000)
def testUnifiedGenotyperLogDirectory() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "unifiedgenotyper_with_log_directory"
spec.args = Array(
" -S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleUnifiedGenotyper.scala",
" -R " + BaseTest.publicTestDir + "exampleFASTA.fasta",
" -I " + BaseTest.publicTestDir + "exampleBAM.bam",
" -logDir exampleUGLogDir").mkString
spec.jobRunners = PipelineTest.allJobRunners
spec.jobRunners = QueueTest.allJobRunners
spec.expectedFilePaths :+= "exampleUGLogDir/exampleBAM.unfiltered.vcf.out"
spec.expectedFilePaths :+= "exampleUGLogDir/exampleBAM.unfiltered.eval.out"
PipelineTest.executeTest(spec)
QueueTest.executeTest(spec)
}
}

64
public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/examples/HelloWorldPipelineTest.scala → public/queue-framework/src/test/scala/org/broadinstitute/sting/queue/pipeline/examples/HelloWorldQueueTest.scala
View File

@ -26,127 +26,127 @@
package org.broadinstitute.sting.queue.pipeline.examples
import org.testng.annotations.Test
import org.broadinstitute.sting.queue.pipeline.{PipelineTest, PipelineTestSpec}
import org.broadinstitute.sting.queue.pipeline.{QueueTest, QueueTestSpec}
class HelloWorldPipelineTest {
class HelloWorldQueueTest {
@Test(timeOut=36000000)
def testHelloWorld() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "HelloWorld"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala"
spec.jobRunners = PipelineTest.allJobRunners
PipelineTest.executeTest(spec)
spec.jobRunners = QueueTest.allJobRunners
QueueTest.executeTest(spec)
}
@Test(timeOut=36000000)
def testHelloWorldWithRunName() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "HelloWorldWithRunName"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" +
" -runName HelloWorld"
spec.jobRunners = PipelineTest.allJobRunners
PipelineTest.executeTest(spec)
spec.jobRunners = QueueTest.allJobRunners
QueueTest.executeTest(spec)
}
@Test(timeOut=36000000)
def testHelloWorldWithMemoryLimit() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "HelloWorldMemoryLimit"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" +
" -memLimit 1.25"
spec.jobRunners = PipelineTest.allJobRunners
PipelineTest.executeTest(spec)
spec.jobRunners = QueueTest.allJobRunners
QueueTest.executeTest(spec)
}
@Test(timeOut=36000000)
def testHelloWorldWithPriority() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "HelloWorldWithPriority"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" +
" -jobPriority 100"
spec.jobRunners = PipelineTest.allJobRunners
PipelineTest.executeTest(spec)
spec.jobRunners = QueueTest.allJobRunners
QueueTest.executeTest(spec)
}
@Test(timeOut=36000000)
def testHelloWorldWithLsfResource() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "HelloWorldWithLsfResource"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" +
" -jobResReq rusage[iodine_io=1] -jobResReq select[swp>0] -jobResReq order[swp]"
spec.jobRunners = Seq("Lsf706")
PipelineTest.executeTest(spec)
QueueTest.executeTest(spec)
}
@Test(timeOut=36000000)
def testHelloWorldWithLsfResourceAndMemoryLimit() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "HelloWorldWithLsfResourceAndMemoryLimit"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" +
" -memLimit 1.25 -jobResReq rusage[iodine_io=1] -jobResReq select[swp>0] -jobResReq order[swp]"
spec.jobRunners = Seq("Lsf706")
PipelineTest.executeTest(spec)
QueueTest.executeTest(spec)
}
@Test(timeOut=36000000)
def testHelloWorldWithLsfEnvironment() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "HelloWorldWithLsfEnvironment"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" +
" -jobEnv tv"
spec.jobRunners = Seq("Lsf706")
PipelineTest.executeTest(spec)
QueueTest.executeTest(spec)
}
@Test(timeOut=36000000)
def testHelloWorldWithGridEngineResource() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "HelloWorldWithGridEngineResource"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" +
" -jobResReq s_core=1000M"
spec.jobRunners = Seq("GridEngine")
PipelineTest.executeTest(spec)
QueueTest.executeTest(spec)
}
@Test(timeOut=36000000)
def testHelloWorldWithGridEngineResourceAndMemoryLimit() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "HelloWorldWithGridEngineResourceAndMemoryLimit"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" +
" -memLimit 1.25 -jobResReq s_core=1000M"
spec.jobRunners = Seq("GridEngine")
PipelineTest.executeTest(spec)
QueueTest.executeTest(spec)
}
@Test(timeOut=36000000)
def testHelloWorldWithGridEngineEnvironment() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "HelloWorldWithGridEngineEnvironment"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" +
" -jobEnv \"make 1\""
spec.jobRunners = Seq("GridEngine")
PipelineTest.executeTest(spec)
QueueTest.executeTest(spec)
}
// disabled because our DRMAA implementation doesn't support wallTime
@Test(enabled=false, timeOut=36000000)
def testHelloWorldWithWalltime() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "HelloWorldWithWalltime"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" +
" -wallTime 100"
spec.jobRunners = PipelineTest.allJobRunners
PipelineTest.executeTest(spec)
spec.jobRunners = QueueTest.allJobRunners
QueueTest.executeTest(spec)
}
@Test(timeOut=36000000)
def testHelloWorldWithLogDirectory() {
val spec = new PipelineTestSpec
val spec = new QueueTestSpec
spec.name = "HelloWorldWithLogDirectory"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" +
" -logDir pipelineLogDir"
spec.jobRunners = PipelineTest.allJobRunners
spec.jobRunners = QueueTest.allJobRunners
spec.expectedFilePaths = Seq("pipelineLogDir/HelloWorld-1.out")
PipelineTest.executeTest(spec)
QueueTest.executeTest(spec)
}
}

4
public/queue-package/pom.xml
View File

@ -144,9 +144,9 @@
</configuration>
</execution>
<execution>
<id>pipeline-tests</id>
<id>queue-tests</id>
<configuration>
<skip>${sting.serialpipelinetests.skipped}</skip>
<skip>${sting.serialqueuetests.skipped}</skip>
<dependenciesToScan>
<scan>org.broadinstitute.sting:.*</scan>
</dependenciesToScan>

20
public/sting-root/pom.xml
View File

@ -31,10 +31,10 @@
<sting.committests.skipped>true</sting.committests.skipped>
<sting.unittests.skipped>${sting.committests.skipped}</sting.unittests.skipped>
<sting.integrationtests.skipped>${sting.committests.skipped}</sting.integrationtests.skipped>
<sting.pipelinetests.skipped>${sting.committests.skipped}</sting.pipelinetests.skipped>
<sting.queuetests.skipped>${sting.committests.skipped}</sting.queuetests.skipped>
<sting.largescaletests.skipped>true</sting.largescaletests.skipped>
<sting.knowledgebasetests.skipped>true</sting.knowledgebasetests.skipped>
<sting.pipelinetests.run>false</sting.pipelinetests.run>
<sting.queuetests.run>false</sting.queuetests.run>
<scala.maxmemory>1g</scala.maxmemory>
<test.maxmemory>4g</test.maxmemory>
<java.gc.threads>4</java.gc.threads>
@ -412,7 +412,7 @@
<clover.pertest.coverage>diff</clover.pertest.coverage>
<java.awt.headless>true</java.awt.headless>
<!-- TODO: Fix BaseTest to not error out if this property is missing. -->
<pipeline.run>${sting.pipelinetests.run}</pipeline.run>
<queuetest.run>${sting.queuetests.run}</queuetest.run>
<java.io.tmpdir>${java.io.tmpdir}</java.io.tmpdir>
</systemPropertyVariables>
</configuration>
@ -435,19 +435,19 @@
</configuration>
</execution>
<execution>
<id>pipeline-tests</id>
<id>queue-tests</id>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
<!-- run pipeline dry run tests -->
<!-- run queue dry run tests -->
<configuration>
<!-- TODO: Re-enable pipeline tests by default? -->
<skip>${sting.pipelinetests.skipped}</skip>
<reportsDirectory>${project.build.directory}/failsafe-reports/pipeline/${it.test}</reportsDirectory>
<summaryFile>${project.build.directory}/failsafe-reports/pipeline/failsafe-summary-${it.test}.xml</summaryFile>
<!-- TODO: Re-enable queue tests by default? -->
<skip>${sting.queuetests.skipped}</skip>
<reportsDirectory>${project.build.directory}/failsafe-reports/queuetest/${it.test}</reportsDirectory>
<summaryFile>${project.build.directory}/failsafe-reports/queuetest/failsafe-summary-${it.test}.xml</summaryFile>
<includes>
<include>**/*PipelineTest.class</include>
<include>**/*QueueTest.class</include>
</includes>
</configuration>
</execution>