Merge pull request #488 from broadinstitute/ks_mvn_serial_test_updates

Ks mvn serial test updates
This commit is contained in:
droazen 2014-02-06 15:12:08 -05:00
commit b2b44c335f
12 changed files with 330 additions and 98 deletions

View File

@ -33,6 +33,7 @@ for arg in "${@}" ; do
elif [[ "${property_name}" == "test.debug.port" ]] ; then
mvn_properties="${mvn_properties} -Dmaven.surefire.debug=\"-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=${property_value}\""
mvn_properties="${mvn_properties} -Dmaven.failsafe.debug=\"-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=${property_value}\""
elif [[ "${property_name}" == "test.default.maxmemory" ]] ; then
mvn_properties="${mvn_properties} -Dtest.maxmemory=${property_value}"
@ -98,18 +99,16 @@ for arg in "${@}" ; do
elif [[ "${arg}" == "gatkfull.binary.release.tests" ]] ; then
local_repo="sitetemprepo"
mvn_args="install -Dmaven.repo.local=${local_repo} && mvn verify"
mvn_args="${mvn_args} -Dmaven.repo.local=${local_repo} -Dmaven.javadoc.skip=true"
mvn_args="${mvn_args} -Dsting.generate-gatk-extensions.skipped=true"
mvn_args="${mvn_args} -Dsting.jar.phase=none -Dsting.unpack.phase=none -Dsting.shade.phase=none"
mvn_args="${mvn_args} -Dmaven.repo.local=${local_repo}"
mvn_args="${mvn_args} -Dsting.packagetests.enabled=true"
mvn_args="${mvn_args} -Dsting.packagecommittests.skipped=false"
# TODO: This runs only the pipeline tests (full, non-dry run), but not the commit tests for Queue.
elif [[ "${arg}" == "queuefull.binary.release.tests" ]] ; then
local_repo="sitetemprepo"
mvn_args="install -Dmaven.repo.local=${local_repo} && mvn verify"
mvn_args="${mvn_args} -Dmaven.repo.local=${local_repo} -Dmaven.javadoc.skip=true"
mvn_args="${mvn_args} -Dsting.generate-gatk-extensions.skipped=true"
mvn_args="${mvn_args} -Dsting.jar.phase=none -Dsting.unpack.phase=none -Dsting.shade.phase=none"
mvn_args="${mvn_args} -Dmaven.repo.local=${local_repo}"
mvn_args="${mvn_args} -Dsting.packagetests.enabled=true"
mvn_args="${mvn_args} -Dsting.packagepipelinetests.skipped=false"
mvn_args="${mvn_args} -Dsting.pipelinetests.run=true"

92
pom.xml
View File

@ -32,13 +32,37 @@
<resource.bundle.skip>false</resource.bundle.skip>
<!-- TODO: Need a better a way to say "don't include hidden" by default -->
<gatkdocs.include.hidden>-build-timestamp "${maven.build.timestamp}"</gatkdocs.include.hidden>
<!--
Phases of the build that may be disabled to speed up compilation.
-->
<sting.jar.phase>package</sting.jar.phase>
<sting.test-utils.jar.phase>none</sting.test-utils.jar.phase>
<sting.generate-resources.phase>generate-resources</sting.generate-resources.phase>
<sting.process-resources.phase>process-resources</sting.process-resources.phase>
<sting.process-test-resources.phase>process-test-resources</sting.process-test-resources.phase>
<!--
Package tests ensure the consistency of the packaged / shaded jars.
It runs the tests where the monolithic jar is the only dependency on the classpath.
-->
<sting.packagecommittests.skipped>true</sting.packagecommittests.skipped>
<sting.packageunittests.skipped>${sting.packagecommittests.skipped}</sting.packageunittests.skipped>
<sting.packageintegrationtests.skipped>${sting.packagecommittests.skipped}</sting.packageintegrationtests.skipped>
<sting.packagepipelinetests.skipped>${sting.packagecommittests.skipped}</sting.packagepipelinetests.skipped>
<sting.packagelargescaletests.skipped>true</sting.packagelargescaletests.skipped>
<!--
Serial tests use the test jars to run tests, such that all tests are run from a single TestNG invocation.
This is different that the invoker, that runs the test classes from the filesystem, but pointing at the packaged JAR files.
TODO: Currently, all tests run within each package, since packages already collect dependencies for shading an uber jar.
TODO: Should there be another level up of tests, possibly running "all tests" via this aggregator level?
TODO: If that require the aggregator to be dependent on the child dependencies, perhaps a better approach might be another monolithic test project.
-->
<sting.serialcommittests.skipped>true</sting.serialcommittests.skipped>
<sting.serialunittests.skipped>${sting.serialcommittests.skipped}</sting.serialunittests.skipped>
<sting.serialintegrationtests.skipped>${sting.serialcommittests.skipped}</sting.serialintegrationtests.skipped>
<sting.serialpipelinetests.skipped>${sting.serialcommittests.skipped}</sting.serialpipelinetests.skipped>
<sting.seriallargescaletests.skipped>true</sting.seriallargescaletests.skipped>
</properties>
<dependencies>
@ -109,6 +133,20 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
<id>default-resources</id>
<goals>
<goal>resources</goal>
</goals>
<phase>${sting.process-resources.phase}</phase>
</execution>
<execution>
<id>default-testResources</id>
<goals>
<goal>testResources</goal>
</goals>
<phase>${sting.process-test-resources.phase}</phase>
</execution>
<execution>
<id>copy-resource-bundle-log4j</id>
<goals>
@ -277,38 +315,13 @@
<phase>${sting.jar.phase}</phase>
</execution>
<execution>
<id>test-utils</id>
<id>test-jar</id>
<goals>
<goal>jar</goal>
<goal>test-jar</goal>
</goals>
<phase>${sting.test-utils-jar.phase}</phase>
<phase>${sting.jar.phase}</phase>
<configuration>
<classesDirectory>${project.build.testOutputDirectory}</classesDirectory>
<classifier>test-utils</classifier>
<!--
TODO: Excluding actual tests from tests-utils jar is required because the same test classes
TODO: cannot be in both the test artifact AND the test classes directory.
TODO: Otherwise maven throws an [ERROR] java.lang.RuntimeException: Duplicate test set
TODO: This is due to AbstractSurefireMojo.scanForTestClasses() adding both scanDirectories() and scanDependencies().
TODO: Pain to debug as exception is three maven invoker levels deep, outer maven, maven-inovker-plugin, then forked tests!
TODO: See also: http://maven.apache.org/plugins/maven-jar-plugin/usage.html#The_preferred_way
TODO: Then that dependency won't have test classes that will also run from the
TODO: test classes directory during the invoked package test.
-->
<excludes>
<exclude>**/*UnitTest.class</exclude>
<exclude>**/*UnitTest$*.class</exclude>
<exclude>**/*IntegrationTest.class</exclude>
<exclude>**/*IntegrationTest$*.class</exclude>
<exclude>**/*PipelineTest.class</exclude>
<exclude>**/*PipelineTest$*.class</exclude>
<exclude>**/*LargeScaleTest.class</exclude>
<exclude>**/*LargeScaleTest.class</exclude>
</excludes>
<skipIfEmpty>true</skipIfEmpty>
</configuration>
</execution>
</executions>
@ -773,6 +786,25 @@
<module>private</module>
</modules>
</profile>
<!-- Collection of properties for use during package testing -->
<profile>
<id>packagetests-enabled</id>
<activation>
<property>
<name>sting.packagetests.enabled</name>
<value>true</value>
</property>
</activation>
<properties>
<maven.javadoc.skip>true</maven.javadoc.skip>
<sting.generate-gatk-extensions.skipped>true</sting.generate-gatk-extensions.skipped>
<sting.jar.phase>none</sting.jar.phase>
<sting.generate-resources.phase>none</sting.generate-resources.phase>
<sting.process-resources.phase>none</sting.process-resources.phase>
<sting.process-test-resources.phase>none</sting.process-test-resources.phase>
</properties>
</profile>
</profiles>
</project>

View File

@ -40,20 +40,10 @@
<artifactId>fastutil</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>sting-utils</artifactId>
<version>${project.version}</version>
<classifier>test-utils</classifier>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>gatk-framework</artifactId>
<version>${project.version}</version>
<classifier>test-utils</classifier>
<type>test-jar</type>
<scope>test</scope>
</dependency>

View File

@ -38,27 +38,10 @@
<version>${sting.version}</version>
</dependency>
<!--
Test dependencies are not transitive,
so must include each explicit test artifact.
TODO: switch from test-utils classifier to a tests-utils artifact
TODO: see also http://maven.apache.org/plugins/maven-jar-plugin/usage.html#The_preferred_way
-->
<dependency>
<groupId>org.broadinstitute.sting</groupId>
<artifactId>sting-utils</artifactId>
<version>${sting.version}</version>
<classifier>test-utils</classifier>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.broadinstitute.sting</groupId>
<artifactId>gatk-framework</artifactId>
<version>${sting.version}</version>
<classifier>test-utils</classifier>
<type>test-jar</type>
<scope>test</scope>
</dependency>
@ -208,7 +191,6 @@
<phase>none</phase>
</execution>
<execution>
<!-- run all non integration tests -->
<id>unit-tests</id>
<goals>
<goal>test</goal>

View File

@ -16,7 +16,6 @@
<properties>
<sting.basedir>${project.basedir}/../..</sting.basedir>
<sting.packagetests.artifactId>gatk-package</sting.packagetests.artifactId>
<sting.test-utils-jar.phase>${sting.jar.phase}</sting.test-utils-jar.phase>
</properties>
<dependencies>
@ -26,15 +25,6 @@
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>sting-utils</artifactId>
<version>${project.version}</version>
<classifier>test-utils</classifier>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
@ -55,7 +45,7 @@
<executions>
<execution>
<id>example-resources</id>
<phase>generate-resources</phase>
<phase>${sting.generate-resources.phase}</phase>
</execution>
</executions>
</plugin>
@ -79,14 +69,6 @@
</execution>
</executions>
</plugin>
<!--
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
</executions>
</plugin>
-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-invoker-plugin</artifactId>

View File

@ -51,10 +51,79 @@
<classifier>example-resources</classifier>
<type>tar.bz2</type>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>gatk-framework</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.caliper</groupId>
<artifactId>caliper</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<executions>
<execution>
<id>unit-tests</id>
<configuration>
<skip>${sting.serialunittests.skipped}</skip>
<dependenciesToScan>
<scan>org.broadinstitute.sting:.*</scan>
</dependenciesToScan>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<executions>
<execution>
<id>integration-tests</id>
<configuration>
<skip>${sting.serialintegrationtests.skipped}</skip>
<dependenciesToScan>
<scan>org.broadinstitute.sting:.*</scan>
</dependenciesToScan>
</configuration>
</execution>
<execution>
<id>pipeline-tests</id>
<configuration>
<skip>${sting.serialpipelinetests.skipped}</skip>
<dependenciesToScan>
<scan>org.broadinstitute.sting:.*</scan>
</dependenciesToScan>
</configuration>
</execution>
<execution>
<id>large-scale-tests</id>
<configuration>
<skip>${sting.seriallargescaletests.skipped}</skip>
<dependenciesToScan>
<scan>org.broadinstitute.sting:.*</scan>
</dependenciesToScan>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
@ -132,6 +201,14 @@
<version>${project.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>gatk-protected</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
<optional>true</optional>
</dependency>
</dependencies>
</profile>
<profile>
@ -148,7 +225,47 @@
<version>${project.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>gatk-private</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
<optional>true</optional>
</dependency>
</dependencies>
<build>
<plugins>
<!-- TODO: All tests require access to private. For now. -->
<plugin>
<groupId>com.pyx4j</groupId>
<artifactId>maven-junction-plugin</artifactId>
<executions>
<execution>
<id>link-private-testdata</id>
<phase>process-test-resources</phase>
</execution>
<execution>
<id>unlink-private-testdata</id>
<phase>clean</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>packagetests-enabled</id>
<activation>
<property>
<name>sting.packagetests.enabled</name>
<value>true</value>
</property>
</activation>
<properties>
<sting.unpack.phase>none</sting.unpack.phase>
<sting.shade.phase>none</sting.shade.phase>
</properties>
</profile>
</profiles>

View File

@ -30,7 +30,7 @@
<goals>
<goal>single</goal>
</goals>
<phase>generate-resources</phase>
<phase>${sting.generate-resources.phase}</phase>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<descriptors>

View File

@ -48,13 +48,29 @@
<artifactId>cofoja</artifactId>
</dependency>
<!--
gatk-framework test-jar added for BaseTest
TODO: Uncomment below to use the explicitly bad "exclude *" to really test correct packaging,
TODO: until we can separate BaseTest and other utilities into their own sting-test-utils artifact.
TODO: We only want the classes in the packaged jars tested, not the classes within the dependency jars too.
See also:
http://stackoverflow.com/questions/547805/how-to-exclude-all-transitive-dependencies-of-a-maven-dependency
http://maven.apache.org/plugins/maven-jar-plugin/usage.html#The_preferred_way
-->
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>gatk-framework</artifactId>
<version>${project.version}</version>
<classifier>test-utils</classifier>
<type>test-jar</type>
<scope>test</scope>
<!--
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
-->
</dependency>
<dependency>
@ -95,13 +111,9 @@
<workingDirectory>${sting.packagetests.basedir}</workingDirectory>
<classesDirectory>${project.build.outputDirectory}/ignored_by_package_test</classesDirectory>
<testClassesDirectory>${sting.packagetests.testClasses}</testClassesDirectory>
<dependenciesToScan>
<scan>org.broadinstitute.sting:.*</scan>
</dependenciesToScan>
</configuration>
<executions>
<execution>
<!-- run all non integration tests -->
<id>unit-tests</id>
<goals>
<goal>test</goal>
@ -137,9 +149,6 @@
<classesDirectory>${project.build.outputDirectory}/ignored_by_package_test</classesDirectory>
<testClassesDirectory>${sting.packagetests.testClasses}</testClassesDirectory>
<summaryFile>${failsafe.summaryFile}</summaryFile>
<dependenciesToScan>
<scan>org.broadinstitute.sting:.*</scan>
</dependenciesToScan>
</configuration>
<executions>
<execution>

View File

@ -18,7 +18,6 @@
<gatk.extensions.sources>${project.build.directory}/generated-sources/gatk-extensions</gatk.extensions.sources>
<sting.generate-gatk-extensions.skipped>false</sting.generate-gatk-extensions.skipped>
<sting.packagetests.artifactId>queue-package</sting.packagetests.artifactId>
<sting.test-utils-jar.phase>${sting.jar.phase}</sting.test-utils-jar.phase>
</properties>
<dependencies>
@ -66,7 +65,6 @@
<groupId>${project.groupId}</groupId>
<artifactId>gatk-framework</artifactId>
<version>${project.version}</version>
<classifier>test-utils</classifier>
<type>test-jar</type>
<scope>test</scope>
</dependency>
@ -130,7 +128,7 @@
<executions>
<execution>
<id>example-resources</id>
<phase>generate-resources</phase>
<phase>${sting.generate-resources.phase}</phase>
</execution>
</executions>
</plugin>

View File

@ -82,10 +82,86 @@
<classifier>example-resources</classifier>
<type>tar.bz2</type>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>queue-framework</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>gatk-framework</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.caliper</groupId>
<artifactId>caliper</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<executions>
<execution>
<id>unit-tests</id>
<configuration>
<skip>${sting.serialunittests.skipped}</skip>
<dependenciesToScan>
<scan>org.broadinstitute.sting:.*</scan>
</dependenciesToScan>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<executions>
<execution>
<id>integration-tests</id>
<configuration>
<skip>${sting.serialintegrationtests.skipped}</skip>
<dependenciesToScan>
<scan>org.broadinstitute.sting:.*</scan>
</dependenciesToScan>
</configuration>
</execution>
<execution>
<id>pipeline-tests</id>
<configuration>
<skip>${sting.serialpipelinetests.skipped}</skip>
<dependenciesToScan>
<scan>org.broadinstitute.sting:.*</scan>
</dependenciesToScan>
</configuration>
</execution>
<execution>
<id>large-scale-tests</id>
<configuration>
<skip>${sting.seriallargescaletests.skipped}</skip>
<dependenciesToScan>
<scan>org.broadinstitute.sting:.*</scan>
</dependenciesToScan>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
@ -163,7 +239,55 @@
<version>${project.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>queue-private</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
<optional>true</optional>
</dependency>
</dependencies>
<build>
<plugins>
<!-- TODO: All tests require access to private. For now. -->
<plugin>
<groupId>com.pyx4j</groupId>
<artifactId>maven-junction-plugin</artifactId>
<executions>
<execution>
<id>link-private-testdata</id>
<phase>process-test-resources</phase>
</execution>
<execution>
<id>unlink-private-testdata</id>
<phase>clean</phase>
</execution>
<execution>
<id>link-private-qscript</id>
<phase>process-test-resources</phase>
</execution>
<execution>
<id>unlink-private-qscript</id>
<phase>clean</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>packagetests-enabled</id>
<activation>
<property>
<name>sting.packagetests.enabled</name>
<value>true</value>
</property>
</activation>
<properties>
<sting.unpack.phase>none</sting.unpack.phase>
<sting.shade.phase>none</sting.shade.phase>
</properties>
</profile>
</profiles>

View File

@ -37,8 +37,8 @@
<groupId>com.sun</groupId>
<artifactId>tools.jar</artifactId>
<version>1.5</version>
<classifier>system</classifier>
<systemPath>${java.home}../lib/tools.jar</systemPath>
<scope>system</scope>
<systemPath>${java.home}/../lib/tools.jar</systemPath>
</dependency>
</dependencies>
</project>

View File

@ -17,7 +17,6 @@
<sting.basedir>${project.basedir}/../..</sting.basedir>
<gsalib.packagedir>org/broadinstitute/sting/utils/R</gsalib.packagedir>
<gsalib.filename>gsalib.tar.gz</gsalib.filename>
<sting.test-utils-jar.phase>${sting.jar.phase}</sting.test-utils-jar.phase>
</properties>
<dependencies>