zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added faster built executable, non-packaged jars. 

Moved top level target symlinks to package jar files to under target/package.
Executable jar files are placed under target/executable with the new target[/lib] directories.
Under top level target, symlinks to *either* the package *or* the executable jars replace what was a symlink to the package jar path.
Allow disabling of the shade package.
ant-bridge.sh by default only builds executable jars, and doesn't package by default, as did the old ant build.xml.
Added a new package_path.sh utility script for other scripts to use instead of anything in the target folder.
This commit is contained in:
Khalid Shakir 2014-11-02 16:18:07 +09:00
parent f6af776590
commit 1cb4b99548
7 changed files with 196 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
ant-bridge.sh
View File

@ -1,6 +1,7 @@
#!/bin/sh
mvn_args="verify"
default_args="verify '-Ddisable.shadepackage'"
mvn_args="${default_args}"
mvn_properties=
mvn_clean=
unknown_args=
@ -44,22 +45,23 @@ for arg in "${@}" ; do
fi
else
if [[ "${arg}" != "dist" && "${mvn_args}" != "" && "${mvn_args}" != "verify" ]] ; then
if [[ "${arg}" != "dist" && "${mvn_args}" != "" && "${mvn_args}" != "${default_args}" ]] ; then
echo "Sorry, this script does not currently support mixing targets." >&2
exit 1
elif [[ "${arg}" == "dist" ]] ; then
mvn_args="verify"
mvn_args="${default_args}"
elif [[ "${arg}" == "gatk" ]] ; then
mvn_args="verify '-P!queue'"
mvn_args="${default_args} '-P!queue'"
elif [[ "${arg}" == "test.compile" ]] ; then
mvn_args="test-compile"
elif [[ "${arg}" == "gatkdocs" ]] ; then
local_repo="sitetemprepo"
mvn_args="install -Dmaven.repo.local=${local_repo} -Ddisable.queue && mvn site -Dmaven.repo.local=${local_repo} -Ddisable.queue"
mvn_args="install -Dmaven.repo.local=${local_repo} '-P!queue' && mvn site -Dmaven.repo.local=${local_repo} '-P!queue'"
mvn_pkg_args=
elif [[ "${arg}" == "package.gatk.full" ]] ; then
mvn_args="package '-P!private,!queue'"
@ -75,11 +77,11 @@ for arg in "${@}" ; do
# elif [[ "${arg}" == "release.gatk.full" ]] ; then
# mvn_args="package '-P!private,!queue'"
# post_script=" && private/src/main/scripts/shell/copy_release.sh public/gatk-package/target/GenomeAnalysisTK-*.tar.bz2"
# post_script=" && private/src/main/scripts/shell/copy_release.sh protected/gatk-package-distribution/target/GenomeAnalysisTK-*.tar.bz2"
# elif [[ "${arg}" == "release.queue.full" ]] ; then
# mvn_args="package '-P!private'"
# post_script=" && private/src/main/scripts/shell/copy_release.sh public/queue-package/target/Queue-*.tar.bz2"
# post_script=" && private/src/main/scripts/shell/copy_release.sh protected/gatk-queue-package-distribution/target/Queue-*.tar.bz2"
elif [[ "${arg}" == "build-picard-private" ]] ; then
mvn_args="mvn install -f private/picard-maven/pom.xml"
@ -113,7 +115,7 @@ for arg in "${@}" ; do
mvn_args="${mvn_args} -Dgatk.queuetests.run=true"
elif [[ "${arg}" == "committests" ]] ; then
mvn_args="verify -Dgatk.committests.skipped=false"
mvn_args="${default_args} -Dgatk.committests.skipped=false"
elif [[ "${arg}" == "test" ]] ; then
mvn_args="test -Dgatk.unittests.skipped=false"
@ -122,19 +124,19 @@ for arg in "${@}" ; do
mvn_args="test -Dgatk.unittests.skipped=false"
elif [[ "${arg}" == "integrationtest" ]] ; then
mvn_args="verify -Dgatk.integrationtests.skipped=false"
mvn_args="${default_args} -Dgatk.integrationtests.skipped=false"
elif [[ "${arg}" == "largescaletest" ]] ; then
mvn_args="verify -Dgatk.largescaletests.skipped=false"
mvn_args="${default_args} -Dgatk.largescaletests.skipped=false"
elif [[ "${arg}" == "knowledgebasetest" ]] ; then
mvn_args="verify -Dgatk.knowledgebasetests.skipped=false"
mvn_args="${default_args} -Dgatk.knowledgebasetests.skipped=false"
elif [[ "${arg}" == "queuetest" ]] ; then
mvn_args="verify -Dgatk.queuetests.skipped=false"
mvn_args="${default_args} -Dgatk.queuetests.skipped=false"
elif [[ "${arg}" == "queuetestrun" ]] ; then
mvn_args="verify -Dgatk.queuetests.skipped=false -Dgatk.queuetests.run=true"
mvn_args="${default_args} -Dgatk.queuetests.skipped=false -Dgatk.queuetests.run=true"
elif [[ "${arg}" == "fasttest" ]] ; then
mvn_args="verify -Dgatk.committests.skipped=false -pl private/gatk-tools-private -am -Dresource.bundle.skip=true"

103
pom.xml
View File

@ -33,11 +33,14 @@
<!-- TODO: Need a better a way to say "don't include hidden" by default -->
<gatkdocs.include.hidden>-build-timestamp "${maven.build.timestamp}"</gatkdocs.include.hidden>
<gatk.shell.directory>${gatk.basedir}/public/src/main/scripts/shell</gatk.shell.directory>
<gatk.assembly.directory>${gatk.basedir}/public/src/main/assembly</gatk.assembly.directory>
<!--
Phases of the build that may be disabled to speed up compilation.
-->
<gatk.jar.phase>package</gatk.jar.phase>
<gatk.unpack.phase>prepare-package</gatk.unpack.phase>
<gatk.shade.phase>package</gatk.shade.phase>
<gatk.generate-resources.phase>generate-resources</gatk.generate-resources.phase>
<gatk.process-resources.phase>process-resources</gatk.process-resources.phase>
<gatk.process-test-resources.phase>process-test-resources</gatk.process-test-resources.phase>
@ -66,6 +69,16 @@
<gatk.serialqueuetests.skipped>${gatk.serialcommittests.skipped}</gatk.serialqueuetests.skipped>
<gatk.seriallargescaletests.skipped>true</gatk.seriallargescaletests.skipped>
<gatk.serialknowledgebasetests.skipped>true</gatk.serialknowledgebasetests.skipped>
<!-- Full path to write the executable MANIFEST.MF only jars, 10 seconds to create at 4KB each, and the accompanying lib directory -->
<gatk.executable.directory>${gatk.basedir}/target/executable</gatk.executable.directory>
<!-- Full path to write (symlinks to) the full shaded package jars, 1+ minute to create at 12MB+ each -->
<gatk.package.directory>${gatk.basedir}/target/package</gatk.package.directory>
<!--
Full path to write symlink to either an executable MANIFEST.MF only jar - OR - a fully shaded package jar.
NOTE: MANIFEST.MF only jars MUST be accompanied by the lib folder, or they will not run.
-->
<gatk.shortcut.directory>${gatk.basedir}/target</gatk.shortcut.directory>
</properties>
<dependencies>
@ -139,6 +152,17 @@
<excludes>${resource.bundle.path}</excludes>
</configuration>
</execution>
<execution>
<id>executable-jar-lib</id>
<goals>
<goal>copy-dependencies</goal>
</goals>
<phase>none</phase>
<configuration>
<outputDirectory>${gatk.executable.directory}/lib</outputDirectory>
<includeScope>runtime</includeScope>
</configuration>
</execution>
</executions>
</plugin>
<!-- TODO: Change the ResourceBundleExtractorDoclet to not require log4j.properties file -->
@ -319,10 +343,40 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<id>executable-jar</id>
<goals>
<goal>jar</goal>
</goals>
<phase>none</phase>
<configuration>
<classesDirectory>${project.build.outputDirectory}/ignored_by_executable_jar</classesDirectory>
<outputDirectory>${gatk.executable.directory}</outputDirectory>
<finalName>${gatk.binary-dist.name}</finalName>
<archive>
<manifest>
<mainClass>${app.main.class}</mainClass>
<addClasspath>true</addClasspath>
<classpathPrefix>lib/</classpathPrefix>
</manifest>
</archive>
</configuration>
</execution>
<execution>
<id>default-jar</id>
<phase>${gatk.jar.phase}</phase>
</execution>
<!--
Maven keeps executing default-jar first, even if it's listed AFTER the executable-jar.
So while packaging: run executable-jar, disable default-jar, then run unshaded-default-jar.
-->
<execution>
<id>unshaded-default-jar</id>
<goals>
<goal>jar</goal>
</goals>
<phase>none</phase>
</execution>
<execution>
<id>test-jar</id>
<goals>
@ -341,7 +395,7 @@
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<id>gatk-executable</id>
<id>package-jar</id>
<goals>
<goal>shade</goal>
</goals>
@ -405,7 +459,7 @@
<phase>none</phase>
<configuration>
<descriptors>
<descriptor>src/main/assembly/binary-dist.xml</descriptor>
<descriptor>${gatk.assembly.directory}/binary-dist.xml</descriptor>
</descriptors>
</configuration>
</execution>
@ -437,7 +491,7 @@
</configuration>
</execution>
<execution>
<id>link-binary-jar</id>
<id>link-executable-jar</id>
<goals>
<goal>link</goal>
</goals>
@ -445,7 +499,26 @@
<configuration>
<links>
<link>
<dst>${gatk.basedir}/target/${gatk.binary-dist.name}.${project.packaging}</dst>
<dst>${gatk.shortcut.directory}/${gatk.binary-dist.name}.${project.packaging}</dst>
<src>${gatk.executable.directory}/${gatk.binary-dist.name}.${project.packaging}</src>
</link>
</links>
</configuration>
</execution>
<execution>
<id>link-package-jar</id>
<goals>
<goal>link</goal>
</goals>
<phase>none</phase>
<configuration>
<links>
<link>
<dst>${gatk.package.directory}/${gatk.binary-dist.name}.${project.packaging}</dst>
<src>${project.build.directory}/${project.build.finalName}.${project.packaging}</src>
</link>
<link>
<dst>${gatk.shortcut.directory}/${gatk.binary-dist.name}.${project.packaging}</dst>
<src>${project.build.directory}/${project.build.finalName}.${project.packaging}</src>
</link>
</links>
@ -748,6 +821,26 @@
</modules>
</profile>
<!-- Optionally do not shade/package jars -->
<!--
NOTE: Profile id "fast" comes from comments in PR #771.
The name is meant to be memorable, but is highly non-specific. Users are forewarned that
behavior of this profile, or the identifier itself, may be heavily modified in the future.
Hardcode usage in non-VCS controlled scripts at your own risk.
-->
<profile>
<id>fast</id>
<activation>
<property>
<name>disable.shadepackage</name>
</property>
</activation>
<properties>
<gatk.unpack.phase>none</gatk.unpack.phase>
<gatk.shade.phase>none</gatk.shade.phase>
</properties>
</profile>
<!-- Collection of properties for use during package testing -->
<profile>
<id>packagetests-enabled</id>
@ -761,6 +854,8 @@
<maven.javadoc.skip>true</maven.javadoc.skip>
<gatk.generate-gatk-extensions.skipped>true</gatk.generate-gatk-extensions.skipped>
<gatk.jar.phase>none</gatk.jar.phase>
<gatk.unpack.phase>none</gatk.unpack.phase>
<gatk.shade.phase>none</gatk.shade.phase>
<gatk.generate-resources.phase>none</gatk.generate-resources.phase>
<gatk.process-resources.phase>none</gatk.process-resources.phase>
<gatk.process-test-resources.phase>none</gatk.process-test-resources.phase>

47
protected/gatk-package-distribution/pom.xml
View File

@ -15,8 +15,6 @@
<properties>
<gatk.basedir>${project.basedir}/../..</gatk.basedir>
<gatk.unpack.phase>prepare-package</gatk.unpack.phase>
<gatk.shade.phase>package</gatk.shade.phase>
<app.main.class>org.broadinstitute.gatk.engine.CommandLineGATK</app.main.class>
<gatk.binary-dist.name>GenomeAnalysisTK</gatk.binary-dist.name>
</properties>
@ -169,6 +167,25 @@
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<id>executable-jar</id>
<phase>${gatk.jar.phase}</phase>
</execution>
<execution>
<id>default-jar</id>
<phase>none</phase>
</execution>
<execution>
<id>unshaded-default-jar</id>
<phase>${gatk.jar.phase}</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
@ -177,6 +194,10 @@
<id>unpack-direct-dependencies</id>
<phase>${gatk.unpack.phase}</phase>
</execution>
<execution>
<id>executable-jar-lib</id>
<phase>${gatk.jar.phase}</phase>
</execution>
</executions>
</plugin>
@ -185,7 +206,7 @@
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<id>gatk-executable</id>
<id>package-jar</id>
<phase>${gatk.shade.phase}</phase>
</execution>
</executions>
@ -207,7 +228,11 @@
<artifactId>maven-junction-plugin</artifactId>
<executions>
<execution>
<id>link-binary-jar</id>
<id>link-executable-jar</id>
<phase>${gatk.jar.phase}</phase>
</execution>
<execution>
<id>link-package-jar</id>
<phase>${gatk.shade.phase}</phase>
</execution>
<execution>
@ -236,20 +261,6 @@
</build>
<profiles>
<profile>
<id>packagetests-enabled</id>
<activation>
<property>
<name>gatk.packagetests.enabled</name>
<value>true</value>
</property>
</activation>
<properties>
<gatk.jar.phase>none</gatk.jar.phase>
<gatk.unpack.phase>none</gatk.unpack.phase>
<gatk.shade.phase>none</gatk.shade.phase>
</properties>
</profile>
<profile>
<id>gsadev</id>
<activation>

47
protected/gatk-queue-package-distribution/pom.xml
View File

@ -15,8 +15,6 @@
<properties>
<gatk.basedir>${project.basedir}/../..</gatk.basedir>
<gatk.unpack.phase>prepare-package</gatk.unpack.phase>
<gatk.shade.phase>package</gatk.shade.phase>
<gatk.binary-dist.name>Queue</gatk.binary-dist.name>
<app.main.class>org.broadinstitute.gatk.queue.QCommandLine</app.main.class>
</properties>
@ -171,6 +169,25 @@
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<id>executable-jar</id>
<phase>${gatk.jar.phase}</phase>
</execution>
<execution>
<id>default-jar</id>
<phase>none</phase>
</execution>
<execution>
<id>unshaded-default-jar</id>
<phase>${gatk.jar.phase}</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
@ -179,6 +196,10 @@
<id>unpack-direct-dependencies</id>
<phase>${gatk.unpack.phase}</phase>
</execution>
<execution>
<id>executable-jar-lib</id>
<phase>${gatk.jar.phase}</phase>
</execution>
</executions>
</plugin>
@ -187,7 +208,7 @@
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<id>gatk-executable</id>
<id>package-jar</id>
<phase>${gatk.shade.phase}</phase>
</execution>
</executions>
@ -209,7 +230,11 @@
<artifactId>maven-junction-plugin</artifactId>
<executions>
<execution>
<id>link-binary-jar</id>
<id>link-executable-jar</id>
<phase>${gatk.jar.phase}</phase>
</execution>
<execution>
<id>link-package-jar</id>
<phase>${gatk.shade.phase}</phase>
</execution>
<execution>
@ -238,20 +263,6 @@
</build>
<profiles>
<profile>
<id>packagetests-enabled</id>
<activation>
<property>
<name>gatk.packagetests.enabled</name>
<value>true</value>
</property>
</activation>
<properties>
<gatk.jar.phase>none</gatk.jar.phase>
<gatk.unpack.phase>none</gatk.unpack.phase>
<gatk.shade.phase>none</gatk.shade.phase>
</properties>
</profile>
<profile>
<id>gsadev</id>
<activation>

23
protected/gatk-queue-package-distribution/src/main/assembly/binary-dist.xml
View File

@ -1,23 +0,0 @@
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<id>binary-dist</id>
<formats>
<format>tar.bz2</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<dependencySets>
<dependencySet>
<includes>
<include>org.broadinstitute.gatk:gatk-queue-package-distribution</include>
</includes>
<outputFileNameMapping>${gatk.binary-dist.name}.${artifact.extension}</outputFileNameMapping>
</dependencySet>
<dependencySet>
<outputDirectory>resources</outputDirectory>
<unpack>true</unpack>
<includes>
<include>org.broadinstitute.gatk:gatk-engine:tar.bz2:example-resources</include>
<include>org.broadinstitute.gatk:gatk-queue-extensions-public:tar.bz2:example-resources</include>
</includes>
</dependencySet>
</dependencySets>
</assembly>

23
public/external-example/pom.xml
View File

@ -28,7 +28,7 @@
<gatk.integrationtests.skipped>${gatk.committests.skipped}</gatk.integrationtests.skipped>
<!-- This flag is used by the package tests to disable re-shading -->
<gatk.unpack.phase>process-resources</gatk.unpack.phase>
<gatk.unpack.phase>prepare-package</gatk.unpack.phase>
<gatk.shade.phase>package</gatk.shade.phase>
</properties>
@ -259,7 +259,28 @@
</plugins>
</build>
<!-- These profiles are used by the GATK build in a multi-module setting. You do NOT need these profiles. -->
<profiles>
<!-- Optionally do not shade/package jars -->
<!--
NOTE: Profile id "fast" comes from comments in PR #771.
The name is meant to be memorable, but is highly non-specific. Users are forewarned that
behavior of this profile, or the identifier itself, may be heavily modified in the future.
Hardcode usage in non-VCS controlled scripts at your own risk.
-->
<profile>
<id>fast</id>
<activation>
<property>
<name>disable.shadepackage</name>
</property>
</activation>
<properties>
<gatk.unpack.phase>none</gatk.unpack.phase>
<gatk.shade.phase>none</gatk.shade.phase>
</properties>
</profile>
<!-- Collection of properties for use during package testing -->
<profile>
<id>packagetests-enabled</id>
<activation>

4
protected/gatk-package-distribution/src/main/assembly/binary-dist.xml → public/src/main/assembly/binary-dist.xml
View File

@ -7,7 +7,7 @@
<dependencySets>
<dependencySet>
<includes>
<include>org.broadinstitute.gatk:gatk-package-distribution</include>
<include>${groupId}:${artifactId}</include>
</includes>
<outputFileNameMapping>${gatk.binary-dist.name}.${artifact.extension}</outputFileNameMapping>
</dependencySet>
@ -15,7 +15,7 @@
<outputDirectory>resources</outputDirectory>
<unpack>true</unpack>
<includes>
<include>org.broadinstitute.gatk:gatk-engine:tar.bz2:example-resources</include>
<include>org.broadinstitute.gatk:*:tar.bz2:example-resources</include>
</includes>
</dependencySet>
</dependencySets>