zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Build system round 2: finished preparing the packaging system for dual protected/lite releases 

* Targets to package and release lite/protected versions of the GATK/Queue
* Still TODO: -determine the actual directories where the protected releases should go
              -update the Bamboo release plan
              -fix a bug in the binary release test targets
This commit is contained in:
David Roazen 2012-05-24 12:05:14 -04:00
parent c3c7f17d90
commit 5e76c811b9
8 changed files with 275 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
build.xml
View File

@ -74,7 +74,7 @@
<property name="package.xml.dir" value="${public.dir}/packages" />
<property name="package.output.dir" value="${dist.dir}/packages" />
<property name="staging.dir" value="staging" />
<property name="executable" value="GenomeAnalysisTK" /> <!-- If running the 'package' task, this property controls the name of the xml file to package -->
<property name="default.executable" value="none" />
<!-- Javadoc/Scaladoc directories -->
<property name="javadoc.dir" value="javadoc" />
@ -326,6 +326,10 @@
<property name="compile.scala" value="false" />
</target>
<target name="init.javaandscala">
<property name="compile.scala" value="true" />
</target>
<target name="init.usecontracts">
<property name="use.contracts" value="true" />
</target>
@ -341,6 +345,7 @@
<property name="build.target" value="${default.build.target}" />
<property name="compile.scala" value="${compile.scala.by.default}" />
<property name="use.contracts" value="${use.contracts.by.default}" />
<property name="executable" value="${default.executable}" />
<!-- Determine what should be built -->
<condition property="include.scala">
@ -842,6 +847,32 @@
<!-- Release-related tasks -->
<!-- ******************************************************************************** -->
<target name="init.buildgatkfull" depends="init.publicprotectedonly, init.javaonly">
<property name="executable" value="GenomeAnalysisTK" />
</target>
<target name="init.buildgatklite" depends="init.publiconly, init.javaonly">
<property name="executable" value="GenomeAnalysisTKLite" />
</target>
<target name="init.buildqueuefull" depends="init.publicprotectedonly, init.javaandscala">
<property name="executable" value="Queue" />
</target>
<target name="init.buildqueuelite" depends="init.publiconly, init.javaandscala">
<property name="executable" value="QueueLite" />
</target>
<target name="require.executable">
<condition property="no.executable.defined">
<or>
<equals arg1="${executable}" arg2="none" />
<equals arg1="${executable}" arg2="$${executable}" />
</or>
</condition>
<fail message="No executable defined. Call a more specific packaging/release target, or define an executable manually" if="no.executable.defined" />
</target>
<!-- Unzip all classes from their current locations and assemble them in a staging directory -->
<target name="stage" description="stage files for distribution">
<mkdir dir="${staging.dir}"/>
@ -863,8 +894,9 @@
</unjar>
</target>
<!-- Build a package consisting of all supporting files -->
<target name="package" depends="dist,stage" description="bundle up an executable for distribution">
<!-- Build a package consisting of all supporting files. Don't call this target directly. Call one of the specific packaging targets below -->
<target name="package" depends="dist,stage,require.executable" description="bundle up an executable for distribution">
<mkdir dir="${package.output.dir}" />
<xslt destdir="${package.output.dir}" style="${package.xml.dir}/CreatePackager.xsl" useImplicitFileset="false">
<flattenmapper/>
@ -878,15 +910,30 @@
<ant antfile="${package.output.dir}/${executable}.xml" target="package" />
</target>
<target name="package.gatk" depends="init.javaonly, package" />
<!-- Package specific versions of the GATK/Queue. ALWAYS do an ant clean before invoking these! -->
<target name="package.gatk.full" depends="init.buildgatkfull,package" />
<target name="release" depends="package" description="release a build, putting each file in a location specified by the package">
<ant antfile="${package.output.dir}/${executable}.xml" target="release" />
<target name="package.gatk.lite" depends="init.buildgatklite,package" />
<target name="package.queue.full" depends="init.buildqueuefull,package" />
<target name="package.queue.lite" depends="init.buildqueuelite,package" />
<!-- Release a build. Don't call this target directly. Call one of the specific release targets below -->
<target name="release" depends="require.executable" description="release a build, putting each file in a location specified by the package">
<ant antfile="${package.output.dir}/${executable}.xml" target="release" />
</target>
<target name="protected.release" depends="package" description="release a protected build, putting each file in a location specified by the package">
<ant antfile="${package.output.dir}/${executable}.xml" target="protectedrelease" />
</target>
<!-- Release specific versions of the GATK/Queue. ALWAYS do an ant clean before invoking these! -->
<target name="release.gatk.full" depends="package.gatk.full,release" />
<target name="release.gatk.lite" depends="package.gatk.lite,release" />
<target name="release.queue.full" depends="package.queue.full,release" />
<target name="release.queue.lite" depends="package.queue.lite,release" />
<!-- Build a subset of picard with only those classes we need by completely abusing the packaging system -->
<!-- TODO: Reuse as much as possible of the 'stage' and 'package' targets -->
@ -1134,6 +1181,7 @@
<mkdir dir="@{outputdir}"/>
<echo message="Sting: Running @{testtype} test cases!"/>
<echo message="Test classpath: ${testng.classpath}"/>
<taskdef resource="testngtasks" classpath="${testng.jar}"/>
<testng outputDir="@{outputdir}"
classpathref="${testng.classpath}"
@ -1188,30 +1236,28 @@
<property name="testng.classpath" value="testng.default.classpath" />
</target>
<target name="init.testgatkjar">
<property name="testng.classpath" value="gatk.binary.release.tests.classpath" />
</target>
<target name="init.testqueuejar">
<property name="testng.classpath" value="queue.binary.release.tests.classpath" />
</target>
<target name="committests">
<antcall target="unittest" inheritAll="false"/>
<antcall target="integrationtest" inheritAll="false"/>
<antcall target="pipelinetest" inheritAll="false"/>
</target>
<target name="gatk.binary.release.tests" depends="package">
<antcall target="unittest" inheritAll="false">
<param name="testng.classpath" value="gatk.binary.release.tests.classpath" />
</antcall>
<antcall target="integrationtest" inheritAll="false">
<param name="testng.classpath" value="gatk.binary.release.tests.classpath" />
</antcall>
</target>
<target name="gatkfull.binary.release.tests" depends="package.gatk.full,init.testgatkjar,unittest,integrationtest" />
<target name="gatklite.binary.release.tests" depends="package.gatk.lite,init.testgatkjar,unittest,integrationtest" />
<target name="queuefull.binary.release.tests" depends="package.queue.full,init.testqueuejar,pipelinetest" />
<target name="queuelite.binary.release.tests" depends="package.queue.lite,init.testqueuejar,pipelinetest" />
<target name="queue.binary.release.tests">
<antcall target="package" inheritAll="false">
<param name="executable" value="Queue" />
</antcall>
<antcall target="pipelinetest" inheritAll="false">
<param name="executable" value="Queue" />
<param name="testng.classpath" value="queue.binary.release.tests.classpath" />
</antcall>
</target>
<!-- Our four different test types: UnitTest, IntegrationTest, LargeScaleTest, PipelineTest -->
<target name="unittest" depends="test.compile,test.init" description="Run unit tests">

15
public/packages/CreatePackager.xsl
View File

@ -100,21 +100,6 @@
</xsl:if>
</xsl:for-each>
</target>
<target name="protectedrelease">
<xsl:for-each select="protectedrelease/executable">
<copy todir="{@directory}/{$package.basename}"><fileset dir="{$package.dir}" /></copy>
<xsl:if test="@symlink">
<symlink link="{@directory}/{@symlink}" resource="{$package.basename}" overwrite="true" />
</xsl:if>
</xsl:for-each>
<xsl:for-each select="protectedrelease/archive">
<copy file="{$dist.dir}/packages/{$package.filename}" todir="{@directory}" />
<xsl:if test="@symlink">
<symlink link="{@directory}/{@symlink}" resource="{$package.filename}" overwrite="true" />
</xsl:if>
</xsl:for-each>
</target>
</project>
</xsl:template>

7
public/packages/GATKEngine.xml
View File

@ -47,5 +47,12 @@
<file name="public/testdata/exampleFASTA.fasta" />
<file name="public/testdata/exampleFASTA.fasta.fai" />
<file name="public/testdata/exampleFASTA.dict" />
<!-- GATK sample code and build scripts -->
<file name="public/java/src/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageWalker.java" />
<file name="public/java/src/org/broadinstitute/sting/gatk/walkers/PileupWalker.java" />
<file name="public/java/src/org/broadinstitute/sting/gatk/walkers/PrintReadsWalker.java" />
<file name="public/java/src/org/broadinstitute/sting/gatk/walkers/qc/CountLociWalker.java" />
<file name="public/java/src/org/broadinstitute/sting/gatk/walkers/qc/CountReadsWalker.java" />
<file name="public/java/src/org/broadinstitute/sting/gatk/walkers/qc/ValidatingPileupWalker.java" />
</resources>
</package>

44
public/packages/GenomeAnalysisTK.xml
View File

@ -1,4 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright (c) 2012, The Broad Institute
~
~ Permission is hereby granted, free of charge, to any person
~ obtaining a copy of this software and associated documentation
~ files (the "Software"), to deal in the Software without
~ restriction, including without limitation the rights to use,
~ copy, modify, merge, publish, distribute, sublicense, and/or sell
~ copies of the Software, and to permit persons to whom the
~ Software is furnished to do so, subject to the following
~ conditions:
~
~ The above copyright notice and this permission notice shall be
~ included in all copies or substantial portions of the Software.
~ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
~ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
~ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
~ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
~ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
~ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
~ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
~ OTHER DEALINGS IN THE SOFTWARE.
-->
<!-- The full release of the GATK contains both public and protected tools (but nothing from private) -->
<package name="GenomeAnalysisTK">
<version file="StingText.properties" property="org.broadinstitute.sting.gatk.version" />
<executable name="GenomeAnalysisTK">
@ -11,22 +36,9 @@
<modules>
<module file="AnalyzeCovariates.xml"/>
</modules>
<resources>
<!-- GATK sample code and build scripts -->
<file name="public/java/src/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageWalker.java" />
<file name="public/java/src/org/broadinstitute/sting/gatk/walkers/PileupWalker.java" />
<file name="public/java/src/org/broadinstitute/sting/gatk/walkers/PrintReadsWalker.java" />
<file name="public/java/src/org/broadinstitute/sting/gatk/walkers/qc/CountLociWalker.java" />
<file name="public/java/src/org/broadinstitute/sting/gatk/walkers/qc/CountReadsWalker.java" />
<file name="public/java/src/org/broadinstitute/sting/gatk/walkers/qc/ValidatingPileupWalker.java" />
</resources>
<release>
<executable directory="/humgen/gsa-hpprojects/GATK/bin" symlink="current" />
<archive directory="/web/ftp/pub/gsa/GenomeAnalysisTK" symlink="GenomeAnalysisTK-latest.tar.bz2" />
<!-- TODO: determine the real directory where the protected GATK release should go -->
<executable directory="/local/gsa-engineering/test_releases/gatk" symlink="current" />
<archive directory="/local/gsa-engineering/test_releases/gatk" symlink="GenomeAnalysisTK-latest.tar.bz2" />
</release>
<protectedrelease>
<!-- TODO: Determine where the protected releases should be stored -->
<executable directory="" symlink="current" />
<archive directory="" symlink="GenomeAnalysisTK-latest.tar.bz2" />
</protectedrelease>
</package>

43
public/packages/GenomeAnalysisTKLite.xml 100644
View File

@ -0,0 +1,43 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright (c) 2012, The Broad Institute
~
~ Permission is hereby granted, free of charge, to any person
~ obtaining a copy of this software and associated documentation
~ files (the "Software"), to deal in the Software without
~ restriction, including without limitation the rights to use,
~ copy, modify, merge, publish, distribute, sublicense, and/or sell
~ copies of the Software, and to permit persons to whom the
~ Software is furnished to do so, subject to the following
~ conditions:
~
~ The above copyright notice and this permission notice shall be
~ included in all copies or substantial portions of the Software.
~ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
~ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
~ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
~ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
~ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
~ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
~ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
~ OTHER DEALINGS IN THE SOFTWARE.
-->
<!-- The lite release of the GATK contains only public tools (nothing from protected or private) -->
<package name="GenomeAnalysisTKLite">
<version file="StingText.properties" property="org.broadinstitute.sting.gatk.version" />
<executable name="GenomeAnalysisTKLite">
<main-class name="org.broadinstitute.sting.gatk.CommandLineGATK" />
<resource-bundle file="StingText.properties" />
<modules>
<module file="GATKEngine.xml"/>
</modules>
</executable>
<modules>
<module file="AnalyzeCovariates.xml"/>
</modules>
<release>
<executable directory="/humgen/gsa-hpprojects/GATK/bin" symlink="current" />
<archive directory="/web/ftp/pub/gsa/GenomeAnalysisTK" symlink="GenomeAnalysisTKLite-latest.tar.bz2" />
</release>
</package>

49
public/packages/Queue.xml
View File

@ -23,60 +23,21 @@
~ OTHER DEALINGS IN THE SOFTWARE.
-->
<!-- The full release of Queue contains both public and protected tools (but nothing from private) -->
<package name="Queue">
<version file="StingText.properties" property="org.broadinstitute.sting.queue.version" />
<executable name="Queue">
<main-class name="org.broadinstitute.sting.queue.QCommandLine" />
<resource-bundle file="StingText.properties" />
<dependencies>
<!-- Queue -->
<package name="org.broadinstitute.sting.queue.**" />
<!-- Pipeline + Utils -->
<package name="org.broadinstitute.sting.pipeline.**" />
<package name="org.broadinstitute.sting.utils.**" />
<!-- Scala -->
<package name="scala.**" />
<!-- Picard -->
<package name="net.sf.picard.**" />
<package name="edu.mit.broad.picard.**" />
<!-- JavaMail -->
<package name="javax.activation.**" />
<package name="javax.mail.**" />
<package name="com.sun.activation.**" />
<package name="com.sun.mail.**" />
<file path="META-INF/javamail.charset.map" />
<file path="META-INF/javamail.default.address.map" />
<file path="META-INF/javamail.default.providers" />
<file path="META-INF/mailcap" />
<file path="META-INF/mailcap.default" />
<file path="META-INF/mimetypes.default" />
<!-- JNA, including embedded native libraries -->
<dir name="com/sun/jna" />
<!-- Contracts for Java -->
<package name="com.google.java.contract.**" />
<!-- snpEff -->
<package name="ca.mcgill.mcb.pcingola.**" />
<file path="snpEff_genes.ftl" />
<file path="snpEff_summary.ftl" />
<!-- R scripts -->
<dir name="org/broadinstitute/sting/queue" includes="**/*.R" />
</dependencies>
<modules>
<module file="QueueEngine.xml"/>
<module file="GATKEngine.xml"/>
<module file="AnalyzeCovariates.xml"/>
</modules>
</executable>
<release>
<executable directory="/humgen/gsa-hpprojects/Queue/bin" symlink="current" />
<archive directory="/web/ftp/pub/gsa/Queue" symlink="Queue-latest.tar.bz2" />
<!-- TODO: determine the real directory where the protected Queue release should go -->
<executable directory="/local/gsa-engineering/test_releases/queue" symlink="current" />
<archive directory="/local/gsa-engineering/test_releases/queue" symlink="Queue-latest.tar.bz2" />
</release>
</package>

78
public/packages/QueueEngine.xml 100644
View File

@ -0,0 +1,78 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright (c) 2012, The Broad Institute
~
~ Permission is hereby granted, free of charge, to any person
~ obtaining a copy of this software and associated documentation
~ files (the "Software"), to deal in the Software without
~ restriction, including without limitation the rights to use,
~ copy, modify, merge, publish, distribute, sublicense, and/or sell
~ copies of the Software, and to permit persons to whom the
~ Software is furnished to do so, subject to the following
~ conditions:
~
~ The above copyright notice and this permission notice shall be
~ included in all copies or substantial portions of the Software.
~ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
~ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
~ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
~ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
~ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
~ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
~ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
~ OTHER DEALINGS IN THE SOFTWARE.
-->
<package name="QueueEngine">
<executable name="QueueEngine">
<dependencies>
<!-- Queue -->
<package name="org.broadinstitute.sting.queue.**" />
<!-- Pipeline + Utils -->
<package name="org.broadinstitute.sting.pipeline.**" />
<package name="org.broadinstitute.sting.utils.**" />
<!-- Scala -->
<package name="scala.**" />
<!-- Picard -->
<package name="net.sf.picard.**" />
<package name="edu.mit.broad.picard.**" />
<!-- JavaMail -->
<package name="javax.activation.**" />
<package name="javax.mail.**" />
<package name="com.sun.activation.**" />
<package name="com.sun.mail.**" />
<file path="META-INF/javamail.charset.map" />
<file path="META-INF/javamail.default.address.map" />
<file path="META-INF/javamail.default.providers" />
<file path="META-INF/mailcap" />
<file path="META-INF/mailcap.default" />
<file path="META-INF/mimetypes.default" />
<!-- JNA, including embedded native libraries -->
<dir name="com/sun/jna" />
<!-- Contracts for Java -->
<package name="com.google.java.contract.**" />
<!-- snpEff -->
<package name="ca.mcgill.mcb.pcingola.**" />
<file path="snpEff_genes.ftl" />
<file path="snpEff_summary.ftl" />
<!-- R scripts -->
<dir name="org/broadinstitute/sting/queue" includes="**/*.R" />
</dependencies>
</executable>
<resources>
<!-- Sample QScripts -->
<file name="public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleCountReads.scala" />
<file name="public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleCountLoci.scala" />
<file name="public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleUnifiedGenotyper.scala" />
<file name="public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleReadFilter.scala" />
<file name="public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleCustomWalker.scala" />
</resources>
</package>

42
public/packages/QueueLite.xml 100644
View File

@ -0,0 +1,42 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright (c) 2012, The Broad Institute
~
~ Permission is hereby granted, free of charge, to any person
~ obtaining a copy of this software and associated documentation
~ files (the "Software"), to deal in the Software without
~ restriction, including without limitation the rights to use,
~ copy, modify, merge, publish, distribute, sublicense, and/or sell
~ copies of the Software, and to permit persons to whom the
~ Software is furnished to do so, subject to the following
~ conditions:
~
~ The above copyright notice and this permission notice shall be
~ included in all copies or substantial portions of the Software.
~ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
~ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
~ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
~ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
~ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
~ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
~ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
~ OTHER DEALINGS IN THE SOFTWARE.
-->
<!-- The lite release of Queue contains only public tools (nothing from protected or private) -->
<package name="QueueLite">
<version file="StingText.properties" property="org.broadinstitute.sting.queue.version" />
<executable name="QueueLite">
<main-class name="org.broadinstitute.sting.queue.QCommandLine" />
<resource-bundle file="StingText.properties" />
<modules>
<module file="QueueEngine.xml"/>
<module file="GATKEngine.xml"/>
<module file="AnalyzeCovariates.xml"/>
</modules>
</executable>
<release>
<executable directory="/humgen/gsa-hpprojects/Queue/bin" symlink="current" />
<archive directory="/web/ftp/pub/gsa/Queue" symlink="QueueLite-latest.tar.bz2" />
</release>
</package>