From d1a689af334580481524bb821e68ebb48535bec1 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Wed, 29 Jan 2014 14:54:24 -0500 Subject: [PATCH] Added new utility files used by maven build, including the ant-bridge script. --- ant-bridge.sh | 158 ++++++++++++++++++ .../org/mycompany/app/MyExampleWalker.java | 56 +++++++ .../app/MyExampleWalkerIntegrationTest.java | 54 ++++++ .../app/MyExampleWalkerUnitTest.java | 41 +++++ .../src/main/assembly/example-resources.xml | 37 ++++ .../src/main/assembly/binary-dist.xml | 22 +++ public/gsalib/src/assembly/gsalib.xml | 13 ++ .../src/main/assembly/example-resources.xml | 20 +++ .../src/main/assembly/binary-dist.xml | 23 +++ .../sting/utils/help/log4j.properties | 7 + 10 files changed, 431 insertions(+) create mode 100755 ant-bridge.sh create mode 100644 public/external-example/src/main/java/org/mycompany/app/MyExampleWalker.java create mode 100644 public/external-example/src/test/java/org/mycompany/app/MyExampleWalkerIntegrationTest.java create mode 100644 public/external-example/src/test/java/org/mycompany/app/MyExampleWalkerUnitTest.java create mode 100644 public/gatk-framework/src/main/assembly/example-resources.xml create mode 100644 public/gatk-package/src/main/assembly/binary-dist.xml create mode 100644 public/gsalib/src/assembly/gsalib.xml create mode 100644 public/queue-framework/src/main/assembly/example-resources.xml create mode 100644 public/queue-package/src/main/assembly/binary-dist.xml create mode 100644 public/sting-utils/src/main/config/org/broadinstitute/sting/utils/help/log4j.properties diff --git a/ant-bridge.sh b/ant-bridge.sh new file mode 100755 index 000000000..f374cd99e --- /dev/null +++ b/ant-bridge.sh @@ -0,0 +1,158 @@ +#!/bin/sh + +mvn_args="test-compile" +mvn_properties= +mvn_clean= +unknown_args= +property_regex='-D(.*)=(.*)' +unit_test_regex='.*UnitTest' +post_script= + +for arg in "${@}" ; do + if [ "${arg}" = "clean" ] ; then + mvn_clean="clean" + mvn_args= + + elif [[ "${arg}" =~ ${property_regex} ]] ; then + property_name=${BASH_REMATCH[1]} + property_value=${BASH_REMATCH[2]} + + if [ "${property_name}" = "single" ] ; then + test_property="test" + test_disabled="it.test" + if [[ ! "${property_value}" =~ ${unit_test_regex} ]] ; then + test_property="it.test" + test_disabled="test" + fi + + mvn_properties="${mvn_properties} -D${test_disabled}=disabled -D${test_property}=${property_value}" + + elif [ "${property_name}" = "test.debug.port" ] ; then + mvn_properties="${mvn_properties} -Dmaven.surefire.debug=\"-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=${property_value}\"" + + elif [ "${property_name}" = "test.default.maxmemory" ] ; then + mvn_properties="${mvn_properties} -Dtest.maxmemory=${property_value}" + + else + unknown_args="${unknown_args} \"${arg}\"" + + fi + + else + if [ "${arg}" != "dist" -a "${mvn_args}" != "test-compile" ]; then + echo "Sorry, this script does not currently support mixing targets." >&2 + exit 1 + + elif [ "${arg}" = "dist" ] ; then + mvn_args="test-compile" + + elif [ "${arg}" = "gatkdocs" ] ; then + local_repo="sitetemprepo" + mvn_args="install -Dmaven.repo.local=${local_repo} -Ddisable.queue && mvn site -Dmaven.repo.local=${local_repo} -Ddisable.queue" + + elif [ "${arg}" = "package.gatk.full" ] ; then + mvn_args="package '-P!private,!queue'" + + elif [ "${arg}" = "package.gatk.all" ] ; then + mvn_args="package '-P!queue'" + + elif [ "${arg}" = "package.queue.full" ] ; then + mvn_args="package '-P!private'" + + elif [ "${arg}" = "package.queue.all" ] ; then + mvn_args="package" + + elif [ "${arg}" = "release.gatk.full" ] ; then + mvn_args="package '-P!private,!queue'" + post_script=" && private/src/main/scripts/shell/copy_release.sh public/gatk-package/target/GenomeAnalysisTK-*.tar.bz2" + + elif [ "${arg}" = "release.queue.full" ] ; then + mvn_args="package '-P!private'" + post_script=" && private/src/main/scripts/shell/copy_release.sh public/queue-package/target/Queue-*.tar.bz2" + + elif [ "${arg}" = "build-picard-private" ] ; then + mvn_args="mvn install -f private/picard-maven/pom.xml" + + # TODO: clover support + # see ant and maven docs for clover: + # https://confluence.atlassian.com/display/CLOVER/1.+QuickStart+Guide + # https://confluence.atlassian.com/display/CLOVER/Clover-for-Maven+2+and+3+User%27s+Guide + # + #elif [ "${arg}" = "clover.report" ] ; then + # mvn_args=... + # + #elif [ "${arg}" = "with.clover" ] ; then + # mvn_args=... + + # TODO: This runs *all* commit tests, including the few on Queue. + elif [ "${arg}" = "gatkfull.binary.release.tests" ] ; then + local_repo="sitetemprepo" + mvn_args="install -Dmaven.repo.local=${local_repo} && verify" + mvn_args="${mvn_args} -Dmaven.repo.local=${local_repo} -Dmaven.javadoc.skip=true" + mvn_args="${mvn_args} -Dsting.generate-gatk-extensions.skipped=true" + mvn_args="${mvn_args} -Dsting.jar.phase=none -Dsting.unpack.phase=none -Dsting.shade.phase=none" + mvn_args="${mvn_args} -Dsting.packagecommittests.skipped=false" + + # TODO: This runs only the pipeline tests (full, non-dry run), but not the commit tests for Queue. + elif [ "${arg}" = "queuefull.binary.release.tests" ] ; then + local_repo="sitetemprepo" + mvn_args="install -Dmaven.repo.local=${local_repo} && verify" + mvn_args="${mvn_args} -Dmaven.repo.local=${local_repo} -Dmaven.javadoc.skip=true" + mvn_args="${mvn_args} -Dsting.generate-gatk-extensions.skipped=true" + mvn_args="${mvn_args} -Dsting.jar.phase=none -Dsting.unpack.phase=none -Dsting.shade.phase=none" + mvn_args="${mvn_args} -Dsting.packagepipelinetests.skipped=false" + mvn_args="${mvn_args} -Dsting.pipelinetests.run=true" + + elif [ "${arg}" = "committests" ] ; then + mvn_args="verify -Dsting.committests.skipped=false" + + elif [ "${arg}" = "test" ] ; then + mvn_args="test -Dsting.unittests.skipped=false" + + elif [ "${arg}" = "unittest" ] ; then + mvn_args="test -Dsting.unittests.skipped=false" + + elif [ "${arg}" = "integrationtest" ] ; then + mvn_args="verify -Dsting.integrationtests.skipped=false" + + elif [ "${arg}" = "largescaletest" ] ; then + mvn_args="verify -Dsting.largescaletests.skipped=false" + + elif [ "${arg}" = "pipelinetest" ] ; then + mvn_args="verify -Dsting.pipelinetests.skipped=false" + + elif [ "${arg}" = "pipelinetestrun" ] ; then + mvn_args="verify -Dsting.pipelinetests.skipped=false -Dsting.pipelinetests.run=true" + + elif [ "${arg}" = "fasttest" ] ; then + mvn_args="verify -Dsting.committests.skipped=false -pl private/gatk-private -am -Dresource.bundle.skip=true" + + else + unknown_args="${unknown_args} \"${arg}\"" + + fi + + fi + +done + +mvn_cmd= +if [ "${mvn_clean}" != "" ]; then + if [ "${mvn_args}" != "" ]; then + mvn_cmd="mvn ${mvn_clean} && mvn ${mvn_args}" + else + mvn_cmd="mvn ${mvn_clean}" + fi +else + mvn_cmd="mvn ${mvn_args}" +fi + +if [ "${unknown_args}" != "" ]; then + echo "Unrecognized arguments:${unknown_args}" >&2 + +else + echo "Equivalent maven command" + echo "${mvn_cmd}${mvn_properties}${post_script}" + sh -c "${mvn_cmd}${mvn_properties}${post_script}" + +fi diff --git a/public/external-example/src/main/java/org/mycompany/app/MyExampleWalker.java b/public/external-example/src/main/java/org/mycompany/app/MyExampleWalker.java new file mode 100644 index 000000000..d65c47c99 --- /dev/null +++ b/public/external-example/src/main/java/org/mycompany/app/MyExampleWalker.java @@ -0,0 +1,56 @@ +/* +* Copyright (c) 2012 The Broad Institute +* +* Permission is hereby granted, free of charge, to any person +* obtaining a copy of this software and associated documentation +* files (the "Software"), to deal in the Software without +* restriction, including without limitation the rights to use, +* copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following +* conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +* THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +package org.mycompany.app; + +import org.broadinstitute.sting.commandline.Output; +import org.broadinstitute.sting.gatk.contexts.AlignmentContext; +import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; +import org.broadinstitute.sting.gatk.walkers.LocusWalker; + +import java.io.PrintStream; + +/** + * An example walker that looks surprisingly like CountLoci. + */ +public class MyExampleWalker extends LocusWalker { + @Output + PrintStream out; + + public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) { + return 1; + } + + public Long reduceInit() { return 0l; } + + public Long reduce(Integer value, Long sum) { + return value + sum; + } + + public void onTraversalDone( Long c ) { + out.println(c); + } +} diff --git a/public/external-example/src/test/java/org/mycompany/app/MyExampleWalkerIntegrationTest.java b/public/external-example/src/test/java/org/mycompany/app/MyExampleWalkerIntegrationTest.java new file mode 100644 index 000000000..777079426 --- /dev/null +++ b/public/external-example/src/test/java/org/mycompany/app/MyExampleWalkerIntegrationTest.java @@ -0,0 +1,54 @@ +/* +* Copyright (c) 2012 The Broad Institute +* +* Permission is hereby granted, free of charge, to any person +* obtaining a copy of this software and associated documentation +* files (the "Software"), to deal in the Software without +* restriction, including without limitation the rights to use, +* copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following +* conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +* THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +package org.mycompany.app; + +import org.broadinstitute.sting.WalkerTest; +import org.testng.annotations.Test; + +import java.io.File; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Collections; +import java.util.MissingResourceException; + +/** + * NOTE: Currently the testing infrastructure for walkers does not support running outside the Broad. + */ +public class MyExampleWalkerIntegrationTest extends WalkerTest { + @Test + public void testMyExampleWalker() throws URISyntaxException { + String gatk_args = String.format("-T MyExampleWalker -I %s -R %s", getResource("/exampleBAM.bam"), getResource("/exampleFASTA.fasta")); + WalkerTestSpec spec = new WalkerTestSpec(gatk_args, Collections.emptyList()); + executeTest("Testing count on the example bam", spec); + } + + private File getResource(String path) throws URISyntaxException { + URL resourceUrl = getClass().getResource(path); + if (resourceUrl == null) + throw new MissingResourceException("Resource not found: " + path, getClass().getSimpleName(), path); + return new File(resourceUrl.toURI()); + } +} diff --git a/public/external-example/src/test/java/org/mycompany/app/MyExampleWalkerUnitTest.java b/public/external-example/src/test/java/org/mycompany/app/MyExampleWalkerUnitTest.java new file mode 100644 index 000000000..e3e0c81ea --- /dev/null +++ b/public/external-example/src/test/java/org/mycompany/app/MyExampleWalkerUnitTest.java @@ -0,0 +1,41 @@ +/* +* Copyright (c) 2012 The Broad Institute +* +* Permission is hereby granted, free of charge, to any person +* obtaining a copy of this software and associated documentation +* files (the "Software"), to deal in the Software without +* restriction, including without limitation the rights to use, +* copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following +* conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +* THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +package org.mycompany.app; + +import org.broadinstitute.sting.BaseTest; +import org.testng.Assert; +import org.testng.annotations.Test; + +/** + * NOTE: Currently the testing infrastructure for walkers does not support running outside the Broad. + */ +public class MyExampleWalkerUnitTest extends BaseTest { + @Test + public void testMyExampleWalker() { + MyExampleWalker walker = new MyExampleWalker(); + Assert.assertEquals((long)walker.reduce(1, 1L), 2L); + } +} diff --git a/public/gatk-framework/src/main/assembly/example-resources.xml b/public/gatk-framework/src/main/assembly/example-resources.xml new file mode 100644 index 000000000..b285cc05f --- /dev/null +++ b/public/gatk-framework/src/main/assembly/example-resources.xml @@ -0,0 +1,37 @@ + + example-resources + + tar.bz2 + + false + + + ${project.build.sourceDirectory}/org/broadinstitute/sting/gatk/walkers/qc + . + + Pileup.java + CountLoci.java + CountReads.java + CheckPileup.java + + + + ${project.build.sourceDirectory}/org/broadinstitute/sting/gatk/walkers/readutils + . + + PrintReads.java + + + + src/test/resources + . + + exampleBAM.bam + exampleBAM.bam.bai + exampleFASTA.fasta + exampleFASTA.fasta.fai + exampleFASTA.dict + + + + diff --git a/public/gatk-package/src/main/assembly/binary-dist.xml b/public/gatk-package/src/main/assembly/binary-dist.xml new file mode 100644 index 000000000..adc52646c --- /dev/null +++ b/public/gatk-package/src/main/assembly/binary-dist.xml @@ -0,0 +1,22 @@ + + binary-dist + + tar.bz2 + + false + + + + org.broadinstitute.sting:gatk-package + + ${sting.binary-dist.name}.${artifact.extension} + + + resources + true + + org.broadinstitute.sting:gatk-framework:tar.bz2:example-resources + + + + diff --git a/public/gsalib/src/assembly/gsalib.xml b/public/gsalib/src/assembly/gsalib.xml new file mode 100644 index 000000000..7650c713d --- /dev/null +++ b/public/gsalib/src/assembly/gsalib.xml @@ -0,0 +1,13 @@ + + gsalib + + tar.gz + + false + + + gsalib + src/R + + + diff --git a/public/queue-framework/src/main/assembly/example-resources.xml b/public/queue-framework/src/main/assembly/example-resources.xml new file mode 100644 index 000000000..7d4ec43ef --- /dev/null +++ b/public/queue-framework/src/main/assembly/example-resources.xml @@ -0,0 +1,20 @@ + + example-resources + + tar.bz2 + + false + + + src/main/qscripts/org/broadinstitute/sting/queue/qscripts/examples + . + + ExampleCountReads.scala + ExampleCountLoci.scala + ExampleUnifiedGenotyper.scala + ExampleReadFilter.scala + ExampleCustomWalker.scala + + + + diff --git a/public/queue-package/src/main/assembly/binary-dist.xml b/public/queue-package/src/main/assembly/binary-dist.xml new file mode 100644 index 000000000..6de236a56 --- /dev/null +++ b/public/queue-package/src/main/assembly/binary-dist.xml @@ -0,0 +1,23 @@ + + binary-dist + + tar.bz2 + + false + + + + org.broadinstitute.sting:queue-package + + ${sting.binary-dist.name}.${artifact.extension} + + + resources + true + + org.broadinstitute.sting:gatk-framework:tar.bz2:example-resources + org.broadinstitute.sting:queue-framework:tar.bz2:example-resources + + + + diff --git a/public/sting-utils/src/main/config/org/broadinstitute/sting/utils/help/log4j.properties b/public/sting-utils/src/main/config/org/broadinstitute/sting/utils/help/log4j.properties new file mode 100644 index 000000000..38c8335c9 --- /dev/null +++ b/public/sting-utils/src/main/config/org/broadinstitute/sting/utils/help/log4j.properties @@ -0,0 +1,7 @@ +# Root logger option +log4j.rootLogger=INFO, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.SimpleLayout