Small bug fixes:
Added a property to allow generating Queue extensions around an external version of the GATK. Updated argument order in -help. Restored the ability to use QScript trait imported annotations in constructors. Removing line feeds from email password files. git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@5416 348d0f76-0448-11de-a6fe-93d51630548a
This commit is contained in:
kshakir
parent
4baeb5979f
commit
2058fc12bc
37
build.xml
37
build.xml
|
|
@ -1,3 +1,27 @@
|
|||
<!--
|
||||
~ Copyright (c) 2011, The Broad Institute
|
||||
~
|
||||
~ Permission is hereby granted, free of charge, to any person
|
||||
~ obtaining a copy of this software and associated documentation
|
||||
~ files (the "Software"), to deal in the Software without
|
||||
~ restriction, including without limitation the rights to use,
|
||||
~ copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
~ copies of the Software, and to permit persons to whom the
|
||||
~ Software is furnished to do so, subject to the following
|
||||
~ conditions:
|
||||
~
|
||||
~ The above copyright notice and this permission notice shall be
|
||||
~ included in all copies or substantial portions of the Software.
|
||||
~ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
~ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
~ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
~ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
~ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
~ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
~ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
~ OTHER DEALINGS IN THE SOFTWARE.
|
||||
-->
|
||||
|
||||
<project name="Sting" default="dist" basedir="." xmlns:ivy="antlib:org.apache.ivy.ant">
|
||||
<description>Compile and distribute the Sting toolkit</description>
|
||||
<property name="build.sysclasspath" value="first" />
|
||||
|
|
@ -34,7 +58,7 @@
|
|||
|
||||
<property environment="env"/>
|
||||
|
||||
<property name="drmaa.file" value="${env.SGE_ROOT}/lib/drmaa.jar" />
|
||||
<property name="drmaa.jar" value="${env.SGE_ROOT}/lib/drmaa.jar" />
|
||||
|
||||
<patternset id="java.source.pattern">
|
||||
<include name="**/*.java"/>
|
||||
|
|
@ -111,7 +135,7 @@
|
|||
</target>
|
||||
|
||||
<target name="init.gridengine" depends="init" if="include.gridengine">
|
||||
<copy todir="lib" file="${drmaa.file}"/>
|
||||
<copy todir="lib" file="${drmaa.jar}"/>
|
||||
</target>
|
||||
|
||||
<target name="resolve" depends="init.resolve,init,init.gridengine"
|
||||
|
|
@ -164,7 +188,7 @@
|
|||
<!-- Include Grid Engine in the compile if SGE_ROOT is available. -->
|
||||
<!-- Based off of http://wikis.sun.com/display/GridEngine/Automating+Grid+Engine+Functions+Through+DRMAA -->
|
||||
<condition property="include.gridengine">
|
||||
<available file="${drmaa.file}"/>
|
||||
<available file="${drmaa.jar}"/>
|
||||
</condition>
|
||||
|
||||
<!-- Get the pipeline run type. Default to dry. -->
|
||||
|
|
@ -245,6 +269,7 @@
|
|||
</not>
|
||||
<uptodate targetfile="${queue-extensions.source.dir}">
|
||||
<srcfiles refid="java.class.files"/>
|
||||
<srcfiles file="${queue-extensions.gatk.jar}"/>
|
||||
</uptodate>
|
||||
</or>
|
||||
</condition>
|
||||
|
|
@ -254,11 +279,15 @@
|
|||
<target name="queue-extensions.generate" depends="extracthelp,init.queue-extensions.generate" unless="uptodate.queue-extensions.generate" description="generate GATK modules for Queue">
|
||||
<mkdir dir="${queue-extensions.source.dir}"/>
|
||||
<echo>Generating Queue GATK extensions...</echo>
|
||||
<java fork="true" failonerror="true" classname="org.broadinstitute.sting.queue.extensions.gatk.GATKExtensionsGenerator" classpathref="scala.dependencies">
|
||||
<java fork="true" failonerror="true" classname="org.broadinstitute.sting.queue.extensions.gatk.GATKExtensionsGenerator" >
|
||||
<arg value="-outDir" />
|
||||
<arg path="${queue-extensions.source.dir}" />
|
||||
<arg value="-l" />
|
||||
<arg value="WARN" />
|
||||
<classpath>
|
||||
<file file="${queue-extensions.gatk.jar}"/>
|
||||
<path refid="scala.dependencies"/>
|
||||
</classpath>
|
||||
</java>
|
||||
<touch>
|
||||
<file file="${queue-extensions.source.dir}"/>
|
||||
|
|
|
|||
|
|
@ -1,8 +1,33 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.queue
|
||||
|
||||
import org.broadinstitute.sting.queue.util.Logging
|
||||
import org.broadinstitute.sting.queue.function.QFunction
|
||||
import org.broadinstitute.sting.utils.text.XReadLines
|
||||
import annotation.target.field
|
||||
|
||||
/**
|
||||
* Defines a Queue pipeline as a collection of CommandLineFunctions.
|
||||
|
|
@ -10,16 +35,22 @@ import org.broadinstitute.sting.utils.text.XReadLines
|
|||
trait QScript extends Logging {
|
||||
// Type aliases so users don't have to import
|
||||
type File = java.io.File
|
||||
type Input = org.broadinstitute.sting.commandline.Input
|
||||
type Output = org.broadinstitute.sting.commandline.Output
|
||||
type Argument = org.broadinstitute.sting.commandline.Argument
|
||||
type ArgumentCollection = org.broadinstitute.sting.commandline.ArgumentCollection
|
||||
type CommandLineFunction = org.broadinstitute.sting.queue.function.CommandLineFunction
|
||||
type InProcessFunction = org.broadinstitute.sting.queue.function.InProcessFunction
|
||||
type ScatterGatherableFunction = org.broadinstitute.sting.queue.function.scattergather.ScatterGatherableFunction
|
||||
type Gather = org.broadinstitute.sting.queue.function.scattergather.Gather
|
||||
type SimpleTextGatherFunction = org.broadinstitute.sting.queue.function.scattergather.SimpleTextGatherFunction
|
||||
|
||||
// Make sure annotations can be used in class constructors but target the fields
|
||||
// ex: class MyClass(@Input var myVar: File) {}
|
||||
// This was implicitly enabled in 2.8.0-RC2 and then updated to this new syntax:
|
||||
// http://lampsvn.epfl.ch/trac/scala/ticket/3596
|
||||
// http://lampsvn.epfl.ch/trac/scala/ticket/3421
|
||||
type Input = org.broadinstitute.sting.commandline.Input @field
|
||||
type Output = org.broadinstitute.sting.commandline.Output @field
|
||||
type Argument = org.broadinstitute.sting.commandline.Argument @field
|
||||
type ArgumentCollection = org.broadinstitute.sting.commandline.ArgumentCollection @field
|
||||
type Gather = org.broadinstitute.sting.queue.function.scattergather.Gather @field
|
||||
|
||||
/**
|
||||
* Builds the CommandLineFunctions that will be used to run this script and adds them to this.functions directly or using the add() utility method.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,3 +1,27 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.queue
|
||||
|
||||
import java.io.File
|
||||
|
|
@ -11,18 +35,15 @@ class QSettings {
|
|||
@Argument(fullName="job_name_prefix", shortName="jobPrefix", doc="Default name prefix for compute farm jobs.", required=false)
|
||||
var jobNamePrefix: String = QSettings.processNamePrefix
|
||||
|
||||
@Argument(fullName="job_queue", shortName="jobQueue", doc="Default queue for compute farm jobs.", required=false)
|
||||
var jobQueue: String = _
|
||||
|
||||
@Argument(fullName="job_project", shortName="jobProject", doc="Default project for compute farm jobs.", required=false)
|
||||
var jobProject: String = "Queue"
|
||||
|
||||
@Argument(fullName="job_queue", shortName="jobQueue", doc="Default queue for compute farm jobs.", required=false)
|
||||
var jobQueue: String = _
|
||||
|
||||
@Argument(fullName="job_priority", shortName="jobPriority", doc="Default priority for jobs.", required=false)
|
||||
var jobPriority: Option[Int] = None
|
||||
|
||||
@Argument(fullName="job_scatter_gather_directory", shortName="jobSGDir", doc="Default directory to place scatter gather output for compute farm jobs.", required=false)
|
||||
var jobScatterGatherDirectory: File = _
|
||||
|
||||
@Argument(fullName="default_memory_limit", shortName="memLimit", doc="Default memory limit for jobs, in gigabytes.", required=false)
|
||||
var memoryLimit: Option[Int] = None
|
||||
|
||||
|
|
@ -32,6 +53,9 @@ class QSettings {
|
|||
@Argument(fullName="temp_directory", shortName="tempDir", doc="Temp directory to pass to functions.", required=false)
|
||||
var tempDirectory = new File(System.getProperty("java.io.tmpdir"))
|
||||
|
||||
@Argument(fullName="job_scatter_gather_directory", shortName="jobSGDir", doc="Default directory to place scatter gather output for compute farm jobs.", required=false)
|
||||
var jobScatterGatherDirectory: File = _
|
||||
|
||||
@ArgumentCollection
|
||||
val emailSettings = new EmailSettings
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,8 +33,8 @@ import org.broadinstitute.sting.queue.util.SystemUtils
|
|||
* Command line options for a QGraph.
|
||||
*/
|
||||
class QGraphSettings {
|
||||
@ArgumentCollection
|
||||
val qSettings = new QSettings
|
||||
@Argument(fullName="run_scripts", shortName="run", doc="Run QScripts. Without this flag set only performs a dry run.", required=false)
|
||||
var run = false
|
||||
|
||||
@Argument(fullName="job_runner", shortName="jobRunner", doc="Use the specified job runner to dispatch command line jobs", required=false)
|
||||
var jobRunner: String = _
|
||||
|
|
@ -42,8 +42,23 @@ class QGraphSettings {
|
|||
@Argument(fullName="bsub", shortName="bsub", doc="Equivalent to -jobRunner Lsf706", required=false)
|
||||
var bsub = false
|
||||
|
||||
@Argument(fullName="run_scripts", shortName="run", doc="Run QScripts. Without this flag set only performs a dry run.", required=false)
|
||||
var run = false
|
||||
@Argument(fullName="status",shortName="status",doc="Get status of jobs for the qscript",required=false)
|
||||
var getStatus = false
|
||||
|
||||
@Argument(fullName="retry_failed", shortName="retry", doc="Retry the specified number of times after a command fails. Defaults to no retries.", required=false)
|
||||
var retries = 0
|
||||
|
||||
@Argument(fullName="start_from_scratch", shortName="startFromScratch", doc="Runs all command line functions even if the outputs were previously output successfully.", required=false)
|
||||
var startFromScratch = false
|
||||
|
||||
@Argument(fullName="keep_intermediate_outputs", shortName="keepIntermediates", doc="After a successful run keep the outputs of any Function marked as intermediate.", required=false)
|
||||
var keepIntermediates = false
|
||||
|
||||
@Argument(fullName="status_email_to", shortName="statusTo", doc="Email address to send emails to upon completion or on error.", required=false)
|
||||
var statusEmailTo: List[String] = Nil
|
||||
|
||||
@Argument(fullName="status_email_from", shortName="statusFrom", doc="Email address to send emails from upon completion or on error.", required=false)
|
||||
var statusEmailFrom: String = System.getProperty("user.name") + "@" + SystemUtils.domainName
|
||||
|
||||
@Argument(fullName="dot_graph", shortName="dot", doc="Outputs the queue graph to a .dot file. See: http://en.wikipedia.org/wiki/DOT_language", required=false)
|
||||
var dotFile: File = _
|
||||
|
|
@ -51,21 +66,6 @@ class QGraphSettings {
|
|||
@Argument(fullName="expanded_dot_graph", shortName="expandedDot", doc="Outputs the queue graph of scatter gather to a .dot file. Otherwise overwrites the dot_graph", required=false)
|
||||
var expandedDotFile: File = _
|
||||
|
||||
@Argument(fullName="start_from_scratch", shortName="startFromScratch", doc="Runs all command line functions even if the outputs were previously output successfully.", required=false)
|
||||
var startFromScratch = false
|
||||
|
||||
@Argument(fullName="status",shortName="status",doc="Get status of jobs for the qscript",required=false)
|
||||
var getStatus = false
|
||||
|
||||
@Argument(fullName="status_email_from", shortName="statusFrom", doc="Email address to send emails from upon completion or on error.", required=false)
|
||||
var statusEmailFrom: String = System.getProperty("user.name") + "@" + SystemUtils.domainName
|
||||
|
||||
@Argument(fullName="status_email_to", shortName="statusTo", doc="Email address to send emails to upon completion or on error.", required=false)
|
||||
var statusEmailTo: List[String] = Nil
|
||||
|
||||
@Argument(fullName="keep_intermediate_outputs", shortName="keepIntermediates", doc="After a successful run keep the outputs of any Function marked as intermediate.", required=false)
|
||||
var keepIntermediates = false
|
||||
|
||||
@Argument(fullName="retry_failed", shortName="retry", doc="Retry the specified number of times after a command fails. Defaults to no retries.", required=false)
|
||||
var retries = 0
|
||||
@ArgumentCollection
|
||||
val qSettings = new QSettings
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,27 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.queue.util
|
||||
|
||||
import org.apache.commons.mail.{MultiPartEmail, EmailAttachment}
|
||||
|
|
@ -37,7 +61,7 @@ class EmailMessage extends Logging {
|
|||
if (settings.passwordFile != null) {
|
||||
val reader = new FileReader(settings.passwordFile)
|
||||
try {
|
||||
org.apache.commons.io.IOUtils.toString(reader)
|
||||
org.apache.commons.io.IOUtils.toString(reader).replaceAll("\\r|\\n", "")
|
||||
} finally {
|
||||
org.apache.commons.io.IOUtils.closeQuietly(reader)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,27 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.queue.util
|
||||
|
||||
import java.io.File
|
||||
|
|
@ -22,9 +46,9 @@ class EmailSettings {
|
|||
@Argument(doc="Email SMTP username. Defaults to none.", shortName="emailUser", fullName="emailUsername", required=false)
|
||||
var username: String = _
|
||||
|
||||
@Argument(doc="Email SMTP password. Defaults to none. Not secure! See emailPassFile.", shortName="emailPass", fullName="emailPassword", required=false)
|
||||
var password: String = _
|
||||
|
||||
@Argument(doc="Email SMTP password file. Defaults to none.", shortName="emailPassFile", fullName="emailPasswordFile", required=false)
|
||||
var passwordFile: File = _
|
||||
|
||||
@Argument(doc="Email SMTP password. Defaults to none. Not secure! See emailPassFile.", shortName="emailPass", fullName="emailPassword", required=false)
|
||||
var password: String = _
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue