From 2058fc12bccf7fbc9426a434136e1536abac84d9 Mon Sep 17 00:00:00 2001 From: kshakir Date: Thu, 10 Mar 2011 19:54:14 +0000 Subject: [PATCH] Small bug fixes: Added a property to allow generating Queue extensions around an external version of the GATK. Updated argument order in -help. Restored the ability to use QScript trait imported annotations in constructors. Removing line feeds from email password files. git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@5416 348d0f76-0448-11de-a6fe-93d51630548a --- build.xml | 37 ++++++++++++++-- .../broadinstitute/sting/queue/QScript.scala | 41 +++++++++++++++--- .../sting/queue/QSettings.scala | 36 +++++++++++++--- .../sting/queue/engine/QGraphSettings.scala | 42 +++++++++---------- .../sting/queue/util/EmailMessage.scala | 26 +++++++++++- .../sting/queue/util/EmailSettings.scala | 30 +++++++++++-- 6 files changed, 172 insertions(+), 40 deletions(-) diff --git a/build.xml b/build.xml index 24710a425..766c987a7 100644 --- a/build.xml +++ b/build.xml @@ -1,3 +1,27 @@ + + Compile and distribute the Sting toolkit @@ -34,7 +58,7 @@ - + @@ -111,7 +135,7 @@ - + - + @@ -245,6 +269,7 @@ + @@ -254,11 +279,15 @@ Generating Queue GATK extensions... - + + + + + diff --git a/scala/src/org/broadinstitute/sting/queue/QScript.scala b/scala/src/org/broadinstitute/sting/queue/QScript.scala index f7224a4d6..9934d5e0e 100755 --- a/scala/src/org/broadinstitute/sting/queue/QScript.scala +++ b/scala/src/org/broadinstitute/sting/queue/QScript.scala @@ -1,8 +1,33 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue import org.broadinstitute.sting.queue.util.Logging import org.broadinstitute.sting.queue.function.QFunction import org.broadinstitute.sting.utils.text.XReadLines +import annotation.target.field /** * Defines a Queue pipeline as a collection of CommandLineFunctions. @@ -10,16 +35,22 @@ import org.broadinstitute.sting.utils.text.XReadLines trait QScript extends Logging { // Type aliases so users don't have to import type File = java.io.File - type Input = org.broadinstitute.sting.commandline.Input - type Output = org.broadinstitute.sting.commandline.Output - type Argument = org.broadinstitute.sting.commandline.Argument - type ArgumentCollection = org.broadinstitute.sting.commandline.ArgumentCollection type CommandLineFunction = org.broadinstitute.sting.queue.function.CommandLineFunction type InProcessFunction = org.broadinstitute.sting.queue.function.InProcessFunction type ScatterGatherableFunction = org.broadinstitute.sting.queue.function.scattergather.ScatterGatherableFunction - type Gather = org.broadinstitute.sting.queue.function.scattergather.Gather type SimpleTextGatherFunction = org.broadinstitute.sting.queue.function.scattergather.SimpleTextGatherFunction + // Make sure annotations can be used in class constructors but target the fields + // ex: class MyClass(@Input var myVar: File) {} + // This was implicitly enabled in 2.8.0-RC2 and then updated to this new syntax: + // http://lampsvn.epfl.ch/trac/scala/ticket/3596 + // http://lampsvn.epfl.ch/trac/scala/ticket/3421 + type Input = org.broadinstitute.sting.commandline.Input @field + type Output = org.broadinstitute.sting.commandline.Output @field + type Argument = org.broadinstitute.sting.commandline.Argument @field + type ArgumentCollection = org.broadinstitute.sting.commandline.ArgumentCollection @field + type Gather = org.broadinstitute.sting.queue.function.scattergather.Gather @field + /** * Builds the CommandLineFunctions that will be used to run this script and adds them to this.functions directly or using the add() utility method. */ diff --git a/scala/src/org/broadinstitute/sting/queue/QSettings.scala b/scala/src/org/broadinstitute/sting/queue/QSettings.scala index 993932bd5..687ea8b0b 100644 --- a/scala/src/org/broadinstitute/sting/queue/QSettings.scala +++ b/scala/src/org/broadinstitute/sting/queue/QSettings.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue import java.io.File @@ -11,18 +35,15 @@ class QSettings { @Argument(fullName="job_name_prefix", shortName="jobPrefix", doc="Default name prefix for compute farm jobs.", required=false) var jobNamePrefix: String = QSettings.processNamePrefix - @Argument(fullName="job_queue", shortName="jobQueue", doc="Default queue for compute farm jobs.", required=false) - var jobQueue: String = _ - @Argument(fullName="job_project", shortName="jobProject", doc="Default project for compute farm jobs.", required=false) var jobProject: String = "Queue" + @Argument(fullName="job_queue", shortName="jobQueue", doc="Default queue for compute farm jobs.", required=false) + var jobQueue: String = _ + @Argument(fullName="job_priority", shortName="jobPriority", doc="Default priority for jobs.", required=false) var jobPriority: Option[Int] = None - @Argument(fullName="job_scatter_gather_directory", shortName="jobSGDir", doc="Default directory to place scatter gather output for compute farm jobs.", required=false) - var jobScatterGatherDirectory: File = _ - @Argument(fullName="default_memory_limit", shortName="memLimit", doc="Default memory limit for jobs, in gigabytes.", required=false) var memoryLimit: Option[Int] = None @@ -32,6 +53,9 @@ class QSettings { @Argument(fullName="temp_directory", shortName="tempDir", doc="Temp directory to pass to functions.", required=false) var tempDirectory = new File(System.getProperty("java.io.tmpdir")) + @Argument(fullName="job_scatter_gather_directory", shortName="jobSGDir", doc="Default directory to place scatter gather output for compute farm jobs.", required=false) + var jobScatterGatherDirectory: File = _ + @ArgumentCollection val emailSettings = new EmailSettings } diff --git a/scala/src/org/broadinstitute/sting/queue/engine/QGraphSettings.scala b/scala/src/org/broadinstitute/sting/queue/engine/QGraphSettings.scala index 650599377..10b795fa0 100644 --- a/scala/src/org/broadinstitute/sting/queue/engine/QGraphSettings.scala +++ b/scala/src/org/broadinstitute/sting/queue/engine/QGraphSettings.scala @@ -33,8 +33,8 @@ import org.broadinstitute.sting.queue.util.SystemUtils * Command line options for a QGraph. */ class QGraphSettings { - @ArgumentCollection - val qSettings = new QSettings + @Argument(fullName="run_scripts", shortName="run", doc="Run QScripts. Without this flag set only performs a dry run.", required=false) + var run = false @Argument(fullName="job_runner", shortName="jobRunner", doc="Use the specified job runner to dispatch command line jobs", required=false) var jobRunner: String = _ @@ -42,8 +42,23 @@ class QGraphSettings { @Argument(fullName="bsub", shortName="bsub", doc="Equivalent to -jobRunner Lsf706", required=false) var bsub = false - @Argument(fullName="run_scripts", shortName="run", doc="Run QScripts. Without this flag set only performs a dry run.", required=false) - var run = false + @Argument(fullName="status",shortName="status",doc="Get status of jobs for the qscript",required=false) + var getStatus = false + + @Argument(fullName="retry_failed", shortName="retry", doc="Retry the specified number of times after a command fails. Defaults to no retries.", required=false) + var retries = 0 + + @Argument(fullName="start_from_scratch", shortName="startFromScratch", doc="Runs all command line functions even if the outputs were previously output successfully.", required=false) + var startFromScratch = false + + @Argument(fullName="keep_intermediate_outputs", shortName="keepIntermediates", doc="After a successful run keep the outputs of any Function marked as intermediate.", required=false) + var keepIntermediates = false + + @Argument(fullName="status_email_to", shortName="statusTo", doc="Email address to send emails to upon completion or on error.", required=false) + var statusEmailTo: List[String] = Nil + + @Argument(fullName="status_email_from", shortName="statusFrom", doc="Email address to send emails from upon completion or on error.", required=false) + var statusEmailFrom: String = System.getProperty("user.name") + "@" + SystemUtils.domainName @Argument(fullName="dot_graph", shortName="dot", doc="Outputs the queue graph to a .dot file. See: http://en.wikipedia.org/wiki/DOT_language", required=false) var dotFile: File = _ @@ -51,21 +66,6 @@ class QGraphSettings { @Argument(fullName="expanded_dot_graph", shortName="expandedDot", doc="Outputs the queue graph of scatter gather to a .dot file. Otherwise overwrites the dot_graph", required=false) var expandedDotFile: File = _ - @Argument(fullName="start_from_scratch", shortName="startFromScratch", doc="Runs all command line functions even if the outputs were previously output successfully.", required=false) - var startFromScratch = false - - @Argument(fullName="status",shortName="status",doc="Get status of jobs for the qscript",required=false) - var getStatus = false - - @Argument(fullName="status_email_from", shortName="statusFrom", doc="Email address to send emails from upon completion or on error.", required=false) - var statusEmailFrom: String = System.getProperty("user.name") + "@" + SystemUtils.domainName - - @Argument(fullName="status_email_to", shortName="statusTo", doc="Email address to send emails to upon completion or on error.", required=false) - var statusEmailTo: List[String] = Nil - - @Argument(fullName="keep_intermediate_outputs", shortName="keepIntermediates", doc="After a successful run keep the outputs of any Function marked as intermediate.", required=false) - var keepIntermediates = false - - @Argument(fullName="retry_failed", shortName="retry", doc="Retry the specified number of times after a command fails. Defaults to no retries.", required=false) - var retries = 0 + @ArgumentCollection + val qSettings = new QSettings } diff --git a/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala b/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala index 12231c873..4f865e0f1 100644 --- a/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala +++ b/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.util import org.apache.commons.mail.{MultiPartEmail, EmailAttachment} @@ -37,7 +61,7 @@ class EmailMessage extends Logging { if (settings.passwordFile != null) { val reader = new FileReader(settings.passwordFile) try { - org.apache.commons.io.IOUtils.toString(reader) + org.apache.commons.io.IOUtils.toString(reader).replaceAll("\\r|\\n", "") } finally { org.apache.commons.io.IOUtils.closeQuietly(reader) } diff --git a/scala/src/org/broadinstitute/sting/queue/util/EmailSettings.scala b/scala/src/org/broadinstitute/sting/queue/util/EmailSettings.scala index f6cfd0cf6..de07702d7 100644 --- a/scala/src/org/broadinstitute/sting/queue/util/EmailSettings.scala +++ b/scala/src/org/broadinstitute/sting/queue/util/EmailSettings.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.util import java.io.File @@ -22,9 +46,9 @@ class EmailSettings { @Argument(doc="Email SMTP username. Defaults to none.", shortName="emailUser", fullName="emailUsername", required=false) var username: String = _ - @Argument(doc="Email SMTP password. Defaults to none. Not secure! See emailPassFile.", shortName="emailPass", fullName="emailPassword", required=false) - var password: String = _ - @Argument(doc="Email SMTP password file. Defaults to none.", shortName="emailPassFile", fullName="emailPasswordFile", required=false) var passwordFile: File = _ + + @Argument(doc="Email SMTP password. Defaults to none. Not secure! See emailPassFile.", shortName="emailPass", fullName="emailPassword", required=false) + var password: String = _ }