2010-05-24 04:21:09 +08:00
package org.broadinstitute.sting.queue
2010-08-10 00:42:48 +08:00
import java.io.File
import java.util.Arrays
import org.broadinstitute.sting.queue.engine.QGraph
2010-08-12 05:58:26 +08:00
import org.broadinstitute.sting.commandline._
2010-10-14 23:58:52 +08:00
import org.broadinstitute.sting.queue.util._
2010-05-24 04:21:09 +08:00
2010-08-10 00:42:48 +08:00
/* *
* Entry point of Queue . Compiles and runs QScripts passed in to the command line .
*/
class QCommandLine extends CommandLineProgram with Logging {
@Input ( fullName = "script" , shortName = "S" , doc = "QScript scala file" , required = true )
@ClassType ( classOf [ File ] )
private var scripts = List . empty [ File ]
2010-05-24 04:21:09 +08:00
2010-08-10 00:42:48 +08:00
@Argument ( fullName = "bsub_all_jobs" , shortName = "bsub" , doc = "Use bsub to submit jobs" , required = false )
private var bsubAllJobs = false
2010-08-12 05:58:26 +08:00
@Argument ( fullName = "run_scripts" , shortName = "run" , doc = "Run QScripts. Without this flag set only performs a dry run." , required = false )
2010-08-10 00:42:48 +08:00
private var run = false
2010-05-24 04:21:09 +08:00
2010-08-10 00:42:48 +08:00
@Argument ( fullName = "dot_graph" , shortName = "dot" , doc = "Outputs the queue graph to a .dot file. See: http://en.wikipedia.org/wiki/DOT_language" , required = false )
2010-08-12 05:58:26 +08:00
private var dotFile : File = _
@Argument ( fullName = "expanded_dot_graph" , shortName = "expandedDot" , doc = "Outputs the queue graph of scatter gather to a .dot file. Otherwise overwrites the dot_graph" , required = false )
private var expandedDotFile : File = _
2010-10-07 09:19:18 +08:00
@Argument ( fullName = "start_clean" , shortName = "clean" , doc = "Runs all command line functions even if the outputs were previously output successfully." , required = false )
private var startClean = false
2010-08-12 05:58:26 +08:00
@Argument ( fullName = "for_reals" , shortName = "forReals" , doc = "Run QScripts" , required = false ) @Hidden
private var runScripts = false
Queue now does job tracking (replace -run with -status in the command line). Produces output that looks like:
INFO 20:58:17,827 QCommandLine - Checking pipeline status
INFO 20:58:23,234 QGraph$$anonfun$formatStatus$1 - Height_Hirschhorn_NHGRI.uncleaned_MergeIndels [DONE]
INFO 20:58:23,236 QGraph$$anonfun$formatStatus$1 - IndelGenotyper_158.bam [DONE] 5t/5d/0r/0p/0f
INFO 20:58:23,237 QGraph$$anonfun$formatStatus$1 - IndelGenotyper_929.bam [DONE] 5t/5d/0r/0p/0f
INFO 20:58:23,238 QGraph$$anonfun$formatStatus$1 - Height_Hirschhorn_NHGRI.uncleaned_SNP_calls [NOT DONE] 5t/0d/0r/5p/0f
INFO 20:58:23,239 QGraph$$anonfun$formatStatus$1 - Height_Hirschhorn_NHGRI.uncleaned_HandFilter [NOT DONE]
INFO 20:58:23,240 QGraph$$anonfun$formatStatus$1 - IndelGenotyper_1122.bam [DONE] 5t/5d/0r/0p/0f
INFO 20:58:23,240 QGraph$$anonfun$formatStatus$1 - Height_Hirschhorn_NHGRI.uncleaned_VariantRecalibrator [NOT DONE]
INFO 20:58:23,241 QGraph$$anonfun$formatStatus$1 - IndelGenotyper_913.bam [DONE] 5t/5d/0r/0p/0f
INFO 20:58:23,242 QGraph$$anonfun$formatStatus$1 - IndelGenotyper_2037.bam [DONE] 5t/5d/0r/0p/0f
INFO 20:58:23,243 QGraph$$anonfun$formatStatus$1 - Height_Hirschhorn_NHGRI.uncleaned_VariantEval [NOT DONE]
INFO 20:58:23,244 QGraph$$anonfun$formatStatus$1 - Height_Hirschhorn_NHGRI.uncleaned_Cluster [NOT DONE]
INFO 20:58:23,245 QGraph$$anonfun$formatStatus$1 - IndelGenotyper_106.bam [DONE] 5t/5d/0r/0p/0f
INFO 20:58:23,246 QGraph$$anonfun$formatStatus$1 - Height_Hirschhorn_NHGRI.uncleaned_Cluster_and_Indel_filter [NOT DONE]
INFO 20:58:23,247 QGraph$$anonfun$formatStatus$1 - Height_Hirschhorn_NHGRI.uncleaned_ApplyVariantCuts [NOT DONE]
INFO 20:58:23,248 QGraph$$anonfun$formatStatus$1 - Height_Hirschhorn_NHGRI.uncleaned_GenomicAnnotator [NOT DONE]
INFO 20:58:23,248 QGraph$$anonfun$formatStatus$1 - IndelGenotyper_1713.bam [DONE] 5t/5d/0r/0p/0f
git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@4340 348d0f76-0448-11de-a6fe-93d51630548a
2010-09-24 08:59:09 +08:00
@Argument ( fullName = "status" , shortName = "status" , doc = "Get status of jobs for the qscript" , required = false )
private var getStatus = false
2010-08-12 05:58:26 +08:00
@ArgumentCollection
private val qSettings = new QSettings
2010-05-24 04:21:09 +08:00
2010-10-14 23:58:52 +08:00
@Argument ( fullName = "statusEmailFrom" , shortName = "statusFrom" , doc = "Email address to send emails from upon completion or on error." , required = false )
private var statusEmailFrom : String = System . getProperty ( "user.name" ) + "@" + SystemUtils . domainName
@Argument ( fullName = "statusEmailTo" , shortName = "statusTo" , doc = "Email address to send emails to upon completion or on error." , required = false )
private var statusEmailTo : List [ String ] = Nil
2010-08-10 00:42:48 +08:00
/* *
* Takes the QScripts passed in , runs their script ( ) methods , retrieves their generated
* functions , and then builds and runs a QGraph based on the dependencies .
*/
def execute = {
2010-10-07 02:29:56 +08:00
2010-08-10 00:42:48 +08:00
val qGraph = new QGraph
2010-08-12 05:58:26 +08:00
qGraph . dryRun = ! ( run || runScripts )
2010-08-10 00:42:48 +08:00
qGraph . bsubAllJobs = bsubAllJobs
2010-10-07 09:19:18 +08:00
qGraph . startClean = startClean
2010-08-12 05:58:26 +08:00
qGraph . dotFile = dotFile
qGraph . expandedDotFile = expandedDotFile
qGraph . qSettings = qSettings
2010-08-13 23:54:08 +08:00
qGraph . debugMode = debugMode == true
2010-10-14 23:58:52 +08:00
qGraph . statusEmailFrom = statusEmailFrom
qGraph . statusEmailTo = statusEmailTo
2010-08-10 00:42:48 +08:00
val scripts = qScriptManager . createScripts ( )
for ( script <- scripts ) {
logger . info ( "Scripting " + qScriptManager . getName ( script . getClass . asSubclass ( classOf [ QScript ] ) ) )
loadArgumentsIntoObject ( script )
script . script
script . functions . foreach ( qGraph . add ( _ ) )
logger . info ( "Added " + script . functions . size + " functions" )
2010-05-24 04:21:09 +08:00
}
2010-10-07 02:29:56 +08:00
Runtime . getRuntime . addShutdownHook ( new Thread {
/* * Kills running processes as the JVM shuts down. */
override def run = {
qGraph . shutdown ( )
ProcessController . shutdown ( )
}
} )
Queue now does job tracking (replace -run with -status in the command line). Produces output that looks like:
INFO 20:58:17,827 QCommandLine - Checking pipeline status
INFO 20:58:23,234 QGraph$$anonfun$formatStatus$1 - Height_Hirschhorn_NHGRI.uncleaned_MergeIndels [DONE]
INFO 20:58:23,236 QGraph$$anonfun$formatStatus$1 - IndelGenotyper_158.bam [DONE] 5t/5d/0r/0p/0f
INFO 20:58:23,237 QGraph$$anonfun$formatStatus$1 - IndelGenotyper_929.bam [DONE] 5t/5d/0r/0p/0f
INFO 20:58:23,238 QGraph$$anonfun$formatStatus$1 - Height_Hirschhorn_NHGRI.uncleaned_SNP_calls [NOT DONE] 5t/0d/0r/5p/0f
INFO 20:58:23,239 QGraph$$anonfun$formatStatus$1 - Height_Hirschhorn_NHGRI.uncleaned_HandFilter [NOT DONE]
INFO 20:58:23,240 QGraph$$anonfun$formatStatus$1 - IndelGenotyper_1122.bam [DONE] 5t/5d/0r/0p/0f
INFO 20:58:23,240 QGraph$$anonfun$formatStatus$1 - Height_Hirschhorn_NHGRI.uncleaned_VariantRecalibrator [NOT DONE]
INFO 20:58:23,241 QGraph$$anonfun$formatStatus$1 - IndelGenotyper_913.bam [DONE] 5t/5d/0r/0p/0f
INFO 20:58:23,242 QGraph$$anonfun$formatStatus$1 - IndelGenotyper_2037.bam [DONE] 5t/5d/0r/0p/0f
INFO 20:58:23,243 QGraph$$anonfun$formatStatus$1 - Height_Hirschhorn_NHGRI.uncleaned_VariantEval [NOT DONE]
INFO 20:58:23,244 QGraph$$anonfun$formatStatus$1 - Height_Hirschhorn_NHGRI.uncleaned_Cluster [NOT DONE]
INFO 20:58:23,245 QGraph$$anonfun$formatStatus$1 - IndelGenotyper_106.bam [DONE] 5t/5d/0r/0p/0f
INFO 20:58:23,246 QGraph$$anonfun$formatStatus$1 - Height_Hirschhorn_NHGRI.uncleaned_Cluster_and_Indel_filter [NOT DONE]
INFO 20:58:23,247 QGraph$$anonfun$formatStatus$1 - Height_Hirschhorn_NHGRI.uncleaned_ApplyVariantCuts [NOT DONE]
INFO 20:58:23,248 QGraph$$anonfun$formatStatus$1 - Height_Hirschhorn_NHGRI.uncleaned_GenomicAnnotator [NOT DONE]
INFO 20:58:23,248 QGraph$$anonfun$formatStatus$1 - IndelGenotyper_1713.bam [DONE] 5t/5d/0r/0p/0f
git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@4340 348d0f76-0448-11de-a6fe-93d51630548a
2010-09-24 08:59:09 +08:00
if ( ! getStatus ) {
qGraph . run
} else {
qGraph . checkStatus
}
2010-08-12 05:58:26 +08:00
2010-10-07 02:29:56 +08:00
if ( qGraph . hasFailed ) {
logger . info ( "Done with errors" )
2010-10-19 06:11:14 +08:00
qGraph . logFailed
2010-10-07 02:29:56 +08:00
1
} else {
logger . info ( "Done" )
0
}
2010-08-10 00:42:48 +08:00
}
/* *
* Returns true as QScripts are located and compiled .
* @return true
*/
override def canAddArgumentsDynamically = true
/* *
* Returns the list of QScripts passed in via - S so that their
* arguments can be inspected before QScript . script is called .
* @return Array of QScripts passed in .
*/
override def getArgumentSources =
qScriptManager . getValues . asInstanceOf [ Array [ Class [ _ ] ] ]
2010-05-26 06:52:29 +08:00
2010-08-10 00:42:48 +08:00
/* *
* Returns the name of a QScript
* @return The name of a QScript
*/
override def getArgumentSourceName ( source : Class [ _ ] ) =
qScriptManager . getName ( source . asSubclass ( classOf [ QScript ] ) )
/* *
* Returns a ScalaCompoundArgumentTypeDescriptor that can parse argument sources into scala collections .
* @return a ScalaCompoundArgumentTypeDescriptor
*/
override def getArgumentTypeDescriptors =
Arrays . asList ( new ScalaCompoundArgumentTypeDescriptor )
/* *
* Loads the QScripts passed in and returns a new QScriptManager than can be used to create them .
*/
private lazy val qScriptManager = {
QScriptManager . loadScripts ( scripts )
new QScriptManager
}
}
/* *
* Entry point of Queue . Compiles and runs QScripts passed in to the command line .
*/
object QCommandLine {
/* *
* Main .
* @param argv Arguments .
*/
def main ( argv : Array [ String ] ) {
try {
CommandLineProgram . start ( new QCommandLine , argv ) ;
if ( CommandLineProgram . result != 0 )
System . exit ( CommandLineProgram . result ) ;
} catch {
case e : Exception => CommandLineProgram . exitSystemWithError ( e )
}
2010-05-24 04:21:09 +08:00
}
}