From 57353294cc63f9d9580557c1b27bcf21d1dab6eb Mon Sep 17 00:00:00 2001 From: kshakir Date: Sun, 30 Jan 2011 06:35:53 +0000 Subject: [PATCH] Copying jobLimitSeconds to clones. Some cleanup and refactoring around copying values to clones. git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@5128 348d0f76-0448-11de-a6fe-93d51630548a --- .../queue/function/CommandLineFunction.scala | 23 ++++++++++++--- .../sting/queue/function/QFunction.scala | 29 +++++++++++++++---- .../ScatterGatherableFunction.scala | 25 ++-------------- .../sting/queue/pipeline/VariantCalling.scala | 2 -- 4 files changed, 45 insertions(+), 34 deletions(-) diff --git a/scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala index 1e8ba68d0..2457f3fea 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala @@ -8,9 +8,6 @@ import org.broadinstitute.sting.queue.util._ trait CommandLineFunction extends QFunction with Logging { def commandLine: String - /* Is it a gather function? */ - var isGather: Boolean = true - /** Upper memory limit */ var memoryLimit: Option[Int] = None @@ -20,6 +17,24 @@ trait CommandLineFunction extends QFunction with Logging { /** Job queue to run the command */ var jobQueue: String = _ + override def copySettingsTo(function: QFunction) { + super.copySettingsTo(function) + function match { + case commandLineFunction: CommandLineFunction => + if (commandLineFunction.memoryLimit.isEmpty) + commandLineFunction.memoryLimit = this.memoryLimit + + if (commandLineFunction.jobProject == null) + commandLineFunction.jobProject = this.jobProject + + if (commandLineFunction.jobQueue == null) + commandLineFunction.jobQueue = this.jobQueue + + commandLineFunction.jobQueue = this.jobQueue + case _ => /* ignore */ + } + } + /** * Returns set of directories required to run the command. * @return Set of directories required to run the command. @@ -38,7 +53,7 @@ trait CommandLineFunction extends QFunction with Logging { if (jobProject == null) jobProject = qSettings.jobProject - if (memoryLimit.isEmpty && qSettings.memoryLimit.isDefined) + if (memoryLimit.isEmpty) memoryLimit = qSettings.memoryLimit super.freezeFieldValues diff --git a/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala index da6a5e0c8..c727c4f47 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala @@ -53,6 +53,29 @@ trait QFunction extends Logging { */ var updateJobRun: PartialFunction[Any,Unit] = null + /** + * If true, unless another unfinished function is dependent on this function, + * this function will NOT be run even if the outputs have not been created. + */ + var isIntermediate = false + + /** + * Copies settings from this function to another function. + * @param function QFunction to copy values to. + */ + def copySettingsTo(function: QFunction) { + function.analysisName = this.analysisName + function.jobName = this.jobName + function.qSettings = this.qSettings + function.commandDirectory = this.commandDirectory + function.jobTempDir = this.jobTempDir + function.addOrder = this.addOrder + function.jobLimitSeconds = this.jobLimitSeconds + function.jobRestartable = this.jobRestartable + function.updateJobRun = this.updateJobRun + function.isIntermediate = this.isIntermediate + } + /** File to redirect any output. Defaults to .out */ @Output(doc="File to redirect any output", required=false) @Gather(classOf[SimpleTextGatherFunction]) @@ -138,12 +161,6 @@ trait QFunction extends Logging { /** The @Argument fields on this CommandLineFunction. */ def argumentFields = QFunction.classFields(this.functionFieldClass).argumentFields - /** - * If true, unless another unfinished function is dependent on this function, - * this function will NOT be run even if the outputs have not been created. - */ - var isIntermediate = false - /** * Returns the class that should be used for looking up fields. */ diff --git a/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala index a4b31b519..3b3ae5442 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala @@ -70,7 +70,7 @@ trait ScatterGatherableFunction extends CommandLineFunction { val outputFieldsWithValues = this.outputFields.filter(hasFieldValue(_)) // Create the scatter function based on @Scatter - syncFunction(scatterFunction) + this.copySettingsTo(scatterFunction) scatterFunction.addOrder = this.addOrder :+ 1 scatterFunction.commandDirectory = this.scatterGatherTempDir("scatter") scatterFunction.originalInputs = this.inputs @@ -89,11 +89,10 @@ trait ScatterGatherableFunction extends CommandLineFunction { for (gatherField <- outputFieldsWithValues) { val gatherFunction = this.newGatherFunction(gatherField) val gatherOutput = getFieldFile(gatherField) - syncFunction(gatherFunction) + this.copySettingsTo(gatherFunction) gatherFunction.addOrder = this.addOrder :+ gatherAddOrder gatherFunction.commandDirectory = this.scatterGatherTempDir("gather-" + gatherField.field.getName) gatherFunction.originalOutput = this.getFieldFile(gatherField) - gatherFunction.isIntermediate = this.isIntermediate initGatherFunction(gatherFunction, gatherField) functions :+= gatherFunction gatherFunctions += gatherField -> gatherFunction @@ -106,11 +105,10 @@ trait ScatterGatherableFunction extends CommandLineFunction { for (i <- 1 to numClones) { val cloneFunction = this.newCloneFunction() - syncFunction(cloneFunction) + this.copySettingsTo(cloneFunction) cloneFunction.originalFunction = this cloneFunction.index = i cloneFunction.addOrder = this.addOrder :+ (i+1) - cloneFunction.memoryLimit = this.memoryLimit cloneFunction.isIntermediate = true // Setup the fields on the clone function, outputting each as a relative file in the sg directory. @@ -224,23 +222,6 @@ trait ScatterGatherableFunction extends CommandLineFunction { this.setupCloneFunction(cloneFunction, index) } - /** - * Copies standard values from this function to the just created function. - * @param newFunction newly created function. - */ - protected def syncFunction(newFunction: QFunction) = { - newFunction.analysisName = this.analysisName - newFunction.qSettings = this.qSettings - newFunction.jobTempDir = this.jobTempDir - newFunction.jobName = this.jobName - newFunction match { - case newCLFFunction: CommandLineFunction => - newCLFFunction.jobQueue = this.jobQueue - newCLFFunction.jobProject = this.jobProject - case _ => /* ignore */ - } - } - /** * The scatter function. */ diff --git a/scala/src/org/broadinstitute/sting/queue/pipeline/VariantCalling.scala b/scala/src/org/broadinstitute/sting/queue/pipeline/VariantCalling.scala index 650e5190e..dc464bc84 100755 --- a/scala/src/org/broadinstitute/sting/queue/pipeline/VariantCalling.scala +++ b/scala/src/org/broadinstitute/sting/queue/pipeline/VariantCalling.scala @@ -92,7 +92,6 @@ class VariantCalling(attribs: Pipeline,gatkJar: File) { */ private def StandardIndelGenotyper(bam : File, output: File, gather: Boolean) : IndelGenotyperV2 = { var ig = StandardIndelGenotyper(bam,output) - ig.isGather = gather return ig } @@ -106,7 +105,6 @@ class VariantCalling(attribs: Pipeline,gatkJar: File) { cv.genotypemergeoption = Some(org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.GenotypeMergeType.UNIQUIFY) cv.variantmergeoption = Some(org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.UNION) cv.analysisName = "IndelGenotyper" - cv.isGather = true cv.priority = (igList.map[String,List[String]](ig => swapExt(ig.out,".vcf","").getAbsolutePath)).mkString(",") //cv.priority = (igList.foldLeft[List[String]](Nil)( (prLs, ig) => prLs ::: List(swapExt(ig.out,".vcf","").getAbsolutePath))).mkString(",") cv.rodBind = igList.map[RodBind,List[RodBind]](ig => new RodBind(swapExt(ig.out,".vcf","").getName,"VCF",ig.out))