diff --git a/packages/Queue.xml b/packages/Queue.xml index 56e7ad2eb..916827dab 100644 --- a/packages/Queue.xml +++ b/packages/Queue.xml @@ -15,6 +15,9 @@ + + + diff --git a/scala/qscript/oneoffs/carneiro/justClean.scala b/scala/qscript/oneoffs/carneiro/justClean.scala index 9b9781a71..36a7b5062 100755 --- a/scala/qscript/oneoffs/carneiro/justClean.scala +++ b/scala/qscript/oneoffs/carneiro/justClean.scala @@ -1,7 +1,5 @@ import org.broadinstitute.sting.queue.QScript import org.broadinstitute.sting.queue.extensions.gatk.{RealignerTargetCreator, RodBind, IndelRealigner} -import org.broadinstitute.sting.commandline.ArgumentSource -import org.broadinstitute.sting.queue.extensions.gatk.BamGatherFunction /** * Created by IntelliJ IDEA. @@ -69,13 +67,6 @@ class justClean extends QScript { clean.memoryLimit = Some(6) clean.scatterCount = 84 - clean.setupGatherFunction = { - case (gather: BamGatherFunction, source: ArgumentSource) => - gather.memoryLimit = Some(6) // Memory limit you expect for the job - gather.jarFile = new File("/seq/software/picard/current/bin/MergeSamFiles.jar") - } - - add(clean); } } diff --git a/scala/qscript/playground/FullCallingPipeline.q b/scala/qscript/playground/FullCallingPipeline.q index a391371ad..54726160d 100755 --- a/scala/qscript/playground/FullCallingPipeline.q +++ b/scala/qscript/playground/FullCallingPipeline.q @@ -1,7 +1,7 @@ import org.broadinstitute.sting.commandline.ArgumentSource import org.broadinstitute.sting.datasources.pipeline.Pipeline import org.broadinstitute.sting.queue.extensions.gatk._ -import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction +import org.broadinstitute.sting.queue.extensions.picard.PicardBamEmbeddedFunction import org.broadinstitute.sting.queue.extensions.samtools._ import org.broadinstitute.sting.queue.function.ListWriterFunction import org.broadinstitute.sting.queue.function.scattergather.{GatherFunction, CloneFunction, ScatterFunction} @@ -16,9 +16,6 @@ class FullCallingPipeline extends QScript { @Argument(doc="the YAML file specifying inputs, interval lists, reference sequence, etc.", shortName="Y") var yamlFile: File = _ - @Input(doc="path to Picard FixMateInformation.jar. See http://picard.sourceforge.net/ .", shortName="P", required=false) - var picardFixMatesJar: File = new java.io.File("/seq/software/picard/current/bin/FixMateInformation.jar") - @Input(doc="path to GATK jar", shortName="G") var gatkJar: File = _ @@ -39,6 +36,8 @@ class FullCallingPipeline extends QScript { private var pipeline: Pipeline = _ + private final val picardFixMatesClass = "net.sf.picard.sam.FixMateInformation" + trait CommandLineGATKArgs extends CommandLineGATK { this.intervals = List(qscript.pipeline.getProject.getIntervalList) this.jarFile = qscript.gatkJar @@ -110,7 +109,7 @@ class FullCallingPipeline extends QScript { gather.commandDirectory = new File("CleanedBams/IntermediateFiles/%s/ScatterGather/Gather_%s".format(sampleId, source.field.getName)) gather.jobOutputFile = new File(".queue/logs/Cleaning/%s/FixMates.out".format(sampleId)) gather.memoryLimit = Some(6) - gather.jarFile = qscript.picardFixMatesJar + gather.mainClass = picardFixMatesClass gather.assumeSorted = None case (gather: GatherFunction, source: ArgumentSource) => gather.commandDirectory = new File("CleanedBams/IntermediateFiles/%s/ScatterGather/Gather_%s".format(sampleId, source.field.getName)) @@ -123,7 +122,7 @@ class FullCallingPipeline extends QScript { realigner.isIntermediate = true // Explicitly run fix mates if the function won't be scattered. - val fixMates = new PicardBamJarFunction { + val fixMates = new PicardBamEmbeddedFunction { @Input(doc="unfixed bam") var unfixed: File = _ @Output(doc="fixed bam") var fixed: File = _ def inputBams = List(unfixed) @@ -132,7 +131,7 @@ class FullCallingPipeline extends QScript { fixMates.jobOutputFile = new File(".queue/logs/Cleaning/%s/FixMates.out".format(sampleId)) fixMates.memoryLimit = Some(6) - fixMates.jarFile = qscript.picardFixMatesJar + fixMates.mainClass = picardFixMatesClass fixMates.unfixed = realigner.out fixMates.fixed = cleaned_bam fixMates.analysisName = "FixMates_"+sampleId diff --git a/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala b/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala index abf4cbe09..1ec8f44a6 100755 --- a/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala +++ b/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala @@ -310,11 +310,10 @@ class QGraph extends Logging { * Dry-runs the jobs by traversing the graph. */ private def dryRunJobs() { - if (settings.startFromScratch) { + if (settings.startFromScratch) logger.info("Will remove outputs from previous runs.") - foreachFunction(_.resetToPending(false)) - } else - updateGraphStatus(false) + + updateGraphStatus(false) var readyJobs = getReadyJobs() while (running && readyJobs.size > 0) { @@ -361,11 +360,10 @@ class QGraph extends Logging { settings.jobRunner = "Shell" commandLineManager = commandLinePluginManager.createByName(settings.jobRunner) - if (settings.startFromScratch) { + if (settings.startFromScratch) logger.info("Removing outputs from previous runs.") - foreachFunction(_.resetToPending(true)) - } else - updateGraphStatus(true) + + updateGraphStatus(true) var readyJobs = TreeSet.empty[FunctionEdge](functionOrdering) readyJobs ++= getReadyJobs() @@ -458,7 +456,10 @@ class QGraph extends Logging { * @param cleanOutputs If true will delete outputs when setting edges to pending. */ private def updateGraphStatus(cleanOutputs: Boolean) { - traverseFunctions(edge => checkDone(edge, cleanOutputs)) + if (settings.startFromScratch) + foreachFunction(edge => edge.resetToPending(cleanOutputs)) + else + traverseFunctions(edge => checkDone(edge, cleanOutputs)) traverseFunctions(edge => recheckDone(edge)) } diff --git a/scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala b/scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala index 8a442f6f5..67dfbba64 100644 --- a/scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala @@ -1,14 +1,19 @@ package org.broadinstitute.sting.queue.extensions.gatk import org.broadinstitute.sting.queue.function.scattergather.GatherFunction -import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction +import org.broadinstitute.sting.queue.extensions.picard.PicardBamEmbeddedFunction /** - * Merges BAM files using Picards MergeSamFiles.jar. - * At the Broad the jar can be found at /seq/software/picard/current/bin/MergeSamFiles.jar. Outside the broad see http://picard.sourceforge.net/") + * Merges BAM files using Picards net.sf.picard.sam.MergeSamFiles. */ -class BamGatherFunction extends GatherFunction with PicardBamJarFunction { +class BamGatherFunction extends GatherFunction with PicardBamEmbeddedFunction { + this.mainClass = "net.sf.picard.sam.MergeSamFiles" this.assumeSorted = Some(true) protected def inputBams = gatherParts protected def outputBam = originalOutput + + override def init() { + // Whatever the original function can handle, merging *should* do less. + this.memoryLimit = originalFunction.memoryLimit + } } diff --git a/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamEmbeddedFunction.scala b/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamEmbeddedFunction.scala new file mode 100644 index 000000000..deb251d89 --- /dev/null +++ b/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamEmbeddedFunction.scala @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.queue.extensions.picard + +import org.broadinstitute.sting.queue.function.EmbeddedCommandLineFunction + +/** + * Wraps a Picard embedded class that operates on BAM files. + * See http://picard.sourceforge.net/ for more info. + * + * Since the command lines take slightly different arguments + * some values are optional. + */ +trait PicardBamEmbeddedFunction extends EmbeddedCommandLineFunction with PicardBamFunction { +} diff --git a/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamFunction.scala b/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamFunction.scala new file mode 100644 index 000000000..45c51c60c --- /dev/null +++ b/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamFunction.scala @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.queue.extensions.picard + +import java.io.File +import org.broadinstitute.sting.queue.function.CommandLineFunction + +/** + * Wraps a Picard function that operates on BAM files. + * See http://picard.sourceforge.net/ for more info. + * + * Since the various BAM utilities take slightly different arguments + * some values are optional. + */ +trait PicardBamFunction extends CommandLineFunction { + var validationStringency = "SILENT" + var sortOrder = "coordinate" + var compressionLevel: Option[Int] = None + var maxRecordsInRam: Option[Int] = None + var assumeSorted: Option[Boolean] = None + + protected def inputBams: List[File] + protected def outputBam: File + + abstract override def commandLine = super.commandLine + + Array(optional(" COMPRESSION_LEVEL=", compressionLevel), optional(" VALIDATION_STRINGENCY=", validationStringency), + optional(" SO=", sortOrder), optional( " MAX_RECORDS_IN_RAM=", maxRecordsInRam), optional(" ASSUME_SORTED=", assumeSorted), + " OUTPUT=" + outputBam, repeat(" INPUT=", inputBams), " TMP_DIR=" + jobTempDir).mkString +} diff --git a/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamJarFunction.scala b/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamJarFunction.scala index c86a389c1..31e95f0d1 100644 --- a/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamJarFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamJarFunction.scala @@ -1,7 +1,30 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.extensions.picard import org.broadinstitute.sting.queue.function.JarCommandLineFunction -import java.io.File /** * Wraps a Picard jar that operates on BAM files. @@ -10,18 +33,5 @@ import java.io.File * Since the jar files take slightly different arguments * some values are optional. */ -trait PicardBamJarFunction extends JarCommandLineFunction { - var validationStringency = "SILENT" - var sortOrder = "coordinate" - var compressionLevel: Option[Int] = None - var maxRecordsInRam: Option[Int] = None - var assumeSorted: Option[Boolean] = None - - protected def inputBams: List[File] - protected def outputBam: File - - override def commandLine = super.commandLine + "%s%s%s%s%s%s%s%s".format( - optional(" COMPRESSION_LEVEL=", compressionLevel), optional(" VALIDATION_STRINGENCY=", validationStringency), - optional(" SO=", sortOrder), optional( " MAX_RECORDS_IN_RAM=", maxRecordsInRam), optional(" ASSUME_SORTED=", assumeSorted), - " OUTPUT=" + outputBam, repeat(" INPUT=", inputBams), " TMP_DIR=" + jobTempDir) +trait PicardBamJarFunction extends JarCommandLineFunction with PicardBamFunction { } diff --git a/scala/src/org/broadinstitute/sting/queue/function/EmbeddedCommandLineFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/EmbeddedCommandLineFunction.scala new file mode 100644 index 000000000..e9987cdef --- /dev/null +++ b/scala/src/org/broadinstitute/sting/queue/function/EmbeddedCommandLineFunction.scala @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.queue.function + +import java.io.File +import org.broadinstitute.sting.queue.util.IOUtils +import org.broadinstitute.sting.commandline.Argument + +/** + * Defines a command line function that runs java code from inside the existing class path. + */ +trait EmbeddedCommandLineFunction extends JavaCommandLineFunction { + @Argument(doc="Main class to run from the current classpath") + var mainClass: String = null + + def javaExecutable = "-cp %s %s".format(EmbeddedCommandLineFunction.classpath, mainClass) +} + +object EmbeddedCommandLineFunction { + private val classpath = System.getProperty("java.class.path") + .split(File.pathSeparatorChar).map(path => IOUtils.absolute(new File(path))) + .mkString("\"", "\"" + File.pathSeparator + "\"", "\"") +}