diff --git a/protected/java/test/org/broadinstitute/sting/gatk/walkers/compression/reducereads/ReduceReadsIntegrationTest.java b/protected/java/test/org/broadinstitute/sting/gatk/walkers/compression/reducereads/ReduceReadsIntegrationTest.java index 1e539dc9d..f0e8b76d4 100755 --- a/protected/java/test/org/broadinstitute/sting/gatk/walkers/compression/reducereads/ReduceReadsIntegrationTest.java +++ b/protected/java/test/org/broadinstitute/sting/gatk/walkers/compression/reducereads/ReduceReadsIntegrationTest.java @@ -17,6 +17,7 @@ public class ReduceReadsIntegrationTest extends WalkerTest { final String COREDUCTION_BAM_A = validationDataLocation + "coreduction.test.A.bam"; final String COREDUCTION_BAM_B = validationDataLocation + "coreduction.test.B.bam"; final String COREDUCTION_L = " -L 1:1,853,860-1,854,354 -L 1:1,884,131-1,892,057"; + final String OFFCONTIG_BAM = privateTestDir + "readOffb37contigMT.bam"; private void RRTest(String testName, String args, String md5) { String base = String.format("-T ReduceReads -npt -R %s -I %s ", REF, BAM) + " -o %s "; @@ -86,5 +87,15 @@ public class ReduceReadsIntegrationTest extends WalkerTest { executeTest("testCoReduction", new WalkerTestSpec(base, Arrays.asList("5c30fde961a1357bf72c15144c01981b"))); } + /** + * Bug happens when reads are soft-clipped off the contig (usually in the MT). This test guarantees no changes to the upstream code will + * break the current hard-clipping routine that protects reduce reads from such reads. + */ + @Test(enabled = true) + public void testReadOffContig() { + String base = String.format("-T ReduceReads -npt -R %s -I %s ", REF, OFFCONTIG_BAM) + " -o %s "; + executeTest("testReadOffContig", new WalkerTestSpec(base, Arrays.asList("53e16367d333da0b7d40a7683a35c95f"))); + } + } diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java index 6c7a162f8..9fdb48b34 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java @@ -397,9 +397,6 @@ public class GATKSAMRecord extends BAMRecord { else if (op != CigarOperator.HARD_CLIP) break; } - - if ( softStart < 1 ) - softStart = 1; } return softStart; } diff --git a/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala b/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala index 637174557..f899af86d 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala @@ -110,95 +110,103 @@ class QCommandLine extends CommandLineProgram with Logging { * functions, and then builds and runs a QGraph based on the dependencies. */ def execute = { - ClassFieldCache.parsingEngine = this.parser + var success = false + var result = 1 + try { + ClassFieldCache.parsingEngine = this.parser - if (settings.qSettings.runName == null) - settings.qSettings.runName = FilenameUtils.removeExtension(scripts.head.getName) - if (IOUtils.isDefaultTempDir(settings.qSettings.tempDirectory)) - settings.qSettings.tempDirectory = IOUtils.absolute(settings.qSettings.runDirectory, ".queue/tmp") - qGraph.initializeWithSettings(settings) + if (settings.qSettings.runName == null) + settings.qSettings.runName = FilenameUtils.removeExtension(scripts.head.getName) + if (IOUtils.isDefaultTempDir(settings.qSettings.tempDirectory)) + settings.qSettings.tempDirectory = IOUtils.absolute(settings.qSettings.runDirectory, ".queue/tmp") + qGraph.initializeWithSettings(settings) - for (commandPlugin <- allCommandPlugins) { - loadArgumentsIntoObject(commandPlugin) - } - - for (commandPlugin <- allCommandPlugins) { - if (commandPlugin.statusMessenger != null) - commandPlugin.statusMessenger.started() - } - - qGraph.messengers = allCommandPlugins.filter(_.statusMessenger != null).map(_.statusMessenger).toSeq - - // TODO: Default command plugin argument? - val remoteFileConverter = ( - for (commandPlugin <- allCommandPlugins if (commandPlugin.remoteFileConverter != null)) - yield commandPlugin.remoteFileConverter - ).headOption.getOrElse(null) - - if (remoteFileConverter != null) - loadArgumentsIntoObject(remoteFileConverter) - - val allQScripts = qScriptPluginManager.createAllTypes() - for (script <- allQScripts) { - logger.info("Scripting " + qScriptPluginManager.getName(script.getClass.asSubclass(classOf[QScript]))) - loadArgumentsIntoObject(script) - allCommandPlugins.foreach(_.initScript(script)) - // TODO: Pulling inputs can be time/io expensive! Some scripts are using the files to generate functions-- even for dry runs-- so pull it all down for now. - //if (settings.run) - script.pullInputs() - script.qSettings = settings.qSettings - try { - script.script() - } catch { - case e: Exception => - throw new UserException.CannotExecuteQScript(script.getClass.getSimpleName + ".script() threw the following exception: " + e, e) + for (commandPlugin <- allCommandPlugins) { + loadArgumentsIntoObject(commandPlugin) } - if (remoteFileConverter != null) { - if (remoteFileConverter.convertToRemoteEnabled) - script.mkRemoteOutputs(remoteFileConverter) - } - - script.functions.foreach(qGraph.add(_)) - logger.info("Added " + script.functions.size + " functions") - } - // Execute the job graph - qGraph.run() - - val functionsAndStatus = qGraph.getFunctionsAndStatus - val success = qGraph.success - - // walk over each script, calling onExecutionDone - for (script <- allQScripts) { - val scriptFunctions = functionsAndStatus.filterKeys(f => script.functions.contains(f)) - script.onExecutionDone(scriptFunctions, success) - } - - logger.info("Script %s with %d total jobs".format(if (success) "completed successfully" else "failed", functionsAndStatus.size)) - - // write the final complete job report - logger.info("Writing final jobs report...") - qGraph.writeJobsReport() - - if (!success) { - logger.info("Done with errors") - qGraph.logFailed() - for (commandPlugin <- allCommandPlugins) + for (commandPlugin <- allCommandPlugins) { if (commandPlugin.statusMessenger != null) - commandPlugin.statusMessenger.exit("Done with errors: %s".format(qGraph.formattedStatusCounts)) - 1 - } else { - if (settings.run) { - allQScripts.foreach(_.pushOutputs()) - for (commandPlugin <- allCommandPlugins) - if (commandPlugin.statusMessenger != null) { - val allInputs = allQScripts.map(_.remoteInputs) - val allOutputs = allQScripts.map(_.remoteOutputs) - commandPlugin.statusMessenger.done(allInputs, allOutputs) - } + commandPlugin.statusMessenger.started() + } + + qGraph.messengers = allCommandPlugins.filter(_.statusMessenger != null).map(_.statusMessenger).toSeq + + // TODO: Default command plugin argument? + val remoteFileConverter = ( + for (commandPlugin <- allCommandPlugins if (commandPlugin.remoteFileConverter != null)) + yield commandPlugin.remoteFileConverter + ).headOption.getOrElse(null) + + if (remoteFileConverter != null) + loadArgumentsIntoObject(remoteFileConverter) + + val allQScripts = qScriptPluginManager.createAllTypes() + for (script <- allQScripts) { + logger.info("Scripting " + qScriptPluginManager.getName(script.getClass.asSubclass(classOf[QScript]))) + loadArgumentsIntoObject(script) + allCommandPlugins.foreach(_.initScript(script)) + // TODO: Pulling inputs can be time/io expensive! Some scripts are using the files to generate functions-- even for dry runs-- so pull it all down for now. + //if (settings.run) + script.pullInputs() + script.qSettings = settings.qSettings + try { + script.script() + } catch { + case e: Exception => + throw new UserException.CannotExecuteQScript(script.getClass.getSimpleName + ".script() threw the following exception: " + e, e) + } + + if (remoteFileConverter != null) { + if (remoteFileConverter.convertToRemoteEnabled) + script.mkRemoteOutputs(remoteFileConverter) + } + + script.functions.foreach(qGraph.add(_)) + logger.info("Added " + script.functions.size + " functions") + } + // Execute the job graph + qGraph.run() + + val functionsAndStatus = qGraph.getFunctionsAndStatus + + // walk over each script, calling onExecutionDone + for (script <- allQScripts) { + val scriptFunctions = functionsAndStatus.filterKeys(f => script.functions.contains(f)) + script.onExecutionDone(scriptFunctions, success) + } + + logger.info("Script %s with %d total jobs".format(if (success) "completed successfully" else "failed", functionsAndStatus.size)) + + // write the final complete job report + logger.info("Writing final jobs report...") + qGraph.writeJobsReport() + + if (qGraph.success) { + if (settings.run) { + allQScripts.foreach(_.pushOutputs()) + for (commandPlugin <- allCommandPlugins) + if (commandPlugin.statusMessenger != null) { + val allInputs = allQScripts.map(_.remoteInputs) + val allOutputs = allQScripts.map(_.remoteOutputs) + commandPlugin.statusMessenger.done(allInputs, allOutputs) + } + } + success = true + result = 0 + } + } finally { + if (!success) { + logger.info("Done with errors") + qGraph.logFailed() + if (settings.run) { + for (commandPlugin <- allCommandPlugins) + if (commandPlugin.statusMessenger != null) + commandPlugin.statusMessenger.exit("Done with errors: %s".format(qGraph.formattedStatusCounts)) + } } - 0 } + result } /** diff --git a/public/scala/src/org/broadinstitute/sting/queue/QScript.scala b/public/scala/src/org/broadinstitute/sting/queue/QScript.scala index eb8be183a..d709d1fb4 100755 --- a/public/scala/src/org/broadinstitute/sting/queue/QScript.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/QScript.scala @@ -124,7 +124,7 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon } /** - * Pull all remote files to the local disk. + * Pull all remote files to the local disk */ def pullInputs() { val inputs = ClassFieldCache.getFieldFiles(this, inputFields) @@ -135,7 +135,7 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon } /** - * Push all remote files from the local disk. + * Push all remote files from the local disk */ def pushOutputs() { val outputs = ClassFieldCache.getFieldFiles(this, outputFields) @@ -145,28 +145,17 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon } } - /** - * List out the remote outputs - * @return the RemoteFile outputs by argument source - */ - def remoteInputs: Map[String, Seq[RemoteFile]] = tagMap(remoteFieldMap(inputFields)) - - /** - * List out the remote outputs - * @return the RemoteFile outputs by argument source - */ - def remoteOutputs: Map[String, Seq[RemoteFile]] = tagMap(remoteFieldMap(outputFields)) - - private def tagMap(remoteFieldMap: Map[ArgumentSource, Seq[RemoteFile]]): Map[String, Seq[RemoteFile]] = { - remoteFieldMap.collect{ case (k, v) => ClassFieldCache.fullName(k) -> v }.toMap - } - - private def remoteFieldMap(fields: Seq[ArgumentSource]): Map[ArgumentSource, Seq[RemoteFile]] = { - fields.map(field => (field -> filterRemoteFiles(ClassFieldCache.getFieldFiles(this, field)))).filter(tuple => !tuple._2.isEmpty).toMap - } - private def filterRemoteFiles(fields: Seq[File]): Seq[RemoteFile] = fields.filter(field => field != null && field.isInstanceOf[RemoteFile]).map(_.asInstanceOf[RemoteFile]) + /** + * @return the inputs or null if there are no inputs + */ + def remoteInputs: AnyRef = null + + /** + * @return the outputs or null if there are no outputs + */ + def remoteOutputs: AnyRef = null /** The complete list of fields. */ def functionFields: Seq[ArgumentSource] = ClassFieldCache.classFunctionFields(this.getClass) diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/QStatusMessenger.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/QStatusMessenger.scala index a1133b944..a69f68b8e 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/QStatusMessenger.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/QStatusMessenger.scala @@ -7,7 +7,7 @@ import org.broadinstitute.sting.queue.util.RemoteFile */ trait QStatusMessenger { def started() - def done(inputs: Seq[Map[String, Seq[RemoteFile]]], outputs: Seq[Map[String, Seq[RemoteFile]]]) + def done(inputs: Seq[_], outputs: Seq[_]) def exit(message: String) def started(job: String)