zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bringing in the changes from the CMI repo

This commit is contained in:
Mauricio Carneiro 2012-12-13 21:59:37 -05:00
parent 696bf95fba 97e880654b
commit 74344a3871
5 changed files with 113 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
protected/java/test/org/broadinstitute/sting/gatk/walkers/compression/reducereads/ReduceReadsIntegrationTest.java
View File

@ -17,6 +17,7 @@ public class ReduceReadsIntegrationTest extends WalkerTest {
final String COREDUCTION_BAM_A = validationDataLocation + "coreduction.test.A.bam";
final String COREDUCTION_BAM_B = validationDataLocation + "coreduction.test.B.bam";
final String COREDUCTION_L = " -L 1:1,853,860-1,854,354 -L 1:1,884,131-1,892,057";
final String OFFCONTIG_BAM = privateTestDir + "readOffb37contigMT.bam";
private void RRTest(String testName, String args, String md5) {
String base = String.format("-T ReduceReads -npt -R %s -I %s ", REF, BAM) + " -o %s ";
@ -86,5 +87,15 @@ public class ReduceReadsIntegrationTest extends WalkerTest {
executeTest("testCoReduction", new WalkerTestSpec(base, Arrays.asList("5c30fde961a1357bf72c15144c01981b")));
}
/**
* Bug happens when reads are soft-clipped off the contig (usually in the MT). This test guarantees no changes to the upstream code will
* break the current hard-clipping routine that protects reduce reads from such reads.
*/
@Test(enabled = true)
public void testReadOffContig() {
String base = String.format("-T ReduceReads -npt -R %s -I %s ", REF, OFFCONTIG_BAM) + " -o %s ";
executeTest("testReadOffContig", new WalkerTestSpec(base, Arrays.asList("53e16367d333da0b7d40a7683a35c95f")));
}
}

3
public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java
View File

@ -397,9 +397,6 @@ public class GATKSAMRecord extends BAMRecord {
else if (op != CigarOperator.HARD_CLIP)
break;
}
if ( softStart < 1 )
softStart = 1;
}
return softStart;
}

172
public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala
View File

@ -110,95 +110,103 @@ class QCommandLine extends CommandLineProgram with Logging {
* functions, and then builds and runs a QGraph based on the dependencies.
*/
def execute = {
ClassFieldCache.parsingEngine = this.parser
var success = false
var result = 1
try {
ClassFieldCache.parsingEngine = this.parser
if (settings.qSettings.runName == null)
settings.qSettings.runName = FilenameUtils.removeExtension(scripts.head.getName)
if (IOUtils.isDefaultTempDir(settings.qSettings.tempDirectory))
settings.qSettings.tempDirectory = IOUtils.absolute(settings.qSettings.runDirectory, ".queue/tmp")
qGraph.initializeWithSettings(settings)
if (settings.qSettings.runName == null)
settings.qSettings.runName = FilenameUtils.removeExtension(scripts.head.getName)
if (IOUtils.isDefaultTempDir(settings.qSettings.tempDirectory))
settings.qSettings.tempDirectory = IOUtils.absolute(settings.qSettings.runDirectory, ".queue/tmp")
qGraph.initializeWithSettings(settings)
for (commandPlugin <- allCommandPlugins) {
loadArgumentsIntoObject(commandPlugin)
}
for (commandPlugin <- allCommandPlugins) {
if (commandPlugin.statusMessenger != null)
commandPlugin.statusMessenger.started()
}
qGraph.messengers = allCommandPlugins.filter(_.statusMessenger != null).map(_.statusMessenger).toSeq
// TODO: Default command plugin argument?
val remoteFileConverter = (
for (commandPlugin <- allCommandPlugins if (commandPlugin.remoteFileConverter != null))
yield commandPlugin.remoteFileConverter
).headOption.getOrElse(null)
if (remoteFileConverter != null)
loadArgumentsIntoObject(remoteFileConverter)
val allQScripts = qScriptPluginManager.createAllTypes()
for (script <- allQScripts) {
logger.info("Scripting " + qScriptPluginManager.getName(script.getClass.asSubclass(classOf[QScript])))
loadArgumentsIntoObject(script)
allCommandPlugins.foreach(_.initScript(script))
// TODO: Pulling inputs can be time/io expensive! Some scripts are using the files to generate functions-- even for dry runs-- so pull it all down for now.
//if (settings.run)
script.pullInputs()
script.qSettings = settings.qSettings
try {
script.script()
} catch {
case e: Exception =>
throw new UserException.CannotExecuteQScript(script.getClass.getSimpleName + ".script() threw the following exception: " + e, e)
for (commandPlugin <- allCommandPlugins) {
loadArgumentsIntoObject(commandPlugin)
}
if (remoteFileConverter != null) {
if (remoteFileConverter.convertToRemoteEnabled)
script.mkRemoteOutputs(remoteFileConverter)
}
script.functions.foreach(qGraph.add(_))
logger.info("Added " + script.functions.size + " functions")
}
// Execute the job graph
qGraph.run()
val functionsAndStatus = qGraph.getFunctionsAndStatus
val success = qGraph.success
// walk over each script, calling onExecutionDone
for (script <- allQScripts) {
val scriptFunctions = functionsAndStatus.filterKeys(f => script.functions.contains(f))
script.onExecutionDone(scriptFunctions, success)
}
logger.info("Script %s with %d total jobs".format(if (success) "completed successfully" else "failed", functionsAndStatus.size))
// write the final complete job report
logger.info("Writing final jobs report...")
qGraph.writeJobsReport()
if (!success) {
logger.info("Done with errors")
qGraph.logFailed()
for (commandPlugin <- allCommandPlugins)
for (commandPlugin <- allCommandPlugins) {
if (commandPlugin.statusMessenger != null)
commandPlugin.statusMessenger.exit("Done with errors: %s".format(qGraph.formattedStatusCounts))
1
} else {
if (settings.run) {
allQScripts.foreach(_.pushOutputs())
for (commandPlugin <- allCommandPlugins)
if (commandPlugin.statusMessenger != null) {
val allInputs = allQScripts.map(_.remoteInputs)
val allOutputs = allQScripts.map(_.remoteOutputs)
commandPlugin.statusMessenger.done(allInputs, allOutputs)
}
commandPlugin.statusMessenger.started()
}
qGraph.messengers = allCommandPlugins.filter(_.statusMessenger != null).map(_.statusMessenger).toSeq
// TODO: Default command plugin argument?
val remoteFileConverter = (
for (commandPlugin <- allCommandPlugins if (commandPlugin.remoteFileConverter != null))
yield commandPlugin.remoteFileConverter
).headOption.getOrElse(null)
if (remoteFileConverter != null)
loadArgumentsIntoObject(remoteFileConverter)
val allQScripts = qScriptPluginManager.createAllTypes()
for (script <- allQScripts) {
logger.info("Scripting " + qScriptPluginManager.getName(script.getClass.asSubclass(classOf[QScript])))
loadArgumentsIntoObject(script)
allCommandPlugins.foreach(_.initScript(script))
// TODO: Pulling inputs can be time/io expensive! Some scripts are using the files to generate functions-- even for dry runs-- so pull it all down for now.
//if (settings.run)
script.pullInputs()
script.qSettings = settings.qSettings
try {
script.script()
} catch {
case e: Exception =>
throw new UserException.CannotExecuteQScript(script.getClass.getSimpleName + ".script() threw the following exception: " + e, e)
}
if (remoteFileConverter != null) {
if (remoteFileConverter.convertToRemoteEnabled)
script.mkRemoteOutputs(remoteFileConverter)
}
script.functions.foreach(qGraph.add(_))
logger.info("Added " + script.functions.size + " functions")
}
// Execute the job graph
qGraph.run()
val functionsAndStatus = qGraph.getFunctionsAndStatus
// walk over each script, calling onExecutionDone
for (script <- allQScripts) {
val scriptFunctions = functionsAndStatus.filterKeys(f => script.functions.contains(f))
script.onExecutionDone(scriptFunctions, success)
}
logger.info("Script %s with %d total jobs".format(if (success) "completed successfully" else "failed", functionsAndStatus.size))
// write the final complete job report
logger.info("Writing final jobs report...")
qGraph.writeJobsReport()
if (qGraph.success) {
if (settings.run) {
allQScripts.foreach(_.pushOutputs())
for (commandPlugin <- allCommandPlugins)
if (commandPlugin.statusMessenger != null) {
val allInputs = allQScripts.map(_.remoteInputs)
val allOutputs = allQScripts.map(_.remoteOutputs)
commandPlugin.statusMessenger.done(allInputs, allOutputs)
}
}
success = true
result = 0
}
} finally {
if (!success) {
logger.info("Done with errors")
qGraph.logFailed()
if (settings.run) {
for (commandPlugin <- allCommandPlugins)
if (commandPlugin.statusMessenger != null)
commandPlugin.statusMessenger.exit("Done with errors: %s".format(qGraph.formattedStatusCounts))
}
}
0
}
result
}
/**

33
public/scala/src/org/broadinstitute/sting/queue/QScript.scala
View File

@ -124,7 +124,7 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon
}
/**
* Pull all remote files to the local disk.
* Pull all remote files to the local disk
*/
def pullInputs() {
val inputs = ClassFieldCache.getFieldFiles(this, inputFields)
@ -135,7 +135,7 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon
}
/**
* Push all remote files from the local disk.
* Push all remote files from the local disk
*/
def pushOutputs() {
val outputs = ClassFieldCache.getFieldFiles(this, outputFields)
@ -145,28 +145,17 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon
}
}
/**
* List out the remote outputs
* @return the RemoteFile outputs by argument source
*/
def remoteInputs: Map[String, Seq[RemoteFile]] = tagMap(remoteFieldMap(inputFields))
/**
* List out the remote outputs
* @return the RemoteFile outputs by argument source
*/
def remoteOutputs: Map[String, Seq[RemoteFile]] = tagMap(remoteFieldMap(outputFields))
private def tagMap(remoteFieldMap: Map[ArgumentSource, Seq[RemoteFile]]): Map[String, Seq[RemoteFile]] = {
remoteFieldMap.collect{ case (k, v) => ClassFieldCache.fullName(k) -> v }.toMap
}
private def remoteFieldMap(fields: Seq[ArgumentSource]): Map[ArgumentSource, Seq[RemoteFile]] = {
fields.map(field => (field -> filterRemoteFiles(ClassFieldCache.getFieldFiles(this, field)))).filter(tuple => !tuple._2.isEmpty).toMap
}
private def filterRemoteFiles(fields: Seq[File]): Seq[RemoteFile] =
fields.filter(field => field != null && field.isInstanceOf[RemoteFile]).map(_.asInstanceOf[RemoteFile])
/**
* @return the inputs or null if there are no inputs
*/
def remoteInputs: AnyRef = null
/**
* @return the outputs or null if there are no outputs
*/
def remoteOutputs: AnyRef = null
/** The complete list of fields. */
def functionFields: Seq[ArgumentSource] = ClassFieldCache.classFunctionFields(this.getClass)

2
public/scala/src/org/broadinstitute/sting/queue/engine/QStatusMessenger.scala
View File

@ -7,7 +7,7 @@ import org.broadinstitute.sting.queue.util.RemoteFile
*/
trait QStatusMessenger {
def started()
def done(inputs: Seq[Map[String, Seq[RemoteFile]]], outputs: Seq[Map[String, Seq[RemoteFile]]])
def done(inputs: Seq[_], outputs: Seq[_])
def exit(message: String)
def started(job: String)