Resolving simple conflicts in the data processing pipeline.

Conflicts:
	public/scala/qscript/org/broadinstitute/sting/queue/qscripts/DataProcessingPipeline.scala

public/scala/qscript/org/broadinstitute/sting/queue/qscripts/DataProcessingPipeline.scala

@@ -72,6 +72,9 @@ class DataProcessingPipeline extends QScript {
   @Input(doc="Number of threads BWA should use", fullName="bwa_threads", shortName="bt", required=false)
   var bwaThreads: Int = 1
 
+  @Input(doc="Dont perform validation on the BAM files", fullName="no_validation", shortName="nv", required=false)
+  var noValidation: Boolean = false
+
 
   /****************************************************************************
   * Global Variables
@@ -135,7 +138,7 @@ class DataProcessingPipeline extends QScript {
       }
     }
 
-    println("\n\n*** DEBUG ***\n")
+    println("\n\n*** INPUT FILES ***\n")
     // Creating one file for each sample in the dataset
     val sampleBamFiles = scala.collection.mutable.Map.empty[String, File]
     for ((sample, flist) <- sampleTable) {
@@ -149,7 +152,7 @@ class DataProcessingPipeline extends QScript {
       sampleBamFiles(sample) = sampleFileName
       add(joinBams(flist, sampleFileName))
     }
-    println("*** DEBUG ***\n\n")
+    println("*** INPUT FILES ***\n\n")
 
     return sampleBamFiles.toMap
   }
@@ -246,7 +249,12 @@ class DataProcessingPipeline extends QScript {
       val preValidateLog  = swapExt(bam, ".bam", ".pre.validation")
       val postValidateLog = swapExt(bam, ".bam", ".post.validation")
 
-      add(validate(bam, preValidateLog))
+      // Validation is an optional step for the BAM file generated after
+      // alignment and the final bam file of the pipeline.
+      if (!noValidation) {
+        add(validate(bam, preValidateLog),
+            validate(recalBam, postValidateLog))
+      }
 
       if (cleaningModel != ConsensusDeterminationModel.KNOWNS_ONLY)
         add(target(bam, targetIntervals))
@@ -257,8 +265,8 @@ class DataProcessingPipeline extends QScript {
           recal(dedupedBam, preRecalFile, recalBam),
           cov(recalBam, postRecalFile),
           analyzeCovariates(preRecalFile, preOutPath),
-          analyzeCovariates(postRecalFile, postOutPath),
+          analyzeCovariates(postRecalFile, postOutPath))
-          validate(recalBam, postValidateLog))
+
 
       cohortList :+= recalBam
     }
@@ -282,6 +290,13 @@ class DataProcessingPipeline extends QScript {
     this.isIntermediate = true
   }
 
+  // General arguments to non-GATK tools
+  trait ExternalCommonArgs extends CommandLineFunction {
+    this.memoryLimit = 4
+    this.isIntermediate = true
+  }
+
+
   case class target (inBams: File, outIntervals: File) extends RealignerTargetCreator with CommandLineGATKArgs {
     if (cleaningModel != ConsensusDeterminationModel.KNOWNS_ONLY)
       this.input_file :+= inBams
@@ -300,8 +315,8 @@ class DataProcessingPipeline extends QScript {
     this.targetIntervals = tIntervals
     this.out = outBam
     this.rodBind :+= RodBind("dbsnp", "VCF", dbSNP)
-    if (!indels.isEmpty)
+    if (!qscript.indels.isEmpty)
-      this.rodBind :+= RodBind("indels", "VCF", indels)
+      this.rodBind :+= RodBind("indels", "VCF", qscript.indels)
     this.consensusDeterminationModel =  consensusDeterminationModel
     this.compress = 0
     this.scatterCount = nContigs
@@ -332,7 +347,6 @@ class DataProcessingPipeline extends QScript {
     this.isIntermediate = false
     this.analysisName = queueLogDir + outBam + ".recalibration"
     this.jobName = queueLogDir + outBam + ".recalibration"
-
   }
 
 
@@ -350,48 +364,41 @@ class DataProcessingPipeline extends QScript {
     this.jobName = queueLogDir + inRecalFile + ".analyze_covariates"
   }
 
-  case class dedup (inBam: File, outBam: File, metricsFile: File) extends MarkDuplicates {
+  case class dedup (inBam: File, outBam: File, metricsFile: File) extends MarkDuplicates with ExternalCommonArgs {
     this.input = List(inBam)
     this.output = outBam
     this.metrics = metricsFile
-    this.memoryLimit = 6
-    this.isIntermediate = true
     this.analysisName = queueLogDir + outBam + ".dedup"
     this.jobName = queueLogDir + outBam + ".dedup"
   }
 
-  case class joinBams (inBams: List[File], outBam: File) extends MergeSamFiles {
+  case class joinBams (inBams: List[File], outBam: File) extends MergeSamFiles with ExternalCommonArgs {
     this.input = inBams
     this.output = outBam
-    this.memoryLimit = 4
-    this.isIntermediate = true
     this.analysisName = queueLogDir + outBam + ".joinBams"
     this.jobName = queueLogDir + outBam + ".joinBams"
   }
 
-  case class sortSam (inSam: File, outBam: File, sortOrderP: SortOrder) extends SortSam {
+  case class sortSam (inSam: File, outBam: File, sortOrderP: SortOrder) extends SortSam with ExternalCommonArgs {
     this.input = List(inSam)
     this.output = outBam
     this.sortOrder = sortOrderP
-    this.memoryLimit = 4
-    this.isIntermediate = true
     this.analysisName = queueLogDir + outBam + ".sortSam"
     this.jobName = queueLogDir + outBam + ".sortSam"
   }
 
-  case class validate (inBam: File, outLog: File) extends ValidateSamFile {
+  case class validate (inBam: File, outLog: File) extends ValidateSamFile with ExternalCommonArgs {
     this.input = List(inBam)
     this.output = outLog
     this.maxRecordsInRam = 100000
     this.REFERENCE_SEQUENCE = qscript.reference
-    this.memoryLimit = 4
     this.isIntermediate = false
     this.analysisName = queueLogDir + outLog + ".validate"
     this.jobName = queueLogDir + outLog + ".validate"
   }
 
 
-  case class addReadGroup (inBam: File, outBam: File, readGroup: ReadGroup) extends AddOrReplaceReadGroups {
+  case class addReadGroup (inBam: File, outBam: File, readGroup: ReadGroup) extends AddOrReplaceReadGroups with ExternalCommonArgs {
     this.input = List(inBam)
     this.output = outBam
     this.RGID = readGroup.id
@@ -407,12 +414,7 @@ class DataProcessingPipeline extends QScript {
     this.jobName = queueLogDir + outBam + ".rg"
   }
 
-  trait BWACommonArgs extends CommandLineFunction {
+  case class bwa_aln_se (inBam: File, outSai: File) extends CommandLineFunction with ExternalCommonArgs {
-    this.memoryLimit = 4
-    this.isIntermediate = true
-  }
-
-  case class bwa_aln_se (inBam: File, outSai: File) extends CommandLineFunction with BWACommonArgs {
     @Input(doc="bam file to be aligned") var bam = inBam
     @Output(doc="output sai file") var sai = outSai
     def commandLine = bwaPath + " aln -t " + bwaThreads + " -q 5 " + reference + " -b " + bam + " > " + sai
@@ -420,7 +422,7 @@ class DataProcessingPipeline extends QScript {
     this.jobName = queueLogDir + outSai + ".bwa_aln_se"
   }
 
-  case class bwa_aln_pe (inBam: File, outSai1: File, index: Int) extends CommandLineFunction with BWACommonArgs {
+  case class bwa_aln_pe (inBam: File, outSai1: File, index: Int) extends CommandLineFunction with ExternalCommonArgs {
     @Input(doc="bam file to be aligned") var bam = inBam
     @Output(doc="output sai file for 1st mating pair") var sai = outSai1
     def commandLine = bwaPath + " aln -t " + bwaThreads + " -q 5 " + reference + " -b" + index + " " + bam + " > " + sai
@@ -428,7 +430,7 @@ class DataProcessingPipeline extends QScript {
     this.jobName = queueLogDir + outSai1 + ".bwa_aln_pe1"
   }
 
-  case class bwa_sam_se (inBam: File, inSai: File, outBam: File) extends CommandLineFunction with BWACommonArgs {
+  case class bwa_sam_se (inBam: File, inSai: File, outBam: File) extends CommandLineFunction with ExternalCommonArgs {
     @Input(doc="bam file to be aligned") var bam = inBam
     @Input(doc="bwa alignment index file") var sai = inSai
     @Output(doc="output aligned bam file") var alignedBam = outBam
@@ -437,7 +439,7 @@ class DataProcessingPipeline extends QScript {
     this.jobName = queueLogDir + outBam + ".bwa_sam_se"
   }
 
-  case class bwa_sam_pe (inBam: File, inSai1: File, inSai2:File, outBam: File) extends CommandLineFunction with BWACommonArgs {
+  case class bwa_sam_pe (inBam: File, inSai1: File, inSai2:File, outBam: File) extends CommandLineFunction with ExternalCommonArgs {
     @Input(doc="bam file to be aligned") var bam = inBam
     @Input(doc="bwa alignment index file for 1st mating pair") var sai1 = inSai1
     @Input(doc="bwa alignment index file for 2nd mating pair") var sai2 = inSai2

public/scala/qscript/org/broadinstitute/sting/queue/qscripts/RecalibrateBaseQualities.scala

@@ -20,14 +20,14 @@ class RecalibrateBaseQualities extends QScript {
   @Input(doc="input BAM file - or list of BAM files", shortName="i", required=true)
   var input: File = _
 
-  @Input(doc="path to R resources folder inside the Sting repository", fullName="path_to_r", shortName="r", required=false)
+  @Input(doc="path to R resources folder inside the Sting repository", fullName="path_to_r", shortName="r", required=true)
-  var R: String = new File("/humgen/gsa-scr1/carneiro/stable/R")
+  var R: String = _
 
-  @Input(doc="Reference fasta file", shortName="R", required=false)
+  @Input(doc="Reference fasta file", shortName="R", required=true)
-  var reference: File = new File("/seq/references/Homo_sapiens_assembly19/v1/Homo_sapiens_assembly19.fasta")
+  var reference: File = _ // new File("/seq/references/Homo_sapiens_assembly19/v1/Homo_sapiens_assembly19.fasta")
 
-  @Input(doc="dbsnp ROD to use (VCF)", shortName="D", required=false)
+  @Input(doc="dbsnp ROD to use (VCF)", shortName="D", required=true)
-  var dbSNP: File = new File("/humgen/gsa-hpprojects/GATK/data/dbsnp_132_b37.leftAligned.vcf")
+  var dbSNP: File = _     // new File("/humgen/gsa-hpprojects/GATK/data/dbsnp_132_b37.leftAligned.vcf")
 
   val queueLogDir: String = ".qlog/"
   var nContigs: Int = 0