diff --git a/scala/qscript/core/MethodsDevelopmentCallingPipeline.scala b/scala/qscript/core/MethodsDevelopmentCallingPipeline.scala index ad8e8cdd0..3ca9e39ca 100755 --- a/scala/qscript/core/MethodsDevelopmentCallingPipeline.scala +++ b/scala/qscript/core/MethodsDevelopmentCallingPipeline.scala @@ -1,7 +1,17 @@ import org.broadinstitute.sting.queue.extensions.gatk._ import org.broadinstitute.sting.queue.QScript import org.broadinstitute.sting.gatk.phonehome.GATKRunReport -import org.broadinstitute.sting.queue.function.scattergather.{GatherFunction, CloneFunction, ScatterFunction} + + + // ToDos: + // reduce the scope of the datasets so the script is more nimble + // figure out how to give names to all the Queue-LSF logs (other than Q-1931@node1434-24.out) so that it is easier to find logs for certain steps + // create gold standard BAQ'd bam files, no reason to always do it on the fly + + // Analysis to add at the end of the script: + // auto generation of the cluster plots + // spike in NA12878 to the exomes and to the lowpass, analysis of how much of her variants are being recovered compared to single sample exome or HiSeq calls + // produce Kiran's Venn plots based on comparison between new VCF and gold standard produced VCF class MethodsDevelopmentCallingPipeline extends QScript { @@ -57,6 +67,7 @@ class MethodsDevelopmentCallingPipeline extends QScript { val goldStandard_VCF: File, val intervals: String, val titvTarget: Double, + val trancheTarget: Double, val isLowpass: Boolean) { val name = qscript.outputDir + baseName val clusterFile = new File(name + ".clusters") @@ -69,7 +80,8 @@ class MethodsDevelopmentCallingPipeline extends QScript { val tsRecalibratedVCF = new File(name + ".ts.recalibrated.vcf") val tsTranchesFile = new File(name + ".ts.tranches") val cutVCF = new File(name + ".cut.vcf") - val evalFile = new File(name + ".eval") + val evalFile = new File(name + ".snp.eval") + val evalIndelFile = new File(name + ".indel.eval") val goldStandardName = qscript.outputDir + "goldStandard/" + baseName val goldStandardClusterFile = new File(goldStandardName + ".clusters") } @@ -81,69 +93,64 @@ class MethodsDevelopmentCallingPipeline extends QScript { val dbSNP_hg18_129 = "/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/dbSNP/dbsnp_129_hg18.rod" // Special case for NA12878 collections that can't use 132 because they are part of it. val dbSNP_b36 = "/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/dbSNP/dbsnp_129_b36.rod" val dbSNP_b37 = "/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/dbSNP/dbsnp_132_b37.leftAligned.vcf" - val dbSNP_b37_129 = "/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/dbSNP/dbsnp_129_b37.rod" // Special case for NA12878 collections that can't use 132 because they are part of it. + val dbSNP_b37_129 = "/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/dbSNP/dbsnp_129_b37.leftAligned.vcf" // Special case for NA12878 collections that can't use 132 because they are part of it. val hapmap_hg18 = "/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/HapMap/3.3/sites_r27_nr.hg18_fwd.vcf" val hapmap_b36 = "/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/HapMap/3.3/sites_r27_nr.b36_fwd.vcf" val hapmap_b37 = "/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/HapMap/3.3/sites_r27_nr.b37_fwd.vcf" + val omni_b37 = "/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/Omni2.5_chip/1212samples.b37.vcf" val indelMask_b36 = "/humgen/1kg/processing/pipeline_test_bams/pilot1.dindel.mask.b36.bed" val indelMask_b37 = "/humgen/1kg/processing/pipeline_test_bams/pilot1.dindel.mask.b37.bed" - // ToDos: - // reduce the scope of the datasets so the script is more nimble - // figure out how to give names to all the Queue-LSF logs (other than Q-1931@node1434-24.out) so that it is easier to find logs for certain steps - // create gold standard BAQ'd bam files, no reason to always do it on the fly - - // Analysis to add at the end of the script: - // auto generation of the cluster plots - // spike in NA12878 to the exomes and to the lowpass, analysis of how much of her variants are being recovered compared to single sample exome or HiSeq calls - // produce Kiran's Venn plots based on comparison between new VCF and gold standard produced VCF - val lowPass: Boolean = true val indels: Boolean = true + val useMask: Boolean = !noMASK + val useCut: Boolean = !noCut + val queueLogDir = ".qlog/" + val targetDataSets: Map[String, Target] = Map( "HiSeq" -> new Target("NA12878.HiSeq", hg18, dbSNP_hg18_129, hapmap_hg18, "/humgen/gsa-hpprojects/dev/depristo/oneOffProjects/1000GenomesProcessingPaper/wgs.v13/HiSeq.WGS.cleaned.indels.10.mask", new File("/humgen/gsa-hpprojects/NA12878Collection/bams/NA12878.HiSeq.WGS.bwa.cleaned.recal.bam"), new File("/home/radon01/depristo/work/oneOffProjects/1000GenomesProcessingPaper/wgs.v13/HiSeq.WGS.cleaned.ug.snpfiltered.indelfiltered.vcf"), - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg18.intervals", 2.07, !lowPass), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg18.intervals", 2.07, 1.0, !lowPass), "HiSeq19" -> new Target("NA12878.HiSeq19", hg19, dbSNP_b37_129, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/NA12878.HiSeq.WGS.bwa.cleaned.recal.hg19.bam"), new File("/humgen/gsa-hpprojects/dev/carneiro/hiseq19/analysis/snps/NA12878.HiSeq19.filtered.vcf"), - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 2.3, !lowPass), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 2.3, 1.0, !lowPass), "GA2hg19" -> new Target("NA12878.GA2.hg19", hg19, dbSNP_b37_129, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/NA12878.GA2.WGS.bwa.cleaned.hg19.bam"), new File("/humgen/gsa-hpprojects/dev/carneiro/hiseq19/analysis/snps/NA12878.GA2.hg19.filtered.vcf"), - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 2.3, !lowPass), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 2.3, 1.0, !lowPass), "WEx" -> new Target("NA12878.WEx", hg18, dbSNP_hg18_129, hapmap_hg18, "/humgen/gsa-hpprojects/dev/depristo/oneOffProjects/1000GenomesProcessingPaper/wgs.v13/GA2.WEx.cleaned.indels.10.mask", new File("/humgen/gsa-hpprojects/NA12878Collection/bams/NA12878.WEx.cleaned.recal.bam"), new File("/home/radon01/depristo/work/oneOffProjects/1000GenomesProcessingPaper/wgs.v13/GA2.WEx.cleaned.ug.snpfiltered.indelfiltered.vcf"), - "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.targets.interval_list", 2.6, !lowPass), + "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.targets.interval_list", 2.6, 3.0, !lowPass), "WExTrio" -> new Target("CEUTrio.WEx", hg19, dbSNP_b37_129, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WEx.bwa.cleaned.recal.bam"), new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"), - "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 2.6, !lowPass), + "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 2.6, 3.0, !lowPass), "FIN" -> new Target("FIN", b37, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/1kg/processing/pipeline_test_bams/FIN.79sample.Nov2010.chr20.bam"), new File("/humgen/gsa-hpprojects/dev/data/AugChr20Calls_v4_3state/ALL.august.v4.chr20.filtered.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.hg19.intervals", 2.3, lowPass), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.hg19.intervals", 2.3, 1.0, lowPass), "TGPWExGdA" -> new Target("1000G.WEx.GdA", b37, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/1kg/processing/pipeline_test_bams/Barcoded_1000G_WEx_Reduced_Plate_1.cleaned.list"), // BUGBUG: reduce from 60 to 20 people new File("/humgen/gsa-scr1/delangel/NewUG/calls/AugustRelease.filtered_Q50_QD5.0_SB0.0.allSamples.SNPs_hg19.WEx_UG_newUG_MQC.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 2.6, !lowPass), + "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 2.6, 1.0, !lowPass), "LowPassN60" -> new Target("lowpass.N60", b36, dbSNP_b36, hapmap_b36, indelMask_b36, new File("/humgen/1kg/analysis/bamsForDataProcessingPapers/lowpass_b36/lowpass.chr20.cleaned.matefixed.bam"), // the bam list to call from new File("/home/radon01/depristo/work/oneOffProjects/VQSRCutByNRS/lowpass.N60.chr20.filtered.vcf"), // the gold standard VCF file to run through the VQSR - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.b36.intervals", 2.3, lowPass), // chunked interval list to use with Queue's scatter/gather functionality + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.b36.intervals", 2.3, 1.0, lowPass), // chunked interval list to use with Queue's scatter/gather functionality "LowPassAugust" -> new Target("ALL.august.v4", b37, dbSNP_b37, hapmap_b37, indelMask_b37, // BUGBUG: kill this, it is too large new File("/humgen/1kg/processing/allPopulations_chr20_august_release.cleaned.merged.bams/ALL.cleaned.merged.list"), new File("/humgen/gsa-hpprojects/dev/data/AugChr20Calls_v4_3state/ALL.august.v4.chr20.filtered.vcf"), - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.hg19.intervals", 2.3, lowPass), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.hg19.intervals", 2.3, 1.0, lowPass), "LowPassEUR363Nov" -> new Target("EUR.nov2010", b37, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/1kg/processing/pipeline_test_bams/EUR.363sample.Nov2010.chr20.bam"), new File("/humgen/gsa-hpprojects/dev/data/AugChr20Calls_v4_3state/ALL.august.v4.chr20.filtered.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.hg19.intervals", 2.3, lowPass) + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.hg19.intervals", 2.3, 1.0, lowPass) ) @@ -182,8 +189,8 @@ class MethodsDevelopmentCallingPipeline extends QScript { val FiltersToIgnore = List("DPFilter", "ABFilter", "ESPStandard", "QualByDepth", "StrandBias", "HomopolymerRun") - // 1a.) Call SNPs with UG - class snpCall (t: Target) extends UnifiedGenotyper with UNIVERSAL_GATK_ARGS { + // 1.) Unified Genotyper Base + class GenotyperBase (t: Target) extends UnifiedGenotyper with UNIVERSAL_GATK_ARGS { this.reference_sequence = t.reference this.intervalsString ++= List(t.intervals) this.scatterCount = 63 // the smallest interval list has 63 intervals, one for each Mb on chr20 @@ -191,175 +198,170 @@ class MethodsDevelopmentCallingPipeline extends QScript { this.stand_call_conf = Some( if ( t.isLowpass ) { 4.0 } else { 30.0 } ) this.stand_emit_conf = Some( if ( t.isLowpass ) { 4.0 } else { 30.0 } ) this.input_file :+= t.bamList + if (t.dbsnpFile.endsWith(".rod")) + this.DBSNP = new File(t.dbsnpFile) + else if (t.dbsnpFile.endsWith(".vcf")) + this.rodBind :+= RodBind("dbsnp", "VCF", t.dbsnpFile) + } + + // 1a.) Call SNPs with UG + class snpCall (t: Target) extends GenotyperBase { this.out = t.rawVCF this.baq = Some( if (noBAQ) {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF} else {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY}) this.analysisName = t.name + "_UGs" - this.jobName = t.name + ".snpcall" - if (t.dbsnpFile.endsWith(".rod")) this.DBSNP = new File(t.dbsnpFile) - else if (t.dbsnpFile.endsWith(".vcf")) this.rodBind :+= RodBind("dbsnp", "VCF", t.dbsnpFile) + this.jobName = queueLogDir + t.name + ".snpcall" } // 1b.) Call Indels with UG - class indelCall (t: Target) extends UnifiedGenotyper with UNIVERSAL_GATK_ARGS { - this.reference_sequence = t.reference - this.intervalsString ++= List(t.intervals) - this.scatterCount = 63 // the smallest interval list has 63 intervals, one for each Mb on chr20 - this.dcov = Some( if ( t.isLowpass ) { 50 } else { 250 } ) - this.stand_call_conf = Some( if ( t.isLowpass ) { 4.0 } else { 30.0 } ) - this.stand_emit_conf = Some( if ( t.isLowpass ) { 4.0 } else { 30.0 } ) - this.input_file :+= t.bamList + class indelCall (t: Target) extends GenotyperBase { this.out = t.rawIndelVCF this.glm = Some(org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeLikelihoodsCalculationModel.Model.DINDEL) this.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF) this.analysisName = t.name + "_UGi" - this.jobName = t.name + ".indelcall" - if (t.dbsnpFile.endsWith(".rod")) this.DBSNP = new File(t.dbsnpFile) - else if (t.dbsnpFile.endsWith(".vcf")) this.rodBind :+= RodBind("dbsnp", "VCF", t.dbsnpFile) + this.jobName = queueLogDir + t.name + ".indelcall" } - // 2a.) Filter SNPs - class snpFilter (t: Target) extends VariantFiltration with UNIVERSAL_GATK_ARGS { + // 2.) Hard Filtering Base + class FilterBase (t: Target) extends VariantFiltration with UNIVERSAL_GATK_ARGS { this.reference_sequence = t.reference this.intervalsString ++= List(t.intervals) this.scatterCount = 10 - this.variantVCF = t.rawVCF - this.out = t.filteredVCF this.filterName ++= List("HARD_TO_VALIDATE") this.filterExpression ++= List("\"MQ0 >= 4 && (MQ0 / (1.0 * DP)) > 0.1\"") - this.analysisName = t.name + "_VF" - this.jobName = t.name + ".snpfilter" - if (!noMASK) { - this.rodBind :+= RodBind("mask", "Bed", t.maskFile) - this.maskName = "InDel" - } } - // 2b.) Filter Indels - class indelFilter (t: Target) extends VariantFiltration with UNIVERSAL_GATK_ARGS { - this.reference_sequence = t.reference - this.intervalsString ++= List(t.intervals) - this.scatterCount = 10 - this.variantVCF = t.rawIndelVCF - this.out = t.filteredIndelVCF - this.filterName ++= List("HARD_TO_VALIDATE", "LowQual", "StrandBias", "QualByDepth", "HomopolymerRun") - this.filterExpression ++= List("\"MQ0 >= 4 && (MQ0 / (1.0 * DP)) > 0.1\"", "\"QUAL<30.0\"", "\"SB>=-1.0\"", "\"QD<1.0\"", "\"HRun>=15\"") - this.analysisName = t.name + "_VF" - this.jobName = t.name + ".indelfilter" - if (!noMASK) { + // 2a.) Hard Filter for SNPs (soon to be obsolete) + class snpFilter (t: Target) extends FilterBase { + this.variantVCF = t.rawVCF + this.out = t.filteredVCF + if (useMask) { this.rodBind :+= RodBind("mask", "Bed", t.maskFile) this.maskName = "InDel" } + this.analysisName = t.name + "_VF" + this.jobName = queueLogDir + t.name + ".snpfilter" + } + + // 2b.) Hard Filter for Indels + class indelFilter (t: Target) extends FilterBase { + this.variantVCF = t.rawIndelVCF + this.out = t.filteredIndelVCF + this.filterName ++= List("LowQual", "StrandBias", "QualByDepth", "HomopolymerRun") + if (t.isLowpass) + this.filterExpression ++= List("\"QUAL<30.0\"", "\"SB>=-1.0\"", "\"QD<1.0\"", "\"HRun>=15\"") + else + this.filterExpression ++= List("\"QUAL<50.0\"", "\"SB>=-1.0\"", "\"QD<5.0\"", "\"HRun>=15\"") + this.analysisName = t.name + "_VF" + this.jobName = queueLogDir + t.name + ".indelfilter" } // 3.) VQSR part1 Generate Gaussian clusters based on truth sites class GenerateClusters(t: Target, goldStandard: Boolean) extends GenerateVariantClusters with UNIVERSAL_GATK_ARGS { - val name: String = if ( goldStandard ) { t.goldStandardName } else { t.name } - this.reference_sequence = t.reference - this.rodBind :+= RodBind("hapmap", "VCF", t.hapmapFile) - if( t.hapmapFile.contains("b37") ) - this.rodBind :+= RodBind("1kg", "VCF", "/humgen/gsa-hpprojects/GATK/data/Comparisons/Unvalidated/1kg_pilot1_projectCalls/ALL.low_coverage.2010_07.hg19.vcf") - this.rodBind :+= RodBind("input", "VCF", if ( goldStandard ) { t.goldStandard_VCF } else { t.filteredVCF } ) - this.clusterFile = if ( goldStandard ) { t.goldStandardClusterFile } else { t.clusterFile } - this.use_annotation ++= List("QD", "SB", "HaplotypeScore", "HRun") - this.intervalsString ++= List(t.intervals) - this.qual = Some(350) // clustering parameters to be updated soon pending new experimentation results - this.std = Some(3.5) - this.mG = Some(10) - this.ignoreFilter ++= FiltersToIgnore - this.analysisName = name + "_GVC" - this.jobName = t.name + ".cluster" - if (t.dbsnpFile.endsWith(".rod")) - this.DBSNP = new File(t.dbsnpFile) - else if (t.dbsnpFile.endsWith(".vcf")) - this.rodBind :+= RodBind("dbsnp", "VCF", t.dbsnpFile) + val name: String = if ( goldStandard ) { t.goldStandardName } else { t.name } + this.reference_sequence = t.reference + this.rodBind :+= RodBind("hapmap", "VCF", t.hapmapFile) + if( t.hapmapFile.contains("b37") ) + this.rodBind :+= RodBind("1kg", "VCF", "/humgen/gsa-hpprojects/GATK/data/Comparisons/Unvalidated/1kg_pilot1_projectCalls/ALL.low_coverage.2010_07.hg19.vcf") + this.rodBind :+= RodBind("input", "VCF", if ( goldStandard ) { t.goldStandard_VCF } else { t.filteredVCF } ) + this.clusterFile = if ( goldStandard ) { t.goldStandardClusterFile } else { t.clusterFile } + this.use_annotation ++= List("QD", "SB", "HaplotypeScore", "HRun") + this.intervalsString ++= List(t.intervals) + this.qual = Some(350) // clustering parameters to be updated soon pending new experimentation results + this.std = Some(3.5) + this.mG = Some(10) + this.ignoreFilter ++= FiltersToIgnore + this.analysisName = name + "_GVC" + this.jobName = queueLogDir + t.name + ".cluster" + if (t.dbsnpFile.endsWith(".rod")) + this.DBSNP = new File(t.dbsnpFile) + else if (t.dbsnpFile.endsWith(".vcf")) + this.rodBind :+= RodBind("dbsnp", "VCF", t.dbsnpFile) } // 4.) VQSR part2 Calculate new LOD for all input SNPs by evaluating the Gaussian clusters class VariantRecalibratorBase(t: Target, goldStandard: Boolean) extends VariantRecalibrator with UNIVERSAL_GATK_ARGS { - val name: String = if ( goldStandard ) { t.goldStandardName } else { t.name } - this.reference_sequence = t.reference - if( t.hapmapFile.contains("b37") ) - this.rodBind :+= RodBind("1kg", "VCF", "/humgen/gsa-hpprojects/GATK/data/Comparisons/Unvalidated/1kg_pilot1_projectCalls/ALL.low_coverage.2010_07.hg19.vcf") - this.rodBind :+= RodBind("hapmap", "VCF", t.hapmapFile) - this.rodBind :+= RodBind("truth", "VCF", t.hapmapFile) - this.rodBind :+= RodBind("input", "VCF", if ( goldStandard ) { t.goldStandard_VCF } else { t.filteredVCF } ) - this.clusterFile = if ( goldStandard ) { t.goldStandardClusterFile } else { t.clusterFile } - this.analysisName = name + "_VR" - this.intervalsString ++= List(t.intervals) - this.ignoreFilter ++= FiltersToIgnore - this.ignoreFilter ++= List("HARD_TO_VALIDATE") - this.target_titv = Some(t.titvTarget) - if (t.dbsnpFile.endsWith(".rod")) - this.DBSNP = new File(t.dbsnpFile) - else if (t.dbsnpFile.endsWith(".vcf")) - this.rodBind :+= RodBind("dbsnp", "VCF", t.dbsnpFile) + val name: String = if ( goldStandard ) { t.goldStandardName } else { t.name } + this.reference_sequence = t.reference + if( t.hapmapFile.contains("b37") ) + this.rodBind :+= RodBind("1kg", "VCF", "/humgen/gsa-hpprojects/GATK/data/Comparisons/Unvalidated/1kg_pilot1_projectCalls/ALL.low_coverage.2010_07.hg19.vcf") + this.rodBind :+= RodBind("hapmap", "VCF", t.hapmapFile) + this.rodBind :+= RodBind("truth", "VCF", t.hapmapFile) + this.rodBind :+= RodBind("input", "VCF", if ( goldStandard ) { t.goldStandard_VCF } else { t.filteredVCF } ) + this.clusterFile = if ( goldStandard ) { t.goldStandardClusterFile } else { t.clusterFile } + this.analysisName = name + "_VR" + this.intervalsString ++= List(t.intervals) + this.ignoreFilter ++= FiltersToIgnore + this.ignoreFilter ++= List("HARD_TO_VALIDATE") + this.target_titv = Some(t.titvTarget) + if (t.dbsnpFile.endsWith(".rod")) + this.DBSNP = new File(t.dbsnpFile) + else if (t.dbsnpFile.endsWith(".vcf")) + this.rodBind :+= RodBind("dbsnp", "VCF", t.dbsnpFile) } // 4a.) Choose VQSR tranches based on novel ti/tv class VariantRecalibratorTiTv(t: Target, goldStandard: Boolean) extends VariantRecalibratorBase(t, goldStandard) { - this.tranche ++= List("0.1", "1.0", "10.0", "100.0") - this.out = t.titvRecalibratedVCF - this.tranchesFile = t.titvTranchesFile - this.jobName = t.name + ".titv" + this.tranche ++= List("0.1", "1.0", "3.0", "5.0", "10.0", "100.0") + this.out = t.titvRecalibratedVCF + this.tranchesFile = t.titvTranchesFile + this.jobName = queueLogDir + t.name + ".titv" } // 4b.) Choose VQSR tranches based on sensitivity to truth set class VariantRecalibratorNRS(t: Target, goldStandard: Boolean) extends VariantRecalibratorBase(t, goldStandard) { - this.sm = Some(org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY) - this.tranche ++= List("0.1", "1.0", "10.0", "100.0") - this.out = t.tsRecalibratedVCF - this.priorDBSNP = Some(2.0) - this.priorHapMap = Some(2.0) - this.prior1KG = Some(2.0) - this.tranchesFile = t.tsTranchesFile - this.jobName = t.name + ".nrs" + this.sm = Some(org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY) + this.tranche ++= List("0.1", "1.0", "3.0", "5.0", "10.0", "100.0") + this.out = t.tsRecalibratedVCF + this.priorDBSNP = Some(2.0) + this.priorHapMap = Some(2.0) + this.prior1KG = Some(2.0) + this.tranchesFile = t.tsTranchesFile + this.jobName = queueLogDir + t.name + ".nrs" } // 5.) Variant Cut filter out the variants marked by recalibration to the 99% tranche class VariantCut(t: Target) extends ApplyVariantCuts with UNIVERSAL_GATK_ARGS { - this.reference_sequence = t.reference - this.rodBind :+= RodBind("input", "VCF", t.tsRecalibratedVCF ) - this.intervalsString ++= List(t.intervals) - this.out = t.cutVCF - this.tranchesFile = t.tsTranchesFile - this.fdr_filter_level = Some(1.0) - this.analysisName = t.name + "_VC" - this.jobName = t.name + ".cut" - if (t.dbsnpFile.endsWith(".rod")) - this.DBSNP = new File(t.dbsnpFile) - else if (t.dbsnpFile.endsWith(".vcf")) - this.rodBind :+= RodBind("dbsnp", "VCF", t.dbsnpFile) + this.reference_sequence = t.reference + this.rodBind :+= RodBind("input", "VCF", t.tsRecalibratedVCF ) + this.intervalsString ++= List(t.intervals) + this.out = t.cutVCF + this.tranchesFile = t.tsTranchesFile + this.fdr_filter_level = Some(t.trancheTarget) + this.analysisName = t.name + "_VC" + this.jobName = queueLogDir + t.name + ".cut" + if (t.dbsnpFile.endsWith(".rod")) + this.DBSNP = new File(t.dbsnpFile) + else if (t.dbsnpFile.endsWith(".vcf")) + this.rodBind :+= RodBind("dbsnp", "VCF", t.dbsnpFile) } - // 6a.) Variant Evaluation (OPTIONAL) based on the sensitivity recalibrated vcf - class snpEvaluation(t: Target) extends VariantEval with UNIVERSAL_GATK_ARGS { - val name: String = t.name - this.reference_sequence = t.reference - this.rodBind :+= RodBind("comphapmap", "VCF", t.hapmapFile) - this.rodBind :+= RodBind("eval", "VCF", if (!noCut) {t.cutVCF} else {t.tsRecalibratedVCF} ) - this.analysisName = name + "_VE" - this.jobName = t.name + ".snpeval" - this.intervalsString ++= List(t.intervals) - this.out = t.evalFile - if (t.dbsnpFile.endsWith(".rod")) - this.DBSNP = new File(t.dbsnpFile) - else if (t.dbsnpFile.endsWith(".vcf")) - this.rodBind :+= RodBind("dbsnp", "VCF", t.dbsnpFile) + // 6.) Variant Evaluation Base(OPTIONAL) + class EvalBase(t: Target) extends VariantEval with UNIVERSAL_GATK_ARGS { + this.reference_sequence = t.reference + this.rodBind :+= RodBind("comphapmap", "VCF", t.hapmapFile) + this.intervalsString ++= List(t.intervals) + if (t.dbsnpFile.endsWith(".rod")) + this.DBSNP = new File(t.dbsnpFile) + else if (t.dbsnpFile.endsWith(".vcf")) + this.rodBind :+= RodBind("dbsnp", "VCF", t.dbsnpFile) } - // 6b.) Variant Evaluation of Indels (OPTIONAL) - class indelEvaluation(t: Target) extends VariantEval with UNIVERSAL_GATK_ARGS { - val name: String = t.name - this.reference_sequence = t.reference - this.rodBind :+= RodBind("comphapmap", "VCF", t.hapmapFile) - this.rodBind :+= RodBind("eval", "VCF", if (!noCut) {t.cutVCF} else {t.tsRecalibratedVCF} ) - this.analysisName = name + "_VE" - this.jobName = t.name + ".indeleval" - this.intervalsString ++= List(t.intervals) - this.out = t.evalFile - if (t.dbsnpFile.endsWith(".rod")) - this.DBSNP = new File(t.dbsnpFile) - else if (t.dbsnpFile.endsWith(".vcf")) - this.rodBind :+= RodBind("dbsnp", "VCF", t.dbsnpFile) + // 6a.) SNP Evaluation (OPTIONAL) based on the cut vcf + class snpEvaluation(t: Target) extends EvalBase { + if (t.reference == b37 || t.reference == hg19) this.rodBind :+= RodBind("compomni", "VCF", omni_b37) + this.rodBind :+= RodBind("eval", "VCF", if (useCut) {t.cutVCF} else {t.tsRecalibratedVCF} ) + this.evalModule :+= "GenotypeConcordance" + this.out = t.evalFile + this.analysisName = t.name + "_VEs" + this.jobName = queueLogDir + t.name + ".snp.eval" + } + + // 6b.) Indel Evaluation (OPTIONAL) + class indelEvaluation(t: Target) extends EvalBase { + this.rodBind :+= RodBind("eval", "VCF", t.filteredIndelVCF) + this.evalModule :+= "IndelStatistics" + this.out = t.evalIndelFile + this.analysisName = t.name + "_VEi" + this.jobName = queueLogDir + queueLogDir + t.name + ".indel.eval" } } diff --git a/scala/qscript/oneoffs/carneiro/dataProcessing.scala b/scala/qscript/oneoffs/carneiro/dataProcessing.scala index 37a7cc7fe..a3ea3168b 100755 --- a/scala/qscript/oneoffs/carneiro/dataProcessing.scala +++ b/scala/qscript/oneoffs/carneiro/dataProcessing.scala @@ -26,7 +26,7 @@ class dataProcessing extends QScript { @Input(doc="Reference fasta file", shortName="R", required=false) var reference: File = new File("/seq/references/Homo_sapiens_assembly19/v1/Homo_sapiens_assembly19.fasta") - @Input(doc="dbsnp ROD to use (VCF)", shortName="D", required=false) // todo -- accept any format. Not only VCF. + @Input(doc="dbsnp ROD to use (VCF)", shortName="D", required=false) val dbSNP: File = new File("/humgen/gsa-hpprojects/GATK/data/dbsnp_132_b37.leftAligned.vcf") @Input(doc="extra VCF files to use as reference indels for Indel Realignment", shortName="indels", required=false) //todo -- once vcfs are merged, this will become the only indel vcf to be used and the merged file will be the default. @@ -51,6 +51,9 @@ class dataProcessing extends QScript { val dindelASNCalls: String = "/humgen/1kg/DCC/ftp/technical/working/20110126_dindel_august/ASN.dindel_august_release_merged_pilot1.20110126.sites.vcf.gz" val dindelEURCalls: String = "/humgen/1kg/DCC/ftp/technical/working/20110126_dindel_august/EUR.dindel_august_release_merged_pilot1.20110126.sites.vcf.gz" + val queueLogDir: String = ".qlog/" + + // Simple boolean definitions for code clarity val knownsOnly: Boolean = true val intermediate: Boolean = true @@ -59,7 +62,7 @@ class dataProcessing extends QScript { trait CommandLineGATKArgs extends CommandLineGATK { this.jarFile = qscript.GATKjar this.reference_sequence = qscript.reference - this.memoryLimit = Some(8) + this.memoryLimit = Some(4) this.isIntermediate = true } @@ -135,7 +138,7 @@ class dataProcessing extends QScript { this.rodBind :+= RodBind("indels2", "VCF", dindelAFRCalls) this.rodBind :+= RodBind("indels3", "VCF", dindelEURCalls) this.rodBind :+= RodBind("indels4", "VCF", dindelASNCalls) - this.jobName = outIntervals + ".ktarget" + this.jobName = queueLogDir + outIntervals + ".ktarget" } class allTargets (inBams: String, outIntervals: String) extends knownTargets(outIntervals) { @@ -148,7 +151,7 @@ class dataProcessing extends QScript { this.rodBind :+= RodBind("indels3", "VCF", dindelEURCalls) this.rodBind :+= RodBind("indels4", "VCF", dindelASNCalls) if (qscript.indels != null) this.rodBind :+= RodBind("indels5", "VCF", qscript.indels) - this.jobName = outIntervals + ".atarget" + this.jobName = queueLogDir + outIntervals + ".atarget" } class clean (inBams: String, tIntervals: String, outBam: String, knownsOnly: Boolean, intermediate: Boolean) extends IndelRealigner with CommandLineGATKArgs { @@ -168,7 +171,7 @@ class dataProcessing extends QScript { this.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY) this.compress = Some(0) this.isIntermediate = intermediate - this.jobName = outBam + ".clean" + this.jobName = queueLogDir + outBam + ".clean" if (!intermediate && !qscript.intervalString.isEmpty()) this.intervalsString ++= List(qscript.intervalString) if (!intermediate && qscript.intervals != null) this.intervals :+= qscript.intervals } @@ -185,7 +188,7 @@ class dataProcessing extends QScript { this.memoryLimit = Some(6) this.jarFile = qscript.dedupJar this.isIntermediate = true - this.jobName = outBam + ".dedup" + this.jobName = queueLogDir + outBam + ".dedup" } class cov (inBam: String, outRecalFile: String) extends CountCovariates with CommandLineGATKArgs { @@ -193,7 +196,7 @@ class dataProcessing extends QScript { this.covariate ++= List("ReadGroupCovariate", "QualityScoreCovariate", "CycleCovariate", "DinucCovariate") this.input_file :+= new File(inBam) this.recal_file = new File(outRecalFile) - this.jobName = outRecalFile + ".covariates" + this.jobName = queueLogDir + outRecalFile + ".covariates" } class recal (inBam: String, inRecalFile: String, outBam: String) extends TableRecalibration with CommandLineGATKArgs { @@ -202,7 +205,7 @@ class dataProcessing extends QScript { this.recal_file = new File(inRecalFile) this.out = new File(outBam) this.index_output_bam_on_the_fly = Some(true) - this.jobName = outBam + ".recalibration" + this.jobName = queueLogDir + outBam + ".recalibration" } class analyzeCovariates (inRecalFile: String, outPath: String) extends AnalyzeCovariates { @@ -210,12 +213,12 @@ class dataProcessing extends QScript { this.resources = qscript.R this.recal_file = new File(inRecalFile) this.output_dir = outPath - this.jobName = inRecalFile + ".analyze_covariates" + this.jobName = queueLogDir + inRecalFile + ".analyze_covariates" } class writeList(inBams: List[File], outBamList: String) extends ListWriterFunction { this.inputFiles = inBams this.listFile = new File(outBamList) - this.jobName = outBamList + ".bamList" + this.jobName = queueLogDir + outBamList + ".bamList" } }