From f3e94ef2be693d2df94617aa79a2f01e0a1d83a8 Mon Sep 17 00:00:00 2001 From: kshakir Date: Thu, 24 Mar 2011 14:03:51 +0000 Subject: [PATCH] Walkers can now specify a class extending from Gatherer to merge custom output formats. Add @Gather(MyGatherer.class) to the walker @Output. JavaCommandLineFunctions can now specify the classpath+mainclass as an alternative to specifying a path to an executable jar. JCLF by default pass on the current classpath and only require the mainclass be specified by the developer extending the JCLF, relieving the QScript author from having to explicitly specify the jar. Like the Picard MergeSamFiles, GATK engine by default is now run from the current classpath. The GATK can still be overridden via .jarFile or .javaClasspath. Walkers from the GATK package are now also embedded into the Queue package. Updated AnalyzeCovariates to make it easier to guess the main class, AnalyzeCovariates instead of AnalyzeCovariatesCLP. Removed the GATK jar argument from the example QScripts. Removed one of the most FAQ when getting started with Scala/Queue, the use of Option[_] in QScripts: 1) Fixed mistaken assumption with java enums. In java enums can be null so they don't need nullable wrappers. 2) Added syntactic sugar for Nullable primitives to the QScript trait. Any variable defined as Option[Int] can just be assigned an Int value or None, ex: myFunc.memoryLimit = 3 Removed other unused code. Re-fixed dry run function ordering. Re-ordered the QCommandline companion object so that IntelliJ doesn't complain about missing main methods. git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@5504 348d0f76-0448-11de-a6fe-93d51630548a --- .../distributedGATKPerformance.scala | 17 +- .../analyzecovariates/AnalyzeCovariates.java | 16 +- .../scattergather => commandline}/Gather.java | 5 +- .../sting/commandline/Gatherer.java | 25 ++- .../AnalyzeConcordance.java | 207 ------------------ .../AnalyzeConcordanceField.java | 41 ---- .../gatk/ArgumentDefinitionField.java | 22 +- .../queue/extensions/gatk/ArgumentField.java | 2 +- .../gatk/GATKExtensionsGenerator.java | 7 +- packages/AnalyzeConcordance.xml | 18 -- packages/GATKEngine.xml | 49 +++++ packages/GenomeAnalysisTK.xml | 21 +- packages/Queue.xml | 28 +++ .../MethodsDevelopmentCallingPipeline.scala | 28 +-- .../core/StandardVariantEvaluation.scala | 2 +- scala/qscript/examples/ExampleCountLoci.scala | 6 +- .../qscript/examples/ExampleCountReads.scala | 7 - .../examples/ExampleCustomWalker.scala | 23 +- .../examples/ExampleUnifiedGenotyper.scala | 8 +- .../oneoffs/carneiro/dataProcessing.scala | 22 +- .../oneoffs/carneiro/dataProcessingV2.scala | 22 +- .../qscript/oneoffs/carneiro/justClean.scala | 10 +- .../qscript/oneoffs/carneiro/pbCalling.scala | 20 +- scala/qscript/oneoffs/chartl/BootstrapCalls.q | 36 +-- .../oneoffs/chartl/RefineGenotypesAndMerge.q | 7 +- .../qscript/oneoffs/chartl/expanded_targets.q | 14 +- .../chartl/old/private_mutations_old.q | 10 +- scala/qscript/oneoffs/chartl/omni_qc.q | 29 ++- .../oneoffs/chartl/private_mutations.q | 6 +- .../oneoffs/delangel/Phase1IndelCalling.scala | 29 ++- .../qscript/oneoffs/depristo/1kg_table1.scala | 12 +- .../oneoffs/depristo/CleaningTest.scala | 22 +- .../depristo/RefineGenotypesWithBeagle.q | 14 +- .../oneoffs/depristo/VQSRCutByNRS.scala | 15 +- .../depristo/manySampleUGPerformance.scala | 16 +- .../depristo/resequencingSamples1KG.scala | 8 +- .../qscript/oneoffs/fromer/PhaseSamples.scala | 4 +- .../oneoffs/fromer/ReadDepthCNVanalysis.scala | 8 +- scala/qscript/oneoffs/hanna/DoC.scala | 4 +- .../oneoffs/kshakir/UGMemoryTests.scala | 4 +- .../LinearIndexBinTests.scala | 4 +- .../qscript/oneoffs/rpoplin/ASHGcalling.scala | 31 ++- .../oneoffs/rpoplin/Phase1Calling.scala | 27 ++- .../oneoffs/rpoplin/Phase1Cleaning.scala | 11 +- .../rpoplin/VQSR_parameterSearch.scala | 19 +- scala/qscript/playground/BatchMerge.q | 28 +-- .../qscript/playground/FullCallingPipeline.q | 32 +-- .../playground/MultiFullCallingPipeline.scala | 6 +- scala/qscript/playground/recalibrate.scala | 16 +- .../sting/queue/QCommandLine.scala | 86 +++++--- .../broadinstitute/sting/queue/QScript.scala | 7 +- .../sting/queue/engine/QGraph.scala | 19 +- .../firehose/ImportSingleValueFunction.scala | 45 ---- .../extensions/gatk/BamGatherFunction.scala | 30 ++- .../extensions/gatk/GATKScatterFunction.scala | 2 +- .../extensions/picard/PicardBamFunction.scala | 22 +- .../EmbeddedCommandLineFunction.scala | 45 ---- .../function/JarCommandLineFunction.scala | 14 -- .../function/JavaCommandLineFunction.scala | 59 ++++- .../sting/queue/function/QFunction.scala | 26 ++- .../scattergather/GathererFunction.scala} | 20 +- .../ScatterGatherableFunction.scala | 27 ++- .../sting/queue/pipeline/BamProcessing.scala | 4 +- .../sting/queue/pipeline/VariantCalling.scala | 8 +- .../util/PrimitiveOptionConversions.scala | 139 ++++++++++++ .../ExampleCountLociPipelineTest.scala | 28 ++- 66 files changed, 800 insertions(+), 799 deletions(-) rename java/src/org/broadinstitute/sting/{queue/function/scattergather => commandline}/Gather.java (86%) rename scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamEmbeddedFunction.scala => java/src/org/broadinstitute/sting/commandline/Gatherer.java (62%) delete mode 100644 java/src/org/broadinstitute/sting/playground/analyzeconcordance/AnalyzeConcordance.java delete mode 100644 java/src/org/broadinstitute/sting/playground/analyzeconcordance/AnalyzeConcordanceField.java delete mode 100644 packages/AnalyzeConcordance.xml create mode 100644 packages/GATKEngine.xml delete mode 100644 scala/src/org/broadinstitute/sting/queue/extensions/firehose/ImportSingleValueFunction.scala delete mode 100644 scala/src/org/broadinstitute/sting/queue/function/EmbeddedCommandLineFunction.scala delete mode 100644 scala/src/org/broadinstitute/sting/queue/function/JarCommandLineFunction.scala rename scala/src/org/broadinstitute/sting/queue/{extensions/picard/PicardBamJarFunction.scala => function/scattergather/GathererFunction.scala} (68%) create mode 100644 scala/src/org/broadinstitute/sting/queue/util/PrimitiveOptionConversions.scala diff --git a/analysis/depristo/distributedGATK/distributedGATKPerformance.scala b/analysis/depristo/distributedGATK/distributedGATKPerformance.scala index 17a95b4bb..8ca54fbe7 100755 --- a/analysis/depristo/distributedGATK/distributedGATKPerformance.scala +++ b/analysis/depristo/distributedGATK/distributedGATKPerformance.scala @@ -1,4 +1,3 @@ -import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction import org.broadinstitute.sting.queue.extensions.gatk._ import org.broadinstitute.sting.queue.extensions.samtools.SamtoolsIndexFunction import org.broadinstitute.sting.queue.QScript @@ -40,7 +39,7 @@ class DistributedGATKPerformance extends QScript { @Argument(shortName="trackerDir", doc="root directory for distributed tracker files", required=false) var trackerDir: String = "" // "/humgen/gsa-scr1/depristo/tmp/" - trait UNIVERSAL_GATK_ARGS extends CommandLineGATK { logging_level = "DEBUG"; jarFile = gatkJarFile; memoryLimit = Some(2); } + trait UNIVERSAL_GATK_ARGS extends CommandLineGATK { logging_level = "DEBUG"; jarFile = gatkJarFile; memoryLimit = 2; } class Target( val baseName: String, @@ -154,9 +153,9 @@ class DistributedGATKPerformance extends QScript { def addUG(ug: UnifiedGenotyper) = { if ( ! long ) - ug.jobLimitSeconds = Some(60 * 60 * 4) + ug.jobLimitSeconds = 60 * 60 * 4 if ( limitTo30Min ) - ug.jobLimitSeconds = Some(60 * 30) + ug.jobLimitSeconds = 60 * 30 add(ug); } @@ -171,7 +170,7 @@ class DistributedGATKPerformance extends QScript { var ug: UnifiedGenotyper = new UnifiedGenotyper(target, aname + ".part" + part) ug.intervalsString ++= getTargetInterval(target) ug.processingTracker = new File(trackerDir + target.name + "." + aname + ".distributed.txt") - ug.processingTrackerID = Some(part) + ug.processingTrackerID = part if ( part == 1 ) ug.performanceLog = new File("%s.%s.pf.log".format(target.name, aname)) ug.processingTrackerStatusFile = new File("%s.%s.%d.ptstatus.log".format(target.name, aname, part)) @@ -186,12 +185,12 @@ class DistributedGATKPerformance extends QScript { // 1.) Call SNPs with UG class UnifiedGenotyper(t: Target, aname: String) extends org.broadinstitute.sting.queue.extensions.gatk.UnifiedGenotyper with UNIVERSAL_GATK_ARGS { this.reference_sequence = t.reference - this.dcov = Some( if ( t.isLowpass ) { 50 } else { 250 } ) - this.stand_call_conf = Some( if ( t.isLowpass ) { 4.0 } else { 30.0 } ) - this.stand_emit_conf = Some( if ( t.isLowpass ) { 4.0 } else { 30.0 } ) + this.dcov = if ( t.isLowpass ) { 50 } else { 250 } + this.stand_call_conf = if ( t.isLowpass ) { 4.0 } else { 30.0 } + this.stand_emit_conf = if ( t.isLowpass ) { 4.0 } else { 30.0 } this.input_file :+= t.bamList this.out = t.rawVCF(aname) - this.baq = Some( if (t.useBAQ) {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.RECALCULATE} else {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF}) + this.baq = if (t.useBAQ) {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.RECALCULATE} else {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF} this.analysisName = t.name + "_UG." + aname if (t.dbsnpFile.endsWith(".rod")) this.DBSNP = new File(t.dbsnpFile) diff --git a/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java b/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java index 47ea006a0..f0d903fc4 100755 --- a/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java +++ b/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java @@ -48,7 +48,7 @@ import java.io.*; * Create collapsed versions of the recal csv file and call R scripts to plot residual error versus the various covariates. */ -class AnalyzeCovariatesCLP extends CommandLineProgram { +public class AnalyzeCovariates extends CommandLineProgram { ///////////////////////////// // Command Line Arguments @@ -311,12 +311,14 @@ class AnalyzeCovariatesCLP extends CommandLineProgram { } } } -} -public class AnalyzeCovariates { - public static void main(String args[]) throws Exception { - AnalyzeCovariatesCLP clp = new AnalyzeCovariatesCLP(); - CommandLineProgram.start( clp, args ); - System.exit(0); + public static void main(String args[]) { + try { + AnalyzeCovariates clp = new AnalyzeCovariates(); + start(clp, args); + System.exit(CommandLineProgram.result); + } catch (Exception e) { + exitSystemWithError(e); + } } } diff --git a/java/src/org/broadinstitute/sting/queue/function/scattergather/Gather.java b/java/src/org/broadinstitute/sting/commandline/Gather.java similarity index 86% rename from java/src/org/broadinstitute/sting/queue/function/scattergather/Gather.java rename to java/src/org/broadinstitute/sting/commandline/Gather.java index 45e0afaba..59c3f50cb 100644 --- a/java/src/org/broadinstitute/sting/queue/function/scattergather/Gather.java +++ b/java/src/org/broadinstitute/sting/commandline/Gather.java @@ -22,13 +22,12 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -package org.broadinstitute.sting.queue.function.scattergather; +package org.broadinstitute.sting.commandline; import java.lang.annotation.*; /** - * Specifies the class type of the CommandLineFunction to gather an @Output - * Written in java because scala doesn't support RetentionPolicy.RUNTIME + * Specifies the class type to gather an @Output */ @Documented @Inherited diff --git a/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamEmbeddedFunction.scala b/java/src/org/broadinstitute/sting/commandline/Gatherer.java similarity index 62% rename from scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamEmbeddedFunction.scala rename to java/src/org/broadinstitute/sting/commandline/Gatherer.java index deb251d89..fd86f5f6b 100644 --- a/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamEmbeddedFunction.scala +++ b/java/src/org/broadinstitute/sting/commandline/Gatherer.java @@ -22,16 +22,25 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -package org.broadinstitute.sting.queue.extensions.picard +package org.broadinstitute.sting.commandline; -import org.broadinstitute.sting.queue.function.EmbeddedCommandLineFunction +import java.io.File; +import java.util.List; /** - * Wraps a Picard embedded class that operates on BAM files. - * See http://picard.sourceforge.net/ for more info. - * - * Since the command lines take slightly different arguments - * some values are optional. + * Combines a list of files into a single output. */ -trait PicardBamEmbeddedFunction extends EmbeddedCommandLineFunction with PicardBamFunction { +public abstract class Gatherer { + /** + * Gathers a list of files into a single output. + * @param inputs Files to combine. + * @param output Path to output file. + */ + public abstract void gather(List inputs, File output); + + /** + * Returns true if the caller should wait for the input files to propagate over NFS before running gather(). + * @return true if the caller should wait for the input files to propagate over NFS before running gather(). + */ + public boolean waitForInputs() { return true; } } diff --git a/java/src/org/broadinstitute/sting/playground/analyzeconcordance/AnalyzeConcordance.java b/java/src/org/broadinstitute/sting/playground/analyzeconcordance/AnalyzeConcordance.java deleted file mode 100644 index ddac5f58f..000000000 --- a/java/src/org/broadinstitute/sting/playground/analyzeconcordance/AnalyzeConcordance.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Copyright (c) 2010 The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR - * THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.playground.analyzeconcordance; - -import org.broadinstitute.sting.commandline.CommandLineProgram; -import org.broadinstitute.sting.commandline.Argument; -import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import org.broadinstitute.sting.utils.text.XReadLines; -import org.broadinstitute.sting.utils.Utils; -import org.broadinstitute.sting.utils.PathUtils; -import org.broadinstitute.sting.playground.utils.ProcessUtils; -import org.apache.log4j.Logger; - -import java.io.*; -import java.util.*; - -/** - * Compares results of VariantEval across a population or a case/control group. - */ -public class AnalyzeConcordance extends CommandLineProgram { - - @Argument(fullName = "group_name", shortName = "groupName", doc = "The name of the group which will be prefixed output files", required = false) - private String baseName = "analyze_concordance"; - @Argument(fullName = "eval_list", shortName = "evalList", doc = "The input list of unfiltered eval files to analyze", required = true) - private String evalListFile = null; - @Argument(fullName = "filtered_eval_list", shortName = "filteredEvalList", doc = "The input list of filtered eval files to analyze", required = false) - private String filteredEvalListFile = null; - @Argument(fullName = "output_dir", shortName = "outputDir", doc = "The directory in which to output all the plots and intermediate data files", required = false) - private String outputDir = "analyzeConcordance"; - @Argument(fullName = "path_to_Rscript", shortName = "Rscript", doc = "The path to your implementation of Rscript", required = false) - private String pathToRscript = "env Rscript"; - @Argument(fullName = "path_to_resources", shortName = "resources", doc = "Path to resources folder holding the Sting analyze concordance R scripts", required = false) - private String pathToResources = "R" + File.separator + "analyzeConcordance"; - - private enum EvalFilterType { - UNFILTERED, FILTERED - } - - private static final AnalyzeConcordanceField[] ANALYZE_CONCORDANCE_FIELDS = AnalyzeConcordanceField.values(); - - private String evalDataFile; - private List data = new ArrayList(); - - private static Logger logger = Logger.getLogger(AnalyzeConcordance.class); - - protected int execute() { - int result; - - try { - createOutputDirectory(); - - // initialize all the data from the csv file and allocate the list of covariates - logger.info("Reading in input csv file..."); - initializeData(); - logger.info("...Done!"); - - // output data tables for Rscript to read in - logger.info("Writing out intermediate tables for R..."); - writeDataTables(); - logger.info("...Done!"); - - // perform the analysis using Rscript and output the plots - logger.info("Calling analysis R scripts and writing out figures..."); - result = callRScripts(); - logger.info("...Done!"); - - // perform the analysis using Rscript and output the plots - logger.info("Generating html report..."); - generateHtmlReport(); - logger.info("...Done!"); - } catch (Exception e) { - throw new ReviewedStingException("Error analyzing concordance", e); - } - - return result; - } - - private void createOutputDirectory() { - // create the output directory where all the data tables and plots will go - File outputDir = new File(this.outputDir); - if (!outputDir.exists() && !outputDir.mkdirs()) { - throw new ReviewedStingException("Couldn't create directory: " + this.outputDir); - } - } - - private void initializeData() throws FileNotFoundException { - // add the column headers to the data - addHeader(); - - // read the list of unfiltered eval files - addEvalListFile(EvalFilterType.UNFILTERED, new File(evalListFile)); - - // if provided, read the list of filtered eval files - if (filteredEvalListFile != null) { - addEvalListFile(EvalFilterType.FILTERED, new File(filteredEvalListFile)); - } - } - - private void addHeader() { - String[] headers = new String[ANALYZE_CONCORDANCE_FIELDS.length + 2]; - int column = 0; - headers[column++] = "eval_id"; - headers[column++] = "filter_type"; - - for (AnalyzeConcordanceField field : ANALYZE_CONCORDANCE_FIELDS) { - headers[column++] = field.getColumnHeader(); - } - - data.add(headers); - } - - private void addEvalListFile(EvalFilterType filterType, File evalListFile) throws FileNotFoundException { - for (String line : new XReadLines(evalListFile)) { - String[] parts = line.split("\t"); - addEvalFile(parts[0], filterType, new File(parts[1])); - } - } - - private void addEvalFile(String evalID, EvalFilterType filterType, File evalFile) throws FileNotFoundException { - SortedMap fieldValues = new TreeMap(); - - for (String line : new XReadLines(evalFile)) { - for (AnalyzeConcordanceField field : ANALYZE_CONCORDANCE_FIELDS) { - String value = field.parseLine(line); - if (value != null) { - fieldValues.put(field, value); - break; // continue to the next line. - } - } - } - - String[] values = new String[ANALYZE_CONCORDANCE_FIELDS.length + 2]; - int column = 0; - values[column++] = evalID; - values[column++] = filterType.toString().toLowerCase(); - - // get all the values, including null if for some reason a value wasn't found - for (AnalyzeConcordanceField field : ANALYZE_CONCORDANCE_FIELDS) { - values[column++] = fieldValues.get(field); - } - - data.add(values); - } - - private void writeDataTables() throws FileNotFoundException { - evalDataFile = baseName + ".eval_data.tsv"; - // Create a PrintStream - PrintStream output = new PrintStream(new File(outputDir, evalDataFile)); - for (String[] line : data) { - output.println(Utils.join("\t", line)); - } - output.close(); - } - - private int callRScripts() { - String command = pathToRscript + " " - + new File(pathToResources, "analyzeConcordance.R") + " " - + new File(outputDir, baseName) + " " - + new File(outputDir, evalDataFile); - - return ProcessUtils.runCommandAndWait(command); - } - - private void generateHtmlReport() throws FileNotFoundException { - // TODO: Enhance the reports - PrintStream output = new PrintStream(new File(outputDir, "report.html")); - output.println(""); - for (File pngFile : new File(outputDir).listFiles(new PathUtils.ExtensionFilter("png"))) { - output.println("
"); - } - output.println(""); - output.close(); - } - - public static void main(String[] argv) { - try { - AnalyzeConcordance instance = new AnalyzeConcordance(); - start(instance, argv); - System.exit(CommandLineProgram.result); - } catch (Exception e) { - exitSystemWithError(e); - } - } -} diff --git a/java/src/org/broadinstitute/sting/playground/analyzeconcordance/AnalyzeConcordanceField.java b/java/src/org/broadinstitute/sting/playground/analyzeconcordance/AnalyzeConcordanceField.java deleted file mode 100644 index b7ff2e50f..000000000 --- a/java/src/org/broadinstitute/sting/playground/analyzeconcordance/AnalyzeConcordanceField.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.broadinstitute.sting.playground.analyzeconcordance; - -import java.util.regex.Pattern; -import java.util.regex.Matcher; - -/** - * Created by IntelliJ IDEA. - * User: kshakir - * Date: Feb 11, 2010 - */ -public enum AnalyzeConcordanceField { - N_BASES_COVERED("all_bases", "all,summary,variant_counts", "n bases covered"), - ALL_DBSNP_RATE("all_dbsnp", "all,summary,db_coverage", "dbsnp_rate"), - ALL_VARIANT_COUNT("all_variants", "all,summary,variant_counts", "variants"), - ALL_TITV_RATIO("all_titv", "all,summary,transitions_transversions", "ratio"), - KNOWN_VARIANT_COUNT("known_variants", "known,summary,variant_counts", "variants"), - KNOWN_TITV_RATIO("known_titv", "known,summary,transitions_transversions", "ratio"), - NOVEL_VARIANT_COUNT("novel_variants", "novel,summary,variant_counts", "variants"), - NOVEL_TITV_RATIO("novel_titv", "novel,summary,transitions_transversions", "ratio"); - - private String columnHeader; - private Pattern pattern; - - private AnalyzeConcordanceField(String columnHeader, String evalHeader, String analysis) { - this.columnHeader = columnHeader; - - String lineRegex = evalHeader + " {2,}" + analysis + " {2,}([0-9.]+).*"; - this.pattern = Pattern.compile(lineRegex); - } - - public String getColumnHeader() { - return this.columnHeader; - } - - public String parseLine(String line) { - Matcher matcher = this.pattern.matcher(line); - if (!matcher.matches()) - return null; - return matcher.group(1); - } -} diff --git a/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java b/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java index 4aaba7cc1..c8c62fbd1 100644 --- a/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java +++ b/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java @@ -111,15 +111,19 @@ public abstract class ArgumentDefinitionField extends ArgumentField { public static List getArgumentFields(ParsingEngine parsingEngine,Class classType) { List argumentFields = new ArrayList(); for (ArgumentSource argumentSource: parsingEngine.extractArgumentSources(classType)) - if (!argumentSource.isDeprecated()) + if (!argumentSource.isDeprecated()) { + Class gatherer = null; + if (argumentSource.field.isAnnotationPresent(Gather.class)) + gatherer = argumentSource.field.getAnnotation(Gather.class).value(); for (ArgumentDefinition argumentDefinition: argumentSource.createArgumentDefinitions()) - argumentFields.addAll(getArgumentFields(argumentDefinition)); + argumentFields.addAll(getArgumentFields(argumentDefinition, gatherer)); + } return argumentFields; } private static final List intervalFields = Arrays.asList("intervals", "excludeIntervals", "targetIntervals"); - private static List getArgumentFields(ArgumentDefinition argumentDefinition) { + private static List getArgumentFields(ArgumentDefinition argumentDefinition, Class gatherer) { if (intervalFields.contains(argumentDefinition.fullName) && argumentDefinition.ioType == ArgumentIOType.INPUT) { return Arrays.asList( new IntervalFileArgumentField(argumentDefinition), @@ -138,7 +142,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField { return Collections.singletonList(new InputArgumentField(argumentDefinition)); } else if (argumentDefinition.ioType == ArgumentIOType.OUTPUT) { - return Collections.singletonList(new OutputArgumentField(argumentDefinition)); + return Collections.singletonList(new OutputArgumentField(argumentDefinition, gatherer)); } else if (argumentDefinition.isFlag) { return Collections.singletonList(new FlagArgumentField(argumentDefinition)); @@ -224,8 +228,10 @@ public abstract class ArgumentDefinitionField extends ArgumentField { // if (argumentDefinition.ioType == ArgumentIOType.OUTPUT) // Map all outputs to files. private static class OutputArgumentField extends ArgumentDefinitionField { - public OutputArgumentField(ArgumentDefinition argumentDefinition) { + private final Class gatherer; + public OutputArgumentField(ArgumentDefinition argumentDefinition, Class gatherer) { super(argumentDefinition); + this.gatherer = gatherer; } @Override protected Class getInnerType() { return File.class; } @@ -235,7 +241,9 @@ public abstract class ArgumentDefinitionField extends ArgumentField { @Override public boolean isGather() { return true; } @Override protected String getGatherAnnotation() { String gather; - if (SAMFileWriter.class.isAssignableFrom(argumentDefinition.argumentType)) + if (gatherer != null) + gather = "@Gather(classOf[" + gatherer.getName() + "])%n"; + else if (SAMFileWriter.class.isAssignableFrom(argumentDefinition.argumentType)) gather = "@Gather(classOf[BamGatherFunction])%n"; else if (VCFWriter.class.isAssignableFrom(argumentDefinition.argumentType)) gather = "@Gather(classOf[VcfGatherFunction])%n"; @@ -272,7 +280,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField { } // if (!argumentDefinition.required && useOption(argumentDefinition.argumentType)) - // Any optional arguments that are primitives / enums are wrapped in options. + // Any optional arguments that are primitives are wrapped in options. private static class OptionedArgumentField extends ArgumentDefinitionField { private final boolean useFormatter; diff --git a/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java b/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java index 056c366a0..f035892ce 100644 --- a/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java +++ b/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java @@ -217,7 +217,7 @@ public abstract class ArgumentField { * @return true if option should be used. */ protected static boolean useOption(Class argType) { - return (argType.isPrimitive()) || (Number.class.isAssignableFrom(argType)) || (argType.isEnum()); + return (argType.isPrimitive()) || (Number.class.isAssignableFrom(argType)); } /** diff --git a/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java b/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java index dbb52eb66..45160090e 100644 --- a/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java +++ b/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java @@ -140,9 +140,10 @@ public class GATKExtensionsGenerator extends CommandLineProgram { continue; String clpClassName = clpManager.getName(clp); + String clpConstructor = String.format("analysisName = \"%s\"%njavaMainClass = \"%s\"%n", clpClassName, clp.getName()); - writeClass("org.broadinstitute.sting.queue.function.JarCommandLineFunction", clpClassName, - false, "", ArgumentDefinitionField.getArgumentFields(parser,clp), dependents); + writeClass("org.broadinstitute.sting.queue.function.JavaCommandLineFunction", clpClassName, + false, clpConstructor, ArgumentDefinitionField.getArgumentFields(parser,clp), dependents); if (clp == CommandLineGATK.class) { for (Entry>> walkersByPackage: walkerManager.getWalkerNamesByPackage(false).entrySet()) { @@ -359,7 +360,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram { baseClass += " with ScatterGatherableFunction"; } if (isGather) - importSet.add("import org.broadinstitute.sting.queue.function.scattergather.Gather"); + importSet.add("import org.broadinstitute.sting.commandline.Gather"); // Sort the imports so that the are always in the same order. List sortedImports = new ArrayList(importSet); diff --git a/packages/AnalyzeConcordance.xml b/packages/AnalyzeConcordance.xml deleted file mode 100644 index d0878089b..000000000 --- a/packages/AnalyzeConcordance.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/packages/GATKEngine.xml b/packages/GATKEngine.xml new file mode 100644 index 000000000..1f35522d9 --- /dev/null +++ b/packages/GATKEngine.xml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/GenomeAnalysisTK.xml b/packages/GenomeAnalysisTK.xml index 54853d524..2aa58952e 100644 --- a/packages/GenomeAnalysisTK.xml +++ b/packages/GenomeAnalysisTK.xml @@ -4,18 +4,9 @@ - - - - - - - - - - - - + + + @@ -28,12 +19,6 @@ - - - - - - diff --git a/packages/Queue.xml b/packages/Queue.xml index 916827dab..97eb9c15b 100644 --- a/packages/Queue.xml +++ b/packages/Queue.xml @@ -1,4 +1,28 @@ + + @@ -21,6 +45,10 @@ + + + + diff --git a/scala/qscript/core/MethodsDevelopmentCallingPipeline.scala b/scala/qscript/core/MethodsDevelopmentCallingPipeline.scala index 42a7323a3..adcf48c31 100755 --- a/scala/qscript/core/MethodsDevelopmentCallingPipeline.scala +++ b/scala/qscript/core/MethodsDevelopmentCallingPipeline.scala @@ -185,8 +185,8 @@ class MethodsDevelopmentCallingPipeline extends QScript { trait UNIVERSAL_GATK_ARGS extends CommandLineGATK { logging_level = "INFO"; jarFile = gatkJarFile; - memoryLimit = Some(4); - phone_home = Some(if ( LOCAL_ET ) GATKRunReport.PhoneHomeOption.STANDARD else GATKRunReport.PhoneHomeOption.AWS_S3) + memoryLimit = 4; + phone_home = if ( LOCAL_ET ) GATKRunReport.PhoneHomeOption.STANDARD else GATKRunReport.PhoneHomeOption.AWS_S3 } def bai(bam: File) = new File(bam + ".bai") @@ -197,9 +197,9 @@ class MethodsDevelopmentCallingPipeline extends QScript { this.reference_sequence = t.reference this.intervalsString ++= List(t.intervals) this.scatterCount = 63 // the smallest interval list has 63 intervals, one for each Mb on chr20 - this.dcov = Some( if ( t.isLowpass ) { 50 } else { 250 } ) - this.stand_call_conf = Some( if ( t.isLowpass ) { 4.0 } else { 30.0 } ) - this.stand_emit_conf = Some( if ( t.isLowpass ) { 4.0 } else { 30.0 } ) + this.dcov = if ( t.isLowpass ) { 50 } else { 250 } + this.stand_call_conf = if ( t.isLowpass ) { 4.0 } else { 30.0 } + this.stand_emit_conf = if ( t.isLowpass ) { 4.0 } else { 30.0 } this.input_file :+= t.bamList if (t.dbsnpFile.endsWith(".rod")) this.DBSNP = new File(t.dbsnpFile) @@ -210,11 +210,11 @@ class MethodsDevelopmentCallingPipeline extends QScript { // 1a.) Call SNPs with UG class snpCall (t: Target) extends GenotyperBase(t) { if (minimumBaseQuality >= 0) - this.min_base_quality_score = Some(minimumBaseQuality) + this.min_base_quality_score = minimumBaseQuality if (qscript.deletions >= 0) - this.max_deletion_fraction = Some(qscript.deletions) + this.max_deletion_fraction = qscript.deletions this.out = t.rawVCF - this.baq = Some( if (noBAQ) {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF} else {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY}) + this.baq = if (noBAQ) {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF} else {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY} this.analysisName = t.name + "_UGs" this.jobName = queueLogDir + t.name + ".snpcall" } @@ -222,8 +222,8 @@ class MethodsDevelopmentCallingPipeline extends QScript { // 1b.) Call Indels with UG class indelCall (t: Target) extends GenotyperBase(t) { this.out = t.rawIndelVCF - this.glm = Some(org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeLikelihoodsCalculationModel.Model.DINDEL) - this.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF) + this.glm = org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeLikelihoodsCalculationModel.Model.DINDEL + this.baq = org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF this.analysisName = t.name + "_UGi" this.jobName = queueLogDir + t.name + ".indelcall" } @@ -248,7 +248,7 @@ class MethodsDevelopmentCallingPipeline extends QScript { // 3.) Variant Quality Score Recalibration - Generate Recalibration table class VQSR(t: Target, goldStandard: Boolean) extends ContrastiveRecalibrator with UNIVERSAL_GATK_ARGS { - this.memoryLimit = Some(6) + this.memoryLimit = 6 this.reference_sequence = t.reference this.intervalsString ++= List(t.intervals) this.rodBind :+= RodBind("input", "VCF", if ( goldStandard ) { t.goldStandard_VCF } else { t.rawVCF } ) @@ -272,13 +272,13 @@ class MethodsDevelopmentCallingPipeline extends QScript { // 4.) Apply the recalibration table to the appropriate tranches class applyVQSR (t: Target, goldStandard: Boolean) extends ApplyRecalibration with UNIVERSAL_GATK_ARGS { - this.memoryLimit = Some(4) + this.memoryLimit = 4 this.reference_sequence = t.reference this.intervalsString ++= List(t.intervals) this.rodBind :+= RodBind("input", "VCF", if ( goldStandard ) { t.goldStandard_VCF } else { t.rawVCF } ) this.tranches_file = if ( goldStandard ) { t.goldStandardTranchesFile } else { t.tranchesFile} this.recal_file = if ( goldStandard ) { t.goldStandardRecalFile } else { t.recalFile } - this.fdr_filter_level = Some(2.0) + this.fdr_filter_level = 2.0 this.out = t.recalibratedVCF this.analysisName = t.name + "_AVQSR" this.jobName = queueLogDir + t.name + ".applyVQSR" @@ -293,7 +293,7 @@ class MethodsDevelopmentCallingPipeline extends QScript { this.intervalsString ++= List(t.intervals) this.out = t.cutVCF this.tranchesFile = t.tranchesFile - this.fdr_filter_level = Some(t.trancheTarget) + this.fdr_filter_level = t.trancheTarget if (t.dbsnpFile.endsWith(".rod")) this.DBSNP = new File(t.dbsnpFile) else if (t.dbsnpFile.endsWith(".vcf")) diff --git a/scala/qscript/core/StandardVariantEvaluation.scala b/scala/qscript/core/StandardVariantEvaluation.scala index 34db7f9bb..f58ae796e 100755 --- a/scala/qscript/core/StandardVariantEvaluation.scala +++ b/scala/qscript/core/StandardVariantEvaluation.scala @@ -65,7 +65,7 @@ class StandardVariantEvaluation extends QScript { this.jarFile = gatkJarFile; this.intervalsString = List(TARGET_INTERVAL); this.reference_sequence = referenceFile; - this.memoryLimit = Some(2) + this.memoryLimit = 2 } def initializeStandardDataFiles() = { diff --git a/scala/qscript/examples/ExampleCountLoci.scala b/scala/qscript/examples/ExampleCountLoci.scala index d3e1add25..ba834e72f 100644 --- a/scala/qscript/examples/ExampleCountLoci.scala +++ b/scala/qscript/examples/ExampleCountLoci.scala @@ -5,9 +5,6 @@ import org.broadinstitute.sting.queue.extensions.gatk._ * An introductory pipeline with integration tests testing the MD5 of the @Output. */ class ExampleCountLoci extends QScript { - @Input(doc="The path to the GenomeAnalysisTK.jar file.", shortName="gatk") - var gatkJar: File = null - @Input(doc="The reference file for the bam files.", shortName="R") var referenceFile: File = null @@ -22,12 +19,11 @@ class ExampleCountLoci extends QScript { def script = { val countLoci = new CountLoci - countLoci.jarFile = gatkJar countLoci.reference_sequence = referenceFile countLoci.input_file = bamFiles countLoci.intervalsString = intervals countLoci.out = out - countLoci.memoryLimit = Some(1) + countLoci.memoryLimit = 1 add(countLoci) } } diff --git a/scala/qscript/examples/ExampleCountReads.scala b/scala/qscript/examples/ExampleCountReads.scala index c02362fa5..85f796a51 100644 --- a/scala/qscript/examples/ExampleCountReads.scala +++ b/scala/qscript/examples/ExampleCountReads.scala @@ -7,9 +7,6 @@ import org.broadinstitute.sting.queue.extensions.gatk._ * All bams must have the same reference. */ class ExampleCountReads extends QScript { - @Input(doc="The path to the GenomeAnalysisTK.jar file.", shortName="gatk") - var gatkJar: File = null - @Input(doc="The reference file for the bam files.", shortName="R") var referenceFile: File = null @@ -33,9 +30,6 @@ class ExampleCountReads extends QScript { // The names of walkers are the same as you would use for '-T ' val jointCountReads = new CountReads - // Set the GATK jar file for this command. - jointCountReads.jarFile = gatkJar - // Each field in the extensions is based off of the full form of the arguments. // To get the list of arguments and their descriptions run // java -jar -T -help @@ -52,7 +46,6 @@ class ExampleCountReads extends QScript { if (bamFiles.size > 1) { for (bamFile <- bamFiles) { val singleCountReads = new CountReads - singleCountReads.jarFile = gatkJar singleCountReads.reference_sequence = referenceFile // ':+' is the scala List append operator singleCountReads.input_file :+= bamFile diff --git a/scala/qscript/examples/ExampleCustomWalker.scala b/scala/qscript/examples/ExampleCustomWalker.scala index 20158aba6..340934b5b 100644 --- a/scala/qscript/examples/ExampleCustomWalker.scala +++ b/scala/qscript/examples/ExampleCustomWalker.scala @@ -2,17 +2,11 @@ import org.broadinstitute.sting.queue.QScript import org.broadinstitute.sting.queue.extensions.gatk._ /** - * A pipeline for Queue that runs a custom walker outside of the GATK jar. + * A pipeline for Queue that runs a custom walker on the classpath. * NOTE: This code is an unsupported example for soliciting feedback on how to improve Queue. * Future syntax will simplify running the GATK so please expect the syntax below to change significantly. */ class ExampleCustomWalker extends QScript { - // The full packaged jar should be used. - // You can build this jar via 'ant package' and then find it under - // 'Sting/dist/packages/GenomeAnalysisTK-*/GenomeAnalysisTK.jar' - @Input(doc="The path to the packaged GenomeAnalysisTK.jar file.", shortName="gatk") - var gatkJar: File = null - @Input(doc="The reference file for the bam files.", shortName="R") var referenceFile: File = null @@ -29,15 +23,18 @@ class ExampleCustomWalker extends QScript { * In script, you create and then add() functions to the pipeline. */ def script = { - val myClasses = "myClassDir" - val customWalker = new CommandLineGATK { // Set the name of your walker, for example this will be passed as -T MyCustomWalker this.analysis_type = "MyCustomWalker" - // NOTE: At this time, you still need to specify the GATK jar or the pipeline won't validate. - this.jarFile = gatkJar - override def javaExecutable = "org.broadinstitute.sting.gatk.CommandLineGATK" - override def javaOpts = "%s -cp %s:%s".format(super.javaOpts, gatkJar, myClasses) + + // If your walker is already on the classpath you shouldn't need to do anything else + + // If your walker is in a GATK jar that is for some reason NOT on the classpath + // nor referenced in the Queue.jar's, specify the jar file here + //this.jarFile = "myGATK.jar" + + // If your walker needs a custom classpath, specify it here + //this.javaClasspath = List("myClasses") } customWalker.reference_sequence = referenceFile diff --git a/scala/qscript/examples/ExampleUnifiedGenotyper.scala b/scala/qscript/examples/ExampleUnifiedGenotyper.scala index 0e4dc6149..2908d23f3 100644 --- a/scala/qscript/examples/ExampleUnifiedGenotyper.scala +++ b/scala/qscript/examples/ExampleUnifiedGenotyper.scala @@ -15,9 +15,6 @@ class ExampleUnifiedGenotyper extends QScript { // Required arguments. All initialized to empty values. - @Input(doc="The path to the GenomeAnalysisTK.jar file.", shortName="gatk") - var gatkJar: File = null // The command line must pass the gatk jar to this script via -gatk. - @Input(doc="The reference file for the bam files.", shortName="R") var referenceFile: File = _ // _ is scala shorthand for null @@ -39,12 +36,11 @@ class ExampleUnifiedGenotyper extends QScript { // This trait allows us set the variables below in one place, // and then reuse this trait on each CommandLineGATK function below. trait UnifiedGenotyperArguments extends CommandLineGATK { - this.jarFile = qscript.gatkJar this.reference_sequence = qscript.referenceFile this.intervals = List(qscript.intervals) - // Some() is how you set the value for an scala Option. + // is how you set the value for an scala Option. // Set the memory limit to 2 gigabytes on each command. - this.memoryLimit = Some(2) + this.memoryLimit = 2 } diff --git a/scala/qscript/oneoffs/carneiro/dataProcessing.scala b/scala/qscript/oneoffs/carneiro/dataProcessing.scala index 4307118f6..e5cd2ae9c 100755 --- a/scala/qscript/oneoffs/carneiro/dataProcessing.scala +++ b/scala/qscript/oneoffs/carneiro/dataProcessing.scala @@ -1,5 +1,5 @@ import org.broadinstitute.sting.queue.extensions.gatk._ -import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction +import org.broadinstitute.sting.queue.extensions.picard.PicardBamFunction import org.broadinstitute.sting.queue.QScript import org.broadinstitute.sting.queue.function.ListWriterFunction import scala.io.Source @@ -55,7 +55,7 @@ class dataProcessing extends QScript { trait CommandLineGATKArgs extends CommandLineGATK { this.jarFile = qscript.GATKjar this.reference_sequence = qscript.reference - this.memoryLimit = Some(4) + this.memoryLimit = 4 this.isIntermediate = true } @@ -127,7 +127,7 @@ class dataProcessing extends QScript { class TargetBase (outIntervals: String) extends RealignerTargetCreator with CommandLineGATKArgs { this.out = new File(outIntervals) - this.mismatchFraction = Some(0.0) + this.mismatchFraction = 0.0 this.rodBind :+= RodBind("dbsnp", "VCF", dbSNP) this.rodBind :+= RodBind("indels", "VCF", indels) @@ -139,7 +139,7 @@ class dataProcessing extends QScript { class allTargets (inBams: String, outIntervals: String) extends TargetBase(outIntervals) { this.input_file :+= new File(inBams) - this.memoryLimit = Some(6) + this.memoryLimit = 6 this.jobName = queueLogDir + outIntervals + ".atarget" } @@ -151,16 +151,16 @@ class dataProcessing extends QScript { this.rodBind :+= RodBind("indels", "VCF", qscript.indels) this.useOnlyKnownIndels = knownsOnly this.doNotUseSW = true - this.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY) - this.compress = Some(0) - this.U = Some(org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION) // todo -- update this with the last consensus between Tim, Matt and Eric. This is ugly! + this.baq = org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY + this.compress = 0 + this.U = org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION // todo -- update this with the last consensus between Tim, Matt and Eric. This is ugly! this.isIntermediate = intermediate this.jobName = queueLogDir + outBam + ".clean" if (!intermediate && !qscript.intervalString.isEmpty()) this.intervalsString ++= List(qscript.intervalString) if (!intermediate && qscript.intervals != null) this.intervals :+= qscript.intervals } - class dedup (inBam: String, outBam: String, metricsFile: String) extends PicardBamJarFunction { + class dedup (inBam: String, outBam: String, metricsFile: String) extends PicardBamFunction { @Input(doc="fixed bam") var clean: File = new File(inBam) @Output(doc="deduped bam") var deduped: File = new File(outBam) @Output(doc="deduped bam index") var dedupedIndex: File = new File(outBam + ".bai") @@ -169,7 +169,7 @@ class dataProcessing extends QScript { override def outputBam = deduped override def commandLine = super.commandLine + " M=" + metricsFile + " CREATE_INDEX=true" sortOrder = null - this.memoryLimit = Some(6) + this.memoryLimit = 6 this.jarFile = qscript.dedupJar this.isIntermediate = true this.jobName = queueLogDir + outBam + ".dedup" @@ -188,8 +188,8 @@ class dataProcessing extends QScript { this.input_file :+= new File (inBam) this.recal_file = new File(inRecalFile) this.out = new File(outBam) - this.U = Some(org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION) // todo -- update this with the last consensus between Tim, Matt and Eric. This is ugly! - this.index_output_bam_on_the_fly = Some(true) + this.U = org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION // todo -- update this with the last consensus between Tim, Matt and Eric. This is ugly! + this.index_output_bam_on_the_fly = true this.jobName = queueLogDir + outBam + ".recalibration" } diff --git a/scala/qscript/oneoffs/carneiro/dataProcessingV2.scala b/scala/qscript/oneoffs/carneiro/dataProcessingV2.scala index e6574e73c..870ed015f 100755 --- a/scala/qscript/oneoffs/carneiro/dataProcessingV2.scala +++ b/scala/qscript/oneoffs/carneiro/dataProcessingV2.scala @@ -1,7 +1,7 @@ package oneoffs.carneiro import org.broadinstitute.sting.queue.extensions.gatk._ -import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction +import org.broadinstitute.sting.queue.extensions.picard.PicardBamFunction import org.broadinstitute.sting.queue.QScript import org.broadinstitute.sting.queue.function.ListWriterFunction @@ -173,11 +173,11 @@ class dataProcessingV2 extends QScript { trait CommandLineGATKArgs extends CommandLineGATK { this.jarFile = qscript.GATKjar this.reference_sequence = qscript.reference - this.memoryLimit = Some(4) + this.memoryLimit = 4 this.isIntermediate = true } - case class joinBams (inBams: List[File], outBam: File) extends PicardBamJarFunction { + case class joinBams (inBams: List[File], outBam: File) extends PicardBamFunction { @Input(doc="input bam list") var join = inBams @Output(doc="joined bam") var joined = outBam @Output(doc="joined bam index") var joinedIndex = new File(outBam + "bai") @@ -194,7 +194,7 @@ class dataProcessingV2 extends QScript { if (!knownsOnly) this.input_file :+= inBams this.out = outIntervals - this.mismatchFraction = Some(0.0) + this.mismatchFraction = 0.0 this.rodBind :+= RodBind("dbsnp", "VCF", dbSNP) this.rodBind :+= RodBind("indels", "VCF", indels) this.scatterCount = nContigs @@ -210,14 +210,14 @@ class dataProcessingV2 extends QScript { this.rodBind :+= RodBind("indels", "VCF", qscript.indels) this.useOnlyKnownIndels = knownsOnly this.doNotUseSW = useSW - this.compress = Some(0) - this.U = Some(org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION) // todo -- update this with the last consensus between Tim, Matt and Eric. This is ugly! + this.compress = 0 + this.U = org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION // todo -- update this with the last consensus between Tim, Matt and Eric. This is ugly! this.scatterCount = nContigs this.analysisName = queueLogDir + outBam + ".clean" this.jobName = queueLogDir + outBam + ".clean" } - case class dedup (inBam: File, outBam: File, metricsFile: File) extends PicardBamJarFunction { + case class dedup (inBam: File, outBam: File, metricsFile: File) extends PicardBamFunction { @Input(doc="fixed bam") var clean = inBam @Output(doc="deduped bam") var deduped = outBam @Output(doc="deduped bam index") var dedupedIndex = new File(outBam + ".bai") @@ -226,7 +226,7 @@ class dataProcessingV2 extends QScript { override def outputBam = deduped override def commandLine = super.commandLine + " M=" + metricsFile + " CREATE_INDEX=true" sortOrder = null - this.memoryLimit = Some(6) + this.memoryLimit = 6 this.jarFile = qscript.dedupJar this.analysisName = queueLogDir + outBam + ".dedup" this.jobName = queueLogDir + outBam + ".dedup" @@ -246,12 +246,12 @@ class dataProcessingV2 extends QScript { @Output(doc="recalibrated bam index") var recalIndex = new File(outBam + ".bai") this.input_file :+= inBam this.recal_file = inRecalFile - this.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY) + this.baq = org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY this.out = outBam if (!qscript.intervalString.isEmpty()) this.intervalsString ++= List(qscript.intervalString) else if (qscript.intervals != null) this.intervals :+= qscript.intervals - this.U = Some(org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION) // todo -- update this with the last consensus between Tim, Matt and Eric. This is ugly! - this.index_output_bam_on_the_fly = Some(true) + this.U = org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION // todo -- update this with the last consensus between Tim, Matt and Eric. This is ugly! + this.index_output_bam_on_the_fly = true this.analysisName = queueLogDir + outBam + ".recalibration" this.jobName = queueLogDir + outBam + ".recalibration" diff --git a/scala/qscript/oneoffs/carneiro/justClean.scala b/scala/qscript/oneoffs/carneiro/justClean.scala index 3eefa811c..e2e8a7153 100755 --- a/scala/qscript/oneoffs/carneiro/justClean.scala +++ b/scala/qscript/oneoffs/carneiro/justClean.scala @@ -42,10 +42,10 @@ class justClean extends QScript { target.input_file :+= input target.out = tIntervals target.reference_sequence = reference - target.mismatchFraction = Some(0.0) + target.mismatchFraction = 0.0 target.rodBind :+= RodBind("dbsnp", "VCF", dbSNP) target.rodBind :+= RodBind("indels", "VCF", indels) - target.memoryLimit = Some(6) + target.memoryLimit = 6 target.jobName = queueLogDir + tIntervals + ".atarget" target.jarFile = GATKjar target.scatterCount = 84 @@ -60,11 +60,11 @@ class justClean extends QScript { clean.rodBind :+= RodBind("dbsnp", "VCF", dbSNP) clean.rodBind :+= RodBind("indels", "VCF", indels) clean.doNotUseSW = true - clean.compress = Some(0) - clean.U = Some(org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION) // todo -- update clean with the last consensus between Tim, Matt and Eric. This is ugly! + clean.compress = 0 + clean.U = org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION // todo -- update clean with the last consensus between Tim, Matt and Eric. This is ugly! clean.jobName = queueLogDir + outBam + ".clean" clean.jarFile = GATKjar - clean.memoryLimit = Some(12) + clean.memoryLimit = 12 clean.scatterCount = 84 add(clean); diff --git a/scala/qscript/oneoffs/carneiro/pbCalling.scala b/scala/qscript/oneoffs/carneiro/pbCalling.scala index 506a24e3f..33baa3369 100755 --- a/scala/qscript/oneoffs/carneiro/pbCalling.scala +++ b/scala/qscript/oneoffs/carneiro/pbCalling.scala @@ -158,20 +158,20 @@ class pbCalling extends QScript { this.reference_sequence = t.reference this.intervalsString ++= List(t.intervals) this.scatterCount = 63 // the smallest interval list has 63 intervals, one for each Mb on chr20 - this.dcov = Some( if ( t.isLowpass ) { 50 } else { 250 } ) - this.stand_call_conf = Some( if ( t.isLowpass ) { 4.0 } else { 30.0 } ) - this.stand_emit_conf = Some( if ( t.isLowpass ) { 4.0 } else { 30.0 } ) + this.dcov = if ( t.isLowpass ) { 50 } else { 250 } + this.stand_call_conf = if ( t.isLowpass ) { 4.0 } else { 30.0 } + this.stand_emit_conf = if ( t.isLowpass ) { 4.0 } else { 30.0 } this.input_file :+= t.bamList this.out = t.rawVCF - this.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY) + this.baq = org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY this.analysisName = t.name + "_UG" if (t.dbsnpFile.endsWith(".rod")) this.DBSNP = new File(t.dbsnpFile) else if (t.dbsnpFile.endsWith(".vcf")) this.rodBind :+= RodBind("dbsnp", "VCF", t.dbsnpFile) // Ridiculous workaround to get pacbio data to run.. never commit this! - this.deletions = Some(0.5) - this.mbq = Some(10) + this.deletions = 0.5 + this.mbq = 10 } // 2.) Filter SNPs @@ -188,7 +188,7 @@ class pbCalling extends QScript { } class VQSR(t: Target, goldStandard: Boolean) extends ContrastiveRecalibrator { - this.memoryLimit = Some(6) + this.memoryLimit = 6 this.intervalsString ++= List(t.intervals) this.rodBind :+= RodBind("input", "VCF", if ( goldStandard ) { t.goldStandard_VCF } else { t.filteredVCF } ) this.rodBind :+= RodBind("hapmap", "VCF", t.hapmapFile) @@ -208,12 +208,12 @@ class pbCalling extends QScript { } class applyVQSR (t: Target, goldStandard: Boolean) extends ApplyRecalibration { - this.memoryLimit = Some(4) + this.memoryLimit = 4 this.intervalsString ++= List(t.intervals) this.rodBind :+= RodBind("input", "VCF", if ( goldStandard ) { t.goldStandard_VCF } else { t.filteredVCF } ) this.tranches_file = if ( goldStandard ) { t.goldStandardTranchesFile } else { t.tranchesFile} this.recal_file = if ( goldStandard ) { t.goldStandardRecalFile } else { t.recalFile } - this.fdr_filter_level = Some(2.0) + this.fdr_filter_level = 2.0 this.out = t.recalibratedVCF } @@ -226,7 +226,7 @@ class pbCalling extends QScript { this.intervalsString ++= List(t.intervals) this.out = t.cutVCF this.tranchesFile = t.tranchesFile - this.fdr_filter_level = Some(1.0) + this.fdr_filter_level = 1.0 if (t.dbsnpFile.endsWith(".rod")) this.DBSNP = new File(t.dbsnpFile) else if (t.dbsnpFile.endsWith(".vcf")) diff --git a/scala/qscript/oneoffs/chartl/BootstrapCalls.q b/scala/qscript/oneoffs/chartl/BootstrapCalls.q index 9e9954446..915e2e3c7 100755 --- a/scala/qscript/oneoffs/chartl/BootstrapCalls.q +++ b/scala/qscript/oneoffs/chartl/BootstrapCalls.q @@ -37,14 +37,14 @@ class BootstrapCalls extends QScript { trait UGArgs extends UnifiedGenotyper { this.input_file = bams this.reference_sequence = reference - this.dcov = Some(downsamplingLevel) + this.dcov = downsamplingLevel this.intervals :+= intervalFile - this.stand_call_conf = Some(standCallConf) - this.stand_emit_conf = Some(standCallConf) + this.stand_call_conf = standCallConf + this.stand_emit_conf = standCallConf this.rodBind :+= new RodBind("dbsnp","vcf",dbsnp) this.scatterCount = 20 this.jarFile = sting - this.memoryLimit = Some(4) + this.memoryLimit = 4 } val bootstrapBase = swapExt(bootstrapMergedOut,".vcf",".boot%d.vcf").getAbsolutePath @@ -63,7 +63,7 @@ class BootstrapCalls extends QScript { this.intervals :+= intervalFile this.scatterCount = 40 this.jarFile = sting - this.memoryLimit = Some(4) + this.memoryLimit = 4 this.rodBind ++= calls.map(u => u.out).zipWithIndex.map(u => new RodBind("bootstrap_%d".format(u._2),"vcf",u._1)) this.out = bootstrapMergedOut } @@ -81,11 +81,11 @@ class BootstrapCalls extends QScript { this.rodBind :+= new RodBind("truth1kg","vcf", new File("/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/Omni2.5_chip/1212samples.b37.sites.vcf")) this.cluster_file = swapExt(bootstrapMergedOut,"vcf","cluster") this.use_annotation ++= List("QD", "SB", "HaplotypeScore", "HRun") - this.qual = Some(100) - this.std = Some(3.5) - this.mG = Some(8) + this.qual = 100 + this.std = 3.5 + this.mG = 8 this.trustAllPolymorphic = true - this.memoryLimit = Some(8) + this.memoryLimit = 8 this.jarFile = sting } @@ -102,11 +102,11 @@ class BootstrapCalls extends QScript { this.rodBind :+= new RodBind("truthHapMap","vcf",new File("/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/HapMap/3.3/sites_r27_nr.b37_fwd.vcf")) this.rodBind :+= new RodBind("truth1kg","vcf", new File("/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/Omni2.5_chip/1212samples.b37.sites.vcf")) this.cluster_file = swapExt(bootstrapMergedOut,"vcf","cluster") - this.sm = Some(org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY) + this.sm = org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY this.tranche ++= List("0.1", "0.5", "0.7", "1.0", "3.0", "5.0", "10.0", "100.0") this.trustAllPolymorphic = true this.tranchesFile = swapExt(bootstrapMergedOut,"vcf","tranche") - this.memoryLimit=Some(8) + this.memoryLimit=8 this.jarFile = sting this.rodBind :+= new RodBind("dbsnp","vcf",dbsnp) } @@ -119,10 +119,10 @@ class BootstrapCalls extends QScript { this.intervals :+= intervalFile this.rodBind :+= new RodBind("input","vcf",recal.out) this.tranchesFile = recal.tranchesFile - this.fdr_filter_level = Some(1.0) + this.fdr_filter_level = 1.0 this.out = swapExt(bootstrapMergedOut,".vcf",".recal.cut.vcf") this.jarFile = sting - this.memoryLimit = Some(4) + this.memoryLimit = 4 this.scatterCount = 5 } @@ -149,12 +149,12 @@ class BootstrapCalls extends QScript { this.intervals :+= intervalFile this.rodBind :+= new RodBind("loCov","vcf",rm.noheadvcf) this.rodBind :+= new RodBind("hiCov","vcf",new File("/humgen/gsa-pipeline/PVQF4/all_batches_v001/batch_001/SnpCalls/ESPGO_Gabriel_NHLBI_EOMI_setone_EOMI_Project.cleaned.annotated.handfiltered.vcf")) - this.variantMergeOptions = Some(VariantMergeType.UNION) - this.genotypeMergeOptions = Some(GenotypeMergeType.PRIORITIZE) + this.variantMergeOptions = VariantMergeType.UNION + this.genotypeMergeOptions = GenotypeMergeType.PRIORITIZE this.priority = "hiCov,loCov" this.out = swapExt(bootstrapMergedOut,".vcf",".merged.combined.vcf") this.jarFile = sting - this.memoryLimit = Some(6) + this.memoryLimit = 6 } var combine : CombineVariants = new CombineVariants with CombineArgs @@ -175,8 +175,8 @@ class BootstrapCalls extends QScript { "\"set == 'hiCov'\"","\"set == 'FilteredInAll'\"") this.EV = List("TiTvVariantEvaluator","CountVariants","CompOverlap") this.out = swapExt(bootstrapMergedOut,".vcf",".merged.combined.eval") - this.nt = Some(8) - this.memoryLimit = Some(12) + this.nt = 8 + this.memoryLimit = 12 } var eval : VariantEval = new VariantEval with EvalArgs diff --git a/scala/qscript/oneoffs/chartl/RefineGenotypesAndMerge.q b/scala/qscript/oneoffs/chartl/RefineGenotypesAndMerge.q index deaadd36a..3c7380caa 100755 --- a/scala/qscript/oneoffs/chartl/RefineGenotypesAndMerge.q +++ b/scala/qscript/oneoffs/chartl/RefineGenotypesAndMerge.q @@ -3,7 +3,6 @@ import net.sf.picard.reference.FastaSequenceFile import org.broadinstitute.sting.datasources.pipeline.Pipeline import org.broadinstitute.sting.gatk.DownsampleType import org.broadinstitute.sting.queue.extensions.gatk._ -import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction import org.broadinstitute.sting.queue.extensions.samtools._ import org.broadinstitute.sting.queue.{QException, QScript} import collection.JavaConversions._ @@ -69,7 +68,7 @@ class RefineGenotypesAndMerge extends QScript { refine.beagleInput = beagleInput.out refine.beagleOutputBase = beagleBase refine.beagleMemoryGigs = 6 - refine.memoryLimit = Some(6) + refine.memoryLimit = 6 refine.freezeOutputs var unzipPhased = new GunzipFile(refine.beaglePhasedFile,swapExt(refine.beaglePhasedFile,".gz",".bgl")) @@ -94,8 +93,8 @@ class RefineGenotypesAndMerge extends QScript { def mergeVCFs(vcfs: List[File], outputVCF: File) : CombineVariants = { var cv = new CombineVariants with GATKArgs cv.out = outputVCF - cv.genotypemergeoption = Some(org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.GenotypeMergeType.UNSORTED) - cv.variantmergeoption = Some(org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.UNION) + cv.genotypemergeoption = org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.GenotypeMergeType.UNSORTED + cv.variantmergeoption = org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.UNION cv.priority = (vcfs.foldLeft[List[String]](Nil)( (bNames,vcf) => bNames ::: List[String](swapExt(vcf,".vcf","").getName))).mkString(",") cv.rodBind = vcfs.foldLeft[List[RodBind]](Nil)( (rods,vcf) => rods ::: List[RodBind](new RodBind(swapExt(vcf,".vcf","").getName,"VCF",vcf))) diff --git a/scala/qscript/oneoffs/chartl/expanded_targets.q b/scala/qscript/oneoffs/chartl/expanded_targets.q index 813e82758..8d1cdb1d8 100755 --- a/scala/qscript/oneoffs/chartl/expanded_targets.q +++ b/scala/qscript/oneoffs/chartl/expanded_targets.q @@ -39,14 +39,14 @@ class expanded_targets extends QScript { rtc.out = swapExt(userDir,u,".bam",".clean.targets.interval_list") rtc.input_file :+= u.getAbsoluteFile rtc.intervals :+= cleanIntervals.outList - rtc.memoryLimit = Some(6) + rtc.memoryLimit = 6 rtc }) val clean : List[IndelRealigner] = realign.map( u => { var cleaner : IndelRealigner = new IndelRealigner with GATKArgs cleaner.targetIntervals = u.out cleaner.input_file = u.input_file - cleaner.memoryLimit = Some(6) + cleaner.memoryLimit = 6 cleaner.out = new File(userDir+"/"+swapExt(u.out,".bam",".expanded.targets.bam").getName) cleaner.intervals :+= cleanIntervals.outList cleaner @@ -71,10 +71,10 @@ class expanded_targets extends QScript { var call : UnifiedGenotyper = new UnifiedGenotyper with GATKArgs call.input_file = bams call.out = swapExt(iList,".interval_list",".raw.vcf") - call.trig_emit_conf = Some(0.0) + call.trig_emit_conf = 0.0 call.rodBind :+= new RodBind("trigger","vcf",thisTrigger) call.scatterCount = 10 - call.memoryLimit = Some(6) + call.memoryLimit = 6 var filter : VariantFiltration = new VariantFiltration with GATKArgs filter.rodBind :+= new RodBind("variant","vcf",call.out) filter.filterExpression :+= "\"QD<5.0\"" @@ -87,7 +87,7 @@ class expanded_targets extends QScript { callHiseq.input_file = List(new File("/seq/picard_aggregation/EXT1/NA12878/v3/NA12878.bam")) callHiseq.rodBind :+= new RodBind("trigger","vcf",filter.out) callHiseq.out = swapExt(iList,".interval_list",".hiSeq.genotypes.vcf") - callHiseq.trig_emit_conf = Some(0.0) + callHiseq.trig_emit_conf = 0.0 callHiseq.scatterCount = 5 add(call,filter,callHiseq) @@ -98,8 +98,8 @@ class expanded_targets extends QScript { eval.rodBind :+= new RodBind("compHiSeq_atSites","vcf",callHiseq.out) eval.rodBind :+= new RodBind("compOMNI","vcf",new File("/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/Omni2.5_chip/764samples.deduped.b37.annot.vcf")) eval.out = swapExt(iList,".interval_list",".eval") - eval.reportType = Option(org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType.CSV) - eval.memoryLimit = Some(4) + eval.reportType = org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType.CSV + eval.memoryLimit = 4 add(eval) eval.out diff --git a/scala/qscript/oneoffs/chartl/old/private_mutations_old.q b/scala/qscript/oneoffs/chartl/old/private_mutations_old.q index 095ad6fac..fc38efe60 100755 --- a/scala/qscript/oneoffs/chartl/old/private_mutations_old.q +++ b/scala/qscript/oneoffs/chartl/old/private_mutations_old.q @@ -165,8 +165,8 @@ class private_mutations_old extends QScript { getVennExS.intervals :+= exome_single.out_list getVennExS.reference_sequence = new File("/humgen/1kg/reference/human_g1k_v37.fasta") getVennExS.jarFile = new File("/humgen/gsa-scr1/chartl/sting/dist/GenomeAnalysisTK.jar") - getVennExS.genotypeMergeOptions = Some(VariantContextUtils.GenotypeMergeType.UNIQUIFY) - getVennExS.variantMergeOptions = Some(VariantContextUtils.VariantMergeType.UNION) + getVennExS.genotypeMergeOptions = VariantContextUtils.GenotypeMergeType.UNIQUIFY + getVennExS.variantMergeOptions = VariantContextUtils.VariantMergeType.UNION getVennExS.out = new File(VCF_DIR + "g1k_exome_plus_lowpass.singlesample.merged.exome.sites.vcf") //add(getVennExS) @@ -178,8 +178,8 @@ class private_mutations_old extends QScript { getVennLPS.intervals :+= lowpass_single.out_list getVennLPS.reference_sequence = new File("/humgen/1kg/reference/human_g1k_v37.fasta") getVennLPS.jarFile = new File("/humgen/gsa-scr1/chartl/sting/dist/GenomeAnalysisTK.jar") - getVennLPS.genotypeMergeOptions = Some(VariantContextUtils.GenotypeMergeType.UNIQUIFY) - getVennLPS.variantMergeOptions = Some(VariantContextUtils.VariantMergeType.UNION) + getVennLPS.genotypeMergeOptions = VariantContextUtils.GenotypeMergeType.UNIQUIFY + getVennLPS.variantMergeOptions = VariantContextUtils.VariantMergeType.UNION getVennLPS.out = new File(VCF_DIR + "g1k_exome_plus_lowpass.singlesample.merged.lowpass.sites.vcf") add(getVennLPS) @@ -397,7 +397,7 @@ class private_mutations_old extends QScript { genotype.intervals :+= sites.out_list genotype.out = new File(VCF_DIR+"g1k_lowpass.%s.exome_sites.vcf".format(s)) genotype.input_file :+= bamList.outList - genotype.memoryLimit = Some(3) + genotype.memoryLimit = 3 genotype.output_all_callable_bases = true add(genotype) diff --git a/scala/qscript/oneoffs/chartl/omni_qc.q b/scala/qscript/oneoffs/chartl/omni_qc.q index be20c4f6b..ea7f1ff92 100755 --- a/scala/qscript/oneoffs/chartl/omni_qc.q +++ b/scala/qscript/oneoffs/chartl/omni_qc.q @@ -5,7 +5,6 @@ import org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils import org.broadinstitute.sting.gatk.DownsampleType import org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeCalculationModel.Model import org.broadinstitute.sting.queue.extensions.gatk._ -import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction import org.broadinstitute.sting.queue.extensions.samtools._ import org.broadinstitute.sting.queue.{QException, QScript} import collection.JavaConversions._ @@ -201,7 +200,7 @@ class omni_qc extends QScript { eval1KG_exclude.evalModule :+= "GenotypeConcordance" eval1KG_exclude.evalModule :+= "SimpleMetricsBySample" eval1KG_exclude.reference_sequence = b36_ref - eval1KG_exclude.reportType = Some(VE2TemplateType.CSV) + eval1KG_exclude.reportType = VE2TemplateType.CSV eval1KG_exclude.intervalsString :+= pilot3_interval_list eval1KG_exclude.out = new File(eval_dir+"%s_vs_%s.%s".format("OMNI_764","Pilot3","exclude.mixups.eval.csv")) @@ -222,7 +221,7 @@ class omni_qc extends QScript { combine.rodBind :+= new RodBind("CEU","VCF",gunzip_p1_ceu.outFile) combine.rodBind :+= new RodBind("ASN","VCF",gunzip_p1_chb.outFile) combine.rodBind :+= new RodBind("YRI","VCF",gunzip_p1_yri.outFile) - combine.genotypeMergeOptions = Some(VariantContextUtils.GenotypeMergeType.UNIQUIFY) + combine.genotypeMergeOptions = VariantContextUtils.GenotypeMergeType.UNIQUIFY combine.priority = "%s,%s,%s".format("CEU","ASN","YRI") combine.out = new File(vcf_dir+"Pilot1_Populations_Combined.vcf") @@ -257,7 +256,7 @@ class omni_qc extends QScript { vEval.evalModule :+= "SimpleMetricsBySample" vEval.intervalsString :+= intervals vEval.reference_sequence = reference - vEval.reportType = Some(VE2TemplateType.CSV) + vEval.reportType = VE2TemplateType.CSV vEval.out = new File(eval_dir+base+".eval.csv") @@ -318,8 +317,8 @@ class omni_qc extends QScript { panelCombine.priority = if ( panelCombine.priority.equals("") ) p else panelCombine.priority + "," + p } panelCombine.out = OMNI_b36_panel_vcf - panelCombine.genotypeMergeOptions = Some(VariantContextUtils.GenotypeMergeType.REQUIRE_UNIQUE) - panelCombine.variantMergeOptions = Some(VariantContextUtils.VariantMergeType.UNION) + panelCombine.genotypeMergeOptions = VariantContextUtils.GenotypeMergeType.REQUIRE_UNIQUE + panelCombine.variantMergeOptions = VariantContextUtils.VariantMergeType.UNION panelCombine.setKey = "panel" add(panelCombine) @@ -348,7 +347,7 @@ class omni_qc extends QScript { eval.rodBind :+= new RodBind("comp%s".format(cBase),"VCF",compVCF) eval.noStandard = true eval.E :+= "AlleleFrequencyComparison" - eval.reportType = Some(VE2TemplateType.CSV) + eval.reportType = VE2TemplateType.CSV eval.out = new File(eval_dir+"%s_vs_%s_allele_frequency.eval".format(eBase,cBase)) add(eval) @@ -358,7 +357,7 @@ class omni_qc extends QScript { combine.rodBind :+= new RodBind(eBase,"VCF",evalVCF) combine.rodBind :+= new RodBind(cBase,"VCF",compVCF) combine.out = new File(vcf_dir+"%s_plus_%s.vcf".format(eBase,cBase)) - combine.genotypeMergeOptions = Some(VariantContextUtils.GenotypeMergeType.UNIQUIFY) + combine.genotypeMergeOptions = VariantContextUtils.GenotypeMergeType.UNIQUIFY combine.priority = "%s,%s".format(eBase,cBase) //add(combine) @@ -386,8 +385,8 @@ class omni_qc extends QScript { combine.priority = "%s%s".format("beagle",c) } } - combine.genotypeMergeOptions = Some(VariantContextUtils.GenotypeMergeType.PRIORITIZE) - combine.variantMergeOptions = Some(VariantContextUtils.VariantMergeType.UNION) + combine.genotypeMergeOptions = VariantContextUtils.GenotypeMergeType.PRIORITIZE + combine.variantMergeOptions = VariantContextUtils.VariantMergeType.UNION combine.out = swapExt(pilot1_with_na12878_vcf,".vcf",".beagle_refined_with_omni.vcf") @@ -408,7 +407,7 @@ class omni_qc extends QScript { eval.E :+= "GenotypeConcordance" eval.out = new File(eval_dir+"NA12878.lowpass.beagle.vs.HiSeq.eval") eval.excludeIntervals :+= new File(pilot1_interval_list) - eval.reportType = Some(VE2TemplateType.CSV) + eval.reportType = VE2TemplateType.CSV add(eval) @@ -420,7 +419,7 @@ class omni_qc extends QScript { eval2.sample :+= "NA12878" eval2.out = new File(eval_dir+"NA12878.lowpass.nochip.vs.Hiseq.eval") eval2.excludeIntervals :+= new File(pilot1_interval_list) - eval2.reportType = Some(VE2TemplateType.CSV) + eval2.reportType = VE2TemplateType.CSV add(eval2) @@ -432,7 +431,7 @@ class omni_qc extends QScript { eval3.sample :+= "NA12878" eval3.out = new File(eval_dir+"NA12878.lowpass.nochip.norefined.vs.Hiseq.eval") eval3.excludeIntervals :+= new File(pilot1_interval_list) - eval3.reportType = Some(VE2TemplateType.CSV) + eval3.reportType = VE2TemplateType.CSV add(eval3) } @@ -443,7 +442,7 @@ class omni_qc extends QScript { beagleInput.intervalsString :+= chr beagleInput.variantVCF = pilot1_with_na12878_vcf beagleInput.rodBind :+= new RodBind("validation","VCF",omnivcf) - beagleInput.validation_genotype_ptrue = Some(0.99) + beagleInput.validation_genotype_ptrue = 0.99 beagleInput.out = new File(scratch_dir+"/"+swapExt(beagleInput.variantVCF.getName,".vcf",".%s.beagle".format(chr))) println (beagleInput.out.getAbsolutePath) @@ -451,7 +450,7 @@ class omni_qc extends QScript { runBeagle.beagleInput = beagleInput.out runBeagle.beagleOutputBase = "Pilot1_NA12878_Beagle_with_OMNI_chr%s".format(chr) runBeagle.beagleMemoryGigs = 6 - runBeagle.memoryLimit = Some(6) + runBeagle.memoryLimit = 6 runBeagle.beagleOutputDir = "" runBeagle.freezeOutputs diff --git a/scala/qscript/oneoffs/chartl/private_mutations.q b/scala/qscript/oneoffs/chartl/private_mutations.q index eb27230ac..40dcb751f 100755 --- a/scala/qscript/oneoffs/chartl/private_mutations.q +++ b/scala/qscript/oneoffs/chartl/private_mutations.q @@ -61,7 +61,7 @@ class private_mutations extends QScript { eval_all.noStandard = true eval_all.E :+= "ACTransitionTable" eval_all.out = swapExt(finalMergedVCF,".vcf",".perm.csv") - eval_all.reportType = Some(org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType.CSV) + eval_all.reportType = org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType.CSV add(eval_all) @@ -70,7 +70,7 @@ class private_mutations extends QScript { eval_afr.rodBind :+= new RodBind("compEUR","VCF",extract_eur.outputVCF) eval_afr.E :+= "ACTransitionTable" eval_afr.out = swapExt(extract_afr.outputVCF,".vcf",".perm.csv") - eval_afr.reportType = Some(org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType.CSV) + eval_afr.reportType = org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType.CSV eval_afr.noStandard = true add(eval_afr) @@ -80,7 +80,7 @@ class private_mutations extends QScript { eval_eur.rodBind :+= new RodBind("evalEUR","VCF",extract_eur.outputVCF) eval_eur.E :+= "ACTransitionTable" eval_eur.out = swapExt(extract_eur.outputVCF,".vcf",".perm.csv") - eval_eur.reportType = Some(org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType.CSV) + eval_eur.reportType = org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType.CSV eval_eur.noStandard = true add(eval_eur) diff --git a/scala/qscript/oneoffs/delangel/Phase1IndelCalling.scala b/scala/qscript/oneoffs/delangel/Phase1IndelCalling.scala index b1b541e33..8b6794041 100755 --- a/scala/qscript/oneoffs/delangel/Phase1IndelCalling.scala +++ b/scala/qscript/oneoffs/delangel/Phase1IndelCalling.scala @@ -4,7 +4,6 @@ import org.broadinstitute.sting.datasources.pipeline.Pipeline import org.broadinstitute.sting.gatk.DownsampleType import org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeLikelihoodsCalculationModel import org.broadinstitute.sting.queue.extensions.gatk._ -import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction import org.broadinstitute.sting.queue.extensions.samtools._ import org.broadinstitute.sting.queue.function.scattergather.{GatherFunction, CloneFunction, ScatterFunction} import org.broadinstitute.sting.queue.{QException, QScript} @@ -55,7 +54,7 @@ class Phase1Calling extends QScript { trait CommandLineGATKArgs extends CommandLineGATK { this.jarFile = qscript.gatkJar this.reference_sequence = qscript.reference - this.memoryLimit = Some(3) + this.memoryLimit = 3 this.jobTempDir = qscript.tmpDir this.jobQueue = "gsa"; @@ -102,15 +101,15 @@ class Phase1Calling extends QScript { } - call.dcov = Some( 50 ) - call.stand_call_conf = Some( 4.0 ) - call.stand_emit_conf = Some( 4.0 ) + call.dcov = 50 + call.stand_call_conf = 4.0 + call.stand_emit_conf = 4.0 call.input_file :+= bamList call.out = rawCalls - call.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF) + call.baq = org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF call.analysisName = baseName + "_UG" call.rodBind :+= RodBind("dbsnp", "VCF", qscript.dbSNPIndels) - call.glm = Some(GenotypeLikelihoodsCalculationModel.Model.DINDEL) + call.glm = GenotypeLikelihoodsCalculationModel.Model.DINDEL var filter = new VariantFiltration with CommandLineGATKArgs @@ -132,9 +131,9 @@ class Phase1Calling extends QScript { gvc.use_annotation ++= List("QD", "SB", "HaplotypeScore", "HRun") gvc.analysisName = baseName + "_GVC" gvc.intervalsString ++= List(qscript.intervals) - //gvc.qual = Some(100) // clustering parameters to be updated soon pending new experimentation results - //gvc.std = Some(4.5) - //gvc.mG = Some(6) + //gvc.qual = 100 // clustering parameters to be updated soon pending new experimentation results + //gvc.std = 4.5 + //gvc.mG = 6 /* var vr = new VariantRecalibrator with CommandLineGATKArgs vr.rodBind :+= RodBind("1kg", "VCF", qscript.omni) @@ -146,13 +145,13 @@ class Phase1Calling extends QScript { vr.analysisName = baseName + "_VR" vr.intervalsString ++= List(qscript.intervals) vr.ignoreFilter ++= List("HARD_TO_VALIDATE") - vr.target_titv = Some(2.3) - vr.sm = Some(org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY) + vr.target_titv = 2.3 + vr.sm = org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY vr.tranche ++= List("0.1", "1.0", "2.0", "3.0", "5.0", "10.0", "100.0") vr.out = recalibratedCalls - vr.priorDBSNP = Some(10.0) - vr.priorHapMap = Some(12.0) - vr.prior1KG = Some(12.0) + vr.priorDBSNP = 10.0 + vr.priorHapMap = 12.0 + vr.prior1KG = 12.0 vr.tranchesFile = tranchesFile add(call, filter, gvc, vr) */ diff --git a/scala/qscript/oneoffs/depristo/1kg_table1.scala b/scala/qscript/oneoffs/depristo/1kg_table1.scala index 2c6968539..020f66e4e 100755 --- a/scala/qscript/oneoffs/depristo/1kg_table1.scala +++ b/scala/qscript/oneoffs/depristo/1kg_table1.scala @@ -127,10 +127,10 @@ class VariantEval(vcfIn: String, evalOut: String, vcfType: String = "VCF") exten this.rodBind :+= RodBind("eval", vcfType, vcfFile) this.out = new File(evalOut) this.DBSNP = new File("/humgen/gsa-hpprojects/GATK/data/dbsnp_129_b36.rod") - this.reportType = Some(VE2TemplateType.Grep) + this.reportType = VE2TemplateType.Grep this.noStandard = true; this.evalModule :+= "CompOverlap" - this.memoryLimit = Some(3) + this.memoryLimit = 3 override def dotString = "VariantEval: " + vcfFile.getName } @@ -147,8 +147,8 @@ class StatPop(target: Target) extends CommandLineFunction { class Combine(vcfsInArg: List[String], vcfOutPath: String) extends org.broadinstitute.sting.queue.extensions.gatk.CombineVariants with UNIVERSAL_GATK_ARGS { val vcfs = vcfsInArg.map((x: String) => new File(x)) val vcfFile = new File(vcfOutPath) - this.variantmergeoption = Some(VariantMergeType.UNION) - this.genotypemergeoption = Some(GenotypeMergeType.PRIORITIZE) + this.variantmergeoption = VariantMergeType.UNION + this.genotypemergeoption = GenotypeMergeType.PRIORITIZE this.out = vcfFile this.rodBind ++= vcfs.map( input => RodBind(input.getName,"VCF",input) ) this.rod_priority_list = vcfs.map( _.getName ).mkString(",") @@ -164,8 +164,8 @@ class DepthOfCoverage(bam: String, docOutPath: String, interval: String) extends val bamFile = new File(bam) this.omitIntervalStatistics = true this.omitDepthOutputAtEachBase = true - this.minBaseQuality = Some(0) - this.minMappingQuality = Some(0) + this.minBaseQuality = 0 + this.minMappingQuality = 0 this.out = new File(docOutPath) this.input_file :+= bamFile if (interval != null) { diff --git a/scala/qscript/oneoffs/depristo/CleaningTest.scala b/scala/qscript/oneoffs/depristo/CleaningTest.scala index 1b08d9ca5..dc360a4b3 100755 --- a/scala/qscript/oneoffs/depristo/CleaningTest.scala +++ b/scala/qscript/oneoffs/depristo/CleaningTest.scala @@ -4,8 +4,8 @@ package oneoffs.depristo import org.broadinstitute.sting.queue.extensions.gatk._ import org.broadinstitute.sting.queue.QScript import collection.JavaConversions._ -import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction -import org.broadinstitute.sting.queue.function.JarCommandLineFunction +import org.broadinstitute.sting.queue.extensions.picard.PicardBamFunction +import org.broadinstitute.sting.queue.function.JavaCommandLineFunction class CleaningTest extends QScript { @@ -45,7 +45,7 @@ class CleaningTest extends QScript { trait CommandLineGATKArgs extends CommandLineGATK { this.jarFile = qscript.gatkJar this.reference_sequence = qscript.reference - this.memoryLimit = Some(4) + this.memoryLimit = 4 this.jobTempDir = qscript.tmpDir } @@ -62,7 +62,7 @@ class CleaningTest extends QScript { target.input_file :+= bamList target.intervalsString :+= interval target.out = targetIntervals - target.mismatchFraction = Some(0.0) + target.mismatchFraction = 0.0 target.rodBind :+= RodBind("dbsnp", "VCF", qscript.dbSNP) target.rodBind :+= RodBind("indels3", "VCF", qscript.dindelEURCalls) //target.jobName = baseName + ".target" @@ -79,7 +79,7 @@ class CleaningTest extends QScript { clean.out = if ( cm ) cleanedBam else new File(cleanedBam + ".intermediate.bam") clean.doNotUseSW = true clean.constrainMovement = cm - clean.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF) + clean.baq = org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF clean.rodBind :+= RodBind("dbsnp", "VCF", qscript.dbSNP) clean.rodBind :+= RodBind("indels3", "VCF", qscript.dindelEURCalls) //clean.sortInCoordinateOrderEvenThoughItIsHighlyUnsafe = true @@ -90,7 +90,7 @@ class CleaningTest extends QScript { if ( ! cm ) { // Explicitly run fix mates if the function won't be scattered. - val fixMates = new PicardBamJarFunction { + val fixMates = new PicardBamFunction { // Declare inputs/outputs for dependency tracking. @Input(doc="unfixed bam") var unfixed: File = _ @Output(doc="fixed bam") var fixed: File = _ @@ -99,7 +99,7 @@ class CleaningTest extends QScript { } //fixMates.jobOutputFile = new File(".queue/logs/Cleaning/%s/FixMates.out".format(sampleId)) - fixMates.memoryLimit = Some(4) + fixMates.memoryLimit = 4 fixMates.jarFile = qscript.picardFixMatesJar fixMates.unfixed = clean.out fixMates.fixed = cleanedBam @@ -110,7 +110,7 @@ class CleaningTest extends QScript { add(fixMates) } - val validate = new JarCommandLineFunction { + val validate = new JavaCommandLineFunction { // Declare inputs/outputs for dependency tracking. @Input(doc="unfixed bam") var unfixed: File = _ def inputBams = List(unfixed) @@ -119,12 +119,12 @@ class CleaningTest extends QScript { } //fixMates.jobOutputFile = new File(".queue/logs/Cleaning/%s/FixMates.out".format(sampleId)) - validate.memoryLimit = Some(2) + validate.memoryLimit = 2 validate.jarFile = qscript.picardValidateJar validate.unfixed = cleanedBam add(validate) - val toQueryName = new PicardBamJarFunction { + val toQueryName = new PicardBamFunction { // Declare inputs/outputs for dependency tracking. @Input(doc="coordiante bam") var cobam: File = _ @Output(doc="query bam") var qnbam: File = _ @@ -133,7 +133,7 @@ class CleaningTest extends QScript { } //fixMates.jobOutputFile = new File(".queue/logs/Cleaning/%s/FixMates.out".format(sampleId)) - toQueryName.memoryLimit = Some(4) + toQueryName.memoryLimit = 4 toQueryName.jarFile = qscript.picardSortSamJar toQueryName.cobam = cleanedBam toQueryName.qnbam = new File(cleanedBam.getAbsolutePath + ".qn.bam") diff --git a/scala/qscript/oneoffs/depristo/RefineGenotypesWithBeagle.q b/scala/qscript/oneoffs/depristo/RefineGenotypesWithBeagle.q index 3aceb0fa3..e9bed79d3 100755 --- a/scala/qscript/oneoffs/depristo/RefineGenotypesWithBeagle.q +++ b/scala/qscript/oneoffs/depristo/RefineGenotypesWithBeagle.q @@ -36,7 +36,7 @@ class RefineGenotypesWithBeagle extends QScript { trait GATKArgs extends CommandLineGATK { this.reference_sequence = qscript.reference this.jarFile = qscript.gatkJarFile - this.memoryLimit = Some(2) + this.memoryLimit = 2 } // -------------------------------------------------------------------------------- @@ -48,9 +48,9 @@ class RefineGenotypesWithBeagle extends QScript { class GenotypeBAMAtSites(@Input bam: File, @Input sitesVCF: File, @Output genotypesVCF: File) extends UnifiedGenotyper with GATKArgs { this.input_file = List(bam) this.o = genotypesVCF - this.stand_call_conf = Some(0.0) - this.out_mode = Some(org.broadinstitute.sting.gatk.walkers.genotyper.UnifiedGenotyperEngine.OUTPUT_MODE.EMIT_ALL_SITES) - this.gt_mode = Some(org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES) + this.stand_call_conf = 0.0 + this.out_mode = org.broadinstitute.sting.gatk.walkers.genotyper.UnifiedGenotyperEngine.OUTPUT_MODE.EMIT_ALL_SITES + this.gt_mode = org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES this.rodBind :+= new RodBind("alleles","VCF",sitesVCF) // we only want chromosome counts annotations @@ -62,7 +62,7 @@ class RefineGenotypesWithBeagle extends QScript { // make sure we have the right intervals if ( interval != null ) { this.intervalsString = List(interval) - this.BTIMR = Some(org.broadinstitute.sting.utils.interval.IntervalSetRule.INTERSECTION) + this.BTIMR = org.broadinstitute.sting.utils.interval.IntervalSetRule.INTERSECTION } } @@ -73,7 +73,7 @@ class RefineGenotypesWithBeagle extends QScript { // -------------------------------------------------------------------------------- class BeagleCommand(outputBase: String) extends CommandLineFunction { - this.memoryLimit = Some(BEAGLE_MEM_IN_GB) + this.memoryLimit = BEAGLE_MEM_IN_GB // Note: These get set @Output val beaglePhasedFile: File = new File(outputBase +".phased.gz") @@ -151,7 +151,7 @@ class RefineGenotypesWithBeagle extends QScript { if ( interval != null ) evalBeagle.intervalsString = List(interval) evalBeagle.variantVCF = evalVCF evalBeagle.out = swapExt(outputVCF,".vcf",".unphased.beagle") - evalBeagle.bs = Some(percentLeftOut) + evalBeagle.bs = percentLeftOut evalBeagle.bsvcf = swapExt(outputVCF,".vcf",".missing.vcf") evalBeagle.missing = MISSING_KEY //evalBeagle.isIntermediate = true diff --git a/scala/qscript/oneoffs/depristo/VQSRCutByNRS.scala b/scala/qscript/oneoffs/depristo/VQSRCutByNRS.scala index 0dfcc6b06..7d5d961f1 100755 --- a/scala/qscript/oneoffs/depristo/VQSRCutByNRS.scala +++ b/scala/qscript/oneoffs/depristo/VQSRCutByNRS.scala @@ -1,4 +1,3 @@ -import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction import org.broadinstitute.sting.queue.extensions.gatk._ import org.broadinstitute.sting.queue.extensions.samtools.SamtoolsIndexFunction import org.broadinstitute.sting.queue.QScript @@ -18,7 +17,7 @@ class VQSRCutByNRS extends QScript { @Argument(fullName = "prefix", doc="Prefix argument", required=false) var prefix: String = "" - trait UNIVERSAL_GATK_ARGS extends CommandLineGATK { logging_level = "INFO"; jarFile = gatkJarFile; memoryLimit = Some(3) } + trait UNIVERSAL_GATK_ARGS extends CommandLineGATK { logging_level = "INFO"; jarFile = gatkJarFile; memoryLimit = 3 } class Target(val name: String, val reference: File, val rodName: String, val VCF: File, val intervals: Option[String], val titvTarget: Double) { def clusterFile = new File(name + ".clusters") @@ -67,9 +66,9 @@ class GenerateVariantClusters(t: Target) extends org.broadinstitute.sting.queue. this.use_annotation ++= List("QD", "SB", "HaplotypeScore", "HRun") this.analysisName = t.name + "_Cluster" if ( t.intervals != None ) this.intervalsString ++= List(t.intervals.get) - this.qual = Some(300) - this.std = Some(3.5) - this.mG = Some(16) // v2 calls + this.qual = 300 + this.std = 3.5 + this.mG = 16 // v2 calls // ignores this.ignoreFilter ++= FiltersToIgnore } @@ -86,8 +85,8 @@ class VariantRecalibratorBase(t: Target, ans: List[String]) extends org.broadins if ( t.intervals != None ) this.intervalsString ++= List(t.intervals.get) this.ignoreFilter ++= FiltersToIgnore this.ignoreFilter ++= List("HARD_TO_VALIDATE") - this.priorDBSNP = Some(2.0) - this.priorHapMap = Some(2.0) + this.priorDBSNP = 2.0 + this.priorHapMap = 2.0 this.target_titv = t.titvTarget this.use_annotation ++= ans this.out = new File("/dev/null") @@ -100,7 +99,7 @@ class VariantRecalibratorTiTv(t: Target, ans: List[String], prefix: String) exte } class VariantRecalibratorNRS(t: Target, ans: List[String], prefix: String) extends VariantRecalibratorBase(t,ans) { - this.sm = Some(org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY) + this.sm = org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY this.tranche ++= List("50", "25", "10", "5", "2", "1", "0.5", "0.1") //this.out = new File(t.name + ".ts.recalibrated.vcf") this.tranchesFile = new File(t.name + prefix + ".ts.tranches") diff --git a/scala/qscript/oneoffs/depristo/manySampleUGPerformance.scala b/scala/qscript/oneoffs/depristo/manySampleUGPerformance.scala index 33d7f16ea..1d9bd008c 100755 --- a/scala/qscript/oneoffs/depristo/manySampleUGPerformance.scala +++ b/scala/qscript/oneoffs/depristo/manySampleUGPerformance.scala @@ -29,7 +29,7 @@ class ManySampleUGPerformanceTesting extends QScript { this.intervals = List(new File(TARGET_INTERVAL)); this.reference_sequence = referenceFile; this.jobQueue = "gsa"; - this.memoryLimit = Some(4) + this.memoryLimit = 4 //this.commandDirectory = new File("results"); } @@ -52,11 +52,11 @@ class ManySampleUGPerformanceTesting extends QScript { // SNP calling //add(new Call(sublist.list, nSamples, "dynamic_merge")) val gt = new Call(bams, nSamples, name); - gt.exactCalculation = Some(org.broadinstitute.sting.gatk.walkers.genotyper.ExactAFCalculationModel.ExactCalculation.N2_GOLD_STANDARD) + gt.exactCalculation = org.broadinstitute.sting.gatk.walkers.genotyper.ExactAFCalculationModel.ExactCalculation.N2_GOLD_STANDARD add(gt) val gtLinear = new Call(bams, nSamples, name + "_linear"); - gtLinear.exactCalculation = Some(org.broadinstitute.sting.gatk.walkers.genotyper.ExactAFCalculationModel.ExactCalculation.LINEAR_EXPERIMENTAL) + gtLinear.exactCalculation = org.broadinstitute.sting.gatk.walkers.genotyper.ExactAFCalculationModel.ExactCalculation.LINEAR_EXPERIMENTAL add(gtLinear) // SNP calling -- no annotations @@ -74,24 +74,24 @@ class ManySampleUGPerformanceTesting extends QScript { } class MergeBAMs(bamList: File) extends PrintReads with UNIVERSAL_GATK_ARGS { - this.memoryLimit = Some(3) + this.memoryLimit = 3 this.input_file :+= bamList - this.memoryLimit = Some(16) + this.memoryLimit = 16 this.o = new File(MERGED_DIR + "/" + bamList.getName + ".bam") } class Call(@Input(doc="foo") bamList: File, n: Int, name: String) extends UnifiedGenotyper with UNIVERSAL_GATK_ARGS { @Output(doc="foo") var outVCF: File = new File("%s.%d.%s.vcf".format(bamList.getName, n, name)) this.input_file :+= bamList - this.stand_call_conf = Option(10.0) - this.dcov = Option(DCOV); + this.stand_call_conf = 10.0 + this.dcov = DCOV; this.o = outVCF } class MyCountLoci(@Input(doc="foo") bamList: File, n: Int, name: String) extends CountLoci with UNIVERSAL_GATK_ARGS { @Output(doc="foo") var outFile: File = new File("%s.%d.%s.txt".format(bamList.getName, n, name)) this.input_file :+= bamList - this.dcov = Option(DCOV); + this.dcov = DCOV; this.o = outFile } diff --git a/scala/qscript/oneoffs/depristo/resequencingSamples1KG.scala b/scala/qscript/oneoffs/depristo/resequencingSamples1KG.scala index 6393fa9a3..b17ccaafc 100644 --- a/scala/qscript/oneoffs/depristo/resequencingSamples1KG.scala +++ b/scala/qscript/oneoffs/depristo/resequencingSamples1KG.scala @@ -20,8 +20,8 @@ class resequencingSamples1KG extends QScript { this.intervals = List(new File(TARGET_INTERVAL)); this.reference_sequence = referenceFile; this.jobQueue = "gsa"; - this.et = Option(org.broadinstitute.sting.gatk.phonehome.GATKRunReport.PhoneHomeOption.STANDARD); - this.dcov = Option(50); + this.et = org.broadinstitute.sting.gatk.phonehome.GATKRunReport.PhoneHomeOption.STANDARD; + this.dcov = 50; } def script = { @@ -30,10 +30,10 @@ class resequencingSamples1KG extends QScript { } class MyQSample(@Input(doc="foo") bamList: File) extends QSample with UNIVERSAL_GATK_ARGS { - this.memoryLimit = Some(4) + this.memoryLimit = 4 this.input_file :+= bamList //this.BTI = "genotypes" - this.nt = Option(10) + this.nt = 10 this.rodBind :+= RodBind("genotypes", "VCF", HM3) this.o = new File("%s.qsample".format(bamList.getName)) } diff --git a/scala/qscript/oneoffs/fromer/PhaseSamples.scala b/scala/qscript/oneoffs/fromer/PhaseSamples.scala index 8f382b230..0c75b557e 100644 --- a/scala/qscript/oneoffs/fromer/PhaseSamples.scala +++ b/scala/qscript/oneoffs/fromer/PhaseSamples.scala @@ -38,7 +38,7 @@ class PhaseSamples extends QScript { this.intervalsString = List(qscript.intervals) this.jarFile = qscript.gatkJarFile this.reference_sequence = qscript.referenceFile - this.memoryLimit = Some(3) + this.memoryLimit = 3 this.logging_level = "INFO" } @@ -102,7 +102,7 @@ class PhaseSamples extends QScript { // add the master call: this.rodBind :+= RodBind("master", "VCF", masterCalls) - this.variantMergeOptions = Some(org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.MASTER) + this.variantMergeOptions = org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.MASTER this.out = outputPhased } diff --git a/scala/qscript/oneoffs/fromer/ReadDepthCNVanalysis.scala b/scala/qscript/oneoffs/fromer/ReadDepthCNVanalysis.scala index 6efd483d8..661aa0197 100644 --- a/scala/qscript/oneoffs/fromer/ReadDepthCNVanalysis.scala +++ b/scala/qscript/oneoffs/fromer/ReadDepthCNVanalysis.scala @@ -48,7 +48,7 @@ class ReadDepthCNVanalysis extends QScript { this.intervalsString = List(qscript.intervals) this.jarFile = qscript.gatkJarFile this.reference_sequence = qscript.referenceFile - //this.memoryLimit = Some(3) + //this.memoryLimit = 3 this.logging_level = "INFO" } @@ -98,8 +98,8 @@ class ReadDepthCNVanalysis extends QScript { this.input_file = t.bams - this.downsample_to_coverage = Some(MAX_DEPTH) - this.downsampling_type = Some(DownsampleType.BY_SAMPLE) + this.downsample_to_coverage = MAX_DEPTH + this.downsampling_type = DownsampleType.BY_SAMPLE this.scatterCount = scatterCountInput this.scatterClass = classOf[IntervalScatterFunction] @@ -132,6 +132,6 @@ class ReadDepthCNVanalysis extends QScript { def commandLine = command // Since loading ALL of the output into the perl script can take significant memory: - this.memoryLimit = Some(9) + this.memoryLimit = 9 } } \ No newline at end of file diff --git a/scala/qscript/oneoffs/hanna/DoC.scala b/scala/qscript/oneoffs/hanna/DoC.scala index 4982e1778..5a60f7166 100644 --- a/scala/qscript/oneoffs/hanna/DoC.scala +++ b/scala/qscript/oneoffs/hanna/DoC.scala @@ -35,14 +35,14 @@ class DoC extends QScript { this.jarFile = DoC.this.gatkJar this.reference_sequence = DoC.this.referenceFile this.intervalsString = DoC.this.intervalsString - this.memoryLimit = Some(8) + this.memoryLimit = 8 } def script = { // Create the four function that we can run. val doc = new DepthOfCoverage with DepthOfCoverageArguments - doc.downsampling_type = Some(DownsampleType.NONE) + doc.downsampling_type = DownsampleType.NONE doc.omitLocusTable = true doc.omitIntervals = true doc.omitSampleSummary = true diff --git a/scala/qscript/oneoffs/kshakir/UGMemoryTests.scala b/scala/qscript/oneoffs/kshakir/UGMemoryTests.scala index 119ea9839..132f12cc8 100644 --- a/scala/qscript/oneoffs/kshakir/UGMemoryTests.scala +++ b/scala/qscript/oneoffs/kshakir/UGMemoryTests.scala @@ -35,13 +35,13 @@ class UGMemoryTests extends QScript { snps.jobOutputFile = new File(dir, "UnifiedGenotyper.out") snps.out = new File(dir, "UnifiedGenotyper.vcf") snps.input_file = squid1Bams.take(numBams/2) ++ squid2Bams.take(numBams/2) - snps.memoryLimit = Some(memoryLimit) + snps.memoryLimit = memoryLimit snps.jarFile = qscript.gatkJar snps.reference_sequence = pipeline.getProject.getReferenceFile snps.intervals = List(pipeline.getProject.getIntervalList) snps.rodBind :+= new RodBind("dbsnp", pipeline.getProject.getGenotypeDbsnpType, pipeline.getProject.getGenotypeDbsnp) - snps.downsample_to_coverage = Some(qscript.downsampling_coverage) + snps.downsample_to_coverage = qscript.downsampling_coverage snps.annotation ++= List("AlleleBalance") snps.group :+= "Standard" diff --git a/scala/qscript/oneoffs/kshakir/linearindexbintests/LinearIndexBinTests.scala b/scala/qscript/oneoffs/kshakir/linearindexbintests/LinearIndexBinTests.scala index 5f73f6d2a..1b23c110d 100644 --- a/scala/qscript/oneoffs/kshakir/linearindexbintests/LinearIndexBinTests.scala +++ b/scala/qscript/oneoffs/kshakir/linearindexbintests/LinearIndexBinTests.scala @@ -49,10 +49,10 @@ class LinearIndexBinTests extends QScript { countRod.jarFile = qscript.gatkJar countRod.reference_sequence = reference - countRod.memoryLimit = Some(memoryLimit) + countRod.memoryLimit = memoryLimit // Some of the BED files don't have a chrM, which makes the GATK angry. Run unsafe. - countRod.U = Some(org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.ALL) + countRod.U = org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.ALL for ((rodFile, index) <- rodFiles.zipWithIndex) { val rodType = rodFile.getName.split("\\.").last diff --git a/scala/qscript/oneoffs/rpoplin/ASHGcalling.scala b/scala/qscript/oneoffs/rpoplin/ASHGcalling.scala index 831e1f7c1..f19215511 100755 --- a/scala/qscript/oneoffs/rpoplin/ASHGcalling.scala +++ b/scala/qscript/oneoffs/rpoplin/ASHGcalling.scala @@ -3,7 +3,6 @@ import org.broadinstitute.sting.datasources.pipeline.Pipeline import org.broadinstitute.sting.gatk.DownsampleType import org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeCalculationModel.Model import org.broadinstitute.sting.queue.extensions.gatk._ -import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction import org.broadinstitute.sting.queue.extensions.samtools._ import org.broadinstitute.sting.queue.{QException, QScript} import collection.JavaConversions._ @@ -42,7 +41,7 @@ class ASHGcalling extends QScript { trait CommandLineGATKArgs extends CommandLineGATK { this.jarFile = qscript.gatkJar this.reference_sequence = qscript.reference - this.memoryLimit = Some(2) + this.memoryLimit = 2 this.DBSNP = qscript.dbSNP this.jobTempDir = qscript.tmpDir } @@ -84,8 +83,8 @@ class ASHGcalling extends QScript { combineVariants = new CombineVariants with CommandLineGATKArgs combineVariants.rodBind = vcfChunks combineVariants.out = new TaggedFile(qscript.baseName + ".chr" + qscript.chr.toString + ".filtered.vcf", "vcf") - combineVariants.variantmergeoption = Some(org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.UNION) - combineVariants.genotypemergeoption = Some(org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.GenotypeMergeType.UNSORTED) + combineVariants.variantmergeoption = org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.UNION + combineVariants.genotypemergeoption = org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.GenotypeMergeType.UNSORTED combineVariants.setKey = "null" add(combineVariants) */ @@ -104,16 +103,16 @@ class ASHGcalling extends QScript { // 1.) Clean at known indels var clean = new IndelRealigner with CommandLineGATKArgs val cleanedBam = new File(baseTmpName + "cleaned.bam") - clean.memoryLimit = Some(4) + clean.memoryLimit = 4 clean.input_file :+= bamList clean.intervalsString :+= interval clean.targetIntervals = qscript.targetIntervals clean.out = cleanedBam clean.rodBind :+= RodBind("indels", "VCF", qscript.dindelCalls) clean.knownsOnly = true - clean.LOD = Some(1.0) + clean.LOD = 1.0 clean.sortInCoordinateOrderEvenThoughItIsHighlyUnsafe = true - clean.compress = Some(2) + clean.compress = 2 clean.jobName = baseName + population + ".clean" //clean.stripBam = true //clean.fileSystemUsage = "indium" @@ -121,7 +120,7 @@ class ASHGcalling extends QScript { // 2.) Apply BAQ calculation var baq = new SamtoolsBaqFunction val baqedBam = new File(baseTmpName + "cleaned.baq.bam") - baq.memoryLimit = Some(4) + baq.memoryLimit = 4 baq.in_bam = cleanedBam baq.out_bam = baqedBam baq.jobName = baseName + population + ".baq" @@ -154,22 +153,22 @@ class ASHGcalling extends QScript { } // 4.) Call with UGv2 - call.memoryLimit = Some(4) + call.memoryLimit = 4 call.intervalsString :+= interval call.out = rawCalls - call.dcov = Some(50) - call.standard_min_confidence_threshold_for_calling = Some(50) - call.standard_min_confidence_threshold_for_emitting = Some(30) - call.min_mapping_quality_score = Some(20) - call.min_base_quality_score = Some(20) - call.pnrm = Some(org.broadinstitute.sting.playground.gatk.walkers.genotyper.AlleleFrequencyCalculationModel.Model.GRID_SEARCH) + call.dcov = 50 + call.standard_min_confidence_threshold_for_calling = 50 + call.standard_min_confidence_threshold_for_emitting = 30 + call.min_mapping_quality_score = 20 + call.min_base_quality_score = 20 + call.pnrm = org.broadinstitute.sting.playground.gatk.walkers.genotyper.AlleleFrequencyCalculationModel.Model.GRID_SEARCH call.jobName = baseName + "call" //call.fileSystemUsage = "iodine" // 5b.) Filter near indels and HARD_TO_VALIDATE var filter = new VariantFiltration with CommandLineGATKArgs val filteredCalls = new File(baseName + "filtered.vcf") - filter.memoryLimit = Some(1) + filter.memoryLimit = 1 filter.out = filteredCalls filter.intervalsString :+= interval filter.variantVCF = rawCalls diff --git a/scala/qscript/oneoffs/rpoplin/Phase1Calling.scala b/scala/qscript/oneoffs/rpoplin/Phase1Calling.scala index f4698b4b8..2dfb1c8cf 100755 --- a/scala/qscript/oneoffs/rpoplin/Phase1Calling.scala +++ b/scala/qscript/oneoffs/rpoplin/Phase1Calling.scala @@ -2,7 +2,6 @@ import net.sf.picard.reference.FastaSequenceFile import org.broadinstitute.sting.datasources.pipeline.Pipeline import org.broadinstitute.sting.gatk.DownsampleType import org.broadinstitute.sting.queue.extensions.gatk._ -import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction import org.broadinstitute.sting.queue.extensions.samtools._ import org.broadinstitute.sting.queue.{QException, QScript} import collection.JavaConversions._ @@ -51,7 +50,7 @@ class Phase1Calling extends QScript { trait CommandLineGATKArgs extends CommandLineGATK { this.jarFile = qscript.gatkJar this.reference_sequence = qscript.reference - this.memoryLimit = Some(3) + this.memoryLimit = 3 this.jobTempDir = qscript.tmpDir this.DBSNP = qscript.dbSNP } @@ -79,12 +78,12 @@ class Phase1Calling extends QScript { var call = new UnifiedGenotyper with CommandLineGATKArgs call.intervalsString ++= List(qscript.intervals) call.scatterCount = 63 // the smallest interval list has 63 intervals, one for each Mb on chr20 - call.dcov = Some( 50 ) - call.stand_call_conf = Some( 4.0 ) - call.stand_emit_conf = Some( 4.0 ) + call.dcov = 50 + call.stand_call_conf = 4.0 + call.stand_emit_conf = 4.0 call.input_file :+= bamList call.out = rawCalls - call.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY) + call.baq = org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY call.analysisName = baseName + "_UG" var filter = new VariantFiltration with CommandLineGATKArgs @@ -106,9 +105,9 @@ class Phase1Calling extends QScript { gvc.use_annotation ++= List("QD", "SB", "HaplotypeScore", "HRun") gvc.analysisName = baseName + "_GVC" gvc.intervalsString ++= List(qscript.intervals) - gvc.qual = Some(100) // clustering parameters to be updated soon pending new experimentation results - gvc.std = Some(4.5) - gvc.mG = Some(6) + gvc.qual = 100 // clustering parameters to be updated soon pending new experimentation results + gvc.std = 4.5 + gvc.mG = 6 var vr = new VariantRecalibrator with CommandLineGATKArgs vr.rodBind :+= RodBind("1kg", "VCF", qscript.omni) @@ -120,13 +119,13 @@ class Phase1Calling extends QScript { vr.analysisName = baseName + "_VR" vr.intervalsString ++= List(qscript.intervals) vr.ignoreFilter ++= List("HARD_TO_VALIDATE") - vr.target_titv = Some(2.3) - vr.sm = Some(org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY) + vr.target_titv = 2.3 + vr.sm = org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY vr.tranche ++= List("0.1", "1.0", "2.0", "3.0", "5.0", "10.0", "100.0") vr.out = recalibratedCalls - vr.priorDBSNP = Some(10.0) - vr.priorHapMap = Some(12.0) - vr.prior1KG = Some(12.0) + vr.priorDBSNP = 10.0 + vr.priorHapMap = 12.0 + vr.prior1KG = 12.0 vr.tranchesFile = tranchesFile add(call, filter, gvc, vr) diff --git a/scala/qscript/oneoffs/rpoplin/Phase1Cleaning.scala b/scala/qscript/oneoffs/rpoplin/Phase1Cleaning.scala index 44269e70d..ca42ed68b 100755 --- a/scala/qscript/oneoffs/rpoplin/Phase1Cleaning.scala +++ b/scala/qscript/oneoffs/rpoplin/Phase1Cleaning.scala @@ -2,7 +2,6 @@ import net.sf.picard.reference.FastaSequenceFile import org.broadinstitute.sting.datasources.pipeline.Pipeline import org.broadinstitute.sting.gatk.DownsampleType import org.broadinstitute.sting.queue.extensions.gatk._ -import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction import org.broadinstitute.sting.queue.extensions.samtools._ import org.broadinstitute.sting.queue.{QException, QScript} import collection.JavaConversions._ @@ -43,7 +42,7 @@ class Phase1Cleaning extends QScript { trait CommandLineGATKArgs extends CommandLineGATK { this.jarFile = qscript.gatkJar this.reference_sequence = qscript.reference - this.memoryLimit = Some(2) + this.memoryLimit = 2 this.jobTempDir = qscript.tmpDir } @@ -62,11 +61,11 @@ class Phase1Cleaning extends QScript { // 1.) Create cleaning targets var target = new RealignerTargetCreator with CommandLineGATKArgs - target.memoryLimit = Some(4) + target.memoryLimit = 4 target.input_file :+= bamList target.intervalsString :+= interval target.out = targetIntervals - target.mismatchFraction = Some(0.0) + target.mismatchFraction = 0.0 target.rodBind :+= RodBind("dbsnp", "VCF", qscript.dbSNP) target.rodBind :+= RodBind("indels1", "VCF", qscript.dindelPilotCalls) target.rodBind :+= RodBind("indels2", "VCF", qscript.dindelAFRCalls) @@ -77,13 +76,13 @@ class Phase1Cleaning extends QScript { // 2.) Clean without SW var clean = new IndelRealigner with CommandLineGATKArgs val cleanedBam = new File(baseTmpName + "cleaned.bam") - clean.memoryLimit = Some(4) + clean.memoryLimit = 4 clean.input_file :+= bamList clean.intervalsString :+= interval clean.targetIntervals = targetIntervals clean.out = cleanedBam clean.doNotUseSW = true - clean.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.RECALCULATE) + clean.baq = org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.RECALCULATE clean.rodBind :+= RodBind("dbsnp", "VCF", qscript.dbSNP) clean.rodBind :+= RodBind("indels1", "VCF", qscript.dindelPilotCalls) clean.rodBind :+= RodBind("indels2", "VCF", qscript.dindelAFRCalls) diff --git a/scala/qscript/oneoffs/rpoplin/VQSR_parameterSearch.scala b/scala/qscript/oneoffs/rpoplin/VQSR_parameterSearch.scala index 96a059998..91e00e8f6 100755 --- a/scala/qscript/oneoffs/rpoplin/VQSR_parameterSearch.scala +++ b/scala/qscript/oneoffs/rpoplin/VQSR_parameterSearch.scala @@ -1,4 +1,3 @@ -import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction import org.broadinstitute.sting.queue.extensions.gatk._ import org.broadinstitute.sting.queue.extensions.samtools.SamtoolsIndexFunction import org.broadinstitute.sting.queue.QScript @@ -19,7 +18,7 @@ class VQSR_parameterSearch extends QScript { @Argument(shortName="skipCalling", doc="If true, skip the calling part of the pipeline and only run VQSR on preset, gold standard VCF files", required=false) var skipCalling: Boolean = false - trait UNIVERSAL_GATK_ARGS extends CommandLineGATK { logging_level = "INFO"; jarFile = gatkJarFile; memoryLimit = Some(2); } + trait UNIVERSAL_GATK_ARGS extends CommandLineGATK { logging_level = "INFO"; jarFile = gatkJarFile; memoryLimit = 2; } class Target(val baseName: String, val reference: File, val rodName: String, val bamList: File, val goldStandard_VCF: File, val intervals: String, val titvTarget: Double, val isLowpass: Boolean) { def name = qscript.outputDir + baseName @@ -309,12 +308,12 @@ class VQSR_parameterSearch extends QScript { } this.analysisName = name + "_GVC" this.intervalsString ++= List(t.intervals) - this.qual = Some(t.qualCutoff) - this.std = Some(t.std) - this.mG = Some(t.gaussian) + this.qual = t.qualCutoff + this.std = t.std + this.mG = t.gaussian this.ignoreFilter ++= FiltersToIgnore - this.dirichlet = Some(t.dirichlet) - this.shrinkage = Some(t.shrinkage) + this.dirichlet = t.dirichlet + this.shrinkage = t.shrinkage } // 4.) VQSR part2 Calculate new LOD for all input SNPs by evaluating the Gaussian clusters @@ -336,8 +335,8 @@ class VQSR_parameterSearch extends QScript { this.intervalsString ++= List(t.intervals) this.ignoreFilter ++= FiltersToIgnore this.ignoreFilter ++= List("HARD_TO_VALIDATE") - this.target_titv = Some(t.titvTarget) - this.backOff = Some(t.backoff) + this.target_titv = t.titvTarget + this.backOff = t.backoff } // 4a.) Choose VQSR tranches based on novel ti/tv @@ -350,7 +349,7 @@ class VQSR_parameterSearch extends QScript { // 4b.) Choose VQSR tranches based on sensitivity to truth set class VariantRecalibratorNRS(t: Target, goldStandard: Boolean) extends VariantRecalibratorBase(t, goldStandard) { - this.sm = Some(org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY) + this.sm = org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY if(t.trainOmni == 0 ) { this.tranche ++= List("1.0") } else { diff --git a/scala/qscript/playground/BatchMerge.q b/scala/qscript/playground/BatchMerge.q index e0102cd70..5bf789a30 100755 --- a/scala/qscript/playground/BatchMerge.q +++ b/scala/qscript/playground/BatchMerge.q @@ -48,22 +48,22 @@ class batchMergePipeline extends QScript { trait CalcLikelihoodArgs extends UGCalcLikelihoods { this.reference_sequence = batchMerge.ref - this.max_mismatches_in_40bp_window = Some(batchMerge.mmb) - this.min_base_quality_score = Some(batchMerge.mbq) - this.min_mapping_quality_score = Some(batchMerge.mmq) + this.max_mismatches_in_40bp_window = batchMerge.mmb + this.min_base_quality_score = batchMerge.mbq + this.min_mapping_quality_score = batchMerge.mmq if ( batchMerge.baq >= 0 ) { - this.baqGapOpenPenalty = Some(batchMerge.baq) - this.baq = Some(BAQ.CalculationMode.CALCULATE_AS_NECESSARY) + this.baqGapOpenPenalty = batchMerge.baq + this.baq = BAQ.CalculationMode.CALCULATE_AS_NECESSARY } this.intervals :+= extractIntervals.listOut this.alleleVCF = combineVCFs.outVCF this.jarFile = new File(stingDir+"/dist/GenomeAnalysisTK.jar") - this.memoryLimit = Some(4) + this.memoryLimit = 4 this.scatterCount = 60 - this.output_mode = Some(UnifiedGenotyperEngine.OUTPUT_MODE.EMIT_ALL_SITES) - this.genotyping_mode = Some(GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES) + this.output_mode = UnifiedGenotyperEngine.OUTPUT_MODE.EMIT_ALL_SITES + this.genotyping_mode = GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES if ( batchMerge.indelMode ) { - this.genotype_likelihoods_model = Some(GenotypeLikelihoodsCalculationModel.Model.DINDEL) + this.genotype_likelihoods_model = GenotypeLikelihoodsCalculationModel.Model.DINDEL } } @@ -82,11 +82,11 @@ class batchMergePipeline extends QScript { this.intervals :+= extractIntervals.listOut this.jarFile = new File(stingDir+"/dist/GenomeAnalysisTK.jar") this.scatterCount = 30 - this.memoryLimit = Some(8) - this.output_mode = Some(UnifiedGenotyperEngine.OUTPUT_MODE.EMIT_ALL_SITES) - this.genotyping_mode = Some(GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES) + this.memoryLimit = 8 + this.output_mode = UnifiedGenotyperEngine.OUTPUT_MODE.EMIT_ALL_SITES + this.genotyping_mode = GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES if ( batchMerge.indelMode ) { - this.genotype_likelihoods_model = Some(GenotypeLikelihoodsCalculationModel.Model.DINDEL) + this.genotype_likelihoods_model = GenotypeLikelihoodsCalculationModel.Model.DINDEL } } @@ -100,7 +100,7 @@ class batchMergePipeline extends QScript { this.intervals :+= extractIntervals.listOut this.jarFile = new File(batchMerge.stingDir+"/dist/GenomeAnalysisTK.jar") this.scatterCount = 10 - this.memoryLimit=Some(4) + this.memoryLimit=4 } var combine : CombineVariants = new CombineVariants with CombineVariantsArgs diff --git a/scala/qscript/playground/FullCallingPipeline.q b/scala/qscript/playground/FullCallingPipeline.q index 54726160d..0c87e0c7a 100755 --- a/scala/qscript/playground/FullCallingPipeline.q +++ b/scala/qscript/playground/FullCallingPipeline.q @@ -1,7 +1,7 @@ import org.broadinstitute.sting.commandline.ArgumentSource import org.broadinstitute.sting.datasources.pipeline.Pipeline import org.broadinstitute.sting.queue.extensions.gatk._ -import org.broadinstitute.sting.queue.extensions.picard.PicardBamEmbeddedFunction +import org.broadinstitute.sting.queue.extensions.picard.PicardBamFunction import org.broadinstitute.sting.queue.extensions.samtools._ import org.broadinstitute.sting.queue.function.ListWriterFunction import org.broadinstitute.sting.queue.function.scattergather.{GatherFunction, CloneFunction, ScatterFunction} @@ -42,7 +42,7 @@ class FullCallingPipeline extends QScript { this.intervals = List(qscript.pipeline.getProject.getIntervalList) this.jarFile = qscript.gatkJar this.reference_sequence = qscript.pipeline.getProject.getReferenceFile - this.memoryLimit = Some(4) + this.memoryLimit = 4 } @@ -77,7 +77,7 @@ class FullCallingPipeline extends QScript { targetCreator.analysisName = "CreateTargets_"+sampleId targetCreator.input_file :+= bam targetCreator.out = indel_targets - targetCreator.memoryLimit = Some(2) + targetCreator.memoryLimit = 2 targetCreator.isIntermediate = true val realigner = new IndelRealigner with CommandLineGATKArgs @@ -108,8 +108,8 @@ class FullCallingPipeline extends QScript { case (gather: BamGatherFunction, source: ArgumentSource) => gather.commandDirectory = new File("CleanedBams/IntermediateFiles/%s/ScatterGather/Gather_%s".format(sampleId, source.field.getName)) gather.jobOutputFile = new File(".queue/logs/Cleaning/%s/FixMates.out".format(sampleId)) - gather.memoryLimit = Some(6) - gather.mainClass = picardFixMatesClass + gather.memoryLimit = 6 + gather.javaMainClass = picardFixMatesClass gather.assumeSorted = None case (gather: GatherFunction, source: ArgumentSource) => gather.commandDirectory = new File("CleanedBams/IntermediateFiles/%s/ScatterGather/Gather_%s".format(sampleId, source.field.getName)) @@ -122,7 +122,7 @@ class FullCallingPipeline extends QScript { realigner.isIntermediate = true // Explicitly run fix mates if the function won't be scattered. - val fixMates = new PicardBamEmbeddedFunction { + val fixMates = new PicardBamFunction { @Input(doc="unfixed bam") var unfixed: File = _ @Output(doc="fixed bam") var fixed: File = _ def inputBams = List(unfixed) @@ -130,8 +130,8 @@ class FullCallingPipeline extends QScript { } fixMates.jobOutputFile = new File(".queue/logs/Cleaning/%s/FixMates.out".format(sampleId)) - fixMates.memoryLimit = Some(6) - fixMates.mainClass = picardFixMatesClass + fixMates.memoryLimit = 6 + fixMates.javaMainClass = picardFixMatesClass fixMates.unfixed = realigner.out fixMates.fixed = cleaned_bam fixMates.analysisName = "FixMates_"+sampleId @@ -173,9 +173,9 @@ class FullCallingPipeline extends QScript { val indels = new UnifiedGenotyper with CommandLineGATKArgs with ExpandedIntervals indels.analysisName = base + "_indels" indels.jobOutputFile = new File(".queue/logs/IndelCalling/UnifiedGenotyper.indels.out") - indels.memoryLimit = Some(6) - indels.downsample_to_coverage = Some(600) - indels.genotype_likelihoods_model = Option(org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeLikelihoodsCalculationModel.Model.DINDEL) + indels.memoryLimit = 6 + indels.downsample_to_coverage = 600 + indels.genotype_likelihoods_model = org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeLikelihoodsCalculationModel.Model.DINDEL indels.input_file = bamFiles indels.rodBind :+= RodBind("dbsnp", qscript.pipeline.getProject.getGenotypeDbsnpType, qscript.pipeline.getProject.getGenotypeDbsnp) indels.out = new File("IndelCalls", base+".indels.vcf") @@ -210,8 +210,8 @@ class FullCallingPipeline extends QScript { val snps = new UnifiedGenotyper with CommandLineGATKArgs with ExpandedIntervals snps.analysisName = base+"_snps" snps.jobOutputFile = new File(".queue/logs/SNPCalling/UnifiedGenotyper.snps.out") - snps.memoryLimit = Some(6) - snps.downsample_to_coverage = Some(600) + snps.memoryLimit = 6 + snps.downsample_to_coverage = 600 snps.input_file = bamFiles snps.rodBind :+= RodBind("dbsnp", qscript.pipeline.getProject.getGenotypeDbsnpType, qscript.pipeline.getProject.getGenotypeDbsnp) snps.out = new File("SnpCalls", base+".snps.vcf") @@ -239,8 +239,8 @@ class FullCallingPipeline extends QScript { filteredSNPs.jobOutputFile = new File(".queue/logs/SNPCalling/VariantFiltration.snps.out") filteredSNPs.filterName ++= List("SNPSBFilter","SNPQDFilter","SNPHRunFilter") filteredSNPs.filterExpression ++= List("\"SB>=0.10\"","\"QD<5.0\"","\"HRun>=4\"") - filteredSNPs.clusterWindowSize = Some(10) - filteredSNPs.clusterSize = Some(3) + filteredSNPs.clusterWindowSize = 10 + filteredSNPs.clusterSize = 3 filteredSNPs.rodBind :+= RodBind("mask", "VCF", filteredIndels.out) filteredSNPs.variantVCF = snps.out filteredSNPs.out = swapExt("SnpCalls",snps.out,".vcf",".filtered.vcf") @@ -249,7 +249,7 @@ class FullCallingPipeline extends QScript { val combineAll = new CombineVariants with CommandLineGATKArgs with ExpandedIntervals combineAll.analysisName = base + "_combineAll" combineAll.jobOutputFile = new File(".queue/logs/Combined/CombineVariants.out") - combineAll.variantMergeOptions = Option(org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.UNION) + combineAll.variantMergeOptions = org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.UNION combineAll.rod_priority_list = "Indels,SNPs" combineAll.rodBind :+= RodBind("Indels", "VCF", filteredIndels.out) combineAll.rodBind :+= RodBind("SNPs", "VCF", filteredSNPs.out) diff --git a/scala/qscript/playground/MultiFullCallingPipeline.scala b/scala/qscript/playground/MultiFullCallingPipeline.scala index dde4cd9db..bbcb8eea7 100644 --- a/scala/qscript/playground/MultiFullCallingPipeline.scala +++ b/scala/qscript/playground/MultiFullCallingPipeline.scala @@ -1,5 +1,5 @@ import collection.JavaConversions -import org.broadinstitute.sting.queue.function.JarCommandLineFunction +import org.broadinstitute.sting.queue.function.JavaCommandLineFunction import org.broadinstitute.sting.queue.QScript import org.broadinstitute.sting.queue.util.IOUtils import org.broadinstitute.sting.utils.text.XReadLines @@ -69,7 +69,7 @@ class MultiFullCallingPipeline extends QScript { * Runs a yaml in a pipeline only after a previous pipeline * run has produced the passed in output file. */ - class RunPipeline(yamlFile: File, lastOutput: File) extends JarCommandLineFunction { + class RunPipeline(yamlFile: File, lastOutput: File) extends JavaCommandLineFunction { private var yamlName = yamlFile.getName.stripSuffix(".yaml") @Input(doc="output file to wait for", required=false) @@ -81,7 +81,7 @@ class MultiFullCallingPipeline extends QScript { commandDirectory = yamlFile.getParentFile jobOutputFile = IOUtils.absolute(commandDirectory, yamlName + ".queue.txt") jarFile = queueJar - memoryLimit = Some(1) + memoryLimit = 1 override def commandLine = super.commandLine + optional(" -statusTo ", qscript.pipelineStatusTo) + diff --git a/scala/qscript/playground/recalibrate.scala b/scala/qscript/playground/recalibrate.scala index 5026a3bee..f57991fee 100755 --- a/scala/qscript/playground/recalibrate.scala +++ b/scala/qscript/playground/recalibrate.scala @@ -1,4 +1,3 @@ -import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction import org.broadinstitute.sting.queue.extensions.gatk._ import org.broadinstitute.sting.queue.extensions.samtools.SamtoolsIndexFunction import org.broadinstitute.sting.queue.QScript @@ -23,9 +22,6 @@ class recalibrate extends QScript { @Argument(shortName = "R", doc="ref") var referenceFile: File = _ - @Argument(doc="X", required=false) - var picardMergeSamFilesJar: File = new File("/seq/software/picard/current/bin/MergeSamFiles.jar") - trait UNIVERSAL_GATK_ARGS extends CommandLineGATK { logging_level = "INFO"; jarFile = gatkJarFile; reference_sequence = referenceFile; } def script = { @@ -40,13 +36,11 @@ def script = { val tableRecal = new TableRecalibrate(bamIn, recalData, recalBam) { useOriginalQualities = true } if ( scatter ) { tableRecal.intervals = List(new File("/humgen/gsa-hpprojects/GATK/data/chromosomes.hg18.interval_list")) - //tableRecal.scatterClass = classOf[ContigScatterFunction] - tableRecal.setupGatherFunction = { case (f: PicardBamJarFunction, _) => f.jarFile = picardMergeSamFilesJar; f.memoryLimit = Some(4) } tableRecal.scatterCount = 25 } add(tableRecal) add(new Index(recalBam)) - add(new CountCovariates(recalBam, recalRecalData) { num_threads = Some(4) }) + add(new CountCovariates(recalBam, recalRecalData) { num_threads = 4 }) add(new AnalyzeCovariates(recalData, new File(recalData.getPath() + ".analyzeCovariates"))) add(new AnalyzeCovariates(recalRecalData, new File(recalRecalData.getPath() + ".analyzeCovariates"))) } @@ -65,7 +59,7 @@ class CountCovariates(bamIn: File, recalDataIn: File) extends org.broadinstitute this.DBSNP = new File("/humgen/gsa-hpprojects/GATK/data/dbsnp_129_hg18.rod") this.logging_level = "INFO" this.covariate ++= List("ReadGroupCovariate", "QualityScoreCovariate", "CycleCovariate", "DinucCovariate") - this.memoryLimit = Some(3) + this.memoryLimit = 3 override def dotString = "CountCovariates: %s [args %s]".format(bamIn.getName, if (this.num_threads.isDefined) "-nt " + this.num_threads else "") } @@ -76,7 +70,7 @@ class TableRecalibrate(bamInArg: File, recalDataIn: File, bamOutArg: File) exten this.recal_file = recalDataIn this.out = bamOutArg this.logging_level = "INFO" - this.memoryLimit = Some(2) + this.memoryLimit = 2 this.skipUQUpdate = skipUQUpdateArg override def dotString = "TableRecalibrate: %s => %s".format(bamInArg.getName, bamOutArg.getName, if (this.useOriginalQualities) " -OQ" else "") @@ -87,9 +81,9 @@ class AnalyzeCovariates(recalDataIn: File, outputDir: File) extends org.broadin this.recal_file = recalDataIn this.output_dir = outputDir.toString this.path_to_resources = "/home/radon01/depristo/dev/GenomeAnalysisTK/trunk/R/" - this.ignoreQ = Some(5) + this.ignoreQ = 5 this.path_to_Rscript = "/broad/software/free/Linux/redhat_5_x86_64/pkgs/r_2.7.2/bin/Rscript" - this.memoryLimit = Some(2) + this.memoryLimit = 2 override def dotString = "AnalyzeCovariates: %s".format(recalDataIn.getName) } diff --git a/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala b/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala index c2ea1ec17..5fb55601a 100755 --- a/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala +++ b/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue import function.QFunction @@ -10,6 +34,37 @@ import collection.JavaConversions._ import org.broadinstitute.sting.utils.classloader.PluginManager import org.broadinstitute.sting.utils.exceptions.UserException +/** + * Entry point of Queue. Compiles and runs QScripts passed in to the command line. + */ +object QCommandLine { + /** + * Main. + * @param argv Arguments. + */ + def main(argv: Array[String]) { + val qCommandLine = new QCommandLine + + Runtime.getRuntime.addShutdownHook(new Thread { + /** Cleanup as the JVM shuts down. */ + override def run = { + ProcessController.shutdown() + qCommandLine.shutdown() + } + }) + + try { + CommandLineProgram.start(qCommandLine, argv); + if (CommandLineProgram.result != 0) + System.exit(CommandLineProgram.result); + } catch { + case e: Exception => CommandLineProgram.exitSystemWithError(e) + } finally { + + } + } +} + /** * Entry point of Queue. Compiles and runs QScripts passed in to the command line. */ @@ -99,34 +154,3 @@ class QCommandLine extends CommandLineProgram with Logging { if (qScriptClasses != null) IOUtils.tryDelete(qScriptClasses) } } - -/** - * Entry point of Queue. Compiles and runs QScripts passed in to the command line. - */ -object QCommandLine { - /** - * Main. - * @param argv Arguments. - */ - def main(argv: Array[String]) { - val qCommandLine = new QCommandLine - - Runtime.getRuntime.addShutdownHook(new Thread { - /** Cleanup as the JVM shuts down. */ - override def run = { - ProcessController.shutdown() - qCommandLine.shutdown() - } - }) - - try { - CommandLineProgram.start(qCommandLine, argv); - if (CommandLineProgram.result != 0) - System.exit(CommandLineProgram.result); - } catch { - case e: Exception => CommandLineProgram.exitSystemWithError(e) - } finally { - - } - } -} diff --git a/scala/src/org/broadinstitute/sting/queue/QScript.scala b/scala/src/org/broadinstitute/sting/queue/QScript.scala index 9934d5e0e..2b509d765 100755 --- a/scala/src/org/broadinstitute/sting/queue/QScript.scala +++ b/scala/src/org/broadinstitute/sting/queue/QScript.scala @@ -24,7 +24,7 @@ package org.broadinstitute.sting.queue -import org.broadinstitute.sting.queue.util.Logging +import util.{PrimitiveOptionConversions, Logging} import org.broadinstitute.sting.queue.function.QFunction import org.broadinstitute.sting.utils.text.XReadLines import annotation.target.field @@ -32,7 +32,8 @@ import annotation.target.field /** * Defines a Queue pipeline as a collection of CommandLineFunctions. */ -trait QScript extends Logging { +trait QScript extends Logging with PrimitiveOptionConversions { + // Type aliases so users don't have to import type File = java.io.File type CommandLineFunction = org.broadinstitute.sting.queue.function.CommandLineFunction @@ -49,7 +50,7 @@ trait QScript extends Logging { type Output = org.broadinstitute.sting.commandline.Output @field type Argument = org.broadinstitute.sting.commandline.Argument @field type ArgumentCollection = org.broadinstitute.sting.commandline.ArgumentCollection @field - type Gather = org.broadinstitute.sting.queue.function.scattergather.Gather @field + type Gather = org.broadinstitute.sting.commandline.Gather @field /** * Builds the CommandLineFunctions that will be used to run this script and adds them to this.functions directly or using the add() utility method. diff --git a/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala b/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala index 1ec8f44a6..a3f8f76a2 100755 --- a/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala +++ b/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala @@ -318,7 +318,7 @@ class QGraph extends Logging { var readyJobs = getReadyJobs() while (running && readyJobs.size > 0) { logger.debug("+++++++") - readyJobs.foreach(edge => { + foreachFunction(readyJobs.toList, edge => { if (running) { logEdge(edge) edge.markAsDone @@ -870,12 +870,17 @@ class QGraph extends Logging { * Utility function for running a method over all function edges. * @param edgeFunction Function to run for each FunctionEdge. */ - private def foreachFunction(f: (FunctionEdge) => Unit) = { - jobGraph.edgeSet.toList - .filter(_.isInstanceOf[FunctionEdge]) - .asInstanceOf[List[FunctionEdge]] - .sorted(functionOrdering) - .foreach(edge => if (running) f(edge)) + private def foreachFunction(f: (FunctionEdge) => Unit) { + foreachFunction(jobGraph.edgeSet.toList.filter(_.isInstanceOf[FunctionEdge]).asInstanceOf[List[FunctionEdge]], f) + } + + /** + * Utility function for running a method over a list of function edges. + * @param edegs Edges to traverse. + * @param edgeFunction Function to run for each FunctionEdge. + */ + private def foreachFunction(edges: List[FunctionEdge], f: (FunctionEdge) => Unit) { + edges.sorted(functionOrdering).foreach(edge => if (running) f(edge)) } /** diff --git a/scala/src/org/broadinstitute/sting/queue/extensions/firehose/ImportSingleValueFunction.scala b/scala/src/org/broadinstitute/sting/queue/extensions/firehose/ImportSingleValueFunction.scala deleted file mode 100644 index 07f158224..000000000 --- a/scala/src/org/broadinstitute/sting/queue/extensions/firehose/ImportSingleValueFunction.scala +++ /dev/null @@ -1,45 +0,0 @@ -package org.broadinstitute.sting.queue.extensions.firehose - -import org.broadinstitute.sting.queue.function.JarCommandLineFunction -import org.broadinstitute.sting.commandline.{Input, Argument} -import java.io.File - -/** - * Runs the Firehose ImportSingleValue jar file. - */ -class ImportSingleValueFunction extends JarCommandLineFunction { - @Argument(doc="firehose host") - var host: String = _ - - @Argument(doc="firehose port") - var port: Int = _ - - @Argument(doc="firehose domain") - var domain: String = _ - - @Argument(doc="firehose entity type") - var entityType: String = _ - - @Argument(doc="firehose entity id") - var entityID: String = _ - - @Argument(doc="firehose annotation type name", shortName="bamFHAnn", required=false) - var annotationTypeName: String = _ - - @Argument(doc="clean bam firehose security token", shortName="bamFHToken", required=false) - var securityToken: String = _ - - @Input(doc="imports the path to this file", exclusiveOf="importValueInFile") - var importValue: File = _ - - @Input(doc="imports the value contained in the file", exclusiveOf="importValue") - var importValueInFile: File = _ - - override def commandLine = super.commandLine + ("" + - " PORT=%s HOST=%s DOMAIN=%s ENTITY_TYPE=%s" + - " ENTITY_ID=%s ANNOTATION_TYPE_NAME=%s SECURITY_TOKEN=%s" + - "%s%s" - ).format( - port, host, domain, entityType, entityID, annotationTypeName, securityToken, - optional(" VALUE=", importValue), optional(" VALUE_FILE=", importValueInFile)) -} diff --git a/scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala b/scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala index 67dfbba64..91b9ae3fb 100644 --- a/scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala @@ -1,13 +1,37 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.extensions.gatk import org.broadinstitute.sting.queue.function.scattergather.GatherFunction -import org.broadinstitute.sting.queue.extensions.picard.PicardBamEmbeddedFunction +import org.broadinstitute.sting.queue.extensions.picard.PicardBamFunction /** * Merges BAM files using Picards net.sf.picard.sam.MergeSamFiles. */ -class BamGatherFunction extends GatherFunction with PicardBamEmbeddedFunction { - this.mainClass = "net.sf.picard.sam.MergeSamFiles" +class BamGatherFunction extends GatherFunction with PicardBamFunction { + this.javaMainClass = "net.sf.picard.sam.MergeSamFiles" this.assumeSorted = Some(true) protected def inputBams = gatherParts protected def outputBam = originalOutput diff --git a/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKScatterFunction.scala b/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKScatterFunction.scala index f753cd341..4dbd7c880 100644 --- a/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKScatterFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKScatterFunction.scala @@ -69,7 +69,7 @@ trait GATKScatterFunction extends ScatterFunction { } override def isScatterGatherable = { - if (this.originalGATK.BTI != null && this.originalGATK.BTIMR.isEmpty) + if (this.originalGATK.BTI != null && this.originalGATK.BTIMR == null) throw new IllegalArgumentException("BTI requires BTIMR for use with scatter-gather (recommended: INTERSECTION)") this.originalGATK.reference_sequence != null } diff --git a/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamFunction.scala b/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamFunction.scala index 45c51c60c..386166421 100644 --- a/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamFunction.scala @@ -25,7 +25,9 @@ package org.broadinstitute.sting.queue.extensions.picard import java.io.File -import org.broadinstitute.sting.queue.function.CommandLineFunction +import org.broadinstitute.sting.queue.function.JavaCommandLineFunction +import net.sf.samtools.SAMFileReader.ValidationStringency +import net.sf.samtools.SAMFileHeader.SortOrder /** * Wraps a Picard function that operates on BAM files. @@ -34,9 +36,9 @@ import org.broadinstitute.sting.queue.function.CommandLineFunction * Since the various BAM utilities take slightly different arguments * some values are optional. */ -trait PicardBamFunction extends CommandLineFunction { - var validationStringency = "SILENT" - var sortOrder = "coordinate" +trait PicardBamFunction extends JavaCommandLineFunction { + var validationStringency: ValidationStringency = ValidationStringency.SILENT + var sortOrder: SortOrder = SortOrder.coordinate var compressionLevel: Option[Int] = None var maxRecordsInRam: Option[Int] = None var assumeSorted: Option[Boolean] = None @@ -45,7 +47,13 @@ trait PicardBamFunction extends CommandLineFunction { protected def outputBam: File abstract override def commandLine = super.commandLine + - Array(optional(" COMPRESSION_LEVEL=", compressionLevel), optional(" VALIDATION_STRINGENCY=", validationStringency), - optional(" SO=", sortOrder), optional( " MAX_RECORDS_IN_RAM=", maxRecordsInRam), optional(" ASSUME_SORTED=", assumeSorted), - " OUTPUT=" + outputBam, repeat(" INPUT=", inputBams), " TMP_DIR=" + jobTempDir).mkString + Array( + repeat(" INPUT=", inputBams), + " OUTPUT=" + outputBam, + " TMP_DIR=" + jobTempDir, + optional(" COMPRESSION_LEVEL=", compressionLevel), + optional(" VALIDATION_STRINGENCY=", validationStringency), + optional(" SO=", sortOrder), + optional(" MAX_RECORDS_IN_RAM=", maxRecordsInRam), + optional(" ASSUME_SORTED=", assumeSorted)).mkString } diff --git a/scala/src/org/broadinstitute/sting/queue/function/EmbeddedCommandLineFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/EmbeddedCommandLineFunction.scala deleted file mode 100644 index e9987cdef..000000000 --- a/scala/src/org/broadinstitute/sting/queue/function/EmbeddedCommandLineFunction.scala +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2011, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.queue.function - -import java.io.File -import org.broadinstitute.sting.queue.util.IOUtils -import org.broadinstitute.sting.commandline.Argument - -/** - * Defines a command line function that runs java code from inside the existing class path. - */ -trait EmbeddedCommandLineFunction extends JavaCommandLineFunction { - @Argument(doc="Main class to run from the current classpath") - var mainClass: String = null - - def javaExecutable = "-cp %s %s".format(EmbeddedCommandLineFunction.classpath, mainClass) -} - -object EmbeddedCommandLineFunction { - private val classpath = System.getProperty("java.class.path") - .split(File.pathSeparatorChar).map(path => IOUtils.absolute(new File(path))) - .mkString("\"", "\"" + File.pathSeparator + "\"", "\"") -} diff --git a/scala/src/org/broadinstitute/sting/queue/function/JarCommandLineFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/JarCommandLineFunction.scala deleted file mode 100644 index 3bd9da270..000000000 --- a/scala/src/org/broadinstitute/sting/queue/function/JarCommandLineFunction.scala +++ /dev/null @@ -1,14 +0,0 @@ -package org.broadinstitute.sting.queue.function - -import org.broadinstitute.sting.commandline.Argument -import java.io.File - -/** - * Defines a command line function that runs from a jar file. - */ -trait JarCommandLineFunction extends JavaCommandLineFunction { - @Argument(doc="jar") - var jarFile: File = _ - - def javaExecutable = "-jar " + jarFile -} diff --git a/scala/src/org/broadinstitute/sting/queue/function/JavaCommandLineFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/JavaCommandLineFunction.scala index ebac3465c..72445442e 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/JavaCommandLineFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/JavaCommandLineFunction.scala @@ -1,24 +1,74 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.function +import org.broadinstitute.sting.commandline.Argument +import org.broadinstitute.sting.queue.util.IOUtils +import java.io.File + /** * Defines a command line function that runs java code. */ trait JavaCommandLineFunction extends CommandLineFunction { + @Argument(doc="jar", exclusiveOf="javaMainClass") + var jarFile: File = _ + + @Argument(doc="Main class to run from javaClasspath", exclusiveOf="jarFile") + var javaMainClass: String = _ + /** - * Returns the java executable to run. + * Class path for the main class. + * Defaults to the current classpath. */ - def javaExecutable: String + var javaClasspath: List[String] = Nil /** * Memory limit for the java executable, or if None will use the default memoryLimit. */ var javaMemoryLimit: Option[Int] = None + /** + * Returns the java executable to run. + */ + def javaExecutable: String = { + if (jarFile != null) + "-jar " + jarFile + else if (javaMainClass != null) + "-cp \"%s\" %s".format(javaClasspath.mkString(File.pathSeparator), javaMainClass) + else + null + } + override def freezeFieldValues = { super.freezeFieldValues if (javaMemoryLimit.isEmpty && memoryLimit.isDefined) javaMemoryLimit = memoryLimit + + if (javaMainClass != null && javaClasspath.isEmpty) + javaClasspath = JavaCommandLineFunction.currentClasspath } def javaOpts = "%s -Djava.io.tmpdir=%s" @@ -27,3 +77,8 @@ trait JavaCommandLineFunction extends CommandLineFunction { def commandLine = "java%s %s" .format(javaOpts, javaExecutable) } + +object JavaCommandLineFunction { + val currentClasspath = System.getProperty("java.class.path") + .split(File.pathSeparatorChar).map(path => IOUtils.absolute(new File(path)).getPath).toList +} diff --git a/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala index 8ba81ea73..d3176da9c 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.function import java.io.File @@ -5,7 +29,7 @@ import java.lang.annotation.Annotation import org.broadinstitute.sting.commandline._ import org.broadinstitute.sting.queue.{QException, QSettings} import collection.JavaConversions._ -import org.broadinstitute.sting.queue.function.scattergather.{Gather, SimpleTextGatherFunction} +import org.broadinstitute.sting.queue.function.scattergather.SimpleTextGatherFunction import org.broadinstitute.sting.queue.util.{Logging, CollectionUtils, IOUtils, ReflectionUtils} /** diff --git a/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamJarFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/scattergather/GathererFunction.scala similarity index 68% rename from scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamJarFunction.scala rename to scala/src/org/broadinstitute/sting/queue/function/scattergather/GathererFunction.scala index 31e95f0d1..2c56e4a86 100644 --- a/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamJarFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/scattergather/GathererFunction.scala @@ -22,16 +22,20 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -package org.broadinstitute.sting.queue.extensions.picard +package org.broadinstitute.sting.queue.function.scattergather -import org.broadinstitute.sting.queue.function.JarCommandLineFunction +import org.broadinstitute.sting.commandline.Gatherer +import org.broadinstitute.sting.queue.function.InProcessFunction +import collection.JavaConversions._ /** - * Wraps a Picard jar that operates on BAM files. - * See http://picard.sourceforge.net/ for more info. - * - * Since the jar files take slightly different arguments - * some values are optional. + * Runs a Gatherer in process. */ -trait PicardBamJarFunction extends JarCommandLineFunction with PicardBamFunction { +class GathererFunction(gathererClass: Class[_ <: Gatherer]) extends InProcessFunction with GatherFunction { + def run() { + val gatherer = gathererClass.newInstance + if (gatherer.waitForInputs) + waitForGatherParts + gatherer.gather(this.gatherParts, this.originalOutput) + } } diff --git a/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala b/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala index e9f034535..9938aca02 100644 --- a/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala +++ b/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala @@ -26,7 +26,7 @@ package org.broadinstitute.sting.queue.function.scattergather import java.io.File import org.broadinstitute.sting.queue.util._ -import org.broadinstitute.sting.commandline.ArgumentSource +import org.broadinstitute.sting.commandline.{Gatherer, Gather, ArgumentSource} import org.broadinstitute.sting.queue.function.{QFunction, CommandLineFunction} import org.broadinstitute.sting.queue.QException @@ -240,14 +240,29 @@ trait ScatterGatherableFunction extends CommandLineFunction { * @return A GatherFunction instantiated from @Gather. */ protected def newGatherFunction(gatherField: ArgumentSource) : GatherFunction = { - var gatherClass: Class[_ <: GatherFunction] = null + var gatherClass: Class[_] = null + + // Check if there is a function that will return the gather class for this field. if (this.gatherClass != null) if (this.gatherClass.isDefinedAt(gatherField)) gatherClass = this.gatherClass(gatherField) - if (gatherClass == null) - gatherClass = ReflectionUtils.getAnnotation(gatherField.field, classOf[Gather]) - .value.asSubclass(classOf[GatherFunction]) - gatherClass.newInstance.asInstanceOf[GatherFunction] + + // Check for an annotation defining the gather class. + if (gatherClass == null) { + if (ReflectionUtils.hasAnnotation(gatherField.field, classOf[Gather])) { + gatherClass = ReflectionUtils.getAnnotation(gatherField.field, classOf[Gather]).value + } else { + throw new QException("Missing @Gather annotation: " + gatherField.field.getName) + } + } + + if (classOf[GatherFunction].isAssignableFrom(gatherClass)) { + gatherClass.newInstance.asInstanceOf[GatherFunction] + } else if (classOf[Gatherer].isAssignableFrom(gatherClass)) { + new GathererFunction(gatherClass.asSubclass(classOf[Gatherer])) + } else { + throw new QException("Unsupported @Gather class type: " + gatherClass) + } } /** diff --git a/scala/src/org/broadinstitute/sting/queue/pipeline/BamProcessing.scala b/scala/src/org/broadinstitute/sting/queue/pipeline/BamProcessing.scala index 3c2aae531..76e5a4fae 100755 --- a/scala/src/org/broadinstitute/sting/queue/pipeline/BamProcessing.scala +++ b/scala/src/org/broadinstitute/sting/queue/pipeline/BamProcessing.scala @@ -2,7 +2,7 @@ package org.broadinstitute.sting.queue.pipeline import org.broadinstitute.sting.queue.extensions.gatk._ import java.io.File -import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction +import org.broadinstitute.sting.queue.extensions.picard.PicardBamFunction import org.broadinstitute.sting.queue.function.CommandLineFunction import org.broadinstitute.sting.utils.yaml.YamlUtils import org.broadinstitute.sting.datasources.pipeline.Pipeline @@ -125,7 +125,7 @@ class BamProcessing(attribs: Pipeline, gatkJar: File, fixMatesJar: File) { return pfm } - class PicardFixMates extends PicardBamJarFunction { + class PicardFixMates extends PicardBamFunction { @Input(doc="input bam files") var bams: List[File] = Nil @Output(doc="output bam file") var outBam: File = null diff --git a/scala/src/org/broadinstitute/sting/queue/pipeline/VariantCalling.scala b/scala/src/org/broadinstitute/sting/queue/pipeline/VariantCalling.scala index a614954a4..12afd50d5 100755 --- a/scala/src/org/broadinstitute/sting/queue/pipeline/VariantCalling.scala +++ b/scala/src/org/broadinstitute/sting/queue/pipeline/VariantCalling.scala @@ -57,7 +57,7 @@ class VariantCalling(attribs: Pipeline,gatkJar: File) { ug.input_file = bams ug.out = output ug.downsample_to_coverage = Some(300) - ug.dt = Some(DownsampleType.BY_SAMPLE) + ug.dt = DownsampleType.BY_SAMPLE ug.scatterCount = 50 if ( bams.size > 40 ) { @@ -105,8 +105,8 @@ class VariantCalling(attribs: Pipeline,gatkJar: File) { def StandardIndelCombine( igList : List[IndelGenotyperV2], output : File ) : CombineVariants = { var cv = new CombineVariants with StandardCommandLineGATK cv.out = output - cv.genotypemergeoption = Some(org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.GenotypeMergeType.UNIQUIFY) - cv.variantmergeoption = Some(org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.UNION) + cv.genotypemergeoption = org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.GenotypeMergeType.UNIQUIFY + cv.variantmergeoption = org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.UNION cv.analysisName = "IndelGenotyper" cv.priority = (igList.map[String,List[String]](ig => swapExt(ig.out,".vcf","").getAbsolutePath)).mkString(",") //cv.priority = (igList.foldLeft[List[String]](Nil)( (prLs, ig) => prLs ::: List(swapExt(ig.out,".vcf","").getAbsolutePath))).mkString(",") @@ -408,7 +408,7 @@ object VariantCalling { ugTokens.foreach( addEntry(_) ) myUG.reference_sequence = new File(ugMap("reference_sequence")) - myUG.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.valueOf(ugMap("baq"))) + myUG.baq = org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.valueOf(ugMap("baq")) myUG.baqGapOpenPenalty = Some(ugMap("baqGapOpenPenalty").toDouble) myUG.DBSNP = new File(ugMap("DBSNP")) diff --git a/scala/src/org/broadinstitute/sting/queue/util/PrimitiveOptionConversions.scala b/scala/src/org/broadinstitute/sting/queue/util/PrimitiveOptionConversions.scala new file mode 100644 index 000000000..7061e7c8e --- /dev/null +++ b/scala/src/org/broadinstitute/sting/queue/util/PrimitiveOptionConversions.scala @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.queue.util + +/** + * An importable object that provides automatic primitive to option conversion. + */ +object PrimitiveOptionConversions { + + // Conversion from Option + implicit def byteOption2byte(x: Option[Byte]) = x.get + implicit def shortOption2short(x: Option[Short]) = x.get + implicit def charOption2char(x: Option[Char]) = x.get + implicit def intOption2int(x: Option[Int]) = x.get + implicit def longOption2long(x: Option[Long]) = x.get + implicit def floatOption2float(x: Option[Float]) = x.get + implicit def doubleOption2double(x: Option[Double]) = x.get + implicit def booleanOption2boolean(x: Option[Boolean]) = x.get + + // Conversion to Option + implicit def byte2byteOption(x: Byte) = Some(x) + implicit def short2shortOption(x: Short) = Some(x) + implicit def char2charOption(x: Char) = Some(x) + implicit def int2intOption(x: Int) = Some(x) + implicit def long2longOption(x: Long) = Some(x) + implicit def float2floatOption(x: Float) = Some(x) + implicit def double2doubleOption(x: Double) = Some(x) + implicit def boolean2booleanOption(x: Boolean) = Some(x) + + // Narrowing for constants to byte, short, and float + implicit def int2byteOption(x: Int) = Some(x.toByte) + implicit def int2shortOption(x: Int) = Some(x.toShort) + implicit def double2floatOption(x: Float) = Some(x.toFloat) + + // Widening + implicit def byte2shortOption(x: Byte) = Some(x.toShort) + implicit def byte2intOption(x: Byte) = Some(x.toInt) + implicit def byte2longOption(x: Byte) = Some(x.toLong) + implicit def byte2floatOption(x: Byte) = Some(x.toFloat) + implicit def byte2doubleOption(x: Byte) = Some(x.toDouble) + + implicit def short2intOption(x: Short) = Some(x.toInt) + implicit def short2longOption(x: Short) = Some(x.toLong) + implicit def short2floatOption(x: Short) = Some(x.toFloat) + implicit def short2doubleOption(x: Short) = Some(x.toDouble) + + implicit def char2intOption(x: Char) = Some(x.toInt) + implicit def char2longOption(x: Char) = Some(x.toLong) + implicit def char2floatOption(x: Char) = Some(x.toFloat) + implicit def char2doubleOption(x: Char) = Some(x.toDouble) + + implicit def int2longOption(x: Int) = Some(x.toLong) + implicit def int2floatOption(x: Int) = Some(x.toFloat) + implicit def int2doubleOption(x: Int) = Some(x.toDouble) + + implicit def long2floatOption(x: Long) = Some(x.toFloat) + implicit def long2doubleOption(x: Long) = Some(x.toDouble) + + implicit def float2doubleOption(x: Float) = Some(x.toDouble) + +} + +/** + * A trait that exposes the above functions to all sub classes as well. + */ +trait PrimitiveOptionConversions { + // How to we import these implicit definitions into the trait so that they are seen by objects extending a trait? + // import PrimitiveOptionConversion._ inside of a trait does not seem to work? + // Declaring them in a trait like this does work but does not seem scala-ish. + + implicit def byteOption2byte(x: Option[Byte]) = PrimitiveOptionConversions.byteOption2byte(x) + implicit def shortOption2short(x: Option[Short]) = PrimitiveOptionConversions.shortOption2short(x) + implicit def charOption2char(x: Option[Char]) = PrimitiveOptionConversions.charOption2char(x) + implicit def intOption2int(x: Option[Int]) = PrimitiveOptionConversions.intOption2int(x) + implicit def longOption2long(x: Option[Long]) = PrimitiveOptionConversions.longOption2long(x) + implicit def floatOption2float(x: Option[Float]) = PrimitiveOptionConversions.floatOption2float(x) + implicit def doubleOption2double(x: Option[Double]) = PrimitiveOptionConversions.doubleOption2double(x) + implicit def booleanOption2boolean(x: Option[Boolean]) = PrimitiveOptionConversions.booleanOption2boolean(x) + + implicit def byte2byteOption(x: Byte) = PrimitiveOptionConversions.byte2byteOption(x) + implicit def short2shortOption(x: Short) = PrimitiveOptionConversions.short2shortOption(x) + implicit def char2charOption(x: Char) = PrimitiveOptionConversions.char2charOption(x) + implicit def int2intOption(x: Int) = PrimitiveOptionConversions.int2intOption(x) + implicit def long2longOption(x: Long) = PrimitiveOptionConversions.long2longOption(x) + implicit def float2floatOption(x: Float) = PrimitiveOptionConversions.float2floatOption(x) + implicit def double2doubleOption(x: Double) = PrimitiveOptionConversions.double2doubleOption(x) + implicit def boolean2booleanOption(x: Boolean) = PrimitiveOptionConversions.boolean2booleanOption(x) + + implicit def int2byteOption(x: Int) = PrimitiveOptionConversions.int2byteOption(x) + implicit def int2shortOption(x: Int) = PrimitiveOptionConversions.int2shortOption(x) + implicit def double2floatOption(x: Float) = PrimitiveOptionConversions.double2floatOption(x) + + implicit def byte2shortOption(x: Byte) = PrimitiveOptionConversions.byte2shortOption(x) + implicit def byte2intOption(x: Byte) = PrimitiveOptionConversions.byte2intOption(x) + implicit def byte2longOption(x: Byte) = PrimitiveOptionConversions.byte2longOption(x) + implicit def byte2floatOption(x: Byte) = PrimitiveOptionConversions.byte2floatOption(x) + implicit def byte2doubleOption(x: Byte) = PrimitiveOptionConversions.byte2doubleOption(x) + + implicit def short2intOption(x: Short) = PrimitiveOptionConversions.short2intOption(x) + implicit def short2longOption(x: Short) = PrimitiveOptionConversions.short2longOption(x) + implicit def short2floatOption(x: Short) = PrimitiveOptionConversions.short2floatOption(x) + implicit def short2doubleOption(x: Short) = PrimitiveOptionConversions.short2doubleOption(x) + + implicit def char2intOption(x: Char) = PrimitiveOptionConversions.char2intOption(x) + implicit def char2longOption(x: Char) = PrimitiveOptionConversions.char2longOption(x) + implicit def char2floatOption(x: Char) = PrimitiveOptionConversions.char2floatOption(x) + implicit def char2doubleOption(x: Char) = PrimitiveOptionConversions.char2doubleOption(x) + + implicit def int2longOption(x: Int) = PrimitiveOptionConversions.int2longOption(x) + implicit def int2floatOption(x: Int) = PrimitiveOptionConversions.int2floatOption(x) + implicit def int2doubleOption(x: Int) = PrimitiveOptionConversions.int2doubleOption(x) + + implicit def long2floatOption(x: Long) = PrimitiveOptionConversions.long2floatOption(x) + implicit def long2doubleOption(x: Long) = PrimitiveOptionConversions.long2doubleOption(x) + + implicit def float2doubleOption(x: Float) = PrimitiveOptionConversions.float2doubleOption(x) +} diff --git a/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleCountLociPipelineTest.scala b/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleCountLociPipelineTest.scala index 9c2f5186e..024c14232 100644 --- a/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleCountLociPipelineTest.scala +++ b/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleCountLociPipelineTest.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.pipeline.examples import org.testng.annotations.Test @@ -10,8 +34,8 @@ class ExampleCountLociPipelineTest { var testOut = "count.out" val spec = new PipelineTestSpec spec.name = "countloci" - spec.args = "-S scala/qscript/examples/ExampleCountLoci.scala -gatk %s -R %s -I %s -o %s".format( - PipelineTest.currentGATK, BaseTest.hg18Reference, BaseTest.validationDataLocation + "small_bam_for_countloci.bam", testOut + spec.args = "-S scala/qscript/examples/ExampleCountLoci.scala -R %s -I %s -o %s".format( + BaseTest.hg18Reference, BaseTest.validationDataLocation + "small_bam_for_countloci.bam", testOut ) spec.fileMD5s += testOut -> "67823e4722495eb10a5e4c42c267b3a6" PipelineTest.executeTest(spec)