zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Walkers can now specify a class extending from Gatherer to merge custom output formats. Add @Gather(MyGatherer.class) to the walker @Output. 

JavaCommandLineFunctions can now specify the classpath+mainclass as an alternative to specifying a path to an executable jar.
JCLF by default pass on the current classpath and only require the mainclass be specified by the developer extending the JCLF, relieving the QScript author from having to explicitly specify the jar.
Like the Picard MergeSamFiles, GATK engine by default is now run from the current classpath. The GATK can still be overridden via .jarFile or .javaClasspath.
Walkers from the GATK package are now also embedded into the Queue package.
Updated AnalyzeCovariates to make it easier to guess the main class, AnalyzeCovariates instead of AnalyzeCovariatesCLP.
Removed the GATK jar argument from the example QScripts.
Removed one of the most FAQ when getting started with Scala/Queue, the use of Option[_] in QScripts:
1) Fixed mistaken assumption with java enums. In java enums can be null so they don't need nullable wrappers.
2) Added syntactic sugar for Nullable primitives to the QScript trait. Any variable defined as Option[Int] can just be assigned an Int value or None, ex: myFunc.memoryLimit = 3
Removed other unused code.
Re-fixed dry run function ordering.
Re-ordered the QCommandline companion object so that IntelliJ doesn't complain about missing main methods.

git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@5504 348d0f76-0448-11de-a6fe-93d51630548a
This commit is contained in:
kshakir 2011-03-24 14:03:51 +00:00
parent 18271aa1f4
commit f3e94ef2be
66 changed files with 800 additions and 799 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
analysis/depristo/distributedGATK/distributedGATKPerformance.scala
View File

@ -1,4 +1,3 @@
import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction
import org.broadinstitute.sting.queue.extensions.gatk._
import org.broadinstitute.sting.queue.extensions.samtools.SamtoolsIndexFunction
import org.broadinstitute.sting.queue.QScript
@ -40,7 +39,7 @@ class DistributedGATKPerformance extends QScript {
@Argument(shortName="trackerDir", doc="root directory for distributed tracker files", required=false)
var trackerDir: String = "" // "/humgen/gsa-scr1/depristo/tmp/"
trait UNIVERSAL_GATK_ARGS extends CommandLineGATK { logging_level = "DEBUG"; jarFile = gatkJarFile; memoryLimit = Some(2); }
trait UNIVERSAL_GATK_ARGS extends CommandLineGATK { logging_level = "DEBUG"; jarFile = gatkJarFile; memoryLimit = 2; }
class Target(
val baseName: String,
@ -154,9 +153,9 @@ class DistributedGATKPerformance extends QScript {
def addUG(ug: UnifiedGenotyper) = {
if ( ! long )
ug.jobLimitSeconds = Some(60 * 60 * 4)
ug.jobLimitSeconds = 60 * 60 * 4
if ( limitTo30Min )
ug.jobLimitSeconds = Some(60 * 30)
ug.jobLimitSeconds = 60 * 30
add(ug);
}
@ -171,7 +170,7 @@ class DistributedGATKPerformance extends QScript {
var ug: UnifiedGenotyper = new UnifiedGenotyper(target, aname + ".part" + part)
ug.intervalsString ++= getTargetInterval(target)
ug.processingTracker = new File(trackerDir + target.name + "." + aname + ".distributed.txt")
ug.processingTrackerID = Some(part)
ug.processingTrackerID = part
if ( part == 1 )
ug.performanceLog = new File("%s.%s.pf.log".format(target.name, aname))
ug.processingTrackerStatusFile = new File("%s.%s.%d.ptstatus.log".format(target.name, aname, part))
@ -186,12 +185,12 @@ class DistributedGATKPerformance extends QScript {
// 1.) Call SNPs with UG
class UnifiedGenotyper(t: Target, aname: String) extends org.broadinstitute.sting.queue.extensions.gatk.UnifiedGenotyper with UNIVERSAL_GATK_ARGS {
this.reference_sequence = t.reference
this.dcov = Some( if ( t.isLowpass ) { 50 } else { 250 } )
this.stand_call_conf = Some( if ( t.isLowpass ) { 4.0 } else { 30.0 } )
this.stand_emit_conf = Some( if ( t.isLowpass ) { 4.0 } else { 30.0 } )
this.dcov = if ( t.isLowpass ) { 50 } else { 250 }
this.stand_call_conf = if ( t.isLowpass ) { 4.0 } else { 30.0 }
this.stand_emit_conf = if ( t.isLowpass ) { 4.0 } else { 30.0 }
this.input_file :+= t.bamList
this.out = t.rawVCF(aname)
this.baq = Some( if (t.useBAQ) {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.RECALCULATE} else {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF})
this.baq = if (t.useBAQ) {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.RECALCULATE} else {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF}
this.analysisName = t.name + "_UG." + aname
if (t.dbsnpFile.endsWith(".rod"))
this.DBSNP = new File(t.dbsnpFile)

16
java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java
View File

@ -48,7 +48,7 @@ import java.io.*;
* Create collapsed versions of the recal csv file and call R scripts to plot residual error versus the various covariates.
*/
class AnalyzeCovariatesCLP extends CommandLineProgram {
public class AnalyzeCovariates extends CommandLineProgram {
/////////////////////////////
// Command Line Arguments
@ -311,12 +311,14 @@ class AnalyzeCovariatesCLP extends CommandLineProgram {
}
}
}
}
public class AnalyzeCovariates {
public static void main(String args[]) throws Exception {
AnalyzeCovariatesCLP clp = new AnalyzeCovariatesCLP();
CommandLineProgram.start( clp, args );
System.exit(0);
public static void main(String args[]) {
try {
AnalyzeCovariates clp = new AnalyzeCovariates();
start(clp, args);
System.exit(CommandLineProgram.result);
} catch (Exception e) {
exitSystemWithError(e);
}
}
}

5
java/src/org/broadinstitute/sting/queue/function/scattergather/Gather.java → java/src/org/broadinstitute/sting/commandline/Gather.java
View File

@ -22,13 +22,12 @@
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.function.scattergather;
package org.broadinstitute.sting.commandline;
import java.lang.annotation.*;
/**
* Specifies the class type of the CommandLineFunction to gather an @Output
* Written in java because scala doesn't support RetentionPolicy.RUNTIME
* Specifies the class type to gather an @Output
*/
@Documented
@Inherited

25
scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamEmbeddedFunction.scala → java/src/org/broadinstitute/sting/commandline/Gatherer.java
View File

@ -22,16 +22,25 @@
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.extensions.picard
package org.broadinstitute.sting.commandline;
import org.broadinstitute.sting.queue.function.EmbeddedCommandLineFunction
import java.io.File;
import java.util.List;
/**
* Wraps a Picard embedded class that operates on BAM files.
* See http://picard.sourceforge.net/ for more info.
*
* Since the command lines take slightly different arguments
* some values are optional.
* Combines a list of files into a single output.
*/
trait PicardBamEmbeddedFunction extends EmbeddedCommandLineFunction with PicardBamFunction {
public abstract class Gatherer {
/**
* Gathers a list of files into a single output.
* @param inputs Files to combine.
* @param output Path to output file.
*/
public abstract void gather(List<File> inputs, File output);
/**
* Returns true if the caller should wait for the input files to propagate over NFS before running gather().
* @return true if the caller should wait for the input files to propagate over NFS before running gather().
*/
public boolean waitForInputs() { return true; }
}

207
java/src/org/broadinstitute/sting/playground/analyzeconcordance/AnalyzeConcordance.java
View File

@ -1,207 +0,0 @@
/*
* Copyright (c) 2010 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
* THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.playground.analyzeconcordance;
import org.broadinstitute.sting.commandline.CommandLineProgram;
import org.broadinstitute.sting.commandline.Argument;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import org.broadinstitute.sting.utils.text.XReadLines;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.PathUtils;
import org.broadinstitute.sting.playground.utils.ProcessUtils;
import org.apache.log4j.Logger;
import java.io.*;
import java.util.*;
/**
* Compares results of VariantEval across a population or a case/control group.
*/
public class AnalyzeConcordance extends CommandLineProgram {
@Argument(fullName = "group_name", shortName = "groupName", doc = "The name of the group which will be prefixed output files", required = false)
private String baseName = "analyze_concordance";
@Argument(fullName = "eval_list", shortName = "evalList", doc = "The input list of unfiltered eval files to analyze", required = true)
private String evalListFile = null;
@Argument(fullName = "filtered_eval_list", shortName = "filteredEvalList", doc = "The input list of filtered eval files to analyze", required = false)
private String filteredEvalListFile = null;
@Argument(fullName = "output_dir", shortName = "outputDir", doc = "The directory in which to output all the plots and intermediate data files", required = false)
private String outputDir = "analyzeConcordance";
@Argument(fullName = "path_to_Rscript", shortName = "Rscript", doc = "The path to your implementation of Rscript", required = false)
private String pathToRscript = "env Rscript";
@Argument(fullName = "path_to_resources", shortName = "resources", doc = "Path to resources folder holding the Sting analyze concordance R scripts", required = false)
private String pathToResources = "R" + File.separator + "analyzeConcordance";
private enum EvalFilterType {
UNFILTERED, FILTERED
}
private static final AnalyzeConcordanceField[] ANALYZE_CONCORDANCE_FIELDS = AnalyzeConcordanceField.values();
private String evalDataFile;
private List<String[]> data = new ArrayList<String[]>();
private static Logger logger = Logger.getLogger(AnalyzeConcordance.class);
protected int execute() {
int result;
try {
createOutputDirectory();
// initialize all the data from the csv file and allocate the list of covariates
logger.info("Reading in input csv file...");
initializeData();
logger.info("...Done!");
// output data tables for Rscript to read in
logger.info("Writing out intermediate tables for R...");
writeDataTables();
logger.info("...Done!");
// perform the analysis using Rscript and output the plots
logger.info("Calling analysis R scripts and writing out figures...");
result = callRScripts();
logger.info("...Done!");
// perform the analysis using Rscript and output the plots
logger.info("Generating html report...");
generateHtmlReport();
logger.info("...Done!");
} catch (Exception e) {
throw new ReviewedStingException("Error analyzing concordance", e);
}
return result;
}
private void createOutputDirectory() {
// create the output directory where all the data tables and plots will go
File outputDir = new File(this.outputDir);
if (!outputDir.exists() && !outputDir.mkdirs()) {
throw new ReviewedStingException("Couldn't create directory: " + this.outputDir);
}
}
private void initializeData() throws FileNotFoundException {
// add the column headers to the data
addHeader();
// read the list of unfiltered eval files
addEvalListFile(EvalFilterType.UNFILTERED, new File(evalListFile));
// if provided, read the list of filtered eval files
if (filteredEvalListFile != null) {
addEvalListFile(EvalFilterType.FILTERED, new File(filteredEvalListFile));
}
}
private void addHeader() {
String[] headers = new String[ANALYZE_CONCORDANCE_FIELDS.length + 2];
int column = 0;
headers[column++] = "eval_id";
headers[column++] = "filter_type";
for (AnalyzeConcordanceField field : ANALYZE_CONCORDANCE_FIELDS) {
headers[column++] = field.getColumnHeader();
}
data.add(headers);
}
private void addEvalListFile(EvalFilterType filterType, File evalListFile) throws FileNotFoundException {
for (String line : new XReadLines(evalListFile)) {
String[] parts = line.split("\t");
addEvalFile(parts[0], filterType, new File(parts[1]));
}
}
private void addEvalFile(String evalID, EvalFilterType filterType, File evalFile) throws FileNotFoundException {
SortedMap<AnalyzeConcordanceField, String> fieldValues = new TreeMap<AnalyzeConcordanceField, String>();
for (String line : new XReadLines(evalFile)) {
for (AnalyzeConcordanceField field : ANALYZE_CONCORDANCE_FIELDS) {
String value = field.parseLine(line);
if (value != null) {
fieldValues.put(field, value);
break; // continue to the next line.
}
}
}
String[] values = new String[ANALYZE_CONCORDANCE_FIELDS.length + 2];
int column = 0;
values[column++] = evalID;
values[column++] = filterType.toString().toLowerCase();
// get all the values, including null if for some reason a value wasn't found
for (AnalyzeConcordanceField field : ANALYZE_CONCORDANCE_FIELDS) {
values[column++] = fieldValues.get(field);
}
data.add(values);
}
private void writeDataTables() throws FileNotFoundException {
evalDataFile = baseName + ".eval_data.tsv";
// Create a PrintStream
PrintStream output = new PrintStream(new File(outputDir, evalDataFile));
for (String[] line : data) {
output.println(Utils.join("\t", line));
}
output.close();
}
private int callRScripts() {
String command = pathToRscript + " "
+ new File(pathToResources, "analyzeConcordance.R") + " "
+ new File(outputDir, baseName) + " "
+ new File(outputDir, evalDataFile);
return ProcessUtils.runCommandAndWait(command);
}
private void generateHtmlReport() throws FileNotFoundException {
// TODO: Enhance the reports
PrintStream output = new PrintStream(new File(outputDir, "report.html"));
output.println("<html><body>");
for (File pngFile : new File(outputDir).listFiles(new PathUtils.ExtensionFilter("png"))) {
output.println("<div><img src=\"" + pngFile.getName() + "\"/></div>");
}
output.println("</body></html>");
output.close();
}
public static void main(String[] argv) {
try {
AnalyzeConcordance instance = new AnalyzeConcordance();
start(instance, argv);
System.exit(CommandLineProgram.result);
} catch (Exception e) {
exitSystemWithError(e);
}
}
}

41
java/src/org/broadinstitute/sting/playground/analyzeconcordance/AnalyzeConcordanceField.java
View File

@ -1,41 +0,0 @@
package org.broadinstitute.sting.playground.analyzeconcordance;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
/**
* Created by IntelliJ IDEA.
* User: kshakir
* Date: Feb 11, 2010
*/
public enum AnalyzeConcordanceField {
N_BASES_COVERED("all_bases", "all,summary,variant_counts", "n bases covered"),
ALL_DBSNP_RATE("all_dbsnp", "all,summary,db_coverage", "dbsnp_rate"),
ALL_VARIANT_COUNT("all_variants", "all,summary,variant_counts", "variants"),
ALL_TITV_RATIO("all_titv", "all,summary,transitions_transversions", "ratio"),
KNOWN_VARIANT_COUNT("known_variants", "known,summary,variant_counts", "variants"),
KNOWN_TITV_RATIO("known_titv", "known,summary,transitions_transversions", "ratio"),
NOVEL_VARIANT_COUNT("novel_variants", "novel,summary,variant_counts", "variants"),
NOVEL_TITV_RATIO("novel_titv", "novel,summary,transitions_transversions", "ratio");
private String columnHeader;
private Pattern pattern;
private AnalyzeConcordanceField(String columnHeader, String evalHeader, String analysis) {
this.columnHeader = columnHeader;
String lineRegex = evalHeader + " {2,}" + analysis + " {2,}([0-9.]+).*";
this.pattern = Pattern.compile(lineRegex);
}
public String getColumnHeader() {
return this.columnHeader;
}
public String parseLine(String line) {
Matcher matcher = this.pattern.matcher(line);
if (!matcher.matches())
return null;
return matcher.group(1);
}
}

22
java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java
View File

@ -111,15 +111,19 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
public static List<? extends ArgumentField> getArgumentFields(ParsingEngine parsingEngine,Class<?> classType) {
List<ArgumentField> argumentFields = new ArrayList<ArgumentField>();
for (ArgumentSource argumentSource: parsingEngine.extractArgumentSources(classType))
if (!argumentSource.isDeprecated())
if (!argumentSource.isDeprecated()) {
Class<?> gatherer = null;
if (argumentSource.field.isAnnotationPresent(Gather.class))
gatherer = argumentSource.field.getAnnotation(Gather.class).value();
for (ArgumentDefinition argumentDefinition: argumentSource.createArgumentDefinitions())
argumentFields.addAll(getArgumentFields(argumentDefinition));
argumentFields.addAll(getArgumentFields(argumentDefinition, gatherer));
}
return argumentFields;
}
private static final List<String> intervalFields = Arrays.asList("intervals", "excludeIntervals", "targetIntervals");
private static List<? extends ArgumentField> getArgumentFields(ArgumentDefinition argumentDefinition) {
private static List<? extends ArgumentField> getArgumentFields(ArgumentDefinition argumentDefinition, Class<?> gatherer) {
if (intervalFields.contains(argumentDefinition.fullName) && argumentDefinition.ioType == ArgumentIOType.INPUT) {
return Arrays.asList(
new IntervalFileArgumentField(argumentDefinition),
@ -138,7 +142,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
return Collections.singletonList(new InputArgumentField(argumentDefinition));
} else if (argumentDefinition.ioType == ArgumentIOType.OUTPUT) {
return Collections.singletonList(new OutputArgumentField(argumentDefinition));
return Collections.singletonList(new OutputArgumentField(argumentDefinition, gatherer));
} else if (argumentDefinition.isFlag) {
return Collections.singletonList(new FlagArgumentField(argumentDefinition));
@ -224,8 +228,10 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
// if (argumentDefinition.ioType == ArgumentIOType.OUTPUT)
// Map all outputs to files.
private static class OutputArgumentField extends ArgumentDefinitionField {
public OutputArgumentField(ArgumentDefinition argumentDefinition) {
private final Class<?> gatherer;
public OutputArgumentField(ArgumentDefinition argumentDefinition, Class<?> gatherer) {
super(argumentDefinition);
this.gatherer = gatherer;
}
@Override protected Class<?> getInnerType() { return File.class; }
@ -235,7 +241,9 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
@Override public boolean isGather() { return true; }
@Override protected String getGatherAnnotation() {
String gather;
if (SAMFileWriter.class.isAssignableFrom(argumentDefinition.argumentType))
if (gatherer != null)
gather = "@Gather(classOf[" + gatherer.getName() + "])%n";
else if (SAMFileWriter.class.isAssignableFrom(argumentDefinition.argumentType))
gather = "@Gather(classOf[BamGatherFunction])%n";
else if (VCFWriter.class.isAssignableFrom(argumentDefinition.argumentType))
gather = "@Gather(classOf[VcfGatherFunction])%n";
@ -272,7 +280,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
}
// if (!argumentDefinition.required && useOption(argumentDefinition.argumentType))
// Any optional arguments that are primitives / enums are wrapped in options.
// Any optional arguments that are primitives are wrapped in options.
private static class OptionedArgumentField extends ArgumentDefinitionField {
private final boolean useFormatter;

2
java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java
View File

@ -217,7 +217,7 @@ public abstract class ArgumentField {
* @return true if option should be used.
*/
protected static boolean useOption(Class<?> argType) {
return (argType.isPrimitive()) || (Number.class.isAssignableFrom(argType)) || (argType.isEnum());
return (argType.isPrimitive()) || (Number.class.isAssignableFrom(argType));
}
/**

7
java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java
View File

@ -140,9 +140,10 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
continue;
String clpClassName = clpManager.getName(clp);
String clpConstructor = String.format("analysisName = \"%s\"%njavaMainClass = \"%s\"%n", clpClassName, clp.getName());
writeClass("org.broadinstitute.sting.queue.function.JarCommandLineFunction", clpClassName,
false, "", ArgumentDefinitionField.getArgumentFields(parser,clp), dependents);
writeClass("org.broadinstitute.sting.queue.function.JavaCommandLineFunction", clpClassName,
false, clpConstructor, ArgumentDefinitionField.getArgumentFields(parser,clp), dependents);
if (clp == CommandLineGATK.class) {
for (Entry<String, Collection<Class<? extends Walker>>> walkersByPackage: walkerManager.getWalkerNamesByPackage(false).entrySet()) {
@ -359,7 +360,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
baseClass += " with ScatterGatherableFunction";
}
if (isGather)
importSet.add("import org.broadinstitute.sting.queue.function.scattergather.Gather");
importSet.add("import org.broadinstitute.sting.commandline.Gather");
// Sort the imports so that the are always in the same order.
List<String> sortedImports = new ArrayList<String>(importSet);

18
packages/AnalyzeConcordance.xml
View File

@ -1,18 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Currently built via:
ant playground package -Dexecutable=AnalyzeConcordance
-->
<package name="AnalyzeConcordance">
<executable name="AnalyzeConcordance">
<main-class name="org.broadinstitute.sting.playground.analyzeconcordance.AnalyzeConcordance" />
<resource-bundle file="StingText.properties" />
<dependencies>
<class name="org.broadinstitute.sting.playground.analyzeconcordance.AnalyzeConcordance" />
</dependencies>
</executable>
<resources>
<!-- Supplemental scripts for graph generation, etc. -->
<file name="R/analyzeConcordance/analyzeConcordance.R" />
</resources>
</package>

49
packages/GATKEngine.xml 100644
View File

@ -0,0 +1,49 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright (c) 2011, The Broad Institute
~
~ Permission is hereby granted, free of charge, to any person
~ obtaining a copy of this software and associated documentation
~ files (the "Software"), to deal in the Software without
~ restriction, including without limitation the rights to use,
~ copy, modify, merge, publish, distribute, sublicense, and/or sell
~ copies of the Software, and to permit persons to whom the
~ Software is furnished to do so, subject to the following
~ conditions:
~
~ The above copyright notice and this permission notice shall be
~ included in all copies or substantial portions of the Software.
~ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
~ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
~ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
~ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
~ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
~ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
~ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
~ OTHER DEALINGS IN THE SOFTWARE.
-->
<package name="GATKEngine">
<executable name="GATKEngine">
<dependencies>
<!-- Core walkers -->
<package name="org.broadinstitute.sting.gatk.walkers.**" />
<!-- All non-oneoff GATK-specific RODs -->
<package name="org.broadinstitute.sting.gatk.refdata.**" />
<!-- Filters -->
<package name="org.broadinstitute.sting.gatk.filters" />
<!-- Tribble codecs -->
<package name="org.broad.tribble.*" />
<!-- Workaround - depend on the logger impl required by JEXL -->
<package name="org.apache.commons.logging.impl" />
</dependencies>
</executable>
<resources>
<!-- Sample reads and reference files -->
<file name="testdata/exampleBAM.bam" />
<file name="testdata/exampleBAM.bam.bai" />
<file name="testdata/exampleFASTA.fasta" />
<file name="testdata/exampleFASTA.fasta.fai" />
<file name="testdata/exampleFASTA.dict" />
</resources>
</package>

21
packages/GenomeAnalysisTK.xml
View File

@ -4,18 +4,9 @@
<executable name="GenomeAnalysisTK">
<main-class name="org.broadinstitute.sting.gatk.CommandLineGATK" />
<resource-bundle file="StingText.properties" />
<dependencies>
<!-- Core walkers -->
<package name="org.broadinstitute.sting.gatk.walkers.**" />
<!-- All non-oneoff GATK-specific RODs -->
<package name="org.broadinstitute.sting.gatk.refdata.**" />
<!-- Filters -->
<package name="org.broadinstitute.sting.gatk.filters" />
<!-- Tribble codecs -->
<package name="org.broad.tribble.*" />
<!-- Workaround - depend on the logger impl required by JEXL -->
<package name="org.apache.commons.logging.impl" />
</dependencies>
<modules>
<module file="GATKEngine.xml"/>
</modules>
</executable>
<modules>
<module file="AnalyzeCovariates.xml"/>
@ -28,12 +19,6 @@
<file name="java/src/org/broadinstitute/sting/gatk/walkers/qc/CountLociWalker.java" />
<file name="java/src/org/broadinstitute/sting/gatk/walkers/qc/CountReadsWalker.java" />
<file name="java/src/org/broadinstitute/sting/gatk/walkers/qc/ValidatingPileupWalker.java" />
<!-- Sample reads and reference files -->
<file name="testdata/exampleBAM.bam" />
<file name="testdata/exampleBAM.bam.bai" />
<file name="testdata/exampleFASTA.fasta" />
<file name="testdata/exampleFASTA.fasta.fai" />
<file name="testdata/exampleFASTA.dict" />
</resources>
<release>
<executable directory="/humgen/gsa-hpprojects/GATK/bin" symlink="current" />

28
packages/Queue.xml
View File

@ -1,4 +1,28 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright (c) 2011, The Broad Institute
~
~ Permission is hereby granted, free of charge, to any person
~ obtaining a copy of this software and associated documentation
~ files (the "Software"), to deal in the Software without
~ restriction, including without limitation the rights to use,
~ copy, modify, merge, publish, distribute, sublicense, and/or sell
~ copies of the Software, and to permit persons to whom the
~ Software is furnished to do so, subject to the following
~ conditions:
~
~ The above copyright notice and this permission notice shall be
~ included in all copies or substantial portions of the Software.
~ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
~ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
~ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
~ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
~ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
~ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
~ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
~ OTHER DEALINGS IN THE SOFTWARE.
-->
<package name="Queue">
<version file="StingText.properties" property="org.broadinstitute.sting.queue.version" />
<executable name="Queue">
@ -21,6 +45,10 @@
<!-- JNA, including embedded native libraries -->
<dir name="com/sun/jna" />
</dependencies>
<modules>
<module file="GenomeAnalysisTK.xml"/>
<module file="AnalyzeCovariates.xml"/>
</modules>
</executable>
<release>
<executable directory="/humgen/gsa-hpprojects/Queue/bin" symlink="current" />

28
scala/qscript/core/MethodsDevelopmentCallingPipeline.scala
View File

@ -185,8 +185,8 @@ class MethodsDevelopmentCallingPipeline extends QScript {
trait UNIVERSAL_GATK_ARGS extends CommandLineGATK {
logging_level = "INFO";
jarFile = gatkJarFile;
memoryLimit = Some(4);
phone_home = Some(if ( LOCAL_ET ) GATKRunReport.PhoneHomeOption.STANDARD else GATKRunReport.PhoneHomeOption.AWS_S3)
memoryLimit = 4;
phone_home = if ( LOCAL_ET ) GATKRunReport.PhoneHomeOption.STANDARD else GATKRunReport.PhoneHomeOption.AWS_S3
}
def bai(bam: File) = new File(bam + ".bai")
@ -197,9 +197,9 @@ class MethodsDevelopmentCallingPipeline extends QScript {
this.reference_sequence = t.reference
this.intervalsString ++= List(t.intervals)
this.scatterCount = 63 // the smallest interval list has 63 intervals, one for each Mb on chr20
this.dcov = Some( if ( t.isLowpass ) { 50 } else { 250 } )
this.stand_call_conf = Some( if ( t.isLowpass ) { 4.0 } else { 30.0 } )
this.stand_emit_conf = Some( if ( t.isLowpass ) { 4.0 } else { 30.0 } )
this.dcov = if ( t.isLowpass ) { 50 } else { 250 }
this.stand_call_conf = if ( t.isLowpass ) { 4.0 } else { 30.0 }
this.stand_emit_conf = if ( t.isLowpass ) { 4.0 } else { 30.0 }
this.input_file :+= t.bamList
if (t.dbsnpFile.endsWith(".rod"))
this.DBSNP = new File(t.dbsnpFile)
@ -210,11 +210,11 @@ class MethodsDevelopmentCallingPipeline extends QScript {
// 1a.) Call SNPs with UG
class snpCall (t: Target) extends GenotyperBase(t) {
if (minimumBaseQuality >= 0)
this.min_base_quality_score = Some(minimumBaseQuality)
this.min_base_quality_score = minimumBaseQuality
if (qscript.deletions >= 0)
this.max_deletion_fraction = Some(qscript.deletions)
this.max_deletion_fraction = qscript.deletions
this.out = t.rawVCF
this.baq = Some( if (noBAQ) {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF} else {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY})
this.baq = if (noBAQ) {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF} else {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY}
this.analysisName = t.name + "_UGs"
this.jobName = queueLogDir + t.name + ".snpcall"
}
@ -222,8 +222,8 @@ class MethodsDevelopmentCallingPipeline extends QScript {
// 1b.) Call Indels with UG
class indelCall (t: Target) extends GenotyperBase(t) {
this.out = t.rawIndelVCF
this.glm = Some(org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeLikelihoodsCalculationModel.Model.DINDEL)
this.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF)
this.glm = org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeLikelihoodsCalculationModel.Model.DINDEL
this.baq = org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF
this.analysisName = t.name + "_UGi"
this.jobName = queueLogDir + t.name + ".indelcall"
}
@ -248,7 +248,7 @@ class MethodsDevelopmentCallingPipeline extends QScript {
// 3.) Variant Quality Score Recalibration - Generate Recalibration table
class VQSR(t: Target, goldStandard: Boolean) extends ContrastiveRecalibrator with UNIVERSAL_GATK_ARGS {
this.memoryLimit = Some(6)
this.memoryLimit = 6
this.reference_sequence = t.reference
this.intervalsString ++= List(t.intervals)
this.rodBind :+= RodBind("input", "VCF", if ( goldStandard ) { t.goldStandard_VCF } else { t.rawVCF } )
@ -272,13 +272,13 @@ class MethodsDevelopmentCallingPipeline extends QScript {
// 4.) Apply the recalibration table to the appropriate tranches
class applyVQSR (t: Target, goldStandard: Boolean) extends ApplyRecalibration with UNIVERSAL_GATK_ARGS {
this.memoryLimit = Some(4)
this.memoryLimit = 4
this.reference_sequence = t.reference
this.intervalsString ++= List(t.intervals)
this.rodBind :+= RodBind("input", "VCF", if ( goldStandard ) { t.goldStandard_VCF } else { t.rawVCF } )
this.tranches_file = if ( goldStandard ) { t.goldStandardTranchesFile } else { t.tranchesFile}
this.recal_file = if ( goldStandard ) { t.goldStandardRecalFile } else { t.recalFile }
this.fdr_filter_level = Some(2.0)
this.fdr_filter_level = 2.0
this.out = t.recalibratedVCF
this.analysisName = t.name + "_AVQSR"
this.jobName = queueLogDir + t.name + ".applyVQSR"
@ -293,7 +293,7 @@ class MethodsDevelopmentCallingPipeline extends QScript {
this.intervalsString ++= List(t.intervals)
this.out = t.cutVCF
this.tranchesFile = t.tranchesFile
this.fdr_filter_level = Some(t.trancheTarget)
this.fdr_filter_level = t.trancheTarget
if (t.dbsnpFile.endsWith(".rod"))
this.DBSNP = new File(t.dbsnpFile)
else if (t.dbsnpFile.endsWith(".vcf"))

2
scala/qscript/core/StandardVariantEvaluation.scala
View File

@ -65,7 +65,7 @@ class StandardVariantEvaluation extends QScript {
this.jarFile = gatkJarFile;
this.intervalsString = List(TARGET_INTERVAL);
this.reference_sequence = referenceFile;
this.memoryLimit = Some(2)
this.memoryLimit = 2
}
def initializeStandardDataFiles() = {

6
scala/qscript/examples/ExampleCountLoci.scala
View File

@ -5,9 +5,6 @@ import org.broadinstitute.sting.queue.extensions.gatk._
* An introductory pipeline with integration tests testing the MD5 of the @Output.
*/
class ExampleCountLoci extends QScript {
@Input(doc="The path to the GenomeAnalysisTK.jar file.", shortName="gatk")
var gatkJar: File = null
@Input(doc="The reference file for the bam files.", shortName="R")
var referenceFile: File = null
@ -22,12 +19,11 @@ class ExampleCountLoci extends QScript {
def script = {
val countLoci = new CountLoci
countLoci.jarFile = gatkJar
countLoci.reference_sequence = referenceFile
countLoci.input_file = bamFiles
countLoci.intervalsString = intervals
countLoci.out = out
countLoci.memoryLimit = Some(1)
countLoci.memoryLimit = 1
add(countLoci)
}
}

7
scala/qscript/examples/ExampleCountReads.scala
View File

@ -7,9 +7,6 @@ import org.broadinstitute.sting.queue.extensions.gatk._
* All bams must have the same reference.
*/
class ExampleCountReads extends QScript {
@Input(doc="The path to the GenomeAnalysisTK.jar file.", shortName="gatk")
var gatkJar: File = null
@Input(doc="The reference file for the bam files.", shortName="R")
var referenceFile: File = null
@ -33,9 +30,6 @@ class ExampleCountReads extends QScript {
// The names of walkers are the same as you would use for '-T <WalkerName>'
val jointCountReads = new CountReads
// Set the GATK jar file for this command.
jointCountReads.jarFile = gatkJar
// Each field in the extensions is based off of the full form of the arguments.
// To get the list of arguments and their descriptions run
// java -jar <path to GenomeAnalysisTK.jar> -T <WalkerName> -help
@ -52,7 +46,6 @@ class ExampleCountReads extends QScript {
if (bamFiles.size > 1) {
for (bamFile <- bamFiles) {
val singleCountReads = new CountReads
singleCountReads.jarFile = gatkJar
singleCountReads.reference_sequence = referenceFile
// ':+' is the scala List append operator
singleCountReads.input_file :+= bamFile

23
scala/qscript/examples/ExampleCustomWalker.scala
View File

@ -2,17 +2,11 @@ import org.broadinstitute.sting.queue.QScript
import org.broadinstitute.sting.queue.extensions.gatk._
/**
* A pipeline for Queue that runs a custom walker outside of the GATK jar.
* A pipeline for Queue that runs a custom walker on the classpath.
* NOTE: This code is an unsupported example for soliciting feedback on how to improve Queue.
* Future syntax will simplify running the GATK so please expect the syntax below to change significantly.
*/
class ExampleCustomWalker extends QScript {
// The full packaged jar should be used.
// You can build this jar via 'ant package' and then find it under
// 'Sting/dist/packages/GenomeAnalysisTK-*/GenomeAnalysisTK.jar'
@Input(doc="The path to the packaged GenomeAnalysisTK.jar file.", shortName="gatk")
var gatkJar: File = null
@Input(doc="The reference file for the bam files.", shortName="R")
var referenceFile: File = null
@ -29,15 +23,18 @@ class ExampleCustomWalker extends QScript {
* In script, you create and then add() functions to the pipeline.
*/
def script = {
val myClasses = "myClassDir"
val customWalker = new CommandLineGATK {
// Set the name of your walker, for example this will be passed as -T MyCustomWalker
this.analysis_type = "MyCustomWalker"
// NOTE: At this time, you still need to specify the GATK jar or the pipeline won't validate.
this.jarFile = gatkJar
override def javaExecutable = "org.broadinstitute.sting.gatk.CommandLineGATK"
override def javaOpts = "%s -cp %s:%s".format(super.javaOpts, gatkJar, myClasses)
// If your walker is already on the classpath you shouldn't need to do anything else
// If your walker is in a GATK jar that is for some reason NOT on the classpath
// nor referenced in the Queue.jar's, specify the jar file here
//this.jarFile = "myGATK.jar"
// If your walker needs a custom classpath, specify it here
//this.javaClasspath = List("myClasses")
}
customWalker.reference_sequence = referenceFile

8
scala/qscript/examples/ExampleUnifiedGenotyper.scala
View File

@ -15,9 +15,6 @@ class ExampleUnifiedGenotyper extends QScript {
// Required arguments. All initialized to empty values.
@Input(doc="The path to the GenomeAnalysisTK.jar file.", shortName="gatk")
var gatkJar: File = null // The command line must pass the gatk jar to this script via -gatk.
@Input(doc="The reference file for the bam files.", shortName="R")
var referenceFile: File = _ // _ is scala shorthand for null
@ -39,12 +36,11 @@ class ExampleUnifiedGenotyper extends QScript {
// This trait allows us set the variables below in one place,
// and then reuse this trait on each CommandLineGATK function below.
trait UnifiedGenotyperArguments extends CommandLineGATK {
this.jarFile = qscript.gatkJar
this.reference_sequence = qscript.referenceFile
this.intervals = List(qscript.intervals)
// Some() is how you set the value for an scala Option.
// is how you set the value for an scala Option.
// Set the memory limit to 2 gigabytes on each command.
this.memoryLimit = Some(2)
this.memoryLimit = 2
}

22
scala/qscript/oneoffs/carneiro/dataProcessing.scala
View File

@ -1,5 +1,5 @@
import org.broadinstitute.sting.queue.extensions.gatk._
import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction
import org.broadinstitute.sting.queue.extensions.picard.PicardBamFunction
import org.broadinstitute.sting.queue.QScript
import org.broadinstitute.sting.queue.function.ListWriterFunction
import scala.io.Source
@ -55,7 +55,7 @@ class dataProcessing extends QScript {
trait CommandLineGATKArgs extends CommandLineGATK {
this.jarFile = qscript.GATKjar
this.reference_sequence = qscript.reference
this.memoryLimit = Some(4)
this.memoryLimit = 4
this.isIntermediate = true
}
@ -127,7 +127,7 @@ class dataProcessing extends QScript {
class TargetBase (outIntervals: String) extends RealignerTargetCreator with CommandLineGATKArgs {
this.out = new File(outIntervals)
this.mismatchFraction = Some(0.0)
this.mismatchFraction = 0.0
this.rodBind :+= RodBind("dbsnp", "VCF", dbSNP)
this.rodBind :+= RodBind("indels", "VCF", indels)
@ -139,7 +139,7 @@ class dataProcessing extends QScript {
class allTargets (inBams: String, outIntervals: String) extends TargetBase(outIntervals) {
this.input_file :+= new File(inBams)
this.memoryLimit = Some(6)
this.memoryLimit = 6
this.jobName = queueLogDir + outIntervals + ".atarget"
}
@ -151,16 +151,16 @@ class dataProcessing extends QScript {
this.rodBind :+= RodBind("indels", "VCF", qscript.indels)
this.useOnlyKnownIndels = knownsOnly
this.doNotUseSW = true
this.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY)
this.compress = Some(0)
this.U = Some(org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION) // todo -- update this with the last consensus between Tim, Matt and Eric. This is ugly!
this.baq = org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY
this.compress = 0
this.U = org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION // todo -- update this with the last consensus between Tim, Matt and Eric. This is ugly!
this.isIntermediate = intermediate
this.jobName = queueLogDir + outBam + ".clean"
if (!intermediate && !qscript.intervalString.isEmpty()) this.intervalsString ++= List(qscript.intervalString)
if (!intermediate && qscript.intervals != null) this.intervals :+= qscript.intervals
}
class dedup (inBam: String, outBam: String, metricsFile: String) extends PicardBamJarFunction {
class dedup (inBam: String, outBam: String, metricsFile: String) extends PicardBamFunction {
@Input(doc="fixed bam") var clean: File = new File(inBam)
@Output(doc="deduped bam") var deduped: File = new File(outBam)
@Output(doc="deduped bam index") var dedupedIndex: File = new File(outBam + ".bai")
@ -169,7 +169,7 @@ class dataProcessing extends QScript {
override def outputBam = deduped
override def commandLine = super.commandLine + " M=" + metricsFile + " CREATE_INDEX=true"
sortOrder = null
this.memoryLimit = Some(6)
this.memoryLimit = 6
this.jarFile = qscript.dedupJar
this.isIntermediate = true
this.jobName = queueLogDir + outBam + ".dedup"
@ -188,8 +188,8 @@ class dataProcessing extends QScript {
this.input_file :+= new File (inBam)
this.recal_file = new File(inRecalFile)
this.out = new File(outBam)
this.U = Some(org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION) // todo -- update this with the last consensus between Tim, Matt and Eric. This is ugly!
this.index_output_bam_on_the_fly = Some(true)
this.U = org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION // todo -- update this with the last consensus between Tim, Matt and Eric. This is ugly!
this.index_output_bam_on_the_fly = true
this.jobName = queueLogDir + outBam + ".recalibration"
}

22
scala/qscript/oneoffs/carneiro/dataProcessingV2.scala
View File

@ -1,7 +1,7 @@
package oneoffs.carneiro
import org.broadinstitute.sting.queue.extensions.gatk._
import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction
import org.broadinstitute.sting.queue.extensions.picard.PicardBamFunction
import org.broadinstitute.sting.queue.QScript
import org.broadinstitute.sting.queue.function.ListWriterFunction
@ -173,11 +173,11 @@ class dataProcessingV2 extends QScript {
trait CommandLineGATKArgs extends CommandLineGATK {
this.jarFile = qscript.GATKjar
this.reference_sequence = qscript.reference
this.memoryLimit = Some(4)
this.memoryLimit = 4
this.isIntermediate = true
}
case class joinBams (inBams: List[File], outBam: File) extends PicardBamJarFunction {
case class joinBams (inBams: List[File], outBam: File) extends PicardBamFunction {
@Input(doc="input bam list") var join = inBams
@Output(doc="joined bam") var joined = outBam
@Output(doc="joined bam index") var joinedIndex = new File(outBam + "bai")
@ -194,7 +194,7 @@ class dataProcessingV2 extends QScript {
if (!knownsOnly)
this.input_file :+= inBams
this.out = outIntervals
this.mismatchFraction = Some(0.0)
this.mismatchFraction = 0.0
this.rodBind :+= RodBind("dbsnp", "VCF", dbSNP)
this.rodBind :+= RodBind("indels", "VCF", indels)
this.scatterCount = nContigs
@ -210,14 +210,14 @@ class dataProcessingV2 extends QScript {
this.rodBind :+= RodBind("indels", "VCF", qscript.indels)
this.useOnlyKnownIndels = knownsOnly
this.doNotUseSW = useSW
this.compress = Some(0)
this.U = Some(org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION) // todo -- update this with the last consensus between Tim, Matt and Eric. This is ugly!
this.compress = 0
this.U = org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION // todo -- update this with the last consensus between Tim, Matt and Eric. This is ugly!
this.scatterCount = nContigs
this.analysisName = queueLogDir + outBam + ".clean"
this.jobName = queueLogDir + outBam + ".clean"
}
case class dedup (inBam: File, outBam: File, metricsFile: File) extends PicardBamJarFunction {
case class dedup (inBam: File, outBam: File, metricsFile: File) extends PicardBamFunction {
@Input(doc="fixed bam") var clean = inBam
@Output(doc="deduped bam") var deduped = outBam
@Output(doc="deduped bam index") var dedupedIndex = new File(outBam + ".bai")
@ -226,7 +226,7 @@ class dataProcessingV2 extends QScript {
override def outputBam = deduped
override def commandLine = super.commandLine + " M=" + metricsFile + " CREATE_INDEX=true"
sortOrder = null
this.memoryLimit = Some(6)
this.memoryLimit = 6
this.jarFile = qscript.dedupJar
this.analysisName = queueLogDir + outBam + ".dedup"
this.jobName = queueLogDir + outBam + ".dedup"
@ -246,12 +246,12 @@ class dataProcessingV2 extends QScript {
@Output(doc="recalibrated bam index") var recalIndex = new File(outBam + ".bai")
this.input_file :+= inBam
this.recal_file = inRecalFile
this.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY)
this.baq = org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY
this.out = outBam
if (!qscript.intervalString.isEmpty()) this.intervalsString ++= List(qscript.intervalString)
else if (qscript.intervals != null) this.intervals :+= qscript.intervals
this.U = Some(org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION) // todo -- update this with the last consensus between Tim, Matt and Eric. This is ugly!
this.index_output_bam_on_the_fly = Some(true)
this.U = org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION // todo -- update this with the last consensus between Tim, Matt and Eric. This is ugly!
this.index_output_bam_on_the_fly = true
this.analysisName = queueLogDir + outBam + ".recalibration"
this.jobName = queueLogDir + outBam + ".recalibration"

10
scala/qscript/oneoffs/carneiro/justClean.scala
View File

@ -42,10 +42,10 @@ class justClean extends QScript {
target.input_file :+= input
target.out = tIntervals
target.reference_sequence = reference
target.mismatchFraction = Some(0.0)
target.mismatchFraction = 0.0
target.rodBind :+= RodBind("dbsnp", "VCF", dbSNP)
target.rodBind :+= RodBind("indels", "VCF", indels)
target.memoryLimit = Some(6)
target.memoryLimit = 6
target.jobName = queueLogDir + tIntervals + ".atarget"
target.jarFile = GATKjar
target.scatterCount = 84
@ -60,11 +60,11 @@ class justClean extends QScript {
clean.rodBind :+= RodBind("dbsnp", "VCF", dbSNP)
clean.rodBind :+= RodBind("indels", "VCF", indels)
clean.doNotUseSW = true
clean.compress = Some(0)
clean.U = Some(org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION) // todo -- update clean with the last consensus between Tim, Matt and Eric. This is ugly!
clean.compress = 0
clean.U = org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.NO_READ_ORDER_VERIFICATION // todo -- update clean with the last consensus between Tim, Matt and Eric. This is ugly!
clean.jobName = queueLogDir + outBam + ".clean"
clean.jarFile = GATKjar
clean.memoryLimit = Some(12)
clean.memoryLimit = 12
clean.scatterCount = 84
add(clean);

20
scala/qscript/oneoffs/carneiro/pbCalling.scala
View File

@ -158,20 +158,20 @@ class pbCalling extends QScript {
this.reference_sequence = t.reference
this.intervalsString ++= List(t.intervals)
this.scatterCount = 63 // the smallest interval list has 63 intervals, one for each Mb on chr20
this.dcov = Some( if ( t.isLowpass ) { 50 } else { 250 } )
this.stand_call_conf = Some( if ( t.isLowpass ) { 4.0 } else { 30.0 } )
this.stand_emit_conf = Some( if ( t.isLowpass ) { 4.0 } else { 30.0 } )
this.dcov = if ( t.isLowpass ) { 50 } else { 250 }
this.stand_call_conf = if ( t.isLowpass ) { 4.0 } else { 30.0 }
this.stand_emit_conf = if ( t.isLowpass ) { 4.0 } else { 30.0 }
this.input_file :+= t.bamList
this.out = t.rawVCF
this.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY)
this.baq = org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY
this.analysisName = t.name + "_UG"
if (t.dbsnpFile.endsWith(".rod"))
this.DBSNP = new File(t.dbsnpFile)
else if (t.dbsnpFile.endsWith(".vcf"))
this.rodBind :+= RodBind("dbsnp", "VCF", t.dbsnpFile)
// Ridiculous workaround to get pacbio data to run.. never commit this!
this.deletions = Some(0.5)
this.mbq = Some(10)
this.deletions = 0.5
this.mbq = 10
}
// 2.) Filter SNPs
@ -188,7 +188,7 @@ class pbCalling extends QScript {
}
class VQSR(t: Target, goldStandard: Boolean) extends ContrastiveRecalibrator {
this.memoryLimit = Some(6)
this.memoryLimit = 6
this.intervalsString ++= List(t.intervals)
this.rodBind :+= RodBind("input", "VCF", if ( goldStandard ) { t.goldStandard_VCF } else { t.filteredVCF } )
this.rodBind :+= RodBind("hapmap", "VCF", t.hapmapFile)
@ -208,12 +208,12 @@ class pbCalling extends QScript {
}
class applyVQSR (t: Target, goldStandard: Boolean) extends ApplyRecalibration {
this.memoryLimit = Some(4)
this.memoryLimit = 4
this.intervalsString ++= List(t.intervals)
this.rodBind :+= RodBind("input", "VCF", if ( goldStandard ) { t.goldStandard_VCF } else { t.filteredVCF } )
this.tranches_file = if ( goldStandard ) { t.goldStandardTranchesFile } else { t.tranchesFile}
this.recal_file = if ( goldStandard ) { t.goldStandardRecalFile } else { t.recalFile }
this.fdr_filter_level = Some(2.0)
this.fdr_filter_level = 2.0
this.out = t.recalibratedVCF
}
@ -226,7 +226,7 @@ class pbCalling extends QScript {
this.intervalsString ++= List(t.intervals)
this.out = t.cutVCF
this.tranchesFile = t.tranchesFile
this.fdr_filter_level = Some(1.0)
this.fdr_filter_level = 1.0
if (t.dbsnpFile.endsWith(".rod"))
this.DBSNP = new File(t.dbsnpFile)
else if (t.dbsnpFile.endsWith(".vcf"))

36
scala/qscript/oneoffs/chartl/BootstrapCalls.q
View File

@ -37,14 +37,14 @@ class BootstrapCalls extends QScript {
trait UGArgs extends UnifiedGenotyper {
this.input_file = bams
this.reference_sequence = reference
this.dcov = Some(downsamplingLevel)
this.dcov = downsamplingLevel
this.intervals :+= intervalFile
this.stand_call_conf = Some(standCallConf)
this.stand_emit_conf = Some(standCallConf)
this.stand_call_conf = standCallConf
this.stand_emit_conf = standCallConf
this.rodBind :+= new RodBind("dbsnp","vcf",dbsnp)
this.scatterCount = 20
this.jarFile = sting
this.memoryLimit = Some(4)
this.memoryLimit = 4
}
val bootstrapBase = swapExt(bootstrapMergedOut,".vcf",".boot%d.vcf").getAbsolutePath
@ -63,7 +63,7 @@ class BootstrapCalls extends QScript {
this.intervals :+= intervalFile
this.scatterCount = 40
this.jarFile = sting
this.memoryLimit = Some(4)
this.memoryLimit = 4
this.rodBind ++= calls.map(u => u.out).zipWithIndex.map(u => new RodBind("bootstrap_%d".format(u._2),"vcf",u._1))
this.out = bootstrapMergedOut
}
@ -81,11 +81,11 @@ class BootstrapCalls extends QScript {
this.rodBind :+= new RodBind("truth1kg","vcf", new File("/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/Omni2.5_chip/1212samples.b37.sites.vcf"))
this.cluster_file = swapExt(bootstrapMergedOut,"vcf","cluster")
this.use_annotation ++= List("QD", "SB", "HaplotypeScore", "HRun")
this.qual = Some(100)
this.std = Some(3.5)
this.mG = Some(8)
this.qual = 100
this.std = 3.5
this.mG = 8
this.trustAllPolymorphic = true
this.memoryLimit = Some(8)
this.memoryLimit = 8
this.jarFile = sting
}
@ -102,11 +102,11 @@ class BootstrapCalls extends QScript {
this.rodBind :+= new RodBind("truthHapMap","vcf",new File("/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/HapMap/3.3/sites_r27_nr.b37_fwd.vcf"))
this.rodBind :+= new RodBind("truth1kg","vcf", new File("/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/Omni2.5_chip/1212samples.b37.sites.vcf"))
this.cluster_file = swapExt(bootstrapMergedOut,"vcf","cluster")
this.sm = Some(org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY)
this.sm = org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY
this.tranche ++= List("0.1", "0.5", "0.7", "1.0", "3.0", "5.0", "10.0", "100.0")
this.trustAllPolymorphic = true
this.tranchesFile = swapExt(bootstrapMergedOut,"vcf","tranche")
this.memoryLimit=Some(8)
this.memoryLimit=8
this.jarFile = sting
this.rodBind :+= new RodBind("dbsnp","vcf",dbsnp)
}
@ -119,10 +119,10 @@ class BootstrapCalls extends QScript {
this.intervals :+= intervalFile
this.rodBind :+= new RodBind("input","vcf",recal.out)
this.tranchesFile = recal.tranchesFile
this.fdr_filter_level = Some(1.0)
this.fdr_filter_level = 1.0
this.out = swapExt(bootstrapMergedOut,".vcf",".recal.cut.vcf")
this.jarFile = sting
this.memoryLimit = Some(4)
this.memoryLimit = 4
this.scatterCount = 5
}
@ -149,12 +149,12 @@ class BootstrapCalls extends QScript {
this.intervals :+= intervalFile
this.rodBind :+= new RodBind("loCov","vcf",rm.noheadvcf)
this.rodBind :+= new RodBind("hiCov","vcf",new File("/humgen/gsa-pipeline/PVQF4/all_batches_v001/batch_001/SnpCalls/ESPGO_Gabriel_NHLBI_EOMI_setone_EOMI_Project.cleaned.annotated.handfiltered.vcf"))
this.variantMergeOptions = Some(VariantMergeType.UNION)
this.genotypeMergeOptions = Some(GenotypeMergeType.PRIORITIZE)
this.variantMergeOptions = VariantMergeType.UNION
this.genotypeMergeOptions = GenotypeMergeType.PRIORITIZE
this.priority = "hiCov,loCov"
this.out = swapExt(bootstrapMergedOut,".vcf",".merged.combined.vcf")
this.jarFile = sting
this.memoryLimit = Some(6)
this.memoryLimit = 6
}
var combine : CombineVariants = new CombineVariants with CombineArgs
@ -175,8 +175,8 @@ class BootstrapCalls extends QScript {
"\"set == 'hiCov'\"","\"set == 'FilteredInAll'\"")
this.EV = List("TiTvVariantEvaluator","CountVariants","CompOverlap")
this.out = swapExt(bootstrapMergedOut,".vcf",".merged.combined.eval")
this.nt = Some(8)
this.memoryLimit = Some(12)
this.nt = 8
this.memoryLimit = 12
}
var eval : VariantEval = new VariantEval with EvalArgs

7
scala/qscript/oneoffs/chartl/RefineGenotypesAndMerge.q
View File

@ -3,7 +3,6 @@ import net.sf.picard.reference.FastaSequenceFile
import org.broadinstitute.sting.datasources.pipeline.Pipeline
import org.broadinstitute.sting.gatk.DownsampleType
import org.broadinstitute.sting.queue.extensions.gatk._
import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction
import org.broadinstitute.sting.queue.extensions.samtools._
import org.broadinstitute.sting.queue.{QException, QScript}
import collection.JavaConversions._
@ -69,7 +68,7 @@ class RefineGenotypesAndMerge extends QScript {
refine.beagleInput = beagleInput.out
refine.beagleOutputBase = beagleBase
refine.beagleMemoryGigs = 6
refine.memoryLimit = Some(6)
refine.memoryLimit = 6
refine.freezeOutputs
var unzipPhased = new GunzipFile(refine.beaglePhasedFile,swapExt(refine.beaglePhasedFile,".gz",".bgl"))
@ -94,8 +93,8 @@ class RefineGenotypesAndMerge extends QScript {
def mergeVCFs(vcfs: List[File], outputVCF: File) : CombineVariants = {
var cv = new CombineVariants with GATKArgs
cv.out = outputVCF
cv.genotypemergeoption = Some(org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.GenotypeMergeType.UNSORTED)
cv.variantmergeoption = Some(org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.UNION)
cv.genotypemergeoption = org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.GenotypeMergeType.UNSORTED
cv.variantmergeoption = org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.UNION
cv.priority = (vcfs.foldLeft[List[String]](Nil)( (bNames,vcf) => bNames ::: List[String](swapExt(vcf,".vcf","").getName))).mkString(",")
cv.rodBind = vcfs.foldLeft[List[RodBind]](Nil)( (rods,vcf) => rods ::: List[RodBind](new RodBind(swapExt(vcf,".vcf","").getName,"VCF",vcf)))

14
scala/qscript/oneoffs/chartl/expanded_targets.q
View File

@ -39,14 +39,14 @@ class expanded_targets extends QScript {
rtc.out = swapExt(userDir,u,".bam",".clean.targets.interval_list")
rtc.input_file :+= u.getAbsoluteFile
rtc.intervals :+= cleanIntervals.outList
rtc.memoryLimit = Some(6)
rtc.memoryLimit = 6
rtc
})
val clean : List[IndelRealigner] = realign.map( u => {
var cleaner : IndelRealigner = new IndelRealigner with GATKArgs
cleaner.targetIntervals = u.out
cleaner.input_file = u.input_file
cleaner.memoryLimit = Some(6)
cleaner.memoryLimit = 6
cleaner.out = new File(userDir+"/"+swapExt(u.out,".bam",".expanded.targets.bam").getName)
cleaner.intervals :+= cleanIntervals.outList
cleaner
@ -71,10 +71,10 @@ class expanded_targets extends QScript {
var call : UnifiedGenotyper = new UnifiedGenotyper with GATKArgs
call.input_file = bams
call.out = swapExt(iList,".interval_list",".raw.vcf")
call.trig_emit_conf = Some(0.0)
call.trig_emit_conf = 0.0
call.rodBind :+= new RodBind("trigger","vcf",thisTrigger)
call.scatterCount = 10
call.memoryLimit = Some(6)
call.memoryLimit = 6
var filter : VariantFiltration = new VariantFiltration with GATKArgs
filter.rodBind :+= new RodBind("variant","vcf",call.out)
filter.filterExpression :+= "\"QD<5.0\""
@ -87,7 +87,7 @@ class expanded_targets extends QScript {
callHiseq.input_file = List(new File("/seq/picard_aggregation/EXT1/NA12878/v3/NA12878.bam"))
callHiseq.rodBind :+= new RodBind("trigger","vcf",filter.out)
callHiseq.out = swapExt(iList,".interval_list",".hiSeq.genotypes.vcf")
callHiseq.trig_emit_conf = Some(0.0)
callHiseq.trig_emit_conf = 0.0
callHiseq.scatterCount = 5
add(call,filter,callHiseq)
@ -98,8 +98,8 @@ class expanded_targets extends QScript {
eval.rodBind :+= new RodBind("compHiSeq_atSites","vcf",callHiseq.out)
eval.rodBind :+= new RodBind("compOMNI","vcf",new File("/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/Omni2.5_chip/764samples.deduped.b37.annot.vcf"))
eval.out = swapExt(iList,".interval_list",".eval")
eval.reportType = Option(org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType.CSV)
eval.memoryLimit = Some(4)
eval.reportType = org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType.CSV
eval.memoryLimit = 4
add(eval)
eval.out

10
scala/qscript/oneoffs/chartl/old/private_mutations_old.q
View File

@ -165,8 +165,8 @@ class private_mutations_old extends QScript {
getVennExS.intervals :+= exome_single.out_list
getVennExS.reference_sequence = new File("/humgen/1kg/reference/human_g1k_v37.fasta")
getVennExS.jarFile = new File("/humgen/gsa-scr1/chartl/sting/dist/GenomeAnalysisTK.jar")
getVennExS.genotypeMergeOptions = Some(VariantContextUtils.GenotypeMergeType.UNIQUIFY)
getVennExS.variantMergeOptions = Some(VariantContextUtils.VariantMergeType.UNION)
getVennExS.genotypeMergeOptions = VariantContextUtils.GenotypeMergeType.UNIQUIFY
getVennExS.variantMergeOptions = VariantContextUtils.VariantMergeType.UNION
getVennExS.out = new File(VCF_DIR + "g1k_exome_plus_lowpass.singlesample.merged.exome.sites.vcf")
//add(getVennExS)
@ -178,8 +178,8 @@ class private_mutations_old extends QScript {
getVennLPS.intervals :+= lowpass_single.out_list
getVennLPS.reference_sequence = new File("/humgen/1kg/reference/human_g1k_v37.fasta")
getVennLPS.jarFile = new File("/humgen/gsa-scr1/chartl/sting/dist/GenomeAnalysisTK.jar")
getVennLPS.genotypeMergeOptions = Some(VariantContextUtils.GenotypeMergeType.UNIQUIFY)
getVennLPS.variantMergeOptions = Some(VariantContextUtils.VariantMergeType.UNION)
getVennLPS.genotypeMergeOptions = VariantContextUtils.GenotypeMergeType.UNIQUIFY
getVennLPS.variantMergeOptions = VariantContextUtils.VariantMergeType.UNION
getVennLPS.out = new File(VCF_DIR + "g1k_exome_plus_lowpass.singlesample.merged.lowpass.sites.vcf")
add(getVennLPS)
@ -397,7 +397,7 @@ class private_mutations_old extends QScript {
genotype.intervals :+= sites.out_list
genotype.out = new File(VCF_DIR+"g1k_lowpass.%s.exome_sites.vcf".format(s))
genotype.input_file :+= bamList.outList
genotype.memoryLimit = Some(3)
genotype.memoryLimit = 3
genotype.output_all_callable_bases = true
add(genotype)

29
scala/qscript/oneoffs/chartl/omni_qc.q
View File

@ -5,7 +5,6 @@ import org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils
import org.broadinstitute.sting.gatk.DownsampleType
import org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeCalculationModel.Model
import org.broadinstitute.sting.queue.extensions.gatk._
import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction
import org.broadinstitute.sting.queue.extensions.samtools._
import org.broadinstitute.sting.queue.{QException, QScript}
import collection.JavaConversions._
@ -201,7 +200,7 @@ class omni_qc extends QScript {
eval1KG_exclude.evalModule :+= "GenotypeConcordance"
eval1KG_exclude.evalModule :+= "SimpleMetricsBySample"
eval1KG_exclude.reference_sequence = b36_ref
eval1KG_exclude.reportType = Some(VE2TemplateType.CSV)
eval1KG_exclude.reportType = VE2TemplateType.CSV
eval1KG_exclude.intervalsString :+= pilot3_interval_list
eval1KG_exclude.out = new File(eval_dir+"%s_vs_%s.%s".format("OMNI_764","Pilot3","exclude.mixups.eval.csv"))
@ -222,7 +221,7 @@ class omni_qc extends QScript {
combine.rodBind :+= new RodBind("CEU","VCF",gunzip_p1_ceu.outFile)
combine.rodBind :+= new RodBind("ASN","VCF",gunzip_p1_chb.outFile)
combine.rodBind :+= new RodBind("YRI","VCF",gunzip_p1_yri.outFile)
combine.genotypeMergeOptions = Some(VariantContextUtils.GenotypeMergeType.UNIQUIFY)
combine.genotypeMergeOptions = VariantContextUtils.GenotypeMergeType.UNIQUIFY
combine.priority = "%s,%s,%s".format("CEU","ASN","YRI")
combine.out = new File(vcf_dir+"Pilot1_Populations_Combined.vcf")
@ -257,7 +256,7 @@ class omni_qc extends QScript {
vEval.evalModule :+= "SimpleMetricsBySample"
vEval.intervalsString :+= intervals
vEval.reference_sequence = reference
vEval.reportType = Some(VE2TemplateType.CSV)
vEval.reportType = VE2TemplateType.CSV
vEval.out = new File(eval_dir+base+".eval.csv")
@ -318,8 +317,8 @@ class omni_qc extends QScript {
panelCombine.priority = if ( panelCombine.priority.equals("") ) p else panelCombine.priority + "," + p
}
panelCombine.out = OMNI_b36_panel_vcf
panelCombine.genotypeMergeOptions = Some(VariantContextUtils.GenotypeMergeType.REQUIRE_UNIQUE)
panelCombine.variantMergeOptions = Some(VariantContextUtils.VariantMergeType.UNION)
panelCombine.genotypeMergeOptions = VariantContextUtils.GenotypeMergeType.REQUIRE_UNIQUE
panelCombine.variantMergeOptions = VariantContextUtils.VariantMergeType.UNION
panelCombine.setKey = "panel"
add(panelCombine)
@ -348,7 +347,7 @@ class omni_qc extends QScript {
eval.rodBind :+= new RodBind("comp%s".format(cBase),"VCF",compVCF)
eval.noStandard = true
eval.E :+= "AlleleFrequencyComparison"
eval.reportType = Some(VE2TemplateType.CSV)
eval.reportType = VE2TemplateType.CSV
eval.out = new File(eval_dir+"%s_vs_%s_allele_frequency.eval".format(eBase,cBase))
add(eval)
@ -358,7 +357,7 @@ class omni_qc extends QScript {
combine.rodBind :+= new RodBind(eBase,"VCF",evalVCF)
combine.rodBind :+= new RodBind(cBase,"VCF",compVCF)
combine.out = new File(vcf_dir+"%s_plus_%s.vcf".format(eBase,cBase))
combine.genotypeMergeOptions = Some(VariantContextUtils.GenotypeMergeType.UNIQUIFY)
combine.genotypeMergeOptions = VariantContextUtils.GenotypeMergeType.UNIQUIFY
combine.priority = "%s,%s".format(eBase,cBase)
//add(combine)
@ -386,8 +385,8 @@ class omni_qc extends QScript {
combine.priority = "%s%s".format("beagle",c)
}
}
combine.genotypeMergeOptions = Some(VariantContextUtils.GenotypeMergeType.PRIORITIZE)
combine.variantMergeOptions = Some(VariantContextUtils.VariantMergeType.UNION)
combine.genotypeMergeOptions = VariantContextUtils.GenotypeMergeType.PRIORITIZE
combine.variantMergeOptions = VariantContextUtils.VariantMergeType.UNION
combine.out = swapExt(pilot1_with_na12878_vcf,".vcf",".beagle_refined_with_omni.vcf")
@ -408,7 +407,7 @@ class omni_qc extends QScript {
eval.E :+= "GenotypeConcordance"
eval.out = new File(eval_dir+"NA12878.lowpass.beagle.vs.HiSeq.eval")
eval.excludeIntervals :+= new File(pilot1_interval_list)
eval.reportType = Some(VE2TemplateType.CSV)
eval.reportType = VE2TemplateType.CSV
add(eval)
@ -420,7 +419,7 @@ class omni_qc extends QScript {
eval2.sample :+= "NA12878"
eval2.out = new File(eval_dir+"NA12878.lowpass.nochip.vs.Hiseq.eval")
eval2.excludeIntervals :+= new File(pilot1_interval_list)
eval2.reportType = Some(VE2TemplateType.CSV)
eval2.reportType = VE2TemplateType.CSV
add(eval2)
@ -432,7 +431,7 @@ class omni_qc extends QScript {
eval3.sample :+= "NA12878"
eval3.out = new File(eval_dir+"NA12878.lowpass.nochip.norefined.vs.Hiseq.eval")
eval3.excludeIntervals :+= new File(pilot1_interval_list)
eval3.reportType = Some(VE2TemplateType.CSV)
eval3.reportType = VE2TemplateType.CSV
add(eval3)
}
@ -443,7 +442,7 @@ class omni_qc extends QScript {
beagleInput.intervalsString :+= chr
beagleInput.variantVCF = pilot1_with_na12878_vcf
beagleInput.rodBind :+= new RodBind("validation","VCF",omnivcf)
beagleInput.validation_genotype_ptrue = Some(0.99)
beagleInput.validation_genotype_ptrue = 0.99
beagleInput.out = new File(scratch_dir+"/"+swapExt(beagleInput.variantVCF.getName,".vcf",".%s.beagle".format(chr)))
println (beagleInput.out.getAbsolutePath)
@ -451,7 +450,7 @@ class omni_qc extends QScript {
runBeagle.beagleInput = beagleInput.out
runBeagle.beagleOutputBase = "Pilot1_NA12878_Beagle_with_OMNI_chr%s".format(chr)
runBeagle.beagleMemoryGigs = 6
runBeagle.memoryLimit = Some(6)
runBeagle.memoryLimit = 6
runBeagle.beagleOutputDir = ""
runBeagle.freezeOutputs

6
scala/qscript/oneoffs/chartl/private_mutations.q
View File

@ -61,7 +61,7 @@ class private_mutations extends QScript {
eval_all.noStandard = true
eval_all.E :+= "ACTransitionTable"
eval_all.out = swapExt(finalMergedVCF,".vcf",".perm.csv")
eval_all.reportType = Some(org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType.CSV)
eval_all.reportType = org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType.CSV
add(eval_all)
@ -70,7 +70,7 @@ class private_mutations extends QScript {
eval_afr.rodBind :+= new RodBind("compEUR","VCF",extract_eur.outputVCF)
eval_afr.E :+= "ACTransitionTable"
eval_afr.out = swapExt(extract_afr.outputVCF,".vcf",".perm.csv")
eval_afr.reportType = Some(org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType.CSV)
eval_afr.reportType = org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType.CSV
eval_afr.noStandard = true
add(eval_afr)
@ -80,7 +80,7 @@ class private_mutations extends QScript {
eval_eur.rodBind :+= new RodBind("evalEUR","VCF",extract_eur.outputVCF)
eval_eur.E :+= "ACTransitionTable"
eval_eur.out = swapExt(extract_eur.outputVCF,".vcf",".perm.csv")
eval_eur.reportType = Some(org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType.CSV)
eval_eur.reportType = org.broadinstitute.sting.utils.report.VE2ReportFactory.VE2TemplateType.CSV
eval_eur.noStandard = true
add(eval_eur)

29
scala/qscript/oneoffs/delangel/Phase1IndelCalling.scala
View File

@ -4,7 +4,6 @@ import org.broadinstitute.sting.datasources.pipeline.Pipeline
import org.broadinstitute.sting.gatk.DownsampleType
import org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeLikelihoodsCalculationModel
import org.broadinstitute.sting.queue.extensions.gatk._
import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction
import org.broadinstitute.sting.queue.extensions.samtools._
import org.broadinstitute.sting.queue.function.scattergather.{GatherFunction, CloneFunction, ScatterFunction}
import org.broadinstitute.sting.queue.{QException, QScript}
@ -55,7 +54,7 @@ class Phase1Calling extends QScript {
trait CommandLineGATKArgs extends CommandLineGATK {
this.jarFile = qscript.gatkJar
this.reference_sequence = qscript.reference
this.memoryLimit = Some(3)
this.memoryLimit = 3
this.jobTempDir = qscript.tmpDir
this.jobQueue = "gsa";
@ -102,15 +101,15 @@ class Phase1Calling extends QScript {
}
call.dcov = Some( 50 )
call.stand_call_conf = Some( 4.0 )
call.stand_emit_conf = Some( 4.0 )
call.dcov = 50
call.stand_call_conf = 4.0
call.stand_emit_conf = 4.0
call.input_file :+= bamList
call.out = rawCalls
call.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF)
call.baq = org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF
call.analysisName = baseName + "_UG"
call.rodBind :+= RodBind("dbsnp", "VCF", qscript.dbSNPIndels)
call.glm = Some(GenotypeLikelihoodsCalculationModel.Model.DINDEL)
call.glm = GenotypeLikelihoodsCalculationModel.Model.DINDEL
var filter = new VariantFiltration with CommandLineGATKArgs
@ -132,9 +131,9 @@ class Phase1Calling extends QScript {
gvc.use_annotation ++= List("QD", "SB", "HaplotypeScore", "HRun")
gvc.analysisName = baseName + "_GVC"
gvc.intervalsString ++= List(qscript.intervals)
//gvc.qual = Some(100) // clustering parameters to be updated soon pending new experimentation results
//gvc.std = Some(4.5)
//gvc.mG = Some(6)
//gvc.qual = 100 // clustering parameters to be updated soon pending new experimentation results
//gvc.std = 4.5
//gvc.mG = 6
/*
var vr = new VariantRecalibrator with CommandLineGATKArgs
vr.rodBind :+= RodBind("1kg", "VCF", qscript.omni)
@ -146,13 +145,13 @@ class Phase1Calling extends QScript {
vr.analysisName = baseName + "_VR"
vr.intervalsString ++= List(qscript.intervals)
vr.ignoreFilter ++= List("HARD_TO_VALIDATE")
vr.target_titv = Some(2.3)
vr.sm = Some(org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY)
vr.target_titv = 2.3
vr.sm = org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY
vr.tranche ++= List("0.1", "1.0", "2.0", "3.0", "5.0", "10.0", "100.0")
vr.out = recalibratedCalls
vr.priorDBSNP = Some(10.0)
vr.priorHapMap = Some(12.0)
vr.prior1KG = Some(12.0)
vr.priorDBSNP = 10.0
vr.priorHapMap = 12.0
vr.prior1KG = 12.0
vr.tranchesFile = tranchesFile
add(call, filter, gvc, vr) */

12
scala/qscript/oneoffs/depristo/1kg_table1.scala
View File

@ -127,10 +127,10 @@ class VariantEval(vcfIn: String, evalOut: String, vcfType: String = "VCF") exten
this.rodBind :+= RodBind("eval", vcfType, vcfFile)
this.out = new File(evalOut)
this.DBSNP = new File("/humgen/gsa-hpprojects/GATK/data/dbsnp_129_b36.rod")
this.reportType = Some(VE2TemplateType.Grep)
this.reportType = VE2TemplateType.Grep
this.noStandard = true;
this.evalModule :+= "CompOverlap"
this.memoryLimit = Some(3)
this.memoryLimit = 3
override def dotString = "VariantEval: " + vcfFile.getName
}
@ -147,8 +147,8 @@ class StatPop(target: Target) extends CommandLineFunction {
class Combine(vcfsInArg: List[String], vcfOutPath: String) extends org.broadinstitute.sting.queue.extensions.gatk.CombineVariants with UNIVERSAL_GATK_ARGS {
val vcfs = vcfsInArg.map((x: String) => new File(x))
val vcfFile = new File(vcfOutPath)
this.variantmergeoption = Some(VariantMergeType.UNION)
this.genotypemergeoption = Some(GenotypeMergeType.PRIORITIZE)
this.variantmergeoption = VariantMergeType.UNION
this.genotypemergeoption = GenotypeMergeType.PRIORITIZE
this.out = vcfFile
this.rodBind ++= vcfs.map( input => RodBind(input.getName,"VCF",input) )
this.rod_priority_list = vcfs.map( _.getName ).mkString(",")
@ -164,8 +164,8 @@ class DepthOfCoverage(bam: String, docOutPath: String, interval: String) extends
val bamFile = new File(bam)
this.omitIntervalStatistics = true
this.omitDepthOutputAtEachBase = true
this.minBaseQuality = Some(0)
this.minMappingQuality = Some(0)
this.minBaseQuality = 0
this.minMappingQuality = 0
this.out = new File(docOutPath)
this.input_file :+= bamFile
if (interval != null) {

22
scala/qscript/oneoffs/depristo/CleaningTest.scala
View File

@ -4,8 +4,8 @@ package oneoffs.depristo
import org.broadinstitute.sting.queue.extensions.gatk._
import org.broadinstitute.sting.queue.QScript
import collection.JavaConversions._
import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction
import org.broadinstitute.sting.queue.function.JarCommandLineFunction
import org.broadinstitute.sting.queue.extensions.picard.PicardBamFunction
import org.broadinstitute.sting.queue.function.JavaCommandLineFunction
class CleaningTest extends QScript {
@ -45,7 +45,7 @@ class CleaningTest extends QScript {
trait CommandLineGATKArgs extends CommandLineGATK {
this.jarFile = qscript.gatkJar
this.reference_sequence = qscript.reference
this.memoryLimit = Some(4)
this.memoryLimit = 4
this.jobTempDir = qscript.tmpDir
}
@ -62,7 +62,7 @@ class CleaningTest extends QScript {
target.input_file :+= bamList
target.intervalsString :+= interval
target.out = targetIntervals
target.mismatchFraction = Some(0.0)
target.mismatchFraction = 0.0
target.rodBind :+= RodBind("dbsnp", "VCF", qscript.dbSNP)
target.rodBind :+= RodBind("indels3", "VCF", qscript.dindelEURCalls)
//target.jobName = baseName + ".target"
@ -79,7 +79,7 @@ class CleaningTest extends QScript {
clean.out = if ( cm ) cleanedBam else new File(cleanedBam + ".intermediate.bam")
clean.doNotUseSW = true
clean.constrainMovement = cm
clean.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF)
clean.baq = org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF
clean.rodBind :+= RodBind("dbsnp", "VCF", qscript.dbSNP)
clean.rodBind :+= RodBind("indels3", "VCF", qscript.dindelEURCalls)
//clean.sortInCoordinateOrderEvenThoughItIsHighlyUnsafe = true
@ -90,7 +90,7 @@ class CleaningTest extends QScript {
if ( ! cm ) {
// Explicitly run fix mates if the function won't be scattered.
val fixMates = new PicardBamJarFunction {
val fixMates = new PicardBamFunction {
// Declare inputs/outputs for dependency tracking.
@Input(doc="unfixed bam") var unfixed: File = _
@Output(doc="fixed bam") var fixed: File = _
@ -99,7 +99,7 @@ class CleaningTest extends QScript {
}
//fixMates.jobOutputFile = new File(".queue/logs/Cleaning/%s/FixMates.out".format(sampleId))
fixMates.memoryLimit = Some(4)
fixMates.memoryLimit = 4
fixMates.jarFile = qscript.picardFixMatesJar
fixMates.unfixed = clean.out
fixMates.fixed = cleanedBam
@ -110,7 +110,7 @@ class CleaningTest extends QScript {
add(fixMates)
}
val validate = new JarCommandLineFunction {
val validate = new JavaCommandLineFunction {
// Declare inputs/outputs for dependency tracking.
@Input(doc="unfixed bam") var unfixed: File = _
def inputBams = List(unfixed)
@ -119,12 +119,12 @@ class CleaningTest extends QScript {
}
//fixMates.jobOutputFile = new File(".queue/logs/Cleaning/%s/FixMates.out".format(sampleId))
validate.memoryLimit = Some(2)
validate.memoryLimit = 2
validate.jarFile = qscript.picardValidateJar
validate.unfixed = cleanedBam
add(validate)
val toQueryName = new PicardBamJarFunction {
val toQueryName = new PicardBamFunction {
// Declare inputs/outputs for dependency tracking.
@Input(doc="coordiante bam") var cobam: File = _
@Output(doc="query bam") var qnbam: File = _
@ -133,7 +133,7 @@ class CleaningTest extends QScript {
}
//fixMates.jobOutputFile = new File(".queue/logs/Cleaning/%s/FixMates.out".format(sampleId))
toQueryName.memoryLimit = Some(4)
toQueryName.memoryLimit = 4
toQueryName.jarFile = qscript.picardSortSamJar
toQueryName.cobam = cleanedBam
toQueryName.qnbam = new File(cleanedBam.getAbsolutePath + ".qn.bam")

14
scala/qscript/oneoffs/depristo/RefineGenotypesWithBeagle.q
View File

@ -36,7 +36,7 @@ class RefineGenotypesWithBeagle extends QScript {
trait GATKArgs extends CommandLineGATK {
this.reference_sequence = qscript.reference
this.jarFile = qscript.gatkJarFile
this.memoryLimit = Some(2)
this.memoryLimit = 2
}
// --------------------------------------------------------------------------------
@ -48,9 +48,9 @@ class RefineGenotypesWithBeagle extends QScript {
class GenotypeBAMAtSites(@Input bam: File, @Input sitesVCF: File, @Output genotypesVCF: File) extends UnifiedGenotyper with GATKArgs {
this.input_file = List(bam)
this.o = genotypesVCF
this.stand_call_conf = Some(0.0)
this.out_mode = Some(org.broadinstitute.sting.gatk.walkers.genotyper.UnifiedGenotyperEngine.OUTPUT_MODE.EMIT_ALL_SITES)
this.gt_mode = Some(org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES)
this.stand_call_conf = 0.0
this.out_mode = org.broadinstitute.sting.gatk.walkers.genotyper.UnifiedGenotyperEngine.OUTPUT_MODE.EMIT_ALL_SITES
this.gt_mode = org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES
this.rodBind :+= new RodBind("alleles","VCF",sitesVCF)
// we only want chromosome counts annotations
@ -62,7 +62,7 @@ class RefineGenotypesWithBeagle extends QScript {
// make sure we have the right intervals
if ( interval != null ) {
this.intervalsString = List(interval)
this.BTIMR = Some(org.broadinstitute.sting.utils.interval.IntervalSetRule.INTERSECTION)
this.BTIMR = org.broadinstitute.sting.utils.interval.IntervalSetRule.INTERSECTION
}
}
@ -73,7 +73,7 @@ class RefineGenotypesWithBeagle extends QScript {
// --------------------------------------------------------------------------------
class BeagleCommand(outputBase: String) extends CommandLineFunction {
this.memoryLimit = Some(BEAGLE_MEM_IN_GB)
this.memoryLimit = BEAGLE_MEM_IN_GB
// Note: These get set
@Output val beaglePhasedFile: File = new File(outputBase +".phased.gz")
@ -151,7 +151,7 @@ class RefineGenotypesWithBeagle extends QScript {
if ( interval != null ) evalBeagle.intervalsString = List(interval)
evalBeagle.variantVCF = evalVCF
evalBeagle.out = swapExt(outputVCF,".vcf",".unphased.beagle")
evalBeagle.bs = Some(percentLeftOut)
evalBeagle.bs = percentLeftOut
evalBeagle.bsvcf = swapExt(outputVCF,".vcf",".missing.vcf")
evalBeagle.missing = MISSING_KEY
//evalBeagle.isIntermediate = true

15
scala/qscript/oneoffs/depristo/VQSRCutByNRS.scala
View File

@ -1,4 +1,3 @@
import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction
import org.broadinstitute.sting.queue.extensions.gatk._
import org.broadinstitute.sting.queue.extensions.samtools.SamtoolsIndexFunction
import org.broadinstitute.sting.queue.QScript
@ -18,7 +17,7 @@ class VQSRCutByNRS extends QScript {
@Argument(fullName = "prefix", doc="Prefix argument", required=false)
var prefix: String = ""
trait UNIVERSAL_GATK_ARGS extends CommandLineGATK { logging_level = "INFO"; jarFile = gatkJarFile; memoryLimit = Some(3) }
trait UNIVERSAL_GATK_ARGS extends CommandLineGATK { logging_level = "INFO"; jarFile = gatkJarFile; memoryLimit = 3 }
class Target(val name: String, val reference: File, val rodName: String, val VCF: File, val intervals: Option[String], val titvTarget: Double) {
def clusterFile = new File(name + ".clusters")
@ -67,9 +66,9 @@ class GenerateVariantClusters(t: Target) extends org.broadinstitute.sting.queue.
this.use_annotation ++= List("QD", "SB", "HaplotypeScore", "HRun")
this.analysisName = t.name + "_Cluster"
if ( t.intervals != None ) this.intervalsString ++= List(t.intervals.get)
this.qual = Some(300)
this.std = Some(3.5)
this.mG = Some(16) // v2 calls
this.qual = 300
this.std = 3.5
this.mG = 16 // v2 calls
// ignores
this.ignoreFilter ++= FiltersToIgnore
}
@ -86,8 +85,8 @@ class VariantRecalibratorBase(t: Target, ans: List[String]) extends org.broadins
if ( t.intervals != None ) this.intervalsString ++= List(t.intervals.get)
this.ignoreFilter ++= FiltersToIgnore
this.ignoreFilter ++= List("HARD_TO_VALIDATE")
this.priorDBSNP = Some(2.0)
this.priorHapMap = Some(2.0)
this.priorDBSNP = 2.0
this.priorHapMap = 2.0
this.target_titv = t.titvTarget
this.use_annotation ++= ans
this.out = new File("/dev/null")
@ -100,7 +99,7 @@ class VariantRecalibratorTiTv(t: Target, ans: List[String], prefix: String) exte
}
class VariantRecalibratorNRS(t: Target, ans: List[String], prefix: String) extends VariantRecalibratorBase(t,ans) {
this.sm = Some(org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY)
this.sm = org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY
this.tranche ++= List("50", "25", "10", "5", "2", "1", "0.5", "0.1")
//this.out = new File(t.name + ".ts.recalibrated.vcf")
this.tranchesFile = new File(t.name + prefix + ".ts.tranches")

16
scala/qscript/oneoffs/depristo/manySampleUGPerformance.scala
View File

@ -29,7 +29,7 @@ class ManySampleUGPerformanceTesting extends QScript {
this.intervals = List(new File(TARGET_INTERVAL));
this.reference_sequence = referenceFile;
this.jobQueue = "gsa";
this.memoryLimit = Some(4)
this.memoryLimit = 4
//this.commandDirectory = new File("results");
}
@ -52,11 +52,11 @@ class ManySampleUGPerformanceTesting extends QScript {
// SNP calling
//add(new Call(sublist.list, nSamples, "dynamic_merge"))
val gt = new Call(bams, nSamples, name);
gt.exactCalculation = Some(org.broadinstitute.sting.gatk.walkers.genotyper.ExactAFCalculationModel.ExactCalculation.N2_GOLD_STANDARD)
gt.exactCalculation = org.broadinstitute.sting.gatk.walkers.genotyper.ExactAFCalculationModel.ExactCalculation.N2_GOLD_STANDARD
add(gt)
val gtLinear = new Call(bams, nSamples, name + "_linear");
gtLinear.exactCalculation = Some(org.broadinstitute.sting.gatk.walkers.genotyper.ExactAFCalculationModel.ExactCalculation.LINEAR_EXPERIMENTAL)
gtLinear.exactCalculation = org.broadinstitute.sting.gatk.walkers.genotyper.ExactAFCalculationModel.ExactCalculation.LINEAR_EXPERIMENTAL
add(gtLinear)
// SNP calling -- no annotations
@ -74,24 +74,24 @@ class ManySampleUGPerformanceTesting extends QScript {
}
class MergeBAMs(bamList: File) extends PrintReads with UNIVERSAL_GATK_ARGS {
this.memoryLimit = Some(3)
this.memoryLimit = 3
this.input_file :+= bamList
this.memoryLimit = Some(16)
this.memoryLimit = 16
this.o = new File(MERGED_DIR + "/" + bamList.getName + ".bam")
}
class Call(@Input(doc="foo") bamList: File, n: Int, name: String) extends UnifiedGenotyper with UNIVERSAL_GATK_ARGS {
@Output(doc="foo") var outVCF: File = new File("%s.%d.%s.vcf".format(bamList.getName, n, name))
this.input_file :+= bamList
this.stand_call_conf = Option(10.0)
this.dcov = Option(DCOV);
this.stand_call_conf = 10.0
this.dcov = DCOV;
this.o = outVCF
}
class MyCountLoci(@Input(doc="foo") bamList: File, n: Int, name: String) extends CountLoci with UNIVERSAL_GATK_ARGS {
@Output(doc="foo") var outFile: File = new File("%s.%d.%s.txt".format(bamList.getName, n, name))
this.input_file :+= bamList
this.dcov = Option(DCOV);
this.dcov = DCOV;
this.o = outFile
}

8
scala/qscript/oneoffs/depristo/resequencingSamples1KG.scala
View File

@ -20,8 +20,8 @@ class resequencingSamples1KG extends QScript {
this.intervals = List(new File(TARGET_INTERVAL));
this.reference_sequence = referenceFile;
this.jobQueue = "gsa";
this.et = Option(org.broadinstitute.sting.gatk.phonehome.GATKRunReport.PhoneHomeOption.STANDARD);
this.dcov = Option(50);
this.et = org.broadinstitute.sting.gatk.phonehome.GATKRunReport.PhoneHomeOption.STANDARD;
this.dcov = 50;
}
def script = {
@ -30,10 +30,10 @@ class resequencingSamples1KG extends QScript {
}
class MyQSample(@Input(doc="foo") bamList: File) extends QSample with UNIVERSAL_GATK_ARGS {
this.memoryLimit = Some(4)
this.memoryLimit = 4
this.input_file :+= bamList
//this.BTI = "genotypes"
this.nt = Option(10)
this.nt = 10
this.rodBind :+= RodBind("genotypes", "VCF", HM3)
this.o = new File("%s.qsample".format(bamList.getName))
}

4
scala/qscript/oneoffs/fromer/PhaseSamples.scala
View File

@ -38,7 +38,7 @@ class PhaseSamples extends QScript {
this.intervalsString = List(qscript.intervals)
this.jarFile = qscript.gatkJarFile
this.reference_sequence = qscript.referenceFile
this.memoryLimit = Some(3)
this.memoryLimit = 3
this.logging_level = "INFO"
}
@ -102,7 +102,7 @@ class PhaseSamples extends QScript {
// add the master call:
this.rodBind :+= RodBind("master", "VCF", masterCalls)
this.variantMergeOptions = Some(org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.MASTER)
this.variantMergeOptions = org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.MASTER
this.out = outputPhased
}

8
scala/qscript/oneoffs/fromer/ReadDepthCNVanalysis.scala
View File

@ -48,7 +48,7 @@ class ReadDepthCNVanalysis extends QScript {
this.intervalsString = List(qscript.intervals)
this.jarFile = qscript.gatkJarFile
this.reference_sequence = qscript.referenceFile
//this.memoryLimit = Some(3)
//this.memoryLimit = 3
this.logging_level = "INFO"
}
@ -98,8 +98,8 @@ class ReadDepthCNVanalysis extends QScript {
this.input_file = t.bams
this.downsample_to_coverage = Some(MAX_DEPTH)
this.downsampling_type = Some(DownsampleType.BY_SAMPLE)
this.downsample_to_coverage = MAX_DEPTH
this.downsampling_type = DownsampleType.BY_SAMPLE
this.scatterCount = scatterCountInput
this.scatterClass = classOf[IntervalScatterFunction]
@ -132,6 +132,6 @@ class ReadDepthCNVanalysis extends QScript {
def commandLine = command
// Since loading ALL of the output into the perl script can take significant memory:
this.memoryLimit = Some(9)
this.memoryLimit = 9
}
}

4
scala/qscript/oneoffs/hanna/DoC.scala
View File

@ -35,14 +35,14 @@ class DoC extends QScript {
this.jarFile = DoC.this.gatkJar
this.reference_sequence = DoC.this.referenceFile
this.intervalsString = DoC.this.intervalsString
this.memoryLimit = Some(8)
this.memoryLimit = 8
}
def script = {
// Create the four function that we can run.
val doc = new DepthOfCoverage with DepthOfCoverageArguments
doc.downsampling_type = Some(DownsampleType.NONE)
doc.downsampling_type = DownsampleType.NONE
doc.omitLocusTable = true
doc.omitIntervals = true
doc.omitSampleSummary = true

4
scala/qscript/oneoffs/kshakir/UGMemoryTests.scala
View File

@ -35,13 +35,13 @@ class UGMemoryTests extends QScript {
snps.jobOutputFile = new File(dir, "UnifiedGenotyper.out")
snps.out = new File(dir, "UnifiedGenotyper.vcf")
snps.input_file = squid1Bams.take(numBams/2) ++ squid2Bams.take(numBams/2)
snps.memoryLimit = Some(memoryLimit)
snps.memoryLimit = memoryLimit
snps.jarFile = qscript.gatkJar
snps.reference_sequence = pipeline.getProject.getReferenceFile
snps.intervals = List(pipeline.getProject.getIntervalList)
snps.rodBind :+= new RodBind("dbsnp", pipeline.getProject.getGenotypeDbsnpType, pipeline.getProject.getGenotypeDbsnp)
snps.downsample_to_coverage = Some(qscript.downsampling_coverage)
snps.downsample_to_coverage = qscript.downsampling_coverage
snps.annotation ++= List("AlleleBalance")
snps.group :+= "Standard"

4
scala/qscript/oneoffs/kshakir/linearindexbintests/LinearIndexBinTests.scala
View File

@ -49,10 +49,10 @@ class LinearIndexBinTests extends QScript {
countRod.jarFile = qscript.gatkJar
countRod.reference_sequence = reference
countRod.memoryLimit = Some(memoryLimit)
countRod.memoryLimit = memoryLimit
// Some of the BED files don't have a chrM, which makes the GATK angry. Run unsafe.
countRod.U = Some(org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.ALL)
countRod.U = org.broadinstitute.sting.gatk.arguments.ValidationExclusion.TYPE.ALL
for ((rodFile, index) <- rodFiles.zipWithIndex) {
val rodType = rodFile.getName.split("\\.").last

31
scala/qscript/oneoffs/rpoplin/ASHGcalling.scala
View File

@ -3,7 +3,6 @@ import org.broadinstitute.sting.datasources.pipeline.Pipeline
import org.broadinstitute.sting.gatk.DownsampleType
import org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeCalculationModel.Model
import org.broadinstitute.sting.queue.extensions.gatk._
import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction
import org.broadinstitute.sting.queue.extensions.samtools._
import org.broadinstitute.sting.queue.{QException, QScript}
import collection.JavaConversions._
@ -42,7 +41,7 @@ class ASHGcalling extends QScript {
trait CommandLineGATKArgs extends CommandLineGATK {
this.jarFile = qscript.gatkJar
this.reference_sequence = qscript.reference
this.memoryLimit = Some(2)
this.memoryLimit = 2
this.DBSNP = qscript.dbSNP
this.jobTempDir = qscript.tmpDir
}
@ -84,8 +83,8 @@ class ASHGcalling extends QScript {
combineVariants = new CombineVariants with CommandLineGATKArgs
combineVariants.rodBind = vcfChunks
combineVariants.out = new TaggedFile(qscript.baseName + ".chr" + qscript.chr.toString + ".filtered.vcf", "vcf")
combineVariants.variantmergeoption = Some(org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.UNION)
combineVariants.genotypemergeoption = Some(org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.GenotypeMergeType.UNSORTED)
combineVariants.variantmergeoption = org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.UNION
combineVariants.genotypemergeoption = org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.GenotypeMergeType.UNSORTED
combineVariants.setKey = "null"
add(combineVariants)
*/
@ -104,16 +103,16 @@ class ASHGcalling extends QScript {
// 1.) Clean at known indels
var clean = new IndelRealigner with CommandLineGATKArgs
val cleanedBam = new File(baseTmpName + "cleaned.bam")
clean.memoryLimit = Some(4)
clean.memoryLimit = 4
clean.input_file :+= bamList
clean.intervalsString :+= interval
clean.targetIntervals = qscript.targetIntervals
clean.out = cleanedBam
clean.rodBind :+= RodBind("indels", "VCF", qscript.dindelCalls)
clean.knownsOnly = true
clean.LOD = Some(1.0)
clean.LOD = 1.0
clean.sortInCoordinateOrderEvenThoughItIsHighlyUnsafe = true
clean.compress = Some(2)
clean.compress = 2
clean.jobName = baseName + population + ".clean"
//clean.stripBam = true
//clean.fileSystemUsage = "indium"
@ -121,7 +120,7 @@ class ASHGcalling extends QScript {
// 2.) Apply BAQ calculation
var baq = new SamtoolsBaqFunction
val baqedBam = new File(baseTmpName + "cleaned.baq.bam")
baq.memoryLimit = Some(4)
baq.memoryLimit = 4
baq.in_bam = cleanedBam
baq.out_bam = baqedBam
baq.jobName = baseName + population + ".baq"
@ -154,22 +153,22 @@ class ASHGcalling extends QScript {
}
// 4.) Call with UGv2
call.memoryLimit = Some(4)
call.memoryLimit = 4
call.intervalsString :+= interval
call.out = rawCalls
call.dcov = Some(50)
call.standard_min_confidence_threshold_for_calling = Some(50)
call.standard_min_confidence_threshold_for_emitting = Some(30)
call.min_mapping_quality_score = Some(20)
call.min_base_quality_score = Some(20)
call.pnrm = Some(org.broadinstitute.sting.playground.gatk.walkers.genotyper.AlleleFrequencyCalculationModel.Model.GRID_SEARCH)
call.dcov = 50
call.standard_min_confidence_threshold_for_calling = 50
call.standard_min_confidence_threshold_for_emitting = 30
call.min_mapping_quality_score = 20
call.min_base_quality_score = 20
call.pnrm = org.broadinstitute.sting.playground.gatk.walkers.genotyper.AlleleFrequencyCalculationModel.Model.GRID_SEARCH
call.jobName = baseName + "call"
//call.fileSystemUsage = "iodine"
// 5b.) Filter near indels and HARD_TO_VALIDATE
var filter = new VariantFiltration with CommandLineGATKArgs
val filteredCalls = new File(baseName + "filtered.vcf")
filter.memoryLimit = Some(1)
filter.memoryLimit = 1
filter.out = filteredCalls
filter.intervalsString :+= interval
filter.variantVCF = rawCalls

27
scala/qscript/oneoffs/rpoplin/Phase1Calling.scala
View File

@ -2,7 +2,6 @@ import net.sf.picard.reference.FastaSequenceFile
import org.broadinstitute.sting.datasources.pipeline.Pipeline
import org.broadinstitute.sting.gatk.DownsampleType
import org.broadinstitute.sting.queue.extensions.gatk._
import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction
import org.broadinstitute.sting.queue.extensions.samtools._
import org.broadinstitute.sting.queue.{QException, QScript}
import collection.JavaConversions._
@ -51,7 +50,7 @@ class Phase1Calling extends QScript {
trait CommandLineGATKArgs extends CommandLineGATK {
this.jarFile = qscript.gatkJar
this.reference_sequence = qscript.reference
this.memoryLimit = Some(3)
this.memoryLimit = 3
this.jobTempDir = qscript.tmpDir
this.DBSNP = qscript.dbSNP
}
@ -79,12 +78,12 @@ class Phase1Calling extends QScript {
var call = new UnifiedGenotyper with CommandLineGATKArgs
call.intervalsString ++= List(qscript.intervals)
call.scatterCount = 63 // the smallest interval list has 63 intervals, one for each Mb on chr20
call.dcov = Some( 50 )
call.stand_call_conf = Some( 4.0 )
call.stand_emit_conf = Some( 4.0 )
call.dcov = 50
call.stand_call_conf = 4.0
call.stand_emit_conf = 4.0
call.input_file :+= bamList
call.out = rawCalls
call.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY)
call.baq = org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY
call.analysisName = baseName + "_UG"
var filter = new VariantFiltration with CommandLineGATKArgs
@ -106,9 +105,9 @@ class Phase1Calling extends QScript {
gvc.use_annotation ++= List("QD", "SB", "HaplotypeScore", "HRun")
gvc.analysisName = baseName + "_GVC"
gvc.intervalsString ++= List(qscript.intervals)
gvc.qual = Some(100) // clustering parameters to be updated soon pending new experimentation results
gvc.std = Some(4.5)
gvc.mG = Some(6)
gvc.qual = 100 // clustering parameters to be updated soon pending new experimentation results
gvc.std = 4.5
gvc.mG = 6
var vr = new VariantRecalibrator with CommandLineGATKArgs
vr.rodBind :+= RodBind("1kg", "VCF", qscript.omni)
@ -120,13 +119,13 @@ class Phase1Calling extends QScript {
vr.analysisName = baseName + "_VR"
vr.intervalsString ++= List(qscript.intervals)
vr.ignoreFilter ++= List("HARD_TO_VALIDATE")
vr.target_titv = Some(2.3)
vr.sm = Some(org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY)
vr.target_titv = 2.3
vr.sm = org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY
vr.tranche ++= List("0.1", "1.0", "2.0", "3.0", "5.0", "10.0", "100.0")
vr.out = recalibratedCalls
vr.priorDBSNP = Some(10.0)
vr.priorHapMap = Some(12.0)
vr.prior1KG = Some(12.0)
vr.priorDBSNP = 10.0
vr.priorHapMap = 12.0
vr.prior1KG = 12.0
vr.tranchesFile = tranchesFile
add(call, filter, gvc, vr)

11
scala/qscript/oneoffs/rpoplin/Phase1Cleaning.scala
View File

@ -2,7 +2,6 @@ import net.sf.picard.reference.FastaSequenceFile
import org.broadinstitute.sting.datasources.pipeline.Pipeline
import org.broadinstitute.sting.gatk.DownsampleType
import org.broadinstitute.sting.queue.extensions.gatk._
import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction
import org.broadinstitute.sting.queue.extensions.samtools._
import org.broadinstitute.sting.queue.{QException, QScript}
import collection.JavaConversions._
@ -43,7 +42,7 @@ class Phase1Cleaning extends QScript {
trait CommandLineGATKArgs extends CommandLineGATK {
this.jarFile = qscript.gatkJar
this.reference_sequence = qscript.reference
this.memoryLimit = Some(2)
this.memoryLimit = 2
this.jobTempDir = qscript.tmpDir
}
@ -62,11 +61,11 @@ class Phase1Cleaning extends QScript {
// 1.) Create cleaning targets
var target = new RealignerTargetCreator with CommandLineGATKArgs
target.memoryLimit = Some(4)
target.memoryLimit = 4
target.input_file :+= bamList
target.intervalsString :+= interval
target.out = targetIntervals
target.mismatchFraction = Some(0.0)
target.mismatchFraction = 0.0
target.rodBind :+= RodBind("dbsnp", "VCF", qscript.dbSNP)
target.rodBind :+= RodBind("indels1", "VCF", qscript.dindelPilotCalls)
target.rodBind :+= RodBind("indels2", "VCF", qscript.dindelAFRCalls)
@ -77,13 +76,13 @@ class Phase1Cleaning extends QScript {
// 2.) Clean without SW
var clean = new IndelRealigner with CommandLineGATKArgs
val cleanedBam = new File(baseTmpName + "cleaned.bam")
clean.memoryLimit = Some(4)
clean.memoryLimit = 4
clean.input_file :+= bamList
clean.intervalsString :+= interval
clean.targetIntervals = targetIntervals
clean.out = cleanedBam
clean.doNotUseSW = true
clean.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.RECALCULATE)
clean.baq = org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.RECALCULATE
clean.rodBind :+= RodBind("dbsnp", "VCF", qscript.dbSNP)
clean.rodBind :+= RodBind("indels1", "VCF", qscript.dindelPilotCalls)
clean.rodBind :+= RodBind("indels2", "VCF", qscript.dindelAFRCalls)

19
scala/qscript/oneoffs/rpoplin/VQSR_parameterSearch.scala
View File

@ -1,4 +1,3 @@
import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction
import org.broadinstitute.sting.queue.extensions.gatk._
import org.broadinstitute.sting.queue.extensions.samtools.SamtoolsIndexFunction
import org.broadinstitute.sting.queue.QScript
@ -19,7 +18,7 @@ class VQSR_parameterSearch extends QScript {
@Argument(shortName="skipCalling", doc="If true, skip the calling part of the pipeline and only run VQSR on preset, gold standard VCF files", required=false)
var skipCalling: Boolean = false
trait UNIVERSAL_GATK_ARGS extends CommandLineGATK { logging_level = "INFO"; jarFile = gatkJarFile; memoryLimit = Some(2); }
trait UNIVERSAL_GATK_ARGS extends CommandLineGATK { logging_level = "INFO"; jarFile = gatkJarFile; memoryLimit = 2; }
class Target(val baseName: String, val reference: File, val rodName: String, val bamList: File, val goldStandard_VCF: File, val intervals: String, val titvTarget: Double, val isLowpass: Boolean) {
def name = qscript.outputDir + baseName
@ -309,12 +308,12 @@ class VQSR_parameterSearch extends QScript {
}
this.analysisName = name + "_GVC"
this.intervalsString ++= List(t.intervals)
this.qual = Some(t.qualCutoff)
this.std = Some(t.std)
this.mG = Some(t.gaussian)
this.qual = t.qualCutoff
this.std = t.std
this.mG = t.gaussian
this.ignoreFilter ++= FiltersToIgnore
this.dirichlet = Some(t.dirichlet)
this.shrinkage = Some(t.shrinkage)
this.dirichlet = t.dirichlet
this.shrinkage = t.shrinkage
}
// 4.) VQSR part2 Calculate new LOD for all input SNPs by evaluating the Gaussian clusters
@ -336,8 +335,8 @@ class VQSR_parameterSearch extends QScript {
this.intervalsString ++= List(t.intervals)
this.ignoreFilter ++= FiltersToIgnore
this.ignoreFilter ++= List("HARD_TO_VALIDATE")
this.target_titv = Some(t.titvTarget)
this.backOff = Some(t.backoff)
this.target_titv = t.titvTarget
this.backOff = t.backoff
}
// 4a.) Choose VQSR tranches based on novel ti/tv
@ -350,7 +349,7 @@ class VQSR_parameterSearch extends QScript {
// 4b.) Choose VQSR tranches based on sensitivity to truth set
class VariantRecalibratorNRS(t: Target, goldStandard: Boolean) extends VariantRecalibratorBase(t, goldStandard) {
this.sm = Some(org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY)
this.sm = org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator.SelectionMetricType.TRUTH_SENSITIVITY
if(t.trainOmni == 0 ) {
this.tranche ++= List("1.0")
} else {

28
scala/qscript/playground/BatchMerge.q
View File

@ -48,22 +48,22 @@ class batchMergePipeline extends QScript {
trait CalcLikelihoodArgs extends UGCalcLikelihoods {
this.reference_sequence = batchMerge.ref
this.max_mismatches_in_40bp_window = Some(batchMerge.mmb)
this.min_base_quality_score = Some(batchMerge.mbq)
this.min_mapping_quality_score = Some(batchMerge.mmq)
this.max_mismatches_in_40bp_window = batchMerge.mmb
this.min_base_quality_score = batchMerge.mbq
this.min_mapping_quality_score = batchMerge.mmq
if ( batchMerge.baq >= 0 ) {
this.baqGapOpenPenalty = Some(batchMerge.baq)
this.baq = Some(BAQ.CalculationMode.CALCULATE_AS_NECESSARY)
this.baqGapOpenPenalty = batchMerge.baq
this.baq = BAQ.CalculationMode.CALCULATE_AS_NECESSARY
}
this.intervals :+= extractIntervals.listOut
this.alleleVCF = combineVCFs.outVCF
this.jarFile = new File(stingDir+"/dist/GenomeAnalysisTK.jar")
this.memoryLimit = Some(4)
this.memoryLimit = 4
this.scatterCount = 60
this.output_mode = Some(UnifiedGenotyperEngine.OUTPUT_MODE.EMIT_ALL_SITES)
this.genotyping_mode = Some(GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES)
this.output_mode = UnifiedGenotyperEngine.OUTPUT_MODE.EMIT_ALL_SITES
this.genotyping_mode = GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES
if ( batchMerge.indelMode ) {
this.genotype_likelihoods_model = Some(GenotypeLikelihoodsCalculationModel.Model.DINDEL)
this.genotype_likelihoods_model = GenotypeLikelihoodsCalculationModel.Model.DINDEL
}
}
@ -82,11 +82,11 @@ class batchMergePipeline extends QScript {
this.intervals :+= extractIntervals.listOut
this.jarFile = new File(stingDir+"/dist/GenomeAnalysisTK.jar")
this.scatterCount = 30
this.memoryLimit = Some(8)
this.output_mode = Some(UnifiedGenotyperEngine.OUTPUT_MODE.EMIT_ALL_SITES)
this.genotyping_mode = Some(GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES)
this.memoryLimit = 8
this.output_mode = UnifiedGenotyperEngine.OUTPUT_MODE.EMIT_ALL_SITES
this.genotyping_mode = GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES
if ( batchMerge.indelMode ) {
this.genotype_likelihoods_model = Some(GenotypeLikelihoodsCalculationModel.Model.DINDEL)
this.genotype_likelihoods_model = GenotypeLikelihoodsCalculationModel.Model.DINDEL
}
}
@ -100,7 +100,7 @@ class batchMergePipeline extends QScript {
this.intervals :+= extractIntervals.listOut
this.jarFile = new File(batchMerge.stingDir+"/dist/GenomeAnalysisTK.jar")
this.scatterCount = 10
this.memoryLimit=Some(4)
this.memoryLimit=4
}
var combine : CombineVariants = new CombineVariants with CombineVariantsArgs

32
scala/qscript/playground/FullCallingPipeline.q
View File

@ -1,7 +1,7 @@
import org.broadinstitute.sting.commandline.ArgumentSource
import org.broadinstitute.sting.datasources.pipeline.Pipeline
import org.broadinstitute.sting.queue.extensions.gatk._
import org.broadinstitute.sting.queue.extensions.picard.PicardBamEmbeddedFunction
import org.broadinstitute.sting.queue.extensions.picard.PicardBamFunction
import org.broadinstitute.sting.queue.extensions.samtools._
import org.broadinstitute.sting.queue.function.ListWriterFunction
import org.broadinstitute.sting.queue.function.scattergather.{GatherFunction, CloneFunction, ScatterFunction}
@ -42,7 +42,7 @@ class FullCallingPipeline extends QScript {
this.intervals = List(qscript.pipeline.getProject.getIntervalList)
this.jarFile = qscript.gatkJar
this.reference_sequence = qscript.pipeline.getProject.getReferenceFile
this.memoryLimit = Some(4)
this.memoryLimit = 4
}
@ -77,7 +77,7 @@ class FullCallingPipeline extends QScript {
targetCreator.analysisName = "CreateTargets_"+sampleId
targetCreator.input_file :+= bam
targetCreator.out = indel_targets
targetCreator.memoryLimit = Some(2)
targetCreator.memoryLimit = 2
targetCreator.isIntermediate = true
val realigner = new IndelRealigner with CommandLineGATKArgs
@ -108,8 +108,8 @@ class FullCallingPipeline extends QScript {
case (gather: BamGatherFunction, source: ArgumentSource) =>
gather.commandDirectory = new File("CleanedBams/IntermediateFiles/%s/ScatterGather/Gather_%s".format(sampleId, source.field.getName))
gather.jobOutputFile = new File(".queue/logs/Cleaning/%s/FixMates.out".format(sampleId))
gather.memoryLimit = Some(6)
gather.mainClass = picardFixMatesClass
gather.memoryLimit = 6
gather.javaMainClass = picardFixMatesClass
gather.assumeSorted = None
case (gather: GatherFunction, source: ArgumentSource) =>
gather.commandDirectory = new File("CleanedBams/IntermediateFiles/%s/ScatterGather/Gather_%s".format(sampleId, source.field.getName))
@ -122,7 +122,7 @@ class FullCallingPipeline extends QScript {
realigner.isIntermediate = true
// Explicitly run fix mates if the function won't be scattered.
val fixMates = new PicardBamEmbeddedFunction {
val fixMates = new PicardBamFunction {
@Input(doc="unfixed bam") var unfixed: File = _
@Output(doc="fixed bam") var fixed: File = _
def inputBams = List(unfixed)
@ -130,8 +130,8 @@ class FullCallingPipeline extends QScript {
}
fixMates.jobOutputFile = new File(".queue/logs/Cleaning/%s/FixMates.out".format(sampleId))
fixMates.memoryLimit = Some(6)
fixMates.mainClass = picardFixMatesClass
fixMates.memoryLimit = 6
fixMates.javaMainClass = picardFixMatesClass
fixMates.unfixed = realigner.out
fixMates.fixed = cleaned_bam
fixMates.analysisName = "FixMates_"+sampleId
@ -173,9 +173,9 @@ class FullCallingPipeline extends QScript {
val indels = new UnifiedGenotyper with CommandLineGATKArgs with ExpandedIntervals
indels.analysisName = base + "_indels"
indels.jobOutputFile = new File(".queue/logs/IndelCalling/UnifiedGenotyper.indels.out")
indels.memoryLimit = Some(6)
indels.downsample_to_coverage = Some(600)
indels.genotype_likelihoods_model = Option(org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeLikelihoodsCalculationModel.Model.DINDEL)
indels.memoryLimit = 6
indels.downsample_to_coverage = 600
indels.genotype_likelihoods_model = org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeLikelihoodsCalculationModel.Model.DINDEL
indels.input_file = bamFiles
indels.rodBind :+= RodBind("dbsnp", qscript.pipeline.getProject.getGenotypeDbsnpType, qscript.pipeline.getProject.getGenotypeDbsnp)
indels.out = new File("IndelCalls", base+".indels.vcf")
@ -210,8 +210,8 @@ class FullCallingPipeline extends QScript {
val snps = new UnifiedGenotyper with CommandLineGATKArgs with ExpandedIntervals
snps.analysisName = base+"_snps"
snps.jobOutputFile = new File(".queue/logs/SNPCalling/UnifiedGenotyper.snps.out")
snps.memoryLimit = Some(6)
snps.downsample_to_coverage = Some(600)
snps.memoryLimit = 6
snps.downsample_to_coverage = 600
snps.input_file = bamFiles
snps.rodBind :+= RodBind("dbsnp", qscript.pipeline.getProject.getGenotypeDbsnpType, qscript.pipeline.getProject.getGenotypeDbsnp)
snps.out = new File("SnpCalls", base+".snps.vcf")
@ -239,8 +239,8 @@ class FullCallingPipeline extends QScript {
filteredSNPs.jobOutputFile = new File(".queue/logs/SNPCalling/VariantFiltration.snps.out")
filteredSNPs.filterName ++= List("SNPSBFilter","SNPQDFilter","SNPHRunFilter")
filteredSNPs.filterExpression ++= List("\"SB>=0.10\"","\"QD<5.0\"","\"HRun>=4\"")
filteredSNPs.clusterWindowSize = Some(10)
filteredSNPs.clusterSize = Some(3)
filteredSNPs.clusterWindowSize = 10
filteredSNPs.clusterSize = 3
filteredSNPs.rodBind :+= RodBind("mask", "VCF", filteredIndels.out)
filteredSNPs.variantVCF = snps.out
filteredSNPs.out = swapExt("SnpCalls",snps.out,".vcf",".filtered.vcf")
@ -249,7 +249,7 @@ class FullCallingPipeline extends QScript {
val combineAll = new CombineVariants with CommandLineGATKArgs with ExpandedIntervals
combineAll.analysisName = base + "_combineAll"
combineAll.jobOutputFile = new File(".queue/logs/Combined/CombineVariants.out")
combineAll.variantMergeOptions = Option(org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.UNION)
combineAll.variantMergeOptions = org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.UNION
combineAll.rod_priority_list = "Indels,SNPs"
combineAll.rodBind :+= RodBind("Indels", "VCF", filteredIndels.out)
combineAll.rodBind :+= RodBind("SNPs", "VCF", filteredSNPs.out)

6
scala/qscript/playground/MultiFullCallingPipeline.scala
View File

@ -1,5 +1,5 @@
import collection.JavaConversions
import org.broadinstitute.sting.queue.function.JarCommandLineFunction
import org.broadinstitute.sting.queue.function.JavaCommandLineFunction
import org.broadinstitute.sting.queue.QScript
import org.broadinstitute.sting.queue.util.IOUtils
import org.broadinstitute.sting.utils.text.XReadLines
@ -69,7 +69,7 @@ class MultiFullCallingPipeline extends QScript {
* Runs a yaml in a pipeline only after a previous pipeline
* run has produced the passed in output file.
*/
class RunPipeline(yamlFile: File, lastOutput: File) extends JarCommandLineFunction {
class RunPipeline(yamlFile: File, lastOutput: File) extends JavaCommandLineFunction {
private var yamlName = yamlFile.getName.stripSuffix(".yaml")
@Input(doc="output file to wait for", required=false)
@ -81,7 +81,7 @@ class MultiFullCallingPipeline extends QScript {
commandDirectory = yamlFile.getParentFile
jobOutputFile = IOUtils.absolute(commandDirectory, yamlName + ".queue.txt")
jarFile = queueJar
memoryLimit = Some(1)
memoryLimit = 1
override def commandLine = super.commandLine +
optional(" -statusTo ", qscript.pipelineStatusTo) +

16
scala/qscript/playground/recalibrate.scala
View File

@ -1,4 +1,3 @@
import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction
import org.broadinstitute.sting.queue.extensions.gatk._
import org.broadinstitute.sting.queue.extensions.samtools.SamtoolsIndexFunction
import org.broadinstitute.sting.queue.QScript
@ -23,9 +22,6 @@ class recalibrate extends QScript {
@Argument(shortName = "R", doc="ref")
var referenceFile: File = _
@Argument(doc="X", required=false)
var picardMergeSamFilesJar: File = new File("/seq/software/picard/current/bin/MergeSamFiles.jar")
trait UNIVERSAL_GATK_ARGS extends CommandLineGATK { logging_level = "INFO"; jarFile = gatkJarFile; reference_sequence = referenceFile; }
def script = {
@ -40,13 +36,11 @@ def script = {
val tableRecal = new TableRecalibrate(bamIn, recalData, recalBam) { useOriginalQualities = true }
if ( scatter ) {
tableRecal.intervals = List(new File("/humgen/gsa-hpprojects/GATK/data/chromosomes.hg18.interval_list"))
//tableRecal.scatterClass = classOf[ContigScatterFunction]
tableRecal.setupGatherFunction = { case (f: PicardBamJarFunction, _) => f.jarFile = picardMergeSamFilesJar; f.memoryLimit = Some(4) }
tableRecal.scatterCount = 25
}
add(tableRecal)
add(new Index(recalBam))
add(new CountCovariates(recalBam, recalRecalData) { num_threads = Some(4) })
add(new CountCovariates(recalBam, recalRecalData) { num_threads = 4 })
add(new AnalyzeCovariates(recalData, new File(recalData.getPath() + ".analyzeCovariates")))
add(new AnalyzeCovariates(recalRecalData, new File(recalRecalData.getPath() + ".analyzeCovariates")))
}
@ -65,7 +59,7 @@ class CountCovariates(bamIn: File, recalDataIn: File) extends org.broadinstitute
this.DBSNP = new File("/humgen/gsa-hpprojects/GATK/data/dbsnp_129_hg18.rod")
this.logging_level = "INFO"
this.covariate ++= List("ReadGroupCovariate", "QualityScoreCovariate", "CycleCovariate", "DinucCovariate")
this.memoryLimit = Some(3)
this.memoryLimit = 3
override def dotString = "CountCovariates: %s [args %s]".format(bamIn.getName, if (this.num_threads.isDefined) "-nt " + this.num_threads else "")
}
@ -76,7 +70,7 @@ class TableRecalibrate(bamInArg: File, recalDataIn: File, bamOutArg: File) exten
this.recal_file = recalDataIn
this.out = bamOutArg
this.logging_level = "INFO"
this.memoryLimit = Some(2)
this.memoryLimit = 2
this.skipUQUpdate = skipUQUpdateArg
override def dotString = "TableRecalibrate: %s => %s".format(bamInArg.getName, bamOutArg.getName, if (this.useOriginalQualities) " -OQ" else "")
@ -87,9 +81,9 @@ class AnalyzeCovariates(recalDataIn: File, outputDir: File) extends org.broadin
this.recal_file = recalDataIn
this.output_dir = outputDir.toString
this.path_to_resources = "/home/radon01/depristo/dev/GenomeAnalysisTK/trunk/R/"
this.ignoreQ = Some(5)
this.ignoreQ = 5
this.path_to_Rscript = "/broad/software/free/Linux/redhat_5_x86_64/pkgs/r_2.7.2/bin/Rscript"
this.memoryLimit = Some(2)
this.memoryLimit = 2
override def dotString = "AnalyzeCovariates: %s".format(recalDataIn.getName)
}

86
scala/src/org/broadinstitute/sting/queue/QCommandLine.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue
import function.QFunction
@ -10,6 +34,37 @@ import collection.JavaConversions._
import org.broadinstitute.sting.utils.classloader.PluginManager
import org.broadinstitute.sting.utils.exceptions.UserException
/**
* Entry point of Queue. Compiles and runs QScripts passed in to the command line.
*/
object QCommandLine {
/**
* Main.
* @param argv Arguments.
*/
def main(argv: Array[String]) {
val qCommandLine = new QCommandLine
Runtime.getRuntime.addShutdownHook(new Thread {
/** Cleanup as the JVM shuts down. */
override def run = {
ProcessController.shutdown()
qCommandLine.shutdown()
}
})
try {
CommandLineProgram.start(qCommandLine, argv);
if (CommandLineProgram.result != 0)
System.exit(CommandLineProgram.result);
} catch {
case e: Exception => CommandLineProgram.exitSystemWithError(e)
} finally {
}
}
}
/**
* Entry point of Queue. Compiles and runs QScripts passed in to the command line.
*/
@ -99,34 +154,3 @@ class QCommandLine extends CommandLineProgram with Logging {
if (qScriptClasses != null) IOUtils.tryDelete(qScriptClasses)
}
}
/**
* Entry point of Queue. Compiles and runs QScripts passed in to the command line.
*/
object QCommandLine {
/**
* Main.
* @param argv Arguments.
*/
def main(argv: Array[String]) {
val qCommandLine = new QCommandLine
Runtime.getRuntime.addShutdownHook(new Thread {
/** Cleanup as the JVM shuts down. */
override def run = {
ProcessController.shutdown()
qCommandLine.shutdown()
}
})
try {
CommandLineProgram.start(qCommandLine, argv);
if (CommandLineProgram.result != 0)
System.exit(CommandLineProgram.result);
} catch {
case e: Exception => CommandLineProgram.exitSystemWithError(e)
} finally {
}
}
}

7
scala/src/org/broadinstitute/sting/queue/QScript.scala
View File

@ -24,7 +24,7 @@
package org.broadinstitute.sting.queue
import org.broadinstitute.sting.queue.util.Logging
import util.{PrimitiveOptionConversions, Logging}
import org.broadinstitute.sting.queue.function.QFunction
import org.broadinstitute.sting.utils.text.XReadLines
import annotation.target.field
@ -32,7 +32,8 @@ import annotation.target.field
/**
* Defines a Queue pipeline as a collection of CommandLineFunctions.
*/
trait QScript extends Logging {
trait QScript extends Logging with PrimitiveOptionConversions {
// Type aliases so users don't have to import
type File = java.io.File
type CommandLineFunction = org.broadinstitute.sting.queue.function.CommandLineFunction
@ -49,7 +50,7 @@ trait QScript extends Logging {
type Output = org.broadinstitute.sting.commandline.Output @field
type Argument = org.broadinstitute.sting.commandline.Argument @field
type ArgumentCollection = org.broadinstitute.sting.commandline.ArgumentCollection @field
type Gather = org.broadinstitute.sting.queue.function.scattergather.Gather @field
type Gather = org.broadinstitute.sting.commandline.Gather @field
/**
* Builds the CommandLineFunctions that will be used to run this script and adds them to this.functions directly or using the add() utility method.

19
scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala
View File

@ -318,7 +318,7 @@ class QGraph extends Logging {
var readyJobs = getReadyJobs()
while (running && readyJobs.size > 0) {
logger.debug("+++++++")
readyJobs.foreach(edge => {
foreachFunction(readyJobs.toList, edge => {
if (running) {
logEdge(edge)
edge.markAsDone
@ -870,12 +870,17 @@ class QGraph extends Logging {
* Utility function for running a method over all function edges.
* @param edgeFunction Function to run for each FunctionEdge.
*/
private def foreachFunction(f: (FunctionEdge) => Unit) = {
jobGraph.edgeSet.toList
.filter(_.isInstanceOf[FunctionEdge])
.asInstanceOf[List[FunctionEdge]]
.sorted(functionOrdering)
.foreach(edge => if (running) f(edge))
private def foreachFunction(f: (FunctionEdge) => Unit) {
foreachFunction(jobGraph.edgeSet.toList.filter(_.isInstanceOf[FunctionEdge]).asInstanceOf[List[FunctionEdge]], f)
}
/**
* Utility function for running a method over a list of function edges.
* @param edegs Edges to traverse.
* @param edgeFunction Function to run for each FunctionEdge.
*/
private def foreachFunction(edges: List[FunctionEdge], f: (FunctionEdge) => Unit) {
edges.sorted(functionOrdering).foreach(edge => if (running) f(edge))
}
/**

45
scala/src/org/broadinstitute/sting/queue/extensions/firehose/ImportSingleValueFunction.scala
View File

@ -1,45 +0,0 @@
package org.broadinstitute.sting.queue.extensions.firehose
import org.broadinstitute.sting.queue.function.JarCommandLineFunction
import org.broadinstitute.sting.commandline.{Input, Argument}
import java.io.File
/**
* Runs the Firehose ImportSingleValue jar file.
*/
class ImportSingleValueFunction extends JarCommandLineFunction {
@Argument(doc="firehose host")
var host: String = _
@Argument(doc="firehose port")
var port: Int = _
@Argument(doc="firehose domain")
var domain: String = _
@Argument(doc="firehose entity type")
var entityType: String = _
@Argument(doc="firehose entity id")
var entityID: String = _
@Argument(doc="firehose annotation type name", shortName="bamFHAnn", required=false)
var annotationTypeName: String = _
@Argument(doc="clean bam firehose security token", shortName="bamFHToken", required=false)
var securityToken: String = _
@Input(doc="imports the path to this file", exclusiveOf="importValueInFile")
var importValue: File = _
@Input(doc="imports the value contained in the file", exclusiveOf="importValue")
var importValueInFile: File = _
override def commandLine = super.commandLine + ("" +
" PORT=%s HOST=%s DOMAIN=%s ENTITY_TYPE=%s" +
" ENTITY_ID=%s ANNOTATION_TYPE_NAME=%s SECURITY_TOKEN=%s" +
"%s%s"
).format(
port, host, domain, entityType, entityID, annotationTypeName, securityToken,
optional(" VALUE=", importValue), optional(" VALUE_FILE=", importValueInFile))
}

30
scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala
View File

@ -1,13 +1,37 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.extensions.gatk
import org.broadinstitute.sting.queue.function.scattergather.GatherFunction
import org.broadinstitute.sting.queue.extensions.picard.PicardBamEmbeddedFunction
import org.broadinstitute.sting.queue.extensions.picard.PicardBamFunction
/**
* Merges BAM files using Picards net.sf.picard.sam.MergeSamFiles.
*/
class BamGatherFunction extends GatherFunction with PicardBamEmbeddedFunction {
this.mainClass = "net.sf.picard.sam.MergeSamFiles"
class BamGatherFunction extends GatherFunction with PicardBamFunction {
this.javaMainClass = "net.sf.picard.sam.MergeSamFiles"
this.assumeSorted = Some(true)
protected def inputBams = gatherParts
protected def outputBam = originalOutput

2
scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKScatterFunction.scala
View File

@ -69,7 +69,7 @@ trait GATKScatterFunction extends ScatterFunction {
}
override def isScatterGatherable = {
if (this.originalGATK.BTI != null && this.originalGATK.BTIMR.isEmpty)
if (this.originalGATK.BTI != null && this.originalGATK.BTIMR == null)
throw new IllegalArgumentException("BTI requires BTIMR for use with scatter-gather (recommended: INTERSECTION)")
this.originalGATK.reference_sequence != null
}

22
scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamFunction.scala
View File

@ -25,7 +25,9 @@
package org.broadinstitute.sting.queue.extensions.picard
import java.io.File
import org.broadinstitute.sting.queue.function.CommandLineFunction
import org.broadinstitute.sting.queue.function.JavaCommandLineFunction
import net.sf.samtools.SAMFileReader.ValidationStringency
import net.sf.samtools.SAMFileHeader.SortOrder
/**
* Wraps a Picard function that operates on BAM files.
@ -34,9 +36,9 @@ import org.broadinstitute.sting.queue.function.CommandLineFunction
* Since the various BAM utilities take slightly different arguments
* some values are optional.
*/
trait PicardBamFunction extends CommandLineFunction {
var validationStringency = "SILENT"
var sortOrder = "coordinate"
trait PicardBamFunction extends JavaCommandLineFunction {
var validationStringency: ValidationStringency = ValidationStringency.SILENT
var sortOrder: SortOrder = SortOrder.coordinate
var compressionLevel: Option[Int] = None
var maxRecordsInRam: Option[Int] = None
var assumeSorted: Option[Boolean] = None
@ -45,7 +47,13 @@ trait PicardBamFunction extends CommandLineFunction {
protected def outputBam: File
abstract override def commandLine = super.commandLine +
Array(optional(" COMPRESSION_LEVEL=", compressionLevel), optional(" VALIDATION_STRINGENCY=", validationStringency),
optional(" SO=", sortOrder), optional( " MAX_RECORDS_IN_RAM=", maxRecordsInRam), optional(" ASSUME_SORTED=", assumeSorted),
" OUTPUT=" + outputBam, repeat(" INPUT=", inputBams), " TMP_DIR=" + jobTempDir).mkString
Array(
repeat(" INPUT=", inputBams),
" OUTPUT=" + outputBam,
" TMP_DIR=" + jobTempDir,
optional(" COMPRESSION_LEVEL=", compressionLevel),
optional(" VALIDATION_STRINGENCY=", validationStringency),
optional(" SO=", sortOrder),
optional(" MAX_RECORDS_IN_RAM=", maxRecordsInRam),
optional(" ASSUME_SORTED=", assumeSorted)).mkString
}

45
scala/src/org/broadinstitute/sting/queue/function/EmbeddedCommandLineFunction.scala
View File

@ -1,45 +0,0 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.function
import java.io.File
import org.broadinstitute.sting.queue.util.IOUtils
import org.broadinstitute.sting.commandline.Argument
/**
* Defines a command line function that runs java code from inside the existing class path.
*/
trait EmbeddedCommandLineFunction extends JavaCommandLineFunction {
@Argument(doc="Main class to run from the current classpath")
var mainClass: String = null
def javaExecutable = "-cp %s %s".format(EmbeddedCommandLineFunction.classpath, mainClass)
}
object EmbeddedCommandLineFunction {
private val classpath = System.getProperty("java.class.path")
.split(File.pathSeparatorChar).map(path => IOUtils.absolute(new File(path)))
.mkString("\"", "\"" + File.pathSeparator + "\"", "\"")
}

14
scala/src/org/broadinstitute/sting/queue/function/JarCommandLineFunction.scala
View File

@ -1,14 +0,0 @@
package org.broadinstitute.sting.queue.function
import org.broadinstitute.sting.commandline.Argument
import java.io.File
/**
* Defines a command line function that runs from a jar file.
*/
trait JarCommandLineFunction extends JavaCommandLineFunction {
@Argument(doc="jar")
var jarFile: File = _
def javaExecutable = "-jar " + jarFile
}

59
scala/src/org/broadinstitute/sting/queue/function/JavaCommandLineFunction.scala
View File

@ -1,24 +1,74 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.function
import org.broadinstitute.sting.commandline.Argument
import org.broadinstitute.sting.queue.util.IOUtils
import java.io.File
/**
* Defines a command line function that runs java code.
*/
trait JavaCommandLineFunction extends CommandLineFunction {
@Argument(doc="jar", exclusiveOf="javaMainClass")
var jarFile: File = _
@Argument(doc="Main class to run from javaClasspath", exclusiveOf="jarFile")
var javaMainClass: String = _
/**
* Returns the java executable to run.
* Class path for the main class.
* Defaults to the current classpath.
*/
def javaExecutable: String
var javaClasspath: List[String] = Nil
/**
* Memory limit for the java executable, or if None will use the default memoryLimit.
*/
var javaMemoryLimit: Option[Int] = None
/**
* Returns the java executable to run.
*/
def javaExecutable: String = {
if (jarFile != null)
"-jar " + jarFile
else if (javaMainClass != null)
"-cp \"%s\" %s".format(javaClasspath.mkString(File.pathSeparator), javaMainClass)
else
null
}
override def freezeFieldValues = {
super.freezeFieldValues
if (javaMemoryLimit.isEmpty && memoryLimit.isDefined)
javaMemoryLimit = memoryLimit
if (javaMainClass != null && javaClasspath.isEmpty)
javaClasspath = JavaCommandLineFunction.currentClasspath
}
def javaOpts = "%s -Djava.io.tmpdir=%s"
@ -27,3 +77,8 @@ trait JavaCommandLineFunction extends CommandLineFunction {
def commandLine = "java%s %s"
.format(javaOpts, javaExecutable)
}
object JavaCommandLineFunction {
val currentClasspath = System.getProperty("java.class.path")
.split(File.pathSeparatorChar).map(path => IOUtils.absolute(new File(path)).getPath).toList
}

26
scala/src/org/broadinstitute/sting/queue/function/QFunction.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.function
import java.io.File
@ -5,7 +29,7 @@ import java.lang.annotation.Annotation
import org.broadinstitute.sting.commandline._
import org.broadinstitute.sting.queue.{QException, QSettings}
import collection.JavaConversions._
import org.broadinstitute.sting.queue.function.scattergather.{Gather, SimpleTextGatherFunction}
import org.broadinstitute.sting.queue.function.scattergather.SimpleTextGatherFunction
import org.broadinstitute.sting.queue.util.{Logging, CollectionUtils, IOUtils, ReflectionUtils}
/**

20
scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamJarFunction.scala → scala/src/org/broadinstitute/sting/queue/function/scattergather/GathererFunction.scala
View File

@ -22,16 +22,20 @@
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.extensions.picard
package org.broadinstitute.sting.queue.function.scattergather
import org.broadinstitute.sting.queue.function.JarCommandLineFunction
import org.broadinstitute.sting.commandline.Gatherer
import org.broadinstitute.sting.queue.function.InProcessFunction
import collection.JavaConversions._
/**
* Wraps a Picard jar that operates on BAM files.
* See http://picard.sourceforge.net/ for more info.
*
* Since the jar files take slightly different arguments
* some values are optional.
* Runs a Gatherer in process.
*/
trait PicardBamJarFunction extends JarCommandLineFunction with PicardBamFunction {
class GathererFunction(gathererClass: Class[_ <: Gatherer]) extends InProcessFunction with GatherFunction {
def run() {
val gatherer = gathererClass.newInstance
if (gatherer.waitForInputs)
waitForGatherParts
gatherer.gather(this.gatherParts, this.originalOutput)
}
}

27
scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala
View File

@ -26,7 +26,7 @@ package org.broadinstitute.sting.queue.function.scattergather
import java.io.File
import org.broadinstitute.sting.queue.util._
import org.broadinstitute.sting.commandline.ArgumentSource
import org.broadinstitute.sting.commandline.{Gatherer, Gather, ArgumentSource}
import org.broadinstitute.sting.queue.function.{QFunction, CommandLineFunction}
import org.broadinstitute.sting.queue.QException
@ -240,14 +240,29 @@ trait ScatterGatherableFunction extends CommandLineFunction {
* @return A GatherFunction instantiated from @Gather.
*/
protected def newGatherFunction(gatherField: ArgumentSource) : GatherFunction = {
var gatherClass: Class[_ <: GatherFunction] = null
var gatherClass: Class[_] = null
// Check if there is a function that will return the gather class for this field.
if (this.gatherClass != null)
if (this.gatherClass.isDefinedAt(gatherField))
gatherClass = this.gatherClass(gatherField)
if (gatherClass == null)
gatherClass = ReflectionUtils.getAnnotation(gatherField.field, classOf[Gather])
.value.asSubclass(classOf[GatherFunction])
gatherClass.newInstance.asInstanceOf[GatherFunction]
// Check for an annotation defining the gather class.
if (gatherClass == null) {
if (ReflectionUtils.hasAnnotation(gatherField.field, classOf[Gather])) {
gatherClass = ReflectionUtils.getAnnotation(gatherField.field, classOf[Gather]).value
} else {
throw new QException("Missing @Gather annotation: " + gatherField.field.getName)
}
}
if (classOf[GatherFunction].isAssignableFrom(gatherClass)) {
gatherClass.newInstance.asInstanceOf[GatherFunction]
} else if (classOf[Gatherer].isAssignableFrom(gatherClass)) {
new GathererFunction(gatherClass.asSubclass(classOf[Gatherer]))
} else {
throw new QException("Unsupported @Gather class type: " + gatherClass)
}
}
/**

4
scala/src/org/broadinstitute/sting/queue/pipeline/BamProcessing.scala
View File

@ -2,7 +2,7 @@ package org.broadinstitute.sting.queue.pipeline
import org.broadinstitute.sting.queue.extensions.gatk._
import java.io.File
import org.broadinstitute.sting.queue.extensions.picard.PicardBamJarFunction
import org.broadinstitute.sting.queue.extensions.picard.PicardBamFunction
import org.broadinstitute.sting.queue.function.CommandLineFunction
import org.broadinstitute.sting.utils.yaml.YamlUtils
import org.broadinstitute.sting.datasources.pipeline.Pipeline
@ -125,7 +125,7 @@ class BamProcessing(attribs: Pipeline, gatkJar: File, fixMatesJar: File) {
return pfm
}
class PicardFixMates extends PicardBamJarFunction {
class PicardFixMates extends PicardBamFunction {
@Input(doc="input bam files") var bams: List[File] = Nil
@Output(doc="output bam file") var outBam: File = null

8
scala/src/org/broadinstitute/sting/queue/pipeline/VariantCalling.scala
View File

@ -57,7 +57,7 @@ class VariantCalling(attribs: Pipeline,gatkJar: File) {
ug.input_file = bams
ug.out = output
ug.downsample_to_coverage = Some(300)
ug.dt = Some(DownsampleType.BY_SAMPLE)
ug.dt = DownsampleType.BY_SAMPLE
ug.scatterCount = 50
if ( bams.size > 40 ) {
@ -105,8 +105,8 @@ class VariantCalling(attribs: Pipeline,gatkJar: File) {
def StandardIndelCombine( igList : List[IndelGenotyperV2], output : File ) : CombineVariants = {
var cv = new CombineVariants with StandardCommandLineGATK
cv.out = output
cv.genotypemergeoption = Some(org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.GenotypeMergeType.UNIQUIFY)
cv.variantmergeoption = Some(org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.UNION)
cv.genotypemergeoption = org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.GenotypeMergeType.UNIQUIFY
cv.variantmergeoption = org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContextUtils.VariantMergeType.UNION
cv.analysisName = "IndelGenotyper"
cv.priority = (igList.map[String,List[String]](ig => swapExt(ig.out,".vcf","").getAbsolutePath)).mkString(",")
//cv.priority = (igList.foldLeft[List[String]](Nil)( (prLs, ig) => prLs ::: List(swapExt(ig.out,".vcf","").getAbsolutePath))).mkString(",")
@ -408,7 +408,7 @@ object VariantCalling {
ugTokens.foreach( addEntry(_) )
myUG.reference_sequence = new File(ugMap("reference_sequence"))
myUG.baq = Some(org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.valueOf(ugMap("baq")))
myUG.baq = org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.valueOf(ugMap("baq"))
myUG.baqGapOpenPenalty = Some(ugMap("baqGapOpenPenalty").toDouble)
myUG.DBSNP = new File(ugMap("DBSNP"))

139
scala/src/org/broadinstitute/sting/queue/util/PrimitiveOptionConversions.scala 100644
View File

@ -0,0 +1,139 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.util
/**
* An importable object that provides automatic primitive to option conversion.
*/
object PrimitiveOptionConversions {
// Conversion from Option
implicit def byteOption2byte(x: Option[Byte]) = x.get
implicit def shortOption2short(x: Option[Short]) = x.get
implicit def charOption2char(x: Option[Char]) = x.get
implicit def intOption2int(x: Option[Int]) = x.get
implicit def longOption2long(x: Option[Long]) = x.get
implicit def floatOption2float(x: Option[Float]) = x.get
implicit def doubleOption2double(x: Option[Double]) = x.get
implicit def booleanOption2boolean(x: Option[Boolean]) = x.get
// Conversion to Option
implicit def byte2byteOption(x: Byte) = Some(x)
implicit def short2shortOption(x: Short) = Some(x)
implicit def char2charOption(x: Char) = Some(x)
implicit def int2intOption(x: Int) = Some(x)
implicit def long2longOption(x: Long) = Some(x)
implicit def float2floatOption(x: Float) = Some(x)
implicit def double2doubleOption(x: Double) = Some(x)
implicit def boolean2booleanOption(x: Boolean) = Some(x)
// Narrowing for constants to byte, short, and float
implicit def int2byteOption(x: Int) = Some(x.toByte)
implicit def int2shortOption(x: Int) = Some(x.toShort)
implicit def double2floatOption(x: Float) = Some(x.toFloat)
// Widening
implicit def byte2shortOption(x: Byte) = Some(x.toShort)
implicit def byte2intOption(x: Byte) = Some(x.toInt)
implicit def byte2longOption(x: Byte) = Some(x.toLong)
implicit def byte2floatOption(x: Byte) = Some(x.toFloat)
implicit def byte2doubleOption(x: Byte) = Some(x.toDouble)
implicit def short2intOption(x: Short) = Some(x.toInt)
implicit def short2longOption(x: Short) = Some(x.toLong)
implicit def short2floatOption(x: Short) = Some(x.toFloat)
implicit def short2doubleOption(x: Short) = Some(x.toDouble)
implicit def char2intOption(x: Char) = Some(x.toInt)
implicit def char2longOption(x: Char) = Some(x.toLong)
implicit def char2floatOption(x: Char) = Some(x.toFloat)
implicit def char2doubleOption(x: Char) = Some(x.toDouble)
implicit def int2longOption(x: Int) = Some(x.toLong)
implicit def int2floatOption(x: Int) = Some(x.toFloat)
implicit def int2doubleOption(x: Int) = Some(x.toDouble)
implicit def long2floatOption(x: Long) = Some(x.toFloat)
implicit def long2doubleOption(x: Long) = Some(x.toDouble)
implicit def float2doubleOption(x: Float) = Some(x.toDouble)
}
/**
* A trait that exposes the above functions to all sub classes as well.
*/
trait PrimitiveOptionConversions {
// How to we import these implicit definitions into the trait so that they are seen by objects extending a trait?
// import PrimitiveOptionConversion._ inside of a trait does not seem to work?
// Declaring them in a trait like this does work but does not seem scala-ish.
implicit def byteOption2byte(x: Option[Byte]) = PrimitiveOptionConversions.byteOption2byte(x)
implicit def shortOption2short(x: Option[Short]) = PrimitiveOptionConversions.shortOption2short(x)
implicit def charOption2char(x: Option[Char]) = PrimitiveOptionConversions.charOption2char(x)
implicit def intOption2int(x: Option[Int]) = PrimitiveOptionConversions.intOption2int(x)
implicit def longOption2long(x: Option[Long]) = PrimitiveOptionConversions.longOption2long(x)
implicit def floatOption2float(x: Option[Float]) = PrimitiveOptionConversions.floatOption2float(x)
implicit def doubleOption2double(x: Option[Double]) = PrimitiveOptionConversions.doubleOption2double(x)
implicit def booleanOption2boolean(x: Option[Boolean]) = PrimitiveOptionConversions.booleanOption2boolean(x)
implicit def byte2byteOption(x: Byte) = PrimitiveOptionConversions.byte2byteOption(x)
implicit def short2shortOption(x: Short) = PrimitiveOptionConversions.short2shortOption(x)
implicit def char2charOption(x: Char) = PrimitiveOptionConversions.char2charOption(x)
implicit def int2intOption(x: Int) = PrimitiveOptionConversions.int2intOption(x)
implicit def long2longOption(x: Long) = PrimitiveOptionConversions.long2longOption(x)
implicit def float2floatOption(x: Float) = PrimitiveOptionConversions.float2floatOption(x)
implicit def double2doubleOption(x: Double) = PrimitiveOptionConversions.double2doubleOption(x)
implicit def boolean2booleanOption(x: Boolean) = PrimitiveOptionConversions.boolean2booleanOption(x)
implicit def int2byteOption(x: Int) = PrimitiveOptionConversions.int2byteOption(x)
implicit def int2shortOption(x: Int) = PrimitiveOptionConversions.int2shortOption(x)
implicit def double2floatOption(x: Float) = PrimitiveOptionConversions.double2floatOption(x)
implicit def byte2shortOption(x: Byte) = PrimitiveOptionConversions.byte2shortOption(x)
implicit def byte2intOption(x: Byte) = PrimitiveOptionConversions.byte2intOption(x)
implicit def byte2longOption(x: Byte) = PrimitiveOptionConversions.byte2longOption(x)
implicit def byte2floatOption(x: Byte) = PrimitiveOptionConversions.byte2floatOption(x)
implicit def byte2doubleOption(x: Byte) = PrimitiveOptionConversions.byte2doubleOption(x)
implicit def short2intOption(x: Short) = PrimitiveOptionConversions.short2intOption(x)
implicit def short2longOption(x: Short) = PrimitiveOptionConversions.short2longOption(x)
implicit def short2floatOption(x: Short) = PrimitiveOptionConversions.short2floatOption(x)
implicit def short2doubleOption(x: Short) = PrimitiveOptionConversions.short2doubleOption(x)
implicit def char2intOption(x: Char) = PrimitiveOptionConversions.char2intOption(x)
implicit def char2longOption(x: Char) = PrimitiveOptionConversions.char2longOption(x)
implicit def char2floatOption(x: Char) = PrimitiveOptionConversions.char2floatOption(x)
implicit def char2doubleOption(x: Char) = PrimitiveOptionConversions.char2doubleOption(x)
implicit def int2longOption(x: Int) = PrimitiveOptionConversions.int2longOption(x)
implicit def int2floatOption(x: Int) = PrimitiveOptionConversions.int2floatOption(x)
implicit def int2doubleOption(x: Int) = PrimitiveOptionConversions.int2doubleOption(x)
implicit def long2floatOption(x: Long) = PrimitiveOptionConversions.long2floatOption(x)
implicit def long2doubleOption(x: Long) = PrimitiveOptionConversions.long2doubleOption(x)
implicit def float2doubleOption(x: Float) = PrimitiveOptionConversions.float2doubleOption(x)
}

28
scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleCountLociPipelineTest.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.pipeline.examples
import org.testng.annotations.Test
@ -10,8 +34,8 @@ class ExampleCountLociPipelineTest {
var testOut = "count.out"
val spec = new PipelineTestSpec
spec.name = "countloci"
spec.args = "-S scala/qscript/examples/ExampleCountLoci.scala -gatk %s -R %s -I %s -o %s".format(
PipelineTest.currentGATK, BaseTest.hg18Reference, BaseTest.validationDataLocation + "small_bam_for_countloci.bam", testOut
spec.args = "-S scala/qscript/examples/ExampleCountLoci.scala -R %s -I %s -o %s".format(
BaseTest.hg18Reference, BaseTest.validationDataLocation + "small_bam_for_countloci.bam", testOut
)
spec.fileMD5s += testOut -> "67823e4722495eb10a5e4c42c267b3a6"
PipelineTest.executeTest(spec)