diff --git a/build.xml b/build.xml
index 37fe3c289..e5ad9daf0 100644
--- a/build.xml
+++ b/build.xml
@@ -43,16 +43,17 @@
+
+
+
-
-
+
+
+
-
-
-
@@ -79,6 +80,16 @@
+
+
+
+
+
+
+
+
+
+
@@ -160,6 +171,16 @@
+
+
+
+
+
+
+
+
+
+
@@ -234,6 +255,7 @@
+
@@ -265,7 +287,7 @@
-
+
@@ -326,7 +348,7 @@
-
+
@@ -457,6 +479,35 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -490,6 +541,7 @@
+
@@ -657,54 +709,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -718,20 +722,113 @@
-
-
-
+
+
+
+
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -741,10 +838,10 @@
-
+
-
-
+
-
-
+
+
+
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
+
+
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
+
+
+
-
+
-
+
+
+
-
+
-
+
+
+
-
+
-
+
+
+
-
+
+
+
+
+
+
-
-
+
+
-
-
+
+
-
-
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
-
-
+
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -1044,7 +1105,7 @@
-
+
diff --git a/ivy.xml b/ivy.xml
index 3f3d1c97f..115f4062a 100644
--- a/ivy.xml
+++ b/ivy.xml
@@ -12,6 +12,9 @@
+
+
+
@@ -30,6 +33,9 @@
+
+
+
diff --git a/public/R/queueJobReport.R b/public/R/queueJobReport.R
new file mode 100644
index 000000000..31916361e
--- /dev/null
+++ b/public/R/queueJobReport.R
@@ -0,0 +1,169 @@
+library(gsalib)
+require("ggplot2")
+require("gplots")
+
+#
+# Standard command line switch. Can we loaded interactively for development
+# or executed with RScript
+#
+args = commandArgs(TRUE)
+onCMDLine = ! is.na(args[1])
+if ( onCMDLine ) {
+ inputFileName = args[1]
+ outputPDF = args[2]
+} else {
+ #inputFileName = "~/Desktop/broadLocal/GATK/unstable/report.txt"
+ inputFileName = "/humgen/gsa-hpprojects/dev/depristo/oneOffProjects/Q-25718@node1149.jobreport.txt"
+ #inputFileName = "/humgen/gsa-hpprojects/dev/depristo/oneOffProjects/rodPerformanceGoals/history/report.082711.txt"
+ outputPDF = NA
+}
+
+RUNTIME_UNITS = "(sec)"
+ORIGINAL_UNITS_TO_SECONDS = 1/1000
+
+#
+# Helper function to aggregate all of the jobs in the report across all tables
+#
+allJobsFromReport <- function(report) {
+ names <- c("jobName", "startTime", "analysisName", "doneTime", "exechosts")
+ sub <- lapply(report, function(table) table[,names])
+ do.call("rbind", sub)
+}
+
+#
+# Creates segmentation plots of time (x) vs. job (y) with segments for the duration of the job
+#
+plotJobsGantt <- function(gatkReport, sortOverall) {
+ allJobs = allJobsFromReport(gatkReport)
+ if ( sortOverall ) {
+ title = "All jobs, by analysis, by start time"
+ allJobs = allJobs[order(allJobs$analysisName, allJobs$startTime, decreasing=T), ]
+ } else {
+ title = "All jobs, sorted by start time"
+ allJobs = allJobs[order(allJobs$startTime, decreasing=T), ]
+ }
+ allJobs$index = 1:nrow(allJobs)
+ minTime = min(allJobs$startTime)
+ allJobs$relStartTime = allJobs$startTime - minTime
+ allJobs$relDoneTime = allJobs$doneTime - minTime
+ allJobs$ganttName = paste(allJobs$jobName, "@", allJobs$exechosts)
+ maxRelTime = max(allJobs$relDoneTime)
+ p <- ggplot(data=allJobs, aes(x=relStartTime, y=index, color=analysisName))
+ p <- p + geom_segment(aes(xend=relDoneTime, yend=index), size=2, arrow=arrow(length = unit(0.1, "cm")))
+ p <- p + geom_text(aes(x=relDoneTime, label=ganttName, hjust=-0.2), size=2)
+ p <- p + xlim(0, maxRelTime * 1.1)
+ p <- p + xlab(paste("Start time (relative to first job)", RUNTIME_UNITS))
+ p <- p + ylab("Job")
+ p <- p + opts(title=title)
+ print(p)
+}
+
+#
+# Plots scheduling efficiency at job events
+#
+plotProgressByTime <- function(gatkReport) {
+ allJobs = allJobsFromReport(gatkReport)
+ nJobs = dim(allJobs)[1]
+ allJobs = allJobs[order(allJobs$startTime, decreasing=F),]
+ allJobs$index = 1:nrow(allJobs)
+
+ minTime = min(allJobs$startTime)
+ allJobs$relStartTime = allJobs$startTime - minTime
+ allJobs$relDoneTime = allJobs$doneTime - minTime
+
+ times = sort(c(allJobs$relStartTime, allJobs$relDoneTime))
+
+ countJobs <- function(p) {
+ s = allJobs$relStartTime
+ e = allJobs$relDoneTime
+ x = c() # I wish I knew how to make this work with apply
+ for ( time in times )
+ x = c(x, sum(p(s, e, time)))
+ x
+ }
+
+ pending = countJobs(function(s, e, t) s > t)
+ done = countJobs(function(s, e, t) e < t)
+ running = nJobs - pending - done
+
+ d = data.frame(times=times, pending=pending, running=running, done=done)
+
+ p <- ggplot(data=melt(d, id.vars=c("times")), aes(x=times, y=value, color=variable))
+ p <- p + facet_grid(variable ~ ., scales="free")
+ p <- p + geom_line(size=2)
+ p <- p + xlab(paste("Time since start of first job", RUNTIME_UNITS))
+ p <- p + opts(title = "Job scheduling")
+ print(p)
+}
+
+#
+# Creates tables for each job in this group
+#
+standardColumns = c("jobName", "startTime", "formattedStartTime", "analysisName", "intermediate", "exechosts", "formattedDoneTime", "doneTime", "runtime")
+plotGroup <- function(groupTable) {
+ name = unique(groupTable$analysisName)[1]
+ groupAnnotations = setdiff(names(groupTable), standardColumns)
+ sub = groupTable[,c("jobName", groupAnnotations, "runtime")]
+ sub = sub[order(sub$iteration, sub$jobName, decreasing=F), ]
+
+ # create a table showing each job and all annotations
+ textplot(sub, show.rownames=F)
+ title(paste("Job summary for", name, "full itemization"), cex=3)
+
+ # create the table for each combination of values in the group, listing iterations in the columns
+ sum = cast(melt(sub, id.vars=groupAnnotations, measure.vars=c("runtime")), ... ~ iteration, fun.aggregate=mean)
+ textplot(as.data.frame(sum), show.rownames=F)
+ title(paste("Job summary for", name, "itemizing each iteration"), cex=3)
+
+ # histogram of job times by groupAnnotations
+ if ( length(groupAnnotations) == 1 && dim(sub)[1] > 1 ) {
+ # todo -- how do we group by annotations?
+ p <- ggplot(data=sub, aes(x=runtime)) + geom_histogram()
+ p <- p + xlab("runtime in seconds") + ylab("No. of jobs")
+ p <- p + opts(title=paste("Job runtime histogram for", name))
+ print(p)
+ }
+
+ # as above, but averaging over all iterations
+ groupAnnotationsNoIteration = setdiff(groupAnnotations, "iteration")
+ if ( dim(sub)[1] > 1 ) {
+ sum = cast(melt(sub, id.vars=groupAnnotationsNoIteration, measure.vars=c("runtime")), ... ~ ., fun.aggregate=c(mean, sd))
+ textplot(as.data.frame(sum), show.rownames=F)
+ title(paste("Job summary for", name, "averaging over all iterations"), cex=3)
+ }
+}
+
+# print out some useful basic information
+print("Report")
+print(paste("Project :", inputFileName))
+
+convertUnits <- function(gatkReportData) {
+ convertGroup <- function(g) {
+ g$runtime = g$runtime * ORIGINAL_UNITS_TO_SECONDS
+ g$startTime = g$startTime * ORIGINAL_UNITS_TO_SECONDS
+ g$doneTime = g$doneTime * ORIGINAL_UNITS_TO_SECONDS
+ g
+ }
+ lapply(gatkReportData, convertGroup)
+}
+
+
+# read the table
+gatkReportData <- gsa.read.gatkreport(inputFileName)
+gatkReportData <- convertUnits(gatkReportData)
+#print(summary(gatkReportData))
+
+if ( ! is.na(outputPDF) ) {
+ pdf(outputPDF, height=8.5, width=11)
+}
+
+plotJobsGantt(gatkReportData, T)
+plotJobsGantt(gatkReportData, F)
+plotProgressByTime(gatkReportData)
+for ( group in gatkReportData ) {
+ plotGroup(group)
+}
+
+if ( ! is.na(outputPDF) ) {
+ dev.off()
+}
diff --git a/public/R/src/gsalib/R/gsa.read.gatkreport.R b/public/R/src/gsalib/R/gsa.read.gatkreport.R
index 9b3ef1ad1..011b5240d 100644
--- a/public/R/src/gsalib/R/gsa.read.gatkreport.R
+++ b/public/R/src/gsalib/R/gsa.read.gatkreport.R
@@ -20,6 +20,20 @@
assign(tableName, d, envir=tableEnv);
}
+# Read a fixed width line of text into a list.
+.gsa.splitFixedWidth <- function(line, columnStarts) {
+ splitStartStop <- function(x) {
+ x = substring(x, starts, stops);
+ x = gsub("^[[:space:]]+|[[:space:]]+$", "", x);
+ x;
+ }
+
+ starts = c(1, columnStarts);
+ stops = c(columnStarts - 1, nchar(line));
+
+ sapply(line, splitStartStop)[,1];
+}
+
# Load all GATKReport tables from a file
gsa.read.gatkreport <- function(filename) {
con = file(filename, "r", blocking = TRUE);
@@ -31,9 +45,10 @@ gsa.read.gatkreport <- function(filename) {
tableName = NA;
tableHeader = c();
tableRows = c();
+ version = NA;
for (line in lines) {
- if (length(grep("^##:GATKReport.v0.1[[:space:]]+", line, ignore.case=TRUE)) > 0) {
+ if (length(grep("^##:GATKReport.v", line, ignore.case=TRUE)) > 0) {
headerFields = unlist(strsplit(line, "[[:space:]]+"));
if (!is.na(tableName)) {
@@ -43,13 +58,37 @@ gsa.read.gatkreport <- function(filename) {
tableName = headerFields[2];
tableHeader = c();
tableRows = c();
+
+ # For differences in versions see
+ # $STING_HOME/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportVersion.java
+ if (length(grep("^##:GATKReport.v0.1[[:space:]]+", line, ignore.case=TRUE)) > 0) {
+ version = "v0.1";
+
+ } else if (length(grep("^##:GATKReport.v0.2[[:space:]]+", line, ignore.case=TRUE)) > 0) {
+ version = "v0.2";
+ columnStarts = c();
+
+ }
+
} else if (length(grep("^[[:space:]]*$", line)) > 0 | length(grep("^[[:space:]]*#", line)) > 0) {
# do nothing
} else if (!is.na(tableName)) {
- row = unlist(strsplit(line, "[[:space:]]+"));
+
+ if (version == "v0.1") {
+ row = unlist(strsplit(line, "[[:space:]]+"));
+
+ } else if (version == "v0.2") {
+ if (length(tableHeader) == 0) {
+ headerChars = unlist(strsplit(line, ""));
+ # Find the first position of non space characters, excluding the first character
+ columnStarts = intersect(grep("[[:space:]]", headerChars, invert=TRUE), grep("[[:space:]]", headerChars) + 1);
+ }
+
+ row = .gsa.splitFixedWidth(line, columnStarts);
+ }
if (length(tableHeader) == 0) {
- tableHeader = row;
+ tableHeader = row;
} else {
tableRows = rbind(tableRows, row);
}
diff --git a/public/java/src/net/sf/picard/reference/FastaSequenceIndexBuilder.java b/public/java/src/net/sf/picard/reference/FastaSequenceIndexBuilder.java
index 8825c3767..6c8fe1834 100644
--- a/public/java/src/net/sf/picard/reference/FastaSequenceIndexBuilder.java
+++ b/public/java/src/net/sf/picard/reference/FastaSequenceIndexBuilder.java
@@ -25,7 +25,6 @@
package net.sf.picard.reference;
-import org.broadinstitute.sting.gatk.datasources.reference.ReferenceDataSourceProgressListener;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import static net.sf.picard.reference.FastaSequenceIndexBuilder.Status.*;
@@ -39,8 +38,8 @@ import org.broadinstitute.sting.utils.exceptions.UserException;
* Produces fai file with same output as samtools faidx
*/
public class FastaSequenceIndexBuilder {
- public File fastaFile;
- ReferenceDataSourceProgressListener progress; // interface that provides a method for updating user on progress of reading file
+ final public File fastaFile;
+ final boolean printProgress;
// keep track of location in file
long bytesRead, endOfLastLine, lastTimestamp, fileLength; // initialized to -1 to keep 0-indexed position in file;
@@ -55,10 +54,10 @@ public class FastaSequenceIndexBuilder {
public enum Status { NONE, CONTIG, FIRST_SEQ_LINE, SEQ_LINE, COMMENT }
Status status = Status.NONE; // keeps state of what is currently being read. better to use int instead of enum?
- public FastaSequenceIndexBuilder(File fastaFile, ReferenceDataSourceProgressListener progress) {
- this.progress = progress;
+ public FastaSequenceIndexBuilder(File fastaFile, boolean printProgress) {
this.fastaFile = fastaFile;
fileLength = fastaFile.length();
+ this.printProgress = printProgress;
}
/**
@@ -252,8 +251,8 @@ public class FastaSequenceIndexBuilder {
if (System.currentTimeMillis() - lastTimestamp > 10000) {
int percentProgress = (int) (100*bytesRead/fileLength);
- if (progress != null)
- progress.percentProgress(percentProgress);
+ if (printProgress)
+ System.out.println(String.format("PROGRESS UPDATE: file is %d percent complete", percentProgress));
lastTimestamp = System.currentTimeMillis();
}
}
diff --git a/public/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java b/public/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java
index b9e380295..7ea515591 100755
--- a/public/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java
+++ b/public/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java
@@ -31,30 +31,85 @@ import org.broadinstitute.sting.commandline.Input;
import org.broadinstitute.sting.gatk.walkers.recalibration.Covariate;
import org.broadinstitute.sting.gatk.walkers.recalibration.RecalDatum;
import org.broadinstitute.sting.gatk.walkers.recalibration.RecalibrationArgumentCollection;
+import org.broadinstitute.sting.utils.R.RScriptExecutor;
import org.broadinstitute.sting.utils.classloader.PluginManager;
import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException;
+import org.broadinstitute.sting.utils.help.DocumentedGATKFeature;
import org.broadinstitute.sting.utils.text.XReadLines;
import java.io.*;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
import java.util.regex.Pattern;
/**
- * Created by IntelliJ IDEA.
- * User: rpoplin
- * Date: Dec 1, 2009
+ * Call R scripts to plot residual error versus the various covariates.
+ *
+ *
+ * After counting covariates in either the initial BAM File or again in the recalibrated BAM File, an analysis tool is available which
+ * reads the .csv file and outputs several PDF (and .dat) files for each read group in the given BAM. These PDF files graphically
+ * show the various metrics and characteristics of the reported quality scores (often in relation to the empirical qualities).
+ * In order to show that any biases in the reported quality scores have been generally fixed through recalibration one should run
+ * CountCovariates again on a bam file produced by TableRecalibration. In this way users can compare the analysis plots generated
+ * by pre-recalibration and post-recalibration .csv files. Our usual chain of commands that we use to generate plots of residual
+ * error is: CountCovariates, TableRecalibrate, samtools index on the recalibrated bam file, CountCovariates again on the recalibrated
+ * bam file, and then AnalyzeCovariates on both the before and after recal_data.csv files to see the improvement in recalibration.
+ *
+ *
+ * The color coding along with the RMSE is included in the plots to give some indication of the number of observations that went into
+ * each of the quality score estimates. It is defined as follows for N, the number of observations:
+ *
+ *
+ *
light blue means N < 1,000
+ *
cornflower blue means 1,000 <= N < 10,000
+ *
dark blue means N >= 10,000
+ *
The pink dots indicate points whose quality scores are special codes used by the aligner and which are mathematically
+ * meaningless and so aren't included in any of the numerical calculations.
+ *
+ *
+ *
+ * NOTE: For those running this tool externally from the Broad, it is crucial to note that both the -Rscript and -resources options
+ * must be changed from the default. -Rscript needs to point to your installation of Rscript (this is the scripting version of R,
+ * not the interactive version) while -resources needs to point to the folder holding the R scripts that are used. For those using
+ * this tool as part of the Binary Distribution the -resources should point to the resources folder that is part of the tarball.
+ * For those using this tool by building from the git repository the -resources should point to the R/ subdirectory of the Sting checkout.
+ *
+ *
+ * See the GATK wiki for a tutorial and example recalibration accuracy plots.
+ * http://www.broadinstitute.org/gsa/wiki/index.php/Base_quality_score_recalibration
+ *
+ *
Input
+ *
+ * The recalibration table file in CSV format that was generated by the CountCovariates walker.
+ *
*
- * Create collapsed versions of the recal csv file and call R scripts to plot residual error versus the various covariates.
*/
+@DocumentedGATKFeature(
+ groupName = "AnalyzeCovariates",
+ summary = "Package to plot residual accuracy versus error covariates for the base quality score recalibrator")
public class AnalyzeCovariates extends CommandLineProgram {
/////////////////////////////
// Command Line Arguments
/////////////////////////////
-
+ /**
+ * After the header, data records occur one per line until the end of the file. The first several items on a line are the
+ * values of the individual covariates and will change depending on which covariates were specified at runtime. The last
+ * three items are the data- that is, number of observations for this combination of covariates, number of reference mismatches,
+ * and the raw empirical quality score calculated by phred-scaling the mismatch rate.
+ */
@Input(fullName = "recal_file", shortName = "recalFile", doc = "The input recal csv file to analyze", required = false)
private String RECAL_FILE = "output.recal_data.csv";
@Argument(fullName = "output_dir", shortName = "outputDir", doc = "The directory in which to output all the plots and intermediate data files", required = false)
@@ -65,13 +120,22 @@ public class AnalyzeCovariates extends CommandLineProgram {
private String PATH_TO_RESOURCES = "public/R/";
@Argument(fullName = "ignoreQ", shortName = "ignoreQ", doc = "Ignore bases with reported quality less than this number.", required = false)
private int IGNORE_QSCORES_LESS_THAN = 5;
- @Argument(fullName = "numRG", shortName = "numRG", doc = "Only process N read groups. Default value: -1 (process all read groups)", required = false)
+ @Argument(fullName = "numRG", shortName = "numRG", doc = "Only process N read groups. Default value: -1 (process all read groups)", required = false)
private int NUM_READ_GROUPS_TO_PROCESS = -1; // -1 means process all read groups
+
+ /**
+ * Combinations of covariates in which there are zero mismatches technically have infinite quality. We get around this situation
+ * by capping at the specified value. We've found that Q40 is too low when using a more completely database of known variation like dbSNP build 132 or later.
+ */
@Argument(fullName="max_quality_score", shortName="maxQ", required = false, doc="The integer value at which to cap the quality scores, default is 50")
private int MAX_QUALITY_SCORE = 50;
+
+ /**
+ * This argument is useful for comparing before/after plots and you want the axes to match each other.
+ */
@Argument(fullName="max_histogram_value", shortName="maxHist", required = false, doc="If supplied, this value will be the max value of the histogram plots")
private int MAX_HISTOGRAM_VALUE = 0;
- @Argument(fullName="do_indel_quality", shortName="indels", required = false, doc="If supplied, this value will be the max value of the histogram plots")
+ @Argument(fullName="do_indel_quality", shortName="indels", required = false, doc="If supplied, do indel quality plotting")
private boolean DO_INDEL_QUALITY = false;
@@ -261,13 +325,14 @@ public class AnalyzeCovariates extends CommandLineProgram {
}
private void callRScripts() {
+ RScriptExecutor.RScriptArgumentCollection argumentCollection =
+ new RScriptExecutor.RScriptArgumentCollection(PATH_TO_RSCRIPT, Arrays.asList(PATH_TO_RESOURCES));
+ RScriptExecutor executor = new RScriptExecutor(argumentCollection, true);
int numReadGroups = 0;
-
+
// for each read group
for( Object readGroupKey : dataManager.getCollapsedTable(0).data.keySet() ) {
-
- Process p;
if(++numReadGroups <= NUM_READ_GROUPS_TO_PROCESS || NUM_READ_GROUPS_TO_PROCESS == -1) {
String readGroup = readGroupKey.toString();
@@ -276,35 +341,19 @@ public class AnalyzeCovariates extends CommandLineProgram {
// for each covariate
for( int iii = 1; iii < requestedCovariates.size(); iii++ ) {
Covariate cov = requestedCovariates.get(iii);
- try {
-
- if (DO_INDEL_QUALITY) {
- p = Runtime.getRuntime().exec(PATH_TO_RSCRIPT + " " + PATH_TO_RESOURCES + "plot_indelQuality.R" + " " +
- OUTPUT_DIR + readGroup + "." + cov.getClass().getSimpleName()+ ".dat" + " " +
- cov.getClass().getSimpleName().split("Covariate")[0]); // The third argument is the name of the covariate in order to make the plots look nice
- p.waitFor();
-
- } else {
+ final String outputFilename = OUTPUT_DIR + readGroup + "." + cov.getClass().getSimpleName()+ ".dat";
+ if (DO_INDEL_QUALITY) {
+ executor.callRScripts("plot_indelQuality.R", outputFilename,
+ cov.getClass().getSimpleName().split("Covariate")[0]); // The third argument is the name of the covariate in order to make the plots look nice
+ } else {
if( iii == 1 ) {
- // Analyze reported quality
- p = Runtime.getRuntime().exec(PATH_TO_RSCRIPT + " " + PATH_TO_RESOURCES + "plot_residualError_QualityScoreCovariate.R" + " " +
- OUTPUT_DIR + readGroup + "." + cov.getClass().getSimpleName()+ ".dat" + " " +
- IGNORE_QSCORES_LESS_THAN + " " + MAX_QUALITY_SCORE + " " + MAX_HISTOGRAM_VALUE); // The third argument is the Q scores that should be turned pink in the plot because they were ignored
- p.waitFor();
- } else { // Analyze all other covariates
- p = Runtime.getRuntime().exec(PATH_TO_RSCRIPT + " " + PATH_TO_RESOURCES + "plot_residualError_OtherCovariate.R" + " " +
- OUTPUT_DIR + readGroup + "." + cov.getClass().getSimpleName()+ ".dat" + " " +
- cov.getClass().getSimpleName().split("Covariate")[0]); // The third argument is the name of the covariate in order to make the plots look nice
- p.waitFor();
- }
+ // Analyze reported quality
+ executor.callRScripts("plot_residualError_QualityScoreCovariate.R", outputFilename,
+ IGNORE_QSCORES_LESS_THAN, MAX_QUALITY_SCORE, MAX_HISTOGRAM_VALUE); // The third argument is the Q scores that should be turned pink in the plot because they were ignored
+ } else { // Analyze all other covariates
+ executor.callRScripts("plot_residualError_OtherCovariate.R", outputFilename,
+ cov.getClass().getSimpleName().split("Covariate")[0]); // The third argument is the name of the covariate in order to make the plots look nice
}
- } catch (InterruptedException e) {
- e.printStackTrace();
- System.exit(-1);
- } catch (IOException e) {
- System.out.println("Fatal Exception: Perhaps RScript jobs are being spawned too quickly? One work around is to process fewer read groups using the -numRG option.");
- e.printStackTrace();
- System.exit(-1);
}
}
} else { // at the maximum number of read groups so break out
diff --git a/public/java/src/org/broadinstitute/sting/analyzecovariates/package-info.java b/public/java/src/org/broadinstitute/sting/analyzecovariates/package-info.java
new file mode 100644
index 000000000..9350e4a66
--- /dev/null
+++ b/public/java/src/org/broadinstitute/sting/analyzecovariates/package-info.java
@@ -0,0 +1,4 @@
+/**
+ * Package to plot residual accuracy versus error covariates for the base quality score recalibrator.
+ */
+package org.broadinstitute.sting.analyzecovariates;
\ No newline at end of file
diff --git a/public/java/src/org/broadinstitute/sting/commandline/Advanced.java b/public/java/src/org/broadinstitute/sting/commandline/Advanced.java
new file mode 100644
index 000000000..7aeefe261
--- /dev/null
+++ b/public/java/src/org/broadinstitute/sting/commandline/Advanced.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.commandline;
+
+import java.lang.annotation.*;
+
+/**
+ * Indicates that a walker argument should is considered an advanced option.
+ *
+ * @author Mark DePristo
+ * @version 0.1
+ */
+@Documented
+@Inherited
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE,ElementType.FIELD})
+public @interface Advanced {
+}
diff --git a/public/java/src/org/broadinstitute/sting/commandline/ArgumentDefinitions.java b/public/java/src/org/broadinstitute/sting/commandline/ArgumentDefinitions.java
index 9f92df6e0..8e3f753a8 100755
--- a/public/java/src/org/broadinstitute/sting/commandline/ArgumentDefinitions.java
+++ b/public/java/src/org/broadinstitute/sting/commandline/ArgumentDefinitions.java
@@ -174,7 +174,8 @@ public class ArgumentDefinitions implements Iterable {
static DefinitionMatcher VerifiableDefinitionMatcher = new DefinitionMatcher() {
public boolean matches( ArgumentDefinition definition, Object key ) {
- return definition.validation != null;
+ // We can perform some sort of validation for anything that isn't a flag.
+ return !definition.isFlag;
}
};
}
diff --git a/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatch.java b/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatch.java
index 60ed8c899..351583c07 100755
--- a/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatch.java
+++ b/public/java/src/org/broadinstitute/sting/commandline/ArgumentMatch.java
@@ -44,7 +44,7 @@ public class ArgumentMatch implements Iterable {
public final String label;
/**
- * Maps indicies of command line arguments to values paired with that argument.
+ * Maps indices of command line arguments to values paired with that argument.
*/
public final SortedMap> indices = new TreeMap>();
diff --git a/public/java/src/org/broadinstitute/sting/commandline/ArgumentSource.java b/public/java/src/org/broadinstitute/sting/commandline/ArgumentSource.java
index f48ca864a..8ec0d650a 100644
--- a/public/java/src/org/broadinstitute/sting/commandline/ArgumentSource.java
+++ b/public/java/src/org/broadinstitute/sting/commandline/ArgumentSource.java
@@ -151,6 +151,14 @@ public class ArgumentSource {
return field.isAnnotationPresent(Hidden.class) || field.isAnnotationPresent(Deprecated.class);
}
+ /**
+ * Is the given argument considered an advanced option when displaying on the command-line argument system.
+ * @return True if so. False otherwise.
+ */
+ public boolean isAdvanced() {
+ return field.isAnnotationPresent(Advanced.class);
+ }
+
/**
* Is this command-line argument dependent on some primitive argument types?
* @return True if this command-line argument depends on other arguments; false otherwise.
@@ -175,13 +183,17 @@ public class ArgumentSource {
return typeDescriptor.createsTypeDefault(this);
}
+ public String typeDefaultDocString() {
+ return typeDescriptor.typeDefaultDocString(this);
+ }
+
/**
* Generates a default for the given type.
* @param parsingEngine the parsing engine used to validate this argument type descriptor.
* @return A default value for the given type.
*/
public Object createTypeDefault(ParsingEngine parsingEngine) {
- return typeDescriptor.createTypeDefault(parsingEngine,this,field.getType());
+ return typeDescriptor.createTypeDefault(parsingEngine,this,field.getGenericType());
}
/**
diff --git a/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java b/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java
index 9c33e084d..5fff8f609 100644
--- a/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java
+++ b/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java
@@ -26,6 +26,8 @@
package org.broadinstitute.sting.commandline;
import org.apache.log4j.Logger;
+import org.broad.tribble.Feature;
+import org.broadinstitute.sting.gatk.refdata.tracks.FeatureManager;
import org.broadinstitute.sting.gatk.walkers.Multiplex;
import org.broadinstitute.sting.gatk.walkers.Multiplexer;
import org.broadinstitute.sting.utils.classloader.JVMUtils;
@@ -33,6 +35,7 @@ import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import org.broadinstitute.sting.utils.exceptions.UserException;
+import java.io.File;
import java.lang.annotation.Annotation;
import java.lang.reflect.*;
import java.util.*;
@@ -80,14 +83,26 @@ public abstract class ArgumentTypeDescriptor {
*/
public boolean createsTypeDefault(ArgumentSource source) { return false; }
+ /**
+ * Returns a documentation-friendly value for the default of a type descriptor.
+ * Must be overridden if createsTypeDefault return true. cannot be called otherwise
+ * @param source Source of the command-line argument.
+ * @return Friendly string of the default value, for documentation. If doesn't create a default, throws
+ * and UnsupportedOperationException
+ */
+ public String typeDefaultDocString(ArgumentSource source) {
+ throw new UnsupportedOperationException();
+ }
+
/**
* Generates a default for the given type.
+ *
* @param parsingEngine the parsing engine used to validate this argument type descriptor.
* @param source Source of the command-line argument.
* @param type Type of value to create, in case the command-line argument system wants influence.
* @return A default value for the given type.
*/
- public Object createTypeDefault(ParsingEngine parsingEngine,ArgumentSource source,Class type) { throw new UnsupportedOperationException("Unable to create default for type " + getClass()); }
+ public Object createTypeDefault(ParsingEngine parsingEngine,ArgumentSource source, Type type) { throw new UnsupportedOperationException("Unable to create default for type " + getClass()); }
/**
* Given the given argument source and attributes, synthesize argument definitions for command-line arguments.
@@ -109,7 +124,7 @@ public abstract class ArgumentTypeDescriptor {
* @return The parsed object.
*/
public Object parse(ParsingEngine parsingEngine, ArgumentSource source, ArgumentMatches matches) {
- return parse(parsingEngine, source, source.field.getType(), matches);
+ return parse(parsingEngine, source, source.field.getGenericType(), matches);
}
/**
@@ -131,18 +146,18 @@ public abstract class ArgumentTypeDescriptor {
protected ArgumentDefinition createDefaultArgumentDefinition( ArgumentSource source ) {
Annotation argumentAnnotation = getArgumentAnnotation(source);
return new ArgumentDefinition( ArgumentIOType.getIOType(argumentAnnotation),
- source.field.getType(),
- ArgumentDefinition.getFullName(argumentAnnotation, source.field.getName()),
- ArgumentDefinition.getShortName(argumentAnnotation),
- ArgumentDefinition.getDoc(argumentAnnotation),
- source.isRequired() && !createsTypeDefault(source) && !source.isFlag() && !source.isDeprecated(),
- source.isFlag(),
- source.isMultiValued(),
- source.isHidden(),
- getCollectionComponentType(source.field),
- ArgumentDefinition.getExclusiveOf(argumentAnnotation),
- ArgumentDefinition.getValidationRegex(argumentAnnotation),
- getValidOptions(source) );
+ source.field.getType(),
+ ArgumentDefinition.getFullName(argumentAnnotation, source.field.getName()),
+ ArgumentDefinition.getShortName(argumentAnnotation),
+ ArgumentDefinition.getDoc(argumentAnnotation),
+ source.isRequired() && !createsTypeDefault(source) && !source.isFlag() && !source.isDeprecated(),
+ source.isFlag(),
+ source.isMultiValued(),
+ source.isHidden(),
+ makeRawTypeIfNecessary(getCollectionComponentType(source.field)),
+ ArgumentDefinition.getExclusiveOf(argumentAnnotation),
+ ArgumentDefinition.getValidationRegex(argumentAnnotation),
+ getValidOptions(source) );
}
/**
@@ -151,7 +166,7 @@ public abstract class ArgumentTypeDescriptor {
* @return The parameterized component type, or String.class if the parameterized type could not be found.
* @throws IllegalArgumentException If more than one parameterized type is found on the field.
*/
- protected Class getCollectionComponentType( Field field ) {
+ protected Type getCollectionComponentType( Field field ) {
return null;
}
@@ -162,7 +177,7 @@ public abstract class ArgumentTypeDescriptor {
* @param matches The argument matches for the argument source, or the individual argument match for a scalar if this is being called to help parse a collection.
* @return The individual parsed object matching the argument match with Class type.
*/
- public abstract Object parse( ParsingEngine parsingEngine, ArgumentSource source, Class type, ArgumentMatches matches );
+ public abstract Object parse( ParsingEngine parsingEngine, ArgumentSource source, Type type, ArgumentMatches matches );
/**
* If the argument source only accepts a small set of options, populate the returned list with
@@ -273,6 +288,123 @@ public abstract class ArgumentTypeDescriptor {
public static boolean isArgumentHidden(Field field) {
return field.isAnnotationPresent(Hidden.class);
}
+
+ public Class makeRawTypeIfNecessary(Type t) {
+ if ( t == null )
+ return null;
+ else if ( t instanceof ParameterizedType )
+ return (Class)((ParameterizedType) t).getRawType();
+ else if ( t instanceof Class ) {
+ return (Class)t;
+ } else {
+ throw new IllegalArgumentException("Unable to determine Class-derived component type of field: " + t);
+ }
+ }
+}
+
+/**
+ * Parser for RodBinding objects
+ */
+class RodBindingArgumentTypeDescriptor extends ArgumentTypeDescriptor {
+ /**
+ * We only want RodBinding class objects
+ * @param type The type to check.
+ * @return true if the provided class is a RodBinding.class
+ */
+ @Override
+ public boolean supports( Class type ) {
+ return isRodBinding(type);
+ }
+
+ public static boolean isRodBinding( Class type ) {
+ return RodBinding.class.isAssignableFrom(type);
+ }
+
+ @Override
+ public boolean createsTypeDefault(ArgumentSource source) { return ! source.isRequired(); }
+
+ @Override
+ public Object createTypeDefault(ParsingEngine parsingEngine, ArgumentSource source, Type type) {
+ Class parameterType = JVMUtils.getParameterizedTypeClass(type);
+ return RodBinding.makeUnbound((Class)parameterType);
+ }
+
+ @Override
+ public String typeDefaultDocString(ArgumentSource source) {
+ return "none";
+ }
+
+ @Override
+ public Object parse(ParsingEngine parsingEngine, ArgumentSource source, Type type, ArgumentMatches matches) {
+ ArgumentDefinition defaultDefinition = createDefaultArgumentDefinition(source);
+ String value = getArgumentValue( defaultDefinition, matches );
+ Class parameterType = JVMUtils.getParameterizedTypeClass(type);
+
+ try {
+ String name = defaultDefinition.fullName;
+ String tribbleType = null;
+ Tags tags = getArgumentTags(matches);
+ // must have one or two tag values here
+ if ( tags.getPositionalTags().size() > 2 ) {
+ throw new UserException.CommandLineException(
+ String.format("Unexpected number of positional tags for argument %s : %s. " +
+ "Rod bindings only suport -X:type and -X:name,type argument styles",
+ value, source.field.getName()));
+ } if ( tags.getPositionalTags().size() == 2 ) {
+ // -X:name,type style
+ name = tags.getPositionalTags().get(0);
+ tribbleType = tags.getPositionalTags().get(1);
+ } else {
+ // case with 0 or 1 positional tags
+ FeatureManager manager = new FeatureManager();
+
+ // -X:type style is a type when we cannot determine the type dynamically
+ String tag1 = tags.getPositionalTags().size() == 1 ? tags.getPositionalTags().get(0) : null;
+ if ( tag1 != null ) {
+ if ( manager.getByName(tag1) != null ) // this a type
+ tribbleType = tag1;
+ else
+ name = tag1;
+ }
+
+ if ( tribbleType == null ) {
+ // try to determine the file type dynamically
+ File file = new File(value);
+ if ( file.canRead() && file.isFile() ) {
+ FeatureManager.FeatureDescriptor featureDescriptor = manager.getByFiletype(file);
+ if ( featureDescriptor != null ) {
+ tribbleType = featureDescriptor.getName();
+ logger.info("Dynamically determined type of " + file + " to be " + tribbleType);
+ }
+ }
+
+ if ( tribbleType == null )
+ if ( ! file.canRead() | ! file.isFile() ) {
+ throw new UserException.BadArgumentValue(name, "Couldn't read file to determine type: " + file);
+ } else {
+ throw new UserException.CommandLineException(
+ String.format("No tribble type was provided on the command line and the type of the file could not be determined dynamically. " +
+ "Please add an explicit type tag :NAME listing the correct type from among the supported types:%n%s",
+ manager.userFriendlyListOfAvailableFeatures(parameterType)));
+ }
+ }
+ }
+
+ Constructor ctor = (makeRawTypeIfNecessary(type)).getConstructor(Class.class, String.class, String.class, String.class, Tags.class);
+ RodBinding result = (RodBinding)ctor.newInstance(parameterType, name, value, tribbleType, tags);
+ parsingEngine.addTags(result,tags);
+ parsingEngine.addRodBinding(result);
+ return result;
+ } catch (InvocationTargetException e) {
+ throw new UserException.CommandLineException(
+ String.format("Failed to parse value %s for argument %s.",
+ value, source.field.getName()));
+ } catch (Exception e) {
+ throw new UserException.CommandLineException(
+ String.format("Failed to parse value %s for argument %s. Message: %s",
+ value, source.field.getName(), e.getMessage()));
+ }
+ }
}
/**
@@ -282,9 +414,10 @@ public abstract class ArgumentTypeDescriptor {
class SimpleArgumentTypeDescriptor extends ArgumentTypeDescriptor {
@Override
public boolean supports( Class type ) {
- if( type.isPrimitive() ) return true;
- if( type.isEnum() ) return true;
- if( primitiveToWrapperMap.containsValue(type) ) return true;
+ if ( RodBindingArgumentTypeDescriptor.isRodBinding(type) ) return false;
+ if ( type.isPrimitive() ) return true;
+ if ( type.isEnum() ) return true;
+ if ( primitiveToWrapperMap.containsValue(type) ) return true;
try {
type.getConstructor(String.class);
@@ -298,7 +431,8 @@ class SimpleArgumentTypeDescriptor extends ArgumentTypeDescriptor {
}
@Override
- public Object parse(ParsingEngine parsingEngine, ArgumentSource source, Class type, ArgumentMatches matches) {
+ public Object parse(ParsingEngine parsingEngine, ArgumentSource source, Type fulltype, ArgumentMatches matches) {
+ Class type = makeRawTypeIfNecessary(fulltype);
if (source.isFlag())
return true;
@@ -339,7 +473,7 @@ class SimpleArgumentTypeDescriptor extends ArgumentTypeDescriptor {
throw e;
} catch (InvocationTargetException e) {
throw new UserException.CommandLineException(String.format("Failed to parse value %s for argument %s. This is most commonly caused by providing an incorrect data type (e.g. a double when an int is required)",
- value, source.field.getName()));
+ value, source.field.getName()));
} catch (Exception e) {
throw new DynamicClassResolutionException(String.class, e);
}
@@ -351,7 +485,7 @@ class SimpleArgumentTypeDescriptor extends ArgumentTypeDescriptor {
return result;
}
-
+
/**
* A mapping of the primitive types to their associated wrapper classes. Is there really no way to infer
@@ -382,10 +516,10 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor {
@Override
@SuppressWarnings("unchecked")
- public Object parse(ParsingEngine parsingEngine,ArgumentSource source, Class type, ArgumentMatches matches) {
- Class componentType;
+ public Object parse(ParsingEngine parsingEngine,ArgumentSource source, Type fulltype, ArgumentMatches matches) {
+ Class type = makeRawTypeIfNecessary(fulltype);
+ Type componentType;
Object result;
- Tags tags;
if( Collection.class.isAssignableFrom(type) ) {
@@ -399,7 +533,7 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor {
}
componentType = getCollectionComponentType( source.field );
- ArgumentTypeDescriptor componentArgumentParser = parsingEngine.selectBestTypeDescriptor(componentType);
+ ArgumentTypeDescriptor componentArgumentParser = parsingEngine.selectBestTypeDescriptor(makeRawTypeIfNecessary(componentType));
Collection collection;
try {
@@ -428,7 +562,7 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor {
}
else if( type.isArray() ) {
componentType = type.getComponentType();
- ArgumentTypeDescriptor componentArgumentParser = parsingEngine.selectBestTypeDescriptor(componentType);
+ ArgumentTypeDescriptor componentArgumentParser = parsingEngine.selectBestTypeDescriptor(makeRawTypeIfNecessary(componentType));
// Assemble a collection of individual values used in this computation.
Collection values = new ArrayList();
@@ -436,7 +570,7 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor {
for( ArgumentMatch value: match )
values.add(value);
- result = Array.newInstance(componentType,values.size());
+ result = Array.newInstance(makeRawTypeIfNecessary(componentType),values.size());
int i = 0;
for( ArgumentMatch value: values ) {
@@ -459,16 +593,16 @@ class CompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor {
* @throws IllegalArgumentException If more than one parameterized type is found on the field.
*/
@Override
- protected Class getCollectionComponentType( Field field ) {
- // If this is a parameterized collection, find the contained type. If blow up if more than one type exists.
- if( field.getGenericType() instanceof ParameterizedType) {
- ParameterizedType parameterizedType = (ParameterizedType)field.getGenericType();
- if( parameterizedType.getActualTypeArguments().length > 1 )
- throw new IllegalArgumentException("Unable to determine collection type of field: " + field.toString());
- return (Class)parameterizedType.getActualTypeArguments()[0];
- }
- else
- return String.class;
+ protected Type getCollectionComponentType( Field field ) {
+ // If this is a parameterized collection, find the contained type. If blow up if more than one type exists.
+ if( field.getGenericType() instanceof ParameterizedType) {
+ ParameterizedType parameterizedType = (ParameterizedType)field.getGenericType();
+ if( parameterizedType.getActualTypeArguments().length > 1 )
+ throw new IllegalArgumentException("Unable to determine collection type of field: " + field.toString());
+ return parameterizedType.getActualTypeArguments()[0];
+ }
+ else
+ return String.class;
}
}
@@ -510,12 +644,12 @@ class MultiplexArgumentTypeDescriptor extends ArgumentTypeDescriptor {
}
@Override
- public Object createTypeDefault(ParsingEngine parsingEngine,ArgumentSource source,Class type) {
+ public Object createTypeDefault(ParsingEngine parsingEngine,ArgumentSource source, Type type) {
if(multiplexer == null || multiplexedIds == null)
throw new ReviewedStingException("No multiplexed ids available");
Map multiplexedMapping = new HashMap();
- Class componentType = getCollectionComponentType(source.field);
+ Class componentType = makeRawTypeIfNecessary(getCollectionComponentType(source.field));
ArgumentTypeDescriptor componentTypeDescriptor = parsingEngine.selectBestTypeDescriptor(componentType);
for(Object id: multiplexedIds) {
@@ -527,15 +661,19 @@ class MultiplexArgumentTypeDescriptor extends ArgumentTypeDescriptor {
return multiplexedMapping;
}
+ @Override
+ public String typeDefaultDocString(ArgumentSource source) {
+ return "None";
+ }
@Override
- public Object parse(ParsingEngine parsingEngine, ArgumentSource source, Class type, ArgumentMatches matches) {
+ public Object parse(ParsingEngine parsingEngine, ArgumentSource source, Type type, ArgumentMatches matches) {
if(multiplexedIds == null)
throw new ReviewedStingException("Cannot directly parse a MultiplexArgumentTypeDescriptor; must create a derivative type descriptor first.");
Map multiplexedMapping = new HashMap();
- Class componentType = getCollectionComponentType(source.field);
+ Class componentType = makeRawTypeIfNecessary(getCollectionComponentType(source.field));
for(Object id: multiplexedIds) {
@@ -606,7 +744,7 @@ class MultiplexArgumentTypeDescriptor extends ArgumentTypeDescriptor {
* @throws IllegalArgumentException If more than one parameterized type is found on the field.
*/
@Override
- protected Class getCollectionComponentType( Field field ) {
+ protected Type getCollectionComponentType( Field field ) {
// Multiplex arguments must resolve to maps from which the clp should extract the second type.
if( field.getGenericType() instanceof ParameterizedType) {
ParameterizedType parameterizedType = (ParameterizedType)field.getGenericType();
diff --git a/public/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java b/public/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java
index aba4fc109..d88e7030e 100644
--- a/public/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java
+++ b/public/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java
@@ -43,7 +43,7 @@ import java.util.Locale;
public abstract class CommandLineProgram {
/** The command-line program and the arguments it returned. */
- protected ParsingEngine parser = null;
+ public ParsingEngine parser = null;
/** the default log level */
@Argument(fullName = "logging_level",
@@ -144,6 +144,11 @@ public abstract class CommandLineProgram {
public static int result = -1;
+ @SuppressWarnings("unchecked")
+ public static void start(CommandLineProgram clp, String[] args) throws Exception {
+ start(clp, args, false);
+ }
+
/**
* This function is called to start processing the command line, and kick
* off the execute message of the program.
@@ -153,7 +158,7 @@ public abstract class CommandLineProgram {
* @throws Exception when an exception occurs
*/
@SuppressWarnings("unchecked")
- public static void start(CommandLineProgram clp, String[] args) throws Exception {
+ public static void start(CommandLineProgram clp, String[] args, boolean dryRun) throws Exception {
try {
// setup our log layout
@@ -180,8 +185,9 @@ public abstract class CommandLineProgram {
// - InvalidArgument in case these arguments are specified by plugins.
// - MissingRequiredArgument in case the user requested help. Handle that later, once we've
// determined the full complement of arguments.
- parser.validate(EnumSet.of(ParsingEngine.ValidationType.MissingRequiredArgument,
- ParsingEngine.ValidationType.InvalidArgument));
+ if ( ! dryRun )
+ parser.validate(EnumSet.of(ParsingEngine.ValidationType.MissingRequiredArgument,
+ ParsingEngine.ValidationType.InvalidArgument));
parser.loadArgumentsIntoObject(clp);
// Initialize the logger using the loaded command line.
@@ -195,36 +201,40 @@ public abstract class CommandLineProgram {
if (isHelpPresent(parser))
printHelpAndExit(clp, parser);
- parser.validate();
+ if ( ! dryRun ) parser.validate();
} else {
parser.parse(args);
- if (isHelpPresent(parser))
- printHelpAndExit(clp, parser);
+ if ( ! dryRun ) {
+ if (isHelpPresent(parser))
+ printHelpAndExit(clp, parser);
- parser.validate();
+ parser.validate();
+ }
parser.loadArgumentsIntoObject(clp);
// Initialize the logger using the loaded command line.
clp.setupLoggerLevel(layout);
}
- // if they specify a log location, output our data there
- if (clp.toFile != null) {
- FileAppender appender;
- try {
- appender = new FileAppender(layout, clp.toFile, false);
- logger.addAppender(appender);
- } catch (IOException e) {
- throw new RuntimeException("Unable to re-route log output to " + clp.toFile + " make sure the destination exists");
+ if ( ! dryRun ) {
+ // if they specify a log location, output our data there
+ if (clp.toFile != null) {
+ FileAppender appender;
+ try {
+ appender = new FileAppender(layout, clp.toFile, false);
+ logger.addAppender(appender);
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to re-route log output to " + clp.toFile + " make sure the destination exists");
+ }
}
+
+ // regardless of what happens next, generate the header information
+ HelpFormatter.generateHeaderInformation(clp.getApplicationDetails(), args);
+
+ // call the execute
+ CommandLineProgram.result = clp.execute();
}
-
- // regardless of what happens next, generate the header information
- HelpFormatter.generateHeaderInformation(clp.getApplicationDetails(), args);
-
- // call the execute
- CommandLineProgram.result = clp.execute();
}
catch (ArgumentException e) {
clp.parser.printHelp(clp.getApplicationDetails());
diff --git a/public/java/src/org/broadinstitute/sting/commandline/Output.java b/public/java/src/org/broadinstitute/sting/commandline/Output.java
index 22565dbf5..f8aef0355 100644
--- a/public/java/src/org/broadinstitute/sting/commandline/Output.java
+++ b/public/java/src/org/broadinstitute/sting/commandline/Output.java
@@ -55,7 +55,7 @@ public @interface Output {
* --help argument is specified.
* @return Doc string associated with this command-line argument.
*/
- String doc() default "An output file presented to the walker. Will overwrite contents if file exists.";
+ String doc() default "An output file created by the walker. Will overwrite contents if file exists";
/**
* Is this argument required. If true, the command-line argument system will
diff --git a/public/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java b/public/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java
index 8423bb2f2..fbf8c6516 100755
--- a/public/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java
+++ b/public/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java
@@ -25,6 +25,7 @@
package org.broadinstitute.sting.commandline;
+import com.google.java.contract.Requires;
import org.apache.log4j.Logger;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.classloader.JVMUtils;
@@ -41,11 +42,16 @@ import java.util.*;
* A parser for Sting command-line arguments.
*/
public class ParsingEngine {
+ /**
+ * The loaded argument sources along with their back definitions.
+ */
+ private Map argumentSourcesByDefinition = new HashMap();
+
/**
* A list of defined arguments against which command lines are matched.
* Package protected for testing access.
*/
- ArgumentDefinitions argumentDefinitions = new ArgumentDefinitions();
+ public ArgumentDefinitions argumentDefinitions = new ArgumentDefinitions();
/**
* A list of matches from defined arguments to command-line text.
@@ -59,11 +65,17 @@ public class ParsingEngine {
*/
private List parsingMethods = new ArrayList();
+ /**
+ * All of the RodBinding objects we've seen while parsing
+ */
+ private List rodBindings = new ArrayList();
+
/**
* Class reference to the different types of descriptors that the create method can create.
* The type of set used must be ordered (but not necessarily sorted).
*/
private static final Set STANDARD_ARGUMENT_TYPE_DESCRIPTORS = new LinkedHashSet( Arrays.asList(new SimpleArgumentTypeDescriptor(),
+ new RodBindingArgumentTypeDescriptor(),
new CompoundArgumentTypeDescriptor(),
new MultiplexArgumentTypeDescriptor()) );
@@ -80,6 +92,7 @@ public class ParsingEngine {
protected static Logger logger = Logger.getLogger(ParsingEngine.class);
public ParsingEngine( CommandLineProgram clp ) {
+ RodBinding.resetNameCounter();
parsingMethods.add( ParsingMethod.FullNameParsingMethod );
parsingMethods.add( ParsingMethod.ShortNameParsingMethod );
@@ -107,8 +120,13 @@ public class ParsingEngine {
*/
public void addArgumentSource( String sourceName, Class sourceClass ) {
List argumentsFromSource = new ArrayList();
- for( ArgumentSource argumentSource: extractArgumentSources(sourceClass) )
- argumentsFromSource.addAll( argumentSource.createArgumentDefinitions() );
+ for( ArgumentSource argumentSource: extractArgumentSources(sourceClass) ) {
+ List argumentDefinitions = argumentSource.createArgumentDefinitions();
+ for(ArgumentDefinition argumentDefinition: argumentDefinitions) {
+ argumentSourcesByDefinition.put(argumentDefinition,argumentSource);
+ argumentsFromSource.add( argumentDefinition );
+ }
+ }
argumentDefinitions.add( new ArgumentDefinitionGroup(sourceName, argumentsFromSource) );
}
@@ -199,16 +217,25 @@ public class ParsingEngine {
throw new InvalidArgumentException( invalidArguments );
}
- // Find invalid argument values (arguments that fail the regexp test.
+ // Find invalid argument values -- invalid arguments are either completely missing or fail the specified 'validation' regular expression.
if( !skipValidationOf.contains(ValidationType.InvalidArgumentValue) ) {
Collection verifiableArguments =
argumentDefinitions.findArgumentDefinitions( null, ArgumentDefinitions.VerifiableDefinitionMatcher );
Collection> invalidValues = new ArrayList>();
for( ArgumentDefinition verifiableArgument: verifiableArguments ) {
ArgumentMatches verifiableMatches = argumentMatches.findMatches( verifiableArgument );
+ // Check to see whether an argument value was specified. Argument values must be provided
+ // when the argument name is specified and the argument is not a flag type.
+ for(ArgumentMatch verifiableMatch: verifiableMatches) {
+ ArgumentSource argumentSource = argumentSourcesByDefinition.get(verifiableArgument);
+ if(verifiableMatch.values().size() == 0 && !verifiableArgument.isFlag && argumentSource.createsTypeDefault())
+ invalidValues.add(new Pair(verifiableArgument,null));
+ }
+
+ // Ensure that the field contents meet the validation criteria specified by the regular expression.
for( ArgumentMatch verifiableMatch: verifiableMatches ) {
for( String value: verifiableMatch.values() ) {
- if( !value.matches(verifiableArgument.validation) )
+ if( verifiableArgument.validation != null && !value.matches(verifiableArgument.validation) )
invalidValues.add( new Pair(verifiableArgument, value) );
}
}
@@ -304,7 +331,17 @@ public class ParsingEngine {
if(!tags.containsKey(key))
return new Tags();
return tags.get(key);
- }
+ }
+
+ /**
+ * Add a RodBinding type argument to this parser. Called during parsing to allow
+ * us to track all of the RodBindings discovered in the command line.
+ * @param rodBinding the rodbinding to add. Must not be added twice
+ */
+ @Requires("rodBinding != null")
+ public void addRodBinding(final RodBinding rodBinding) {
+ rodBindings.add(rodBinding);
+ }
/**
* Notify the user that a deprecated command-line argument has been used.
@@ -327,7 +364,7 @@ public class ParsingEngine {
*/
private void loadValueIntoObject( ArgumentSource source, Object instance, ArgumentMatches argumentMatches ) {
// Nothing to load
- if( argumentMatches.size() == 0 && !(source.createsTypeDefault() && source.isRequired()))
+ if( argumentMatches.size() == 0 && ! source.createsTypeDefault() )
return;
// Target instance into which to inject the value.
@@ -344,6 +381,10 @@ public class ParsingEngine {
}
}
+ public Collection getRodBindings() {
+ return Collections.unmodifiableCollection(rodBindings);
+ }
+
/**
* Gets a collection of the container instances of the given type stored within the given target.
* @param source Argument source.
@@ -390,7 +431,6 @@ public class ParsingEngine {
return ArgumentTypeDescriptor.selectBest(argumentTypeDescriptors,type);
}
-
private List extractArgumentSources(Class sourceClass, Field[] parentFields) {
// now simply call into the truly general routine extract argument bindings but with a null
// object so bindings aren't computed
@@ -515,10 +555,14 @@ class InvalidArgumentValueException extends ArgumentException {
private static String formatArguments( Collection> invalidArgumentValues ) {
StringBuilder sb = new StringBuilder();
for( Pair invalidValue: invalidArgumentValues ) {
- sb.append( String.format("%nArgument '--%s' has value of incorrect format: %s (should match %s)",
- invalidValue.first.fullName,
- invalidValue.second,
- invalidValue.first.validation) );
+ if(invalidValue.getSecond() == null)
+ sb.append( String.format("%nArgument '--%s' requires a value but none was provided",
+ invalidValue.first.fullName) );
+ else
+ sb.append( String.format("%nArgument '--%s' has value of incorrect format: %s (should match %s)",
+ invalidValue.first.fullName,
+ invalidValue.second,
+ invalidValue.first.validation) );
}
return sb.toString();
}
diff --git a/public/java/src/org/broadinstitute/sting/commandline/RodBinding.java b/public/java/src/org/broadinstitute/sting/commandline/RodBinding.java
new file mode 100644
index 000000000..e0b1154c4
--- /dev/null
+++ b/public/java/src/org/broadinstitute/sting/commandline/RodBinding.java
@@ -0,0 +1,187 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.commandline;
+
+import com.google.java.contract.Ensures;
+import com.google.java.contract.Requires;
+import org.broad.tribble.Feature;
+
+import java.util.*;
+
+/**
+ * A RodBinding representing a walker argument that gets bound to a ROD track.
+ *
+ * The RodBinding is a formal GATK argument that bridges between a walker and
+ * the RefMetaDataTracker to obtain data about this rod track at runtime. The RodBinding
+ * is explicitly typed with type of the Tribble.Feature expected to be produced by this
+ * argument. The GATK Engine takes care of initializing the binding and connecting it
+ * to the RMD system.
+ *
+ * It is recommended that optional RodBindings be initialized to the value returned
+ * by the static method makeUnbound().
+ *
+ * Note that this class is immutable.
+ */
+public final class RodBinding {
+ protected final static String UNBOUND_VARIABLE_NAME = "";
+ protected final static String UNBOUND_SOURCE = "UNBOUND";
+ protected final static String UNBOUND_TRIBBLE_TYPE = "";
+
+ /**
+ * Create an unbound Rodbinding of type. This is the correct programming
+ * style for an optional RodBinding
+ *
+ * At Input()
+ * RodBinding x = RodBinding.makeUnbound(T.class)
+ *
+ * The unbound binding is guaranteed to never match any binding. It uniquely
+ * returns false to isBound().
+ *
+ * @param type the Class type produced by this unbound object
+ * @param any class extending Tribble Feature
+ * @return the UNBOUND RodBinding producing objects of type T
+ */
+ @Requires("type != null")
+ protected final static RodBinding makeUnbound(Class type) {
+ return new RodBinding(type);
+ }
+
+ /** The name of this binding. Often the name of the field itself, but can be overridden on cmdline */
+ final private String name;
+ /** where the data for this ROD is coming from. A file or special value if coming from stdin */
+ final private String source;
+ /** the string name of the tribble type, such as vcf, bed, etc. */
+ final private String tribbleType;
+ /** The command line tags associated with this RodBinding */
+ final private Tags tags;
+ /** The Java class expected for this RodBinding. Must correspond to the type emited by Tribble */
+ final private Class type;
+ /** True for all RodBindings except the special UNBOUND binding, which is the default for optional arguments */
+ final private boolean bound;
+
+ /**
+ * The name counter. This is how we create unique names for collections of RodBindings
+ * on the command line. If you have provide the GATK with -X file1 and -X file2 to a
+ * RodBinding argument as List> then each binding will receive automatically
+ * the name of X and X2.
+ */
+ final private static Map nameCounter = new HashMap();
+
+ /** for UnitTests */
+ final public static void resetNameCounter() {
+ nameCounter.clear();
+ }
+
+ @Requires("rawName != null")
+ @Ensures("result != null")
+ final private static synchronized String countedVariableName(final String rawName) {
+ Integer count = nameCounter.get(rawName);
+ if ( count == null ) {
+ nameCounter.put(rawName, 1);
+ return rawName;
+ } else {
+ nameCounter.put(rawName, count + 1);
+ return rawName + (count + 1);
+ }
+ }
+
+ @Requires({"type != null", "rawName != null", "source != null", "tribbleType != null", "tags != null"})
+ public RodBinding(Class type, final String rawName, final String source, final String tribbleType, final Tags tags) {
+ this.type = type;
+ this.name = countedVariableName(rawName);
+ this.source = source;
+ this.tribbleType = tribbleType;
+ this.tags = tags;
+ this.bound = true;
+ }
+
+ /**
+ * Make an unbound RodBinding. Only available for creating the globally unique UNBOUND object
+ * @param type class this unbound RodBinding creates
+ */
+ @Requires({"type != null"})
+ private RodBinding(Class type) {
+ this.type = type;
+ this.name = UNBOUND_VARIABLE_NAME; // special value can never be found in RefMetaDataTracker
+ this.source = UNBOUND_SOURCE;
+ this.tribbleType = UNBOUND_TRIBBLE_TYPE;
+ this.tags = new Tags();
+ this.bound = false;
+ }
+
+
+ /**
+ * @return True for all RodBindings except the special UNBOUND binding, which is the default for optional arguments
+ */
+ final public boolean isBound() {
+ return bound;
+ }
+
+ /**
+ * @return The name of this binding. Often the name of the field itself, but can be overridden on cmdline
+ */
+ @Ensures({"result != null"})
+ final public String getName() {
+ return name;
+ }
+
+ /**
+ * @return the string name of the tribble type, such as vcf, bed, etc.
+ */
+ @Ensures({"result != null"})
+ final public Class getType() {
+ return type;
+ }
+
+ /**
+ * @return where the data for this ROD is coming from. A file or special value if coming from stdin
+ */
+ @Ensures({"result != null"})
+ final public String getSource() {
+ return source;
+ }
+
+ /**
+ * @return The command line tags associated with this RodBinding. Will include the tags used to
+ * determine the name and type of this RodBinding
+ */
+ @Ensures({"result != null"})
+ final public Tags getTags() {
+ return tags;
+ }
+
+ /**
+ * @return The Java class expected for this RodBinding. Must correspond to the type emited by Tribble
+ */
+ @Ensures({"result != null"})
+ final public String getTribbleType() {
+ return tribbleType;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("(RodBinding name=%s source=%s)", getName(), getSource());
+ }
+}
diff --git a/public/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java b/public/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java
index a080ab439..32002e093 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java
@@ -25,21 +25,20 @@
package org.broadinstitute.sting.gatk;
-import org.broadinstitute.sting.commandline.ArgumentTypeDescriptor;
-import org.broadinstitute.sting.commandline.CommandLineProgram;
+import org.apache.log4j.Logger;
+import org.broadinstitute.sting.commandline.*;
import org.broadinstitute.sting.gatk.arguments.GATKArgumentCollection;
import org.broadinstitute.sting.gatk.filters.ReadFilter;
import org.broadinstitute.sting.gatk.io.stubs.OutputStreamArgumentTypeDescriptor;
-import org.broadinstitute.sting.gatk.io.stubs.SAMFileReaderArgumentTypeDescriptor;
import org.broadinstitute.sting.gatk.io.stubs.SAMFileWriterArgumentTypeDescriptor;
import org.broadinstitute.sting.gatk.io.stubs.VCFWriterArgumentTypeDescriptor;
import org.broadinstitute.sting.gatk.phonehome.GATKRunReport;
+import org.broadinstitute.sting.gatk.refdata.utils.RMDTriplet;
import org.broadinstitute.sting.gatk.walkers.Walker;
+import org.broadinstitute.sting.utils.classloader.JVMUtils;
import org.broadinstitute.sting.utils.text.ListFileUtils;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
+import java.util.*;
/**
* @author aaron
@@ -64,6 +63,8 @@ public abstract class CommandLineExecutable extends CommandLineProgram {
*/
private final Collection