zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' of ssh://gsa1/humgen/gsa-scr1/gsa-engineering/git/unstable

This commit is contained in:
Mark DePristo 2011-12-07 09:03:17 -05:00
parent 6bf18899df 5869a87e48
commit 5d2212bc8e
62 changed files with 775 additions and 993 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java
View File

@ -280,8 +280,24 @@ public class GenomeAnalysisEngine {
*/
private void determineThreadAllocation() {
Tags tags = parsingEngine.getTags(argCollection.numberOfThreads);
Integer numCPUThreads = tags.containsKey("cpu") ? Integer.parseInt(tags.getValue("cpu")) : null;
Integer numIOThreads = tags.containsKey("io") ? Integer.parseInt(tags.getValue("io")) : null;
// TODO: Kill this complicated logic once Queue supports arbitrary tagged parameters.
Integer numCPUThreads = null;
if(tags.containsKey("cpu") && argCollection.numberOfCPUThreads != null)
throw new UserException("Number of CPU threads specified both directly on the command-line and as a tag to the nt argument. Please specify only one or the other.");
else if(tags.containsKey("cpu"))
numCPUThreads = Integer.parseInt(tags.getValue("cpu"));
else if(argCollection.numberOfCPUThreads != null)
numCPUThreads = argCollection.numberOfCPUThreads;
Integer numIOThreads = null;
if(tags.containsKey("io") && argCollection.numberOfIOThreads != null)
throw new UserException("Number of IO threads specified both directly on the command-line and as a tag to the nt argument. Please specify only one or the other.");
else if(tags.containsKey("io"))
numIOThreads = Integer.parseInt(tags.getValue("io"));
else if(argCollection.numberOfIOThreads != null)
numIOThreads = argCollection.numberOfIOThreads;
this.threadAllocation = new ThreadAllocation(argCollection.numberOfThreads,numCPUThreads,numIOThreads);
}

12
public/java/src/org/broadinstitute/sting/gatk/ReadProperties.java
View File

@ -30,7 +30,6 @@ public class ReadProperties {
private Collection<SAMReaderID> readers = null;
private SAMFileHeader header = null;
private SAMFileReader.ValidationStringency validationStringency = SAMFileReader.ValidationStringency.STRICT;
private Integer readBufferSize = null;
private DownsamplingMethod downsamplingMethod = null;
private ValidationExclusion exclusionList = null;
private Collection<ReadFilter> supplementalFilters = null;
@ -91,14 +90,6 @@ public class ReadProperties {
return validationStringency;
}
/**
* Gets a list of the total number of reads that the sharding system should buffer per BAM file.
* @return
*/
public Integer getReadBufferSize() {
return readBufferSize;
}
/**
* Gets the method and parameters used when downsampling reads.
* @return Downsample fraction.
@ -150,7 +141,6 @@ public class ReadProperties {
* @param header sam file header.
* @param useOriginalBaseQualities True if original base qualities should be used.
* @param strictness Stringency of reads file parsing.
* @param readBufferSize Number of reads to hold in memory per BAM.
* @param downsamplingMethod Method for downsampling reads at a given locus.
* @param exclusionList what safety checks we're willing to let slide
* @param supplementalFilters additional filters to dynamically apply.
@ -169,7 +159,6 @@ public class ReadProperties {
SAMFileHeader header,
boolean useOriginalBaseQualities,
SAMFileReader.ValidationStringency strictness,
Integer readBufferSize,
DownsamplingMethod downsamplingMethod,
ValidationExclusion exclusionList,
Collection<ReadFilter> supplementalFilters,
@ -181,7 +170,6 @@ public class ReadProperties {
byte defaultBaseQualities) {
this.readers = samFiles;
this.header = header;
this.readBufferSize = readBufferSize;
this.validationStringency = strictness;
this.downsamplingMethod = downsamplingMethod == null ? DownsamplingMethod.NONE : downsamplingMethod;
this.exclusionList = exclusionList == null ? new ValidationExclusion() : exclusionList;

19
public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java
View File

@ -198,6 +198,17 @@ public class GATKArgumentCollection {
@Argument(fullName = "num_threads", shortName = "nt", doc = "How many threads should be allocated to running this analysis.", required = false)
public Integer numberOfThreads = 1;
/**
* The following two arguments (num_cpu_threads, num_io_threads are TEMPORARY since Queue cannot currently support arbitrary tagged data types.
* TODO: Kill this when I can do a tagged integer in Queue.
*/
@Argument(fullName="num_cpu_threads", shortName = "nct", doc="How many of the given threads should be allocated to the CPU", required = false)
@Hidden
public Integer numberOfCPUThreads = null;
@Argument(fullName="num_io_threads", shortName = "nit", doc="How many of the given threads should be allocated to IO", required = false)
@Hidden
public Integer numberOfIOThreads = null;
@Argument(fullName = "num_bam_file_handles", shortName = "bfh", doc="The total number of BAM file handles to keep open simultaneously", required=false)
public Integer numberOfBAMFileHandles = null;
@ -369,6 +380,14 @@ public class GATKArgumentCollection {
if (!other.numberOfThreads.equals(this.numberOfThreads)) {
return false;
}
if ((this.numberOfCPUThreads == null && other.numberOfCPUThreads != null) ||
this.numberOfCPUThreads.equals(other.numberOfCPUThreads) ) {
return false;
}
if ((this.numberOfIOThreads == null && other.numberOfIOThreads != null) ||
this.numberOfIOThreads.equals(other.numberOfIOThreads) ) {
return false;
}
if ((other.numberOfBAMFileHandles == null && this.numberOfBAMFileHandles != null) ||
(other.numberOfBAMFileHandles != null && !other.numberOfBAMFileHandles.equals(this.numberOfBAMFileHandles))) {
return false;

11
public/java/src/org/broadinstitute/sting/gatk/datasources/reads/ReadShard.java
View File

@ -38,7 +38,7 @@ public class ReadShard extends Shard {
/**
* What is the maximum number of reads which should go into a read shard.
*/
public static final int MAX_READS = 10000;
public static int MAX_READS = 10000;
/**
* The reads making up this shard.
@ -49,6 +49,15 @@ public class ReadShard extends Shard {
super(parser, ShardType.READ, loci, readsDataSource, fileSpans, isUnmapped);
}
/**
* Sets the maximum number of reads buffered in a read shard. Implemented as a weirdly static interface
* until we know what effect tuning this parameter has.
* @param bufferSize New maximum number
*/
static void setReadBufferSize(final int bufferSize) {
MAX_READS = bufferSize;
}
/**
* Returns true if this shard is meant to buffer reads, rather
* than just holding pointers to their locations.

18
public/java/src/org/broadinstitute/sting/gatk/datasources/reads/ReadShardBalancer.java
View File

@ -89,11 +89,10 @@ public class ReadShardBalancer extends ShardBalancer {
for(SAMReaderID id: shardPosition.keySet()) {
SAMFileSpan fileSpan = new GATKBAMFileSpan(shardPosition.get(id).removeContentsBefore(position.get(id)));
if(!fileSpan.isEmpty())
selectedReaders.put(id,fileSpan);
selectedReaders.put(id,fileSpan);
}
if(selectedReaders.size() > 0) {
if(!isEmpty(selectedReaders)) {
Shard shard = new ReadShard(parser,readsDataSource,selectedReaders,currentFilePointer.locations,currentFilePointer.isRegionUnmapped);
readsDataSource.fillShard(shard);
@ -109,6 +108,19 @@ public class ReadShardBalancer extends ShardBalancer {
position = readsDataSource.getCurrentPosition();
}
/**
* Detects whether the list of file spans contain any read data.
* @param selectedSpans Mapping of readers to file spans.
* @return True if file spans are completely empty; false otherwise.
*/
private boolean isEmpty(Map<SAMReaderID,SAMFileSpan> selectedSpans) {
for(SAMFileSpan fileSpan: selectedSpans.values()) {
if(!fileSpan.isEmpty())
return false;
}
return true;
}
};
}

10
public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java
View File

@ -242,12 +242,17 @@ public class SAMDataSource {
this.threadAllocation = threadAllocation;
// TODO: Consider a borrowed-thread dispatcher implementation.
if(this.threadAllocation.getNumIOThreads() > 0)
if(this.threadAllocation.getNumIOThreads() > 0) {
logger.info("Running in asynchronous I/O mode; number of threads = " + this.threadAllocation.getNumIOThreads());
dispatcher = new BGZFBlockLoadingDispatcher(this.threadAllocation.getNumIOThreads(), numFileHandles != null ? numFileHandles : 1);
}
else
dispatcher = null;
validationStringency = strictness;
if(readBufferSize != null)
ReadShard.setReadBufferSize(readBufferSize);
for (SAMReaderID readerID : samFiles) {
if (!readerID.samFile.canRead())
throw new UserException.CouldNotReadInputFile(readerID.samFile,"file is not present or user does not have appropriate permissions. " +
@ -291,7 +296,6 @@ public class SAMDataSource {
mergedHeader,
useOriginalBaseQualities,
strictness,
readBufferSize,
downsamplingMethod,
exclusionList,
supplementalFilters,
@ -549,8 +553,6 @@ public class SAMDataSource {
inputStream.submitAccessPlan(new SAMReaderPosition(id,inputStream,(GATKBAMFileSpan)shard.getFileSpans().get(id)));
}
iterator = readers.getReader(id).iterator(shard.getFileSpans().get(id));
if(readProperties.getReadBufferSize() != null)
iterator = new BufferingReadIterator(iterator,readProperties.getReadBufferSize());
if(shard.getGenomeLocs().size() > 0)
iterator = new IntervalOverlapFilteringIterator(iterator,shard.getGenomeLocs());
mergingIterator.addIterator(readers.getReader(id),iterator);

11
public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMReaderID.java
View File

@ -67,6 +67,7 @@ public class SAMReaderID implements Comparable {
* @param other The other identifier.
* @return True iff the two readers point to the same file.
*/
@Override
public boolean equals(Object other) {
if(other == null) return false;
if(!(other instanceof SAMReaderID)) return false;
@ -79,10 +80,20 @@ public class SAMReaderID implements Comparable {
* Generate a hash code for this object.
* @return A hash code, based solely on the file name at this point.
*/
@Override
public int hashCode() {
return samFile.hashCode();
}
/**
* Best string representation for a SAM file reader is the path of the source file.
*/
@Override
public String toString() {
return getSamFilePath();
}
@Override
public int compareTo(Object other) {
return this.samFile.getAbsolutePath().compareTo(((SAMReaderID)other).samFile.getAbsolutePath());
}

80
public/java/src/org/broadinstitute/sting/gatk/iterators/BufferingReadIterator.java
View File

@ -1,80 +0,0 @@
/*
* Copyright (c) 2010, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.iterators;
import net.sf.samtools.SAMRecord;
import net.sf.samtools.util.CloseableIterator;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import java.util.LinkedList;
import java.util.NoSuchElementException;
import java.util.Queue;
/**
* Buffers access to a large stream of reads, replenishing the buffer only when the reads
*
* @author mhanna
* @version 0.1
*/
public class BufferingReadIterator implements CloseableIterator<SAMRecord> {
private final CloseableIterator<SAMRecord> wrappedIterator;
private final Queue<SAMRecord> buffer;
private final int bufferSize;
public BufferingReadIterator(final CloseableIterator<SAMRecord> readIterator, final int bufferSize) {
this.wrappedIterator = readIterator;
this.buffer = new LinkedList<SAMRecord>();
this.bufferSize = bufferSize;
}
public boolean hasNext() {
assureBufferFull();
return !buffer.isEmpty();
}
public SAMRecord next() {
assureBufferFull();
if(!hasNext()) throw new NoSuchElementException("No next element available");
return buffer.remove();
}
public void close() {
wrappedIterator.close();
}
public void remove() {
throw new ReviewedStingException("Unable to remove from a BufferingReadIterator");
}
/**
* If the buffer is empty but there are more elements in the iterator,
*/
private void assureBufferFull() {
if(!buffer.isEmpty())
return;
while(buffer.size() < bufferSize && wrappedIterator.hasNext())
buffer.add(wrappedIterator.next());
}
}

44
public/java/src/org/broadinstitute/sting/gatk/report/GATKReportColumn.java
View File

@ -1,6 +1,8 @@
package org.broadinstitute.sting.gatk.report;
import java.util.TreeMap;
import org.apache.commons.lang.math.NumberUtils;
import java.util.*;
/**
* Holds values for a column in a GATK report table
@ -16,12 +18,9 @@ public class GATKReportColumn extends TreeMap<Object, Object> {
*
* @param columnName the name of the column
* @param defaultValue the default value of the column
* @param display if true, the column will be displayed in the final output
* @param display if true, the column will be displayed in the final output
* @param format format string
*/
public GATKReportColumn(String columnName, Object defaultValue, boolean display) {
this(columnName, defaultValue, display, null);
}
public GATKReportColumn(String columnName, Object defaultValue, boolean display, String format) {
this.columnName = columnName;
this.defaultValue = defaultValue;
@ -77,22 +76,47 @@ public class GATKReportColumn extends TreeMap<Object, Object> {
/**
* Get the display width for this column. This allows the entire column to be displayed with the appropriate, fixed width.
* @return the width of this column
* @return the format string for this column
*/
public int getColumnWidth() {
public GATKReportColumnFormat getColumnFormat() {
int maxWidth = columnName.length();
GATKReportColumnFormat.Alignment alignment = GATKReportColumnFormat.Alignment.RIGHT;
for (Object obj : this.values()) {
if (obj != null) {
int width = formatValue(obj).length();
String formatted = formatValue(obj);
int width = formatted.length();
if (width > maxWidth) {
maxWidth = width;
}
if (alignment == GATKReportColumnFormat.Alignment.RIGHT) {
if (!isRightAlign(formatted)) {
alignment = GATKReportColumnFormat.Alignment.LEFT;
}
}
}
}
return maxWidth;
return new GATKReportColumnFormat(maxWidth, alignment);
}
private static final Collection<String> RIGHT_ALIGN_STRINGS = Arrays.asList(
"null",
"NA",
String.valueOf(Double.POSITIVE_INFINITY),
String.valueOf(Double.NEGATIVE_INFINITY),
String.valueOf(Double.NaN));
/**
* Check if the value can be right aligned. Does not trim the values before checking if numeric since it assumes
* the spaces mean that the value is already padded.
* @param value to check
* @return true if the value is a right alignable
*/
protected static boolean isRightAlign(String value) {
return value == null || RIGHT_ALIGN_STRINGS.contains(value) || NumberUtils.isNumber(value);
}
/**

50
public/java/src/org/broadinstitute/sting/pipeline/PipelineSample.java → public/java/src/org/broadinstitute/sting/gatk/report/GATKReportColumnFormat.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2010, The Broad Institute
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -22,41 +22,41 @@
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.pipeline;
import java.io.File;
import java.util.Map;
import java.util.TreeMap;
package org.broadinstitute.sting.gatk.report;
/**
* Java bean defining a sample for a pipeline.
* Column width and left/right alignment.
*/
public class PipelineSample {
private String id;
private Map<String, File> bamFiles = new TreeMap<String, File>();
private Map<String, String> tags = new TreeMap<String, String>();
public class GATKReportColumnFormat {
public static enum Alignment { LEFT, RIGHT }
public int width;
public Alignment alignment;
public String getId() {
return id;
public GATKReportColumnFormat(int width, Alignment alignment) {
this.width = width;
this.alignment = alignment;
}
public void setId(String id) {
this.id = id;
public int getWidth() {
return width;
}
public Map<String, File> getBamFiles() {
return bamFiles;
public Alignment getAlignment() {
return alignment;
}
public void setBamFiles(Map<String, File> bamFiles) {
this.bamFiles = bamFiles;
public String getNameFormat() {
return "%-" + width + "s";
}
public Map<String, String> getTags() {
return tags;
}
public void setTags(Map<String, String> tags) {
this.tags = tags;
public String getValueFormat() {
switch (alignment) {
case LEFT:
return "%-" + width + "s";
case RIGHT:
return "%" + width + "s";
default:
throw new UnsupportedOperationException("Unknown alignment: " + alignment);
}
}
}

11
public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java
View File

@ -608,12 +608,9 @@ public class GATKReportTable {
*/
public void write(PrintStream out) {
// Get the column widths for everything
HashMap<String, String> columnWidths = new HashMap<String, String>();
HashMap<String, GATKReportColumnFormat> columnFormats = new HashMap<String, GATKReportColumnFormat>();
for (String columnName : columns.keySet()) {
int width = columns.get(columnName).getColumnWidth();
String format = "%-" + String.valueOf(width) + "s";
columnWidths.put(columnName, format);
columnFormats.put(columnName, columns.get(columnName).getColumnFormat());
}
String primaryKeyFormat = "%-" + getPrimaryKeyColumnWidth() + "s";
@ -630,7 +627,7 @@ public class GATKReportTable {
for (String columnName : columns.keySet()) {
if (columns.get(columnName).isDisplayable()) {
if (needsPadding) { out.printf(" "); }
out.printf(columnWidths.get(columnName), columnName);
out.printf(columnFormats.get(columnName).getNameFormat(), columnName);
needsPadding = true;
}
@ -650,7 +647,7 @@ public class GATKReportTable {
if (columns.get(columnName).isDisplayable()) {
if (needsPadding) { out.printf(" "); }
String value = columns.get(columnName).getStringValue(primaryKey);
out.printf(columnWidths.get(columnName), value);
out.printf(columnFormats.get(columnName).getValueFormat(), value);
needsPadding = true;
}

4
public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/GATKReportDiffableReader.java
View File

@ -31,7 +31,6 @@ import org.broadinstitute.sting.gatk.report.GATKReportTable;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.Map;
/**
@ -68,7 +67,8 @@ public class GATKReportDiffableReader implements DiffableReader {
for ( GATKReportColumn column : table.getColumns().values() ) {
DiffNode columnRoot = DiffNode.empty(column.getColumnName(), tableRoot);
columnRoot.add("Width", column.getColumnWidth());
columnRoot.add("Width", column.getColumnFormat().getWidth());
// NOTE: as the values are trimmed during parsing left/right alignment is not currently preserved
columnRoot.add("Displayable", column.isDisplayable());
int n = 1;

62
public/java/src/org/broadinstitute/sting/pipeline/Pipeline.java
View File

@ -1,62 +0,0 @@
/*
* Copyright (c) 2010, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.pipeline;
import java.util.ArrayList;
import java.util.List;
/**
* Java bean for storing a list of samples for a pipeline.
*
* NOTE: This class is used in a very similar way to the classes in
* org.broadinstitute.sting.gatk.datasources.sample.
*
* Both store / load sample information from the file system as YAML.
*
* This package will likely be refactored to share common functionality
* with the other at a future date as requirements coalesce.
*
* - kshakir September 22, 2010
*/
public class Pipeline {
private PipelineProject project = new PipelineProject();
private List<PipelineSample> samples = new ArrayList<PipelineSample>();
public PipelineProject getProject() {
return project;
}
public void setProject(PipelineProject project) {
this.project = project;
}
public List<PipelineSample> getSamples() {
return samples;
}
public void setSamples(List<PipelineSample> samples) {
this.samples = samples;
}
}

115
public/java/src/org/broadinstitute/sting/pipeline/PipelineProject.java
View File

@ -1,115 +0,0 @@
/*
* Copyright (c) 2010, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.pipeline;
import java.io.File;
import java.util.Map;
import java.util.TreeMap;
/**
* Java bean defining the project for a pipeline.
*/
public class PipelineProject {
private String name;
private File referenceFile;
private File intervalList;
private File genotypeDbsnp;
private File evalDbsnp;
private File refseqTable;
private Map<String, String> tags = new TreeMap<String, String>();
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public File getIntervalList() {
return intervalList;
}
public void setIntervalList(File intervalList) {
this.intervalList = intervalList;
}
public File getReferenceFile() {
return referenceFile;
}
public void setReferenceFile(File referenceFile) {
this.referenceFile = referenceFile;
}
public File getGenotypeDbsnp() {
return genotypeDbsnp;
}
public void setGenotypeDbsnp(File genotypeDbsnp) {
this.genotypeDbsnp = genotypeDbsnp;
}
public String getGenotypeDbsnpType() {
return getDbsnpType(genotypeDbsnp);
}
public File getEvalDbsnp() {
return evalDbsnp;
}
public void setEvalDbsnp(File evalDbsnp) {
this.evalDbsnp = evalDbsnp;
}
public String getEvalDbsnpType() {
return getDbsnpType(evalDbsnp);
}
public File getRefseqTable() {
return refseqTable;
}
public void setRefseqTable(File refseqTable) {
this.refseqTable = refseqTable;
}
public Map<String, String> getTags() {
return tags;
}
public void setTags(Map<String, String> tags) {
this.tags = tags;
}
private String getDbsnpType(File file) {
if (file == null)
return null;
else if (file.getName().toLowerCase().endsWith(".vcf"))
return "vcf";
else
return "dbsnp";
}
}

20
public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java
View File

@ -83,10 +83,10 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
getShortFieldSetter());
}
protected static final String REQUIRED_TEMPLATE = " + \" %1$s \" + %2$s.format(%3$s)";
protected static final String REPEAT_TEMPLATE = " + repeat(\" %1$s \", %3$s, format=formatValue(%2$s))";
protected static final String OPTIONAL_TEMPLATE = " + optional(\" %1$s \", %3$s, format=formatValue(%2$s))";
protected static final String FLAG_TEMPLATE = " + (if (%3$s) \" %1$s\" else \"\")";
protected static final String REQUIRED_TEMPLATE = " + required(\"%1$s\", %3$s, spaceSeparated=true, escape=true, format=%2$s)";
protected static final String REPEAT_TEMPLATE = " + repeat(\"%1$s\", %3$s, spaceSeparated=true, escape=true, format=%2$s)";
protected static final String OPTIONAL_TEMPLATE = " + optional(\"%1$s\", %3$s, spaceSeparated=true, escape=true, format=%2$s)";
protected static final String FLAG_TEMPLATE = " + conditional(%3$s, \"%1$s\", escape=true, format=%2$s)";
public final String getCommandLineAddition() {
return String.format(getCommandLineTemplate(), getCommandLineParam(), getCommandLineFormat(), getFieldName());
@ -136,7 +136,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
new IntervalStringArgumentField(argumentDefinition));
// ROD Bindings are set by the RodBindField
} else if (RodBindField.ROD_BIND_FIELD.equals(argumentDefinition.fullName) && argumentDefinition.ioType == ArgumentIOType.INPUT) {
} else if (RodBindArgumentField.ROD_BIND_FIELD.equals(argumentDefinition.fullName) && argumentDefinition.ioType == ArgumentIOType.INPUT) {
// TODO: Once everyone is using @Allows and @Requires correctly, we can stop blindly allowing Triplets
return Arrays.asList(new RodBindArgumentField(argumentDefinition), new InputIndexesArgumentField(argumentDefinition, Tribble.STANDARD_INDEX_EXTENSION));
//return Collections.<ArgumentField>emptyList();
@ -337,6 +337,8 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
// Allows the user to specify the track name, track type, and the file.
public static class RodBindArgumentField extends ArgumentDefinitionField {
public static final String ROD_BIND_FIELD = "rodBind";
public RodBindArgumentField(ArgumentDefinition argumentDefinition) {
super(argumentDefinition);
}
@ -344,7 +346,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
@Override protected String getFieldType() { return "List[RodBind]"; }
@Override protected String getDefaultValue() { return "Nil"; }
@Override protected String getCommandLineTemplate() {
return " + repeat(\"\", %3$s, format=RodBind.formatCommandLine(\"%1$s\"))";
return " + repeat(\"%1$s\", %3$s, formatPrefix=RodBind.formatCommandLineParameter, spaceSeparated=true, escape=true, format=%2$s)";
}
}
@ -358,11 +360,11 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
@Override protected String getDefaultValue() { return argumentDefinition.isMultiValued ? "Nil" : "_"; }
@Override protected String getCommandLineTemplate() {
if (argumentDefinition.isMultiValued) {
return " + repeat(\"\", %3$s, format=TaggedFile.formatCommandLine(\"%1$s\"))";
return " + repeat(\"%1$s\", %3$s, formatPrefix=TaggedFile.formatCommandLineParameter, spaceSeparated=true, escape=true, format=%2$s)";
} else if (!argumentDefinition.required) {
return " + optional(\"\", %3$s, format=TaggedFile.formatCommandLine(\"%1$s\"))";
return " + optional(TaggedFile.formatCommandLineParameter(\"%1$s\", %3$s), %3$s, spaceSeparated=true, escape=true, format=%2$s)";
} else {
return " + TaggedFile.formatCommandLine(\"%1$s\")(\"\", %3$s, \"\")";
return " + required(TaggedFile.formatCommandLineParameter(\"%1$s\", %3$s), %3$s, spaceSeparated=true, escape=true, format=%2$s)";
}
}
}

3
public/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java
View File

@ -374,6 +374,9 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
if (isGather)
importSet.add("import org.broadinstitute.sting.commandline.Gather");
// Needed for ShellUtils.escapeShellArgument()
importSet.add("import org.broadinstitute.sting.queue.util.ShellUtils");
// Sort the imports so that the are always in the same order.
List<String> sortedImports = new ArrayList<String>(importSet);
Collections.sort(sortedImports);

129
public/java/src/org/broadinstitute/sting/queue/extensions/gatk/RodBindField.java
View File

@ -1,129 +0,0 @@
/*
* Copyright (c) 2010, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.extensions.gatk;
import org.broadinstitute.sting.commandline.Input;
import org.broadinstitute.sting.gatk.WalkerManager;
import org.broadinstitute.sting.gatk.refdata.tracks.RMDTrackBuilder;
import org.broadinstitute.sting.gatk.walkers.RMD;
import org.broadinstitute.sting.gatk.walkers.Walker;
import java.io.File;
import java.lang.annotation.Annotation;
import java.util.ArrayList;
import java.util.List;
/**
* Allows user to specify the rod file but locks in the track name and the track type.
*/
public class RodBindField extends ArgumentField {
public static final String ROD_BIND_FIELD = "rodBind";
private final String trackName;
private final String typeName;
private final List<RodBindField> relatedFields;
private final boolean isRequired;
public RodBindField(String trackName, String typeName, List<RodBindField> relatedFields, boolean isRequired) {
this.trackName = trackName;
this.typeName = typeName;
this.relatedFields = relatedFields;
this.isRequired = isRequired;
}
@SuppressWarnings("unchecked")
@Override protected Class<? extends Annotation> getAnnotationIOClass() { return Input.class; }
@Override protected Class<?> getInnerType() { return File.class; }
@Override protected String getFullName() { return escape(getRawFieldName()); }
@Override protected String getFieldType() { return "File"; }
@Override protected String getDefaultValue() { return "_"; }
@Override protected String getRawFieldName() { return this.trackName + this.typeName; }
@Override protected String getDoc() { return escape(this.typeName + " " + this.trackName); }
@Override protected boolean isRequired() { return this.isRequired; }
@Override public String getCommandLineAddition() {
// TODO: Stop allowing the generic "rodBind" triplets to satisfy the requirement after @Requires are fixed.
return String.format(" + optional(\" -B:%s,%s \", %s)",
/*
return String.format(this.useOption()
? " + optional(\" -B:%s,%s \", %s)"
: " + \" -B:%s,%s \" + %s",
*/
this.trackName, this.typeName, getFieldName());
}
private boolean useOption() {
return !this.isRequired || (relatedFields.size() > 1);
}
@Override protected String getExclusiveOf() {
StringBuilder exclusiveOf = new StringBuilder();
// TODO: Stop allowing the generic "rodBind" triplets to satisfy the requirement after @Requires are fixed.
if (this.isRequired)
exclusiveOf.append(ROD_BIND_FIELD);
for (RodBindField relatedField: relatedFields)
if (relatedField != this) {
if (exclusiveOf.length() > 0)
exclusiveOf.append(",");
exclusiveOf.append(relatedField.getFieldName());
}
return exclusiveOf.toString();
}
//
// public static List<ArgumentField> getRodArguments(Class<? extends Walker> walkerClass, RMDTrackBuilder trackBuilder) {
// List<ArgumentField> argumentFields = new ArrayList<ArgumentField>();
//
// List<RMD> requires = WalkerManager.getRequiredMetaData(walkerClass);
// List<RMD> allows = WalkerManager.getAllowsMetaData(walkerClass);
//
// for (RMD required: requires) {
// List<RodBindField> fields = new ArrayList<RodBindField>();
// String trackName = required.name();
// if ("*".equals(trackName)) {
// // TODO: Add the field triplet for name=* after @Allows and @Requires are fixed on walkers
// //fields.add(new RodBindArgumentField(argumentDefinition, true));
// } else {
// for (String typeName: trackBuilder.getFeatureManager().getTrackRecordTypeNames(required.type()))
// fields.add(new RodBindField(trackName, typeName, fields, true));
// }
// argumentFields.addAll(fields);
// }
//
// for (RMD allowed: allows) {
// List<RodBindField> fields = new ArrayList<RodBindField>();
// String trackName = allowed.name();
// if ("*".equals(trackName)) {
// // TODO: Add the field triplet for name=* after @Allows and @Requires are fixed on walkers
// //fields.add(new RodBindArgumentField(argumentDefinition, false));
// } else {
// for (String typeName: trackBuilder.getFeatureManager().getTrackRecordTypeNames(allowed.type()))
// fields.add(new RodBindField(trackName, typeName, fields, true));
// }
// argumentFields.addAll(fields);
// }
//
// return argumentFields;
// }
}

23
public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java
View File

@ -362,4 +362,27 @@ public class IOUtils {
org.apache.commons.io.IOUtils.closeQuietly(outputStream);
}
}
/**
* Returns a file throwing a UserException if the file cannot be read.
* @param path File path
* @return LineIterator
*/
public static LineIterator lineIterator(String path) {
return lineIterator(new File(path));
}
/**
* Returns a file throwing a UserException if the file cannot be read.
* @param file File
* @return LineIterator
*/
public static LineIterator lineIterator(File file) {
try {
return FileUtils.lineIterator(file);
} catch (IOException e) {
throw new UserException.CouldNotReadInputFile(file, e);
}
}
}

52
public/java/src/org/broadinstitute/sting/utils/yaml/FieldOrderComparator.java
View File

@ -1,52 +0,0 @@
/*
* Copyright (c) 2010, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.utils.yaml;
import org.yaml.snakeyaml.introspector.Property;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
/**
* Orders properties based on the order of the fields in the Java Bean.
*/
class FieldOrderComparator implements Comparator<Property> {
private final List<String> propertyOrder;
public FieldOrderComparator(Class<?> clazz) {
propertyOrder = new ArrayList<String>();
for (Field field : clazz.getDeclaredFields())
propertyOrder.add(field.getName());
}
@Override
public int compare(Property one, Property two) {
Integer index1 = propertyOrder.indexOf(one.getName());
Integer index2 = propertyOrder.indexOf(two.getName());
return index1.compareTo(index2);
}
}

88
public/java/src/org/broadinstitute/sting/utils/yaml/StingYamlRepresenter.java
View File

@ -1,88 +0,0 @@
/*
* Copyright (c) 2010, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.utils.yaml;
import org.yaml.snakeyaml.introspector.Property;
import org.yaml.snakeyaml.nodes.*;
import org.yaml.snakeyaml.representer.Represent;
import org.yaml.snakeyaml.representer.Representer;
import java.beans.IntrospectionException;
import java.io.File;
import java.util.Set;
import java.util.TreeSet;
/**
* A representer with Sting prefered settings.
* - Fields are ordered in the order of the class declaration, instead of alphabetically.
* - Empty maps and sequences are not output.
* - Files are converted to their absolute paths.
*/
public class StingYamlRepresenter extends Representer {
public StingYamlRepresenter() {
super();
this.representers.put(File.class, new RepresentFile());
}
@Override
protected Set<Property> getProperties(Class<?> type) throws IntrospectionException {
TreeSet<Property> properties = new TreeSet<Property>(new FieldOrderComparator(type));
properties.addAll(super.getProperties(type));
return properties;
}
@Override
protected NodeTuple representJavaBeanProperty(Object javaBean, Property property,
Object propertyValue, Tag customTag) {
NodeTuple tuple = super.representJavaBeanProperty(javaBean, property, propertyValue, customTag);
Node valueNode = tuple.getValueNode();
if (Tag.NULL.equals(valueNode.getTag())) {
return null;// skip 'null' values
}
if (valueNode instanceof CollectionNode) {
if (Tag.SEQ.equals(valueNode.getTag())) {
SequenceNode seq = (SequenceNode) valueNode;
if (seq.getValue().isEmpty()) {
return null;// skip empty lists
}
}
if (Tag.MAP.equals(valueNode.getTag())) {
MappingNode seq = (MappingNode) valueNode;
if (seq.getValue().isEmpty()) {
return null;// skip empty maps
}
}
}
return tuple;
}
private class RepresentFile implements Represent {
@Override
public Node representData(Object o) {
return StingYamlRepresenter.this.representScalar(Tag.STR, ((File)o).getPath());
}
}
}

107
public/java/src/org/broadinstitute/sting/utils/yaml/YamlUtils.java
View File

@ -1,107 +0,0 @@
/*
* Copyright (c) 2010, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.utils.yaml;
import org.broadinstitute.sting.utils.exceptions.UserException;
import org.yaml.snakeyaml.DumperOptions;
import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.constructor.Constructor;
import org.yaml.snakeyaml.nodes.Tag;
import org.yaml.snakeyaml.representer.Representer;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
/**
* A collection of utilities for operating on YAML.
* Uses the FLOW style of writing YAML, versus the BLOCK style.
* By default uses a representer that prunes empty lists and maps.
*/
public class YamlUtils {
private static Representer representer = new StingYamlRepresenter();
private static DumperOptions options = new DumperOptions();
static {
options.setCanonical(false);
options.setExplicitRoot(Tag.MAP);
options.setDefaultFlowStyle(DumperOptions.FlowStyle.FLOW);
options.setPrettyFlow(true);
}
/**
* Serialize an object to the file system.
* @param o Object to serialize.
* @param file Path to write the serialized YAML.
*/
public static void dump(Object o, File file) {
dump(o, file, representer);
}
/**
* Serialize an object to the file system.
* @param o Object to serialize.
* @param file Path to write the serialized YAML.
* @param representer Custom representer with rules on how to serialize YAML.
*/
public static void dump(Object o, File file, Representer representer) {
Constructor constructor = new Constructor(o.getClass());
Yaml yaml = new Yaml(constructor, representer, options);
try {
yaml.dump(o, new FileWriter(file));
} catch (IOException ioe) {
throw new UserException.CouldNotCreateOutputFile(file, ioe);
}
}
/**
* Deserialize an object from the file system.
* @param clazz Clazz to deserialize.
* @param file Path to read the deserialized YAML.
* @return Object deserialized from the file system.
*/
public static <T> T load(Class<? extends T> clazz, File file) {
return load(clazz, file, representer);
}
/**
* Deserialize an object from the file system.
* @param clazz Clazz to deserialize.
* @param file Path to read the deserialized YAML.
* @param representer Custom representer with rules on how to deserialize YAML.
* @return Object deserialized from the file system.
*/
@SuppressWarnings("unchecked")
public static <T> T load(Class<? extends T> clazz, File file, Representer representer) {
Constructor constructor = new Constructor(clazz);
Yaml yaml = new Yaml(constructor, representer, options);
try {
return (T) yaml.load(new FileReader(file));
} catch (IOException ioe) {
throw new UserException.CouldNotReadInputFile(file, ioe);
}
}
}

3
public/java/test/org/broadinstitute/sting/BaseTest.java
View File

@ -78,7 +78,7 @@ public abstract class BaseTest {
public static final String hg19Intervals = intervalsLocation + "whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list";
public static final String hg19Chr20Intervals = intervalsLocation + "whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.chr20.interval_list";
public static final String networkTempDir = "/broad/shptmp/";
public static final String networkTempDir = "/broad/shptmp/" + System.getProperty("user.name") + "/";
public static final File networkTempDirFile = new File(networkTempDir);
public static final File testDirFile = new File("public/testdata/");
@ -239,6 +239,7 @@ public abstract class BaseTest {
*/
public static File createNetworkTempFile(String name, String extension) {
try {
FileUtils.forceMkdir(networkTempDirFile);
File file = File.createTempFile(name, extension, networkTempDirFile);
file.deleteOnExit();
return file;

1
public/java/test/org/broadinstitute/sting/gatk/datasources/reads/DownsamplerBenchmark.java
View File

@ -72,7 +72,6 @@ public class DownsamplerBenchmark extends ReadProcessingBenchmark {
reader.getFileHeader(),
false,
SAMFileReader.ValidationStringency.SILENT,
0,
downsampling.create(),
new ValidationExclusion(Collections.singletonList(ValidationExclusion.TYPE.ALL)),
Collections.<ReadFilter>emptyList(),

1
public/java/test/org/broadinstitute/sting/gatk/iterators/LocusIteratorByStateUnitTest.java
View File

@ -301,7 +301,6 @@ public class LocusIteratorByStateUnitTest extends BaseTest {
false,
SAMFileReader.ValidationStringency.STRICT,
null,
null,
new ValidationExclusion(),
Collections.<ReadFilter>emptyList(),
false,

28
public/java/test/org/broadinstitute/sting/gatk/report/GATKReportUnitTest.java
View File

@ -26,6 +26,7 @@ package org.broadinstitute.sting.gatk.report;
import org.broadinstitute.sting.BaseTest;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
public class GATKReportUnitTest extends BaseTest {
@ -45,4 +46,31 @@ public class GATKReportUnitTest extends BaseTest {
Object validationReportPK = countVariants.getPrimaryKey("none.eval.none.known");
Assert.assertEquals(validationReport.get(validationReportPK, "sensitivity"), "NaN");
}
@DataProvider(name = "rightAlignValues")
public Object[][] getRightAlignValues() {
return new Object[][] {
new Object[] {null, true},
new Object[] {"null", true},
new Object[] {"NA", true},
new Object[] {"0", true},
new Object[] {"0.0", true},
new Object[] {"-0", true},
new Object[] {"-0.0", true},
new Object[] {String.valueOf(Long.MAX_VALUE), true},
new Object[] {String.valueOf(Long.MIN_VALUE), true},
new Object[] {String.valueOf(Float.MIN_NORMAL), true},
new Object[] {String.valueOf(Double.MAX_VALUE), true},
new Object[] {String.valueOf(Double.MIN_VALUE), true},
new Object[] {String.valueOf(Double.POSITIVE_INFINITY), true},
new Object[] {String.valueOf(Double.NEGATIVE_INFINITY), true},
new Object[] {String.valueOf(Double.NaN), true},
new Object[] {"hello", false}
};
}
@Test(dataProvider = "rightAlignValues")
public void testIsRightAlign(String value, boolean expected) {
Assert.assertEquals(GATKReportColumn.isRightAlign(value), expected, "right align of '" + value + "'");
}
}

4
public/java/test/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsIntegrationTest.java
View File

@ -50,8 +50,8 @@ public class DiffObjectsIntegrationTest extends WalkerTest {
@DataProvider(name = "data")
public Object[][] createData() {
new TestParams(testDir + "diffTestMaster.vcf", testDir + "diffTestTest.vcf", "ed377322c615abc7dceb97025076078d");
new TestParams(testDir + "exampleBAM.bam", testDir + "exampleBAM.simple.bam", "02e46f5d2ebb3d49570850595b3f792e");
new TestParams(testDir + "diffTestMaster.vcf", testDir + "diffTestTest.vcf", "da3dc85a0e35a9aade5520591891b4fa");
new TestParams(testDir + "exampleBAM.bam", testDir + "exampleBAM.simple.bam", "7dc8200730313e6753237a696296fb73");
return TestParams.getTests(TestParams.class);
}

44
public/java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalIntegrationTest.java
View File

@ -30,7 +30,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
"-o %s"
),
1,
Arrays.asList("abe943d1aac120d7e75b9b9e5dac2399")
Arrays.asList("f909fd8374f663e983b9b3fda4cf5cf1")
);
executeTest("testFunctionClassWithSnpeff", spec);
}
@ -50,7 +50,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
"-o %s"
),
1,
Arrays.asList("5fd9624c7a35ffb79d0feb1e233fc757")
Arrays.asList("081fcaa532c7ba8f23da739389e6f7c3")
);
executeTest("testStratifySamplesAndExcludeMonomorphicSites", spec);
}
@ -70,7 +70,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
"-o %s"
),
1,
Arrays.asList("4a8765cd02d36e63f6d0f0c10a6c674b")
Arrays.asList("b3852f84d07c270b8a12874083c3e31b")
);
executeTest("testFundamentalsCountVariantsSNPsandIndels", spec);
}
@ -91,7 +91,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
"-o %s"
),
1,
Arrays.asList("4106ab8f742ad1c3138c29220151503c")
Arrays.asList("cf70468b5ebaec408419da69b0a7fcb9")
);
executeTest("testFundamentalsCountVariantsSNPsandIndelsWithNovelty", spec);
}
@ -113,7 +113,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
"-o %s"
),
1,
Arrays.asList("6cee3a8d68307a118944f2df5401ac89")
Arrays.asList("5e3b8b85acfc41365c8208c23abf746b")
);
executeTest("testFundamentalsCountVariantsSNPsandIndelsWithNoveltyAndFilter", spec);
}
@ -134,7 +134,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
"-o %s"
),
1,
Arrays.asList("af5dd27354d5dfd0d2fe03149af09b55")
Arrays.asList("ccdbc50d30ece6d0d3b199c397f03ed3")
);
executeTest("testFundamentalsCountVariantsSNPsandIndelsWithCpG", spec);
}
@ -155,7 +155,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
"-o %s"
),
1,
Arrays.asList("062a231e203671e19aa9c6507710d762")
Arrays.asList("95c690d5af8ed51573eb2f0503dcd9c2")
);
executeTest("testFundamentalsCountVariantsSNPsandIndelsWithFunctionalClass", spec);
}
@ -176,7 +176,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
"-o %s"
),
1,
Arrays.asList("75abdd2b17c0a5e04814b6969a3d4d7e")
Arrays.asList("8e8547eb38b34bec0095b0500fd9641d")
);
executeTest("testFundamentalsCountVariantsSNPsandIndelsWithDegeneracy", spec);
}
@ -197,7 +197,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
"-o %s"
),
1,
Arrays.asList("bdbb5f8230a4a193058750c5e506c733")
Arrays.asList("158a4651a656aea7f84c79548f6fe519")
);
executeTest("testFundamentalsCountVariantsSNPsandIndelsWithSample", spec);
}
@ -220,7 +220,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
"-o %s"
),
1,
Arrays.asList("f076120da22930294840fcc396f5f141")
Arrays.asList("76c8a0b28d2993644120f7afa5833ab2")
);
executeTest("testFundamentalsCountVariantsSNPsandIndelsWithJexlExpression", spec);
}
@ -245,7 +245,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
"-o %s"
),
1,
Arrays.asList("69201f4a2a7a44b38805a4aeeb8830b6")
Arrays.asList("34682193f458b93b39efac00b4fc6723")
);
executeTest("testFundamentalsCountVariantsSNPsandIndelsWithMultipleJexlExpressions", spec);
}
@ -264,7 +264,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
"-o %s"
),
1,
Arrays.asList("c3bd3cb6cfb21a8c2b4d5f69104bf6c2")
Arrays.asList("52f6655f1532bcea24b402010d93ce73")
);
executeTest("testFundamentalsCountVariantsNoCompRod", spec);
}
@ -277,7 +277,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
" --eval " + validationDataLocation + "yri.trio.gatk_glftrio.intersection.annotated.filtered.chr1.vcf" +
" --comp:comp_genotypes,VCF3 " + validationDataLocation + "yri.trio.gatk.ug.head.vcf";
WalkerTestSpec spec = new WalkerTestSpec(withSelect(tests, "DP < 50", "DP50") + " " + extraArgs + " -ST CpG -o %s",
1, Arrays.asList("861f94e3237d62bd5bc00757319241f7"));
1, Arrays.asList("6fa6e77f149de3d13c31d410a98043a0"));
executeTestParallel("testSelect1", spec);
}
@ -287,14 +287,14 @@ public class VariantEvalIntegrationTest extends WalkerTest {
WalkerTestSpec spec = new WalkerTestSpec(cmdRoot + " -ST CpG --eval:VCF3 " + validationDataLocation + vcfFile + " --comp:VCF3 " + validationDataLocation + "GenotypeConcordanceComp.vcf -noEV -EV GenotypeConcordance -o %s",
1,
Arrays.asList("96f27163f16bb945f19c6623cd6db34e"));
Arrays.asList("9a56c20a7b9a554a7b530f2cb1dd776d"));
executeTestParallel("testVEGenotypeConcordance" + vcfFile, spec);
}
@Test
public void testCompVsEvalAC() {
String extraArgs = "-T VariantEval -R "+b36KGReference+" -o %s -ST CpG -EV GenotypeConcordance --eval:evalYRI,VCF3 " + validationDataLocation + "yri.trio.gatk.ug.very.few.lines.vcf --comp:compYRI,VCF3 " + validationDataLocation + "yri.trio.gatk.fake.genotypes.ac.test.vcf";
WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("955c33365e017679047fabec0f14d5e0"));
WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("aeff16bb43be03a2a7e5b9d0108b4999"));
executeTestParallel("testCompVsEvalAC",spec);
}
@ -312,7 +312,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
@Test
public void testCompOverlap() {
String extraArgs = "-T VariantEval -R " + b37KGReference + " -L " + validationDataLocation + "VariantEval/pacbio.hg19.intervals --comp:comphapmap " + comparisonDataLocation + "Validated/HapMap/3.3/genotypes_r27_nr.b37_fwd.vcf --eval " + validationDataLocation + "VariantEval/pacbio.ts.recalibrated.vcf -noEV -EV CompOverlap -sn NA12878 -noST -ST Novelty -o %s";
WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("fb7d989e44bd74c5376cb5732f9f3f64"));
WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("9002023b8aa8fc2c9aac58b8a79bca1e"));
executeTestParallel("testCompOverlap",spec);
}
@ -324,7 +324,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
" --dbsnp " + b37dbSNP132 +
" --eval:evalBI " + validationDataLocation + "VariantEval/ALL.20100201.chr20.bi.sites.vcf" +
" -noST -ST Novelty -o %s";
WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("da5bcb305c5ef207ce175821efdbdefd"));
WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("38ed9d216bd43f1cceceea24146fae38"));
executeTestParallel("testEvalTrackWithoutGenotypes",spec);
}
@ -336,7 +336,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
" --eval:evalBI " + validationDataLocation + "VariantEval/ALL.20100201.chr20.bi.sites.vcf" +
" --eval:evalBC " + validationDataLocation + "VariantEval/ALL.20100201.chr20.bc.sites.vcf" +
" -noST -ST Novelty -o %s";
WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("fde839ece1442388f21a2f0b936756a8"));
WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("453c6b1f7165913e8b1787e22bac1281"));
executeTestParallel("testMultipleEvalTracksWithoutGenotypes",spec);
}
@ -353,13 +353,13 @@ public class VariantEvalIntegrationTest extends WalkerTest {
" -noST -noEV -ST Novelty -EV CompOverlap" +
" -o %s";
WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("1efae6b3b88c752b771e0c8fae24464e"));
WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("61052c19211e7eb61fbbb62db5e40b56"));
executeTestParallel("testMultipleCompTracks",spec);
}
@Test
public void testPerSampleAndSubsettedSampleHaveSameResults1() {
String md5 = "bc9bcabc3105e2515d9a2d41506d2de1";
String md5 = "0edded1cd578db62fa296c99c34a909d";
WalkerTestSpec spec = new WalkerTestSpec(
buildCommandLine(
@ -414,7 +414,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
"-o %s"
),
1,
Arrays.asList("e53546243250634fc03e83b4e61ec55f")
Arrays.asList("ee22604616b3e9fc48a6dcbbf73a056d")
);
executeTest("testAlleleCountStrat", spec);
}
@ -435,7 +435,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
"-o %s"
),
1,
Arrays.asList("c8086f0525bc13e666afeb670c2e13ae")
Arrays.asList("240369cd651c77e05e8a6659f4a6237e")
);
executeTest("testIntervalStrat", spec);
}

2
public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/VCFStreamingIntegrationTest.java
View File

@ -98,7 +98,7 @@ public class VCFStreamingIntegrationTest extends WalkerTest {
" -EV CompOverlap -noEV -noST" +
" -o %s",
1,
Arrays.asList("1f7ed8c0f671dd227ab764624ef0d64c")
Arrays.asList("addf5f4596ddacef40808f6d3d281111")
);
executeTest("testVCFStreamingChain", selectTestSpec);

88
public/java/test/org/broadinstitute/sting/pipeline/PipelineUnitTest.java
View File

@ -1,88 +0,0 @@
/*
* Copyright (c) 2010, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.pipeline;
import org.broadinstitute.sting.pipeline.Pipeline;
import org.broadinstitute.sting.pipeline.PipelineSample;
import org.testng.Assert;
import org.broadinstitute.sting.utils.yaml.YamlUtils;
import org.testng.annotations.Test;
import java.io.File;
import java.util.Map;
public class PipelineUnitTest {
@Test
public void testDumpAndLoad() throws Exception {
Pipeline pipeline = new Pipeline();
pipeline.getProject().setName("PRJ_NAME");
pipeline.getProject().setReferenceFile(new File("my.fasta"));
pipeline.getProject().setGenotypeDbsnp(new File("my.vcf"));
pipeline.getProject().setEvalDbsnp(new File("my.dbsnp"));
pipeline.getProject().getTags().put("testProjectTag", "project value here");
PipelineSample sample = new PipelineSample();
sample.setId("SMP_ID");
sample.getBamFiles().put("recalibrated", new File("recalibrated.bam"));
sample.getBamFiles().put("cleaned", new File("/absolute/path/to/cleaned.bam"));
sample.getTags().put("testSampleTag", "sample value here");
pipeline.getSamples().add(sample);
File file = File.createTempFile("testDumpAndLoad", ".yaml");
YamlUtils.dump(pipeline, file);
Pipeline pipelineLoad = YamlUtils.load(Pipeline.class, file);
Assert.assertEquals(pipelineLoad.getProject().getName(), pipeline.getProject().getName());
Assert.assertEquals(pipeline.getProject().getReferenceFile(), pipelineLoad.getProject().getReferenceFile());
Assert.assertEquals(pipeline.getProject().getIntervalList(), pipelineLoad.getProject().getIntervalList());
Assert.assertEquals(pipeline.getProject().getGenotypeDbsnp(), pipelineLoad.getProject().getGenotypeDbsnp());
Assert.assertEquals(pipeline.getProject().getGenotypeDbsnpType(), pipelineLoad.getProject().getGenotypeDbsnpType());
Assert.assertEquals(pipeline.getProject().getEvalDbsnp(), pipelineLoad.getProject().getEvalDbsnp());
Assert.assertEquals(pipeline.getProject().getEvalDbsnpType(), pipelineLoad.getProject().getEvalDbsnpType());
Assert.assertEquals(pipelineLoad.getProject().getTags().size(), pipeline.getProject().getTags().size());
for (Map.Entry<String, String> entry : pipeline.getProject().getTags().entrySet())
Assert.assertEquals(pipeline.getProject().getTags().get(entry.getKey()), entry.getValue());
Assert.assertEquals(pipelineLoad.getSamples().size(), pipeline.getSamples().size());
for (int i = 0; i < pipeline.getSamples().size(); i++) {
PipelineSample pipelineSample = pipeline.getSamples().get(i);
PipelineSample pipelineLoadSample = pipelineLoad.getSamples().get(i);
Assert.assertEquals(pipelineLoadSample.getId(), pipelineSample.getId());
Assert.assertEquals(pipelineLoadSample.getBamFiles().size(), pipelineSample.getBamFiles().size());
for (Map.Entry<String, File> entry : pipelineSample.getBamFiles().entrySet())
Assert.assertEquals(entry.getValue(), pipelineSample.getBamFiles().get(entry.getKey()));
Assert.assertEquals(pipelineLoadSample.getTags().size(), pipelineSample.getTags().size());
for (Map.Entry<String, String> entry : pipelineSample.getTags().entrySet())
Assert.assertEquals(pipelineSample.getTags().get(entry.getKey()), entry.getValue());
}
}
}

3
public/packages/PicardPrivate.xml
View File

@ -12,15 +12,12 @@
<class name="edu.mit.broad.picard.variation.KnownVariantCodecV2" />
<class name="edu.mit.broad.picard.variation.KnownVariantCodec" />
<class name="edu.mit.broad.picard.variation.KnownVariantFileHeader" />
<class name="edu.mit.broad.picard.variation.GenerateDbSnpFile" />
<class name="edu.mit.broad.picard.variation.VariantType" />
<class name="edu.mit.broad.picard.variation.ObsoleteKnownVariantCodecV1" />
<class name="edu.mit.broad.picard.variation.AbstractKnownVariantCodec" />
<class name="edu.mit.broad.picard.variation.KnownVariantIterator" />
<class name="edu.mit.broad.picard.variation.KnownVariantCodecFactory" />
<class name="edu.mit.broad.picard.variation.KnownVariant" />
<class name="edu.mit.broad.picard.variation.DbSnpFileReader" />
<class name="edu.mit.broad.picard.variation.DbSnpFileGenerator" />
<class name="net.sf.picard.util.IlluminaUtil" />
<class name="edu.mit.broad.picard.genotype.fingerprint.v2.FingerprintingSummaryMetrics" />
<class name="edu.mit.broad.picard.genotype.concordance.DbSnpMatchMetrics" />

2
public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala
View File

@ -314,7 +314,7 @@ class GATKResourcesBundle extends QScript {
class MakeDBSNP129(@Input dbsnp: File, @Input ref: File, @Output dbsnp129: File) extends SelectVariants with UNIVERSAL_GATK_ARGS {
this.variant = dbsnp
this.select ++= List("\"dbSNPBuildID <= 129\"")
this.select ++= List("dbSNPBuildID <= 129")
this.reference_sequence = ref
this.out = dbsnp129
}

4
public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala
View File

@ -248,10 +248,10 @@ class MethodsDevelopmentCallingPipeline extends QScript {
this.V = t.rawIndelVCF
this.out = t.filteredIndelVCF
this.filterName ++= List("IndelQD", "IndelReadPosRankSum", "IndelFS")
this.filterExpression ++= List("\"QD < 2.0\"", "\"ReadPosRankSum < -20.0\"", "\"FS > 200.0\"")
this.filterExpression ++= List("QD < 2.0", "ReadPosRankSum < -20.0", "FS > 200.0")
if (t.nSamples >= 10) {
this.filterName ++= List("IndelInbreedingCoeff")
this.filterExpression ++= List("\"InbreedingCoeff < -0.8\"")
this.filterExpression ++= List("InbreedingCoeff < -0.8")
}
this.analysisName = t.name + "_VF"
this.jobName = queueLogDir + t.name + ".indelfilter"

2
public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleUnifiedGenotyper.scala
View File

@ -62,7 +62,7 @@ class ExampleUnifiedGenotyper extends QScript {
variantFilter.variant = genotyper.out
variantFilter.out = swapExt(qscript.bamFile, "bam", "filtered.vcf")
variantFilter.filterName = filterNames
variantFilter.filterExpression = filterExpressions.map("\"" + _ + "\"")
variantFilter.filterExpression = filterExpressions
evalFiltered.eval :+= variantFilter.out
evalFiltered.out = swapExt(variantFilter.out, "vcf", "eval")

17
public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/RodBind.scala
View File

@ -28,21 +28,14 @@ object RodBind {
def apply(trackName: String, trackType: String, file: File, tag: String) = new RodBind(trackName, trackType, file, tag)
def apply(trackName: String, trackType: String, file: File) = new RodBind(trackName, trackType, file, null)
/**
* Formats the rod binding on the command line.
* Used for optional and repeat.
* @param cmdLineParam command line parameter, ex: -B
* @param prefix unused
* @param value RodBind to add.
* @param suffix unused
* @return The command line addition.
*/
def formatCommandLine(cmdLineParam: String)(prefix: String, value: Any, suffix: String) = {
def formatCommandLineParameter( cmdLineParam: String, value: Any ) = {
value match {
case rodBind: RodBind if (rodBind.tag != null) =>
" %s:%s,%s,%s %s".format(cmdLineParam, rodBind.trackName, rodBind.trackType, rodBind.tag, rodBind.getPath)
"%s:%s,%s,%s".format(cmdLineParam, rodBind.trackName, rodBind.trackType, rodBind.tag)
case rodBind: RodBind =>
" %s:%s,%s %s".format(cmdLineParam, rodBind.trackName, rodBind.trackType, rodBind.getPath)
"%s:%s,%s".format(cmdLineParam, rodBind.trackName, rodBind.trackType)
case x =>
""
}
}
}

18
public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/TaggedFile.scala
View File

@ -2,6 +2,7 @@ package org.broadinstitute.sting.queue.extensions.gatk
import java.io.File
import org.broadinstitute.sting.utils.io.FileExtension
import org.broadinstitute.sting.queue.util.ShellUtils
/**
* Used to provide tagged -I input_file arguments to the GATK.
@ -19,21 +20,14 @@ object TaggedFile {
def apply(path: String, tag: String) = new TaggedFile(path, tag)
def apply(file: File, tag: String) = new TaggedFile(file, tag)
/**
* Formats the rod binding on the command line.
* Used for optional and repeat.
* @param cmdLineParam command line parameter, ex: -I
* @param prefix unused
* @param value TaggedFile to add.
* @param suffix unused
* @return The command line addition.
*/
def formatCommandLine(cmdLineParam: String)(prefix: String, value: Any, suffix: String) = {
def formatCommandLineParameter(cmdLineParam: String, value: Any) = {
value match {
case taggedFile: TaggedFile if (taggedFile.tag != null) =>
" %s:%s %s".format(cmdLineParam, taggedFile.tag, taggedFile.getPath)
"%s:%s".format(cmdLineParam, taggedFile.tag)
case file: File =>
" %s %s".format(cmdLineParam, file.getPath)
cmdLineParam
case x =>
""
}
}
}

14
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/AddOrReplaceReadGroups.scala
View File

@ -55,11 +55,11 @@ class AddOrReplaceReadGroups extends org.broadinstitute.sting.queue.function.Jav
override def outputBam = output
this.createIndex = Some(true)
override def commandLine = super.commandLine +
" RGID=" + RGID +
" RGLB=" + RGLB +
" RGPL=" + RGPL +
" RGPU=" + RGPU +
" RGSM=" + RGSM +
conditionalParameter(RGCN != null && !RGCN.isEmpty, " RGCN=" + RGCN) +
conditionalParameter(RGDS != null && !RGDS.isEmpty, " RGDS=" + RGDS)
required("RGID=" + RGID) +
required("RGLB=" + RGLB) +
required("RGPL=" + RGPL) +
required("RGPU=" + RGPU) +
required("RGSM=" + RGSM) +
conditional(RGCN != null && !RGCN.isEmpty, "RGCN=" + RGCN) +
conditional(RGDS != null && !RGDS.isEmpty, "RGDS=" + RGDS)
}

10
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MarkDuplicates.scala
View File

@ -47,10 +47,8 @@ class MarkDuplicates extends org.broadinstitute.sting.queue.function.JavaCommand
this.sortOrder = null
this.createIndex = Some(true)
override def commandLine = super.commandLine +
" M=" + metrics +
conditionalParameter(REMOVE_DUPLICATES, " REMOVE_DUPLICATES=true") +
conditionalParameter(MAX_FILE_HANDLES_FOR_READ_ENDS_MAP > 0, " MAX_FILE_HANDLES_FOR_READ_ENDS_MAP=" + MAX_FILE_HANDLES_FOR_READ_ENDS_MAP.toString) +
conditionalParameter(SORTING_COLLECTION_SIZE_RATIO > 0, " SORTING_COLLECTION_SIZE_RATIO=" + SORTING_COLLECTION_SIZE_RATIO.toString)
required("M=" + metrics) +
conditional(REMOVE_DUPLICATES, "REMOVE_DUPLICATES=true") +
conditional(MAX_FILE_HANDLES_FOR_READ_ENDS_MAP > 0, "MAX_FILE_HANDLES_FOR_READ_ENDS_MAP=" + MAX_FILE_HANDLES_FOR_READ_ENDS_MAP.toString) +
conditional(SORTING_COLLECTION_SIZE_RATIO > 0, "SORTING_COLLECTION_SIZE_RATIO=" + SORTING_COLLECTION_SIZE_RATIO.toString)
}

8
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MergeSamFiles.scala
View File

@ -44,9 +44,7 @@ class MergeSamFiles extends org.broadinstitute.sting.queue.function.JavaCommandL
override def outputBam = output
this.createIndex = Some(true)
override def commandLine = super.commandLine +
conditionalParameter(MERGE_SEQUENCE_DICTIONARIES, " MERGE_SEQUENCE_DICTIONARIES=true") +
conditionalParameter(USE_THREADING, " USE_THREADING=true") +
conditionalParameter(COMMENT != null && !COMMENT.isEmpty, " COMMENT=" + COMMENT)
conditional(MERGE_SEQUENCE_DICTIONARIES, "MERGE_SEQUENCE_DICTIONARIES=true") +
conditional(USE_THREADING, "USE_THREADING=true") +
conditional(COMMENT != null && !COMMENT.isEmpty, "COMMENT=" + COMMENT)
}

19
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamFunction.scala
View File

@ -48,14 +48,13 @@ trait PicardBamFunction extends JavaCommandLineFunction {
protected def outputBam: File
abstract override def commandLine = super.commandLine +
Array(
repeat(" INPUT=", inputBams),
" TMP_DIR=" + jobTempDir,
optional(" OUTPUT=", outputBam),
optional(" COMPRESSION_LEVEL=", compressionLevel),
optional(" VALIDATION_STRINGENCY=", validationStringency),
optional(" SO=", sortOrder),
optional(" MAX_RECORDS_IN_RAM=", maxRecordsInRam),
optional(" ASSUME_SORTED=", assumeSorted),
optional(" CREATE_INDEX=", createIndex)).mkString
repeat("INPUT=", inputBams, spaceSeparated=false) +
required("TMP_DIR=" + jobTempDir) +
optional("OUTPUT=", outputBam, spaceSeparated=false) +
optional("COMPRESSION_LEVEL=", compressionLevel, spaceSeparated=false) +
optional("VALIDATION_STRINGENCY=", validationStringency, spaceSeparated=false) +
optional("SO=", sortOrder, spaceSeparated=false) +
optional("MAX_RECORDS_IN_RAM=", maxRecordsInRam, spaceSeparated=false) +
optional("ASSUME_SORTED=", assumeSorted, spaceSeparated=false) +
optional("CREATE_INDEX=", createIndex, spaceSeparated=false)
}

6
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ReorderSam.scala
View File

@ -42,7 +42,7 @@ class ReorderSam extends org.broadinstitute.sting.queue.function.JavaCommandLine
this.createIndex = Some(true)
this.sortOrder = null
override def commandLine = super.commandLine +
" REFERENCE=" + sortReference +
optional(" ALLOW_INCOMPLETE_DICT_CONCORDANCE=", ALLOW_INCOMPLETE_DICT_CONCORDANCE)
optional(" ALLOW_CONTIG_LENGTH_DISCORDANCE=", ALLOW_CONTIG_LENGTH_DISCORDANCE)
required("REFERENCE=" + sortReference) +
optional("ALLOW_INCOMPLETE_DICT_CONCORDANCE=", ALLOW_INCOMPLETE_DICT_CONCORDANCE, spaceSeparated=false)
optional("ALLOW_CONTIG_LENGTH_DISCORDANCE=", ALLOW_CONTIG_LENGTH_DISCORDANCE, spaceSeparated=false)
}

12
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/RevertSam.scala
View File

@ -52,10 +52,10 @@ class RevertSam extends org.broadinstitute.sting.queue.function.JavaCommandLineF
override def outputBam = output
this.createIndex = Some(true)
override def commandLine = super.commandLine +
conditionalParameter(!restoreOriginalQualities, " RESTORE_ORIGINAL_QUALITIES=false") +
conditionalParameter(!removeDuplicateInformation, " REMOVE_DUPLICATE_INFORMATION=false") +
conditionalParameter(!removeAlignmentInformation, " REMOVE_ALIGNMENT_INFORMATION=false") +
conditionalParameter(!attributesToClear.isEmpty, repeat(" ATTRIBUTE_TO_CLEAR=", attributesToClear)) +
conditionalParameter(sampleAlias != null, " SAMPLE_ALIAS=" + sampleAlias) +
conditionalParameter(libraryName != null, " LIBRARY_NAME=" + libraryName)
conditional(!restoreOriginalQualities, "RESTORE_ORIGINAL_QUALITIES=false") +
conditional(!removeDuplicateInformation, "REMOVE_DUPLICATE_INFORMATION=false") +
conditional(!removeAlignmentInformation, "REMOVE_ALIGNMENT_INFORMATION=false") +
repeat("ATTRIBUTE_TO_CLEAR=", attributesToClear, spaceSeparated=false) + // repeat() returns "" for null/empty list
conditional(sampleAlias != null, "SAMPLE_ALIAS=" + sampleAlias) +
conditional(libraryName != null, "LIBRARY_NAME=" + libraryName)
}

26
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SamToFastq.scala
View File

@ -61,17 +61,17 @@ class SamToFastq extends org.broadinstitute.sting.queue.function.JavaCommandLine
this.sortOrder = null
override def commandLine = super.commandLine +
" FASTQ=" + fastq +
optional(" SECOND_END_FASTQ=", secondEndFastQ) +
conditionalParameter(outputPerReadGroup, optional(" OUTPUT_PER_RG=", outputPerReadGroup)) +
optional(" OUTPUT_DIR=", outputDir) +
conditionalParameter(!reReverse, optional(" RE_REVERSE=", reReverse)) +
conditionalParameter(includeNonPFReads, optional(" INCLUDE_NON_PF_READS=", includeNonPFReads)) +
optional(" CLIPPING_ATTRIBUTE=", clippingAttribute) +
optional(" CLIPPING_ACTION=", clippingAction) +
conditionalParameter (readOneTrim >= 0, optional(" READ1_TRIM=", readOneTrim)) +
conditionalParameter (readOneMaxBasesToWrite >= 0, optional(" READ1_MAX_BASES_TO_WRITE=", readOneMaxBasesToWrite)) +
conditionalParameter (readTwoTrim >= 0, optional(" READ2_TRIM=", readTwoTrim)) +
conditionalParameter (readTwoMaxBasesToWrite >=0, optional(" READ2_MAX_BASES_TO_WRITE=", readTwoMaxBasesToWrite)) +
conditionalParameter (includeNonPrimaryAlignments, optional(" INCLUDE_NON_PRIMARY_ALIGNMENTS=", includeNonPrimaryAlignments))
required("FASTQ=" + fastq) +
optional("SECOND_END_FASTQ=", secondEndFastQ, spaceSeparated=false) +
conditional(outputPerReadGroup, "OUTPUT_PER_RG=" + outputPerReadGroup) +
optional("OUTPUT_DIR=", outputDir, spaceSeparated=false) +
conditional(!reReverse, "RE_REVERSE=" + reReverse) +
conditional(includeNonPFReads, "INCLUDE_NON_PF_READS=" + includeNonPFReads) +
optional("CLIPPING_ATTRIBUTE=", clippingAttribute, spaceSeparated=false) +
optional("CLIPPING_ACTION=", clippingAction, spaceSeparated=false) +
conditional(readOneTrim >= 0, "READ1_TRIM=" + readOneTrim) +
conditional(readOneMaxBasesToWrite >= 0, "READ1_MAX_BASES_TO_WRITE=" + readOneMaxBasesToWrite) +
conditional(readTwoTrim >= 0, "READ2_TRIM=" + readTwoTrim) +
conditional(readTwoMaxBasesToWrite >= 0, "READ2_MAX_BASES_TO_WRITE=" + readTwoMaxBasesToWrite) +
conditional(includeNonPrimaryAlignments, "INCLUDE_NON_PRIMARY_ALIGNMENTS=" + includeNonPrimaryAlignments)
}

14
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ValidateSamFile.scala
View File

@ -50,11 +50,11 @@ class ValidateSamFile extends org.broadinstitute.sting.queue.function.JavaComman
override def inputBams = input
override def outputBam = output
override def commandLine = super.commandLine +
" MODE=" + MODE +
" MAX_OUTPUT=" + MAX_OUTPUT +
" MAX_OPEN_TEMP_FILES=" + MAX_OPEN_TEMP_FILES +
conditionalParameter(!VALIDATE_INDEX, " VALIDATE_INDEX=false") +
conditionalParameter(IGNORE_WARNINGS, " IGNORE_WARNINGS=true") +
conditionalParameter(IS_BISULFITE_SEQUENCED, " IS_BISULFITE_SEQUENCED=true") +
conditionalParameter(IGNORE != null && !IGNORE.isEmpty, repeat(" IGNORE=", IGNORE))
required("MODE=" + MODE) +
required("MAX_OUTPUT=" + MAX_OUTPUT) +
required("MAX_OPEN_TEMP_FILES=" + MAX_OPEN_TEMP_FILES) +
conditional(!VALIDATE_INDEX, "VALIDATE_INDEX=false") +
conditional(IGNORE_WARNINGS, "IGNORE_WARNINGS=true") +
conditional(IS_BISULFITE_SEQUENCED, "IS_BISULFITE_SEQUENCED=true") +
repeat("IGNORE=", IGNORE, spaceSeparated=false) // repeat() returns "" for null/empty list
}

5
public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsIndexFunction.scala
View File

@ -48,7 +48,10 @@ class SamtoolsIndexFunction extends SamtoolsCommandLineFunction {
bamFileIndex = new File(bamFile.getPath + ".bai")
}
def commandLine = "%s index %s %s".format(samtools, bamFile, bamFileIndex)
def commandLine = required(samtools) +
required("index") +
required(bamFile) +
required(bamFileIndex)
override def dotString = "Index: %s".format(bamFile.getName)
}

8
public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsMergeFunction.scala
View File

@ -55,7 +55,9 @@ class SamtoolsMergeFunction extends SamtoolsCommandLineFunction {
))
}
def commandLine = "%s merge%s %s%s".format(
samtools, optional(" -R ", region),
outputBam, repeat(" ", inputBams))
def commandLine = required(samtools) +
required("merge") +
optional("-R", region) +
required(outputBam) +
repeat(inputBams)
}

17
public/scala/src/org/broadinstitute/sting/queue/extensions/snpeff/SnpEff.scala
View File

@ -50,11 +50,14 @@ class SnpEff extends JavaCommandLineFunction {
@Output(doc="snp eff output")
var outVcf: File = _
override def commandLine = Array(
super.commandLine,
" eff",
if (verbose) " -v" else "",
optional(" -c ", config),
" -i vcf -o vcf %s %s > %s".format(genomeVersion, inVcf, outVcf)
).mkString
override def commandLine = super.commandLine +
required("eff") +
conditional(verbose, "-v") +
optional("-c", config) +
required("-i", "vcf") +
required("-o", "vcf") +
required(genomeVersion) +
required(inVcf) +
required(">", escape=false) +
required(outVcf)
}

209
public/scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala
View File

@ -110,61 +110,178 @@ trait CommandLineFunction extends QFunction with Logging {
}
/**
* Repeats parameters with a prefix/suffix if they are set otherwise returns "".
* Skips null, Nil, None. Unwraps Some(x) to x. Everything else is called with x.toString.
* @param prefix Command line prefix per parameter.
* @param params Traversable parameters.
* @param suffix Optional suffix per parameter.
* @param separator Optional separator per parameter.
* @param format Format function if the value has a value
* @return The generated string
* Safely construct a full required command-line argument with consistent quoting, whitespace separation, etc.
*
* @param prefix Prefix to insert before the argument value (eg., "-f")
* @param param The argument value itself
* @param suffix Suffix to append after the argument value
* @param spaceSeparated If true, insert a space between the prefix, param, and suffix
* @param escape If true, quote the generated argument to avoid interpretation by the shell
* @param format Format String used to convert param to a String
* @return The combined and formatted argument, surrounded by whitespace
*/
protected def repeat(prefix: String, params: Traversable[_], suffix: String = "", separator: String = "",
format: (String, Any, String) => String = formatValue("%s")) =
if (params == null)
""
else
params.filter(param => hasValue(param)).map(param => format(prefix, param, suffix)).mkString(separator)
protected def required( prefix: String, param: Any, suffix: String = "", spaceSeparated: Boolean = true,
escape: Boolean = true, format: String = "%s" ): String = {
" %s ".format(formatArgument(prefix, param, suffix, spaceSeparated, escape, format))
}
/**
* Returns parameter with a prefix/suffix if it is set otherwise returns "".
* Does not output null, Nil, None. Unwraps Some(x) to x. Everything else is called with x.toString.
* @param prefix Command line prefix per parameter.
* @param param Parameter to check for a value.
* @param suffix Optional suffix per parameter.
* @param format Format function if the value has a value
* @return The generated string
* Safely construct a one-token required command-line argument with quoting
*
* @param param The command-line argument value
* @return The argument value quoted and surrounded by whitespace
*/
protected def optional(prefix: String, param: Any, suffix: String = "",
format: (String, Any, String) => String = formatValue("%s")) =
if (hasValue(param)) format(prefix, param, suffix) else ""
protected def required( param: Any ): String = {
required("", param)
}
/**
* Returns "" if the value is null or an empty collection, otherwise return the value.toString.
* @param format Format string if the value has a value
* @param prefix Command line prefix per parameter.
* @param param Parameter to check for a value.
* @param suffix Optional suffix per parameter.
* @return "" if the value is null, or "" if the collection is empty, otherwise the value.toString.
* Safely construct a one-token required command-line argument, and specify whether you want quoting
*
* @param param The command-line argument value
* @param escape If true, quote the generated argument to avoid interpretation by the shell
* @return The argument value, quoted if quoting was requested, and surrounded by whitespace
*/
protected def formatValue(format: String)(prefix: String, param: Any, suffix: String): String =
if (CollectionUtils.isNullOrEmpty(param))
""
else
prefix + (param match {
case Some(x) => format.format(x)
case x => format.format(x)
}) + suffix
protected def required( param: Any, escape: Boolean ): String = {
required("", param, escape=escape)
}
/**
* Returns the parameter if the condition is true. Useful for long string of parameters
* @param condition the condition to validate
* @param param the string to be returned in case condition is true
* @return param if condition is true, "" otherwise
* Safely construct a full optional command-line argument with consistent quoting, whitespace separation, etc.
* If the argument has no value, returns an empty String.
*
* @param prefix Prefix to insert before the argument value (eg., "-f")
* @param param The argument value itself (if null/empty, the method returns empty String)
* @param suffix Suffix to append after the argument value
* @param spaceSeparated If true, insert a space between the prefix, param, and suffix
* @param escape If true, quote the generated argument to avoid interpretation by the shell
* @param format Format String used to convert param to a String
* @return The combined and formatted argument, surrounded by whitespace, or an empty String
* if the argument has no value
*/
protected def conditionalParameter(condition: Boolean, param: String): String =
if (condition == true)
param
else
protected def optional( prefix: String, param: Any, suffix: String = "", spaceSeparated: Boolean = true,
escape: Boolean = true, format: String = "%s" ): String = {
if ( hasValue(param) ) " %s ".format(formatArgument(prefix, param, suffix, spaceSeparated, escape, format)) else ""
}
/**
* Safely construct a one-token optional command-line argument with quoting.
* If the argument has no value, returns an empty String.
*
* @param param The command-line argument value
* @return The argument value quoted and surrounded by whitespace, or an empty String
* if the argument has no value
*/
protected def optional( param: Any ): String = {
optional("", param)
}
/**
* Safely construct a one-token conditional command-line argument. If the provided condition
* is false, an empty String is returned.
*
* @param condition The condition to check
* @param param The command-line argument value
* @param escape If true, quote the generated argument to avoid interpretation by the shell
* @param format Format String used to convert param to a String
* @return The command-line argument value, quoted if quoting was requested and surrounded
* by whitespace, or an empty String if the argument has no value.
*/
protected def conditional( condition: Boolean, param: Any, escape: Boolean = true, format: String = "%s" ): String = {
if ( condition ) {
" %s ".format(formatArgument("", param, "", false, escape, format))
}
else {
""
}
}
/**
* Safely construct a series of full command-line arguments with consistent quoting, whitespace separation, etc.
*
* Each argument value is preceded by a prefix/suffix if they are set. A function can be provided to vary
* each prefix for each argument value (eg., -f:tag1 file1 -f:tag2 file2) -- the default is to use
* the same prefix for all arguments.
*
* @param prefix Prefix to insert before each argument value (eg., "-f")
* @param params The collection of argument values
* @param suffix Suffix to append after each argument value
* @param separator Specifies how to separate the various arguments from each other
* (eg., what should go between '-f' 'file1' and '-f' 'file2'?)
* Default is one space character.
* @param spaceSeparated If true, insert a space between each individual prefix, param, and suffix
* @param escape If true, quote the generated argument to avoid interpretation by the shell
* @param format Format String used to convert each individual param within params to a String
* @param formatPrefix Function mapping (prefix, argumentValue) pairs to prefixes. Can be used to
* vary each prefix depending on the argument value (useful for tags, etc.).
* Default is to use the same prefix for all argument values.
* @return The series of command-line arguments, quoted and whitespace-delimited as requested,
* or an empty String if params was null/Nil/None.
*/
protected def repeat(prefix: String, params: Traversable[_], suffix: String = "", separator: String = " ",
spaceSeparated: Boolean = true, escape: Boolean = true, format: String = "%s",
formatPrefix: (String, Any) => String = (prefix, value) => prefix): String = {
if (CollectionUtils.isNullOrEmpty(params))
""
else
" %s ".format(params.filter(param => hasValue(param)).map(param => formatArgument(formatPrefix(prefix, param), param, suffix, spaceSeparated, escape, format)).mkString(separator))
}
/**
* Safely construct a series of one-token command-line arguments with quoting and space separation.
*
* @param params The collection of argument values
* @return The argument values quoted and space-delimited, or an empty String if params was null/Nil/None
*/
protected def repeat( params: Traversable[_] ): String = {
repeat("", params)
}
/**
* Given an (optional) prefix, an argument value, and an (optional) suffix, formats a command-line
* argument with the specified level of quoting and space-separation.
*
* Helper method for required(), optional(), conditional(), and repeat() -- do not use this
* method directly!
*
* @param prefix Prefix to insert before the argument value (eg., "-f"). Ignored if empty/null.
* @param param The argument value itself. If this is Some(x), it is unwrapped to x before processing.
* @param suffix Suffix to append after the argument value. Ignored if empty/null.
* @param spaceSeparated If true, insert a space between the prefix, param, and suffix
* @param escape If true, quote the generated argument to avoid interpretation by the shell
* @param paramFormat Format string used to convert param to a String
* @return The combined and formatted argument, NOT surrounded by any whitespace.
* Returns an empty String if param was null/empty.
*/
protected def formatArgument( prefix: String, param: Any, suffix: String, spaceSeparated: Boolean, escape: Boolean,
paramFormat: String ): String = {
if (CollectionUtils.isNullOrEmpty(param)) {
return ""
}
// Trim leading and trailing whitespace off our three tokens, and unwrap Some(x) to x for the param
val trimmedValues : List[String] = List((if ( prefix != null ) prefix.trim else ""),
(param match {
case Some(x) => paramFormat.format(x).trim
case x => paramFormat.format(x).trim
}),
(if ( suffix != null ) suffix.trim else ""))
var joinedArgument : String = null
// If the user requested space-separation, join the tokens with a space, and escape individual
// NON-EMPTY tokens if escaping was requested (eg., ("-f", "foo", "") -> "'-f' 'foo'")
if ( spaceSeparated ) {
joinedArgument = trimmedValues.map(x => if ( x.length > 0 && escape ) ShellUtils.escapeShellArgument(x) else x).mkString(" ").trim()
}
// Otherwise join the tokens without any intervening whitespace, and if quoting was requested
// quote the entire concatenated value (eg., ("-Xmx", "4", "G") -> "'-Xmx4G'")
else {
joinedArgument = if ( escape ) ShellUtils.escapeShellArgument(trimmedValues.mkString("")) else trimmedValues.mkString("")
}
// If the user requested escaping and we ended up with an empty String after joining, quote the empty
// String to preserve the command line token. Otherwise just return the joined argument
if ( joinedArgument.length == 0 && escape ) ShellUtils.escapeShellArgument(joinedArgument) else joinedArgument
}
}

34
public/scala/src/org/broadinstitute/sting/queue/function/JavaCommandLineFunction.scala
View File

@ -49,18 +49,6 @@ trait JavaCommandLineFunction extends CommandLineFunction {
*/
var javaMemoryLimit: Option[Double] = None
/**
* Returns the java executable to run.
*/
def javaExecutable: String = {
if (jarFile != null)
"-jar " + jarFile
else if (javaMainClass != null)
"-cp \"%s\" %s".format(javaClasspath.mkString(File.pathSeparator), javaMainClass)
else
null
}
override def freezeFieldValues() {
super.freezeFieldValues()
@ -71,11 +59,25 @@ trait JavaCommandLineFunction extends CommandLineFunction {
javaClasspath = JavaCommandLineFunction.currentClasspath
}
def javaOpts = "%s -Djava.io.tmpdir=%s"
.format(optional(" -Xmx", javaMemoryLimit.map(gb => (gb * 1024).ceil.toInt), "m"), jobTempDir)
/**
* Returns the java executable to run.
*/
def javaExecutable: String = {
if (jarFile != null)
required("-jar", jarFile)
else if (javaMainClass != null)
required("-cp", javaClasspath.mkString(File.pathSeparator)) +
required(javaMainClass)
else
null
}
def commandLine = "java%s %s"
.format(javaOpts, javaExecutable)
def javaOpts = optional("-Xmx", javaMemoryLimit.map(gb => (gb * 1024).ceil.toInt), "m", spaceSeparated=false) +
required("-Djava.io.tmpdir=", jobTempDir, spaceSeparated=false)
def commandLine = required("java") +
javaOpts +
javaExecutable
}
object JavaCommandLineFunction {

36
public/scala/src/org/broadinstitute/sting/queue/util/ShellUtils.scala 100644
View File

@ -0,0 +1,36 @@
package org.broadinstitute.sting.queue.util
import java.lang.IllegalArgumentException
object ShellUtils {
/**
* Escapes the String it's passed so that it will be interpreted literally when
* parsed by sh/bash. Can correctly escape all characters except \0, \r, and \n
*
* Replaces all instances of ' with '\'', and then surrounds the resulting String
* with single quotes.
*
* Examples:
* ab -> 'ab'
* a'b -> 'a'\''b'
* '' -> ''\'''\'''
*
* Since \' is not supported inside single quotes in the shell (ie., '\'' does not work),
* whenever we encounter a single quote we need to terminate the existing single-quoted
* string, place the \' outside of single quotes, and then start a new single-quoted
* string. As long as we don't insert spaces between the separate strings, the shell will
* concatenate them together into a single argument value for us.
*
* @param str the String to escape
* @return the same String quoted so that it will be interpreted literally when
* parsed by sh/bash
*/
def escapeShellArgument ( str : String ) : String = {
if ( str == null ) {
throw new IllegalArgumentException("escapeShellArgument() was passed a null String")
}
"'" + str.replaceAll("'", "'\\\\''") + "'"
}
}

171
public/scala/test/org/broadinstitute/sting/queue/function/CommandLineFunctionUnitTest.scala 100644
View File

@ -0,0 +1,171 @@
package org.broadinstitute.sting.queue.function
import org.testng.Assert
import org.testng.annotations.{DataProvider, Test}
// Since "protected" in Scala is subclass-only and doesn't allow package-level access, we need to
// extend a class if we want to test it
class CommandLineFunctionUnitTest extends CommandLineFunction {
def commandLine = ""
@DataProvider( name="formatArgumentTestData" )
def formatArgumentDataProvider = {
Array(Array("", "argvalue", "", true, true, "'argvalue'"),
Array("", "argvalue", "", true, false, "argvalue"),
Array("", "argvalue", "", false, true, "'argvalue'"),
Array("", "argvalue", "", false, false, "argvalue"),
Array("-arg", "argvalue", "", true, true, "'-arg' 'argvalue'"),
Array("-arg", "argvalue", "", true, false, "-arg argvalue"),
Array("ARGNAME=", "ARGVALUE", "", false, true, "'ARGNAME=ARGVALUE'"),
Array("ARGNAME=", "ARGVALUE", "", false, false, "ARGNAME=ARGVALUE"),
Array("-Xmx", "4", "G", true, true, "'-Xmx' '4' 'G'"),
Array("-Xmx", "4", "G", true, false, "-Xmx 4 G"),
Array("-Xmx", "4", "G", false, true, "'-Xmx4G'"),
Array("-Xmx", "4", "G", false, false, "-Xmx4G"),
Array("", "", "", true, true, "''"),
Array("", "", "", true, false, ""),
Array("", "", "", false, true, "''"),
Array("", "", "", false, false, ""),
Array("", null, "", true, true, ""),
Array("", Nil, "", true, true, ""),
Array("", None, "", true, true, ""),
Array(null, null, null, true, true, ""),
Array("", Some("argvalue"), "", true, true, "'argvalue'")
)
}
@Test( dataProvider="formatArgumentTestData" )
def testFormatArgument( prefix: String, param: Any, suffix: String, spaceSeparated: Boolean, escape: Boolean, expectedReturnValue: String ) {
Assert.assertEquals(formatArgument(prefix, param, suffix, spaceSeparated, escape, "%s"),
expectedReturnValue)
}
@Test
def testFormatArgumentCustomFormatString() {
Assert.assertEquals(formatArgument("", "argvalue", "", true, true, "%.3s"), "'arg'")
}
@DataProvider( name = "requiredTestData" )
def requiredDataProvider = {
Array(Array("", "argvalue", "", true, true, " 'argvalue' "),
Array("", "argvalue", "", true, false, " argvalue "),
Array("", "argvalue", "", false, true, " 'argvalue' "),
Array("", "argvalue", "", false, false, " argvalue "),
Array("-arg", "argvalue", "", true, true, " '-arg' 'argvalue' "),
Array("-arg", "argvalue", "", true, false, " -arg argvalue "),
Array("ARGNAME=", "ARGVALUE", "", false, true, " 'ARGNAME=ARGVALUE' "),
Array("ARGNAME=", "ARGVALUE", "", false, false, " ARGNAME=ARGVALUE "),
Array("-Xmx", "4", "G", true, true, " '-Xmx' '4' 'G' "),
Array("-Xmx", "4", "G", true, false, " -Xmx 4 G "),
Array("-Xmx", "4", "G", false, true, " '-Xmx4G' "),
Array("-Xmx", "4", "G", false, false, " -Xmx4G "),
Array("", "", "", true, true, " '' "),
Array("", "", "", true, false, " "),
Array("", "", "", false, true, " '' "),
Array("", "", "", false, false, " "),
Array("", null, "", true, true, " "),
Array("", Nil, "", true, true, " "),
Array("", None, "", true, true, " ")
)
}
@Test( dataProvider="requiredTestData" )
def testRequired( prefix: String, param: Any, suffix: String, spaceSeparated: Boolean, escape: Boolean, expectedReturnValue: String ) {
Assert.assertEquals(required(prefix, param, suffix, spaceSeparated, escape),
expectedReturnValue)
}
@DataProvider( name = "optionalTestData" )
def optionalDataProvider = {
Array(Array("-arg", "argvalue", "", true, true, " '-arg' 'argvalue' "),
Array("-arg", null, "", true, true, ""),
Array("-arg", Nil, "", true, true, ""),
Array("-arg", None, "", true, true, ""),
Array("-arg", "", "", true, true, " '-arg' ")
)
}
@Test( dataProvider="optionalTestData" )
def testOptional( prefix: String, param: Any, suffix: String, spaceSeparated: Boolean, escape: Boolean, expectedReturnValue: String ) {
Assert.assertEquals(optional(prefix, param, suffix, spaceSeparated, escape),
expectedReturnValue)
}
@DataProvider( name = "conditionalTestData" )
def conditionalDataProvider = {
Array(Array(true, "-FLAG", true, " '-FLAG' "),
Array(true, "-FLAG", false, " -FLAG "),
Array(false, "-FLAG", true, ""),
Array(false, "-FLAG", false, ""),
Array(true, null, true, " "),
Array(true, Nil, true, " "),
Array(true, None, true, " "),
Array(false, null, true, ""),
Array(false, Nil, true, ""),
Array(false, None, true, "")
)
}
@Test( dataProvider="conditionalTestData" )
def testConditional( condition: Boolean, param: Any, escape: Boolean, expectedReturnValue: String ) {
Assert.assertEquals(conditional(condition, param, escape),
expectedReturnValue)
}
@DataProvider( name = "repeatTestData" )
def repeatDataProvider = {
Array(Array("", List("a", "bc", "d"), "", " ", true, true, " 'a' 'bc' 'd' "),
Array("", List("a", "bc", "d"), "", " ", true, false, " a bc d "),
Array("", List("a", "bc", "d"), "", "", true, true, " 'a''bc''d' "),
Array("", List("a", "bc", "d"), "", "", true, false, " abcd "),
Array("-f", List("file1", "file2", "file3"), "", " ", true, true, " '-f' 'file1' '-f' 'file2' '-f' 'file3' "),
Array("-f", List("file1", "file2", "file3"), "", " ", true, false, " -f file1 -f file2 -f file3 "),
Array("-f", List("file1", "file2", "file3"), "", " ", false, true, " '-ffile1' '-ffile2' '-ffile3' "),
Array("-f", List("file1", "file2", "file3"), "", " ", false, false, " -ffile1 -ffile2 -ffile3 "),
Array("-f", List("file1", "file2", "file3"), "", "", false, true, " '-ffile1''-ffile2''-ffile3' "),
Array("-f", List("file1", "file2", "file3"), "", "", false, false, " -ffile1-ffile2-ffile3 "),
Array("-f", List("file1", "file2", "file3"), "suffix", " ", true, true, " '-f' 'file1' 'suffix' '-f' 'file2' 'suffix' '-f' 'file3' 'suffix' "),
Array("-f", List("file1", "file2", "file3"), "suffix", " ", true, false, " -f file1 suffix -f file2 suffix -f file3 suffix "),
Array("-f", List("file1", "file2", "file3"), "suffix", " ", false, true, " '-ffile1suffix' '-ffile2suffix' '-ffile3suffix' "),
Array("-f", List("file1", "file2", "file3"), "suffix", " ", false, false, " -ffile1suffix -ffile2suffix -ffile3suffix "),
Array("-f", null, "", " ", true, true, ""),
Array("-f", Nil, "", " ", true, true, "")
)
}
@Test( dataProvider="repeatTestData" )
def testRepeat( prefix: String, params: Traversable[_], suffix: String, separator: String,
spaceSeparated: Boolean, escape: Boolean, expectedReturnValue: String ) {
Assert.assertEquals(repeat(prefix, params, suffix, separator, spaceSeparated, escape),
expectedReturnValue)
}
// Need to test None separately due to implicit conversion issues when using None in a TestNG data provider
@Test
def testRepeatNone() {
testRepeat("", None, "", " ", true, true, "")
}
@DataProvider( name = "repeatWithPrefixFormattingTestData" )
def repeatWithPrefixFormattingDataProvider = {
Array(Array("-f", List("file1", "file2", "file3"), "", " ", true, true, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value),
" '-f:tagfile1' 'file1' '-f:tagfile2' 'file2' '-f:tagfile3' 'file3' "),
Array("-f", List("file1", "file2", "file3"), "", " ", true, false, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value),
" -f:tagfile1 file1 -f:tagfile2 file2 -f:tagfile3 file3 "),
Array("", List("file1", "file2", "file3"), "", " ", true, true, (prefix: String, value: Any) => "-%s".format(value),
" '-file1' 'file1' '-file2' 'file2' '-file3' 'file3' "),
Array("-f", null, "", " ", true, true, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value),
""),
Array("-f", Nil, "", " ", true, true, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value),
"")
)
}
@Test( dataProvider = "repeatWithPrefixFormattingTestData" )
def testRepeatWithPrefixFormatting( prefix: String, params: Traversable[_], suffix: String, separator: String,
spaceSeparated: Boolean, escape: Boolean, formatPrefix: (String, Any) => String,
expectedReturnValue: String ) {
Assert.assertEquals(repeat(prefix, params, suffix, separator, spaceSeparated, escape, "%s", formatPrefix),
expectedReturnValue)
}
}

57
public/scala/test/org/broadinstitute/sting/queue/util/ShellUtilsUnitTest.scala 100644
View File

@ -0,0 +1,57 @@
package org.broadinstitute.sting.queue.util
import org.testng.annotations.Test
import org.testng.Assert
import java.io.{InputStreamReader, BufferedReader}
class ShellUtilsUnitTest {
@Test
def testEscapeShellArgumentOneCharSequences() {
// Test all ASCII characters except \0, \n, and \r, which we do not support escaping
for ( asciiCode <- 1 to 127 if asciiCode != 10 && asciiCode != 13 ) {
val originalString: String = "%c".format(asciiCode.toChar)
val quotedString: String = ShellUtils.escapeShellArgument(originalString)
val child : Process = new ProcessBuilder("/bin/sh", "-c", "printf \"%s\" " + quotedString).start()
val childReader : BufferedReader = new BufferedReader(new InputStreamReader(child.getInputStream))
val childOutputBuffer : StringBuilder = new StringBuilder
val childReaderThread : Thread = new Thread(new Runnable() {
def run() {
var line : String = childReader.readLine()
while ( line != null ) {
childOutputBuffer.append(line)
line = childReader.readLine()
}
}
})
childReaderThread.start()
val childReturnValue = child.waitFor()
childReaderThread.join()
childReader.close()
val childOutput = childOutputBuffer.toString()
if ( childReturnValue != 0 ) {
Assert.fail("With character ASCII %d, sh child process returned: %d".format(asciiCode, childReturnValue))
}
else if ( ! originalString.equals(childOutput) ) {
Assert.fail("With character ASCII %d, sh child process output \"%s\" instead of the expected \"%s\"".format(
asciiCode, childOutput, originalString))
}
}
}
@Test(expectedExceptions = Array(classOf[IllegalArgumentException]))
def testEscapeShellArgumentNullString() {
ShellUtils.escapeShellArgument(null)
}
@Test
def testEscapeShellArgumentEmptyString() {
Assert.assertEquals(ShellUtils.escapeShellArgument(""), "''")
}
}

BIN
settings/repository/edu.mit.broad/picard-private-parts-2068.jar
View File

Binary file not shown.

BIN
settings/repository/edu.mit.broad/picard-private-parts-2125.jar 100644
View File

Binary file not shown.

2
settings/repository/edu.mit.broad/picard-private-parts-2068.xml → settings/repository/edu.mit.broad/picard-private-parts-2125.xml
View File

@ -1,3 +1,3 @@
<ivy-module version="1.0">
<info organisation="edu.mit.broad" module="picard-private-parts" revision="2068" status="integration" publication="20111024162900" />
<info organisation="edu.mit.broad" module="picard-private-parts" revision="2125" status="integration" publication="20111024162900" />
</ivy-module>

3
settings/repository/net.sf/picard-1.55.985.xml
View File

@ -1,3 +0,0 @@
<ivy-module version="1.0">
<info organisation="net.sf" module="picard" revision="1.55.985" status="release" />
</ivy-module>

BIN
settings/repository/net.sf/picard-1.55.985.jar → settings/repository/net.sf/picard-1.57.1030.jar
View File

Binary file not shown.

3
settings/repository/net.sf/picard-1.57.1030.xml 100644
View File

@ -0,0 +1,3 @@
<ivy-module version="1.0">
<info organisation="net.sf" module="picard" revision="1.57.1030" status="release" />
</ivy-module>

3
settings/repository/net.sf/sam-1.55.985.xml
View File

@ -1,3 +0,0 @@
<ivy-module version="1.0">
<info organisation="net.sf" module="sam" revision="1.55.985" status="release" />
</ivy-module>

BIN
settings/repository/net.sf/sam-1.55.985.jar → settings/repository/net.sf/sam-1.57.1030.jar
View File

Binary file not shown.

3
settings/repository/net.sf/sam-1.57.1030.xml 100644
View File

@ -0,0 +1,3 @@
<ivy-module version="1.0">
<info organisation="net.sf" module="sam" revision="1.57.1030" status="release" />
</ivy-module>

BIN
settings/repository/org.broad/tribble-42.jar → settings/repository/org.broad/tribble-46.jar
View File

Binary file not shown.

2
settings/repository/org.broad/tribble-42.xml → settings/repository/org.broad/tribble-46.xml
View File

@ -1,3 +1,3 @@
<ivy-module version="1.0">
<info organisation="org.broad" module="tribble" revision="42" status="integration" />
<info organisation="org.broad" module="tribble" revision="46" status="integration" />
</ivy-module>