Added ability to specify arguments in files via -args/--arg_file

Pushing back downsample and read filter args so they show up in getApproximateCommandLineArgs()
This commit is contained in:
Khalid Shakir 2011-10-14 12:06:41 -04:00
parent f6ccac889b
commit 89a581a66f
14 changed files with 610 additions and 91 deletions

View File

@ -46,7 +46,7 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
/**
* Maps indices of command line arguments to values paired with that argument.
*/
public final SortedMap<Integer,List<String>> indices = new TreeMap<Integer,List<String>>();
public final SortedMap<ArgumentMatchSite,List<String>> sites = new TreeMap<ArgumentMatchSite,List<String>>();
/**
* An ordered, freeform collection of tags.
@ -72,32 +72,32 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
}
/**
* A simple way of indicating that an argument with the given label and definition exists at this index.
* A simple way of indicating that an argument with the given label and definition exists at this site.
* @param label Label of the argument match. Must not be null.
* @param definition The associated definition, if one exists. May be null.
* @param index Position of the argument. Must not be null.
* @param site Position of the argument. Must not be null.
* @param tags ordered freeform text tags associated with this argument.
*/
public ArgumentMatch(final String label, final ArgumentDefinition definition, final int index, final Tags tags) {
this( label, definition, index, null, tags );
public ArgumentMatch(final String label, final ArgumentDefinition definition, final ArgumentMatchSite site, final Tags tags) {
this( label, definition, site, null, tags );
}
/**
* A simple way of indicating that an argument with the given label and definition exists at this index.
* A simple way of indicating that an argument with the given label and definition exists at this site.
* @param label Label of the argument match. Must not be null.
* @param definition The associated definition, if one exists. May be null.
* @param index Position of the argument. Must not be null.
* @param site Position of the argument. Must not be null.
* @param value Value for the argument at this position.
* @param tags ordered freeform text tags associated with this argument.
*/
private ArgumentMatch(final String label, final ArgumentDefinition definition, final int index, final String value, final Tags tags) {
private ArgumentMatch(final String label, final ArgumentDefinition definition, final ArgumentMatchSite site, final String value, final Tags tags) {
this.label = label;
this.definition = definition;
ArrayList<String> values = new ArrayList<String>();
if( value != null )
values.add(value);
indices.put(index,values );
sites.put(site,values );
this.tags = tags;
}
@ -117,7 +117,7 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
ArgumentMatch otherArgumentMatch = (ArgumentMatch)other;
return this.definition.equals(otherArgumentMatch.definition) &&
this.label.equals(otherArgumentMatch.label) &&
this.indices.equals(otherArgumentMatch.indices) &&
this.sites.equals(otherArgumentMatch.sites) &&
this.tags.equals(otherArgumentMatch.tags);
}
@ -129,16 +129,17 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
* @param key Key which specifies the transform.
* @return A variant of this ArgumentMatch with all keys transformed.
*/
@SuppressWarnings("unchecked")
ArgumentMatch transform(Multiplexer multiplexer, Object key) {
SortedMap<Integer,List<String>> newIndices = new TreeMap<Integer,List<String>>();
for(Map.Entry<Integer,List<String>> index: indices.entrySet()) {
SortedMap<ArgumentMatchSite,List<String>> newIndices = new TreeMap<ArgumentMatchSite,List<String>>();
for(Map.Entry<ArgumentMatchSite,List<String>> site: sites.entrySet()) {
List<String> newEntries = new ArrayList<String>();
for(String entry: index.getValue())
for(String entry: site.getValue())
newEntries.add(multiplexer.transformArgument(key,entry));
newIndices.put(index.getKey(),newEntries);
newIndices.put(site.getKey(),newEntries);
}
ArgumentMatch newArgumentMatch = new ArgumentMatch(label,definition);
newArgumentMatch.indices.putAll(newIndices);
newArgumentMatch.sites.putAll(newIndices);
return newArgumentMatch;
}
@ -157,9 +158,9 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
public Iterator<ArgumentMatch> iterator() {
return new Iterator<ArgumentMatch>() {
/**
* Iterate over each the available index.
* Iterate over each the available site.
*/
private Iterator<Integer> indexIterator = null;
private Iterator<ArgumentMatchSite> siteIterator = null;
/**
* Iterate over each available token.
@ -167,9 +168,9 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
private Iterator<String> tokenIterator = null;
/**
* The next index to return. Null if none remain.
* The next site to return. Null if none remain.
*/
Integer nextIndex = null;
ArgumentMatchSite nextSite = null;
/**
* The next token to return. Null if none remain.
@ -177,7 +178,7 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
String nextToken = null;
{
indexIterator = indices.keySet().iterator();
siteIterator = sites.keySet().iterator();
prepareNext();
}
@ -186,7 +187,7 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
* @return True if there's another token waiting in the wings. False otherwise.
*/
public boolean hasNext() {
return nextToken != null;
return nextToken != null;
}
/**
@ -194,32 +195,32 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
* @return The next ArgumentMatch in the series. Should never be null.
*/
public ArgumentMatch next() {
if( nextIndex == null || nextToken == null )
if( nextSite == null || nextToken == null )
throw new IllegalStateException( "No more ArgumentMatches are available" );
ArgumentMatch match = new ArgumentMatch( label, definition, nextIndex, nextToken, tags );
ArgumentMatch match = new ArgumentMatch( label, definition, nextSite, nextToken, tags );
prepareNext();
return match;
}
/**
* Initialize the next ArgumentMatch to return. If no ArgumentMatches are available,
* initialize nextIndex / nextToken to null.
* initialize nextSite / nextToken to null.
*/
private void prepareNext() {
if( tokenIterator != null && tokenIterator.hasNext() ) {
nextToken = tokenIterator.next();
}
else {
nextIndex = null;
nextSite = null;
nextToken = null;
// Do a nested loop. While more data is present in the inner loop, grab that data.
// Otherwise, troll the outer iterator looking for more data.
while( indexIterator.hasNext() ) {
nextIndex = indexIterator.next();
if( indices.get(nextIndex) != null ) {
tokenIterator = indices.get(nextIndex).iterator();
while( siteIterator.hasNext() ) {
nextSite = siteIterator.next();
if( sites.get(nextSite) != null ) {
tokenIterator = sites.get(nextSite).iterator();
if( tokenIterator.hasNext() ) {
nextToken = tokenIterator.next();
break;
@ -245,29 +246,29 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
* @param other The other match to merge into.
*/
public void mergeInto( ArgumentMatch other ) {
indices.putAll(other.indices);
sites.putAll(other.sites);
}
/**
* Associate a value with this merge maapping.
* @param index index of the command-line argument to which this value is mated.
* @param site site of the command-line argument to which this value is mated.
* @param value Text representation of value to add.
*/
public void addValue( int index, String value ) {
if( !indices.containsKey(index) || indices.get(index) == null )
indices.put(index, new ArrayList<String>() );
indices.get(index).add(value);
public void addValue( ArgumentMatchSite site, String value ) {
if( !sites.containsKey(site) || sites.get(site) == null )
sites.put(site, new ArrayList<String>() );
sites.get(site).add(value);
}
/**
* Does this argument already have a value at the given site?
* Arguments are only allowed to be single-valued per site, and
* flags aren't allowed a value at all.
* @param index Index at which to check for values.
* @param site Site at which to check for values.
* @return True if the argument has a value at the given site. False otherwise.
*/
public boolean hasValueAtSite( int index ) {
return (indices.get(index) != null && indices.get(index).size() >= 1) || isArgumentFlag();
public boolean hasValueAtSite( ArgumentMatchSite site ) {
return (sites.get(site) != null && sites.get(site).size() >= 1) || isArgumentFlag();
}
/**
@ -276,9 +277,9 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
*/
public List<String> values() {
List<String> values = new ArrayList<String>();
for( int index: indices.keySet() ) {
if( indices.get(index) != null )
values.addAll(indices.get(index));
for( ArgumentMatchSite site: sites.keySet() ) {
if( sites.get(site) != null )
values.addAll(sites.get(site));
}
return values;
}

View File

@ -0,0 +1,76 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.commandline;
/**
* Which source and the index within the source where an argument match was found.
*/
public class ArgumentMatchSite implements Comparable<ArgumentMatchSite> {
private final ArgumentMatchSource source;
private final int index;
public ArgumentMatchSite(ArgumentMatchSource source, int index) {
this.source = source;
this.index = index;
}
public ArgumentMatchSource getSource() {
return source;
}
public int getIndex() {
return index;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ArgumentMatchSite that = (ArgumentMatchSite) o;
return (index == that.index) && (source == null ? that.source == null : source.equals(that.source));
}
@Override
public int hashCode() {
int result = source != null ? source.hashCode() : 0;
// Generated by intellij. No other special reason to this implementation. -ks
result = 31 * result + index;
return result;
}
@Override
public int compareTo(ArgumentMatchSite that) {
int comp = this.source.compareTo(that.source);
if (comp != 0)
return comp;
// Both files are the same.
if (this.index == that.index)
return 0;
return this.index < that.index ? -1 : 1;
}
}

View File

@ -0,0 +1,98 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.commandline;
import java.io.File;
/**
* Where an argument match originated, via the commandline or a file.
*/
public class ArgumentMatchSource implements Comparable<ArgumentMatchSource> {
public static final ArgumentMatchSource COMMAND_LINE = new ArgumentMatchSource(ArgumentMatchSourceType.CommandLine, null);
private final ArgumentMatchSourceType type;
private final File file;
/**
* Creates an argument match source from the specified file.
* @param file File specifying the arguments. Must not be null.
*/
public ArgumentMatchSource(File file) {
this(ArgumentMatchSourceType.File, file);
}
private ArgumentMatchSource(ArgumentMatchSourceType type, File file) {
if (type == ArgumentMatchSourceType.File && file == null)
throw new IllegalArgumentException("An argument match source of type File cannot have a null file.");
this.type = type;
this.file = file;
}
public ArgumentMatchSourceType getType() {
return type;
}
public File getFile() {
return file;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ArgumentMatchSource that = (ArgumentMatchSource) o;
return (type == that.type) && (file == null ? that.file == null : file.equals(that.file));
}
@Override
public int hashCode() {
int result = type != null ? type.hashCode() : 0;
result = 31 * result + (file != null ? file.hashCode() : 0);
return result;
}
/**
* Compares two sources, putting the command line first, then files.
*/
@Override
public int compareTo(ArgumentMatchSource that) {
int comp = this.type.compareTo(that.type);
if (comp != 0)
return comp;
File f1 = this.file;
File f2 = that.file;
if ((f1 == null) ^ (f2 == null)) {
// If one of the files is null and the other is not
// put the null file first
return f1 == null ? -1 : 1;
}
return f1 == null ? 0 : f1.compareTo(f2);
}
}

View File

@ -0,0 +1,32 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.commandline;
/**
* Type of where an argument match originated, via the commandline or a file.
*/
public enum ArgumentMatchSourceType {
CommandLine, File
}

View File

@ -37,7 +37,7 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
* Collection matches from argument definition to argument value.
* Package protected access is deliberate.
*/
Map<Integer,ArgumentMatch> argumentMatches = new TreeMap<Integer,ArgumentMatch>();
Map<ArgumentMatchSite,ArgumentMatch> argumentMatches = new TreeMap<ArgumentMatchSite,ArgumentMatch>();
/**
* Provide a place to put command-line argument values that don't seem to belong to
@ -80,7 +80,7 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
* @param site Site at which to check.
* @return True if the site has a match. False otherwise.
*/
boolean hasMatch( int site ) {
boolean hasMatch( ArgumentMatchSite site ) {
return argumentMatches.containsKey( site );
}
@ -90,7 +90,7 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
* @return The match present at the given site.
* @throws IllegalArgumentException if site does not contain a match.
*/
ArgumentMatch getMatch( int site ) {
ArgumentMatch getMatch( ArgumentMatchSite site ) {
if( !argumentMatches.containsKey(site) )
throw new IllegalArgumentException( "Site does not contain an argument: " + site );
return argumentMatches.get(site);
@ -107,6 +107,7 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
/**
* Return all argument matches of this source.
* @param parsingEngine Parsing engine.
* @param argumentSource Argument source to match.
* @return List of all matches.
*/
@ -167,6 +168,7 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
* TODO: Generify this.
* @param multiplexer Multiplexer that controls the transformation process.
* @param key Key which specifies the transform.
* @return new argument matches.
*/
ArgumentMatches transform(Multiplexer multiplexer, Object key) {
ArgumentMatches newArgumentMatches = new ArgumentMatches();
@ -187,15 +189,15 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
for( ArgumentMatch argumentMatch: getUniqueMatches() ) {
if( argumentMatch.definition == match.definition && argumentMatch.tags.equals(match.tags) ) {
argumentMatch.mergeInto( match );
for( int index: match.indices.keySet() )
argumentMatches.put( index, argumentMatch );
for( ArgumentMatchSite site: match.sites.keySet() )
argumentMatches.put( site, argumentMatch );
definitionExists = true;
}
}
if( !definitionExists ) {
for( int index: match.indices.keySet() )
argumentMatches.put( index, match );
for( ArgumentMatchSite site: match.sites.keySet() )
argumentMatches.put( site, match );
}
}

View File

@ -35,10 +35,7 @@ import org.broadinstitute.sting.utils.help.ApplicationDetails;
import org.broadinstitute.sting.utils.help.HelpFormatter;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Locale;
import java.util.*;
public abstract class CommandLineProgram {
@ -155,6 +152,7 @@ public abstract class CommandLineProgram {
*
* @param clp the command line program to execute
* @param args the command line arguments passed in
* @param dryRun dry run
* @throws Exception when an exception occurs
*/
@SuppressWarnings("unchecked")
@ -176,6 +174,8 @@ public abstract class CommandLineProgram {
ParsingEngine parser = clp.parser = new ParsingEngine(clp);
parser.addArgumentSource(clp.getClass());
Map<ArgumentMatchSource, List<String>> parsedArgs;
// process the args
if (clp.canAddArgumentsDynamically()) {
// if the command-line program can toss in extra args, fetch them and reparse the arguments.
@ -196,14 +196,14 @@ public abstract class CommandLineProgram {
Class[] argumentSources = clp.getArgumentSources();
for (Class argumentSource : argumentSources)
parser.addArgumentSource(clp.getArgumentSourceName(argumentSource), argumentSource);
parser.parse(args);
parsedArgs = parser.parse(args);
if (isHelpPresent(parser))
printHelpAndExit(clp, parser);
if ( ! dryRun ) parser.validate();
} else {
parser.parse(args);
parsedArgs = parser.parse(args);
if ( ! dryRun ) {
if (isHelpPresent(parser))
@ -230,7 +230,7 @@ public abstract class CommandLineProgram {
}
// regardless of what happens next, generate the header information
HelpFormatter.generateHeaderInformation(clp.getApplicationDetails(), args);
HelpFormatter.generateHeaderInformation(clp.getApplicationDetails(), parsedArgs);
// call the execute
CommandLineProgram.result = clp.execute();

View File

@ -26,6 +26,7 @@
package org.broadinstitute.sting.commandline;
import com.google.java.contract.Requires;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.classloader.JVMUtils;
@ -35,6 +36,8 @@ import org.broadinstitute.sting.utils.exceptions.UserException;
import org.broadinstitute.sting.utils.help.ApplicationDetails;
import org.broadinstitute.sting.utils.help.HelpFormatter;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.*;
@ -100,6 +103,8 @@ public class ParsingEngine {
if(clp != null)
argumentTypeDescriptors.addAll(clp.getArgumentTypeDescriptors());
argumentTypeDescriptors.addAll(STANDARD_ARGUMENT_TYPE_DESCRIPTORS);
addArgumentSource(ParsingEngineArgumentFiles.class);
}
/**
@ -148,21 +153,43 @@ public class ParsingEngine {
* command-line arguments to the arguments that are actually
* required.
* @param tokens Tokens passed on the command line.
* @return The parsed arguments by file.
*/
public void parse( String[] tokens ) {
public SortedMap<ArgumentMatchSource, List<String>> parse( String[] tokens ) {
argumentMatches = new ArgumentMatches();
SortedMap<ArgumentMatchSource, List<String>> parsedArgs = new TreeMap<ArgumentMatchSource, List<String>>();
int lastArgumentMatchSite = -1;
List<String> cmdLineTokens = Arrays.asList(tokens);
parse(ArgumentMatchSource.COMMAND_LINE, cmdLineTokens, argumentMatches, parsedArgs);
for( int i = 0; i < tokens.length; i++ ) {
String token = tokens[i];
ParsingEngineArgumentFiles argumentFiles = new ParsingEngineArgumentFiles();
// Load the arguments ONLY into the argument files.
// Validation may optionally run on the rest of the arguments.
loadArgumentsIntoObject(argumentFiles);
for (File file: argumentFiles.files) {
List<String> fileTokens = getArguments(file);
parse(new ArgumentMatchSource(file), fileTokens, argumentMatches, parsedArgs);
}
return parsedArgs;
}
private void parse(ArgumentMatchSource matchSource, List<String> tokens,
ArgumentMatches argumentMatches, SortedMap<ArgumentMatchSource, List<String>> parsedArgs) {
ArgumentMatchSite lastArgumentMatchSite = new ArgumentMatchSite(matchSource, -1);
int i = 0;
for (String token: tokens) {
// If the token is of argument form, parse it into its own argument match.
// Otherwise, pair it with the most recently used argument discovered.
ArgumentMatchSite site = new ArgumentMatchSite(matchSource, i);
if( isArgumentForm(token) ) {
ArgumentMatch argumentMatch = parseArgument( token, i );
ArgumentMatch argumentMatch = parseArgument( token, site );
if( argumentMatch != null ) {
argumentMatches.mergeInto( argumentMatch );
lastArgumentMatchSite = i;
lastArgumentMatchSite = site;
}
}
else {
@ -170,10 +197,31 @@ public class ParsingEngine {
!argumentMatches.getMatch(lastArgumentMatchSite).hasValueAtSite(lastArgumentMatchSite))
argumentMatches.getMatch(lastArgumentMatchSite).addValue( lastArgumentMatchSite, token );
else
argumentMatches.MissingArgument.addValue( i, token );
argumentMatches.MissingArgument.addValue( site, token );
}
i++;
}
parsedArgs.put(matchSource, tokens);
}
private List<String> getArguments(File file) {
try {
if (file.getAbsolutePath().endsWith(".list")) {
return getListArguments(file);
}
} catch (IOException e) {
throw new UserException.CouldNotReadInputFile(file, e);
}
throw new UserException.CouldNotReadInputFile(file, "file extension is not .list");
}
private List<String> getListArguments(File file) throws IOException {
ArrayList<String> argsList = new ArrayList<String>();
for (String line: FileUtils.readLines(file))
argsList.addAll(Arrays.asList(Utils.escapeExpressions(line)));
return argsList;
}
public enum ValidationType { MissingRequiredArgument,
@ -494,7 +542,7 @@ public class ParsingEngine {
* @param position The position of the token in question.
* @return ArgumentMatch associated with this token, or null if no match exists.
*/
private ArgumentMatch parseArgument( String token, int position ) {
private ArgumentMatch parseArgument( String token, ArgumentMatchSite position ) {
if( !isArgumentForm(token) )
throw new IllegalArgumentException( "Token is not recognizable as an argument: " + token );
@ -579,9 +627,21 @@ class UnmatchedArgumentException extends ArgumentException {
private static String formatArguments( ArgumentMatch invalidValues ) {
StringBuilder sb = new StringBuilder();
for( int index: invalidValues.indices.keySet() )
for( String value: invalidValues.indices.get(index) ) {
sb.append( String.format("%nInvalid argument value '%s' at position %d.", value, index) );
for( ArgumentMatchSite site: invalidValues.sites.keySet() )
for( String value: invalidValues.sites.get(site) ) {
switch (site.getSource().getType()) {
case CommandLine:
sb.append( String.format("%nInvalid argument value '%s' at position %d.",
value, site.getIndex()) );
break;
case File:
sb.append( String.format("%nInvalid argument value '%s' in file %s at position %d.",
value, site.getSource().getFile().getAbsolutePath(), site.getIndex()) );
break;
default:
throw new RuntimeException( String.format("Unexpected argument match source type: %s",
site.getSource().getType()));
}
if(value != null && Utils.dupString(' ',value.length()).equals(value))
sb.append(" Please make sure any line continuation backslashes on your command line are not followed by whitespace.");
}
@ -634,4 +694,13 @@ class UnknownEnumeratedValueException extends ArgumentException {
private static String formatArguments(ArgumentDefinition definition, String argumentPassed) {
return String.format("Invalid value %s specified for argument %s; valid options are (%s).", argumentPassed, definition.fullName, Utils.join(",",definition.validOptions));
}
}
}
/**
* Container class to store the list of argument files.
* The files will be parsed after the command line arguments.
*/
class ParsingEngineArgumentFiles {
@Argument(fullName = "arg_file", shortName = "args", doc = "Reads arguments from the specified file", required = false)
public List<File> files = new ArrayList<File>();
}

View File

@ -68,7 +68,7 @@ public abstract class ParsingMethod {
* @return An argument match. Definition field will be populated if a match was found or
* empty if no appropriate definition could be found.
*/
public ArgumentMatch match( ArgumentDefinitions definitions, String token, int position ) {
public ArgumentMatch match( ArgumentDefinitions definitions, String token, ArgumentMatchSite position ) {
// If the argument is valid, parse out the argument.
Matcher matcher = pattern.matcher(token);
@ -102,9 +102,7 @@ public abstract class ParsingMethod {
// Try to find a matching argument. If found, label that as the match. If not found, add the argument
// with a null definition.
ArgumentMatch argumentMatch = new ArgumentMatch(argument,argumentDefinition,position,tags);
return argumentMatch;
return new ArgumentMatch(argument,argumentDefinition,position,tags);
}
/**

View File

@ -220,12 +220,12 @@ public class GenomeAnalysisEngine {
ShardStrategy shardStrategy = getShardStrategy(readsDataSource,microScheduler.getReference(),intervals);
// execute the microscheduler, storing the results
Object result = microScheduler.execute(this.walker, shardStrategy);
return microScheduler.execute(this.walker, shardStrategy);
//monitor.stop();
//logger.info(String.format("Maximum heap size consumed: %d",monitor.getMaxMemoryUsed()));
return result;
//return result;
}
/**
@ -296,10 +296,14 @@ public class GenomeAnalysisEngine {
else if(WalkerManager.getDownsamplingMethod(walker) != null)
method = WalkerManager.getDownsamplingMethod(walker);
else
method = argCollection.getDefaultDownsamplingMethod();
method = GATKArgumentCollection.getDefaultDownsamplingMethod();
return method;
}
protected void setDownsamplingMethod(DownsamplingMethod method) {
argCollection.setDownsamplingMethod(method);
}
public BAQ.QualityMode getWalkerBAQQualityMode() { return WalkerManager.getBAQQualityMode(walker); }
public BAQ.ApplicationTime getWalkerBAQApplicationTime() { return WalkerManager.getBAQApplicationTime(walker); }
@ -389,7 +393,9 @@ public class GenomeAnalysisEngine {
/**
* Get the sharding strategy given a driving data source.
*
* @param readsDataSource readsDataSource
* @param drivingDataSource Data on which to shard.
* @param intervals intervals
* @return the sharding strategy
*/
protected ShardStrategy getShardStrategy(SAMDataSource readsDataSource, ReferenceSequenceFile drivingDataSource, GenomeLocSortedSet intervals) {
@ -426,7 +432,7 @@ public class GenomeAnalysisEngine {
return new MonolithicShardStrategy(getGenomeLocParser(), readsDataSource,shardType,region);
}
ShardStrategy shardStrategy = null;
ShardStrategy shardStrategy;
ShardStrategyFactory.SHATTER_STRATEGY shardType;
long SHARD_SIZE = 100000L;
@ -435,6 +441,8 @@ public class GenomeAnalysisEngine {
if (walker instanceof RodWalker) SHARD_SIZE *= 1000;
if (intervals != null && !intervals.isEmpty()) {
if (readsDataSource == null)
throw new IllegalArgumentException("readsDataSource is null");
if(!readsDataSource.isEmpty() && readsDataSource.getSortOrder() != SAMFileHeader.SortOrder.coordinate)
throw new UserException.MissortedBAM(SAMFileHeader.SortOrder.coordinate, "Locus walkers can only traverse coordinate-sorted data. Please resort your input BAM file(s) or set the Sort Order tag in the header appropriately.");
@ -498,7 +506,8 @@ public class GenomeAnalysisEngine {
*/
private void initializeTempDirectory() {
File tempDir = new File(System.getProperty("java.io.tmpdir"));
tempDir.mkdirs();
if (!tempDir.exists() && !tempDir.mkdirs())
throw new UserException.BadTmpDir("Unable to create directory");
}
/**
@ -729,6 +738,7 @@ public class GenomeAnalysisEngine {
* @param reads Reads data source.
* @param reference Reference data source.
* @param rods a collection of the reference ordered data tracks
* @param manager manager
*/
private void validateSourcesAgainstReference(SAMDataSource reads, ReferenceSequenceFile reference, Collection<ReferenceOrderedDataSource> rods, RMDTrackBuilder manager) {
if ((reads.isEmpty() && (rods == null || rods.isEmpty())) || reference == null )
@ -757,15 +767,22 @@ public class GenomeAnalysisEngine {
/**
* Gets a data source for the given set of reads.
*
* @param argCollection arguments
* @param genomeLocParser parser
* @param refReader reader
* @return A data source for the given set of reads.
*/
private SAMDataSource createReadsDataSource(GATKArgumentCollection argCollection, GenomeLocParser genomeLocParser, IndexedFastaSequenceFile refReader) {
DownsamplingMethod method = getDownsamplingMethod();
// Synchronize the method back into the collection so that it shows up when
// interrogating for the downsample method during command line recreation.
setDownsamplingMethod(method);
if ( getWalkerBAQApplicationTime() == BAQ.ApplicationTime.FORBIDDEN && argCollection.BAQMode != BAQ.CalculationMode.OFF)
throw new UserException.BadArgumentValue("baq", "Walker cannot accept BAQ'd base qualities, and yet BAQ mode " + argCollection.BAQMode + " was requested.");
SAMDataSource dataSource = new SAMDataSource(
return new SAMDataSource(
samReaderIDs,
genomeLocParser,
argCollection.useOriginalBaseQualities,
@ -781,14 +798,12 @@ public class GenomeAnalysisEngine {
refReader,
argCollection.defaultBaseQualities,
!argCollection.disableLowMemorySharding);
return dataSource;
}
/**
* Opens a reference sequence file paired with an index. Only public for testing purposes
*
* @param refFile Handle to a reference sequence file. Non-null.
* @return A thread-safe file wrapper.
*/
public void setReferenceDataSource(File refFile) {
this.referenceDataSource = new ReferenceDataSource(refFile);

View File

@ -135,8 +135,8 @@ public class GATKArgumentCollection {
/**
* Gets the downsampling method explicitly specified by the user. If the user didn't specify
* a default downsampling mechanism, return null.
* @return The explicitly specified downsampling mechanism, or null if none exists.
* a default downsampling mechanism, return the default.
* @return The explicitly specified downsampling mechanism, or the default if none exists.
*/
public DownsamplingMethod getDownsamplingMethod() {
if(downsamplingType == null && downsampleFraction == null && downsampleCoverage == null)
@ -146,6 +146,18 @@ public class GATKArgumentCollection {
return new DownsamplingMethod(downsamplingType,downsampleCoverage,downsampleFraction);
}
/**
* Set the downsampling method stored in the argument collection so that it is read back out when interrogating the command line arguments.
* @param method The downsampling mechanism.
*/
public void setDownsamplingMethod(DownsamplingMethod method) {
if (method == null)
throw new IllegalArgumentException("method is null");
downsamplingType = method.type;
downsampleCoverage = method.toCoverage;
downsampleFraction = method.toFraction;
}
// --------------------------------------------------------------------------------------------------------------
//
// BAQ arguments

View File

@ -29,6 +29,7 @@ import org.apache.log4j.Logger;
import org.broadinstitute.sting.commandline.ArgumentDefinition;
import org.broadinstitute.sting.commandline.ArgumentDefinitionGroup;
import org.broadinstitute.sting.commandline.ArgumentDefinitions;
import org.broadinstitute.sting.commandline.ArgumentMatchSource;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.text.TextFormattingUtils;
@ -47,6 +48,7 @@ public class HelpFormatter {
/**
* Prints the help, given a collection of argument definitions.
* @param applicationDetails Application details
* @param argumentDefinitions Argument definitions for which help should be printed.
*/
public void printHelp( ApplicationDetails applicationDetails, ArgumentDefinitions argumentDefinitions ) {
@ -233,7 +235,7 @@ public class HelpFormatter {
private List<ArgumentDefinitionGroup> prepareArgumentGroups( ArgumentDefinitions argumentDefinitions ) {
// Sort the list of argument definitions according to how they should be shown.
// Put the sorted results into a new cloned data structure.
Comparator definitionComparator = new Comparator<ArgumentDefinition>() {
Comparator<ArgumentDefinition> definitionComparator = new Comparator<ArgumentDefinition>() {
public int compare( ArgumentDefinition lhs, ArgumentDefinition rhs ) {
if( lhs.required && rhs.required ) return 0;
if( lhs.required ) return -1;
@ -242,15 +244,15 @@ public class HelpFormatter {
}
};
List<ArgumentDefinitionGroup> argumentGroups = new ArrayList();
List<ArgumentDefinitionGroup> argumentGroups = new ArrayList<ArgumentDefinitionGroup>();
for( ArgumentDefinitionGroup argumentGroup: argumentDefinitions.getArgumentDefinitionGroups() ) {
List<ArgumentDefinition> sortedDefinitions = new ArrayList( argumentGroup.argumentDefinitions );
List<ArgumentDefinition> sortedDefinitions = new ArrayList<ArgumentDefinition>( argumentGroup.argumentDefinitions );
Collections.sort( sortedDefinitions, definitionComparator );
argumentGroups.add( new ArgumentDefinitionGroup(argumentGroup.groupName,sortedDefinitions) );
}
// Sort the argument groups themselves with main arguments first, followed by plugins sorted in name order.
Comparator groupComparator = new Comparator<ArgumentDefinitionGroup>() {
Comparator<ArgumentDefinitionGroup> groupComparator = new Comparator<ArgumentDefinitionGroup>() {
public int compare( ArgumentDefinitionGroup lhs, ArgumentDefinitionGroup rhs ) {
if( lhs.groupName == null && rhs.groupName == null ) return 0;
if( lhs.groupName == null ) return -1;
@ -271,9 +273,9 @@ public class HelpFormatter {
* Generate a standard header for the logger
*
* @param applicationDetails details of the application to run.
* @param args the command line arguments passed in
* @param parsedArgs the command line arguments passed in
*/
public static void generateHeaderInformation(ApplicationDetails applicationDetails, String[] args) {
public static void generateHeaderInformation(ApplicationDetails applicationDetails, Map<ArgumentMatchSource, List<String>> parsedArgs) {
DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
java.util.Date date = new java.util.Date();
@ -283,11 +285,22 @@ public class HelpFormatter {
logger.info(barrier);
for (String headerLine : applicationDetails.applicationHeader)
logger.info(headerLine);
String output = "";
for (String str : args) {
output = output + str + " ";
logger.debug("Current directory: " + System.getProperty("user.dir"));
for (Map.Entry<ArgumentMatchSource, List<String>> entry: parsedArgs.entrySet()) {
ArgumentMatchSource matchSource = entry.getKey();
final String sourceName;
switch (matchSource.getType()) {
case CommandLine: sourceName = "Program"; break;
case File: sourceName = matchSource.getFile().getPath(); break;
default: throw new RuntimeException("Unexpected argument match source type: " + matchSource.getType());
}
String output = sourceName + " Args:";
for (String str : entry.getValue()) {
output = output + " " + str;
}
logger.info(output);
}
logger.info("Program Args: " + output);
logger.info("Date/Time: " + dateFormat.format(date));
logger.info(barrier);

View File

@ -0,0 +1,79 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.commandline;
import org.testng.Assert;
import org.testng.annotations.Test;
import java.io.File;
public class ArgumentMatchSiteUnitTest {
@Test
public void testCommandLine() {
ArgumentMatchSite site = new ArgumentMatchSite(ArgumentMatchSource.COMMAND_LINE, 1);
Assert.assertEquals(site.getSource(), ArgumentMatchSource.COMMAND_LINE);
Assert.assertEquals(site.getIndex(), 1);
}
@Test
public void testFile() {
ArgumentMatchSource source = new ArgumentMatchSource(new File("test"));
ArgumentMatchSite site = new ArgumentMatchSite(source, 1);
Assert.assertEquals(site.getSource(), source);
Assert.assertEquals(site.getIndex(), 1);
}
@Test
public void testEquals() {
ArgumentMatchSource cmdLine = ArgumentMatchSource.COMMAND_LINE;
ArgumentMatchSite site1 = new ArgumentMatchSite(cmdLine, 1);
ArgumentMatchSite site2 = new ArgumentMatchSite(cmdLine, 2);
Assert.assertFalse(site1.equals(null));
Assert.assertTrue(site1.equals(site1));
Assert.assertFalse(site1.equals(site2));
Assert.assertFalse(site2.equals(site1));
Assert.assertTrue(site2.equals(site2));
}
@Test
public void testCompareTo() {
ArgumentMatchSource cmdLine = ArgumentMatchSource.COMMAND_LINE;
ArgumentMatchSite site1 = new ArgumentMatchSite(cmdLine, 1);
ArgumentMatchSite site2 = new ArgumentMatchSite(cmdLine, 2);
Assert.assertTrue(site1.compareTo(site1) == 0);
Assert.assertTrue(site1.compareTo(site2) < 0);
Assert.assertTrue(site2.compareTo(site1) > 0);
Assert.assertTrue(site2.compareTo(site2) == 0);
}
@Test(expectedExceptions = NullPointerException.class)
public void testCompareToNull() {
new ArgumentMatchSite(ArgumentMatchSource.COMMAND_LINE, 0).compareTo(null);
}
}

View File

@ -0,0 +1,98 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.commandline;
import org.broadinstitute.sting.BaseTest;
import org.testng.Assert;
import org.testng.annotations.Test;
import java.io.File;
public class ArgumentMatchSourceUnitTest extends BaseTest {
@Test
public void testCommandLine() {
ArgumentMatchSource source = ArgumentMatchSource.COMMAND_LINE;
Assert.assertEquals(source.getType(), ArgumentMatchSourceType.CommandLine);
Assert.assertNull(source.getFile());
}
@Test
public void testFile() {
File f = new File("test");
ArgumentMatchSource source = new ArgumentMatchSource(f);
Assert.assertEquals(source.getType(), ArgumentMatchSourceType.File);
Assert.assertEquals(source.getFile(), f);
}
@Test(expectedExceptions = IllegalArgumentException.class)
public void testNullFile() {
new ArgumentMatchSource(null);
}
@Test
public void testEquals() {
ArgumentMatchSource cmdLine = ArgumentMatchSource.COMMAND_LINE;
ArgumentMatchSource fileA = new ArgumentMatchSource(new File("a"));
ArgumentMatchSource fileB = new ArgumentMatchSource(new File("b"));
Assert.assertFalse(cmdLine.equals(null));
Assert.assertTrue(cmdLine.equals(cmdLine));
Assert.assertFalse(cmdLine.equals(fileA));
Assert.assertFalse(cmdLine.equals(fileB));
Assert.assertFalse(fileA.equals(cmdLine));
Assert.assertTrue(fileA.equals(fileA));
Assert.assertFalse(fileA.equals(fileB));
Assert.assertFalse(fileB.equals(cmdLine));
Assert.assertFalse(fileB.equals(fileA));
Assert.assertTrue(fileB.equals(fileB));
}
@Test
public void testCompareTo() {
ArgumentMatchSource cmdLine = ArgumentMatchSource.COMMAND_LINE;
ArgumentMatchSource fileA = new ArgumentMatchSource(new File("a"));
ArgumentMatchSource fileB = new ArgumentMatchSource(new File("b"));
Assert.assertTrue(cmdLine.compareTo(cmdLine) == 0);
Assert.assertTrue(cmdLine.compareTo(fileA) < 0);
Assert.assertTrue(cmdLine.compareTo(fileB) < 0);
Assert.assertTrue(fileA.compareTo(cmdLine) > 0);
Assert.assertTrue(fileA.compareTo(fileA) == 0);
Assert.assertTrue(fileA.compareTo(fileB) < 0);
Assert.assertTrue(fileB.compareTo(cmdLine) > 0);
Assert.assertTrue(fileB.compareTo(fileA) > 0);
Assert.assertTrue(fileB.compareTo(fileB) == 0);
}
@Test(expectedExceptions = NullPointerException.class)
public void testCompareToNull() {
ArgumentMatchSource.COMMAND_LINE.compareTo(null);
}
}

View File

@ -25,6 +25,7 @@
package org.broadinstitute.sting.commandline;
import org.apache.commons.io.FileUtils;
import org.broad.tribble.Feature;
import org.broadinstitute.sting.utils.exceptions.UserException;
import org.broadinstitute.sting.utils.variantcontext.VariantContext;
@ -34,6 +35,8 @@ import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.EnumSet;
/**
@ -493,6 +496,7 @@ public class ParsingEngineUnitTest extends BaseTest {
Assert.assertNotNull(definition, "Invalid default argument name assigned");
}
@SuppressWarnings("unused")
private class CamelCaseArgProvider {
@Argument(doc="my arg")
Integer myArg;
@ -507,6 +511,7 @@ public class ParsingEngineUnitTest extends BaseTest {
parsingEngine.validate();
}
@SuppressWarnings("unused")
private class BooleanArgProvider {
@Argument(doc="my bool")
boolean myBool;
@ -561,6 +566,7 @@ public class ParsingEngineUnitTest extends BaseTest {
parsingEngine.validate();
}
@SuppressWarnings("unused")
private class MutuallyExclusiveArgProvider {
@Argument(doc="foo",exclusiveOf="bar")
Integer foo;
@ -618,6 +624,7 @@ public class ParsingEngineUnitTest extends BaseTest {
parsingEngine.addArgumentSource( MultipleArgumentCollectionProvider.class );
}
@SuppressWarnings("unused")
private class MultipleArgumentCollectionProvider {
@ArgumentCollection
RequiredArgProvider rap1 = new RequiredArgProvider();
@ -937,4 +944,23 @@ public class ParsingEngineUnitTest extends BaseTest {
VariantContextRodBindingArgProvider argProvider = new VariantContextRodBindingArgProvider();
parsingEngine.loadArgumentsIntoObject( argProvider );
}
@Test
public void argumentListTest() throws IOException {
File argsFile = BaseTest.createTempFile("args.", ".list");
try {
FileUtils.write(argsFile, "-I na12878.bam");
final String[] commandLine = new String[] {"-args", argsFile.getPath()};
parsingEngine.addArgumentSource(InputFileArgProvider.class);
parsingEngine.parse(commandLine);
parsingEngine.validate();
InputFileArgProvider argProvider = new InputFileArgProvider();
parsingEngine.loadArgumentsIntoObject(argProvider);
Assert.assertEquals(argProvider.inputFile, "na12878.bam", "Argument is not correctly initialized");
} finally {
FileUtils.deleteQuietly(argsFile);
}
}
}