zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' of ssh://gsa2/humgen/gsa-scr1/gsa-engineering/git/unstable

This commit is contained in:
Eric Banks 2012-10-19 12:01:49 -04:00
parent d3cf37dfaf 403654d40a
commit deca564aef
61 changed files with 1460 additions and 375 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.gitignore vendored
View File

@ -18,10 +18,8 @@ queueScatterGather
/bar*
integrationtests/
public/testdata/onTheFlyOutputTest.vcf
private/testdata/onTheFlyOutputTest.vcf
lib
html
gatkdocs
dist
build
resources
build/
dist/
dump/
lib/
out/

42
build.xml
View File

@ -22,7 +22,9 @@
~ OTHER DEALINGS IN THE SOFTWARE.
-->
<project name="Sting" default="dist" basedir="." xmlns:ivy="antlib:org.apache.ivy.ant">
<project name="Sting" default="dist" basedir="."
xmlns:artifact="antlib:org.apache.maven.artifact.ant"
xmlns:ivy="antlib:org.apache.ivy.ant">
<description>Compile and distribute the Sting toolkit</description>
<!-- ******************************************************************************** -->
@ -250,11 +252,14 @@
<property name="ivy.jar.file" value="ivy-${ivy.install.version}.jar"/>
<property name="ivy.settings.dir" value="settings"/>
<property file="${ivy.settings.dir}/ivysettings.properties"/>
<property name="maven-ant-tasks.install.version" value="2.1.3"/>
<property name="maven-ant-tasks.jar.file" value="maven-ant-tasks-${maven-ant-tasks.install.version}.jar"/>
<mkdir dir="${lib.dir}"/>
<mkdir dir="${ivy.jar.dir}"/>
<!-- Comment out the following two lines to build the GATK without a network connection, assuming you have all of the libraries cached already -->
<!-- Comment out the following lines to build the GATK without a network connection, assuming you have all of the libraries cached already -->
<get src="http://repo1.maven.org/maven2/org/apache/ivy/ivy/${ivy.install.version}/${ivy.jar.file}"
dest="${ivy.jar.dir}/${ivy.jar.file}"
usetimestamp="true"/>
@ -262,6 +267,15 @@
uri="antlib:org.apache.ivy.ant"
classpath="${ivy.jar.dir}/${ivy.jar.file}"/>
<get src="http://repo1.maven.org/maven2/org/apache/maven/maven-ant-tasks/${maven-ant-tasks.install.version}/${maven-ant-tasks.jar.file}"
dest="${ivy.jar.dir}/${maven-ant-tasks.jar.file}"
usetimestamp="true"/>
<taskdef resource="org/apache/maven/artifact/ant/antlib.xml"
uri="antlib:antlib:org.apache.maven.artifact.ant"
classpath="${ivy.jar.dir}/${maven-ant-tasks.jar.file}"/>
<!-- End network lines -->
<ivy:settings file="${ivy.settings.dir}/ivysettings.xml"/>
<property name="init.resolve.done" value="true"/>
</target>
@ -295,7 +309,7 @@
<target name="git.rev-parse" depends="git.describe" unless="git.describe.succeeded">
<exec executable="git" outputproperty="git.rev-parse.output" resultproperty="git.rev-parse.exit.value" failonerror="false">
<arg line="rev-parse HEAD" />
<arg line="rev-parse --short HEAD" />
</exec>
<condition property="git.rev-parse.succeeded">
<equals arg1="${git.rev-parse.exit.value}" arg2="0" />
@ -942,6 +956,28 @@
</ant>
</target>
<!-- Maven install a package consisting of all supporting files. Don't call this target directly. Call one of the specific packaging targets below -->
<target name="mvninstall" depends="package" description="maven install a package into .m2/repository">
<property name="mvn.build.version" value="0.0.1" />
<!--
We should use the build version or better yet a git tag version, but tags are currently missing. Alternatively how do we then depend on the LATEST?
<property name="mvn.build.version" value="${build.version}" />
-->
<artifact:pom id="${executable}.pom" groupId="org.broadinstitute.sting" artifactId="${executable}" version="${mvn.build.version}" name="${executable}" />
<artifact:writepom pomRefId="${executable}.pom" file="${package.output.dir}/${executable}-${build.version}/${executable}.pom.xml"/>
<artifact:install file="${package.output.dir}/${executable}-${build.version}/${executable}.jar">
<artifact:pom file="${package.output.dir}/${executable}-${build.version}/${executable}.pom.xml" />
</artifact:install>
</target>
<!-- Maven install specific versions of the GATK/Queue. ALWAYS do an ant clean before invoking these! -->
<target name="mvninstall.gatk.full" depends="package.gatk.full,mvninstall" />
<target name="mvninstall.gatk.lite" depends="package.gatk.lite,mvninstall" />
<target name="mvninstall.queue.full" depends="package.queue.full,mvninstall" />
<target name="mvninstall.queue.lite" depends="package.queue.lite,mvninstall" />
<!-- ******************************************************************************** -->
<!-- Clean -->

BIN
intellij_example.tar.bz2 100644
View File

Binary file not shown.

7
ivy.xml
View File

@ -46,7 +46,8 @@
<dependency org="org.apache.bcel" name="bcel" rev="5.2"/>
<!-- Dependencies for reflections mvn repository -->
<dependency org="org.reflections" name="reflections" rev="0.9.5-RC2"/>
<dependency org="org.reflections" name="reflections" rev="0.9.8"/>
<dependency org="org.slf4j" name="slf4j-log4j12" rev="1.6.1"/>
<!-- Matrix package from math.nist.gov -->
<dependency org="gov.nist" name="Jama" rev="1.0.2"/>
@ -78,8 +79,8 @@
<dependency org="net.sf.gridscheduler" name="drmaa" rev="latest.integration"/>
<!-- Scala dependancies -->
<dependency org="org.scala-lang" name="scala-compiler" rev="2.8.1"/>
<dependency org="org.scala-lang" name="scala-library" rev="2.8.1"/>
<dependency org="org.scala-lang" name="scala-compiler" rev="2.9.2"/>
<dependency org="org.scala-lang" name="scala-library" rev="2.9.2"/>
<!-- testing and evaluation dependencies -->
<dependency org="org.testng" name="testng" rev="5.14.1" conf="test"/>

17
protected/java/src/org/broadinstitute/sting/gatk/walkers/compression/reducereads/ReduceReads.java
View File

@ -181,15 +181,6 @@ public class ReduceReads extends ReadWalker<LinkedList<GATKSAMRecord>, ReduceRea
@Argument(fullName = "minimum_del_proportion_to_trigger_variant", shortName = "mindel", doc = "", required = false)
private double minIndelProportionToTriggerVariant = 0.05;
/**
* Minimum proportion of indels in a site to trigger a variant region. Anything below this will be
* considered consensus.
*/
@Argument(fullName = "contigs", shortName = "ctg", doc = "", required = false)
private int nContigs = 2;
/**
* Downsamples the coverage of a variable region approximately (guarantees the minimum to be equal to this).
* A value of 0 turns downsampling off.
@ -197,6 +188,14 @@ public class ReduceReads extends ReadWalker<LinkedList<GATKSAMRecord>, ReduceRea
@Argument(fullName = "downsample_coverage", shortName = "ds", doc = "", required = false)
private int downsampleCoverage = 250;
/**
* Number of chromossomes in the sample (this is used for the polyploid consensus compression). Only
* tested for humans (or organisms with n=2). Use at your own risk!
*/
@Hidden
@Argument(fullName = "contigs", shortName = "ctg", doc = "", required = false)
private int nContigs = 2;
@Hidden
@Argument(fullName = "", shortName = "dl", doc = "", required = false)
private int debugLevel = 0;

21
protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalcPerformanceTest.java
View File

@ -10,6 +10,7 @@ import org.broadinstitute.sting.utils.MathUtils;
import org.broadinstitute.sting.utils.SimpleTimer;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.fasta.CachingIndexedFastaSequenceFile;
import org.broadinstitute.sting.utils.variantcontext.Allele;
import org.broadinstitute.sting.utils.variantcontext.Genotype;
import org.broadinstitute.sting.utils.variantcontext.VariantContext;
import org.broadinstitute.sting.utils.variantcontext.VariantContextBuilder;
@ -225,12 +226,24 @@ public class AFCalcPerformanceTest {
final AFCalcTestBuilder testBuilder = new AFCalcTestBuilder(call.vc.getNSamples(), 1,
AFCalcFactory.Calculation.EXACT_INDEPENDENT,
AFCalcTestBuilder.PriorType.human);
logger.info(call);
final SimpleTimer timer = new SimpleTimer().start();
final AFCalcResult result = testBuilder.makeModel().getLog10PNonRef(call.vc, testBuilder.makePriors());
call.newNanoTime = timer.getElapsedTimeNano();
call.newPNonRef = result.getLog10PosteriorOfAFGT0();
logger.info(call);
logger.info("\t\t" + result);
final long newNanoTime = timer.getElapsedTimeNano();
if ( call.originalCall.anyPolymorphic(-1) || result.anyPolymorphic(-1) ) {
logger.info("**** ONE IS POLY");
}
logger.info("\t\t getLog10PosteriorOfAFGT0: " + call.originalCall.getLog10PosteriorOfAFGT0() + " vs " + result.getLog10PosteriorOfAFGT0());
final double speedup = call.runtime / (1.0 * newNanoTime);
logger.info("\t\t runtime: " + call.runtime + " vs " + newNanoTime + " speedup " + String.format("%.2f", speedup) + "x");
for ( final Allele a : call.originalCall.getAllelesUsedInGenotyping() ) {
if ( a.isNonReference() ) {
final String warningmeMLE = call.originalCall.getAlleleCountAtMLE(a) != result.getAlleleCountAtMLE(a) ? " DANGER-MLE-DIFFERENT" : "";
logger.info("\t\t MLE " + a + ": " + call.originalCall.getAlleleCountAtMLE(a) + " vs " + result.getAlleleCountAtMLE(a) + warningmeMLE);
final String warningmePost = call.originalCall.getLog10PosteriorOfAFGt0ForAllele(a) == 0 && result.getLog10PosteriorOfAFGt0ForAllele(a) < -10 ? " DANGER-POSTERIORS-DIFFERENT" : "";
logger.info("\t\t Posterior " + a + ": " + call.originalCall.getLog10PosteriorOfAFGt0ForAllele(a) + " vs " + result.getLog10PosteriorOfAFGt0ForAllele(a) + warningmePost);
}
}
}
}

6
protected/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalcTestBuilder.java
View File

@ -45,12 +45,16 @@ public class AFCalcTestBuilder {
human
}
public int getNumAltAlleles() {
return numAltAlleles;
}
public int getnSamples() {
return nSamples;
}
public AFCalc makeModel() {
return AFCalcFactory.createAFCalc(modelType, nSamples, 4, 4, 2);
return AFCalcFactory.createAFCalc(modelType, nSamples, getNumAltAlleles(), getNumAltAlleles(), 2);
}
public double[] makePriors() {

4
protected/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperGeneralPloidyIntegrationTest.java
View File

@ -70,12 +70,12 @@ public class UnifiedGenotyperGeneralPloidyIntegrationTest extends WalkerTest {
@Test(enabled = true)
public void testINDEL_maxAltAlleles2_ploidy3_Pools_noRef() {
PC_LSV_Test_NoRef(" -maxAltAlleles 2 -ploidy 3","LSV_INDEL_DISC_NOREF_p3","INDEL","7d6f319b9edcb1ff8c290fef150a2df8");
PC_LSV_Test_NoRef(" -maxAltAlleles 2 -ploidy 3","LSV_INDEL_DISC_NOREF_p3","INDEL","9acfe0019efdc91217ee070acb071228");
}
@Test(enabled = true)
public void testINDEL_maxAltAlleles2_ploidy1_Pools_noRef() {
PC_LSV_Test_NoRef(" -maxAltAlleles 2 -ploidy 1","LSV_INDEL_DISC_NOREF_p1","INDEL","dd02890123e07e7412a49475cb6280f1");
PC_LSV_Test_NoRef(" -maxAltAlleles 2 -ploidy 1","LSV_INDEL_DISC_NOREF_p1","INDEL","c1d4dd793f61710a1b1fc5d82803210f");
}
@Test(enabled = true)

87
protected/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalcPerformanceUnitTest.java 100644
View File

@ -0,0 +1,87 @@
package org.broadinstitute.sting.gatk.walkers.genotyper.afcalc;
import org.broadinstitute.sting.BaseTest;
import org.broadinstitute.sting.utils.MathUtils;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.collections.Pair;
import org.broadinstitute.sting.utils.variantcontext.Allele;
import org.broadinstitute.sting.utils.variantcontext.VariantContext;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
public class AFCalcPerformanceUnitTest extends BaseTest {
@DataProvider(name = "ScalingTests")
public Object[][] makepolyTestProviderLotsOfAlleles() {
List<Object[]> tests = new ArrayList<Object[]>();
// list of all high-quality models in the system
final List<AFCalcFactory.Calculation> biAllelicModels = Arrays.asList(
AFCalcFactory.Calculation.EXACT_INDEPENDENT,
AFCalcFactory.Calculation.EXACT_REFERENCE);
final List<AFCalcFactory.Calculation> multiAllelicModels = Arrays.asList(
AFCalcFactory.Calculation.EXACT_INDEPENDENT);
// for ( final int nonTypePLs : Arrays.asList(100) ) {
// for ( final int nSamples : Arrays.asList(10000) ) {
// final List<Integer> alleleCounts = Arrays.asList(50);
// for ( final int nAltAlleles : Arrays.asList(1) ) {
for ( final int nonTypePLs : Arrays.asList(100) ) {
for ( final int nSamples : Arrays.asList(100, 1000) ) {
final List<Integer> alleleCounts = Arrays.asList(0, 1, 2, 3, 4, 5, 10, 50, 500);
for ( final int nAltAlleles : Arrays.asList(1, 2, 3) ) {
final List<AFCalcFactory.Calculation> models = nAltAlleles > 1 ? multiAllelicModels : biAllelicModels;
for ( final AFCalcFactory.Calculation model : models ) {
for ( final List<Integer> ACs : Utils.makePermutations(alleleCounts, nAltAlleles, true) ) {
if ( MathUtils.sum(ACs) < nSamples * 2 ) {
final AFCalcTestBuilder testBuilder
= new AFCalcTestBuilder(nSamples, nAltAlleles, model, AFCalcTestBuilder.PriorType.human);
tests.add(new Object[]{testBuilder, ACs, nonTypePLs});
}
}
}
}
}
}
return tests.toArray(new Object[][]{});
}
private Pair<Integer, Integer> estNumberOfEvaluations(final AFCalcTestBuilder testBuilder, final VariantContext vc, final int nonTypePL) {
final int evalOverhead = 2; // 2
final int maxEvalsPerSamplePerAC = 3;
int minEvals = 0, maxEvals = 0;
for ( final Allele alt : vc.getAlternateAlleles() ) {
final int AC = vc.getCalledChrCount(alt);
minEvals += AC + evalOverhead; // everyone is hom-var
maxEvals += AC * maxEvalsPerSamplePerAC + 10;
}
return new Pair<Integer, Integer>(minEvals, maxEvals);
}
@Test(dataProvider = "ScalingTests")
private void testScaling(final AFCalcTestBuilder testBuilder, final List<Integer> ACs, final int nonTypePL) {
final AFCalc calc = testBuilder.makeModel();
final double[] priors = testBuilder.makePriors();
final VariantContext vc = testBuilder.makeACTest(ACs, 0, nonTypePL);
final AFCalcResult result = calc.getLog10PNonRef(vc, priors);
final Pair<Integer, Integer> expectedNEvaluation = estNumberOfEvaluations(testBuilder, vc, nonTypePL);
final int minEvals = expectedNEvaluation.getFirst();
final int maxEvals = expectedNEvaluation.getSecond();
logger.warn(" min " + minEvals + " obs " + result.getnEvaluations() + " max " + maxEvals + " for test " + testBuilder + " sum(ACs)=" + (int)MathUtils.sum(ACs));
Assert.assertTrue(result.getnEvaluations() >= minEvals,
"Actual number of evaluations " + result.getnEvaluations() + " < min number of evals " + minEvals);
Assert.assertTrue(result.getnEvaluations() <= maxEvals,
"Actual number of evaluations " + result.getnEvaluations() + " > max number of evals " + minEvals);
}
}

4
protected/java/test/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCallerIntegrationTest.java
View File

@ -21,7 +21,7 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
@Test
public void testHaplotypeCallerMultiSample() {
HCTest(CEUTRIO_BAM, "", "8c52c0955099cca3215a0d78fd455894");
HCTest(CEUTRIO_BAM, "", "75013fa6a884104f0b1797502b636698");
}
@Test
@ -31,7 +31,7 @@ public class HaplotypeCallerIntegrationTest extends WalkerTest {
@Test
public void testHaplotypeCallerMultiSampleGGA() {
HCTest(CEUTRIO_BAM, "--max_alternate_alleles_for_indels 3 -gt_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "combined.phase1.chr20.raw.indels.sites.vcf", "61c1a0fb62d909229af6b5a91dad8b35");
HCTest(CEUTRIO_BAM, "--max_alternate_alleles_for_indels 3 -gt_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "combined.phase1.chr20.raw.indels.sites.vcf", "3cd3363976b1937d801f9f82996f4abe");
}
private void HCTestComplexVariants(String bam, String args, String md5) {

28
public/java/src/org/broadinstitute/sting/commandline/ArgumentMatch.java
View File

@ -46,7 +46,7 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
/**
* Maps indices of command line arguments to values paired with that argument.
*/
public final SortedMap<ArgumentMatchSite,List<String>> sites = new TreeMap<ArgumentMatchSite,List<String>>();
public final SortedMap<ArgumentMatchSite,List<ArgumentMatchValue>> sites = new TreeMap<ArgumentMatchSite,List<ArgumentMatchValue>>();
/**
* An ordered, freeform collection of tags.
@ -90,11 +90,11 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
* @param value Value for the argument at this position.
* @param tags ordered freeform text tags associated with this argument.
*/
private ArgumentMatch(final String label, final ArgumentDefinition definition, final ArgumentMatchSite site, final String value, final Tags tags) {
private ArgumentMatch(final String label, final ArgumentDefinition definition, final ArgumentMatchSite site, final ArgumentMatchValue value, final Tags tags) {
this.label = label;
this.definition = definition;
ArrayList<String> values = new ArrayList<String>();
ArrayList<ArgumentMatchValue> values = new ArrayList<ArgumentMatchValue>();
if( value != null )
values.add(value);
sites.put(site,values );
@ -131,11 +131,11 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
*/
@SuppressWarnings("unchecked")
ArgumentMatch transform(Multiplexer multiplexer, Object key) {
SortedMap<ArgumentMatchSite,List<String>> newIndices = new TreeMap<ArgumentMatchSite,List<String>>();
for(Map.Entry<ArgumentMatchSite,List<String>> site: sites.entrySet()) {
List<String> newEntries = new ArrayList<String>();
for(String entry: site.getValue())
newEntries.add(multiplexer.transformArgument(key,entry));
SortedMap<ArgumentMatchSite,List<ArgumentMatchValue>> newIndices = new TreeMap<ArgumentMatchSite,List<ArgumentMatchValue>>();
for(Map.Entry<ArgumentMatchSite,List<ArgumentMatchValue>> site: sites.entrySet()) {
List<ArgumentMatchValue> newEntries = new ArrayList<ArgumentMatchValue>();
for(ArgumentMatchValue entry: site.getValue())
newEntries.add(new ArgumentMatchStringValue(multiplexer.transformArgument(key,entry.asString())));
newIndices.put(site.getKey(),newEntries);
}
ArgumentMatch newArgumentMatch = new ArgumentMatch(label,definition);
@ -165,7 +165,7 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
/**
* Iterate over each available token.
*/
private Iterator<String> tokenIterator = null;
private Iterator<ArgumentMatchValue> tokenIterator = null;
/**
* The next site to return. Null if none remain.
@ -175,7 +175,7 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
/**
* The next token to return. Null if none remain.
*/
String nextToken = null;
ArgumentMatchValue nextToken = null;
{
siteIterator = sites.keySet().iterator();
@ -254,9 +254,9 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
* @param site site of the command-line argument to which this value is mated.
* @param value Text representation of value to add.
*/
public void addValue( ArgumentMatchSite site, String value ) {
public void addValue( ArgumentMatchSite site, ArgumentMatchValue value ) {
if( !sites.containsKey(site) || sites.get(site) == null )
sites.put(site, new ArrayList<String>() );
sites.put(site, new ArrayList<ArgumentMatchValue>() );
sites.get(site).add(value);
}
@ -275,8 +275,8 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
* Return the values associated with this argument match.
* @return A collection of the string representation of these value.
*/
public List<String> values() {
List<String> values = new ArrayList<String>();
public List<ArgumentMatchValue> values() {
List<ArgumentMatchValue> values = new ArrayList<ArgumentMatchValue>();
for( ArgumentMatchSite site: sites.keySet() ) {
if( sites.get(site) != null )
values.addAll(sites.get(site));

27
public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchFileValue.java 100644
View File

@ -0,0 +1,27 @@
package org.broadinstitute.sting.commandline;
import java.io.File;
/**
* Holds a reference to a file as an argument match value.
*
* This is useful when the type of the stored file may be a subclass of java.io.File,
* for example a Queue RemoteFile.
*/
public class ArgumentMatchFileValue extends ArgumentMatchValue {
private final File file;
public ArgumentMatchFileValue(File file) {
this.file = file;
}
@Override
public String asString() {
return file == null ? null : file.getAbsolutePath();
}
@Override
public File asFile() {
return file;
}
}

42
public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchSource.java
View File

@ -24,38 +24,36 @@
package org.broadinstitute.sting.commandline;
import java.io.File;
/**
* Where an argument match originated, via the commandline or a file.
* Where an argument match originated, via the commandline or a custom provider.
*/
public class ArgumentMatchSource implements Comparable<ArgumentMatchSource> {
public static final ArgumentMatchSource COMMAND_LINE = new ArgumentMatchSource(ArgumentMatchSourceType.CommandLine, null);
private final ArgumentMatchSourceType type;
private final File file;
private final String description;
/**
* Creates an argument match source from the specified file.
* @param file File specifying the arguments. Must not be null.
* @param description Where the arguments originated.
*/
public ArgumentMatchSource(File file) {
this(ArgumentMatchSourceType.File, file);
public ArgumentMatchSource(String description) {
this(ArgumentMatchSourceType.Provider, description);
}
private ArgumentMatchSource(ArgumentMatchSourceType type, File file) {
if (type == ArgumentMatchSourceType.File && file == null)
throw new IllegalArgumentException("An argument match source of type File cannot have a null file.");
private ArgumentMatchSource(ArgumentMatchSourceType type, String description) {
if (type == ArgumentMatchSourceType.Provider && description == null)
throw new IllegalArgumentException("An argument match source provider cannot have a null description.");
this.type = type;
this.file = file;
this.description = description;
}
public ArgumentMatchSourceType getType() {
return type;
}
public File getFile() {
return file;
public String getDescription() {
return description;
}
@Override
@ -65,13 +63,13 @@ public class ArgumentMatchSource implements Comparable<ArgumentMatchSource> {
ArgumentMatchSource that = (ArgumentMatchSource) o;
return (type == that.type) && (file == null ? that.file == null : file.equals(that.file));
return (type == that.type) && (description == null ? that.description == null : description.equals(that.description));
}
@Override
public int hashCode() {
int result = type != null ? type.hashCode() : 0;
result = 31 * result + (file != null ? file.hashCode() : 0);
result = 31 * result + (description != null ? description.hashCode() : 0);
return result;
}
@ -84,15 +82,15 @@ public class ArgumentMatchSource implements Comparable<ArgumentMatchSource> {
if (comp != 0)
return comp;
File f1 = this.file;
File f2 = that.file;
String d1 = this.description;
String d2 = that.description;
if ((f1 == null) ^ (f2 == null)) {
// If one of the files is null and the other is not
// put the null file first
return f1 == null ? -1 : 1;
if ((d1 == null) ^ (d2 == null)) {
// If one of the descriptions is null and the other is not
// put the null description first
return d1 == null ? -1 : 1;
}
return f1 == null ? 0 : f1.compareTo(f2);
return d1 == null ? 0 : d1.compareTo(d2);
}
}

4
public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchSourceType.java
View File

@ -25,8 +25,8 @@
package org.broadinstitute.sting.commandline;
/**
* Type of where an argument match originated, via the commandline or a file.
* Type of where an argument match originated, via the commandline or a some other provider.
*/
public enum ArgumentMatchSourceType {
CommandLine, File
CommandLine, Provider
}

24
public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchStringValue.java 100644
View File

@ -0,0 +1,24 @@
package org.broadinstitute.sting.commandline;
import java.io.File;
/**
* Argument values that originated from a string.
*/
public class ArgumentMatchStringValue extends ArgumentMatchValue {
private final String value;
public ArgumentMatchStringValue(String value) {
this.value = value;
}
@Override
public String asString() {
return value;
}
@Override
public File asFile() {
return value == null ? null : new File(value);
}
}

18
public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchValue.java 100644
View File

@ -0,0 +1,18 @@
package org.broadinstitute.sting.commandline;
import java.io.File;
/**
* Returns argument values as either strings or values.
*/
public abstract class ArgumentMatchValue {
/**
* @return the value of this argument as a String object.
*/
public abstract String asString();
/**
* @return the value of this argument as a File object.
*/
public abstract File asFile();
}

34
public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java
View File

@ -215,8 +215,8 @@ public abstract class ArgumentTypeDescriptor {
* @param matches The matches for the given argument.
* @return The value of the argument if available, or null if not present.
*/
protected String getArgumentValue( ArgumentDefinition definition, ArgumentMatches matches ) {
Collection<String> argumentValues = getArgumentValues( definition, matches );
protected ArgumentMatchValue getArgumentValue( ArgumentDefinition definition, ArgumentMatches matches ) {
Collection<ArgumentMatchValue> argumentValues = getArgumentValues( definition, matches );
if( argumentValues.size() > 1 )
throw new UserException.CommandLineException("Multiple values associated with given definition, but this argument expects only one: " + definition.fullName);
return argumentValues.size() > 0 ? argumentValues.iterator().next() : null;
@ -244,8 +244,8 @@ public abstract class ArgumentTypeDescriptor {
* @param matches The matches for the given argument.
* @return The value of the argument if available, or an empty collection if not present.
*/
protected Collection<String> getArgumentValues( ArgumentDefinition definition, ArgumentMatches matches ) {
Collection<String> values = new ArrayList<String>();
protected Collection<ArgumentMatchValue> getArgumentValues( ArgumentDefinition definition, ArgumentMatches matches ) {
Collection<ArgumentMatchValue> values = new ArrayList<ArgumentMatchValue>();
for( ArgumentMatch match: matches ) {
if( match.definition.equals(definition) )
values.addAll(match.values());
@ -310,7 +310,7 @@ public abstract class ArgumentTypeDescriptor {
*/
protected Object parseBinding(ArgumentSource source, Type type, ArgumentMatches matches, Tags tags) {
ArgumentDefinition defaultDefinition = createDefaultArgumentDefinition(source);
String value = getArgumentValue(defaultDefinition, matches);
ArgumentMatchValue value = getArgumentValue(defaultDefinition, matches);
@SuppressWarnings("unchecked")
Class<? extends Feature> parameterType = JVMUtils.getParameterizedTypeClass(type);
String name = defaultDefinition.fullName;
@ -328,7 +328,7 @@ public abstract class ArgumentTypeDescriptor {
* @param fieldName The name of the field that was parsed. Used for error reporting.
* @return The newly created binding object of type bindingClass.
*/
public static Object parseBinding(String value, Class<? extends Feature> parameterType, Type bindingClass,
public static Object parseBinding(ArgumentMatchValue value, Class<? extends Feature> parameterType, Type bindingClass,
String bindingName, Tags tags, String fieldName) {
try {
String tribbleType = null;
@ -337,7 +337,7 @@ public abstract class ArgumentTypeDescriptor {
throw new UserException.CommandLineException(
String.format("Unexpected number of positional tags for argument %s : %s. " +
"Rod bindings only support -X:type and -X:name,type argument styles",
value, fieldName));
value.asString(), fieldName));
} else if ( tags.getPositionalTags().size() == 2 ) {
// -X:name,type style
bindingName = tags.getPositionalTags().get(0);
@ -366,7 +366,7 @@ public abstract class ArgumentTypeDescriptor {
if ( tribbleType == null ) {
// try to determine the file type dynamically
File file = new File(value);
File file = value.asFile();
if ( file.canRead() && file.isFile() ) {
FeatureManager.FeatureDescriptor featureDescriptor = manager.getByFiletype(file);
if ( featureDescriptor != null ) {
@ -379,7 +379,7 @@ public abstract class ArgumentTypeDescriptor {
// IntervalBinding can be created from a normal String
Class rawType = (makeRawTypeIfNecessary(bindingClass));
try {
return rawType.getConstructor(String.class).newInstance(value);
return rawType.getConstructor(String.class).newInstance(value.asString());
} catch (NoSuchMethodException e) {
/* ignore */
}
@ -399,14 +399,14 @@ public abstract class ArgumentTypeDescriptor {
}
Constructor ctor = (makeRawTypeIfNecessary(bindingClass)).getConstructor(Class.class, String.class, String.class, String.class, Tags.class);
return ctor.newInstance(parameterType, bindingName, value, tribbleType, tags);
return ctor.newInstance(parameterType, bindingName, value.asString(), tribbleType, tags);
} catch (Exception e) {
if ( e instanceof UserException )
throw ((UserException)e);
else
throw new UserException.CommandLineException(
String.format("Failed to parse value %s for argument %s. Message: %s",
value, fieldName, e.getMessage()));
value.asString(), fieldName, e.getMessage()));
}
}
}
@ -517,7 +517,7 @@ class SimpleArgumentTypeDescriptor extends ArgumentTypeDescriptor {
return true;
ArgumentDefinition defaultDefinition = createDefaultArgumentDefinition(source);
String value = getArgumentValue( defaultDefinition, matches );
ArgumentMatchValue value = getArgumentValue(defaultDefinition, matches);
Object result;
Tags tags = getArgumentTags(matches);
@ -527,12 +527,12 @@ class SimpleArgumentTypeDescriptor extends ArgumentTypeDescriptor {
Method valueOf = primitiveToWrapperMap.get(type).getMethod("valueOf",String.class);
if(value == null)
throw new MissingArgumentValueException(createDefaultArgumentDefinition(source));
result = valueOf.invoke(null,value.trim());
result = valueOf.invoke(null,value.asString().trim());
} else if (type.isEnum()) {
Object[] vals = type.getEnumConstants();
Object defaultEnumeration = null; // as we look at options, record the default option if it exists
for (Object val : vals) {
if (String.valueOf(val).equalsIgnoreCase(value)) return val;
if (String.valueOf(val).equalsIgnoreCase(value == null ? null : value.asString())) return val;
try { if (type.getField(val.toString()).isAnnotationPresent(EnumerationArgumentDefault.class)) defaultEnumeration = val; }
catch (NoSuchFieldException e) { throw new ReviewedStingException("parsing " + type.toString() + "doesn't contain the field " + val.toString()); }
}
@ -544,10 +544,12 @@ class SimpleArgumentTypeDescriptor extends ArgumentTypeDescriptor {
else if (value == null)
throw new MissingArgumentValueException(createDefaultArgumentDefinition(source));
else
throw new UnknownEnumeratedValueException(createDefaultArgumentDefinition(source),value);
throw new UnknownEnumeratedValueException(createDefaultArgumentDefinition(source),value.asString());
} else if (type.equals(File.class)) {
result = value == null ? null : value.asFile();
} else {
Constructor ctor = type.getConstructor(String.class);
result = ctor.newInstance(value);
result = ctor.newInstance(value == null ? null : value.asString());
}
} catch (UserException e) {
throw e;

2
public/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java
View File

@ -174,7 +174,7 @@ public abstract class CommandLineProgram {
ParsingEngine parser = clp.parser = new ParsingEngine(clp);
parser.addArgumentSource(clp.getClass());
Map<ArgumentMatchSource, List<String>> parsedArgs;
Map<ArgumentMatchSource, ParsedArgs> parsedArgs;
// process the args
if (clp.canAddArgumentsDynamically()) {

13
public/java/src/org/broadinstitute/sting/commandline/ParsedArgs.java 100644
View File

@ -0,0 +1,13 @@
package org.broadinstitute.sting.commandline;
/**
* Represents a collection of parsed arguments for an argument source.
*
* Useful for printing out help documents.
*/
public abstract class ParsedArgs {
/**
* @return A compact description of the arguments from an provider/source.
*/
public abstract String getDescription();
}

30
public/java/src/org/broadinstitute/sting/commandline/ParsedListArgs.java 100644
View File

@ -0,0 +1,30 @@
package org.broadinstitute.sting.commandline;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* A list of string arguments, usually from the command line or an args list file.
*/
public class ParsedListArgs extends ParsedArgs {
private final List<String> args = new ArrayList<String>();
public ParsedListArgs() {
}
public ParsedListArgs(List<String> args) {
this.args.addAll(args);
}
public void add(String... args) {
this.args.addAll(Arrays.asList(args));
}
@Override
public String getDescription() {
return StringUtils.join(this.args, " ");
}
}

102
public/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java
View File

@ -30,6 +30,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.classloader.JVMUtils;
import org.broadinstitute.sting.utils.classloader.PluginManager;
import org.broadinstitute.sting.utils.collections.Pair;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import org.broadinstitute.sting.utils.exceptions.UserException;
@ -61,7 +62,7 @@ public class ParsingEngine {
* Indicates as best as possible where command-line text remains unmatched
* to existing arguments.
*/
ArgumentMatches argumentMatches = null;
private ArgumentMatches argumentMatches = null;
/**
* Techniques for parsing and for argument lookup.
@ -88,7 +89,10 @@ public class ParsingEngine {
/**
* List of tags associated with the given instantiation of the command-line argument.
*/
private final Map<Object,Tags> tags = new IdentityHashMap<Object,Tags>();
private final Map<Object,Tags> tags = new IdentityHashMap<Object,Tags>();
private PluginManager<ParsingEngineArgumentProvider> argumentProviderPluginManager =
new PluginManager<ParsingEngineArgumentProvider>(ParsingEngineArgumentProvider.class);
/**
* our log, which we want to capture anything from org.broadinstitute.sting
@ -105,7 +109,10 @@ public class ParsingEngine {
argumentTypeDescriptors.addAll(clp.getArgumentTypeDescriptors());
argumentTypeDescriptors.addAll(STANDARD_ARGUMENT_TYPE_DESCRIPTORS);
addArgumentSource(ParsingEngineArgumentFiles.class);
List<Class<? extends ParsingEngineArgumentProvider>> providers = argumentProviderPluginManager.getPlugins();
for (Class<? extends ParsingEngineArgumentProvider> provider: providers) {
addArgumentSource(provider);
}
}
/**
@ -117,6 +124,10 @@ public class ParsingEngine {
addArgumentSource(null, source);
}
public ArgumentMatches getArgumentMatches() {
return argumentMatches;
}
/**
* Add an argument source. Argument sources are expected to have
* any number of fields with an @Argument annotation attached.
@ -156,29 +167,30 @@ public class ParsingEngine {
* @param tokens Tokens passed on the command line.
* @return The parsed arguments by file.
*/
public SortedMap<ArgumentMatchSource, List<String>> parse( String[] tokens ) {
public SortedMap<ArgumentMatchSource, ParsedArgs> parse( String[] tokens ) {
argumentMatches = new ArgumentMatches();
SortedMap<ArgumentMatchSource, List<String>> parsedArgs = new TreeMap<ArgumentMatchSource, List<String>>();
SortedMap<ArgumentMatchSource, ParsedArgs> parsedArgs = new TreeMap<ArgumentMatchSource, ParsedArgs>();
List<String> cmdLineTokens = Arrays.asList(tokens);
parse(ArgumentMatchSource.COMMAND_LINE, cmdLineTokens, argumentMatches, parsedArgs);
ParsingEngineArgumentFiles argumentFiles = new ParsingEngineArgumentFiles();
List<ParsingEngineArgumentProvider> providers = argumentProviderPluginManager.createAllTypes();
// Load the arguments ONLY into the argument files.
// Validation may optionally run on the rest of the arguments.
loadArgumentsIntoObject(argumentFiles);
for (ParsingEngineArgumentProvider provider: providers) {
// Load the arguments ONLY into the provider.
// Validation may optionally run on the rest of the arguments.
loadArgumentsIntoObject(provider);
}
for (File file: argumentFiles.files) {
List<String> fileTokens = getArguments(file);
parse(new ArgumentMatchSource(file), fileTokens, argumentMatches, parsedArgs);
for (ParsingEngineArgumentProvider provider: providers) {
provider.parse(this, parsedArgs);
}
return parsedArgs;
}
private void parse(ArgumentMatchSource matchSource, List<String> tokens,
ArgumentMatches argumentMatches, SortedMap<ArgumentMatchSource, List<String>> parsedArgs) {
public void parse(ArgumentMatchSource matchSource, List<String> tokens,
ArgumentMatches argumentMatches, SortedMap<ArgumentMatchSource, ParsedArgs> parsedArgs) {
ArgumentMatchSite lastArgumentMatchSite = new ArgumentMatchSite(matchSource, -1);
int i = 0;
@ -195,19 +207,44 @@ public class ParsingEngine {
}
else {
if( argumentMatches.hasMatch(lastArgumentMatchSite) &&
!argumentMatches.getMatch(lastArgumentMatchSite).hasValueAtSite(lastArgumentMatchSite))
argumentMatches.getMatch(lastArgumentMatchSite).addValue( lastArgumentMatchSite, token );
!argumentMatches.getMatch(lastArgumentMatchSite).hasValueAtSite(lastArgumentMatchSite))
argumentMatches.getMatch(lastArgumentMatchSite).addValue( lastArgumentMatchSite, new ArgumentMatchStringValue(token) );
else
argumentMatches.MissingArgument.addValue( site, token );
argumentMatches.MissingArgument.addValue( site, new ArgumentMatchStringValue(token) );
}
i++;
}
parsedArgs.put(matchSource, tokens);
parsedArgs.put(matchSource, new ParsedListArgs(tokens));
}
private List<String> getArguments(File file) {
public void parsePairs(ArgumentMatchSource matchSource, List<Pair<String, ArgumentMatchValue>> tokens,
ArgumentMatches argumentMatches, ParsedArgs matchSourceArgs,
SortedMap<ArgumentMatchSource, ParsedArgs> parsedArgs) {
int i = 0;
for (Pair<String, ArgumentMatchValue> pair: tokens) {
ArgumentMatchSite site = new ArgumentMatchSite(matchSource, i);
List<DefinitionMatcher> matchers = Arrays.asList(ArgumentDefinitions.FullNameDefinitionMatcher, ArgumentDefinitions.ShortNameDefinitionMatcher);
ArgumentDefinition definition = null;
for (DefinitionMatcher matcher: matchers) {
definition = argumentDefinitions.findArgumentDefinition( pair.getFirst(), matcher );
if (definition != null)
break;
}
if (definition == null)
continue;
ArgumentMatch argumentMatch = new ArgumentMatch(pair.getFirst(), definition, site, new Tags());
argumentMatches.mergeInto(argumentMatch);
argumentMatch.addValue(site, pair.getSecond());
i++;
}
parsedArgs.put(matchSource, matchSourceArgs);
}
protected List<String> getArguments(File file) {
try {
if (file.getAbsolutePath().endsWith(".list")) {
return getListArguments(file);
@ -283,9 +320,9 @@ public class ParsingEngine {
// Ensure that the field contents meet the validation criteria specified by the regular expression.
for( ArgumentMatch verifiableMatch: verifiableMatches ) {
for( String value: verifiableMatch.values() ) {
if( verifiableArgument.validation != null && !value.matches(verifiableArgument.validation) )
invalidValues.add( new Pair<ArgumentDefinition,String>(verifiableArgument, value) );
for( ArgumentMatchValue value: verifiableMatch.values() ) {
if( verifiableArgument.validation != null && !value.asString().matches(verifiableArgument.validation) )
invalidValues.add( new Pair<ArgumentDefinition,String>(verifiableArgument, value.asString()) );
}
}
}
@ -629,21 +666,21 @@ class UnmatchedArgumentException extends ArgumentException {
private static String formatArguments( ArgumentMatch invalidValues ) {
StringBuilder sb = new StringBuilder();
for( ArgumentMatchSite site: invalidValues.sites.keySet() )
for( String value: invalidValues.sites.get(site) ) {
for( ArgumentMatchValue value: invalidValues.sites.get(site) ) {
switch (site.getSource().getType()) {
case CommandLine:
sb.append( String.format("%nInvalid argument value '%s' at position %d.",
value, site.getIndex()) );
value.asString(), site.getIndex()) );
break;
case File:
sb.append( String.format("%nInvalid argument value '%s' in file %s at position %d.",
value, site.getSource().getFile().getAbsolutePath(), site.getIndex()) );
case Provider:
sb.append( String.format("%nInvalid argument value '%s' in %s at position %d.",
value.asString(), site.getSource().getDescription(), site.getIndex()) );
break;
default:
throw new RuntimeException( String.format("Unexpected argument match source type: %s",
site.getSource().getType()));
}
if(value != null && Utils.dupString(' ',value.length()).equals(value))
if(value.asString() != null && Utils.dupString(' ',value.asString().length()).equals(value.asString()))
sb.append(" Please make sure any line continuation backslashes on your command line are not followed by whitespace.");
}
return sb.toString();
@ -696,12 +733,3 @@ class UnknownEnumeratedValueException extends ArgumentException {
return String.format("Invalid value %s specified for argument %s; valid options are (%s).", argumentPassed, definition.fullName, Utils.join(",",definition.validOptions));
}
}
/**
* Container class to store the list of argument files.
* The files will be parsed after the command line arguments.
*/
class ParsingEngineArgumentFiles {
@Argument(fullName = "arg_file", shortName = "args", doc = "Reads arguments from the specified file", required = false)
public List<File> files = new ArrayList<File>();
}

30
public/java/src/org/broadinstitute/sting/commandline/ParsingEngineArgumentFiles.java 100644
View File

@ -0,0 +1,30 @@
package org.broadinstitute.sting.commandline;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.SortedMap;
/**
* Container class to store the list of argument files.
* The files will be parsed after the command line arguments.
*/
public class ParsingEngineArgumentFiles extends ParsingEngineArgumentProvider {
@Argument(fullName = "arg_file", shortName = "args", doc = "Reads arguments from the specified file", required = false)
public List<File> files = new ArrayList<File>();
@Override
public void parse(ParsingEngine parsingEngine, SortedMap<ArgumentMatchSource, ParsedArgs> parsedArgs) {
ArgumentMatches argumentMatches = parsingEngine.getArgumentMatches();
for (File file: this.files) {
List<String> fileTokens = parsingEngine.getArguments(file);
parsingEngine.parse(new ArgumentMatchFileSource(file), fileTokens, argumentMatches, parsedArgs);
}
}
}
class ArgumentMatchFileSource extends ArgumentMatchSource {
ArgumentMatchFileSource(File file) {
super("file " + file.getAbsolutePath());
}
}

12
public/java/src/org/broadinstitute/sting/commandline/ParsingEngineArgumentProvider.java 100644
View File

@ -0,0 +1,12 @@
package org.broadinstitute.sting.commandline;
import java.util.List;
import java.util.SortedMap;
/**
* A class that can parse arguments for the engine
*/
public abstract class ParsingEngineArgumentProvider {
public abstract void parse(ParsingEngine parsingEngine, SortedMap<ArgumentMatchSource, ParsedArgs> parsedArgs);
}

14
public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java
View File

@ -157,18 +157,22 @@ public abstract class MicroScheduler implements MicroSchedulerMBean {
if ( ! (walker instanceof TreeReducible) ) {
throw badNT("nt", engine, walker);
} else {
return new HierarchicalMicroScheduler(engine, walker, reads, reference, rods, threadAllocation);
}
}
if ( threadAllocation.getNumCPUThreadsPerDataThread() > 1 && ! (walker instanceof NanoSchedulable) ) {
throw badNT("nct", engine, walker);
}
if ( threadAllocation.getNumDataThreads() > 1 ) {
return new HierarchicalMicroScheduler(engine, walker, reads, reference, rods, threadAllocation);
} else {
if ( threadAllocation.getNumCPUThreadsPerDataThread() > 1 && ! (walker instanceof NanoSchedulable) )
throw badNT("nct", engine, walker);
return new LinearMicroScheduler(engine, walker, reads, reference, rods, threadAllocation);
}
}
private static UserException badNT(final String parallelArg, final GenomeAnalysisEngine engine, final Walker walker) {
throw new UserException.BadArgumentValue("nt",
throw new UserException.BadArgumentValue(parallelArg,
String.format("The analysis %s currently does not support parallel execution with %s. " +
"Please run your analysis without the %s option.", engine.getWalkerName(walker.getClass()), parallelArg, parallelArg));
}

2
public/java/src/org/broadinstitute/sting/gatk/io/stubs/OutputStreamArgumentTypeDescriptor.java
View File

@ -86,7 +86,7 @@ public class OutputStreamArgumentTypeDescriptor extends ArgumentTypeDescriptor {
@Override
public Object parse( ParsingEngine parsingEngine, ArgumentSource source, Type type, ArgumentMatches matches ) {
ArgumentDefinition definition = createDefaultArgumentDefinition(source);
String fileName = getArgumentValue( definition, matches );
String fileName = getArgumentValue( definition, matches ).asString();
// This parser has been passed a null filename and the GATK is not responsible for creating a type default for the object;
// therefore, the user must have failed to specify a type default

12
public/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileReaderArgumentTypeDescriptor.java
View File

@ -25,15 +25,11 @@
package org.broadinstitute.sting.gatk.io.stubs;
import net.sf.samtools.SAMFileReader;
import org.broadinstitute.sting.commandline.ArgumentMatches;
import org.broadinstitute.sting.commandline.ArgumentSource;
import org.broadinstitute.sting.commandline.ArgumentTypeDescriptor;
import org.broadinstitute.sting.commandline.ParsingEngine;
import org.broadinstitute.sting.commandline.*;
import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
import org.broadinstitute.sting.utils.exceptions.UserException;
import org.broadinstitute.sting.utils.sam.SAMFileReaderBuilder;
import java.io.File;
import java.lang.reflect.Type;
/**
@ -47,7 +43,7 @@ public class SAMFileReaderArgumentTypeDescriptor extends ArgumentTypeDescriptor
/**
* Create a new SAMFileReader argument, notifying the given engine when that argument has been created.
* @param engine
* @param engine engine
*/
public SAMFileReaderArgumentTypeDescriptor( GenomeAnalysisEngine engine ) {
this.engine = engine;
@ -62,12 +58,12 @@ public class SAMFileReaderArgumentTypeDescriptor extends ArgumentTypeDescriptor
public Object parse( ParsingEngine parsingEngine, ArgumentSource source, Type type, ArgumentMatches matches ) {
SAMFileReaderBuilder builder = new SAMFileReaderBuilder();
String readerFileName = getArgumentValue( createDefaultArgumentDefinition(source), matches );
ArgumentMatchValue readerFileName = getArgumentValue( createDefaultArgumentDefinition(source), matches );
if( readerFileName == null )
throw new UserException.CommandLineException("SAM file compression was supplied, but no associated writer was supplied with it.");
builder.setSAMFile(new File(readerFileName));
builder.setSAMFile(readerFileName.asFile());
// WARNING: Skipping required side-effect because stub is impossible to generate.
engine.addInput(source, builder);

45
public/java/src/org/broadinstitute/sting/gatk/io/stubs/SAMFileWriterArgumentTypeDescriptor.java
View File

@ -31,7 +31,6 @@ import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
import org.broadinstitute.sting.gatk.io.StingSAMFileWriter;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import java.io.File;
import java.io.OutputStream;
import java.lang.annotation.Annotation;
import java.lang.reflect.Type;
@ -111,10 +110,10 @@ public class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor
public Object parse( ParsingEngine parsingEngine, ArgumentSource source, Type type, ArgumentMatches matches ) {
// Extract all possible parameters that could be passed to a BAM file writer?
ArgumentDefinition bamArgumentDefinition = createBAMArgumentDefinition(source);
String writerFileName = getArgumentValue( bamArgumentDefinition, matches );
ArgumentMatchValue writerFileName = getArgumentValue( bamArgumentDefinition, matches );
String compressionLevelText = getArgumentValue( createBAMCompressionArgumentDefinition(source), matches );
Integer compressionLevel = compressionLevelText != null ? Integer.valueOf(compressionLevelText) : null;
ArgumentMatchValue compressionLevelText = getArgumentValue( createBAMCompressionArgumentDefinition(source), matches );
Integer compressionLevel = compressionLevelText != null ? Integer.valueOf(compressionLevelText.asString()) : null;
boolean indexOnTheFly = !argumentIsPresent(disableWriteIndexArgumentDefinition(source),matches);
boolean generateMD5 = argumentIsPresent(this.enableMD5GenerationArgumentDefinition(source),matches);
@ -124,32 +123,28 @@ public class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor
// This parser has been passed a null filename and the GATK is not responsible for creating a type default for the object;
// therefore, the user must have failed to specify a type default
if(writerFileName == null) {
if(!source.isRequired())
throw new MissingArgumentValueException(bamArgumentDefinition);
if(generateMD5)
if(writerFileName != null && writerFileName.asFile() == null && generateMD5)
throw new ArgumentException("MD5 generation specified, but no output file specified. If md5 generation is desired, please specify a BAM output file and an md5 file will be written alongside.");
}
// Create the stub and set parameters.
SAMFileWriterStub stub;
if ( writerFileName != null )
stub = new SAMFileWriterStub(engine, new File(writerFileName));
else
stub = new SAMFileWriterStub(engine, defaultOutputStream);
SAMFileWriterStub stub = null; // stub = new SAMFileWriterStub(engine, defaultOutputStream);
if ( compressionLevel != null )
stub.setCompressionLevel(compressionLevel);
if ( indexOnTheFly )
stub.setIndexOnTheFly(indexOnTheFly);
if ( generateMD5 )
stub.setGenerateMD5(generateMD5);
if ( simplifyBAM )
stub.setSimplifyBAM(simplifyBAM);
if (writerFileName != null && writerFileName.asFile() != null ) {
stub = new SAMFileWriterStub(engine, writerFileName.asFile());
// WARNING: Side effects required by engine!
parsingEngine.addTags(stub,getArgumentTags(matches));
engine.addOutput(stub);
if ( compressionLevel != null )
stub.setCompressionLevel(compressionLevel);
if ( indexOnTheFly )
stub.setIndexOnTheFly(indexOnTheFly);
if ( generateMD5 )
stub.setGenerateMD5(generateMD5);
if ( simplifyBAM )
stub.setSimplifyBAM(simplifyBAM);
// WARNING: Side effects required by engine!
parsingEngine.addTags(stub,getArgumentTags(matches));
engine.addOutput(stub);
}
return stub;
}

6
public/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterArgumentTypeDescriptor.java
View File

@ -138,8 +138,8 @@ public class VCFWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor {
public Object parse( ParsingEngine parsingEngine, ArgumentSource source, Type type, ArgumentMatches matches ) {
ArgumentDefinition defaultArgumentDefinition = createDefaultArgumentDefinition(source);
// Get the filename for the genotype file, if it exists. If not, we'll need to send output to out.
String writerFileName = getArgumentValue(defaultArgumentDefinition,matches);
File writerFile = writerFileName != null ? new File(writerFileName) : null;
ArgumentMatchValue writerFileName = getArgumentValue(defaultArgumentDefinition,matches);
File writerFile = writerFileName != null ? writerFileName.asFile() : null;
// This parser has been passed a null filename and the GATK is not responsible for creating a type default for the object;
// therefore, the user must have failed to specify a type default
@ -151,7 +151,7 @@ public class VCFWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor {
? new VariantContextWriterStub(engine, writerFile, argumentSources)
: new VariantContextWriterStub(engine, defaultOutputStream, argumentSources);
stub.setCompressed(isCompressed(writerFileName));
stub.setCompressed(isCompressed(writerFileName.asString()));
stub.setDoNotWriteGenotypes(argumentIsPresent(createSitesOnlyArgumentDefinition(),matches));
stub.setSkipWritingCommandLineHeader(argumentIsPresent(createNoCommandLineHeaderArgumentDefinition(),matches));
stub.setForceBCF(argumentIsPresent(createBCFArgumentDefinition(),matches));

2
public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/InbreedingCoeff.java
View File

@ -48,7 +48,7 @@ public class InbreedingCoeff extends InfoFieldAnnotation implements StandardAnno
private Map<String, Object> calculateIC(final VariantContext vc) {
final GenotypesContext genotypes = (founderIds == null || founderIds.isEmpty()) ? vc.getGenotypes() : vc.getGenotypes(founderIds);
if ( genotypes == null || genotypes.size() < MIN_SAMPLES )
if ( genotypes == null || genotypes.size() < MIN_SAMPLES || !vc.isVariant())
return null;
int idxAA = 0, idxAB = 1, idxBB = 2;

19
public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/PerReadAlleleLikelihoodMap.java
View File

@ -113,23 +113,20 @@ public class PerReadAlleleLikelihoodMap {
return likelihoodReadMap.get(p.getRead());
}
public static Allele getMostLikelyAllele(Map<Allele,Double> alleleMap) {
double minLike = Double.POSITIVE_INFINITY, maxLike = Double.NEGATIVE_INFINITY;
public static Allele getMostLikelyAllele( final Map<Allele,Double> alleleMap ) {
double maxLike = Double.NEGATIVE_INFINITY;
double prevMaxLike = Double.NEGATIVE_INFINITY;
Allele mostLikelyAllele = Allele.NO_CALL;
for (Map.Entry<Allele,Double> el : alleleMap.entrySet()) {
for (final Map.Entry<Allele,Double> el : alleleMap.entrySet()) {
if (el.getValue() > maxLike) {
prevMaxLike = maxLike;
maxLike = el.getValue();
mostLikelyAllele = el.getKey();
} else if( el.getValue() > prevMaxLike ) {
prevMaxLike = el.getValue();
}
if (el.getValue() < minLike)
minLike = el.getValue();
}
if (maxLike-minLike > INDEL_LIKELIHOOD_THRESH)
return mostLikelyAllele;
else
return Allele.NO_CALL;
return (maxLike - prevMaxLike > INDEL_LIKELIHOOD_THRESH ? mostLikelyAllele : Allele.NO_CALL );
}
}

13
public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/AFCalcResult.java
View File

@ -238,6 +238,19 @@ public class AFCalcResult {
return getLog10PosteriorOfAFGt0ForAllele(allele) >= log10minPNonRef;
}
/**
* Are any of the alleles polymorphic w.r.t. #isPolymorphic?
*
* @param log10minPNonRef the confidence threshold, in log10 space
* @return true if any are poly, false otherwise
*/
public boolean anyPolymorphic(final double log10minPNonRef) {
for ( final Allele a : getAllelesUsedInGenotyping() )
if ( a.isNonReference() && isPolymorphic(a, log10minPNonRef) )
return true;
return false;
}
/**
* Returns the log10 probability that allele is segregating
*

75
public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/afcalc/ExactCallLogger.java
View File

@ -1,20 +1,18 @@
package org.broadinstitute.sting.gatk.walkers.genotyper.afcalc;
import org.broadinstitute.sting.utils.AutoFormattingTime;
import org.broadinstitute.sting.utils.GenomeLoc;
import org.broadinstitute.sting.utils.GenomeLocParser;
import org.broadinstitute.sting.utils.Utils;
import com.google.java.contract.Requires;
import org.apache.commons.lang.ArrayUtils;
import org.broadinstitute.sting.utils.*;
import org.broadinstitute.sting.utils.exceptions.UserException;
import org.broadinstitute.sting.utils.variantcontext.*;
import java.io.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.*;
/**
* Allows us to write out and read in information about exact calls (site, alleles, PLs, etc) in tabular format
*
* Once opened, calls can be writen to disk with printCallInfo
*/
public class ExactCallLogger implements Cloneable {
private PrintStream callReport = null;
@ -26,7 +24,7 @@ public class ExactCallLogger implements Cloneable {
*/
public ExactCallLogger(final File outputFile) {
try {
callReport = new PrintStream(new FileOutputStream(outputFile));
callReport = new PrintStream(new BufferedOutputStream(new FileOutputStream(outputFile), 10000000));
callReport.println(Utils.join("\t", Arrays.asList("loc", "variable", "key", "value")));
} catch (FileNotFoundException e) {
throw new UserException.CouldNotCreateOutputFile(outputFile, e);
@ -38,32 +36,28 @@ public class ExactCallLogger implements Cloneable {
*/
public static class ExactCall {
final VariantContext vc;
final long origNanoTime;
long newNanoTime = -1;
final double origPNonRef;
double newPNonRef = -1;
final long runtime;
final AFCalcResult originalCall;
public ExactCall(VariantContext vc, long origNanoTime, double origPNonRef) {
public ExactCall(VariantContext vc, final long runtime, final AFCalcResult originalCall) {
this.vc = vc;
this.origNanoTime = origNanoTime;
this.origPNonRef = origPNonRef;
this.runtime = runtime;
this.originalCall = originalCall;
}
@Override
public String toString() {
return String.format("ExactCall %s:%d alleles=%s nSamples=%s orig.pNonRef=%.2f orig.runtime=%s new.pNonRef=%.2f new.runtime=%s",
return String.format("ExactCall %s:%d alleles=%s nSamples=%s orig.pNonRef=%.2f orig.runtime=%s",
vc.getChr(), vc.getStart(), vc.getAlleles(), vc.getNSamples(),
origPNonRef,
new AutoFormattingTime(origNanoTime / 1e9).toString(),
newPNonRef,
newNanoTime == -1 ? "not.run" : new AutoFormattingTime(newNanoTime / 1e9).toString());
originalCall.getLog10PosteriorOfAFGT0(),
new AutoFormattingTime(runtime / 1e9).toString());
}
}
protected void printCallInfo(final VariantContext vc,
final double[] log10AlleleFrequencyPriors,
final long runtimeNano,
final AFCalcResult result) {
protected final void printCallInfo(final VariantContext vc,
final double[] log10AlleleFrequencyPriors,
final long runtimeNano,
final AFCalcResult result) {
printCallElement(vc, "type", "ignore", vc.getType());
int allelei = 0;
@ -90,6 +84,7 @@ public class ExactCallLogger implements Cloneable {
callReport.flush();
}
@Requires({"vc != null", "variable != null", "key != null", "value != null", "callReport != null"})
private void printCallElement(final VariantContext vc,
final Object variable,
final Object key,
@ -102,10 +97,10 @@ public class ExactCallLogger implements Cloneable {
* Read in a list of ExactCall objects from reader, keeping only those
* with starts in startsToKeep or all sites (if this is empty)
*
* @param reader
* @param startsToKeep
* @param parser
* @return
* @param reader a just-opened reader sitting at the start of the file
* @param startsToKeep a list of start position of the calls to keep, or empty if all calls should be kept
* @param parser a genome loc parser to create genome locs
* @return a list of ExactCall objects in reader
* @throws IOException
*/
public static List<ExactCall> readExactLog(final BufferedReader reader, final List<Integer> startsToKeep, GenomeLocParser parser) throws IOException {
@ -118,10 +113,17 @@ public class ExactCallLogger implements Cloneable {
// skip the header line
reader.readLine();
// skip the first "type" line
reader.readLine();
while (true) {
final VariantContextBuilder builder = new VariantContextBuilder();
final List<Allele> alleles = new ArrayList<Allele>();
final List<Genotype> genotypes = new ArrayList<Genotype>();
final double[] posteriors = new double[2];
final double[] priors = MathUtils.normalizeFromLog10(new double[]{0.5, 0.5}, true);
final List<Integer> mle = new ArrayList<Integer>();
final Map<Allele, Double> log10pNonRefByAllele = new HashMap<Allele, Double>();
long runtimeNano = -1;
GenomeLoc currentLoc = null;
@ -139,13 +141,15 @@ public class ExactCallLogger implements Cloneable {
if (currentLoc == null)
currentLoc = lineLoc;
if (variable.equals("log10PosteriorOfAFzero")) {
if (variable.equals("type")) {
if (startsToKeep.isEmpty() || startsToKeep.contains(currentLoc.getStart())) {
builder.alleles(alleles);
final int stop = currentLoc.getStart() + alleles.get(0).length() - 1;
builder.chr(currentLoc.getContig()).start(currentLoc.getStart()).stop(stop);
builder.genotypes(genotypes);
calls.add(new ExactCall(builder.make(), runtimeNano, Double.valueOf(value)));
final int[] mleInts = ArrayUtils.toPrimitive(mle.toArray(new Integer[]{}));
final AFCalcResult result = new AFCalcResult(mleInts, 1, alleles, posteriors, priors, log10pNonRefByAllele);
calls.add(new ExactCall(builder.make(), runtimeNano, result));
}
break;
} else if (variable.equals("allele")) {
@ -155,6 +159,15 @@ public class ExactCallLogger implements Cloneable {
final GenotypeBuilder gb = new GenotypeBuilder(key);
gb.PL(GenotypeLikelihoods.fromPLField(value).getAsPLs());
genotypes.add(gb.make());
} else if (variable.equals("log10PosteriorOfAFEq0")) {
posteriors[0] = Double.valueOf(value);
} else if (variable.equals("log10PosteriorOfAFGt0")) {
posteriors[1] = Double.valueOf(value);
} else if (variable.equals("MLE")) {
mle.add(Integer.valueOf(value));
} else if (variable.equals("pNonRefByAllele")) {
final Allele a = Allele.create(key);
log10pNonRefByAllele.put(a, Double.valueOf(value));
} else if (variable.equals("runtime.nano")) {
runtimeNano = Long.valueOf(value);
} else {

4
public/java/src/org/broadinstitute/sting/gatk/walkers/indels/IndelRealigner.java
View File

@ -370,8 +370,6 @@ public class IndelRealigner extends ReadWalker<Integer, Integer> {
currentInterval = intervals.hasNext() ? intervals.next() : null;
writerToUse = writer;
if ( N_WAY_OUT != null ) {
boolean createIndex = true;
@ -383,9 +381,9 @@ public class IndelRealigner extends ReadWalker<Integer, Integer> {
createIndex, generateMD5s,createProgramRecord(),KEEP_ALL_PG_RECORDS);
}
} else {
// set up the output writer
setupWriter(getToolkit().getSAMFileHeader());
writerToUse = writer;
}
manager = new ConstrainedMateFixingManager(writerToUse, getToolkit().getGenomeLocParser(), MAX_ISIZE_FOR_MOVEMENT, MAX_POS_MOVE_ALLOWED, MAX_RECORDS_IN_MEMORY);

2
public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/PhaseByTransmission.java
View File

@ -36,7 +36,7 @@ import java.util.*;
* <ul>
* <li>In parent/child pairs: If an individual genotype is missing at one site, the other one is phased if it is homozygous. No phasing probability is emitted.</li>
* <li>In trios: If the child is missing, parents are treated as separate individuals and phased if homozygous. No phasing probability is emitted.</li>
* <li>In trios: If one of the parents is missing, it is handled like a parent/child pair. Phasing is done unless both the parent and child are heterozygous and a phasing probabilitt is emitted.</li>
* <li>In trios: If one of the parents is missing, it is handled like a parent/child pair. Phasing is done unless both the parent and child are heterozygous and a phasing probability is emitted.</li>
* <li>In trios: If two individuals are missing, the remaining individual is phased if it is homozygous. No phasing probability is emitted.</li>
* </ul>
*

12
public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java
View File

@ -42,11 +42,11 @@ import org.broadinstitute.sting.utils.MendelianViolation;
import org.broadinstitute.sting.utils.SampleUtils;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.codecs.vcf.*;
import org.broadinstitute.sting.utils.help.DocumentedGATKFeature;
import org.broadinstitute.sting.utils.variantcontext.writer.VariantContextWriter;
import org.broadinstitute.sting.utils.exceptions.UserException;
import org.broadinstitute.sting.utils.help.DocumentedGATKFeature;
import org.broadinstitute.sting.utils.text.XReadLines;
import org.broadinstitute.sting.utils.variantcontext.*;
import org.broadinstitute.sting.utils.variantcontext.writer.VariantContextWriter;
import java.io.File;
import java.io.FileNotFoundException;
@ -542,9 +542,11 @@ public class SelectVariants extends RodWalker<Integer, Integer> implements TreeR
VariantContext sub = subsetRecord(vc, EXCLUDE_NON_VARIANTS);
if ( REGENOTYPE && sub.isPolymorphicInSamples() && hasPLs(sub) ) {
final VariantContextBuilder builder = new VariantContextBuilder(UG_engine.calculateGenotypes(sub)).filters(sub.getFiltersMaybeNull());
addAnnotations(builder, sub);
sub = builder.make();
synchronized (UG_engine) {
final VariantContextBuilder builder = new VariantContextBuilder(UG_engine.calculateGenotypes(sub)).filters(sub.getFiltersMaybeNull());
addAnnotations(builder, sub);
sub = builder.make();
}
}
if ( (!EXCLUDE_NON_VARIANTS || sub.isPolymorphicInSamples()) && (!EXCLUDE_FILTERED || !sub.isFiltered()) ) {

7
public/java/src/org/broadinstitute/sting/utils/classloader/JVMUtils.java
View File

@ -32,7 +32,6 @@ import org.reflections.util.ClasspathHelper;
import java.io.File;
import java.io.IOException;
import java.lang.annotation.Annotation;
import java.lang.reflect.*;
import java.net.URL;
import java.util.*;
@ -198,7 +197,7 @@ public class JVMUtils {
* @return the list of class path urls.
*/
public static Set<URL> getClasspathURLs() {
return ClasspathHelper.getUrlsForManifestsCurrentClasspath();
return ClasspathHelper.forManifest();
}
/**
@ -240,8 +239,8 @@ public class JVMUtils {
/**
* Returns a comma-separated list of the names of the interfaces implemented by this class
*
* @param covClass
* @return
* @param covClass class
* @return names of interfaces
*/
public static String classInterfaces(final Class covClass) {
final List<String> interfaces = new ArrayList<String>();

14
public/java/src/org/broadinstitute/sting/utils/classloader/PluginManager.java
View File

@ -25,8 +25,6 @@
package org.broadinstitute.sting.utils.classloader;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.Logger;
import org.broadinstitute.sting.gatk.WalkerManager;
import org.broadinstitute.sting.gatk.filters.FilterManager;
import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException;
@ -35,7 +33,6 @@ import org.broadinstitute.sting.utils.exceptions.UserException;
import org.reflections.Reflections;
import org.reflections.scanners.SubTypesScanner;
import org.reflections.util.ConfigurationBuilder;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.lang.reflect.Constructor;
@ -57,9 +54,8 @@ public class PluginManager<PluginType> {
private static final Reflections defaultReflections;
static {
// turn off logging in the reflections library - they talk too much (to the wrong logger factory as well, logback)
Logger logger = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Reflections.class);
logger.setLevel(Level.OFF);
// turn off logging in the reflections library - they talk too much
Reflections.log = null;
Set<URL> classPathUrls = new LinkedHashSet<URL>();
@ -179,9 +175,9 @@ public class PluginManager<PluginType> {
/**
* Sorts, in place, the list of plugins according to getName() on each element
*
* @param unsortedPlugins
* @param unsortedPlugins unsorted plugins
*/
private final void sortPlugins(final List<Class<? extends PluginType>> unsortedPlugins) {
private void sortPlugins(final List<Class<? extends PluginType>> unsortedPlugins) {
Collections.sort(unsortedPlugins, new ComparePluginsByName());
}
@ -235,7 +231,7 @@ public class PluginManager<PluginType> {
* @param plugin Name of the plugin for which to search.
* @return True if the plugin exists, false otherwise.
*/
public boolean exists(Class<?> plugin) {
public boolean exists(Class<? extends PluginType> plugin) {
return pluginsByName.containsValue(plugin);
}

18
public/java/src/org/broadinstitute/sting/utils/help/HelpFormatter.java
View File

@ -26,10 +26,7 @@
package org.broadinstitute.sting.utils.help;
import org.apache.log4j.Logger;
import org.broadinstitute.sting.commandline.ArgumentDefinition;
import org.broadinstitute.sting.commandline.ArgumentDefinitionGroup;
import org.broadinstitute.sting.commandline.ArgumentDefinitions;
import org.broadinstitute.sting.commandline.ArgumentMatchSource;
import org.broadinstitute.sting.commandline.*;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.text.TextFormattingUtils;
@ -273,9 +270,9 @@ public class HelpFormatter {
* Generate a standard header for the logger
*
* @param applicationDetails details of the application to run.
* @param parsedArgs the command line arguments passed in
* @param parsedArgs the arguments passed in
*/
public static void generateHeaderInformation(ApplicationDetails applicationDetails, Map<ArgumentMatchSource, List<String>> parsedArgs) {
public static void generateHeaderInformation(ApplicationDetails applicationDetails, Map<ArgumentMatchSource, ParsedArgs> parsedArgs) {
DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
java.util.Date date = new java.util.Date();
@ -286,19 +283,16 @@ public class HelpFormatter {
for (String headerLine : applicationDetails.applicationHeader)
logger.info(headerLine);
logger.debug("Current directory: " + System.getProperty("user.dir"));
for (Map.Entry<ArgumentMatchSource, List<String>> entry: parsedArgs.entrySet()) {
for (Map.Entry<ArgumentMatchSource, ParsedArgs> entry: parsedArgs.entrySet()) {
ArgumentMatchSource matchSource = entry.getKey();
final String sourceName;
switch (matchSource.getType()) {
case CommandLine: sourceName = "Program"; break;
case File: sourceName = matchSource.getFile().getPath(); break;
case Provider: sourceName = matchSource.getDescription(); break;
default: throw new RuntimeException("Unexpected argument match source type: " + matchSource.getType());
}
String output = sourceName + " Args:";
for (String str : entry.getValue()) {
output = output + " " + str;
}
String output = sourceName + " Args: " + entry.getValue().getDescription();
logger.info(output);
}
logger.info("Date/Time: " + dateFormat.format(date));

2
public/java/test/org/broadinstitute/sting/commandline/ArgumentMatchSiteUnitTest.java
View File

@ -39,7 +39,7 @@ public class ArgumentMatchSiteUnitTest {
@Test
public void testFile() {
ArgumentMatchSource source = new ArgumentMatchSource(new File("test"));
ArgumentMatchSource source = new ArgumentMatchFileSource(new File("test"));
ArgumentMatchSite site = new ArgumentMatchSite(source, 1);
Assert.assertEquals(site.getSource(), source);
Assert.assertEquals(site.getIndex(), 1);

16
public/java/test/org/broadinstitute/sting/commandline/ArgumentMatchSourceUnitTest.java
View File

@ -35,15 +35,15 @@ public class ArgumentMatchSourceUnitTest extends BaseTest {
public void testCommandLine() {
ArgumentMatchSource source = ArgumentMatchSource.COMMAND_LINE;
Assert.assertEquals(source.getType(), ArgumentMatchSourceType.CommandLine);
Assert.assertNull(source.getFile());
Assert.assertNull(source.getDescription());
}
@Test
public void testFile() {
File f = new File("test");
ArgumentMatchSource source = new ArgumentMatchSource(f);
Assert.assertEquals(source.getType(), ArgumentMatchSourceType.File);
Assert.assertEquals(source.getFile(), f);
ArgumentMatchSource source = new ArgumentMatchFileSource(f);
Assert.assertEquals(source.getType(), ArgumentMatchSourceType.Provider);
Assert.assertEquals(source.getDescription(), "file " + f.getAbsolutePath());
}
@Test(expectedExceptions = IllegalArgumentException.class)
@ -54,8 +54,8 @@ public class ArgumentMatchSourceUnitTest extends BaseTest {
@Test
public void testEquals() {
ArgumentMatchSource cmdLine = ArgumentMatchSource.COMMAND_LINE;
ArgumentMatchSource fileA = new ArgumentMatchSource(new File("a"));
ArgumentMatchSource fileB = new ArgumentMatchSource(new File("b"));
ArgumentMatchSource fileA = new ArgumentMatchFileSource(new File("a"));
ArgumentMatchSource fileB = new ArgumentMatchFileSource(new File("b"));
Assert.assertFalse(cmdLine.equals(null));
@ -75,8 +75,8 @@ public class ArgumentMatchSourceUnitTest extends BaseTest {
@Test
public void testCompareTo() {
ArgumentMatchSource cmdLine = ArgumentMatchSource.COMMAND_LINE;
ArgumentMatchSource fileA = new ArgumentMatchSource(new File("a"));
ArgumentMatchSource fileB = new ArgumentMatchSource(new File("b"));
ArgumentMatchSource fileA = new ArgumentMatchFileSource(new File("a"));
ArgumentMatchSource fileB = new ArgumentMatchFileSource(new File("b"));
Assert.assertTrue(cmdLine.compareTo(cmdLine) == 0);
Assert.assertTrue(cmdLine.compareTo(fileA) < 0);

11
public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
View File

@ -60,7 +60,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
public void testMultipleSNPAlleles() {
WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
"-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + privateTestDir + "multiallelic.snps.bam -o %s -L " + privateTestDir + "multiallelic.snps.intervals", 1,
Arrays.asList("26af30187316f742878c85f0ed091837"));
Arrays.asList("48b4f4b05461be276bffc91350f08cbc"));
executeTest("test Multiple SNP alleles", spec);
}
@ -76,7 +76,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
public void testReverseTrim() {
WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
"-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm INDEL -I " + validationDataLocation + "CEUTrio.HiSeq.b37.chr20.10_11mb.bam -o %s -L 20:10289124 -L 20:10090289", 1,
Arrays.asList("aa9cf96ab8f5aa844387e3aef1f27249"));
Arrays.asList("04affcc9d720ee17bc221759707e0cd2"));
executeTest("test reverse trim", spec);
}
@ -84,7 +84,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
public void testMismatchedPLs() {
WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
"-T UnifiedGenotyper -R " + b37KGReference + " -nosl --no_cmdline_in_header -glm INDEL -I " + privateTestDir + "mismatchedPLs.bam -o %s -L 1:24020341", 1,
Arrays.asList("d210ee1baa75dd4a0c63aef6b1fa7a8a"));
Arrays.asList("112e7bedfd284d4d9390aa006118c733"));
executeTest("test mismatched PLs", spec);
}
@ -343,13 +343,13 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
public void testMultiSampleIndels1() {
WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
baseCommandIndels + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10450700-10551000", 1,
Arrays.asList("7fc488fe16dea9f023bfcfdaa908a548"));
Arrays.asList("863ee56b3594f09795644127f2f9539f"));
List<File> result = executeTest("test MultiSample Pilot1 CEU indels", spec1).getFirst();
WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + result.get(0).getAbsolutePath() + " -I " + validationDataLocation +
"low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10450700-10551000", 1,
Arrays.asList("f3ff7fe0f15f31eadd726c711d6bf3de"));
Arrays.asList("503ca1b75cc7b2679eaa80f7b5e7ef1c"));
executeTest("test MultiSample Pilot1 CEU indels using GENOTYPE_GIVEN_ALLELES", spec2);
}
@ -452,5 +452,4 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
Arrays.asList("bbf16e1873e525ee5975021cfb8988cf"));
executeTest("test calling on a ReducedRead BAM", spec);
}
}

10
public/java/test/org/broadinstitute/sting/gatk/walkers/indels/IndelRealignerIntegrationTest.java
View File

@ -113,4 +113,14 @@ public class IndelRealignerIntegrationTest extends WalkerTest {
executeTest(String.format("realigner [%s]", entry.getKey()), spec);
}
}
@Test
public void testNWayOut() {
WalkerTestSpec spec1 = new WalkerTestSpec(
baseCommandPrefix + " -nWayOut .clean.bam ",
1,
Arrays.asList("d41d8cd98f00b204e9800998ecf8427e"));
executeTest("test realigner nWayOut", spec1);
}
}

25
public/scala/qscript/org/broadinstitute/sting/queue/qscripts/DataProcessingPipeline.scala
View File

@ -13,6 +13,7 @@ import net.sf.samtools.SAMFileHeader.SortOrder
import org.broadinstitute.sting.queue.util.QScriptUtils
import org.broadinstitute.sting.queue.function.ListWriterFunction
import org.broadinstitute.sting.commandline.Hidden
import org.broadinstitute.sting.commandline
class DataProcessingPipeline extends QScript {
qscript =>
@ -41,34 +42,34 @@ class DataProcessingPipeline extends QScript {
@Input(doc="The path to the binary of bwa (usually BAM files have already been mapped - but if you want to remap this is the option)", fullName="path_to_bwa", shortName="bwa", required=false)
var bwaPath: File = _
@Input(doc="the project name determines the final output (BAM file) base name. Example NA12878 yields NA12878.processed.bam", fullName="project", shortName="p", required=false)
@Argument(doc="the project name determines the final output (BAM file) base name. Example NA12878 yields NA12878.processed.bam", fullName="project", shortName="p", required=false)
var projectName: String = "project"
@Input(doc="Output path for the processed BAM files.", fullName="output_directory", shortName="outputDir", required=false)
@Argument(doc="Output path for the processed BAM files.", fullName="output_directory", shortName="outputDir", required=false)
var outputDir: String = ""
@Input(doc="the -L interval string to be used by GATK - output bams at interval only", fullName="gatk_interval_string", shortName="L", required=false)
@Argument(doc="the -L interval string to be used by GATK - output bams at interval only", fullName="gatk_interval_string", shortName="L", required=false)
var intervalString: String = ""
@Input(doc="an intervals file to be used by GATK - output bams at intervals only", fullName="gatk_interval_file", shortName="intervals", required=false)
var intervals: File = _
@Input(doc="Cleaning model: KNOWNS_ONLY, USE_READS or USE_SW", fullName="clean_model", shortName="cm", required=false)
@Argument(doc="Cleaning model: KNOWNS_ONLY, USE_READS or USE_SW", fullName="clean_model", shortName="cm", required=false)
var cleaningModel: String = "USE_READS"
@Input(doc="Decompose input BAM file and fully realign it using BWA and assume Single Ended reads", fullName="use_bwa_single_ended", shortName="bwase", required=false)
@Argument(doc="Decompose input BAM file and fully realign it using BWA and assume Single Ended reads", fullName="use_bwa_single_ended", shortName="bwase", required=false)
var useBWAse: Boolean = false
@Input(doc="Decompose input BAM file and fully realign it using BWA and assume Pair Ended reads", fullName="use_bwa_pair_ended", shortName="bwape", required=false)
@Argument(doc="Decompose input BAM file and fully realign it using BWA and assume Pair Ended reads", fullName="use_bwa_pair_ended", shortName="bwape", required=false)
var useBWApe: Boolean = false
@Input(doc="Decompose input BAM file and fully realign it using BWA SW", fullName="use_bwa_sw", shortName="bwasw", required=false)
@Argument(doc="Decompose input BAM file and fully realign it using BWA SW", fullName="use_bwa_sw", shortName="bwasw", required=false)
var useBWAsw: Boolean = false
@Input(doc="Number of threads BWA should use", fullName="bwa_threads", shortName="bt", required=false)
@Argument(doc="Number of threads BWA should use", fullName="bwa_threads", shortName="bt", required=false)
var bwaThreads: Int = 1
@Input(doc="Perform validation on the BAM files", fullName="validation", shortName="vs", required=false)
@Argument(doc="Perform validation on the BAM files", fullName="validation", shortName="vs", required=false)
var validation: Boolean = false
@ -76,15 +77,15 @@ class DataProcessingPipeline extends QScript {
* Hidden Parameters
****************************************************************************/
@Hidden
@Input(doc="How many ways to scatter/gather", fullName="scatter_gather", shortName="sg", required=false)
@Argument(doc="How many ways to scatter/gather", fullName="scatter_gather", shortName="sg", required=false)
var nContigs: Int = -1
@Hidden
@Input(doc="Define the default platform for Count Covariates -- useful for techdev purposes only.", fullName="default_platform", shortName="dp", required=false)
@Argument(doc="Define the default platform for Count Covariates -- useful for techdev purposes only.", fullName="default_platform", shortName="dp", required=false)
var defaultPlatform: String = ""
@Hidden
@Input(doc="Run the pipeline in test mode only", fullName = "test_mode", shortName = "test", required=false)
@Argument(doc="Run the pipeline in test mode only", fullName = "test_mode", shortName = "test", required=false)
var testMode: Boolean = false

65
public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala
View File

@ -28,7 +28,7 @@ import function.QFunction
import java.io.File
import org.broadinstitute.sting.commandline._
import org.broadinstitute.sting.queue.util._
import org.broadinstitute.sting.queue.engine.{QGraphSettings, QGraph}
import org.broadinstitute.sting.queue.engine.{QStatusMessenger, QGraphSettings, QGraph}
import collection.JavaConversions._
import org.broadinstitute.sting.utils.classloader.PluginManager
import org.broadinstitute.sting.utils.exceptions.UserException
@ -90,29 +90,46 @@ class QCommandLine extends CommandLineProgram with Logging {
private var qScriptClasses: File = _
private var shuttingDown = false
private lazy val pluginManager = {
private lazy val qScriptPluginManager = {
qScriptClasses = IOUtils.tempDir("Q-Classes-", "", settings.qSettings.tempDirectory)
qScriptManager.loadScripts(scripts, qScriptClasses)
new PluginManager[QScript](classOf[QScript], Seq(qScriptClasses.toURI.toURL))
}
QFunction.parsingEngine = new ParsingEngine(this)
private lazy val qStatusMessengerPluginManager = {
new PluginManager[QStatusMessenger](classOf[QStatusMessenger])
}
ClassFieldCache.parsingEngine = new ParsingEngine(this)
/**
* Takes the QScripts passed in, runs their script() methods, retrieves their generated
* functions, and then builds and runs a QGraph based on the dependencies.
*/
def execute = {
val allStatusMessengers = qStatusMessengerPluginManager.createAllTypes()
if (settings.qSettings.runName == null)
settings.qSettings.runName = FilenameUtils.removeExtension(scripts.head.getName)
if (IOUtils.isDefaultTempDir(settings.qSettings.tempDirectory))
settings.qSettings.tempDirectory = IOUtils.absolute(settings.qSettings.runDirectory, ".queue/tmp")
qGraph.initializeWithSettings(settings)
val allQScripts = pluginManager.createAllTypes()
for (statusMessenger <- allStatusMessengers) {
loadArgumentsIntoObject(statusMessenger)
}
for (statusMessenger <- allStatusMessengers) {
statusMessenger.started()
}
val allQScripts = qScriptPluginManager.createAllTypes()
for (script <- allQScripts) {
logger.info("Scripting " + pluginManager.getName(script.getClass.asSubclass(classOf[QScript])))
logger.info("Scripting " + qScriptPluginManager.getName(script.getClass.asSubclass(classOf[QScript])))
loadArgumentsIntoObject(script)
// TODO: Pulling inputs can be time/io expensive! Some scripts are using the files to generate functions-- even for dry runs-- so pull it all down for now.
//if (settings.run)
script.pullInputs()
script.qSettings = settings.qSettings
try {
script.script()
@ -142,11 +159,18 @@ class QCommandLine extends CommandLineProgram with Logging {
logger.info("Writing final jobs report...")
qGraph.writeJobsReport()
if (!qGraph.success) {
if (!success) {
logger.info("Done with errors")
qGraph.logFailed()
for (statusMessenger <- allStatusMessengers)
statusMessenger.exit("Done with errors")
1
} else {
if (settings.run) {
allQScripts.foreach(_.pushOutputs())
for (statusMessenger <- allStatusMessengers)
statusMessenger.done(allQScripts.map(_.remoteOutputs))
}
0
}
}
@ -158,19 +182,30 @@ class QCommandLine extends CommandLineProgram with Logging {
override def canAddArgumentsDynamically = true
/**
* Returns the list of QScripts passed in via -S so that their
* arguments can be inspected before QScript.script is called.
* @return Array of QScripts passed in.
* Returns the list of QScripts passed in via -S and other plugins
* so that their arguments can be inspected before QScript.script is called.
* @return Array of dynamic sources
*/
override def getArgumentSources =
pluginManager.getPlugins.toIterable.toArray.asInstanceOf[Array[Class[_]]]
override def getArgumentSources = {
var plugins = Seq.empty[Class[_]]
plugins ++= qScriptPluginManager.getPlugins
plugins ++= qStatusMessengerPluginManager.getPlugins
plugins.toArray
}
/**
* Returns the name of a QScript
* @return The name of a QScript
* Returns the name of a script/plugin
* @return The name of a script/plugin
*/
override def getArgumentSourceName(source: Class[_]) =
pluginManager.getName(source.asSubclass(classOf[QScript]))
override def getArgumentSourceName(source: Class[_]) = {
if (classOf[QScript].isAssignableFrom(source))
qScriptPluginManager.getName(source.asSubclass(classOf[QScript]))
else if (classOf[QStatusMessenger].isAssignableFrom(source))
qStatusMessengerPluginManager.getName(source.asSubclass(classOf[QStatusMessenger]))
else
null
}
/**
* Returns a ScalaCompoundArgumentTypeDescriptor that can parse argument sources into scala collections.

34
public/scala/src/org/broadinstitute/sting/queue/QScript.scala
View File

@ -27,7 +27,8 @@ package org.broadinstitute.sting.queue
import engine.JobRunInfo
import org.broadinstitute.sting.queue.function.QFunction
import annotation.target.field
import util.{StringFileConversions, PrimitiveOptionConversions, Logging}
import util._
import org.broadinstitute.sting.commandline.ArgumentSource
/**
* Defines a Queue pipeline as a collection of CommandLineFunctions.
@ -106,6 +107,37 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon
def addAll(functions: Seq[QFunction]) {
functions.foreach( f => add(f) )
}
def pullInputs() {
val inputs = ClassFieldCache.getFieldFiles(this, inputFields)
for (remoteFile <- filterRemoteFiles(inputs)) {
logger.info("Pulling %s from %s".format(remoteFile.getAbsolutePath, remoteFile.remoteDescription))
remoteFile.pullToLocal()
}
}
def pushOutputs() {
val outputs = ClassFieldCache.getFieldFiles(this, outputFields)
for (remoteFile <- filterRemoteFiles(outputs)) {
logger.info("Pushing %s to %s".format(remoteFile.getAbsolutePath, remoteFile.remoteDescription))
remoteFile.pushToRemote()
}
}
def remoteOutputs: Map[ArgumentSource, Seq[RemoteFile]] =
outputFields.map(field => (field -> filterRemoteFiles(ClassFieldCache.getFieldFiles(this, field)))).filter(tuple => !tuple._2.isEmpty).toMap
private def filterRemoteFiles(fields: Seq[File]): Seq[RemoteFile] =
fields.filter(field => field != null && field.isInstanceOf[RemoteFile]).map(_.asInstanceOf[RemoteFile])
/** The complete list of fields. */
def functionFields: Seq[ArgumentSource] = ClassFieldCache.classFunctionFields(this.getClass)
/** The @Input fields. */
def inputFields: Seq[ArgumentSource] = ClassFieldCache.classInputFields(this.getClass)
/** The @Output fields. */
def outputFields: Seq[ArgumentSource] = ClassFieldCache.classOutputFields(this.getClass)
/** The @Argument fields. */
def argumentFields: Seq[ArgumentSource] = ClassFieldCache.classArgumentFields(this.getClass)
}
object QScript {

7
public/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala
View File

@ -11,6 +11,7 @@ import org.apache.log4j.Level
import scala.tools.nsc.util.{FakePos, NoPosition, Position}
import org.broadinstitute.sting.queue.util.TextFormatUtils._
import org.broadinstitute.sting.utils.classloader.JVMUtils
import tools.util.StringOps
/**
* Plugin manager for QScripts which loads QScripts into the current class loader.
@ -63,7 +64,7 @@ object QScriptManager extends Logging {
* Heavily based on scala/src/compiler/scala/tools/nsc/reporters/ConsoleReporter.scala
*/
private class Log4JReporter(val settings: Settings) extends AbstractReporter {
def displayPrompt { throw new UnsupportedOperationException("Unable to prompt the user. Prompting should be off.") }
def displayPrompt() { throw new UnsupportedOperationException("Unable to prompt the user. Prompting should be off.") }
/**
* Displays the message at position with severity.
@ -98,9 +99,9 @@ object QScriptManager extends Logging {
*/
def printSummary() {
if (WARNING.count > 0)
printMessage(Level.WARN, countElementsAsString(WARNING.count, "warning") + " found")
printMessage(Level.WARN, StringOps.countElementsAsString(WARNING.count, "warning") + " found")
if (ERROR.count > 0)
printMessage(Level.ERROR, countElementsAsString(ERROR.count, "error") + " found")
printMessage(Level.ERROR, StringOps.countElementsAsString(ERROR.count, "error") + " found")
}
/**

2
public/scala/src/org/broadinstitute/sting/queue/engine/FunctionEdge.scala
View File

@ -185,7 +185,7 @@ class FunctionEdge(val function: QFunction, val inputs: QNode, val outputs: QNod
val tailLines = IOUtils.tail(errorFile, maxLines)
val nl = "%n".format()
val summary = if (tailLines.size > maxLines) "Last %d lines".format(maxLines) else "Contents"
this.function.jobErrorLines = collection.JavaConversions.asScalaIterable(tailLines).toSeq
this.function.jobErrorLines = collection.JavaConversions.collectionAsScalaIterable(tailLines).toSeq
logger.error("%s of %s:%n%s".format(summary, errorFile, StringUtils.join(tailLines, nl)))
} else {
logger.error("Unable to access log file: %s".format(errorFile))

13
public/scala/src/org/broadinstitute/sting/queue/engine/QStatusMessenger.scala 100644
View File

@ -0,0 +1,13 @@
package org.broadinstitute.sting.queue.engine
import org.broadinstitute.sting.commandline.ArgumentSource
import org.broadinstitute.sting.queue.util.RemoteFile
/**
* Plugin to sends QStatus messages
*/
trait QStatusMessenger {
def started()
def done(files: Seq[Map[ArgumentSource, Seq[RemoteFile]]])
def exit(message: String)
}

407
public/scala/src/org/broadinstitute/sting/queue/extensions/cancer/MuTect.scala 100644
View File

File diff suppressed because one or more lines are too long

7
public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala
View File

@ -28,6 +28,7 @@ import org.broadinstitute.sting.queue.function.scattergather.GatherFunction
import org.broadinstitute.sting.queue.extensions.picard.PicardBamFunction
import org.broadinstitute.sting.queue.function.{RetryMemoryLimit, QFunction}
import org.broadinstitute.sting.gatk.io.stubs.SAMFileWriterArgumentTypeDescriptor
import org.broadinstitute.sting.queue.util.ClassFieldCache
/**
* Merges BAM files using net.sf.picard.sam.MergeSamFiles.
@ -47,13 +48,13 @@ class BamGatherFunction extends GatherFunction with PicardBamFunction with Retry
// bam_compression and index_output_bam_on_the_fly from SAMFileWriterArgumentTypeDescriptor
// are added by the GATKExtensionsGenerator to the subclass of CommandLineGATK
val compression = QFunction.findField(originalFunction.getClass, SAMFileWriterArgumentTypeDescriptor.COMPRESSION_FULLNAME)
val compression = ClassFieldCache.findField(originalFunction.getClass, SAMFileWriterArgumentTypeDescriptor.COMPRESSION_FULLNAME)
this.compressionLevel = originalGATK.getFieldValue(compression).asInstanceOf[Option[Int]]
val disableIndex = QFunction.findField(originalFunction.getClass, SAMFileWriterArgumentTypeDescriptor.DISABLE_INDEXING_FULLNAME)
val disableIndex = ClassFieldCache.findField(originalFunction.getClass, SAMFileWriterArgumentTypeDescriptor.DISABLE_INDEXING_FULLNAME)
this.createIndex = Some(!originalGATK.getFieldValue(disableIndex).asInstanceOf[Boolean])
val enableMD5 = QFunction.findField(originalFunction.getClass, SAMFileWriterArgumentTypeDescriptor.ENABLE_MD5_FULLNAME)
val enableMD5 = ClassFieldCache.findField(originalFunction.getClass, SAMFileWriterArgumentTypeDescriptor.ENABLE_MD5_FULLNAME)
this.createMD5 = Some(originalGATK.getFieldValue(enableMD5).asInstanceOf[Boolean])
super.freezeFieldValues()

2
public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala
View File

@ -92,6 +92,6 @@ object GATKIntervals {
}
private def createBinding(interval: String, argumentName: String, tags: Tags): IntervalBinding[Feature] = {
ArgumentTypeDescriptor.parseBinding(interval, classOf[Feature], classOf[IntervalBinding[Feature]], argumentName, tags, argumentName).asInstanceOf[IntervalBinding[Feature]]
ArgumentTypeDescriptor.parseBinding(new ArgumentMatchStringValue(interval), classOf[Feature], classOf[IntervalBinding[Feature]], argumentName, tags, argumentName).asInstanceOf[IntervalBinding[Feature]]
}
}

5
public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/VcfGatherFunction.scala
View File

@ -27,6 +27,7 @@ package org.broadinstitute.sting.queue.extensions.gatk
import org.broadinstitute.sting.queue.function.scattergather.GatherFunction
import org.broadinstitute.sting.queue.function.{RetryMemoryLimit, QFunction}
import org.broadinstitute.sting.gatk.io.stubs.VCFWriterArgumentTypeDescriptor
import org.broadinstitute.sting.queue.util.ClassFieldCache
/**
* Merges a vcf text file.
@ -46,10 +47,10 @@ class VcfGatherFunction extends CombineVariants with GatherFunction with RetryMe
// NO_HEADER and sites_only from VCFWriterArgumentTypeDescriptor
// are added by the GATKExtensionsGenerator to the subclass of CommandLineGATK
val noHeader = QFunction.findField(originalFunction.getClass, VCFWriterArgumentTypeDescriptor.NO_HEADER_ARG_NAME)
val noHeader = ClassFieldCache.findField(originalFunction.getClass, VCFWriterArgumentTypeDescriptor.NO_HEADER_ARG_NAME)
this.no_cmdline_in_header = originalGATK.getFieldValue(noHeader).asInstanceOf[Boolean]
val sitesOnly = QFunction.findField(originalFunction.getClass, VCFWriterArgumentTypeDescriptor.SITES_ONLY_ARG_NAME)
val sitesOnly = ClassFieldCache.findField(originalFunction.getClass, VCFWriterArgumentTypeDescriptor.SITES_ONLY_ARG_NAME)
this.sites_only = originalGATK.getFieldValue(sitesOnly).asInstanceOf[Boolean]
// ensure that the gather function receives the same unsafe parameter as the scattered function

60
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/CalculateHsMetrics.scala 100644
View File

@ -0,0 +1,60 @@
package org.broadinstitute.sting.queue.extensions.picard
import org.broadinstitute.sting.commandline.{Argument, Output, Input}
import java.io.File
/**
* Created with IntelliJ IDEA.
* User: delangel
* Date: 10/9/12
* Time: 5:59 PM
* To change this template use File | Settings | File Templates.
*/
class CalculateHsMetrics extends org.broadinstitute.sting.queue.function.JavaCommandLineFunction with PicardBamFunction {
analysisName = "CalculateHsMetrics"
javaMainClass = "net.sf.picard.sam.CalculateHsMetrics"
@Input(doc="The input SAM or BAM files to analyze. Must be coordinate sorted.", shortName = "input", fullName = "input_bam_files", required = true)
var input: Seq[File] = Nil
@Output(doc="The output file to write statistics to", shortName = "output", fullName = "output_file", required = true)
var output: File = _
@Argument(doc="Interval list with targets", shortName = "targets", fullName = "target_list", required = true)
var targets: File = _
@Argument(doc="Interval list with baits", shortName = "baits", fullName = "bait_list", required = true)
var baits: File = _
@Argument(doc="Reference file", shortName = "reference", fullName = "reference", required = true)
var reference: File = _
/*
@Argument(doc = "Maximum number of file handles to keep open when spilling read ends to disk. Set this number a little lower than the per-process maximum number of file that may be open. This number can be found by executing the 'ulimit -n' command on a Unix system.", shortName = "max_file_handles", fullName ="max_file_handles_for_read_ends_maps", required=false)
var MAX_FILE_HANDLES_FOR_READ_ENDS_MAP: Int = -1;
@Argument(doc = "This number, plus the maximum RAM available to the JVM, determine the memory footprint used by some of the sorting collections. If you are running out of memory, try reducing this number.", shortName = "sorting_ratio", fullName = "sorting_collection_size_ratio", required = false)
var SORTING_COLLECTION_SIZE_RATIO: Double = -1
*/
override def freezeFieldValues() {
super.freezeFieldValues()
// if (outputIndex == null && output != null)
// outputIndex = new File(output.getName.stripSuffix(".bam") + ".bai")
}
val level = "SAMPLE"
override def inputBams = input
override def outputBam = output
//this.sortOrder = null
//this.createIndex = Some(true)
override def commandLine = super.commandLine +
required("BAIT_INTERVALS=" + baits) +
required("TARGET_INTERVALS=" + targets) +
required("REFERENCE_SEQUENCE=" + reference) +
optional("METRIC_ACCUMULATION_LEVEL="+level)/*+
conditional(REMOVE_DUPLICATES, "REMOVE_DUPLICATES=true") +
conditional(MAX_FILE_HANDLES_FOR_READ_ENDS_MAP > 0, "MAX_FILE_HANDLES_FOR_READ_ENDS_MAP=" + MAX_FILE_HANDLES_FOR_READ_ENDS_MAP.toString) +
conditional(SORTING_COLLECTION_SIZE_RATIO > 0, "SORTING_COLLECTION_SIZE_RATIO=" + SORTING_COLLECTION_SIZE_RATIO.toString) */
}

32
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/CollectGcBiasMetrics.scala 100644
View File

@ -0,0 +1,32 @@
package org.broadinstitute.sting.queue.extensions.picard
import org.broadinstitute.sting.commandline.{Argument, Output, Input}
import java.io.File
/**
* Created with IntelliJ IDEA.
* User: delangel
* Date: 10/10/12
* Time: 10:37 AM
* To change this template use File | Settings | File Templates.
*/
class CollectGcBiasMetrics extends org.broadinstitute.sting.queue.function.JavaCommandLineFunction with PicardBamFunction {
analysisName = "CalculateGcMetrics"
javaMainClass = "net.sf.picard.sam.CalculateGcMetrics"
@Input(doc="The input SAM or BAM files to analyze. Must be coordinate sorted.", shortName = "input", fullName = "input_bam_files", required = true)
var input: Seq[File] = Nil
@Output(doc="The output file to write statistics to", shortName = "output", fullName = "output_file", required = true)
var output: File = _
@Argument(doc="Reference file", shortName = "reference", fullName = "reference", required = true)
var reference: File = _
override def inputBams = input
override def outputBam = output
override def commandLine = super.commandLine +
required("CHART_OUTPUT=" + output+".pdf") +
required("REFERENCE_SEQUENCE=" + reference) +
required("ASSUME_SORTED=true")
}

36
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/CollectMultipleMetrics.scala 100644
View File

@ -0,0 +1,36 @@
package org.broadinstitute.sting.queue.extensions.picard
import org.broadinstitute.sting.commandline.{Argument, Output, Input}
import java.io.File
/**
* Created with IntelliJ IDEA.
* User: delangel
* Date: 10/10/12
* Time: 10:37 AM
* To change this template use File | Settings | File Templates.
*/
class CollectMultipleMetrics extends org.broadinstitute.sting.queue.function.JavaCommandLineFunction with PicardBamFunction{
analysisName = "CalculateMultipleMetrics"
javaMainClass = "net.sf.picard.sam.CalculateMultipleMetrics"
@Input(doc="The input SAM or BAM files to analyze. Must be coordinate sorted.", shortName = "input", fullName = "input_bam_files", required = true)
var input: Seq[File] = Nil
@Output(doc="The output file to write statistics to", shortName = "output", fullName = "output_file", required = true)
var output: File = _
@Argument(doc="Reference file", shortName = "reference", fullName = "reference", required = true)
var reference: File = _
override def inputBams = input
override def outputBam = output
override def commandLine = super.commandLine +
required("REFERENCE_SEQUENCE=" + reference) +
required("ASSUME_SORTED=true") +
required("PROGRAM=QualityScoreDistribution") +
required("PROGRAM=MeanQualityByCycle") +
required("PROGRAM=CollectAlignmentSummaryMetrics" )
}

80
public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala
View File

@ -28,7 +28,6 @@ import java.io.File
import java.lang.annotation.Annotation
import org.broadinstitute.sting.commandline._
import org.broadinstitute.sting.queue.{QException, QSettings}
import collection.JavaConversions._
import java.lang.IllegalStateException
import org.broadinstitute.sting.queue.util._
import org.broadinstitute.sting.utils.io.IOUtils
@ -194,13 +193,13 @@ trait QFunction extends Logging with QJobReport {
def failOutputs: Seq[File] = statusPrefixes.map(path => new File(path + ".fail"))
/** The complete list of fields on this CommandLineFunction. */
def functionFields = QFunction.classFields(this.functionFieldClass).functionFields
def functionFields: Seq[ArgumentSource] = ClassFieldCache.classFunctionFields(this.functionFieldClass)
/** The @Input fields on this CommandLineFunction. */
def inputFields = QFunction.classFields(this.functionFieldClass).inputFields
def inputFields: Seq[ArgumentSource] = ClassFieldCache.classInputFields(this.functionFieldClass)
/** The @Output fields on this CommandLineFunction. */
def outputFields = QFunction.classFields(this.functionFieldClass).outputFields
def outputFields: Seq[ArgumentSource] = ClassFieldCache.classOutputFields(this.functionFieldClass)
/** The @Argument fields on this CommandLineFunction. */
def argumentFields = QFunction.classFields(this.functionFieldClass).argumentFields
def argumentFields: Seq[ArgumentSource] = ClassFieldCache.classArgumentFields(this.functionFieldClass)
/**
* Returns the class that should be used for looking up fields.
@ -475,79 +474,12 @@ trait QFunction extends Logging with QJobReport {
* @param source Field to get the value for.
* @return value of the field.
*/
def getFieldValue(source: ArgumentSource) = ReflectionUtils.getValue(invokeObj(source), source.field)
def getFieldValue(source: ArgumentSource) = ClassFieldCache.getFieldValue(this, source)
/**
* Gets the value of a field.
* @param source Field to set the value for.
* @return value of the field.
*/
def setFieldValue(source: ArgumentSource, value: Any) = ReflectionUtils.setValue(invokeObj(source), source.field, value)
/**
* Walks gets the fields in this object or any collections in that object
* recursively to find the object holding the field to be retrieved or set.
* @param source Field find the invoke object for.
* @return Object to invoke the field on.
*/
private def invokeObj(source: ArgumentSource) = source.parentFields.foldLeft[AnyRef](this)(ReflectionUtils.getValue(_, _))
}
object QFunction {
var parsingEngine: ParsingEngine = _
/**
* The list of fields defined on a class
* @param clazz The class to lookup fields.
*/
private class ClassFields(clazz: Class[_]) {
/** The complete list of fields on this CommandLineFunction. */
val functionFields: Seq[ArgumentSource] = parsingEngine.extractArgumentSources(clazz).toSeq
/** The @Input fields on this CommandLineFunction. */
val inputFields = functionFields.filter(source => ReflectionUtils.hasAnnotation(source.field, classOf[Input]))
/** The @Output fields on this CommandLineFunction. */
val outputFields = functionFields.filter(source => ReflectionUtils.hasAnnotation(source.field, classOf[Output]))
/** The @Argument fields on this CommandLineFunction. */
val argumentFields = functionFields.filter(source => ReflectionUtils.hasAnnotation(source.field, classOf[Argument]))
}
/**
* The mapping from class to fields.
*/
private var classFieldsMap = Map.empty[Class[_], ClassFields]
/**
* Returns the field on clazz.
* @param clazz Class to search.
* @param name Name of the field to return.
* @return Argument source for the field.
*/
def findField(clazz: Class[_], name: String) = {
classFields(clazz).functionFields.find(_.field.getName == name) match {
case Some(source) => source
case None => throw new QException("Could not find a field on class %s with name %s".format(clazz, name))
}
}
/**
* Returns the fields for a class.
* @param clazz Class to retrieve fields for.
* @return the fields for the class.
*/
private def classFields(clazz: Class[_]) = {
classFieldsMap.get(clazz) match {
case Some(classFields) => classFields
case None =>
val classFields = new ClassFields(clazz)
classFieldsMap += clazz -> classFields
classFields
}
}
/**
* Returns the Seq of fields for a QFunction class.
* @param clazz Class to retrieve fields for.
* @return the fields of the class.
*/
def classFunctionFields(clazz: Class[_]) = classFields(clazz).functionFields
def setFieldValue(source: ArgumentSource, value: Any) = ClassFieldCache.setFieldValue(this, source, value)
}

9
public/scala/src/org/broadinstitute/sting/queue/function/scattergather/CloneFunction.scala
View File

@ -25,13 +25,14 @@
package org.broadinstitute.sting.queue.function.scattergather
import org.broadinstitute.sting.commandline.ArgumentSource
import org.broadinstitute.sting.queue.function.{QFunction, CommandLineFunction}
import org.broadinstitute.sting.queue.function.CommandLineFunction
import org.broadinstitute.sting.queue.util.ClassFieldCache
/**
* Shadow clones another command line function.
*/
object CloneFunction {
private lazy val cloneFunctionFields = QFunction.classFunctionFields(classOf[CloneFunction])
private lazy val cloneFunctionFields = ClassFieldCache.classFunctionFields(classOf[CloneFunction])
}
class CloneFunction extends CommandLineFunction {
@ -76,7 +77,7 @@ class CloneFunction extends CommandLineFunction {
def commandLine = withScatterPart(() => originalFunction.commandLine)
def getFieldValue(field: String): AnyRef = {
val source = QFunction.findField(originalFunction.getClass, field)
val source = ClassFieldCache.findField(originalFunction.getClass, field)
getFieldValue(source)
}
@ -98,7 +99,7 @@ class CloneFunction extends CommandLineFunction {
}
def setFieldValue(field: String, value: Any) {
val source = QFunction.findField(originalFunction.getClass, field)
val source = ClassFieldCache.findField(originalFunction.getClass, field)
setFieldValue(source, value)
}

183
public/scala/src/org/broadinstitute/sting/queue/util/ClassFieldCache.scala 100644
View File

@ -0,0 +1,183 @@
package org.broadinstitute.sting.queue.util
import org.broadinstitute.sting.commandline._
import scala.Some
import org.broadinstitute.sting.queue.QException
import collection.JavaConversions._
import java.io.File
/**
* Utilities and a static cache of argument fields for various classes populated by the parsingEngine.
* Because this class works with the ParsingEngine it can walk @ArgumentCollection hierarchies.
*/
object ClassFieldCache {
var parsingEngine: ParsingEngine = _
//
// Field caching
//
/**
* The list of fields defined on a class
* @param clazz The class to lookup fields.
*/
private class ClassFields(clazz: Class[_]) {
/** The complete list of fields on this CommandLineFunction. */
val functionFields: Seq[ArgumentSource] = parsingEngine.extractArgumentSources(clazz).toSeq
/** The @Input fields on this CommandLineFunction. */
val inputFields: Seq[ArgumentSource] = functionFields.filter(source => ReflectionUtils.hasAnnotation(source.field, classOf[Input]))
/** The @Output fields on this CommandLineFunction. */
val outputFields: Seq[ArgumentSource] = functionFields.filter(source => ReflectionUtils.hasAnnotation(source.field, classOf[Output]))
/** The @Argument fields on this CommandLineFunction. */
val argumentFields: Seq[ArgumentSource] = functionFields.filter(source => ReflectionUtils.hasAnnotation(source.field, classOf[Argument]))
}
/**
* The mapping from class to fields.
*/
private var classFieldsMap = Map.empty[Class[_], ClassFields]
/**
* Returns the fields for a class.
* @param clazz Class to retrieve fields for.
* @return the fields for the class.
*/
private def classFields(clazz: Class[_]): ClassFields = {
classFieldsMap.get(clazz) match {
case Some(classFields) => classFields
case None =>
val classFields = new ClassFields(clazz)
classFieldsMap += clazz -> classFields
classFields
}
}
/**
* Returns the field on clazz.
* @param clazz Class to search.
* @param name Name of the field to return.
* @return Argument source for the field.
*/
def findField(clazz: Class[_], name: String): ArgumentSource = {
classFields(clazz).functionFields.find(_.field.getName == name) match {
case Some(source) => source
case None => throw new QException("Could not find a field on class %s with name %s".format(clazz, name))
}
}
/**
* Returns the Seq of fields for a QFunction class.
* @param clazz Class to retrieve fields for.
* @return the fields of the class.
*/
def classFunctionFields(clazz: Class[_]): Seq[ArgumentSource] = classFields(clazz).functionFields
/**
* Returns the Seq of inputs for a QFunction class.
* @param clazz Class to retrieve inputs for.
* @return the inputs of the class.
*/
def classInputFields(clazz: Class[_]): Seq[ArgumentSource] = classFields(clazz).inputFields
/**
* Returns the Seq of outputs for a QFunction class.
* @param clazz Class to retrieve outputs for.
* @return the outputs of the class.
*/
def classOutputFields(clazz: Class[_]): Seq[ArgumentSource] = classFields(clazz).outputFields
/**
* Returns the Seq of arguments for a QFunction class.
* @param clazz Class to retrieve arguments for.
* @return the arguments of the class.
*/
def classArgumentFields(clazz: Class[_]): Seq[ArgumentSource] = classFields(clazz).argumentFields
//
// get/set fields as AnyRef
//
/**
* Gets the value of a field.
* @param obj Top level object storing the source info.
* @param source Field to get the value for.
* @return value of the field.
*/
def getFieldValue(obj: AnyRef, source: ArgumentSource) = ReflectionUtils.getValue(invokeObj(obj, source), source.field)
/**
* Gets the value of a field.
* @param obj Top level object storing the source info.
* @param source Field to set the value for.
* @return value of the field.
*/
def setFieldValue(obj: AnyRef, source: ArgumentSource, value: Any) = ReflectionUtils.setValue(invokeObj(obj, source), source.field, value)
/**
* Walks gets the fields in this object or any collections in that object
* recursively to find the object holding the field to be retrieved or set.
* @param obj Top level object storing the source info.
* @param source Field find the invoke object for.
* @return Object to invoke the field on.
*/
private def invokeObj(obj: AnyRef, source: ArgumentSource) = source.parentFields.foldLeft[AnyRef](obj)(ReflectionUtils.getValue(_, _))
//
// get/set fields as java.io.File
//
/**
* Gets the files from the fields. The fields must be a File, a FileExtension, or a Seq or Set of either.
* @param obj Top level object storing the source info.
* @param fields Fields to get files.
* @return for the fields.
*/
def getFieldFiles(obj: AnyRef, fields: Seq[ArgumentSource]): Seq[File] = {
var files: Seq[File] = Nil
for (field <- fields)
files ++= getFieldFiles(obj, field)
files.distinct
}
/**
* Gets the files from the field. The field must be a File, a FileExtension, or a Seq or Set of either.
* @param obj Top level object storing the source info.
* @param field Field to get files.
* @return for the field.
*/
def getFieldFiles(obj: AnyRef, field: ArgumentSource): Seq[File] = {
var files: Seq[File] = Nil
CollectionUtils.foreach(getFieldValue(obj, field), (fieldValue) => {
val file = fieldValueToFile(field, fieldValue)
if (file != null)
files :+= file
})
files.distinct
}
/**
* Gets the file from the field. The field must be a File or a FileExtension and not a Seq or Set.
* @param obj Top level object storing the source info.
* @param field Field to get the file.
* @return for the field.
*/
def getFieldFile(obj: AnyRef, field: ArgumentSource): File =
fieldValueToFile(field, getFieldValue(obj, field))
/**
* Converts the field value to a file. The field must be a File or a FileExtension.
* @param field Field to get the file.
* @param value Value of the File or FileExtension or null.
* @return Null if value is null, otherwise the File.
* @throws QException if the value is not a File or FileExtension.
*/
private def fieldValueToFile(field: ArgumentSource, value: Any): File = value match {
case file: File => file
case null => null
case unknown => throw new QException("Non-file found. Try removing the annotation, change the annotation to @Argument, or extend File with FileExtension: %s: %s".format(field.field, unknown))
}
}

3
public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala
View File

@ -57,7 +57,8 @@ object QScriptUtils {
for (file <- fromFile(in).getLines())
if (!file.startsWith("#") && !file.isEmpty )
list :+= new File(file.trim())
list.sortWith(_.compareTo(_) < 0)
// list.sortWith(_.compareTo(_) < 0)
list
}
/**

14
public/scala/src/org/broadinstitute/sting/queue/util/RemoteFile.scala 100644
View File

@ -0,0 +1,14 @@
package org.broadinstitute.sting.queue.util
import java.io.File
import org.broadinstitute.sting.utils.io.FileExtension
/**
* An extension of java.io.File that can be pulled from or pushed to a remote location.
*/
trait RemoteFile extends File with FileExtension {
def pullToLocal()
def pushToRemote()
def deleteRemote()
def remoteDescription: String
}

1
settings/ivysettings.xml
View File

@ -7,7 +7,6 @@
<artifact pattern="${repository.dir}/[organisation]/[artifact]-[revision].[ext]" />
<ivy pattern="${repository.dir}/[organisation]/[module]-[revision].xml" />
</filesystem>
<ibiblio name="reflections-repo" m2compatible="true" root="http://reflections.googlecode.com/svn/repo" />
<ibiblio name="maven" root="http://repo1.maven.org/maven2" m2compatible="true" />
</chain>
</resolvers>