Marked boolean SAMFileWriterATD arguments as flags so scala generator maps them to Boolean instead of Option[Boolean].
Using the VCFWriterATD isCompressed to check if the VCF index will be auto generated. Tracking BAM and Tribble indexes as @Inputs and @Outputs in generated QFunctions. Updates to the BamGatherFunction to disable the index during merge when disable_bam_indexing = true. Made a shortcut for live-running pipelinetest, pipelinetestrun. git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@5606 348d0f76-0448-11de-a6fe-93d51630548a
This commit is contained in:
kshakir
parent
866f4fd569
commit
0a58d7aa1a
17
build.xml
17
build.xml
|
|
@ -191,11 +191,6 @@
|
|||
<available file="${drmaa.jar}"/>
|
||||
</condition>
|
||||
|
||||
<!-- Get the pipeline run type. Default to dry. -->
|
||||
<condition property="pipeline.run" value="dry" else="${pipeline.run}">
|
||||
<equals arg1="${pipeline.run}" arg2="$${pipeline.run}" />
|
||||
</condition>
|
||||
|
||||
<echo message="GATK build : ${gatk.target}"/>
|
||||
<echo message="Scala build : ${scala.target}"/>
|
||||
<echo message="source revision : ${build.version}"/>
|
||||
|
|
@ -624,6 +619,11 @@
|
|||
<macrodef name="run-test">
|
||||
<attribute name="testtype"/>
|
||||
<sequential>
|
||||
<!-- Get the pipeline run type. Default to dry. -->
|
||||
<condition property="pipeline.run" value="dry" else="${pipeline.run}">
|
||||
<equals arg1="${pipeline.run}" arg2="$${pipeline.run}" />
|
||||
</condition>
|
||||
|
||||
<mkdir dir="${report}/@{testtype}"/>
|
||||
<echo message="Sting: Running @{testtype} test cases!"/>
|
||||
<taskdef resource="testngtasks" classpath="lib/testng-5.14.1.jar"/>
|
||||
|
|
@ -687,6 +687,13 @@
|
|||
</condition>
|
||||
<run-test testtype="${pipetype}"/>
|
||||
</target>
|
||||
<target name="pipelinetestrun" depends="test.compile" description="Run pipeline tests">
|
||||
<property name="pipeline.run" value="run"/>
|
||||
<condition property="pipetype" value="*PipelineTest" else="${single}">
|
||||
<not><isset property="single"/></not>
|
||||
</condition>
|
||||
<run-test testtype="${pipetype}"/>
|
||||
</target>
|
||||
|
||||
<!-- ***************************************************************************** -->
|
||||
<!-- *********** Tribble ********* -->
|
||||
|
|
|
|||
|
|
@ -178,7 +178,7 @@ public class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor
|
|||
null,
|
||||
"Turn off on-the-fly creation of indices for output BAM files.",
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
source.isHidden(),
|
||||
null,
|
||||
|
|
@ -194,7 +194,7 @@ public class SAMFileWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor
|
|||
SIMPLIFY_BAM_SHORTNAME,
|
||||
"If provided, output BAM files will be simplified to include just key reads for downstream variation discovery analyses (removing duplicates, PF-, non-primary reads), as well stripping all extended tags from the kept reads except the read group identifier",
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
source.isHidden(),
|
||||
null,
|
||||
|
|
|
|||
|
|
@ -133,7 +133,7 @@ public class VCFWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor {
|
|||
throw new MissingArgumentValueException(defaultArgumentDefinition);
|
||||
|
||||
// Should we compress the output stream?
|
||||
boolean compress = writerFileName != null && SUPPORTED_ZIPPED_SUFFIXES.contains(getFileSuffix(writerFileName));
|
||||
boolean compress = isCompressed(writerFileName);
|
||||
|
||||
boolean skipWritingHeader = argumentIsPresent(createNoHeaderArgumentDefinition(),matches);
|
||||
boolean doNotWriteGenotypes = argumentIsPresent(createSitesOnlyArgumentDefinition(),matches);
|
||||
|
|
@ -189,13 +189,21 @@ public class VCFWriterArgumentTypeDescriptor extends ArgumentTypeDescriptor {
|
|||
null );
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the file will be compressed.
|
||||
* @param writerFileName Name of the file
|
||||
* @return true if the file will be compressed.
|
||||
*/
|
||||
public static boolean isCompressed(String writerFileName) {
|
||||
return writerFileName != null && SUPPORTED_ZIPPED_SUFFIXES.contains(getFileSuffix(writerFileName));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a lower-cased version of the suffix of the provided file.
|
||||
* @param fileName the file name. Must not be null.
|
||||
* @return lower-cased version of the file suffix. Will not be null.
|
||||
*/
|
||||
private String getFileSuffix(String fileName) {
|
||||
private static String getFileSuffix(String fileName) {
|
||||
int indexOfLastDot = fileName.lastIndexOf(".");
|
||||
if ( indexOfLastDot == -1 )
|
||||
return "";
|
||||
|
|
|
|||
|
|
@ -24,9 +24,12 @@
|
|||
|
||||
package org.broadinstitute.sting.queue.extensions.gatk;
|
||||
|
||||
import net.sf.samtools.BAMIndex;
|
||||
import net.sf.samtools.SAMFileWriter;
|
||||
import org.broad.tribble.Tribble;
|
||||
import org.broad.tribble.vcf.VCFWriter;
|
||||
import org.broadinstitute.sting.commandline.*;
|
||||
import org.broadinstitute.sting.gatk.io.stubs.SAMFileWriterArgumentTypeDescriptor;
|
||||
|
||||
import java.io.File;
|
||||
import java.lang.annotation.Annotation;
|
||||
|
|
@ -70,7 +73,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
|
|||
" * Short name of %1$s%n" +
|
||||
" * @param value Short name of %1$s%n" +
|
||||
" */%n" +
|
||||
"def %4$s(value: %2$s) = this.%1$s = value%n",
|
||||
"def %4$s(value: %2$s) { this.%1$s = value }%n",
|
||||
getFieldName(),
|
||||
getFieldType(),
|
||||
getShortFieldGetter(),
|
||||
|
|
@ -132,17 +135,37 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
|
|||
// ROD Bindings are set by the RodBindField
|
||||
} else if (RodBindField.ROD_BIND_FIELD.equals(argumentDefinition.fullName) && argumentDefinition.ioType == ArgumentIOType.INPUT) {
|
||||
// TODO: Once everyone is using @Allows and @Requires correctly, we can stop blindly allowing Triplets
|
||||
return Collections.singletonList(new RodBindArgumentField(argumentDefinition));
|
||||
return Arrays.asList(new RodBindArgumentField(argumentDefinition), new InputIndexesArgumentField(argumentDefinition, Tribble.STANDARD_INDEX_EXTENSION));
|
||||
//return Collections.<ArgumentField>emptyList();
|
||||
|
||||
} else if ("input_file".equals(argumentDefinition.fullName) && argumentDefinition.ioType == ArgumentIOType.INPUT) {
|
||||
return Arrays.asList(new InputTaggedFileDefinitionField(argumentDefinition), new IndexFilesField());
|
||||
return Arrays.asList(new InputTaggedFileDefinitionField(argumentDefinition), new InputIndexesArgumentField(argumentDefinition, BAMIndex.BAMIndexSuffix, ".bam"));
|
||||
|
||||
} else if (argumentDefinition.ioType == ArgumentIOType.INPUT) {
|
||||
return Collections.singletonList(new InputArgumentField(argumentDefinition));
|
||||
|
||||
} else if (argumentDefinition.ioType == ArgumentIOType.OUTPUT) {
|
||||
return Collections.singletonList(new OutputArgumentField(argumentDefinition, gatherer));
|
||||
|
||||
List<ArgumentField> fields = new ArrayList<ArgumentField>();
|
||||
|
||||
String gatherClass;
|
||||
if (gatherer != null)
|
||||
gatherClass = gatherer.getName();
|
||||
else if (SAMFileWriter.class.isAssignableFrom(argumentDefinition.argumentType))
|
||||
gatherClass = "BamGatherFunction";
|
||||
else if (VCFWriter.class.isAssignableFrom(argumentDefinition.argumentType))
|
||||
gatherClass = "VcfGatherFunction";
|
||||
else
|
||||
gatherClass = "org.broadinstitute.sting.queue.function.scattergather.SimpleTextGatherFunction";
|
||||
|
||||
fields.add(new OutputArgumentField(argumentDefinition, gatherClass));
|
||||
|
||||
if (SAMFileWriter.class.isAssignableFrom(argumentDefinition.argumentType))
|
||||
fields.add(new SAMFileWriterIndexArgumentField(argumentDefinition));
|
||||
else if (VCFWriter.class.isAssignableFrom(argumentDefinition.argumentType))
|
||||
fields.add(new VCFWriterIndexArgumentField(argumentDefinition));
|
||||
|
||||
return fields;
|
||||
|
||||
} else if (argumentDefinition.isFlag) {
|
||||
return Collections.singletonList(new FlagArgumentField(argumentDefinition));
|
||||
|
|
@ -228,10 +251,10 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
|
|||
// if (argumentDefinition.ioType == ArgumentIOType.OUTPUT)
|
||||
// Map all outputs to files.
|
||||
private static class OutputArgumentField extends ArgumentDefinitionField {
|
||||
private final Class<?> gatherer;
|
||||
public OutputArgumentField(ArgumentDefinition argumentDefinition, Class<?> gatherer) {
|
||||
private final String gatherClass;
|
||||
public OutputArgumentField(ArgumentDefinition argumentDefinition, String gatherClass) {
|
||||
super(argumentDefinition);
|
||||
this.gatherer = gatherer;
|
||||
this.gatherClass = gatherClass;
|
||||
}
|
||||
|
||||
@Override protected Class<?> getInnerType() { return File.class; }
|
||||
|
|
@ -240,16 +263,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
|
|||
|
||||
@Override public boolean isGather() { return true; }
|
||||
@Override protected String getGatherAnnotation() {
|
||||
String gather;
|
||||
if (gatherer != null)
|
||||
gather = "@Gather(classOf[" + gatherer.getName() + "])%n";
|
||||
else if (SAMFileWriter.class.isAssignableFrom(argumentDefinition.argumentType))
|
||||
gather = "@Gather(classOf[BamGatherFunction])%n";
|
||||
else if (VCFWriter.class.isAssignableFrom(argumentDefinition.argumentType))
|
||||
gather = "@Gather(classOf[VcfGatherFunction])%n";
|
||||
else
|
||||
gather = "@Gather(classOf[org.broadinstitute.sting.queue.function.scattergather.SimpleTextGatherFunction])%n";
|
||||
return String.format(gather);
|
||||
return String.format("@Gather(classOf[%s])%n", gatherClass);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -348,19 +362,94 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
|
|||
/**
|
||||
* Adds optional inputs for the indexes of any bams or sams added to this function.
|
||||
*/
|
||||
private static class IndexFilesField extends ArgumentField {
|
||||
private static class InputIndexesArgumentField extends ArgumentField {
|
||||
private final String indexFieldName;
|
||||
private final String originalFieldName;
|
||||
private final String indexSuffix;
|
||||
private final String originalSuffix;
|
||||
public InputIndexesArgumentField(ArgumentDefinition originalArgumentDefinition, String indexSuffix) {
|
||||
this(originalArgumentDefinition, indexSuffix, null);
|
||||
}
|
||||
public InputIndexesArgumentField(ArgumentDefinition originalArgumentDefinition, String indexSuffix, String originalSuffix) {
|
||||
this.indexFieldName = originalArgumentDefinition.fullName + "Indexes";
|
||||
this.originalFieldName = originalArgumentDefinition.fullName;
|
||||
this.indexSuffix = indexSuffix;
|
||||
this.originalSuffix = originalSuffix;
|
||||
}
|
||||
@Override protected Class<? extends Annotation> getAnnotationIOClass() { return Input.class; }
|
||||
@Override public String getCommandLineAddition() { return ""; }
|
||||
@Override protected String getDoc() { return "Dependencies on any index files for any bams added to input_files"; }
|
||||
@Override protected String getFullName() { return "index_files"; }
|
||||
@Override protected String getDoc() { return "Dependencies on any indexes of " + this.originalFieldName; }
|
||||
@Override protected String getFullName() { return this.indexFieldName; }
|
||||
@Override protected boolean isRequired() { return false; }
|
||||
@Override protected String getFieldType() { return "List[File]"; }
|
||||
@Override protected String getDefaultValue() { return "Nil"; }
|
||||
@Override protected Class<?> getInnerType() { return File.class; }
|
||||
@Override protected String getRawFieldName() { return "index_files"; }
|
||||
@Override protected String getRawFieldName() { return this.indexFieldName; }
|
||||
@Override protected String getFreezeFields() {
|
||||
if (originalSuffix == null) {
|
||||
return String.format(
|
||||
("%1$s ++= %2$s" +
|
||||
".filter(orig => orig != null)" +
|
||||
".map(orig => new File(orig.getPath + \"%3$s\"))%n"),
|
||||
indexFieldName, originalFieldName, indexSuffix);
|
||||
} else {
|
||||
return String.format(
|
||||
("%1$s ++= %2$s" +
|
||||
".filter(orig => orig != null && orig.getName.endsWith(\"%4$s\"))" +
|
||||
".flatMap(orig => Array(" +
|
||||
" new File(orig.getPath + \"%3$s\")," +
|
||||
" new File(orig.getPath.stripSuffix(\"%4$s\") + \"%3$s\") ))%n"),
|
||||
indexFieldName, originalFieldName, indexSuffix, originalSuffix);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static abstract class OutputIndexArgumentField extends ArgumentField {
|
||||
protected final String indexFieldName;
|
||||
protected final String originalFieldName;
|
||||
public OutputIndexArgumentField(ArgumentDefinition originalArgumentDefinition) {
|
||||
this.indexFieldName = originalArgumentDefinition.fullName + "Index";
|
||||
this.originalFieldName = originalArgumentDefinition.fullName;
|
||||
}
|
||||
@Override protected Class<? extends Annotation> getAnnotationIOClass() { return Output.class; }
|
||||
@Override public String getCommandLineAddition() { return ""; }
|
||||
@Override protected String getDoc() { return "Automatically generated index for " + this.originalFieldName; }
|
||||
@Override protected String getFullName() { return this.indexFieldName; }
|
||||
@Override protected boolean isRequired() { return false; }
|
||||
@Override protected String getFieldType() { return "File"; }
|
||||
@Override protected String getDefaultValue() { return "_"; }
|
||||
@Override protected Class<?> getInnerType() { return File.class; }
|
||||
@Override protected String getRawFieldName() { return this.indexFieldName; }
|
||||
|
||||
@Override public boolean isGather() { return true; }
|
||||
@Override protected String getGatherAnnotation() {
|
||||
return String.format("@Gather(classOf[AutoIndexGatherFunction])%n");
|
||||
}
|
||||
}
|
||||
|
||||
private static class VCFWriterIndexArgumentField extends OutputIndexArgumentField {
|
||||
public VCFWriterIndexArgumentField(ArgumentDefinition originalArgumentDefinition) {
|
||||
super(originalArgumentDefinition);
|
||||
}
|
||||
@Override protected String getFreezeFields() {
|
||||
return String.format(
|
||||
"index_files ++= input_file.filter(bam => bam != null && bam.getName.endsWith(\".bam\")).map(bam => new File(bam.getPath + \".bai\"))%n");
|
||||
("if (%2$s != null)%n" +
|
||||
" if (!org.broadinstitute.sting.gatk.io.stubs.VCFWriterArgumentTypeDescriptor.isCompressed(%2$s.getPath))%n" +
|
||||
" %1$s = new File(%2$s.getPath + \"%3$s\")%n"),
|
||||
indexFieldName, originalFieldName, Tribble.STANDARD_INDEX_EXTENSION);
|
||||
}
|
||||
}
|
||||
|
||||
private static class SAMFileWriterIndexArgumentField extends OutputIndexArgumentField {
|
||||
public SAMFileWriterIndexArgumentField(ArgumentDefinition originalArgumentDefinition) {
|
||||
super(originalArgumentDefinition);
|
||||
}
|
||||
@Override protected String getFreezeFields() {
|
||||
return String.format(
|
||||
("if (%2$s != null)%n" +
|
||||
" if (!%3$s)%n" +
|
||||
" %1$s = new File(%2$s.getPath.stripSuffix(\".bam\") + \"%4$s\")%n"),
|
||||
indexFieldName, originalFieldName, SAMFileWriterArgumentTypeDescriptor.DISABLE_INDEXING_FULLNAME, BAMIndex.BAMIndexSuffix);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.queue.extensions.gatk
|
||||
|
||||
import org.broadinstitute.sting.queue.function.scattergather.GatherFunction
|
||||
import org.broadinstitute.sting.queue.function.InProcessFunction
|
||||
|
||||
/**
|
||||
* A no-op for index files that were automatically generated during the gather step.
|
||||
* TODO: Allow graph to know that this isn't needed, and/or that one gather job can actually gather N-outputs, and/or look more into generic source->sinks.
|
||||
*/
|
||||
class AutoIndexGatherFunction extends InProcessFunction with GatherFunction {
|
||||
def run() {}
|
||||
}
|
||||
|
|
@ -50,8 +50,8 @@ class BamGatherFunction extends GatherFunction with PicardBamFunction {
|
|||
val compression = QFunction.findField(originalFunction.getClass, SAMFileWriterArgumentTypeDescriptor.COMPRESSION_FULLNAME)
|
||||
this.compressionLevel = originalGATK.getFieldValue(compression).asInstanceOf[Option[Int]]
|
||||
|
||||
val indexBam = QFunction.findField(originalFunction.getClass, SAMFileWriterArgumentTypeDescriptor.DISABLE_INDEXING_FULLNAME)
|
||||
this.createIndex = originalGATK.getFieldValue(indexBam).asInstanceOf[Option[Boolean]]
|
||||
val disableIndex = QFunction.findField(originalFunction.getClass, SAMFileWriterArgumentTypeDescriptor.DISABLE_INDEXING_FULLNAME)
|
||||
this.createIndex = Some(!originalGATK.getFieldValue(disableIndex).asInstanceOf[Boolean])
|
||||
|
||||
super.freezeFieldValues
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue