zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

No more "Q-<pid>@<host>". Generated log file names now use the first output + ".out" (ex. my.vcf.out) or the name of the first QScript plus the order the function was added (ex. MyScript-1.out). The same function added twice with the same outputs will now have the same default logs, meaning the 2nd instance of the function won't be added to the graph twice. 

QScript accessor to QSettings to specify a default runName and other default function settings.
Because log files are no longer pseudo-random their presense can be used to tell if a job without other file outputs is "done". For now still using the log's .done file in addition to original outputs.
Gathered log files concatenate all log files together into the stdout.
InProcessFunctions now have PrintStreams for stdout and stderr.
Updated ivy to use commons-io 2.1 for copying logs to the stdout PrintStream. Removed snakeyaml.
During graph tracking of outputs the Index files, and now BAM MD5s, are tracked with the gathering of the original file.
In Queue generated wrappers for the GATK the Index and MD5s used for tracking are switched to private scope.
Added more detailed output when running with -l DEBUG.
Simplified graphviz visualization for additional debugging.
Switched usage of the scala class 'List' to the trait 'Seq' (think java.util.ArrayList vs. using the interface java.util.List)
Minor cleanup to build including sending ant gsalib to R's default libloc.
This commit is contained in:
Khalid Shakir 2012-01-08 12:11:55 -05:00
parent 90cc17ee2a
commit 5793625592
61 changed files with 1357 additions and 699 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
build.xml
View File

@ -1,5 +1,5 @@
<!--
~ Copyright (c) 2011, The Broad Institute
~ Copyright (c) 2012, The Broad Institute
~
~ Permission is hereby granted, free of charge, to any person
~ obtaining a copy of this software and associated documentation
@ -1210,7 +1210,14 @@
<!-- Build gsalib R module -->
<target name="gsalib">
<exec executable="R" failonerror="true">
<arg line="R CMD INSTALL -l ${R.library.dir} ${R.public.src.dir}/${R.package.path}/gsalib" />
<arg line="R CMD INSTALL --preclean ${R.public.src.dir}/${R.package.path}/gsalib" />
</exec>
</target>
<target name="clean.gsalib">
<!-- Currently not cleaning out the lib during 'ant clean' -->
<exec executable="R" failonerror="false">
<arg line="R CMD REMOVE gsalib" />
</exec>
</target>
</project>

26
ivy.xml
View File

@ -1,3 +1,26 @@
<!--
~ Copyright (c) 2012, The Broad Institute
~
~ Permission is hereby granted, free of charge, to any person
~ obtaining a copy of this software and associated documentation
~ files (the "Software"), to deal in the Software without
~ restriction, including without limitation the rights to use,
~ copy, modify, merge, publish, distribute, sublicense, and/or sell
~ copies of the Software, and to permit persons to whom the
~ Software is furnished to do so, subject to the following
~ conditions:
~
~ The above copyright notice and this permission notice shall be
~ included in all copies or substantial portions of the Software.
~ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
~ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
~ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
~ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
~ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
~ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
~ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
~ OTHER DEALINGS IN THE SOFTWARE.
-->
<ivy-module version="1.0">
<info organisation="org.broadinstitute" module="Sting"/>
@ -21,7 +44,6 @@
<dependency org="jboss" name="javassist" rev="3.7.ga"/>
<dependency org="org.simpleframework" name="simple-xml" rev="2.0.4"/>
<dependency org="org.apache.bcel" name="bcel" rev="5.2"/>
<dependency org="org.yaml" name="snakeyaml" rev="1.7"/>
<!-- Dependencies for reflections mvn repository -->
<dependency org="org.reflections" name="reflections" rev="0.9.5-RC2"/>
@ -40,7 +62,7 @@
<dependency org="org.apache.commons" name="commons-jexl" rev="2.0"/>
<dependency org="commons-lang" name="commons-lang" rev="2.5"/>
<dependency org="commons-logging" name="commons-logging" rev="1.1.1"/>
<dependency org="commons-io" name="commons-io" rev="2.0"/>
<dependency org="commons-io" name="commons-io" rev="2.1"/>
<dependency org="org.apache.commons" name="commons-math" rev="2.2" />
<!-- Lucene core utilities -->

5
public/java/src/org/broadinstitute/sting/commandline/Gather.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2010, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -34,5 +34,6 @@ import java.lang.annotation.*;
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.FIELD})
public @interface Gather {
Class value();
Class value() default Gather.class;
boolean enabled() default true;
}

97
public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2010, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -70,17 +70,18 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
" * Short name of %1$s%n" +
" * @return Short name of %1$s%n" +
" */%n" +
"def %3$s = this.%1$s%n" +
"%5$sdef %3$s = this.%1$s%n" +
"%n" +
"/**%n" +
" * Short name of %1$s%n" +
" * @param value Short name of %1$s%n" +
" */%n" +
"def %4$s(value: %2$s) { this.%1$s = value }%n",
"%5$sdef %4$s(value: %2$s) { this.%1$s = value }%n",
getFieldName(),
getFieldType(),
getShortFieldGetter(),
getShortFieldSetter());
getShortFieldSetter(),
getPrivacy());
}
protected static final String REQUIRED_TEMPLATE = " + required(\"%1$s\", %3$s, spaceSeparated=true, escape=true, format=%2$s)";
@ -135,11 +136,8 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
new IntervalFileArgumentField(argumentDefinition),
new IntervalStringArgumentField(argumentDefinition));
// ROD Bindings are set by the RodBindField
} else if (RodBindArgumentField.ROD_BIND_FIELD.equals(argumentDefinition.fullName) && argumentDefinition.ioType == ArgumentIOType.INPUT) {
// TODO: Once everyone is using @Allows and @Requires correctly, we can stop blindly allowing Triplets
return Arrays.asList(new RodBindArgumentField(argumentDefinition), new InputIndexesArgumentField(argumentDefinition, Tribble.STANDARD_INDEX_EXTENSION));
//return Collections.<ArgumentField>emptyList();
} else if (NumThreadsArgumentField.NUM_THREADS_FIELD.equals(argumentDefinition.fullName)) {
return Arrays.asList(new NumThreadsArgumentField(argumentDefinition));
} else if ("input_file".equals(argumentDefinition.fullName) && argumentDefinition.ioType == ArgumentIOType.INPUT) {
return Arrays.asList(new InputTaggedFileDefinitionField(argumentDefinition), new InputIndexesArgumentField(argumentDefinition, BAMIndex.BAMIndexSuffix, ".bam"));
@ -166,10 +164,13 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
fields.add(new OutputArgumentField(argumentDefinition, gatherClass));
if (SAMFileWriter.class.isAssignableFrom(argumentDefinition.argumentType))
if (SAMFileWriter.class.isAssignableFrom(argumentDefinition.argumentType)) {
fields.add(new SAMFileWriterIndexArgumentField(argumentDefinition));
else if (VCFWriter.class.isAssignableFrom(argumentDefinition.argumentType))
fields.add(new SAMFileWriterMD5ArgumentField(argumentDefinition));
}
else if (VCFWriter.class.isAssignableFrom(argumentDefinition.argumentType)) {
fields.add(new VCFWriterIndexArgumentField(argumentDefinition));
}
return fields;
@ -228,7 +229,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
@Override protected String getRawFieldName() { return super.getRawFieldName() + "String"; }
@Override protected String getFullName() { return super.getFullName() + "String"; }
@Override protected String getRawShortFieldName() { return super.getRawShortFieldName() + "String"; }
@Override protected String getFieldType() { return "List[String]"; }
@Override protected String getFieldType() { return "Seq[String]"; }
@Override protected String getDefaultValue() { return "Nil"; }
@Override public String getCommandLineTemplate() { return REPEAT_TEMPLATE; }
@ -250,7 +251,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
}
@Override protected Class<?> getInnerType() { return File.class; }
@Override protected String getFieldType() { return isMultiValued() ? "List[File]" : "File"; }
@Override protected String getFieldType() { return isMultiValued() ? "Seq[File]" : "File"; }
@Override protected String getDefaultValue() { return isMultiValued() ? "Nil" : "_"; }
}
@ -294,7 +295,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
}
@Override protected Class<?> getInnerType() { return mapType(argumentDefinition.componentType); }
@Override protected String getFieldType() { return String.format("List[%s]", getType(getInnerType())); }
@Override protected String getFieldType() { return String.format("Seq[%s]", getType(getInnerType())); }
@Override protected String getDefaultValue() { return "Nil"; }
@Override protected String getCommandLineTemplate() { return REPEAT_TEMPLATE; }
}
@ -336,17 +337,16 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
}
// Allows the user to specify the track name, track type, and the file.
public static class RodBindArgumentField extends ArgumentDefinitionField {
public static final String ROD_BIND_FIELD = "rodBind";
public static class NumThreadsArgumentField extends OptionedArgumentField {
public static final String NUM_THREADS_FIELD = "num_threads";
public RodBindArgumentField(ArgumentDefinition argumentDefinition) {
super(argumentDefinition);
public NumThreadsArgumentField(ArgumentDefinition argumentDefinition) {
super(argumentDefinition, false);
}
@Override protected Class<?> getInnerType() { return null; } // RodBind does not need to be imported.
@Override protected String getFieldType() { return "List[RodBind]"; }
@Override protected String getDefaultValue() { return "Nil"; }
@Override protected String getCommandLineTemplate() {
return " + repeat(\"%1$s\", %3$s, formatPrefix=RodBind.formatCommandLineParameter, spaceSeparated=true, escape=true, format=%2$s)";
@Override
protected String getFreezeFields() {
return String.format("if (num_threads.isDefined) nCoresRequest = num_threads%n");
}
}
@ -356,7 +356,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
super(argumentDefinition);
}
@Override protected Class<?> getInnerType() { return null; } // TaggedFile does not need to be imported.
@Override protected String getFieldType() { return argumentDefinition.isMultiValued ? "List[File]" : "File"; }
@Override protected String getFieldType() { return argumentDefinition.isMultiValued ? "Seq[File]" : "File"; }
@Override protected String getDefaultValue() { return argumentDefinition.isMultiValued ? "Nil" : "_"; }
@Override protected String getCommandLineTemplate() {
if (argumentDefinition.isMultiValued) {
@ -395,10 +395,11 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
}
@Override protected String getFullName() { return this.indexFieldName; }
@Override protected boolean isRequired() { return false; }
@Override protected String getFieldType() { return "List[File]"; }
@Override protected String getFieldType() { return "Seq[File]"; }
@Override protected String getDefaultValue() { return "Nil"; }
@Override protected Class<?> getInnerType() { return File.class; }
@Override protected String getRawFieldName() { return this.indexFieldName; }
@Override protected String getPrivacy() { return "private "; }
@Override protected String getFreezeFields() {
if (originalIsMultiValued) {
if (originalSuffix == null) {
@ -434,53 +435,69 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
}
}
// Tracks an automatically generated index
private static abstract class OutputIndexArgumentField extends ArgumentField {
protected final String indexFieldName;
// Tracks an automatically generated index, md5, etc.
private static abstract class AuxilliaryOutputArgumentField extends ArgumentField {
protected final String originalFieldName;
public OutputIndexArgumentField(ArgumentDefinition originalArgumentDefinition) {
this.indexFieldName = originalArgumentDefinition.fullName + "Index";
protected final String auxFieldName;
protected final String auxFieldLabel;
public AuxilliaryOutputArgumentField(ArgumentDefinition originalArgumentDefinition, String auxFieldLabel) {
this.originalFieldName = originalArgumentDefinition.fullName;
this.auxFieldName = originalArgumentDefinition.fullName + auxFieldLabel;
this.auxFieldLabel = auxFieldLabel;
}
@Override protected Class<? extends Annotation> getAnnotationIOClass() { return Output.class; }
@Override public String getCommandLineAddition() { return ""; }
@Override protected String getDoc() { return "Automatically generated index for " + this.originalFieldName; }
@Override protected String getFullName() { return this.indexFieldName; }
@Override protected String getDoc() { return String.format("Automatically generated %s for %s", auxFieldLabel.toLowerCase(), this.originalFieldName); }
@Override protected String getFullName() { return this.auxFieldName; }
@Override protected boolean isRequired() { return false; }
@Override protected String getFieldType() { return "File"; }
@Override protected String getDefaultValue() { return "_"; }
@Override protected Class<?> getInnerType() { return File.class; }
@Override protected String getRawFieldName() { return this.indexFieldName; }
@Override protected String getRawFieldName() { return this.auxFieldName; }
@Override protected String getPrivacy() { return "private "; }
@Override public boolean isGather() { return true; }
@Override protected String getGatherAnnotation() {
return String.format("@Gather(classOf[AutoIndexGatherFunction])%n");
return String.format("@Gather(enabled=false)%n");
}
}
private static class VCFWriterIndexArgumentField extends OutputIndexArgumentField {
private static class VCFWriterIndexArgumentField extends AuxilliaryOutputArgumentField {
public VCFWriterIndexArgumentField(ArgumentDefinition originalArgumentDefinition) {
super(originalArgumentDefinition);
super(originalArgumentDefinition, "Index");
}
@Override protected String getFreezeFields() {
return String.format(
("if (%2$s != null)%n" +
" if (!org.broadinstitute.sting.gatk.io.stubs.VCFWriterArgumentTypeDescriptor.isCompressed(%2$s.getPath))%n" +
" %1$s = new File(%2$s.getPath + \"%3$s\")%n"),
indexFieldName, originalFieldName, Tribble.STANDARD_INDEX_EXTENSION);
auxFieldName, originalFieldName, Tribble.STANDARD_INDEX_EXTENSION);
}
}
private static class SAMFileWriterIndexArgumentField extends OutputIndexArgumentField {
private static class SAMFileWriterIndexArgumentField extends AuxilliaryOutputArgumentField {
public SAMFileWriterIndexArgumentField(ArgumentDefinition originalArgumentDefinition) {
super(originalArgumentDefinition);
super(originalArgumentDefinition, "Index");
}
@Override protected String getFreezeFields() {
return String.format(
("if (%2$s != null)%n" +
" if (!%3$s)%n" +
" %1$s = new File(%2$s.getPath.stripSuffix(\".bam\") + \"%4$s\")%n"),
indexFieldName, originalFieldName, SAMFileWriterArgumentTypeDescriptor.DISABLE_INDEXING_FULLNAME, BAMIndex.BAMIndexSuffix);
auxFieldName, originalFieldName, SAMFileWriterArgumentTypeDescriptor.DISABLE_INDEXING_FULLNAME, BAMIndex.BAMIndexSuffix);
}
}
private static class SAMFileWriterMD5ArgumentField extends AuxilliaryOutputArgumentField {
public SAMFileWriterMD5ArgumentField(ArgumentDefinition originalArgumentDefinition) {
super(originalArgumentDefinition, "MD5");
}
@Override protected String getFreezeFields() {
return String.format(
("if (%2$s != null)%n" +
" if (%3$s)%n" +
" %1$s = new File(%2$s.getPath + \"%4$s\")%n"),
auxFieldName, originalFieldName, SAMFileWriterArgumentTypeDescriptor.ENABLE_MD5_FULLNAME, ".md5");
}
}

9
public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2010, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -56,7 +56,7 @@ public abstract class ArgumentField {
return String.format("%n" +
"/** %s */%n" +
"@%s(fullName=\"%s\", shortName=\"%s\", doc=\"%s\", required=%s, exclusiveOf=\"%s\", validation=\"%s\")%n" +
"%svar %s: %s = %s%n" +
"%s%svar %s: %s = %s%n" +
"%s",
getDoc(),
getAnnotationIOClass().getSimpleName(),
@ -66,7 +66,7 @@ public abstract class ArgumentField {
isRequired(),
getExclusiveOf(),
getValidation(),
getGatherAnnotation(), getFieldName(), getFieldType(), getDefaultValue(),
getGatherAnnotation(), getPrivacy(), getFieldName(), getFieldType(), getDefaultValue(),
getDefineAddition());
}
@ -143,6 +143,9 @@ public abstract class ArgumentField {
/** @return True if this field uses @Gather. */
public boolean isGather() { return false; }
/** @return Privacy for the field. */
protected String getPrivacy() { return ""; }
/** @return The raw field name, which will be checked against scala build in types. */
protected abstract String getRawFieldName();
/** @return The field name checked against reserved words. */

32
public/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2010, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -34,13 +34,11 @@ import org.broadinstitute.sting.commandline.ParsingEngine;
import org.broadinstitute.sting.gatk.CommandLineGATK;
import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
import org.broadinstitute.sting.gatk.WalkerManager;
import org.broadinstitute.sting.gatk.arguments.ValidationExclusion;
import org.broadinstitute.sting.gatk.filters.FilterManager;
import org.broadinstitute.sting.gatk.filters.ReadFilter;
import org.broadinstitute.sting.gatk.io.stubs.OutputStreamArgumentTypeDescriptor;
import org.broadinstitute.sting.gatk.io.stubs.SAMFileWriterArgumentTypeDescriptor;
import org.broadinstitute.sting.gatk.io.stubs.VCFWriterArgumentTypeDescriptor;
import org.broadinstitute.sting.gatk.refdata.tracks.RMDTrackBuilder;
import org.broadinstitute.sting.gatk.walkers.PartitionBy;
import org.broadinstitute.sting.gatk.walkers.PartitionType;
import org.broadinstitute.sting.gatk.walkers.Walker;
@ -85,7 +83,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
"%n" +
"/** A dynamicly generated list of classes that the GATK Extensions depend on, but are not be detected by default by BCEL. */%n" +
"class %s {%n" +
"val types = List(%n%s)%n" +
"val types = Seq(%n%s)%n" +
"}%n";
@Output(fullName="output_directory", shortName="outDir", doc="Directory to output the generated scala", required=true)
@ -95,10 +93,6 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
GenomeAnalysisEngine GATKEngine = new GenomeAnalysisEngine();
WalkerManager walkerManager = new WalkerManager();
FilterManager filterManager = new FilterManager();
// HACK: We're currently relying on the fact that RMDTrackBuilder is used only from RMD type lookups, not
// RMD track location. Therefore, no sequence dictionary is required. In the future, we should separate
// RMD track lookups from track creation.
RMDTrackBuilder trackBuilder = new RMDTrackBuilder(null,null,ValidationExclusion.TYPE.ALL);
/**
* Required main method implementation.
@ -147,7 +141,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
String clpConstructor = String.format("analysisName = \"%s\"%njavaMainClass = \"%s\"%n", clpClassName, clp.getName());
writeClass("org.broadinstitute.sting.queue.function.JavaCommandLineFunction", clpClassName,
false, clpConstructor, ArgumentDefinitionField.getArgumentFields(parser,clp), dependents, false);
false, clpConstructor, ArgumentDefinitionField.getArgumentFields(parser,clp), dependents);
if (clp == CommandLineGATK.class) {
for (Entry<String, Collection<Class<? extends Walker>>> walkersByPackage: walkerManager.getWalkerNamesByPackage(false).entrySet()) {
@ -169,7 +163,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
}
writeClass(GATK_EXTENSIONS_PACKAGE_NAME + "." + clpClassName, walkerName,
isScatter, constructor, argumentFields, dependents, true);
isScatter, constructor, argumentFields, dependents);
} catch (Exception e) {
throw new ReviewedStingException("Error generating wrappers for walker " + walkerType, e);
}
@ -242,8 +236,8 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
*/
private void writeClass(String baseClass, String className, boolean isScatter,
String constructor, List<? extends ArgumentField> argumentFields,
Set<Class<?>> dependents, boolean isGATKWalker) throws IOException {
String content = getContent(CLASS_TEMPLATE, baseClass, className, constructor, isScatter, "", argumentFields, dependents, isGATKWalker);
Set<Class<?>> dependents) throws IOException {
String content = getContent(CLASS_TEMPLATE, baseClass, className, constructor, isScatter, "", argumentFields, dependents);
writeFile(GATK_EXTENSIONS_PACKAGE_NAME + "." + className, content);
}
@ -257,7 +251,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
*/
private void writeFilter(String className, List<? extends ArgumentField> argumentFields, Set<Class<?>> dependents) throws IOException {
String content = getContent(TRAIT_TEMPLATE, "org.broadinstitute.sting.queue.function.CommandLineFunction",
className, "", false, String.format(" + \" -read_filter %s\"", className), argumentFields, dependents, false);
className, "", false, String.format(" + \" -read_filter %s\"", className), argumentFields, dependents);
writeFile(GATK_EXTENSIONS_PACKAGE_NAME + "." + className, content);
}
@ -351,8 +345,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
*/
private static String getContent(String scalaTemplate, String baseClass, String className,
String constructor, boolean isScatter, String commandLinePrefix,
List<? extends ArgumentField> argumentFields, Set<Class<?>> dependents,
boolean isGATKWalker) {
List<? extends ArgumentField> argumentFields, Set<Class<?>> dependents) {
StringBuilder arguments = new StringBuilder();
StringBuilder commandLine = new StringBuilder(commandLinePrefix);
@ -376,9 +369,6 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
if (isGather)
importSet.add("import org.broadinstitute.sting.commandline.Gather");
// Needed for ShellUtils.escapeShellArgument()
importSet.add("import org.broadinstitute.sting.queue.util.ShellUtils");
// Sort the imports so that the are always in the same order.
List<String> sortedImports = new ArrayList<String>(importSet);
Collections.sort(sortedImports);
@ -386,10 +376,8 @@ public class GATKExtensionsGenerator extends CommandLineProgram {
StringBuffer freezeFieldOverride = new StringBuffer();
for (String freezeField: freezeFields)
freezeFieldOverride.append(freezeField);
if (freezeFieldOverride.length() > 0 || isGATKWalker) {
freezeFieldOverride.insert(0, String.format("override def freezeFieldValues = {%nsuper.freezeFieldValues%n"));
if ( isGATKWalker )
freezeFieldOverride.append(String.format("if ( num_threads.isDefined ) nCoresRequest = num_threads%n"));
if (freezeFieldOverride.length() > 0) {
freezeFieldOverride.insert(0, String.format("override def freezeFieldValues() {%nsuper.freezeFieldValues()%n"));
freezeFieldOverride.append(String.format("}%n%n"));
}

54
public/scala/qscript/org/broadinstitute/sting/queue/qscripts/DataProcessingPipeline.scala
View File

@ -29,14 +29,14 @@ class DataProcessingPipeline extends QScript {
var reference: File = _
@Input(doc="dbsnp ROD to use (must be in VCF format)", fullName="dbsnp", shortName="D", required=true)
var dbSNP: List[File] = List()
var dbSNP: Seq[File] = Seq()
/****************************************************************************
* Optional Parameters
****************************************************************************/
@Input(doc="extra VCF files to use as reference indels for Indel Realignment", fullName="extra_indels", shortName="indels", required=false)
var indels: List[File] = List()
var indels: Seq[File] = Seq()
@Input(doc="The path to the binary of bwa (usually BAM files have already been mapped - but if you want to remap this is the option)", fullName="path_to_bwa", shortName="bwa", required=false)
var bwaPath: File = _
@ -118,13 +118,13 @@ class DataProcessingPipeline extends QScript {
// Because the realignment only happens after these scripts are executed, in case you are using
// bwa realignment, this function will operate over the original bam files and output over the
// (to be realigned) bam files.
def createSampleFiles(bamFiles: List[File], realignedBamFiles: List[File]): Map[String, List[File]] = {
def createSampleFiles(bamFiles: Seq[File], realignedBamFiles: Seq[File]): Map[String, Seq[File]] = {
// Creating a table with SAMPLE information from each input BAM file
val sampleTable = scala.collection.mutable.Map.empty[String, List[File]]
val sampleTable = scala.collection.mutable.Map.empty[String, Seq[File]]
val realignedIterator = realignedBamFiles.iterator
for (bam <- bamFiles) {
val rBam = realignedIterator.next // advance to next element in the realignedBam list so they're in sync.
val rBam = realignedIterator.next() // advance to next element in the realignedBam list so they're in sync.
val samReader = new SAMFileReader(bam)
val header = samReader.getFileHeader
@ -138,12 +138,12 @@ class DataProcessingPipeline extends QScript {
for (rg <- readGroups) {
val sample = rg.getSample
if (!sampleTable.contains(sample))
sampleTable(sample) = List(rBam)
sampleTable(sample) = Seq(rBam)
else if ( !sampleTable(sample).contains(rBam))
sampleTable(sample) :+= rBam
}
}
return sampleTable.toMap
sampleTable.toMap
}
// Rebuilds the Read Group string to give BWA
@ -161,8 +161,8 @@ class DataProcessingPipeline extends QScript {
// Takes a list of processed BAM files and realign them using the BWA option requested (bwase or bwape).
// Returns a list of realigned BAM files.
def performAlignment(bams: List[File]): List[File] = {
var realignedBams: List[File] = List()
def performAlignment(bams: Seq[File]): Seq[File] = {
var realignedBams: Seq[File] = Seq()
var index = 1
for (bam <- bams) {
// first revert the BAM file to the original qualities
@ -194,10 +194,10 @@ class DataProcessingPipeline extends QScript {
realignedBams :+= rgRealignedBamFile
index = index + 1
}
return realignedBams
realignedBams
}
def getIndelCleaningModel(): ConsensusDeterminationModel = {
def getIndelCleaningModel: ConsensusDeterminationModel = {
if (cleaningModel == "KNOWNS_ONLY")
ConsensusDeterminationModel.KNOWNS_ONLY
else if (cleaningModel == "USE_SW")
@ -206,17 +206,17 @@ class DataProcessingPipeline extends QScript {
ConsensusDeterminationModel.USE_READS
}
def revertBams(bams: List[File], removeAlignmentInformation: Boolean): List[File] = {
var revertedBAMList: List[File] = List()
def revertBams(bams: Seq[File], removeAlignmentInformation: Boolean): Seq[File] = {
var revertedBAMList: Seq[File] = Seq()
for (bam <- bams)
revertedBAMList :+= revertBAM(bam, removeAlignmentInformation)
return revertedBAMList
revertedBAMList
}
def revertBAM(bam: File, removeAlignmentInformation: Boolean): File = {
val revertedBAM = swapExt(bam, ".bam", ".reverted.bam")
add(revert(bam, revertedBAM, removeAlignmentInformation))
return revertedBAM
revertedBAM
}
/****************************************************************************
@ -224,22 +224,22 @@ class DataProcessingPipeline extends QScript {
****************************************************************************/
def script = {
def script() {
// final output list of processed bam files
var cohortList: List[File] = List()
var cohortList: Seq[File] = Seq()
// sets the model for the Indel Realigner
cleanModelEnum = getIndelCleaningModel()
cleanModelEnum = getIndelCleaningModel
// keep a record of the number of contigs in the first bam file in the list
val bams = QScriptUtils.createListFromFile(input)
val bams = QScriptUtils.createSeqFromFile(input)
if (nContigs < 0)
nContigs = QScriptUtils.getNumberOfContigs(bams(0))
val realignedBAMs = if (useBWApe || useBWAse || useBWAsw) {performAlignment(bams)} else {revertBams(bams, false)}
// generate a BAM file per sample joining all per lane files if necessary
val sampleBAMFiles: Map[String, List[File]] = createSampleFiles(bams, realignedBAMs)
val sampleBAMFiles: Map[String, Seq[File]] = createSampleFiles(bams, realignedBAMs)
// if this is a 'knowns only' indel realignment run, do it only once for all samples.
val globalIntervals = new File(outputDir + projectName + ".intervals")
@ -317,7 +317,7 @@ class DataProcessingPipeline extends QScript {
this.maxRecordsInRam = 100000
}
case class target (inBams: List[File], outIntervals: File) extends RealignerTargetCreator with CommandLineGATKArgs {
case class target (inBams: Seq[File], outIntervals: File) extends RealignerTargetCreator with CommandLineGATKArgs {
if (cleanModelEnum != ConsensusDeterminationModel.KNOWNS_ONLY)
this.input_file = inBams
this.out = outIntervals
@ -330,7 +330,7 @@ class DataProcessingPipeline extends QScript {
this.jobName = queueLogDir + outIntervals + ".target"
}
case class clean (inBams: List[File], tIntervals: File, outBam: File) extends IndelRealigner with CommandLineGATKArgs {
case class clean (inBams: Seq[File], tIntervals: File, outBam: File) extends IndelRealigner with CommandLineGATKArgs {
this.input_file = inBams
this.targetIntervals = tIntervals
this.out = outBam
@ -347,11 +347,11 @@ class DataProcessingPipeline extends QScript {
case class cov (inBam: File, outRecalFile: File) extends CountCovariates with CommandLineGATKArgs {
this.knownSites ++= qscript.dbSNP
this.covariate ++= List("ReadGroupCovariate", "QualityScoreCovariate", "CycleCovariate", "DinucCovariate")
this.covariate ++= Seq("ReadGroupCovariate", "QualityScoreCovariate", "CycleCovariate", "DinucCovariate")
this.input_file :+= inBam
this.recal_file = outRecalFile
if (!defaultPlatform.isEmpty) this.default_platform = defaultPlatform
if (!qscript.intervalString.isEmpty()) this.intervalsString ++= List(qscript.intervalString)
if (!qscript.intervalString.isEmpty) this.intervalsString ++= Seq(qscript.intervalString)
else if (qscript.intervals != null) this.intervals :+= qscript.intervals
this.scatterCount = nContigs
this.analysisName = queueLogDir + outRecalFile + ".covariates"
@ -363,7 +363,7 @@ class DataProcessingPipeline extends QScript {
this.recal_file = inRecalFile
this.baq = CalculationMode.CALCULATE_AS_NECESSARY
this.out = outBam
if (!qscript.intervalString.isEmpty()) this.intervalsString ++= List(qscript.intervalString)
if (!qscript.intervalString.isEmpty) this.intervalsString ++= Seq(qscript.intervalString)
else if (qscript.intervals != null) this.intervals :+= qscript.intervals
this.no_pg_tag = qscript.testMode
this.scatterCount = nContigs
@ -395,7 +395,7 @@ class DataProcessingPipeline extends QScript {
this.jobName = queueLogDir + outBam + ".dedup"
}
case class joinBams (inBams: List[File], outBam: File) extends MergeSamFiles with ExternalCommonArgs {
case class joinBams (inBams: Seq[File], outBam: File) extends MergeSamFiles with ExternalCommonArgs {
this.input = inBams
this.output = outBam
this.analysisName = queueLogDir + outBam + ".joinBams"
@ -495,7 +495,7 @@ class DataProcessingPipeline extends QScript {
this.jobName = queueLogDir + outBam + ".bwasw"
}
case class writeList(inBams: List[File], outBamList: File) extends ListWriterFunction {
case class writeList(inBams: Seq[File], outBamList: File) extends ListWriterFunction {
this.inputFiles = inBams
this.listFile = outBamList
this.analysisName = queueLogDir + outBamList + ".bamList"

4
public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala
View File

@ -53,9 +53,9 @@ class PacbioProcessingPipeline extends QScript {
val queueLogDir: String = ".qlog/"
def script = {
def script() {
val fileList: List[File] = QScriptUtils.createListFromFile(input)
val fileList: Seq[File] = QScriptUtils.createSeqFromFile(input)
for (file: File <- fileList) {

62
public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -36,6 +36,7 @@ import org.broadinstitute.sting.utils.io.IOUtils
import org.broadinstitute.sting.utils.help.ApplicationDetails
import java.util.{ResourceBundle, Arrays}
import org.broadinstitute.sting.utils.text.TextFormattingUtils
import org.apache.commons.io.FilenameUtils
/**
* Entry point of Queue. Compiles and runs QScripts passed in to the command line.
@ -61,6 +62,7 @@ object QCommandLine extends Logging {
CommandLineProgram.start(qCommandLine, argv)
try {
Runtime.getRuntime.removeShutdownHook(shutdownHook)
qCommandLine.shutdown()
} catch {
case _ => /* ignore, example 'java.lang.IllegalStateException: Shutdown in progress' */
}
@ -78,10 +80,10 @@ object QCommandLine extends Logging {
class QCommandLine extends CommandLineProgram with Logging {
@Input(fullName="script", shortName="S", doc="QScript scala file", required=true)
@ClassType(classOf[File])
private var scripts = List.empty[File]
var scripts = Seq.empty[File]
@ArgumentCollection
private val settings = new QGraphSettings
val settings = new QGraphSettings
private val qScriptManager = new QScriptManager
private val qGraph = new QGraph
@ -91,7 +93,7 @@ class QCommandLine extends CommandLineProgram with Logging {
private lazy val pluginManager = {
qScriptClasses = IOUtils.tempDir("Q-Classes-", "", settings.qSettings.tempDirectory)
qScriptManager.loadScripts(scripts, qScriptClasses)
new PluginManager[QScript](classOf[QScript], List(qScriptClasses.toURI.toURL))
new PluginManager[QScript](classOf[QScript], Seq(qScriptClasses.toURI.toURL))
}
QFunction.parsingEngine = new ParsingEngine(this)
@ -101,12 +103,16 @@ class QCommandLine extends CommandLineProgram with Logging {
* functions, and then builds and runs a QGraph based on the dependencies.
*/
def execute = {
if (settings.qSettings.runName == null)
settings.qSettings.runName = FilenameUtils.removeExtension(scripts.head.getName)
qGraph.settings = settings
val allQScripts = pluginManager.createAllTypes();
for (script <- allQScripts) {
logger.info("Scripting " + pluginManager.getName(script.getClass.asSubclass(classOf[QScript])))
loadArgumentsIntoObject(script)
script.qSettings = settings.qSettings
try {
script.script()
} catch {
@ -120,22 +126,34 @@ class QCommandLine extends CommandLineProgram with Logging {
// Execute the job graph
qGraph.run()
val functionsAndStatus = qGraph.getFunctionsAndStatus
val success = qGraph.success
// walk over each script, calling onExecutionDone
for (script <- allQScripts) {
script.onExecutionDone(qGraph.getFunctionsAndStatus(script.functions), qGraph.success)
if ( ! settings.disableJobReport ) {
val jobStringName = (QScriptUtils.?(settings.jobReportFile)).getOrElse(settings.qSettings.jobNamePrefix + ".jobreport.txt")
val scriptFunctions = functionsAndStatus.filterKeys(f => script.functions.contains(f))
script.onExecutionDone(scriptFunctions, success)
}
if (!shuttingDown) {
val reportFile = new File(jobStringName)
logger.info("Writing JobLogging GATKReport to file " + reportFile)
QJobReport.printReport(qGraph.getFunctionsAndStatus(script.functions), reportFile)
logger.info("Script %s with %d total jobs".format(if (success) "completed successfully" else "failed", functionsAndStatus.size))
if ( settings.run ) {
val pdfFile = new File(jobStringName + ".pdf")
logger.info("Plotting JobLogging GATKReport to file " + pdfFile)
QJobReport.plotReport(reportFile, pdfFile)
}
if (!settings.disableJobReport) {
val jobStringName = {
if (settings.jobReportFile != null)
settings.jobReportFile
else
settings.qSettings.runName + ".jobreport.txt"
}
if (!shuttingDown) {
val reportFile = IOUtils.absolute(settings.qSettings.runDirectory, jobStringName)
logger.info("Writing JobLogging GATKReport to file " + reportFile)
QJobReport.printReport(functionsAndStatus, reportFile)
if (settings.run) {
val pdfFile = IOUtils.absolute(settings.qSettings.runDirectory, FilenameUtils.removeExtension(jobStringName) + ".pdf")
logger.info("Plotting JobLogging GATKReport to file " + pdfFile)
QJobReport.plotReport(reportFile, pdfFile)
}
}
}
@ -179,20 +197,20 @@ class QCommandLine extends CommandLineProgram with Logging {
override def getApplicationDetails : ApplicationDetails = {
new ApplicationDetails(createQueueHeader(),
List.empty[String],
Seq.empty[String],
ApplicationDetails.createDefaultRunningInstructions(getClass.asInstanceOf[Class[CommandLineProgram]]),
"")
}
private def createQueueHeader() : List[String] = {
List(String.format("Queue v%s, Compiled %s", getQueueVersion, getBuildTimestamp),
"Copyright (c) 2011 The Broad Institute",
private def createQueueHeader() : Seq[String] = {
Seq(String.format("Queue v%s, Compiled %s", getQueueVersion, getBuildTimestamp),
"Copyright (c) 2012 The Broad Institute",
"Please view our documentation at http://www.broadinstitute.org/gsa/wiki",
"For support, please view our support site at http://getsatisfaction.com/gsa")
}
private def getQueueVersion : String = {
var stingResources : ResourceBundle = TextFormattingUtils.loadResourceBundle("StingText")
val stingResources : ResourceBundle = TextFormattingUtils.loadResourceBundle("StingText")
if ( stingResources.containsKey("org.broadinstitute.sting.queue.QueueVersion.version") ) {
stingResources.getString("org.broadinstitute.sting.queue.QueueVersion.version")
@ -203,7 +221,7 @@ class QCommandLine extends CommandLineProgram with Logging {
}
private def getBuildTimestamp : String = {
var stingResources : ResourceBundle = TextFormattingUtils.loadResourceBundle("StingText")
val stingResources : ResourceBundle = TextFormattingUtils.loadResourceBundle("StingText")
if ( stingResources.containsKey("build.timestamp") ) {
stingResources.getString("build.timestamp")

20
public/scala/src/org/broadinstitute/sting/queue/QScript.scala
View File

@ -27,7 +27,6 @@ package org.broadinstitute.sting.queue
import engine.JobRunInfo
import org.broadinstitute.sting.queue.function.QFunction
import annotation.target.field
import io.Source
import util.{StringFileConversions, PrimitiveOptionConversions, Logging}
/**
@ -53,6 +52,11 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon
type ArgumentCollection = org.broadinstitute.sting.commandline.ArgumentCollection @field
type Gather = org.broadinstitute.sting.commandline.Gather @field
/**
* Default settings for QFunctions
*/
var qSettings: QSettings = _
/**
* Builds the CommandLineFunctions that will be used to run this script and adds them to this.functions directly or using the add() utility method.
*/
@ -60,18 +64,14 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon
/**
* A default handler for the onExecutionDone() function. By default this doesn't do anything
* except print out a fine status message.
*/
def onExecutionDone(jobs: Map[QFunction, JobRunInfo], success: Boolean) {
logger.info("Script %s with %d total jobs".format(if (success) "completed successfully" else "failed", jobs.size))
// this is too much output
// for ( (f, info) <- jobs ) logger.info(" %s %s".format(f.jobName, info))
}
/**
* The command line functions that will be executed for this QScript.
*/
var functions = List.empty[QFunction]
var functions = Seq.empty[QFunction]
/**
* Exchanges the extension on a file.
@ -98,22 +98,20 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon
* Adds one or more command line functions to be run.
* @param functions Functions to add.
*/
def add(functions: QFunction*) = {
def add(functions: QFunction*) {
functions.foreach(function => function.addOrder = QScript.nextAddOrder)
this.functions ++= functions
}
def addAll(functions: List[QFunction]) {
def addAll(functions: Seq[QFunction]) {
functions.foreach( f => add(f) )
}
def extractFileEntries(in: File): List[File] = Source.fromFile(in).getLines().toList
}
object QScript {
private var addOrder = 0
private def nextAddOrder = {
addOrder += 1
List(addOrder)
Seq(addOrder)
}
}

4
public/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala
View File

@ -20,7 +20,7 @@ class QScriptManager() extends Logging {
* Compiles and loads the scripts in the files into the current classloader.
* Heavily based on scala/src/compiler/scala/tools/ant/Scalac.scala
*/
def loadScripts(scripts: List[File], tempDir: File) {
def loadScripts(scripts: Seq[File], tempDir: File) {
if (scripts.size > 0) {
val settings = new Settings((error: String) => logger.error(error))
settings.deprecation.value = true
@ -36,7 +36,7 @@ class QScriptManager() extends Logging {
logger.info("Compiling %s QScript%s".format(scripts.size, plural(scripts.size)))
logger.debug("Compilation directory: " + settings.outdir.value)
run.compileFiles(scripts.map(new PlainFile(_)))
run.compileFiles(scripts.toList.map(new PlainFile(_)))
reporter.printSummary()
if (reporter.hasErrors) {

26
public/scala/src/org/broadinstitute/sting/queue/QSettings.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -25,15 +25,14 @@
package org.broadinstitute.sting.queue
import java.io.File
import org.broadinstitute.sting.commandline.{ArgumentCollection, Argument}
import org.broadinstitute.sting.queue.util.{SystemUtils, EmailSettings}
import org.broadinstitute.sting.commandline.Argument
/**
* Default settings settable on the command line and passed to CommandLineFunctions.
*/
class QSettings {
@Argument(fullName="job_name_prefix", shortName="jobPrefix", doc="Default name prefix for compute farm jobs.", required=false)
var jobNamePrefix: String = QSettings.processNamePrefix
@Argument(fullName="run_name", shortName="runName", doc="A name for this run used for various status messages.", required=false)
var runName: String = _
@Argument(fullName="job_project", shortName="jobProject", doc="Default project for compute farm jobs.", required=false)
var jobProject: String = _
@ -45,13 +44,13 @@ class QSettings {
var jobPriority: Option[Int] = None
@Argument(fullName="job_native_arg", shortName="jobNative", doc="Native arguments to pass to the job runner.", required=false)
var jobNativeArgs: List[String] = Nil
var jobNativeArgs: Seq[String] = Nil
@Argument(fullName="job_resource_request", shortName="jobResReq", doc="Resource requests to pass to the job runner.", required=false)
var jobResourceRequests: List[String] = Nil
var jobResourceRequests: Seq[String] = Nil
@Argument(fullName="job_environment_name", shortName="jobEnv", doc="Environment names for the job runner.", required=false)
var jobEnvironmentNames: List[String] = Nil
var jobEnvironmentNames: Seq[String] = Nil
@Argument(fullName="memory_limit", shortName="memLimit", doc="Default memory limit for jobs, in gigabytes.", required=false)
var memoryLimit: Option[Double] = None
@ -77,15 +76,4 @@ class QSettings {
@Argument(fullName="job_scatter_gather_directory", shortName="jobSGDir", doc="Default directory to place scatter gather output for compute farm jobs.", required=false)
var jobScatterGatherDirectory: File = _
@ArgumentCollection
val emailSettings = new EmailSettings
}
/**
* Default settings settable on the command line and passed to CommandLineFunctions.
*/
object QSettings {
/** A semi-unique job prefix using the host name and the process id. */
private val processNamePrefix = "Q-" + SystemUtils.pidAtHost
}

39
public/scala/src/org/broadinstitute/sting/queue/engine/FunctionEdge.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.engine
import org.broadinstitute.sting.queue.function.QFunction
@ -28,15 +52,18 @@ class FunctionEdge(val function: QFunction, val inputs: QNode, val outputs: QNod
val myRunInfo: JobRunInfo = JobRunInfo.default // purely for dryRun testing
/**
* When using reset status this variable tracks the old status
*/
var resetFromStatus: RunnerStatus.Value = null
/**
* Initializes with the current status of the function.
*/
private var currentStatus = {
val isDone = function.isDone
val isFail = function.isFail
if (isFail.isDefined && isFail.get)
if (function.isFail)
RunnerStatus.FAILED
else if (isDone.isDefined && isDone.get)
else if (function.isDone)
RunnerStatus.DONE
else
RunnerStatus.PENDING
@ -136,13 +163,15 @@ class FunctionEdge(val function: QFunction, val inputs: QNode, val outputs: QNod
* Resets the edge to pending status.
*/
def resetToPending(cleanOutputs: Boolean) {
if (resetFromStatus == null)
resetFromStatus = currentStatus
currentStatus = RunnerStatus.PENDING
if (cleanOutputs)
function.deleteOutputs()
runner = null
}
override def dotString = function.dotString
override def shortDescription = function.shortDescription
/**
* Returns the path to the file to use for logging errors.

23
public/scala/src/org/broadinstitute/sting/queue/engine/InProcessRunner.scala
View File

@ -3,7 +3,8 @@ package org.broadinstitute.sting.queue.engine
import org.broadinstitute.sting.queue.function.InProcessFunction
import java.util.Date
import org.broadinstitute.sting.utils.Utils
import org.apache.commons.io.FileUtils
import org.apache.commons.io.{IOUtils, FileUtils}
import java.io.PrintStream
/**
* Runs a function that executes in process and does not fork out an external process.
@ -16,12 +17,24 @@ class InProcessRunner(val function: InProcessFunction) extends JobRunner[InProce
getRunInfo.exechosts = Utils.resolveHostname()
runStatus = RunnerStatus.RUNNING
function.run()
function.jobOutputStream = new PrintStream(FileUtils.openOutputStream(function.jobOutputFile))
function.jobErrorStream = {
if (function.jobErrorFile != null)
new PrintStream(FileUtils.openOutputStream(function.jobErrorFile))
else
function.jobOutputStream
}
try {
function.run()
function.jobOutputStream.println("%s%nDone.".format(function.description))
} finally {
IOUtils.closeQuietly(function.jobOutputStream)
if (function.jobErrorFile != null)
IOUtils.closeQuietly(function.jobErrorStream)
}
getRunInfo.doneTime = new Date()
val content = "%s%nDone.".format(function.description)
FileUtils.writeStringToFile(function.jobOutputFile, content)
runStatus = RunnerStatus.DONE
getRunInfo.doneTime = new Date()
}
def status = runStatus

26
public/scala/src/org/broadinstitute/sting/queue/engine/MappingEdge.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.engine
/**
@ -10,5 +34,5 @@ class MappingEdge(val inputs: QNode, val outputs: QNode) extends QEdge {
* @return <map>
*/
override def toString = "<map>"
override def dotString = "<map>"
override def shortDescription = ""
}

28
public/scala/src/org/broadinstitute/sting/queue/engine/QEdge.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.engine
/**
@ -15,9 +39,9 @@ trait QEdge {
def outputs: QNode
/**
* The function description in .dot files
* The short description
*/
def dotString = ""
def shortDescription = ""
override def hashCode = inputs.hashCode + outputs.hashCode

214
public/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -30,7 +30,6 @@ import scala.collection.JavaConversions._
import org.jgrapht.alg.CycleDetector
import org.jgrapht.EdgeFactory
import org.jgrapht.ext.DOTExporter
import java.io.File
import org.jgrapht.event.{TraversalListenerAdapter, EdgeTraversalEvent}
import org.broadinstitute.sting.queue.QException
import org.broadinstitute.sting.queue.function.{InProcessFunction, CommandLineFunction, QFunction}
@ -40,7 +39,8 @@ import collection.immutable.{TreeSet, TreeMap}
import org.broadinstitute.sting.queue.function.scattergather.{ScatterFunction, CloneFunction, GatherFunction, ScatterGatherableFunction}
import java.util.Date
import org.broadinstitute.sting.utils.Utils
import org.broadinstitute.sting.utils.io.IOUtils
import org.apache.commons.io.{FileUtils, IOUtils}
import java.io.{OutputStreamWriter, File}
/**
* The internal dependency tracker between sets of function input and output files.
@ -69,7 +69,7 @@ class QGraph extends Logging {
private val commandLinePluginManager = new CommandLinePluginManager
private var commandLineManager: CommandLineJobManager[CommandLineJobRunner] = _
private val inProcessManager = new InProcessJobManager
private def managers = List[Any](inProcessManager, commandLineManager)
private def managers = Seq[Any](inProcessManager, commandLineManager)
private class StatusCounts {
var pending = 0
@ -88,9 +88,14 @@ class QGraph extends Logging {
runningLock.synchronized {
if (running) {
command.qSettings = settings.qSettings
command.freeze
val inputs = getQNode(command.inputs.toList.sorted(fileOrdering))
val outputs = getQNode(command.outputs.toList.sorted(fileOrdering))
command.freeze()
val inputFiles = command.inputs
var outputFiles = command.outputs
outputFiles :+= command.jobOutputFile
if (command.jobErrorFile != null)
outputFiles :+= command.jobErrorFile
val inputs = getQNode(inputFiles.sorted(fileOrdering))
val outputs = getQNode(outputFiles.sorted(fileOrdering))
addEdge(new FunctionEdge(command, inputs, outputs))
}
}
@ -106,8 +111,8 @@ class QGraph extends Logging {
def run() {
runningLock.synchronized {
if (running) {
IOUtils.checkTempDir(settings.qSettings.tempDirectory)
fillGraph
org.broadinstitute.sting.utils.io.IOUtils.checkTempDir(settings.qSettings.tempDirectory)
fillGraph()
val isReady = numMissingValues == 0
if (this.jobGraph.edgeSet.isEmpty) {
@ -133,11 +138,11 @@ class QGraph extends Logging {
}
}
private def fillGraph {
private def fillGraph() {
logger.info("Generating graph.")
fill
if (settings.dotFile != null)
renderToDot(settings.dotFile)
fill()
if (settings.graphvizFile != null)
renderGraph(settings.graphvizFile)
validate()
if (running && numMissingValues == 0) {
@ -145,7 +150,7 @@ class QGraph extends Logging {
if (!scatterGathers.isEmpty) {
logger.info("Generating scatter gather jobs.")
var addedFunctions = List.empty[QFunction]
var addedFunctions = Seq.empty[QFunction]
for (scatterGather <- scatterGathers) {
val functions = scatterGather.asInstanceOf[FunctionEdge]
.function.asInstanceOf[ScatterGatherableFunction]
@ -161,10 +166,10 @@ class QGraph extends Logging {
addedFunctions.foreach(function => if (running) this.add(function))
logger.info("Regenerating graph.")
fill
val scatterGatherDotFile = if (settings.expandedDotFile != null) settings.expandedDotFile else settings.dotFile
fill()
val scatterGatherDotFile = if (settings.graphvizScatterGatherFile != null) settings.graphvizScatterGatherFile else settings.graphvizFile
if (scatterGatherDotFile != null)
renderToDot(scatterGatherDotFile)
renderGraph(scatterGatherDotFile)
validate()
}
}
@ -187,8 +192,8 @@ class QGraph extends Logging {
* @param edge Graph edge to examine for the previous functions.
* @return A list of prior function edges.
*/
private def previousFunctions(edge: QEdge): List[FunctionEdge] = {
var previous = List.empty[FunctionEdge]
private def previousFunctions(edge: QEdge): Seq[FunctionEdge] = {
var previous = Seq.empty[FunctionEdge]
val source = this.jobGraph.getEdgeSource(edge)
for (incomingEdge <- this.jobGraph.incomingEdgesOf(source)) {
incomingEdge match {
@ -208,8 +213,8 @@ class QGraph extends Logging {
* @param edge Graph edge to examine for the next functions.
* @return A list of prior function edges.
*/
private def nextFunctions(edge: QEdge): List[FunctionEdge] = {
var next = List.empty[FunctionEdge]
private def nextFunctions(edge: QEdge): Seq[FunctionEdge] = {
var next = Seq.empty[FunctionEdge]
val target = this.jobGraph.getEdgeTarget(edge)
for (outgoingEdge <- this.jobGraph.outgoingEdgesOf(target)) {
outgoingEdge match {
@ -238,7 +243,7 @@ class QGraph extends Logging {
*/
private def fillIn() {
// clone since edgeSet is backed by the graph
asScalaSet(jobGraph.edgeSet).clone.foreach(edge => {
asScalaSet(jobGraph.edgeSet).clone().foreach(edge => {
if (running) edge match {
case cmd: FunctionEdge => {
addCollectionOutputs(cmd.outputs)
@ -249,7 +254,7 @@ class QGraph extends Logging {
})
}
private def getReadyJobs(): Set[FunctionEdge] = {
private def getReadyJobs: Set[FunctionEdge] = {
jobGraph.edgeSet.filter{
case f: FunctionEdge =>
this.previousFunctions(f).forall(_.status == RunnerStatus.DONE) && f.status == RunnerStatus.PENDING
@ -317,33 +322,39 @@ class QGraph extends Logging {
updateGraphStatus(false)
var readyJobs = getReadyJobs()
var readyJobs = getReadyJobs
while (running && readyJobs.size > 0) {
logger.debug("+++++++")
foreachFunction(readyJobs.toList, edge => {
foreachFunction(readyJobs.toSeq, edge => {
if (running) {
edge.myRunInfo.startTime = new Date()
edge.getRunInfo.exechosts = Utils.resolveHostname()
logEdge(edge)
edge.myRunInfo.doneTime = new Date()
edge.markAsDone
edge.markAsDone()
}
})
readyJobs = getReadyJobs()
readyJobs = getReadyJobs
}
}
private def logEdge(edge: FunctionEdge) {
logger.info("-------")
logger.info("%-8s %s".format(StringUtils.capitalize(edge.status.toString) + ":", edge.function.description))
if (logger.isDebugEnabled) {
logger.debug("Inputs: " + edge.inputs)
logger.debug("Inputs: " + edge.inputs)
logger.debug("Outputs: " + edge.outputs)
logger.debug("Done+: " + edge.function.doneOutputs.filter(_.exists()))
logger.debug("Done-: " + edge.function.doneOutputs.filterNot(_.exists()))
logger.debug("CmdDir: " + edge.function.commandDirectory)
logger.debug("Temp?: " + edge.function.isIntermediate)
logger.debug("Prev: " +
(if (edge.resetFromStatus == null) "none" else StringUtils.capitalize(edge.resetFromStatus.toString)) +
" (reset = " + (edge.resetFromStatus != null && edge.resetFromStatus != edge.status) + ")" )
}
logger.info(StringUtils.capitalize(edge.status.toString) + ": " + edge.function.description)
if (logger.isDebugEnabled)
logger.debug(edge.function.commandDirectory + " > " + edge.function.description)
logger.info("Log: " + edge.function.jobOutputFile.getAbsolutePath)
logger.info("Log: " + edge.function.jobOutputFile.getAbsolutePath)
if (edge.function.jobErrorFile != null)
logger.info("Error: " + edge.function.jobErrorFile.getAbsolutePath)
logger.info("Error: " + edge.function.jobErrorFile.getAbsolutePath)
}
/**
@ -380,7 +391,7 @@ class QGraph extends Logging {
updateGraphStatus(true)
var readyJobs = TreeSet.empty[FunctionEdge](functionOrdering)
readyJobs ++= getReadyJobs()
readyJobs ++= getReadyJobs
runningJobs = Set.empty[FunctionEdge]
var lastRunningCheck = System.currentTimeMillis
var logNextStatusCounts = true
@ -407,7 +418,7 @@ class QGraph extends Logging {
statusCounts.running += startedJobs.size
if (logNextStatusCounts)
logStatusCounts
logStatusCounts()
logNextStatusCounts = false
deleteCleanup(lastRunningCheck)
@ -456,10 +467,10 @@ class QGraph extends Logging {
checkRetryJobs(failedJobs)
}
readyJobs ++= getReadyJobs()
readyJobs ++= getReadyJobs
}
logStatusCounts
logStatusCounts()
deleteCleanup(-1)
} catch {
case e =>
@ -476,7 +487,7 @@ class QGraph extends Logging {
private def nextRunningCheck(lastRunningCheck: Long) =
((30 * 1000L) - (System.currentTimeMillis - lastRunningCheck))
private def logStatusCounts {
private def logStatusCounts() {
logger.info("%d Pend, %d Run, %d Fail, %d Done".format(
statusCounts.pending, statusCounts.running, statusCounts.failed, statusCounts.done))
}
@ -532,7 +543,8 @@ class QGraph extends Logging {
}
if (edge.status == RunnerStatus.DONE || edge.status == RunnerStatus.SKIPPED) {
logger.debug("Already done: " + edge.function.description)
if (logger.isDebugEnabled)
logEdge(edge)
addCleanup(edge)
}
}
@ -546,12 +558,12 @@ class QGraph extends Logging {
}
/**
* Checks if the function should have their outptus removed after they finish running
* @param edges Function to check
* Checks if the function should have their outputs removed after they finish running
* @param edge Function to check
*/
private def addCleanup(edge: FunctionEdge) {
if (!settings.keepIntermediates)
if (edge.function.isIntermediate && edge.function.deleteIntermediateOutputs)
if (edge.function.isIntermediate)
cleanupJobs += edge
}
@ -601,14 +613,16 @@ class QGraph extends Logging {
* From the previous edges, resets any that are marked as skipped to pending.
* If those that are reset have skipped edges, those skipped edges are recursively also set
* to pending.
* Any edges after this edge are also reset to pending.
* @param edge Dependent edge.
* @param previous Previous edges that provide inputs to edge.
* @param cleanOutputs If true will clean up the output files when resetting skipped jobs to pending.
* @param cleanOutputs If true will clean up the output files when resetting jobs to pending.
*/
private def resetPreviousSkipped(edge: FunctionEdge, previous: List[FunctionEdge], cleanOutputs: Boolean) {
for (previousEdge <- previous.filter(_.status == RunnerStatus.SKIPPED)) {
previousEdge.resetToPending(cleanOutputs)
resetPreviousSkipped(previousEdge, this.previousFunctions(previousEdge), cleanOutputs)
private def resetPreviousSkipped(edge: FunctionEdge, previous: Seq[FunctionEdge], cleanOutputs: Boolean) {
val edges = previous.filter(_.status == RunnerStatus.SKIPPED) ++ this.nextFunctions(edge).filter(_.status != RunnerStatus.PENDING)
for (resetEdge <- edges) {
resetEdge.resetToPending(cleanOutputs)
resetPreviousSkipped(resetEdge, this.previousFunctions(resetEdge), cleanOutputs)
}
}
@ -628,9 +642,9 @@ class QGraph extends Logging {
val emailMessage = new EmailMessage
emailMessage.from = settings.statusEmailFrom
emailMessage.to = settings.statusEmailTo
emailMessage.subject = "Queue function: Started: " + settings.qSettings.jobNamePrefix
addStartedFunctions(emailMessage, started.toList)
emailMessage.trySend(settings.qSettings.emailSettings)
emailMessage.subject = "Queue function: Started: " + settings.qSettings.runName
addStartedFunctions(emailMessage, started.toSeq)
emailMessage.trySend(settings.emailSettings)
}
}
@ -639,9 +653,9 @@ class QGraph extends Logging {
val emailMessage = new EmailMessage
emailMessage.from = settings.statusEmailFrom
emailMessage.to = settings.statusEmailTo
emailMessage.subject = "Queue function: Failure: " + settings.qSettings.jobNamePrefix
addFailedFunctions(emailMessage, failed.toList)
emailMessage.trySend(settings.qSettings.emailSettings)
emailMessage.subject = "Queue function: Failure: " + settings.qSettings.runName
addFailedFunctions(emailMessage, failed.toSeq)
emailMessage.trySend(settings.emailSettings)
}
}
@ -665,7 +679,7 @@ class QGraph extends Logging {
private def emailStatus() {
if (running && settings.statusEmailTo.size > 0) {
var failed = List.empty[FunctionEdge]
var failed = Seq.empty[FunctionEdge]
foreachFunction(edge => {
if (edge.status == RunnerStatus.FAILED) {
failed :+= edge
@ -677,16 +691,16 @@ class QGraph extends Logging {
emailMessage.to = settings.statusEmailTo
emailMessage.body = getStatus + nl
if (failed.size == 0) {
emailMessage.subject = "Queue run: Success: " + settings.qSettings.jobNamePrefix
emailMessage.subject = "Queue run: Success: " + settings.qSettings.runName
} else {
emailMessage.subject = "Queue run: Failure: " + settings.qSettings.jobNamePrefix
emailMessage.subject = "Queue run: Failure: " + settings.qSettings.runName
addFailedFunctions(emailMessage, failed)
}
emailMessage.trySend(settings.qSettings.emailSettings)
emailMessage.trySend(settings.emailSettings)
}
}
private def addStartedFunctions(emailMessage: EmailMessage, started: List[FunctionEdge]) {
private def addStartedFunctions(emailMessage: EmailMessage, started: Seq[FunctionEdge]) {
if (emailMessage.body == null)
emailMessage.body = ""
emailMessage.body += """
@ -697,7 +711,7 @@ class QGraph extends Logging {
started.map(edge => emailDescription(edge)).mkString(nl+nl))
}
private def addFailedFunctions(emailMessage: EmailMessage, failed: List[FunctionEdge]) {
private def addFailedFunctions(emailMessage: EmailMessage, failed: Seq[FunctionEdge]) {
val logs = failed.flatMap(edge => logFiles(edge))
if (emailMessage.body == null)
@ -725,7 +739,7 @@ class QGraph extends Logging {
}
private def logFiles(edge: FunctionEdge) = {
var failedOutputs = List.empty[File]
var failedOutputs = Seq.empty[File]
failedOutputs :+= edge.function.jobOutputFile
if (edge.function.jobErrorFile != null)
failedOutputs :+= edge.function.jobErrorFile
@ -762,14 +776,14 @@ class QGraph extends Logging {
private def getStatus = {
val buffer = new StringBuilder
doStatus(status => buffer.append(status).append(nl))
buffer.toString
buffer.toString()
}
/**
* Gets job statuses by traversing the graph and looking for status-related files
*/
private def doStatus(statusFunc: String => Unit) = {
var statuses = List.empty[AnalysisStatus]
private def doStatus(statusFunc: String => Unit) {
var statuses = Seq.empty[AnalysisStatus]
var maxWidth = 0
foreachFunction(edge => {
val name = edge.function.analysisName
@ -860,7 +874,7 @@ class QGraph extends Logging {
private def newGraph = new SimpleDirectedGraph[QNode, QEdge](new EdgeFactory[QNode, QEdge] {
def createEdge(input: QNode, output: QNode) = new MappingEdge(input, output)})
private def getQNode(files: List[File]) = {
private def getQNode(files: Seq[File]) = {
nodeMap.get(files) match {
case Some(node) =>
node
@ -888,7 +902,7 @@ class QGraph extends Logging {
if (inputs.files.size > 1)
for (file <- inputs.files) {
if (running) {
val input = getQNode(List(file))
val input = getQNode(Seq(file))
if (!jobGraph.containsEdge(input, inputs))
addEdge(new MappingEdge(input, inputs))
}
@ -903,7 +917,7 @@ class QGraph extends Logging {
if (outputs.files.size > 1)
for (file <- outputs.files) {
if (running) {
val output = getQNode(List(file))
val output = getQNode(Seq(file))
if (!jobGraph.containsEdge(outputs, output))
addEdge(new MappingEdge(outputs, output))
}
@ -937,37 +951,36 @@ class QGraph extends Logging {
/**
* Utility function for running a method over all function edges.
* @param edgeFunction Function to run for each FunctionEdge.
* @param f Function to run for each FunctionEdge.
*/
private def foreachFunction(f: (FunctionEdge) => Unit) {
foreachFunction(jobGraph.edgeSet.toList.filter(_.isInstanceOf[FunctionEdge]).asInstanceOf[List[FunctionEdge]], f)
foreachFunction(jobGraph.edgeSet.toSeq.filter(_.isInstanceOf[FunctionEdge]).asInstanceOf[Seq[FunctionEdge]], f)
}
/**
* Utility function for running a method over a list of function edges.
* @param edegs Edges to traverse.
* @param edgeFunction Function to run for each FunctionEdge.
* @param edges Edges to traverse.
* @param f Function to run for each FunctionEdge.
*/
private def foreachFunction(edges: List[FunctionEdge], f: (FunctionEdge) => Unit) {
private def foreachFunction(edges: Seq[FunctionEdge], f: (FunctionEdge) => Unit) {
edges.sorted(functionOrdering).foreach(edge => if (running) f(edge))
}
/**
* Utility function for running a method over all function edges.
* @param edgeFunction Function to run for each FunctionEdge.
* Utility function returning all function edges.
*/
private def getFunctionEdges: List[FunctionEdge] = {
jobGraph.edgeSet.toList.filter(_.isInstanceOf[FunctionEdge]).asInstanceOf[List[FunctionEdge]]
private def getFunctionEdges: Seq[FunctionEdge] = {
jobGraph.edgeSet.toSeq.filter(_.isInstanceOf[FunctionEdge]).asInstanceOf[Seq[FunctionEdge]]
}
/**
* Utility function for running a method over all functions, but traversing the nodes in order of dependency.
* @param edgeFunction Function to run for each FunctionEdge.
* @param f Function to run for each FunctionEdge.
*/
private def traverseFunctions(f: (FunctionEdge) => Unit) {
val iterator = new TopologicalOrderIterator(this.jobGraph)
iterator.addTraversalListener(new TraversalListenerAdapter[QNode, QEdge] {
override def edgeTraversed(event: EdgeTraversalEvent[QNode, QEdge]) = {
override def edgeTraversed(event: EdgeTraversalEvent[QNode, QEdge]) {
if (running) {
event.getEdge match {
case functionEdge: FunctionEdge => f(functionEdge)
@ -980,23 +993,44 @@ class QGraph extends Logging {
}
/**
* Outputs the graph to a .dot file.
* Outputs the graph to a .gv DOT file.
* http://www.graphviz.org/Documentation.php
* http://en.wikipedia.org/wiki/DOT_language
* @param file Path to output the .dot file.
* @param file Path to output the .gv file.
*/
private def renderToDot(file: java.io.File) {
val out = new java.io.FileWriter(file)
// todo -- we need a nice way to visualize the key pieces of information about commands. Perhaps a
// todo -- visualizeString() command, or something that shows inputs / outputs
val ve = new org.jgrapht.ext.EdgeNameProvider[QEdge] {
def getEdgeName(function: QEdge) = if (function.dotString == null) "" else function.dotString.replace("\"", "\\\"")
private def renderGraph(file: java.io.File) {
val vertexIDProvider = new org.jgrapht.ext.VertexNameProvider[QNode] {
def getVertexName(node: QNode) = node.id.toString
}
//val iterator = new TopologicalOrderIterator(qGraph.jobGraph)
(new DOTExporter(new org.jgrapht.ext.IntegerNameProvider[QNode](), null, ve)).export(out, jobGraph)
val vertexLabelProvider = new org.jgrapht.ext.VertexNameProvider[QNode] {
// The QGraph fills in with single file nodes between nodes that contain more than one file.
// We only need to display the single element nodes.
def getVertexName(node: QNode) = {
if (!node.files.isEmpty && node.files.tail.isEmpty)
node.files.head.getName
else
""
}
}
out.close
val edgeNameProvider = new org.jgrapht.ext.EdgeNameProvider[QEdge] {
def getEdgeName(edge: QEdge) = {
if (edge.shortDescription != null)
edge.shortDescription.replace("\"", "\\\"")
else
""
}
}
val exporter = new DOTExporter(vertexIDProvider, vertexLabelProvider, edgeNameProvider)
val out = new OutputStreamWriter(FileUtils.openOutputStream(file))
try {
exporter.export(out, jobGraph)
} finally {
IOUtils.closeQuietly(out)
}
}
/**
@ -1054,7 +1088,7 @@ class QGraph extends Logging {
*/
def isShutdown = !running
def getFunctionsAndStatus(functions: List[QFunction]): Map[QFunction, JobRunInfo] = {
def getFunctionsAndStatus: Map[QFunction, JobRunInfo] = {
getFunctionEdges.map(edge => (edge.function, edge.getRunInfo)).toMap
}

17
public/scala/src/org/broadinstitute/sting/queue/engine/QGraphSettings.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -26,7 +26,7 @@ package org.broadinstitute.sting.queue.engine
import java.io.File
import org.broadinstitute.sting.queue.QSettings
import org.broadinstitute.sting.queue.util.SystemUtils
import org.broadinstitute.sting.queue.util.{EmailSettings, SystemUtils}
import org.broadinstitute.sting.commandline.{Advanced, ArgumentCollection, Argument}
/**
@ -58,16 +58,16 @@ class QGraphSettings {
var keepIntermediates = false
@Argument(fullName="status_email_to", shortName="statusTo", doc="Email address to send emails to upon completion or on error.", required=false)
var statusEmailTo: List[String] = Nil
var statusEmailTo: Seq[String] = Nil
@Argument(fullName="status_email_from", shortName="statusFrom", doc="Email address to send emails from upon completion or on error.", required=false)
var statusEmailFrom: String = System.getProperty("user.name") + "@" + SystemUtils.mailName
@Argument(fullName="dot_graph", shortName="dot", doc="Outputs the queue graph to a .dot file. See: http://en.wikipedia.org/wiki/DOT_language", required=false)
var dotFile: File = _
@Argument(fullName="graphviz", shortName="gv", doc="Outputs the queue graph to a Graphviz .gv file. See: http://www.graphviz.org/Documentation.php", required=false)
var graphvizFile: File = _
@Argument(fullName="expanded_dot_graph", shortName="expandedDot", doc="Outputs the queue graph of scatter gather to a .dot file. Otherwise overwrites the dot_graph", required=false)
var expandedDotFile: File = _
@Argument(fullName="graphviz_scatter_gather", shortName="gvsg", doc="Outputs the scatter/gather queue graph to a Graphviz .gv file. Otherwise overwrites the --graphviz file.", required=false)
var graphvizScatterGatherFile: File = _
@Argument(fullName="jobReport", shortName="jobReport", doc="File where we will write the Queue job report", required=false)
var jobReportFile: String = _
@ -76,6 +76,9 @@ class QGraphSettings {
@Argument(fullName="disableJobReport", shortName="disabpleJobReport", doc="If provided, we will not create a job report", required=false)
var disableJobReport: Boolean = false
@ArgumentCollection
val emailSettings = new EmailSettings
@ArgumentCollection
val qSettings = new QSettings
}

28
public/scala/src/org/broadinstitute/sting/queue/engine/QNode.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.engine
import java.io.File
@ -6,7 +30,7 @@ import java.io.File
* Represents a state between QFunctions the directed acyclic QGraph
* @param files The list of files that represent this node state ordered by file name.
*/
class QNode (val id: Int, val files: List[File]) {
class QNode (val id: Int, val files: Seq[File]) {
override def equals(obj: Any) = {
obj match {
case other: QNode => this.id == other.id
@ -16,5 +40,5 @@ class QNode (val id: Int, val files: List[File]) {
override def hashCode = id
override def toString = files.toString
override def toString = files.toString()
}

4
public/scala/src/org/broadinstitute/sting/queue/engine/gridengine/GridEngineJobRunner.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -69,7 +69,7 @@ class GridEngineJobRunner(session: Session, function: CommandLineFunction) exten
if ( function.nCoresRequest.getOrElse(1) > 1 ) {
if ( function.qSettings.dontRequestMultipleCores )
logger.warn("Sending multicore job %s to farm without requesting appropriate number of cores (%d)".format(
function.jobName, function.nCoresRequest.get))
function.shortDescription, function.nCoresRequest.get))
else
nativeSpec += " -pe %s %d".format(function.qSettings.parallelEnvironmentName, function.nCoresRequest.get)
}

13
public/scala/src/org/broadinstitute/sting/queue/engine/lsf/Lsf706JobRunner.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -35,8 +35,8 @@ import org.broadinstitute.sting.queue.engine.{RunnerStatus, CommandLineJobRunner
import java.util.regex.Pattern
import java.lang.StringBuffer
import java.util.Date
import com.sun.jna.{Pointer, Structure, StringArray, NativeLong}
import com.sun.jna.ptr.{PointerByReference, IntByReference}
import com.sun.jna.{Structure, StringArray, NativeLong}
import com.sun.jna.ptr.IntByReference
/**
* Runs jobs on an LSF compute cluster.
@ -60,7 +60,6 @@ class Lsf706JobRunner(val function: CommandLineFunction) extends CommandLineJobR
/**
* Dispatches the function on the LSF cluster.
* @param function Command to run.
*/
def start() {
Lsf706JobRunner.lsfLibLock.synchronized {
@ -110,7 +109,7 @@ class Lsf706JobRunner(val function: CommandLineFunction) extends CommandLineJobR
if ( function.nCoresRequest.getOrElse(1) > 1 ) {
if ( function.qSettings.dontRequestMultipleCores )
logger.warn("Sending multicore job %s to farm without requesting appropriate number of cores (%d)".format(
function.jobName, function.nCoresRequest.get))
function.shortDescription, function.nCoresRequest.get))
else {
request.numProcessors = function.nCoresRequest.get
request.maxNumProcessors = request.numProcessors
@ -298,7 +297,7 @@ object Lsf706JobRunner extends Logging {
runner.getRunInfo.doneTime = new Date(jobInfo.endTime.longValue * 1000)
val exHostsRaw = jobInfo.exHosts.getStringArray(0)
//logger.warn("exHostsRaw = " + exHostsRaw)
val exHostsList = exHostsRaw.toList
val exHostsList = exHostsRaw.toSeq
//logger.warn("exHostsList = " + exHostsList)
val exHosts = exHostsList.reduceLeft(_ + "," + _)
//logger.warn("exHosts = " + exHosts)
@ -363,7 +362,7 @@ object Lsf706JobRunner extends Logging {
/**
* Returns the run limit in seconds for the queue.
* If the queue name is null returns the length of the default queue.
* @param queue Name of the queue or null for the default queue.
* @param queueName Name of the queue or null for the default queue.
* @return the run limit in seconds for the queue.
*/
private def getRlimitRun(queueName: String) = {

5
public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala
View File

@ -53,6 +53,9 @@ class BamGatherFunction extends GatherFunction with PicardBamFunction {
val disableIndex = QFunction.findField(originalFunction.getClass, SAMFileWriterArgumentTypeDescriptor.DISABLE_INDEXING_FULLNAME)
this.createIndex = Some(!originalGATK.getFieldValue(disableIndex).asInstanceOf[Boolean])
super.freezeFieldValues
val enableMD5 = QFunction.findField(originalFunction.getClass, SAMFileWriterArgumentTypeDescriptor.ENABLE_MD5_FULLNAME)
this.createMD5 = Some(originalGATK.getFieldValue(enableMD5).asInstanceOf[Boolean])
super.freezeFieldValues()
}
}

15
public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -32,9 +32,8 @@ import net.sf.samtools.SAMFileHeader
import java.util.Collections
import org.broadinstitute.sting.utils.{GenomeLoc, GenomeLocSortedSet, GenomeLocParser}
case class GATKIntervals(reference: File, intervals: List[String]) {
case class GATKIntervals(reference: File, intervals: Seq[String]) {
private lazy val referenceDataSource = new ReferenceDataSource(reference)
// private var splitsBySize = Map.empty[Int, java.util.List[java.lang.Integer]]
lazy val samFileHeader = {
val header = new SAMFileHeader
@ -53,13 +52,5 @@ case class GATKIntervals(reference: File, intervals: List[String]) {
Collections.unmodifiableList(parsedLocs)
}
lazy val contigs = locs.map(_.getContig).distinct.toList
// def getSplits(size: Int) = {
// splitsBySize.getOrElse(size, {
// val splits: java.util.List[java.lang.Integer] = IntervalUtils.splitFixedIntervals(locs, size)
// splitsBySize += size -> splits
// splits
// })
// }
lazy val contigs = locs.map(_.getContig).distinct.toSeq
}

21
public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKScatterFunction.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -26,7 +26,6 @@ package org.broadinstitute.sting.queue.extensions.gatk
import org.broadinstitute.sting.utils.interval.IntervalUtils
import java.io.File
import collection.JavaConversions._
import org.broadinstitute.sting.utils.io.IOUtils
import org.broadinstitute.sting.queue.function.scattergather.{CloneFunction, ScatterFunction}
import org.broadinstitute.sting.commandline.Output
@ -39,7 +38,7 @@ trait GATKScatterFunction extends ScatterFunction {
private final val intervalsStringField = "intervalsString"
@Output(doc="Scatter function outputs")
var scatterOutputFiles: List[File] = Nil
var scatterOutputFiles: Seq[File] = Nil
/** The original GATK function. */
protected var originalGATK: CommandLineGATK = _
@ -48,7 +47,7 @@ trait GATKScatterFunction extends ScatterFunction {
protected var referenceSequence: File = _
/** The list of interval files ("/path/to/interval.list") or interval strings ("chr1", "chr2") to parse into smaller parts. */
protected var intervals: List[String] = Nil
protected var intervals: Seq[String] = Nil
/** Whether the last scatter job should also include any unmapped reads. */
protected var includeUnmapped: Boolean = _
@ -57,7 +56,7 @@ trait GATKScatterFunction extends ScatterFunction {
this.originalGATK = this.originalFunction.asInstanceOf[CommandLineGATK]
this.referenceSequence = this.originalGATK.reference_sequence
if (this.originalGATK.intervals.isEmpty && (this.originalGATK.intervalsString == null || this.originalGATK.intervalsString.isEmpty)) {
this.intervals ++= GATKScatterFunction.getGATKIntervals(this.referenceSequence, List.empty[String]).contigs
this.intervals ++= GATKScatterFunction.getGATKIntervals(this.referenceSequence, Seq.empty[String]).contigs
} else {
this.intervals ++= this.originalGATK.intervals.map(_.toString)
this.intervals ++= this.originalGATK.intervalsString.filterNot(interval => IntervalUtils.isUnmapped(interval))
@ -70,16 +69,16 @@ trait GATKScatterFunction extends ScatterFunction {
}
override def initCloneInputs(cloneFunction: CloneFunction, index: Int) {
cloneFunction.setFieldValue(this.intervalsField, List(new File("scatter.intervals")))
cloneFunction.setFieldValue(this.intervalsField, Seq(new File("scatter.intervals")))
if (index == this.scatterCount && this.includeUnmapped)
cloneFunction.setFieldValue(this.intervalsStringField, List("unmapped"))
cloneFunction.setFieldValue(this.intervalsStringField, Seq("unmapped"))
else
cloneFunction.setFieldValue(this.intervalsStringField, List.empty[String])
cloneFunction.setFieldValue(this.intervalsStringField, Seq.empty[String])
}
override def bindCloneInputs(cloneFunction: CloneFunction, index: Int) {
val scatterPart = cloneFunction.getFieldValue(this.intervalsField)
.asInstanceOf[List[File]]
.asInstanceOf[Seq[File]]
.map(file => IOUtils.absolute(cloneFunction.commandDirectory, file))
cloneFunction.setFieldValue(this.intervalsField, scatterPart)
this.scatterOutputFiles ++= scatterPart
@ -100,9 +99,9 @@ trait GATKScatterFunction extends ScatterFunction {
}
object GATKScatterFunction {
var gatkIntervals = List.empty[GATKIntervals]
var gatkIntervals = Seq.empty[GATKIntervals]
def getGATKIntervals(reference: File, intervals: List[String]) = {
def getGATKIntervals(reference: File, intervals: Seq[String]) = {
gatkIntervals.find(gi => gi.reference == reference && gi.intervals == intervals) match {
case Some(gi) => gi
case None =>

41
public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/RodBind.scala
View File

@ -1,41 +0,0 @@
package org.broadinstitute.sting.queue.extensions.gatk
import java.io.File
import org.broadinstitute.sting.utils.io.FileExtension
import java.lang.String
/**
* Used to provide -B rodBind arguments to the GATK.
*/
class RodBind(var trackName: String, var trackType: String, path: String, val tag: String) extends File(path) with FileExtension {
def this(trackName: String, trackType: String, path: String) =
this(trackName, trackType, path, null)
def this(trackName: String, trackType: String, file: File, tag: String) =
this(trackName, trackType, file.getPath, tag)
def this(trackName: String, trackType: String, file: File) =
this(trackName, trackType, file.getPath, null)
require(trackName != null, "RodBind trackName cannot be null")
require(trackType != null, "RodBind trackType cannot be null")
def withPath(newPath: String) = new RodBind(trackName, trackType, newPath, tag)
}
/**
* Used to provide -B rodBind arguments to the GATK.
*/
object RodBind {
def apply(trackName: String, trackType: String, path: String, tag: String) = new RodBind(trackName, trackType, path, tag)
def apply(trackName: String, trackType: String, path: String) = new RodBind(trackName, trackType, path, null)
def apply(trackName: String, trackType: String, file: File, tag: String) = new RodBind(trackName, trackType, file, tag)
def apply(trackName: String, trackType: String, file: File) = new RodBind(trackName, trackType, file, null)
def formatCommandLineParameter( cmdLineParam: String, value: Any ) = {
value match {
case rodBind: RodBind if (rodBind.tag != null) =>
"%s:%s,%s,%s".format(cmdLineParam, rodBind.trackName, rodBind.trackType, rodBind.tag)
case rodBind: RodBind =>
"%s:%s,%s".format(cmdLineParam, rodBind.trackName, rodBind.trackType)
case x =>
""
}
}
}

26
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/AddOrReplaceReadGroups.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.extensions.picard
import org.broadinstitute.sting.commandline._
@ -15,7 +39,7 @@ class AddOrReplaceReadGroups extends org.broadinstitute.sting.queue.function.Jav
javaMainClass = "net.sf.picard.sam.AddOrReplaceReadGroups"
@Input(doc="The input SAM or BAM files to analyze. Must be coordinate sorted.", shortName = "input", fullName = "input_bam_files", required = true)
var input: List[File] = Nil
var input: Seq[File] = Nil
@Output(doc="The output BAM file with the modified/added read groups", shortName = "output", fullName = "output_bam_file", required = true)
var output: File = _

26
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MarkDuplicates.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.extensions.picard
import org.broadinstitute.sting.commandline._
@ -15,7 +39,7 @@ class MarkDuplicates extends org.broadinstitute.sting.queue.function.JavaCommand
javaMainClass = "net.sf.picard.sam.MarkDuplicates"
@Input(doc="The input SAM or BAM files to analyze. Must be coordinate sorted.", shortName = "input", fullName = "input_bam_files", required = true)
var input: List[File] = Nil
var input: Seq[File] = Nil
@Output(doc="The output file to write marked records to", shortName = "output", fullName = "output_bam_file", required = true)
var output: File = _

27
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MergeSamFiles.scala
View File

@ -1,9 +1,32 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.extensions.picard
import org.broadinstitute.sting.commandline._
import java.io.File
import org.broadinstitute.sting.queue.QScript._
/*
* Created by IntelliJ IDEA.
@ -16,7 +39,7 @@ class MergeSamFiles extends org.broadinstitute.sting.queue.function.JavaCommandL
javaMainClass = "net.sf.picard.sam.MergeSamFiles"
@Input(doc="The input SAM or BAM files to analyze. Must be coordinate sorted.", shortName = "input", fullName = "input_bam_files", required = true)
var input: List[File] = Nil
var input: Seq[File] = Nil
@Output(doc="The output merged BAM file", shortName = "output", fullName = "output_bam_file", required = true)
var output: File = _

24
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamFunction.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -41,20 +41,22 @@ trait PicardBamFunction extends JavaCommandLineFunction {
var sortOrder = SortOrder.coordinate
var compressionLevel: Option[Int] = None
var createIndex: Option[Boolean] = None
var createMD5: Option[Boolean] = None
var maxRecordsInRam: Option[Int] = None
var assumeSorted: Option[Boolean] = None
protected def inputBams: List[File]
protected def inputBams: Seq[File]
protected def outputBam: File
abstract override def commandLine = super.commandLine +
repeat("INPUT=", inputBams, spaceSeparated=false) +
required("TMP_DIR=" + jobTempDir) +
optional("OUTPUT=", outputBam, spaceSeparated=false) +
optional("COMPRESSION_LEVEL=", compressionLevel, spaceSeparated=false) +
optional("VALIDATION_STRINGENCY=", validationStringency, spaceSeparated=false) +
optional("SO=", sortOrder, spaceSeparated=false) +
optional("MAX_RECORDS_IN_RAM=", maxRecordsInRam, spaceSeparated=false) +
optional("ASSUME_SORTED=", assumeSorted, spaceSeparated=false) +
optional("CREATE_INDEX=", createIndex, spaceSeparated=false)
repeat("INPUT=", inputBams, spaceSeparated=false) +
required("TMP_DIR=" + jobTempDir) +
optional("OUTPUT=", outputBam, spaceSeparated=false) +
optional("COMPRESSION_LEVEL=", compressionLevel, spaceSeparated=false) +
optional("VALIDATION_STRINGENCY=", validationStringency, spaceSeparated=false) +
optional("SO=", sortOrder, spaceSeparated=false) +
optional("MAX_RECORDS_IN_RAM=", maxRecordsInRam, spaceSeparated=false) +
optional("ASSUME_SORTED=", assumeSorted, spaceSeparated=false) +
optional("CREATE_INDEX=", createIndex, spaceSeparated=false) +
optional("CREATE_MD5_FILE=", createMD5, spaceSeparated=false)
}

26
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ReorderSam.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.extensions.picard
import org.broadinstitute.sting.commandline._
@ -14,7 +38,7 @@ class ReorderSam extends org.broadinstitute.sting.queue.function.JavaCommandLine
javaMainClass = "net.sf.picard.sam.ReorderSam"
@Input(doc="Input file (bam or sam) to extract reads from.", shortName = "input", fullName = "input_bam_files", required = true)
var input: List[File] = Nil
var input: Seq[File] = Nil
@Output(doc="Output file (bam or sam) to write extracted reads to.", shortName = "output", fullName = "output_bam_file", required = true)
var output: File = _

28
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/RevertSam.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.extensions.picard
import org.broadinstitute.sting.commandline._
@ -15,7 +39,7 @@ class RevertSam extends org.broadinstitute.sting.queue.function.JavaCommandLineF
javaMainClass = "net.sf.picard.sam.RevertSam"
@Input(shortName = "input", fullName = "input_bam_files", required = true, doc = "The input SAM or BAM files to revert.")
var input: List[File] = Nil
var input: Seq[File] = Nil
@Output(shortName = "output", fullName = "output_bam_file", required = true, doc = "The reverted BAM or SAM output file.")
var output: File = _
@ -33,7 +57,7 @@ class RevertSam extends org.broadinstitute.sting.queue.function.JavaCommandLineF
var removeAlignmentInformation: Boolean = true
@Argument(shortName = "atc", fullName = "attributes_to_clear", required = false, doc = "When removing alignment information, the set of optional tags to remove.")
var attributesToClear: List[String] = Nil
var attributesToClear: Seq[String] = Nil
@Argument(shortName = "sa", fullName = "sample_alias", required = false, doc = "The sample alias to use in the reverted output file. This will override the existing sample alias in the file and is used only if all the read groups in the input file have the same sample alias.")
var sampleAlias: String = null

26
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SamToFastq.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.extensions.picard
import org.broadinstitute.sting.commandline._
@ -15,7 +39,7 @@ class SamToFastq extends org.broadinstitute.sting.queue.function.JavaCommandLine
javaMainClass = "net.sf.picard.sam.SamToFastq"
@Input(shortName = "input", fullName = "input_bam_files", required = true, doc = "Input SAM/BAM file to extract reads from.")
var input: List[File] = Nil
var input: Seq[File] = Nil
@Output(shortName = "fastq", fullName = "output_fastq_file", required = true, doc = "Output fastq file (single-end fastq or, if paired, first end of the pair fastq).")
var fastq: File = _

27
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SortSam.scala
View File

@ -1,9 +1,32 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.extensions.picard
import org.broadinstitute.sting.commandline._
import java.io.File
import org.broadinstitute.sting.queue.QScript._
/*
* Created by IntelliJ IDEA.
@ -16,7 +39,7 @@ class SortSam extends org.broadinstitute.sting.queue.function.JavaCommandLineFun
javaMainClass = "net.sf.picard.sam.SortSam"
@Input(doc="The input SAM or BAM files to sort.", shortName = "input", fullName = "input_bam_files", required = true)
var input: List[File] = Nil
var input: Seq[File] = Nil
@Output(doc="The sorted BAM or SAM output file.", shortName = "output", fullName = "output_bam_file", required = true)
var output: File = _

28
public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ValidateSamFile.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.extensions.picard
import org.broadinstitute.sting.commandline._
@ -17,7 +41,7 @@ class ValidateSamFile extends org.broadinstitute.sting.queue.function.JavaComman
javaMainClass = "net.sf.picard.sam.ValidateSamFile"
@Input(doc="The input SAM or BAM files to analyze. Must be coordinate sorted.", shortName = "input", fullName = "input_bam_files", required = true)
var input: List[File] = Nil
var input: Seq[File] = Nil
@Output(doc="Send output to a file instead of stdout", shortName = "output", fullName = "output_file", required = false)
var output: File = _
@ -26,7 +50,7 @@ class ValidateSamFile extends org.broadinstitute.sting.queue.function.JavaComman
var MODE: Mode = Mode.VERBOSE
@Argument(doc="List of validation error types to ignore.", shortName = "ignore", fullName = "ignore_error_types", required = false)
var IGNORE: List[String] = Nil
var IGNORE: Seq[String] = Nil
@Argument(doc = "The maximum number of lines output in verbose mode.", shortName = "max", fullName = "max_output", required = false)
var MAX_OUTPUT: Int = 100

2
public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsIndexFunction.scala
View File

@ -52,6 +52,4 @@ class SamtoolsIndexFunction extends SamtoolsCommandLineFunction {
required("index") +
required(bamFile) +
required(bamFileIndex)
override def dotString = "Index: %s".format(bamFile.getName)
}

10
public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsMergeFunction.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -34,7 +34,7 @@ class SamtoolsMergeFunction extends SamtoolsCommandLineFunction {
analysisName = "samtools merge"
@Input(doc="BAM file input")
var inputBams: List[File] = Nil
var inputBams: Seq[File] = Nil
@Output(doc="BAM file output")
var outputBam: File = _
@ -43,10 +43,10 @@ class SamtoolsMergeFunction extends SamtoolsCommandLineFunction {
var region: String = _
@Input(doc="BAM file input indexes")
var inputBamIndexes: List[File] = Nil
var inputBamIndexes: Seq[File] = Nil
override def freezeFieldValues = {
super.freezeFieldValues
override def freezeFieldValues() {
super.freezeFieldValues()
inputBamIndexes ++= inputBams
.filter(orig => orig != null && orig.getName.endsWith(".bam"))
.flatMap(orig => Array(

32
public/scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.function
import org.broadinstitute.sting.queue.util._
@ -27,13 +51,13 @@ trait CommandLineFunction extends QFunction with Logging {
var jobQueue: String = _
/** Native arguments to pass to the job runner */
var jobNativeArgs: List[String] = Nil
var jobNativeArgs: Seq[String] = Nil
/** Native arguments to pass to the job runner */
var jobResourceRequests: List[String] = Nil
var jobResourceRequests: Seq[String] = Nil
/** Environment names to pass to the job runner */
var jobEnvironmentNames: List[String] = Nil
var jobEnvironmentNames: Seq[String] = Nil
override def copySettingsTo(function: QFunction) {
super.copySettingsTo(function)
@ -270,7 +294,7 @@ trait CommandLineFunction extends QFunction with Logging {
}
// Trim leading and trailing whitespace off our three tokens, and unwrap Some(x) to x for the param
val trimmedValues : List[String] = List((if ( prefix != null ) prefix.trim else ""),
val trimmedValues : Seq[String] = Seq((if ( prefix != null ) prefix.trim else ""),
(param match {
case Some(x) => paramFormat.format(x).trim
case x => paramFormat.format(x).trim

18
public/scala/src/org/broadinstitute/sting/queue/function/InProcessFunction.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -24,10 +24,24 @@
package org.broadinstitute.sting.queue.function
import java.io.PrintStream
/**
* Runs a function in process.
*/
trait InProcessFunction extends QFunction {
analysisName = this.getClass.getSimpleName
def run()
def description = this.getClass.getSimpleName + " " + this.commandOutputs.mkString(" ")
/**
* During run() this stream will write to the stdout.
*/
var jobOutputStream: PrintStream = null
/**
* Write errors to this stream run().
*/
var jobErrorStream: PrintStream = null
}

6
public/scala/src/org/broadinstitute/sting/queue/function/JavaCommandLineFunction.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -42,7 +42,7 @@ trait JavaCommandLineFunction extends CommandLineFunction {
* Class path for the main class.
* Defaults to the current classpath.
*/
var javaClasspath: List[String] = Nil
var javaClasspath: Seq[String] = Nil
/**
* Memory limit for the java executable, or if None will use the default memoryLimit.
@ -82,5 +82,5 @@ trait JavaCommandLineFunction extends CommandLineFunction {
object JavaCommandLineFunction {
val currentClasspath = System.getProperty("java.class.path")
.split(File.pathSeparatorChar).map(path => IOUtils.absolute(new File(path)).getPath).toList
.split(File.pathSeparatorChar).map(path => IOUtils.absolute(new File(path)).getPath).toSeq
}

30
public/scala/src/org/broadinstitute/sting/queue/function/ListWriterFunction.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.function
import org.broadinstitute.sting.commandline.{Input, Output}
@ -9,10 +33,12 @@ import org.apache.commons.io.IOUtils
* Custom formats can override addFile.
*/
class ListWriterFunction extends InProcessFunction {
@Input(doc="input files") var inputFiles: List[File] = Nil
analysisName = "WriteList"
@Input(doc="input files") var inputFiles: Seq[File] = Nil
@Output(doc="output file") var listFile: File = _
def run {
def run() {
val writer = new PrintWriter(listFile)
try {
for (inputFile <- inputFiles)

193
public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -29,7 +29,7 @@ import java.lang.annotation.Annotation
import org.broadinstitute.sting.commandline._
import org.broadinstitute.sting.queue.{QException, QSettings}
import collection.JavaConversions._
import org.broadinstitute.sting.queue.function.scattergather.SimpleTextGatherFunction
import java.lang.IllegalStateException
import org.broadinstitute.sting.queue.util._
import org.broadinstitute.sting.utils.io.IOUtils
@ -39,13 +39,18 @@ import org.broadinstitute.sting.utils.io.IOUtils
* Inputs are matched to other outputs by using .equals()
*/
trait QFunction extends Logging with QJobReport {
/** A short description of this step in the graph */
/**
* A short description of what this class of function does.
* By default does not include the output specific to this function.
* See shortDescription for a description of what this instance of the function outputs.
*/
var analysisName: String = "<function>"
/** Prefix for automatic job name creation */
var jobNamePrefix: String = _
/** The name name of the job */
/**
* The name name of the job, must be file system safe and unique to the graph.
* Defaults to "runName-<order_function_added>".
* Use shortDescription for an alternative that is display friendly.
*/
var jobName: String = _
/** Default settings */
@ -58,7 +63,7 @@ trait QFunction extends Logging with QJobReport {
var jobTempDir: File = null
/** Order the function was added to the graph. */
var addOrder: List[Int] = Nil
var addOrder: Seq[Int] = Nil
/** Job priority */
var jobPriority: Option[Int] = None
@ -78,12 +83,6 @@ trait QFunction extends Logging with QJobReport {
*/
var isIntermediate = false
/**
* If true and isIntermediate is true, the files listed
* via outputs will deleted after the command completes.
*/
var deleteIntermediateOutputs = true
// -------------------------------------------------------
//
// job run information
@ -95,8 +94,6 @@ trait QFunction extends Logging with QJobReport {
* @param function QFunction to copy values to.
*/
override def copySettingsTo(function: QFunction) {
function.analysisName = this.analysisName
function.jobName = this.jobName
function.qSettings = this.qSettings
function.commandDirectory = this.commandDirectory
function.jobTempDir = this.jobTempDir
@ -105,79 +102,80 @@ trait QFunction extends Logging with QJobReport {
function.jobRestartable = this.jobRestartable
function.updateJobRun = this.updateJobRun
function.isIntermediate = this.isIntermediate
function.deleteIntermediateOutputs = this.deleteIntermediateOutputs
function.reportGroup = this.reportGroup
function.reportFeatures = this.reportFeatures
}
/** File to redirect any output. Defaults to <jobName>.out */
@Output(doc="File to redirect any output", required=false)
@Gather(classOf[SimpleTextGatherFunction])
var jobOutputFile: File = _
/** File to redirect any errors. Defaults to <jobName>.out */
@Output(doc="File to redirect any errors", required=false)
@Gather(classOf[SimpleTextGatherFunction])
var jobErrorFile: File = _
/**
* Description of this command line function.
*/
def description: String
def description: String = "%s: %s > %s".format(analysisName, inputs, outputs)
/**
* The function description in .dot files
* A short description of the function.
*/
def dotString = jobName + " => " + description
def shortDescription = {
firstOutput match {
case file: File => analysisName + ": " + file.getName
case _ => analysisName
}
}
/**
* Returns true if the function is done, false if it's
* not done and None if the done status is unknown.
* Returns true if the function is done.
*/
def isDone = {
def isDone: Boolean = {
val files = doneOutputs
if (files.size == 0)
None
else
Some(files.forall(_.exists))
throw new IllegalStateException("Function should have at least one output: " + analysisName)
files.forall(_.exists)
}
/**
* Returns true if the function has failed, false if it
* has not failed and None if the fail status is unknown.
* Returns true if the function has failed.
*/
def isFail = {
def isFail: Boolean = {
val files = failOutputs
if (files.size == 0)
None
else
Some(files.exists(_.exists))
throw new IllegalStateException("Function should have at least one output: " + analysisName)
files.exists(_.exists)
}
/**
* Returns true if the file is a log file for this function.
* Returns files to track for hidden done/fail files.
* @return Seq[String] files.
*/
protected def isLogFile(file: File) =
file == jobOutputFile || file == jobErrorFile
protected def statusPaths = {
var paths = outputs
paths :+= jobOutputFile
if (jobErrorFile != null)
paths :+= jobErrorFile
paths
}
/**
* Returns prefixes for hidden done/fail files.
* @return prefixes.
*/
private def statusPrefixes = statusPaths.map(file => file.getParentFile + "/." + file.getName)
/**
* Returns the output files for this function.
* @return Set[File] outputs for this function.
* @return outputs for this function.
*/
private def statusPaths =
commandOutputs.map(file => file.getParentFile + "/." + file.getName)
/**
* Returns the output files for this function.
* @return Set[File] outputs for this function.
*/
def doneOutputs = statusPaths.map(path => new File(path + ".done"))
def doneOutputs: Seq[File] = statusPrefixes.map(path => new File(path + ".done"))
/**
* Returns the output files for this function.
* @return Set[File] outputs for this function.
* @return outputs for this function.
*/
def failOutputs = statusPaths.map(path => new File(path + ".fail"))
def failOutputs: Seq[File] = statusPrefixes.map(path => new File(path + ".fail"))
/** The complete list of fields on this CommandLineFunction. */
def functionFields = QFunction.classFields(this.functionFieldClass).functionFields
@ -195,21 +193,21 @@ trait QFunction extends Logging with QJobReport {
/**
* Returns the input files for this function.
* @return Set[File] inputs for this function.
* @return inputs for this function.
*/
def inputs = getFieldFiles(inputFields)
def inputs: Seq[File] = getFieldFiles(inputFields)
/**
* Returns the output files for this function.
* @return Set[File] outputs for this function.
* @return outputs for this function.
*/
def outputs = getFieldFiles(outputFields)
def outputs: Seq[File] = getFieldFiles(outputFields)
/**
* Returns the non-log outputs for this function.
* @return the non-log outputs for this function.
* Returns the first output file.
* @return first output for this function.
*/
def commandOutputs = outputs.filterNot(file => isLogFile(file))
def firstOutput: File = outputs.headOption.getOrElse(null)
/**
* Returns the set of directories where files may be written.
@ -218,6 +216,9 @@ trait QFunction extends Logging with QJobReport {
var dirs = Set.empty[File]
dirs += commandDirectory
dirs += jobTempDir
dirs += jobOutputFile.getParentFile
if (jobErrorFile != null)
dirs += jobErrorFile.getParentFile
dirs ++= outputs.map(_.getParentFile)
dirs
}
@ -235,7 +236,7 @@ trait QFunction extends Logging with QJobReport {
* Deletes the output files and all the status files for this function.
*/
def deleteOutputs() {
commandOutputs.foreach(file => IOUtils.tryDelete(file))
outputs.foreach(file => IOUtils.tryDelete(file))
doneOutputs.foreach(file => IOUtils.tryDelete(file))
failOutputs.foreach(file => IOUtils.tryDelete(file))
}
@ -252,63 +253,63 @@ trait QFunction extends Logging with QJobReport {
/**
* Returns fields that do not have values which are required.
* @return List[String] names of fields missing values.
* @return Seq[String] names of fields missing values.
*/
def missingFields: List[String] = {
def missingFields: Seq[String] = {
val missingInputs = missingFields(inputFields, classOf[Input])
val missingOutputs = missingFields(outputFields, classOf[Output])
val missingArguments = missingFields(argumentFields, classOf[Argument])
(missingInputs | missingOutputs | missingArguments).toList.sorted
(missingInputs ++ missingOutputs ++ missingArguments).distinct.sorted
}
/**
* Returns fields that do not have values which are required.
* @param sources Fields to check.
* @param annotation Annotation.
* @return Set[String] names of fields missing values.
* @return names of fields missing values.
*/
private def missingFields(sources: List[ArgumentSource], annotation: Class[_ <: Annotation]): Set[String] = {
var missing = Set.empty[String]
private def missingFields(sources: Seq[ArgumentSource], annotation: Class[_ <: Annotation]): Seq[String] = {
var missing: Seq[String] = Nil
for (source <- sources) {
if (isRequired(source, annotation))
if (!hasFieldValue(source))
if (!exclusiveOf(source, annotation).exists(otherSource => hasFieldValue(otherSource)))
missing += "@%s: %s - %s".format(annotation.getSimpleName, source.field.getName, doc(source, annotation))
missing :+= "@%s: %s - %s".format(annotation.getSimpleName, source.field.getName, doc(source, annotation))
}
missing
}
/**
* Gets the files from the fields. The fields must be a File, a FileExtension, or a List or Set of either.
* Gets the files from the fields. The fields must be a File, a FileExtension, or a Seq or Set of either.
* @param fields Fields to get files.
* @return Set[File] for the fields.
* @return for the fields.
*/
private def getFieldFiles(fields: List[ArgumentSource]): Set[File] = {
var files = Set.empty[File]
private def getFieldFiles(fields: Seq[ArgumentSource]): Seq[File] = {
var files: Seq[File] = Nil
for (field <- fields)
files ++= getFieldFiles(field)
files
files.distinct
}
/**
* Gets the files from the field. The field must be a File, a FileExtension, or a List or Set of either.
* @param fields Field to get files.
* @return Set[File] for the field.
* Gets the files from the field. The field must be a File, a FileExtension, or a Seq or Set of either.
* @param field Field to get files.
* @return for the field.
*/
def getFieldFiles(field: ArgumentSource): Set[File] = {
var files = Set.empty[File]
def getFieldFiles(field: ArgumentSource): Seq[File] = {
var files: Seq[File] = Nil
CollectionUtils.foreach(getFieldValue(field), (fieldValue) => {
val file = fieldValueToFile(field, fieldValue)
if (file != null)
files += file
files :+= file
})
files
files.distinct
}
/**
* Gets the file from the field. The field must be a File or a FileExtension and not a List or Set.
* Gets the file from the field. The field must be a File or a FileExtension and not a Seq or Set.
* @param field Field to get the file.
* @return File for the field.
* @return for the field.
*/
def getFieldFile(field: ArgumentSource): File =
fieldValueToFile(field, getFieldValue(field))
@ -340,14 +341,15 @@ trait QFunction extends Logging with QJobReport {
* Sets all field values.
*/
def freezeFieldValues() {
if (jobNamePrefix == null)
jobNamePrefix = qSettings.jobNamePrefix
if (jobName == null)
jobName = QFunction.nextJobName(jobNamePrefix)
jobName = qSettings.runName + "-" + this.addOrder.mkString("-")
if (jobOutputFile == null)
jobOutputFile = new File(jobName + ".out")
if (jobOutputFile == null) {
jobOutputFile = firstOutput match {
case file: File => new File(file.getParentFile, file.getName + ".out")
case _ => new File(jobName + ".out")
}
}
if (jobTempDir == null)
jobTempDir = qSettings.tempDirectory
@ -378,6 +380,10 @@ trait QFunction extends Logging with QJobReport {
fieldValue = CollectionUtils.updated(fieldValue, canon).asInstanceOf[AnyRef]
this.setFieldValue(field, fieldValue)
}
this.jobOutputFile = canon(this.jobOutputFile).asInstanceOf[File]
if (this.jobErrorFile != null)
this.jobErrorFile = canon(this.jobErrorFile).asInstanceOf[File]
}
/**
@ -443,7 +449,7 @@ trait QFunction extends Logging with QJobReport {
/**
* Returns false if the value is null or an empty collection.
* @param value Value to test for null, or a collection to test if it is empty.
* @param param Value to test for null, or a collection to test if it is empty.
* @return false if the value is null, or false if the collection is empty, otherwise true.
*/
protected def hasValue(param: Any) = CollectionUtils.isNotNullOrNotEmpty(param)
@ -472,28 +478,15 @@ trait QFunction extends Logging with QJobReport {
}
object QFunction {
/** Job index counter for this run of Queue. */
private var jobIndex = 0
var parsingEngine: ParsingEngine = _
/**
* Returns the next job name using the prefix.
* @param prefix Prefix of the job name.
* @return the next job name.
*/
private def nextJobName(prefix: String) = {
jobIndex += 1
prefix + "-" + jobIndex
}
/**
* The list of fields defined on a class
* @param clazz The class to lookup fields.
*/
private class ClassFields(clazz: Class[_]) {
/** The complete list of fields on this CommandLineFunction. */
val functionFields: List[ArgumentSource] = parsingEngine.extractArgumentSources(clazz).toList
val functionFields: Seq[ArgumentSource] = parsingEngine.extractArgumentSources(clazz).toSeq
/** The @Input fields on this CommandLineFunction. */
val inputFields = functionFields.filter(source => ReflectionUtils.hasAnnotation(source.field, classOf[Input]))
/** The @Output fields on this CommandLineFunction. */

32
public/scala/src/org/broadinstitute/sting/queue/function/scattergather/CloneFunction.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -25,7 +25,6 @@
package org.broadinstitute.sting.queue.function.scattergather
import org.broadinstitute.sting.commandline.ArgumentSource
import java.io.File
import org.broadinstitute.sting.queue.function.{QFunction, CommandLineFunction}
/**
@ -62,9 +61,8 @@ class CloneFunction extends CommandLineFunction {
}
}
override def commandOutputs = withScatterPart(() => originalFunction.commandOutputs)
override def dotString = withScatterPart(() => originalFunction.dotString)
override def description = withScatterPart(() => originalFunction.description)
override def shortDescription = withScatterPart(() => originalFunction.shortDescription)
override protected def functionFieldClass = originalFunction.getClass
def commandLine = withScatterPart(() => originalFunction.commandLine)
@ -75,30 +73,22 @@ class CloneFunction extends CommandLineFunction {
}
override def getFieldValue(source: ArgumentSource): AnyRef = {
source.field.getName match {
case "jobOutputFile" => jobOutputFile
case "jobErrorFile" => jobErrorFile
case _ => overriddenFields.get(source) match {
case Some(value) => value.asInstanceOf[AnyRef]
case None => {
val value = originalFunction.getFieldValue(source)
overriddenFields += source -> value
value
}
overriddenFields.get(source) match {
case Some(value) => value.asInstanceOf[AnyRef]
case None => {
val value = originalFunction.getFieldValue(source)
overriddenFields += source -> value
value
}
}
}
def setFieldValue(field: String, value: Any): Unit = {
def setFieldValue(field: String, value: Any) {
val source = QFunction.findField(originalFunction.getClass, field)
setFieldValue(source, value)
}
override def setFieldValue(source: ArgumentSource, value: Any): Unit = {
source.field.getName match {
case "jobOutputFile" => jobOutputFile = value.asInstanceOf[File]
case "jobErrorFile" => jobErrorFile = value.asInstanceOf[File]
case _ => overriddenFields += source -> value
}
override def setFieldValue(source: ArgumentSource, value: Any) {
overriddenFields += source -> value
}
}

33
public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/AutoIndexGatherFunction.scala → public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ConcatenateLogsFunction.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -22,15 +22,34 @@
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.extensions.gatk
package org.broadinstitute.sting.queue.function.scattergather
import org.broadinstitute.sting.queue.function.scattergather.GatherFunction
import org.broadinstitute.sting.queue.function.InProcessFunction
import org.broadinstitute.sting.queue.QException
import org.broadinstitute.sting.commandline.Input
import org.apache.commons.io.FileUtils
import java.io.File
import collection.JavaConversions._
/**
* A no-op for index files that were automatically generated during the gather step.
* TODO: Allow graph to know that this isn't needed, and/or that one gather job can actually gather N-outputs, and/or look more into generic source->sinks.
* Concatenate log files to the jobOutputFile.
*/
class AutoIndexGatherFunction extends InProcessFunction with GatherFunction {
def run() {}
class ConcatenateLogsFunction extends InProcessFunction {
analysisName = "Concat"
@Input(doc="Parts to gather back into the original output")
var logs: Seq[File] = Nil
override def description = "%s: %s > %s".format(analysisName, logs, jobOutputFile)
override def shortDescription = analysisName + ": " + jobOutputFile.getName
def run() {
val missing = org.broadinstitute.sting.utils.io.IOUtils.waitFor(logs, 120)
if (!missing.isEmpty)
throw new QException("Unable to find log: " + missing.mkString(", "))
logs.foreach(log => {
FileUtils.copyFile(log, this.jobOutputStream)
this.jobOutputStream.println()
})
}
}

41
public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GatherFunction.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.function.scattergather
import java.io.File
@ -11,22 +35,31 @@ import collection.JavaConversions._
* Base class for Gather command line functions.
*/
trait GatherFunction extends QFunction {
analysisName = "Gather"
var originalFunction: ScatterGatherableFunction = _
@Output(doc="The original output of the scattered function")
var originalOutput: File = _
@Input(doc="Parts to gather back into the original output")
var gatherParts: List[File] = Nil
@Input(doc="Other log files that will be gathered before this output", required=false)
var originalLogFiles: List[File] = Nil
var gatherParts: Seq[File] = Nil
/**
* Called to initialize the gather function values after all other values have been setup for this function.
*/
def init() {}
/**
* Don't include this @Gather's log file when tracking .done.
* The done files for original log file being produced will do.
*
* The logs from the scatter/gather jobs are concatenated together into the original log.
* Without removing the logs a .done file would be created for the logs. If a SGF is switched
* from scatterCount=1 to >1 then this Gather would be "missing" its logs and re-run.
*/
override protected def statusPaths = outputs
/**
* Waits for gather parts to propagate over NFS or throws an exception.
*/

7
public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GathererFunction.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -32,11 +32,12 @@ import collection.JavaConversions._
* Runs a Gatherer in process.
*/
class GathererFunction(gathererClass: Class[_ <: Gatherer]) extends InProcessFunction with GatherFunction {
analysisName = this.gathererClass.getSimpleName
def run() {
val gatherer = gathererClass.newInstance
if (gatherer.waitForInputs)
waitForGatherParts
waitForGatherParts()
gatherer.gather(this.gatherParts, this.originalOutput)
}
override def description = this.gathererClass.getSimpleName + " " + this.commandOutputs.mkString(" ")
}

5
public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterFunction.scala
View File

@ -32,14 +32,17 @@ import org.broadinstitute.sting.queue.function.QFunction
* Base class for Scatter functions.
*/
trait ScatterFunction extends QFunction {
analysisName = "Scatter"
var originalFunction: ScatterGatherableFunction = _
@Input(doc="Original inputs to scatter")
var originalInputs: Set[File] = _
override def shortDescription = analysisName + ": %s ...".format(firstOutput.getName)
/**
* Called to initialize scatter function values after all other values have been setup for this function.
* @param originalFunction The original function to with inputs bind to this scatter function.
*/
def init() {}

148
public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -30,6 +30,7 @@ import org.broadinstitute.sting.commandline.{Gatherer, Gather, ArgumentSource}
import org.broadinstitute.sting.queue.function.{QFunction, CommandLineFunction}
import org.broadinstitute.sting.queue.QException
import org.broadinstitute.sting.utils.io.IOUtils
import collection.immutable.ListMap
/**
* A function that can be run faster by splitting it up into pieces and then joining together the results.
@ -47,28 +48,28 @@ trait ScatterGatherableFunction extends CommandLineFunction {
/**
* Function that returns the class to use for gathering a directory. If it returns null then @Gather annotation will be used.
* @param gatherField Field that is to be gathered.
* PartialFunction param gatherField Field that is to be gathered.
* @return The class of the GatherFunction to be used or null.
*/
var gatherClass: PartialFunction[ArgumentSource, Class[_ <: GatherFunction]] = _
/**
* Allows external modification of the ScatterFunction that will create the scatter pieces in the temporary directories.
* @param scatterFunction The function that will create the scatter pieces in the temporary directories.
* PartialFunction param scatterFunction The function that will create the scatter pieces in the temporary directories.
*/
var setupScatterFunction: PartialFunction[ScatterFunction, Unit] = _
/**
* Allows external modification of the GatherFunction that will collect the gather pieces in the temporary directories.
* @param gatherFunction The function that will merge the gather pieces from the temporary directories.
* @param gatherField The output field being gathered.
* PartialFunction param gatherFunction The function that will merge the gather pieces from the temporary directories.
* PartialFunction param gatherField The output field being gathered.
*/
var setupGatherFunction: PartialFunction[(GatherFunction, ArgumentSource), Unit] = _
/**
* Allows external modification of the cloned function.
* @param cloneFunction A clone wrapper of this ScatterGatherableFunction
* @param index The one based index (from 1..scatterCount inclusive) of the scatter piece.
* PartialFunction param cloneFunction A clone wrapper of this ScatterGatherableFunction
* PartialFunction param index The one based index (from 1..scatterCount inclusive) of the scatter piece.
*/
var setupCloneFunction: PartialFunction[(CloneFunction, Int), Unit] = _
@ -108,8 +109,9 @@ trait ScatterGatherableFunction extends CommandLineFunction {
scatterFunction.originalFunction = this
scatterFunction.originalInputs = inputFiles
scatterFunction.commandDirectory = this.scatterGatherTempDir("scatter")
scatterFunction.isIntermediate = true
scatterFunction.jobOutputFile = new File("scatter.out")
scatterFunction.addOrder = this.addOrder :+ 1
scatterFunction.isIntermediate = true
initScatterFunction(scatterFunction)
scatterFunction.absoluteCommandDirectory()
@ -121,69 +123,61 @@ trait ScatterGatherableFunction extends CommandLineFunction {
* Returns a list of scatter / gather and clones of this function
* that can be run in parallel to produce the same output as this
* command line function.
* @return List[QFunction] to run instead of this function.
* @return Seq[QFunction] to run instead of this function.
*/
def generateFunctions() = {
var functions = List.empty[QFunction]
// Only gather up fields that will have a value
val outputFieldsWithValues = this.outputFields.filter(hasFieldValue(_))
// Create the scatter function based on @Scatter
functions :+= scatterFunction
// Ask the scatter function how many clones to create.
val numClones = scatterFunction.scatterCount
// List of the log files that are output by this function.
var logFiles = List(this.jobOutputFile)
if (this.jobErrorFile != null)
logFiles :+= this.jobErrorFile
// Create the gather functions for each output field
var gatherFunctions = Map.empty[ArgumentSource, GatherFunction]
var gatherOutputs = Map.empty[ArgumentSource, File]
var gatherFunctions = ListMap.empty[ArgumentSource, GatherFunction]
var gatherOutputs = ListMap.empty[ArgumentSource, File]
var gatherAddOrder = numClones + 2
// Only track fields that will have a value
val outputFieldsWithValues = this.outputFields.filter(hasFieldValue(_))
for (gatherField <- outputFieldsWithValues) {
val gatherOutput = getFieldFile(gatherField)
gatherOutputs += gatherField -> getFieldFile(gatherField)
}
// Only gather fields that are @Gather(enabled=true)
val outputFieldsWithGathers = outputFieldsWithValues.filter(hasGatherFunction(_))
for (gatherField <- outputFieldsWithGathers) {
val gatherOutput = gatherOutputs(gatherField)
val gatherFunction = this.newGatherFunction(gatherField)
this.copySettingsTo(gatherFunction)
gatherFunction.originalFunction = this
gatherFunction.originalOutput = gatherOutput
gatherFunction.commandDirectory = this.scatterGatherTempDir("gather-" + gatherField.field.getName)
// If this is a gather for a log file, make the gather intermediate just in case the log file name changes
// Otherwise have the regular output function wait on the log files to gather
if (isLogFile(gatherOutput)) {
gatherFunction.isIntermediate = true
// Only delete the log files if the original function is an intermediate
// and the intermediate files are supposed to be deleted
gatherFunction.deleteIntermediateOutputs = this.isIntermediate && this.deleteIntermediateOutputs
} else {
gatherFunction.originalLogFiles = logFiles
}
gatherFunction.jobOutputFile = new File("gather-" + gatherOutput.getName + ".out")
gatherFunction.addOrder = this.addOrder :+ gatherAddOrder
initGatherFunction(gatherFunction, gatherField)
gatherFunction.absoluteCommandDirectory()
gatherFunction.init()
functions :+= gatherFunction
gatherFunctions += gatherField -> gatherFunction
gatherOutputs += gatherField -> gatherOutput
gatherAddOrder += 1
}
// Create the clone functions for running the parallel jobs
var cloneFunctions = List.empty[CloneFunction]
var cloneFunctions = Seq.empty[CloneFunction]
val dirFormat = "temp_%%0%dd_of_%d".format(numClones.toString.length(), numClones)
for (i <- 1 to numClones) {
val cloneFunction = this.newCloneFunction()
this.copySettingsTo(cloneFunction)
cloneFunction.originalFunction = this
cloneFunction.analysisName = this.analysisName
cloneFunction.cloneIndex = i
cloneFunction.commandDirectory = this.scatterGatherTempDir("temp-"+i)
cloneFunction.commandDirectory = this.scatterGatherTempDir(dirFormat.format(i))
cloneFunction.jobOutputFile = new File(this.jobOutputFile.getName)
if (this.jobErrorFile != null)
cloneFunction.jobErrorFile = new File(this.jobErrorFile.getName)
cloneFunction.addOrder = this.addOrder :+ (i+1)
cloneFunction.isIntermediate = true
@ -200,17 +194,39 @@ trait ScatterGatherableFunction extends CommandLineFunction {
// If the command directory is relative, insert the run directory ahead of it.
cloneFunction.absoluteCommandDirectory()
// Get absolute paths to the files and bind the sg functions to the clone function via the absolute paths.
// Allow the scatter function to set the specific input for this clone
scatterFunction.bindCloneInputs(cloneFunction, i)
// Set each of the clone outputs to be absolute paths.
for (gatherField <- outputFieldsWithValues) {
val gatherPart = IOUtils.absolute(cloneFunction.commandDirectory, cloneFunction.getFieldFile(gatherField))
cloneFunction.setFieldValue(gatherField, gatherPart)
gatherFunctions(gatherField).gatherParts :+= gatherPart
}
// For the outputs that are being gathered add this clone's output to be gathered.
for (gatherField <- outputFieldsWithGathers) {
gatherFunctions(gatherField).gatherParts :+= cloneFunction.getFieldFile(gatherField)
}
cloneFunctions :+= cloneFunction
}
functions ++= cloneFunctions
// Track the functions starting with the scatter function.
var functions: Seq[QFunction] = Seq(scatterFunction) ++ cloneFunctions ++ gatherFunctions.values
// Make all log file paths absolute.
for (function <- functions) {
function.jobOutputFile = IOUtils.absolute(function.commandDirectory, function.jobOutputFile)
if (function.jobErrorFile != null)
function.jobErrorFile = IOUtils.absolute(function.commandDirectory, function.jobErrorFile)
}
val jobOutputGather = gatherLogFile(_.jobOutputFile, functions, gatherAddOrder)
if (this.jobErrorFile != null) {
val jobErrorGather = gatherLogFile(_.jobErrorFile, functions, gatherAddOrder + 1)
functions :+= jobErrorGather
}
functions :+= jobOutputGather
// Return all the various created functions.
functions
@ -237,6 +253,25 @@ trait ScatterGatherableFunction extends CommandLineFunction {
this.setupScatterFunction(scatterFunction)
}
/**
* Returns true if the field should be gathered.
* @param gatherField Field that defined @Gather.
* @return true if the field should be gathered.
*/
protected def hasGatherFunction(gatherField: ArgumentSource) : Boolean = {
// Check if there is a function that will return the gather class for this field.
if (this.gatherClass != null && this.gatherClass.isDefinedAt(gatherField))
true
// Check for an annotation defining the gather class.
else if (ReflectionUtils.hasAnnotation(gatherField.field, classOf[Gather]))
ReflectionUtils.getAnnotation(gatherField.field, classOf[Gather]).enabled
// Nothing else to disable this field.
else
true
}
/**
* Creates a new GatherFunction for the gatherField.
* @param gatherField Field that defined @Gather.
@ -255,16 +290,18 @@ trait ScatterGatherableFunction extends CommandLineFunction {
if (ReflectionUtils.hasAnnotation(gatherField.field, classOf[Gather])) {
gatherClass = ReflectionUtils.getAnnotation(gatherField.field, classOf[Gather]).value
} else {
throw new QException("Missing @Gather annotation: " + gatherField.field)
throw new QException("Missing @Gather annotation on %s".format(gatherField.field))
}
}
if (classOf[GatherFunction].isAssignableFrom(gatherClass)) {
if (gatherClass == classOf[GatherFunction]) {
throw new QException("@Gather did not specify class type on %s".format(gatherField.field))
} else if (classOf[GatherFunction].isAssignableFrom(gatherClass)) {
gatherClass.newInstance.asInstanceOf[GatherFunction]
} else if (classOf[Gatherer].isAssignableFrom(gatherClass)) {
new GathererFunction(gatherClass.asSubclass(classOf[Gatherer]))
} else {
throw new QException("Unsupported @Gather class type: " + gatherClass)
throw new QException("Unsupported @Gather class type on %s: %s".format(gatherField.field, gatherClass))
}
}
@ -298,10 +335,27 @@ trait ScatterGatherableFunction extends CommandLineFunction {
this.setupCloneFunction(cloneFunction, index)
}
/**
* Gathers up the logs files from other functions.
* @param logFile Takes the QFunction and return the log file.
* @param functions The functions for which the logs will be concatenated.
* @param addOrder The order this function should be added in the graph.
*/
private def gatherLogFile(logFile: (QFunction) => File, functions: Seq[QFunction], addOrder: Int) = {
val gatherLogFunction = new ConcatenateLogsFunction
this.copySettingsTo(gatherLogFunction)
gatherLogFunction.logs = functions.map(logFile).filter(_ != null)
gatherLogFunction.jobOutputFile = logFile(this)
gatherLogFunction.commandDirectory = this.scatterGatherTempDir()
gatherLogFunction.addOrder = this.addOrder :+ addOrder
gatherLogFunction.isIntermediate = false
gatherLogFunction
}
/**
* Returns a temporary directory under this scatter gather directory.
* @param Sub directory under the scatter gather directory.
* @param subDir directory under the scatter gather directory.
* @return temporary directory under this scatter gather directory.
*/
private def scatterGatherTempDir(subDir: String) = IOUtils.absolute(this.scatterGatherDirectory, this.jobName + "-sg/" + subDir)
private def scatterGatherTempDir(subDir: String = "") = IOUtils.absolute(this.scatterGatherDirectory, this.jobName + "-sg/" + subDir)
}

14
public/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -34,12 +34,12 @@ import scala.collection.JavaConversions._
*/
class EmailMessage extends Logging {
var from: String = _
var to: List[String] = Nil
var cc: List[String] = Nil
var bcc: List[String] = Nil
var to: Seq[String] = Nil
var cc: Seq[String] = Nil
var bcc: Seq[String] = Nil
var subject: String = _
var body: String = _
var attachments: List[File] = Nil
var attachments: Seq[File] = Nil
/**
* Sends the email and throws an exception if the email can't be sent.
@ -111,10 +111,10 @@ class EmailMessage extends Logging {
/**
* Converts the email addresses to a collection of InternetAddress which can bypass client side validation,
* specifically that the domain is specified.
* @param addresses List of email addresses.
* @param addresses Seq of email addresses.
* @return java.util.List of InternetAddress'es
*/
private def convert(addresses: List[String]): java.util.List[InternetAddress] = {
private def convert(addresses: Seq[String]): java.util.List[InternetAddress] = {
addresses.map(address => new InternetAddress(address, false))
}

22
public/scala/src/org/broadinstitute/sting/queue/util/QJobReport.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -28,9 +28,10 @@ import org.broadinstitute.sting.queue.function.QFunction
import org.broadinstitute.sting.gatk.report.{GATKReportTable, GATKReport}
import org.broadinstitute.sting.utils.exceptions.UserException
import org.broadinstitute.sting.queue.engine.JobRunInfo
import java.io.{FileOutputStream, PrintStream, File}
import java.io.{PrintStream, File}
import org.broadinstitute.sting.utils.R.{RScriptLibrary, RScriptExecutor}
import org.broadinstitute.sting.utils.io.Resource
import org.apache.commons.io.{IOUtils, FileUtils}
/**
* A mixin to add Job info to the class
@ -67,7 +68,7 @@ trait QJobReport extends Logging {
def getReportGroup = self.analysisName.replaceAll(GATKReportTable.INVALID_TABLE_NAME_REGEX, "_")
def getReportFeatures = reportFeatures
def getReportFeatureNames: List[String] = getReportFeatures.keys.toList
def getReportFeatureNames: Seq[String] = getReportFeatures.keys.toSeq
def getReportFeature(key: String): String = {
getReportFeatures.get(key) match {
case Some(x) => x
@ -102,9 +103,12 @@ object QJobReport {
def printReport(jobsRaw: Map[QFunction, JobRunInfo], dest: File) {
val jobs = jobsRaw.filter(_._2.isFilledIn).filter(_._1.includeInReport)
jobs foreach {case (qf, info) => qf.setRunInfo(info)}
val stream = new PrintStream(new FileOutputStream(dest))
printJobLogging(jobs.keys.toList, stream)
stream.close()
val stream = new PrintStream(FileUtils.openOutputStream(dest))
try {
printJobLogging(jobs.keys.toSeq, stream)
} finally {
IOUtils.closeQuietly(stream)
}
}
def plotReport(reportFile: File, pdfFile: File) {
@ -129,7 +133,7 @@ object QJobReport {
* Prints the JobLogging logs to a GATKReport. First splits up the
* logs by group, and for each group generates a GATKReportTable
*/
private def printJobLogging(logs: List[QFunction], stream: PrintStream) {
private def printJobLogging(logs: Seq[QFunction], stream: PrintStream) {
// create the report
val report: GATKReport = new GATKReport
@ -151,11 +155,11 @@ object QJobReport {
report.print(stream)
}
private def groupLogs(logs: List[QFunction]): Map[String, List[QFunction]] = {
private def groupLogs(logs: Seq[QFunction]): Map[String, Seq[QFunction]] = {
logs.groupBy(_.getReportGroup)
}
private def logKeys(logs: List[QFunction]): Set[String] = {
private def logKeys(logs: Seq[QFunction]): Set[String] = {
// the keys should be the same for each log, but we will check that
val keys = Set[String](logs(0).getReportFeatureNames : _*)

41
public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.util
import java.io.File
@ -12,23 +36,22 @@ import collection.JavaConversions._
* User: carneiro
* Date: 7/14/11
* Time: 4:57 PM
* To change this template use File | Settings | File Templates.
*/
object QScriptUtils {
/**
* Takes a bam list file and produces a scala list with each file allowing the bam list
* Takes a bam list file and produces a scala sequence with each file allowing the bam list
* to have empty lines and comment lines (lines starting with #).
*/
def createListFromFile(in: File):List[File] = {
def createSeqFromFile(in: File):Seq[File] = {
// If the file provided ends with .bam, .fasta or .fq, it is not a bam list, we treat it as a single file.
// and return a list with only this file.
if (in.toString.endsWith(".bam") || in.toString.endsWith(".fasta") || in.toString.endsWith(".fq"))
return List(in)
return Seq(in)
var list: List[File] = List()
for (file <- fromFile(in).getLines)
var list: Seq[File] = Seq()
for (file <- fromFile(in).getLines())
if (!file.startsWith("#") && !file.isEmpty )
list :+= new File(file.trim())
list.sortWith(_.compareTo(_) < 0)
@ -55,8 +78,4 @@ object QScriptUtils {
}
false
}
def ?[A <: AnyRef](ref: A): Option[A] =
if (ref eq null) None else Some(ref)
}
}

32
public/scala/src/org/broadinstitute/sting/queue/util/ReflectionUtils.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.util
import org.broadinstitute.sting.queue.QException
@ -64,17 +88,17 @@ object ReflectionUtils {
/**
* Returns all the declared fields on a class in order of sub type to super type.
* @param clazz Base class to start looking for fields.
* @return List[Field] found on the class and all super classes.
* @return Seq[Field] found on the class and all super classes.
*/
def getAllFields(clazz: Class[_]) = getAllTypes(clazz).map(_.getDeclaredFields).flatMap(_.toList)
def getAllFields(clazz: Class[_]) = getAllTypes(clazz).map(_.getDeclaredFields).flatMap(_.toSeq)
/**
* Gets all the types on a class in order of sub type to super type.
* @param clazz Base class.
* @return List[Class] including the class and all super classes.
* @return Seq[Class] including the class and all super classes.
*/
def getAllTypes(clazz: Class[_]) = {
var types = List.empty[Class[_]]
var types = Seq.empty[Class[_]]
var c = clazz
while (c != null) {
types :+= c

46
public/scala/src/org/broadinstitute/sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.util
import collection.JavaConversions._
@ -14,32 +38,34 @@ class ScalaCompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor {
/**
* Checks if the class type is a scala collection.
* @param classType Class type to check.
* @return true if the class is a List, Set, or an Option.
* @return true if the class is a Seq, Set, or an Option.
*/
def supports(classType: Class[_]) = isCompound(classType)
/**
* Checks if the class type is a scala collection.
* @param source Argument source to check.
* @return true if the source is a List, Set, or an Option.
* @return true if the source is a Seq, Set, or an Option.
*/
override def isMultiValued(source: ArgumentSource) = isCompound(source.field.getType)
/**
* Checks if the class type is a scala collection.
* @param classType Class type to check.
* @return true if the class is a List, Set, or an Option.
* @return true if the class is a Seq, Set, or an Option.
*/
private def isCompound(classType: Class[_]) = {
classOf[List[_]].isAssignableFrom(classType) ||
classOf[Seq[_]].isAssignableFrom(classType) ||
classOf[List[_]].isAssignableFrom(classType) || // see comment below re: List vs. Seq
classOf[Set[_]].isAssignableFrom(classType) ||
classOf[Option[_]].isAssignableFrom(classType)
}
/**
* Parses the argument matches based on the class type of the argument source's field.
* @param parsingEngine Parsing engine.
* @param source Argument source that contains the field being populated.
* @param classType Class type being parsed.
* @param typeType Type of the argument source's field.
* @param argumentMatches The argument match strings that were found for this argument source.
* @return The parsed object.
*/
@ -51,7 +77,15 @@ class ScalaCompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor {
val componentType = ReflectionUtils.getCollectionType(source.field)
val componentArgumentParser = parsingEngine.selectBestTypeDescriptor(componentType)
if (classOf[List[_]].isAssignableFrom(classType)) {
if (classOf[Seq[_]].isAssignableFrom(classType)) {
var seq = Seq.empty[Any]
for (argumentMatch <- argumentMatches)
for (value <- argumentMatch)
seq :+= componentArgumentParser.parse(parsingEngine, source, componentType, new ArgumentMatches(value))
seq
} else if (classOf[List[_]].isAssignableFrom(classType)) {
// QScripts should be using the interface Seq instead of the class List.
// Leaving this here for now for legacy support until the effects of switching have been tested for a while. -ks
var list = List.empty[Any]
for (argumentMatch <- argumentMatches)
for (value <- argumentMatch)

34
public/scala/src/org/broadinstitute/sting/queue/util/StringFileConversions.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -44,7 +44,7 @@ object StringFileConversions {
// and mixins all correct so this doesn't have to be duplicated with concrete implementations?
// http://programming-scala.labs.oreilly.com/ch12.html is your friend.
implicit def stringsAsFiles(x: List[Comparable[_ >: String with File <: Comparable[_ >: String with File <: Serializable] with Serializable] with Serializable]): List[File] = {
implicit def stringsAsFiles(x: Seq[Comparable[_ >: String with File <: Comparable[_ >: String with File <: Serializable] with Serializable] with Serializable]): Seq[File] = {
x.map(_ match {
case string: String => stringAsFile(string)
case file: File => file
@ -52,7 +52,23 @@ object StringFileConversions {
})
}
implicit def filesAsStrings(x: List[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable]): List[String] = {
implicit def filesAsStrings(x: Seq[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable]): Seq[String] = {
x.map(_ match {
case file: File => fileAsString(file)
case string: String => string
case null => null
})
}
implicit def stringsAsFilesList(x: List[Comparable[_ >: String with File <: Comparable[_ >: String with File <: Serializable] with Serializable] with Serializable]): List[File] = {
x.map(_ match {
case string: String => stringAsFile(string)
case file: File => file
case null => null
})
}
implicit def filesAsStringsList(x: List[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable]): List[String] = {
x.map(_ match {
case file: File => fileAsString(file)
case string: String => string
@ -91,14 +107,22 @@ trait StringFileConversions {
StringFileConversions.fileAsString(x)
}
implicit def stringsAsFiles(x: List[Comparable[_ >: String with File <: Comparable[_ >: String with File <: Serializable] with Serializable] with Serializable]): List[File] = {
implicit def stringsAsFiles(x: Seq[Comparable[_ >: String with File <: Comparable[_ >: String with File <: Serializable] with Serializable] with Serializable]): Seq[File] = {
StringFileConversions.stringsAsFiles(x)
}
implicit def filesAsStrings(x: List[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable]): List[String] = {
implicit def filesAsStrings(x: Seq[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable]): Seq[String] = {
StringFileConversions.filesAsStrings(x)
}
implicit def stringsAsFilesList(x: List[Comparable[_ >: String with File <: Comparable[_ >: String with File <: Serializable] with Serializable] with Serializable]): List[File] = {
StringFileConversions.stringsAsFilesList(x)
}
implicit def filesAsStringsList(x: List[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable]): List[String] = {
StringFileConversions.filesAsStringsList(x)
}
implicit def stringsAsFiles(x: Set[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable] with Serializable]): Set[File] = {
StringFileConversions.stringsAsFiles(x)
}

5
public/scala/src/org/broadinstitute/sting/queue/util/SystemUtils.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -24,7 +24,6 @@
package org.broadinstitute.sting.queue.util
import java.lang.management.ManagementFactory
import java.net.InetAddress
import java.io.File
import io.Source
@ -56,6 +55,4 @@ object SystemUtils extends Logging {
else
hostName.split('.').takeRight(2).mkString(".")
}
val pidAtHost = ManagementFactory.getRuntimeMXBean.getName.split('.').head
}

23
public/scala/test/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervalsUnitTest.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -40,9 +40,6 @@ class GATKIntervalsUnitTest {
createSetFromSequenceDictionary(new ReferenceDataSource(hg18Reference).getReference.getSequenceDictionary).toList
private final lazy val hg19Reference = new File(BaseTest.hg19Reference)
private final lazy val hg19GenomeLocParser = new GenomeLocParser(new CachingIndexedFastaSequenceFile(hg19Reference))
private final lazy val hg19ReferenceLocs = GenomeLocSortedSet.
createSetFromSequenceDictionary(new ReferenceDataSource(hg19Reference).getReference.getSequenceDictionary).toList
@Test
def testWithIntervals() {
@ -50,16 +47,14 @@ class GATKIntervalsUnitTest {
val chr2 = hg18GenomeLocParser.parseGenomeLoc("chr2:2-3")
val chr3 = hg18GenomeLocParser.parseGenomeLoc("chr3:3-5")
val gi = new GATKIntervals(hg18Reference, List("chr1:1-1", "chr2:2-3", "chr3:3-5"))
Assert.assertEquals(gi.locs.toList, List(chr1, chr2, chr3))
Assert.assertEquals(gi.contigs, List("chr1", "chr2", "chr3"))
// Assert.assertEquals(gi.getSplits(2).toList, List(2, 3))
// Assert.assertEquals(gi.getSplits(3).toList, List(1, 2, 3))
val gi = new GATKIntervals(hg18Reference, Seq("chr1:1-1", "chr2:2-3", "chr3:3-5"))
Assert.assertEquals(gi.locs.toSeq, Seq(chr1, chr2, chr3))
Assert.assertEquals(gi.contigs, Seq("chr1", "chr2", "chr3"))
}
@Test(timeOut = 30000)
@Test(timeOut = 30000L)
def testIntervalFile() {
var gi = new GATKIntervals(hg19Reference, List(BaseTest.hg19Intervals))
var gi = new GATKIntervals(hg19Reference, Seq(BaseTest.hg19Intervals))
Assert.assertEquals(gi.locs.size, 189894)
// Timeout check is because of bad:
// for(Item item: javaConvertedScalaList)
@ -74,14 +69,12 @@ class GATKIntervalsUnitTest {
val gi = new GATKIntervals(hg18Reference, Nil)
Assert.assertEquals(gi.locs, hg18ReferenceLocs)
Assert.assertEquals(gi.contigs.size, hg18ReferenceLocs.size)
// Assert.assertEquals(gi.getSplits(2).toList, List(10, 45))
// Assert.assertEquals(gi.getSplits(4).toList, List(5, 10, 16, 45))
}
@Test
def testContigCounts() {
Assert.assertEquals(new GATKIntervals(hg18Reference, Nil).contigs, hg18ReferenceLocs.map(_.getContig))
Assert.assertEquals(new GATKIntervals(hg18Reference, List("chr1", "chr2", "chr3")).contigs, List("chr1", "chr2", "chr3"))
Assert.assertEquals(new GATKIntervals(hg18Reference, List("chr1:1-2", "chr1:4-5", "chr2:1-1", "chr3:2-2")).contigs, List("chr1", "chr2", "chr3"))
Assert.assertEquals(new GATKIntervals(hg18Reference, Seq("chr1", "chr2", "chr3")).contigs, Seq("chr1", "chr2", "chr3"))
Assert.assertEquals(new GATKIntervals(hg18Reference, Seq("chr1:1-2", "chr1:4-5", "chr2:1-1", "chr3:2-2")).contigs, Seq("chr1", "chr2", "chr3"))
}
}

58
public/scala/test/org/broadinstitute/sting/queue/function/CommandLineFunctionUnitTest.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.function
import org.testng.Assert
@ -114,20 +138,20 @@ class CommandLineFunctionUnitTest extends CommandLineFunction {
@DataProvider( name = "repeatTestData" )
def repeatDataProvider = {
Array(Array("", List("a", "bc", "d"), "", " ", true, true, " 'a' 'bc' 'd' "),
Array("", List("a", "bc", "d"), "", " ", true, false, " a bc d "),
Array("", List("a", "bc", "d"), "", "", true, true, " 'a''bc''d' "),
Array("", List("a", "bc", "d"), "", "", true, false, " abcd "),
Array("-f", List("file1", "file2", "file3"), "", " ", true, true, " '-f' 'file1' '-f' 'file2' '-f' 'file3' "),
Array("-f", List("file1", "file2", "file3"), "", " ", true, false, " -f file1 -f file2 -f file3 "),
Array("-f", List("file1", "file2", "file3"), "", " ", false, true, " '-ffile1' '-ffile2' '-ffile3' "),
Array("-f", List("file1", "file2", "file3"), "", " ", false, false, " -ffile1 -ffile2 -ffile3 "),
Array("-f", List("file1", "file2", "file3"), "", "", false, true, " '-ffile1''-ffile2''-ffile3' "),
Array("-f", List("file1", "file2", "file3"), "", "", false, false, " -ffile1-ffile2-ffile3 "),
Array("-f", List("file1", "file2", "file3"), "suffix", " ", true, true, " '-f' 'file1' 'suffix' '-f' 'file2' 'suffix' '-f' 'file3' 'suffix' "),
Array("-f", List("file1", "file2", "file3"), "suffix", " ", true, false, " -f file1 suffix -f file2 suffix -f file3 suffix "),
Array("-f", List("file1", "file2", "file3"), "suffix", " ", false, true, " '-ffile1suffix' '-ffile2suffix' '-ffile3suffix' "),
Array("-f", List("file1", "file2", "file3"), "suffix", " ", false, false, " -ffile1suffix -ffile2suffix -ffile3suffix "),
Array(Array("", Seq("a", "bc", "d"), "", " ", true, true, " 'a' 'bc' 'd' "),
Array("", Seq("a", "bc", "d"), "", " ", true, false, " a bc d "),
Array("", Seq("a", "bc", "d"), "", "", true, true, " 'a''bc''d' "),
Array("", Seq("a", "bc", "d"), "", "", true, false, " abcd "),
Array("-f", Seq("file1", "file2", "file3"), "", " ", true, true, " '-f' 'file1' '-f' 'file2' '-f' 'file3' "),
Array("-f", Seq("file1", "file2", "file3"), "", " ", true, false, " -f file1 -f file2 -f file3 "),
Array("-f", Seq("file1", "file2", "file3"), "", " ", false, true, " '-ffile1' '-ffile2' '-ffile3' "),
Array("-f", Seq("file1", "file2", "file3"), "", " ", false, false, " -ffile1 -ffile2 -ffile3 "),
Array("-f", Seq("file1", "file2", "file3"), "", "", false, true, " '-ffile1''-ffile2''-ffile3' "),
Array("-f", Seq("file1", "file2", "file3"), "", "", false, false, " -ffile1-ffile2-ffile3 "),
Array("-f", Seq("file1", "file2", "file3"), "suffix", " ", true, true, " '-f' 'file1' 'suffix' '-f' 'file2' 'suffix' '-f' 'file3' 'suffix' "),
Array("-f", Seq("file1", "file2", "file3"), "suffix", " ", true, false, " -f file1 suffix -f file2 suffix -f file3 suffix "),
Array("-f", Seq("file1", "file2", "file3"), "suffix", " ", false, true, " '-ffile1suffix' '-ffile2suffix' '-ffile3suffix' "),
Array("-f", Seq("file1", "file2", "file3"), "suffix", " ", false, false, " -ffile1suffix -ffile2suffix -ffile3suffix "),
Array("-f", null, "", " ", true, true, ""),
Array("-f", Nil, "", " ", true, true, "")
)
@ -148,11 +172,11 @@ class CommandLineFunctionUnitTest extends CommandLineFunction {
@DataProvider( name = "repeatWithPrefixFormattingTestData" )
def repeatWithPrefixFormattingDataProvider = {
Array(Array("-f", List("file1", "file2", "file3"), "", " ", true, true, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value),
Array(Array("-f", Seq("file1", "file2", "file3"), "", " ", true, true, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value),
" '-f:tagfile1' 'file1' '-f:tagfile2' 'file2' '-f:tagfile3' 'file3' "),
Array("-f", List("file1", "file2", "file3"), "", " ", true, false, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value),
Array("-f", Seq("file1", "file2", "file3"), "", " ", true, false, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value),
" -f:tagfile1 file1 -f:tagfile2 file2 -f:tagfile3 file3 "),
Array("", List("file1", "file2", "file3"), "", " ", true, true, (prefix: String, value: Any) => "-%s".format(value),
Array("", Seq("file1", "file2", "file3"), "", " ", true, true, (prefix: String, value: Any) => "-%s".format(value),
" '-file1' 'file1' '-file2' 'file2' '-file3' 'file3' "),
Array("-f", null, "", " ", true, true, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value),
""),

6
public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -47,10 +47,10 @@ object PipelineTest extends BaseTest with Logging {
final val allJobRunners = {
val commandLinePluginManager = new CommandLinePluginManager
commandLinePluginManager.getPlugins.map(commandLinePluginManager.getName(_)).toList
commandLinePluginManager.getPlugins.map(commandLinePluginManager.getName(_)).toSeq
}
final val defaultJobRunners = List("Lsf706", "GridEngine")
final val defaultJobRunners = Seq("Lsf706", "GridEngine")
/**
* Returns the top level output path to this test.

28
public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTestEvalSpec.scala
View File

@ -1,14 +1,38 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.pipeline
/**
* Data validations to evaluate on a GATKReport.
*/
class PipelineTestEvalSpec {
/** List of eval modules to output. */
/** Eval modules to output. */
var evalReport: String = _
/** Validations to assert. */
var validations: List[PipelineValidation[_]] = Nil
var validations: Seq[PipelineValidation[_]] = Nil
}
/** A VariantEval JEXL and range of values to validate. */

26
public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTestSpec.scala
View File

@ -1,3 +1,27 @@
/*
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.queue.pipeline
class PipelineTestSpec(var name: String = null) {
@ -9,7 +33,7 @@ class PipelineTestSpec(var name: String = null) {
var jobQueue: String = _
/** Job runners to run the test. Default is null which means use the default. */
var jobRunners: List[String] = _
var jobRunners: Seq[String] = _
/** Expected MD5 results for each file path. */
var fileMD5s = Map.empty[String, String]

20
public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/HelloWorldPipelineTest.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -38,11 +38,11 @@ class HelloWorldPipelineTest {
}
@Test
def testHelloWorldWithPrefix() {
def testHelloWorldWithRunName() {
val spec = new PipelineTestSpec
spec.name = "HelloWorldWithPrefix"
spec.name = "HelloWorldWithRunName"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" +
" -jobPrefix HelloWorld"
" -runName HelloWorld"
spec.jobRunners = PipelineTest.allJobRunners
PipelineTest.executeTest(spec)
}
@ -73,7 +73,7 @@ class HelloWorldPipelineTest {
spec.name = "HelloWorldWithLsfResource"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" +
" -jobResReq rusage[iodine_io=1] -jobResReq select[swp>0] -jobResReq order[swp]"
spec.jobRunners = List("Lsf706")
spec.jobRunners = Seq("Lsf706")
PipelineTest.executeTest(spec)
}
@ -83,7 +83,7 @@ class HelloWorldPipelineTest {
spec.name = "HelloWorldWithLsfResourceAndMemoryLimit"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" +
" -memLimit 1.25 -jobResReq rusage[iodine_io=1] -jobResReq select[swp>0] -jobResReq order[swp]"
spec.jobRunners = List("Lsf706")
spec.jobRunners = Seq("Lsf706")
PipelineTest.executeTest(spec)
}
@ -93,7 +93,7 @@ class HelloWorldPipelineTest {
spec.name = "HelloWorldWithLsfEnvironment"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" +
" -jobEnv tv"
spec.jobRunners = List("Lsf706")
spec.jobRunners = Seq("Lsf706")
PipelineTest.executeTest(spec)
}
@ -103,7 +103,7 @@ class HelloWorldPipelineTest {
spec.name = "HelloWorldWithGridEngineResource"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" +
" -jobResReq s_core=1000M"
spec.jobRunners = List("GridEngine")
spec.jobRunners = Seq("GridEngine")
PipelineTest.executeTest(spec)
}
@ -113,7 +113,7 @@ class HelloWorldPipelineTest {
spec.name = "HelloWorldWithGridEngineResourceAndMemoryLimit"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" +
" -memLimit 1.25 -jobResReq s_core=1000M"
spec.jobRunners = List("GridEngine")
spec.jobRunners = Seq("GridEngine")
PipelineTest.executeTest(spec)
}
@ -123,7 +123,7 @@ class HelloWorldPipelineTest {
spec.name = "HelloWorldWithGridEngineEnvironment"
spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" +
" -jobEnv \"make 1\""
spec.jobRunners = List("GridEngine")
spec.jobRunners = Seq("GridEngine")
PipelineTest.executeTest(spec)
}
}

82
public/scala/test/org/broadinstitute/sting/queue/util/StringFileConversionsUnitTest.scala
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, The Broad Institute
* Copyright (c) 2012, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
@ -50,40 +50,40 @@ class StringFileConversionsUnitTest {
@Test
def testStringToFileList() {
var files = List(new File("foo"))
var files = Seq(new File("foo"))
files :+= "bar"
Assert.assertEquals(files, List(new File("foo"), new File("bar")))
Assert.assertEquals(files, Seq(new File("foo"), new File("bar")))
files = List(new File("foo"))
files = Seq(new File("foo"))
files :+= null.asInstanceOf[String]
Assert.assertEquals(files, List(new File("foo"), null))
Assert.assertEquals(files, Seq(new File("foo"), null))
files = List[File](null)
files = Seq[File](null)
files :+= "foo"
Assert.assertEquals(files, List(null, new File("foo")))
Assert.assertEquals(files, Seq(null, new File("foo")))
files = List[File](null)
files = Seq[File](null)
files :+= null.asInstanceOf[String]
Assert.assertEquals(files, List(null, null))
Assert.assertEquals(files, Seq(null, null))
}
@Test
def testFileToStringList() {
var strings = List("foo")
var strings = Seq("foo")
strings :+= new File("bar")
Assert.assertEquals(strings, List("foo", "bar"))
Assert.assertEquals(strings, Seq("foo", "bar"))
strings = List("foo")
strings = Seq("foo")
strings :+= null.asInstanceOf[File]
Assert.assertEquals(strings, List("foo", null))
Assert.assertEquals(strings, Seq("foo", null))
strings = List[String](null)
strings = Seq[String](null)
strings :+= new File("foo")
Assert.assertEquals(strings, List(null, "foo"))
Assert.assertEquals(strings, Seq(null, "foo"))
strings = List[String](null)
strings = Seq[String](null)
strings :+= null.asInstanceOf[File]
Assert.assertEquals(strings, List(null, null))
Assert.assertEquals(strings, Seq(null, null))
}
@Test
@ -126,40 +126,40 @@ class StringFileConversionsUnitTest {
@Test
def testStringListToFileList() {
var files = List(new File("foo"))
files ++= List("bar")
Assert.assertEquals(files, List(new File("foo"), new File("bar")))
var files = Seq(new File("foo"))
files ++= Seq("bar")
Assert.assertEquals(files, Seq(new File("foo"), new File("bar")))
files = List(new File("foo"))
files ++= List[String](null)
Assert.assertEquals(files, List(new File("foo"), null))
files = Seq(new File("foo"))
files ++= Seq[String](null)
Assert.assertEquals(files, Seq(new File("foo"), null))
files = List[File](null)
files ++= List("foo")
Assert.assertEquals(files, List(null, new File("foo")))
files = Seq[File](null)
files ++= Seq("foo")
Assert.assertEquals(files, Seq(null, new File("foo")))
files = List[File](null)
files ++= List[String](null)
Assert.assertEquals(files, List(null, null))
files = Seq[File](null)
files ++= Seq[String](null)
Assert.assertEquals(files, Seq(null, null))
}
@Test
def testFileListToStringList() {
var strings = List("foo")
strings ++= List(new File("bar"))
Assert.assertEquals(strings, List("foo", "bar"))
var strings = Seq("foo")
strings ++= Seq(new File("bar"))
Assert.assertEquals(strings, Seq("foo", "bar"))
strings = List("foo")
strings ++= List[File](null)
Assert.assertEquals(strings, List("foo", null))
strings = Seq("foo")
strings ++= Seq[File](null)
Assert.assertEquals(strings, Seq("foo", null))
strings = List[String](null)
strings ++= List(new File("foo"))
Assert.assertEquals(strings, List(null, "foo"))
strings = Seq[String](null)
strings ++= Seq(new File("foo"))
Assert.assertEquals(strings, Seq(null, "foo"))
strings = List[String](null)
strings ++= List[File](null)
Assert.assertEquals(strings, List(null, null))
strings = Seq[String](null)
strings ++= Seq[File](null)
Assert.assertEquals(strings, Seq(null, null))
}
@Test