Embedding gsalib source and queueJobReport R scripts in the dist and package jars.
Moved gsalib and queueJobReport.R to embeddable namespaced locations. Updated packager dependencies/dir to add an @includes which filters the embedded fileset. RScriptExecutor can now JIT compiles the gsalib. RScriptExecutor uses ProcessController and sends the Rscript output to java's stdout when run under -l DEBUG. Refactored ProcessController and IOUtils from Queue to Sting Utils. Added more unit tests to ProcessController along with a utility class to hard stop OutputStreams at a specified byte count. Replaced uses of some IOUtils with Apache Commons IO. ShellJobRunner refactored to use direct ProcessController and now kills jobs on shutdown. Better QGraph responsiveness on shutdown by using Object.wait() instead of Thread.sleep().
This commit is contained in:
Khalid Shakir
parent
89a581a66f
commit
fac9932938
138
build.xml
138
build.xml
|
|
@ -28,6 +28,8 @@
|
|||
|
||||
<property name="build.dir" value="build" />
|
||||
<property name="dist.dir" value="dist" />
|
||||
<property name="contract.dump.dir" value="dump" />
|
||||
<property name="pipelinetest.dir" value="pipelinetests" />
|
||||
<property name="lib.dir" value="lib" />
|
||||
<property name="external.dir" value="external" />
|
||||
<property name="public.dir" value="public" />
|
||||
|
|
@ -35,18 +37,25 @@
|
|||
<property name="java.public.source.dir" value="${public.dir}/java/src" />
|
||||
<property name="java.private.source.dir" value="${private.dir}/java/src" />
|
||||
<property name="java.classes" value="${build.dir}/java/classes" />
|
||||
<property name="R.public.scripts.dir" value="${public.dir}/R/scripts" />
|
||||
<property name="R.public.src.dir" value="${public.dir}/R/src" />
|
||||
<!-- Legacy: Installing libraries back into the source directory
|
||||
instead of the build or dist directory... intentionally avoids ant clean?? -->
|
||||
<property name="R.library.dir" value="${public.dir}/R" />
|
||||
<property name="R.tar.dir" value="${build.dir}/R/src" />
|
||||
<property name="R.package.path" value="org/broadinstitute/sting/utils/R" />
|
||||
<property name="resource.file" value="StingText.properties" />
|
||||
<property name="resource.path" value="${java.classes}/StingText.properties" />
|
||||
|
||||
<property name="scala.public.source.dir" value="${public.dir}/scala/src" />
|
||||
<property name="scala.private.source.dir" value="${private.dir}/scala/src" />
|
||||
<property name="scala.private.source.dir" value="${private.dir}/scala/src" />
|
||||
<property name="scala.classes" value="${build.dir}/scala/classes" />
|
||||
|
||||
<property name="queue-extensions.source.dir" value="${build.dir}/queue-extensions/src" />
|
||||
|
||||
<property name="javadoc.dir" value="javadoc" />
|
||||
<property name="scaladoc.dir" value="scaladoc" />
|
||||
|
||||
|
||||
<!-- Contracts for Java -->
|
||||
<!-- By default, enabled only for test targets -->
|
||||
<!-- To disable for test targets, run with -Duse.contracts=false -->
|
||||
|
|
@ -60,7 +69,7 @@
|
|||
|
||||
<!-- do we want to halt on failure of a unit test? default to yes (Bamboo uses 'no') -->
|
||||
<property name="halt" value="yes" />
|
||||
|
||||
|
||||
<!-- should our unit test output go to a file or the screen?
|
||||
false means it goes to the screen (default) true to file -->
|
||||
<property name="usefile" value="false" />
|
||||
|
|
@ -82,7 +91,7 @@
|
|||
<patternset refid="java.source.pattern" />
|
||||
</fileset>
|
||||
|
||||
<!-- terrible hack to get gatkdocs to see all files -->
|
||||
<!-- terrible hack to get gatkdocs to see all files -->
|
||||
<patternset id="all.java.source.pattern">
|
||||
<include name="${java.public.source.dir}/**/*.java" />
|
||||
<include name="${java.private.source.dir}/**/*.java" />
|
||||
|
|
@ -113,7 +122,7 @@
|
|||
<exclude name="testng*.jar" />
|
||||
<exclude name="bcel*.jar" />
|
||||
</patternset>
|
||||
|
||||
|
||||
<path id="external.dependencies">
|
||||
<fileset dir="${lib.dir}">
|
||||
<patternset refid="dependency.mask" />
|
||||
|
|
@ -154,7 +163,7 @@
|
|||
<property name="ivy.jar.file" value="ivy-${ivy.install.version}.jar"/>
|
||||
<property name="ivy.settings.dir" value="settings"/>
|
||||
<property file="${ivy.settings.dir}/ivysettings.properties"/>
|
||||
|
||||
|
||||
<mkdir dir="${lib.dir}"/>
|
||||
<mkdir dir="${ivy.jar.dir}"/>
|
||||
|
||||
|
|
@ -211,11 +220,11 @@
|
|||
<equals arg1="${git.describe.exit.value}" arg2="0" />
|
||||
</condition>
|
||||
</target>
|
||||
|
||||
|
||||
<target name="tagged.build.version" depends="git.describe" if="git.describe.succeeded">
|
||||
<property name="build.version" value="${git.describe.output}" />
|
||||
</target>
|
||||
|
||||
|
||||
<target name="git.rev-parse" depends="git.describe" unless="git.describe.succeeded">
|
||||
<exec executable="git" outputproperty="git.rev-parse.output" resultproperty="git.rev-parse.exit.value" failonerror="false">
|
||||
<arg line="rev-parse HEAD" />
|
||||
|
|
@ -224,11 +233,11 @@
|
|||
<equals arg1="${git.rev-parse.exit.value}" arg2="0" />
|
||||
</condition>
|
||||
</target>
|
||||
|
||||
|
||||
<target name="untagged.build.version" depends="git.rev-parse" if="git.rev-parse.succeeded">
|
||||
<property name="build.version" value="${git.rev-parse.output}" />
|
||||
</target>
|
||||
|
||||
|
||||
<target name="generate.build.version" depends="tagged.build.version, untagged.build.version">
|
||||
<!-- Set build.version to exported if no other value has been set -->
|
||||
<property name="build.version" value="exported" />
|
||||
|
|
@ -266,7 +275,7 @@
|
|||
<echo message="Scala build : ${scala.target}"/>
|
||||
<echo message="source revision : ${build.version}"/>
|
||||
<echo message="build time : ${build.timestamp}" />
|
||||
|
||||
|
||||
<condition property="include.private">
|
||||
<equals arg1="${gatk.target}" arg2="private" casesensitive="false" />
|
||||
</condition>
|
||||
|
|
@ -312,13 +321,13 @@
|
|||
<target name="gatk.compile.public.source" depends="init,resolve">
|
||||
<javac fork="true" srcdir="${java.public.source.dir}" memoryMaximumSize="512m" destdir="${java.classes}" debug="true" debuglevel="lines,vars,source" classpathref="external.dependencies" tempdir="${java.io.tmpdir}">
|
||||
<compilerarg value="-proc:none"/>
|
||||
</javac>
|
||||
</javac>
|
||||
</target>
|
||||
|
||||
<target name="gatk.compile.private.source" depends="gatk.compile.public.source" if="include.private">
|
||||
<javac fork="true" srcdir="${java.private.source.dir}" memoryMaximumSize="512m" destdir="${java.classes}" debug="true" debuglevel="lines,vars,source" classpathref="external.dependencies" tempdir="${java.io.tmpdir}">
|
||||
<compilerarg value="-proc:none"/>
|
||||
</javac>
|
||||
</javac>
|
||||
</target>
|
||||
|
||||
<target name="gatk.compile.external.source" depends="gatk.compile.public.source,gatk.compile.private.source">
|
||||
|
|
@ -327,11 +336,11 @@
|
|||
<property name="dist.dir" value="${external.dist.dir}" />
|
||||
<property name="gatk.classpath" value="${external.gatk.classpath}" />
|
||||
<fileset dir="${external.dir}" includes="*/build.xml" erroronmissingdir="false" />
|
||||
</subant>
|
||||
</subant>
|
||||
</target>
|
||||
|
||||
<target name="gatk.compile.source"
|
||||
depends="gatk.compile.public.source,gatk.compile.private.source,gatk.compile.external.source"
|
||||
<target name="gatk.compile.source"
|
||||
depends="gatk.compile.public.source,gatk.compile.private.source,gatk.compile.external.source"
|
||||
description="compile the GATK source" />
|
||||
|
||||
<target name="gatk.contracts.public" depends="gatk.compile.source" if="include.contracts">
|
||||
|
|
@ -341,9 +350,9 @@
|
|||
<pathelement path="${java.classes}" />
|
||||
</classpath>
|
||||
<compilerarg value="-Acom.google.java.contract.debug"/>
|
||||
<compilerarg value="-Acom.google.java.contract.dump=dump/"/>
|
||||
<compilerarg value="-Acom.google.java.contract.dump=${contract.dump.dir}"/>
|
||||
<compilerarg value="-proc:only"/>
|
||||
</javac>
|
||||
</javac>
|
||||
</target>
|
||||
|
||||
<target name="check.contracts.private" depends="gatk.contracts.public">
|
||||
|
|
@ -362,14 +371,14 @@
|
|||
<pathelement path="${java.classes}" />
|
||||
</classpath>
|
||||
<compilerarg value="-Acom.google.java.contract.debug"/>
|
||||
<compilerarg value="-Acom.google.java.contract.dump=dump/"/>
|
||||
<compilerarg value="-Acom.google.java.contract.dump=${contract.dump.dir}"/>
|
||||
<compilerarg value="-proc:only"/>
|
||||
</javac>
|
||||
</javac>
|
||||
</target>
|
||||
|
||||
<target name="gatk.contracts" depends="gatk.contracts.public,gatk.contracts.private"
|
||||
<target name="gatk.contracts" depends="gatk.contracts.public,gatk.contracts.private"
|
||||
description="create GATK contracts" if="include.contracts" />
|
||||
|
||||
|
||||
<target name="gatk.compile" depends="init,resolve,gatk.compile.source,gatk.contracts" />
|
||||
|
||||
<target name="init.queue-extensions.generate" depends="gatk.compile">
|
||||
|
|
@ -413,9 +422,9 @@
|
|||
<src path="${scala.public.source.dir}" />
|
||||
<src path="${queue-extensions.source.dir}" />
|
||||
<include name="**/*.scala"/>
|
||||
</scalac>
|
||||
</scalac>
|
||||
</target>
|
||||
|
||||
|
||||
<target name="check.scala.private" depends="scala.compile.public">
|
||||
<condition property="include.scala.private">
|
||||
<and>
|
||||
|
|
@ -424,12 +433,12 @@
|
|||
</and>
|
||||
</condition>
|
||||
</target>
|
||||
|
||||
|
||||
<target name="scala.compile.private" depends="check.scala.private" if="include.scala.private">
|
||||
<scalac fork="true" jvmargs="-Xmx512m" destdir="${scala.classes}" classpathref="scala.dependencies" deprecation="yes" unchecked="yes">
|
||||
<src path="${scala.private.source.dir}" />
|
||||
<include name="**/*.scala"/>
|
||||
</scalac>
|
||||
</scalac>
|
||||
</target>
|
||||
|
||||
<target name="scala.compile" depends="scala.compile.public,scala.compile.private" if="scala.include" description="compile Scala" />
|
||||
|
|
@ -532,6 +541,11 @@
|
|||
|
||||
<target name="sting.compile" depends="gatk.compile, scala.compile" />
|
||||
|
||||
<target name="R.public.tar">
|
||||
<mkdir dir="${R.tar.dir}/${R.package.path}" />
|
||||
<tar compression="gzip" basedir="${R.public.src.dir}/${R.package.path}" includes="gsalib/**" destfile="${R.tar.dir}/${R.package.path}/gsalib.tar.gz" />
|
||||
</target>
|
||||
|
||||
<target name="init.jar" depends="sting.compile,extracthelp">
|
||||
<mkdir dir="${dist.dir}"/>
|
||||
<copy todir="${dist.dir}">
|
||||
|
|
@ -539,7 +553,7 @@
|
|||
</copy>
|
||||
</target>
|
||||
|
||||
<target name="sting-utils.jar" depends="gatk.compile, init.jar">
|
||||
<target name="sting-utils.jar" depends="gatk.compile, init.jar, R.public.tar">
|
||||
<jar jarfile="${dist.dir}/StingUtils.jar">
|
||||
<fileset dir="${java.classes}">
|
||||
<include name="**/utils/**/*.class"/>
|
||||
|
|
@ -551,6 +565,12 @@
|
|||
<fileset dir="${java.classes}" includes="**/sting/jna/**/*.class"/>
|
||||
<fileset dir="${java.classes}" includes="net/sf/picard/**/*.class"/>
|
||||
<fileset dir="${java.classes}" includes="net/sf/samtools/**/*.class"/>
|
||||
<fileset dir="${R.tar.dir}">
|
||||
<include name="**/${R.package.path}/**/*.tar.gz"/>
|
||||
</fileset>
|
||||
<fileset dir="${R.public.scripts.dir}">
|
||||
<include name="**/utils/**/*.R"/>
|
||||
</fileset>
|
||||
<manifest>
|
||||
<attribute name="Premain-Class" value="org.broadinstitute.sting.utils.instrumentation.Sizeof" />
|
||||
</manifest>
|
||||
|
|
@ -579,6 +599,10 @@
|
|||
<include name="**/gatk/**/*.class" />
|
||||
<include name="**/alignment/**/*.class"/>
|
||||
</fileset>
|
||||
<fileset dir="${R.public.scripts.dir}">
|
||||
<include name="**/gatk/**/*.R"/>
|
||||
<include name="**/alignment/**/*.R"/>
|
||||
</fileset>
|
||||
<manifest>
|
||||
<attribute name="Main-Class" value="org.broadinstitute.sting.gatk.CommandLineGATK" />
|
||||
</manifest>
|
||||
|
|
@ -593,6 +617,10 @@
|
|||
<include name="**/analyzecovariates/**/*.class" />
|
||||
<include name="**/gatk/walkers/recalibration/*.class" />
|
||||
</fileset>
|
||||
<fileset dir="${R.public.scripts.dir}">
|
||||
<include name="**/analyzecovariates/**/*.R"/>
|
||||
<include name="**/gatk/walkers/recalibration/**/*.R"/>
|
||||
</fileset>
|
||||
<manifest>
|
||||
<attribute name="Main-Class" value="org.broadinstitute.sting.analyzecovariates.AnalyzeCovariates" />
|
||||
</manifest>
|
||||
|
|
@ -605,28 +633,7 @@
|
|||
<fileset dir="${external.dir}" includes="*/build.xml" erroronmissingdir="false" />
|
||||
</subant>
|
||||
</target>
|
||||
<!--
|
||||
<target name="gatk.oneoffs.jar" depends="gatk.compile, init.jar"
|
||||
description="generate the GATK oneoffs distribution" if="include.oneoffs">
|
||||
<jar jarfile="${dist.dir}/CompareBAMAlignments.jar" whenmanifestonly="skip">
|
||||
<fileset dir="${java.classes}">
|
||||
<include name="**/tools/**/*.class" />
|
||||
</fileset>
|
||||
<manifest>
|
||||
<attribute name="Main-Class" value="org.broadinstitute.sting.oneoffprojects.tools.CompareBAMAlignments" />
|
||||
</manifest>
|
||||
</jar>
|
||||
|
||||
<jar jarfile="${dist.dir}/SliceBams.jar" whenmanifestonly="skip">
|
||||
<fileset dir="${java.classes}">
|
||||
<include name="**/tools/**/*.class" />
|
||||
</fileset>
|
||||
<manifest>
|
||||
<attribute name="Main-Class" value="org.broadinstitute.sting.playground.tools.SliceBams" />
|
||||
</manifest>
|
||||
</jar>
|
||||
</target>
|
||||
-->
|
||||
<target name="scala.jar" depends="scala.compile, init.jar" if="scala.include">
|
||||
<jar jarfile="${dist.dir}/GATKScala.jar">
|
||||
<fileset dir="${scala.classes}">
|
||||
|
|
@ -643,6 +650,9 @@
|
|||
<fileset dir="${java.classes}">
|
||||
<include name="org/broadinstitute/sting/queue/**/*.class" />
|
||||
</fileset>
|
||||
<fileset dir="${R.public.scripts.dir}">
|
||||
<include name="org/broadinstitute/sting/queue/**/*.R"/>
|
||||
</fileset>
|
||||
<manifest>
|
||||
<attribute name="Main-Class" value="org.broadinstitute.sting.queue.QCommandLine" />
|
||||
</manifest>
|
||||
|
|
@ -682,20 +692,7 @@
|
|||
</jar>
|
||||
|
||||
</target>
|
||||
<!--
|
||||
<target name="gatk.oneoffs.manifests" depends="gatk.oneoffs.jar, init.manifests" if="include.oneoffs">
|
||||
<jar jarfile="${dist.dir}/CompareBAMAlignments.jar" update="true" whenmanifestonly="skip">
|
||||
<manifest>
|
||||
<attribute name="Class-Path" value="${jar.classpath}" />
|
||||
</manifest>
|
||||
</jar>
|
||||
<jar jarfile="${dist.dir}/SliceBams.jar" update="true" whenmanifestonly="skip">
|
||||
<manifest>
|
||||
<attribute name="Class-Path" value="${jar.classpath}" />
|
||||
</manifest>
|
||||
</jar>
|
||||
</target>
|
||||
-->
|
||||
|
||||
<target name="queue.manifests" depends="queue.jar, init.manifests" if="scala.include">
|
||||
<jar jarfile="${dist.dir}/Queue.jar" update="true" >
|
||||
<manifest>
|
||||
|
|
@ -780,10 +777,6 @@
|
|||
<pathelement location="${testng.jar}"/>
|
||||
</classpath>
|
||||
<compilerarg value="-proc:none"/>
|
||||
<!--
|
||||
<compilerarg value="-Acom.google.java.contract.debug"/>
|
||||
<compilerarg value="-Acom.google.java.contract.dump=dump/"/>
|
||||
-->
|
||||
</javac>
|
||||
</target>
|
||||
|
||||
|
|
@ -800,10 +793,6 @@
|
|||
<pathelement location="${testng.jar}"/>
|
||||
</classpath>
|
||||
<compilerarg value="-proc:none"/>
|
||||
<!--
|
||||
<compilerarg value="-Acom.google.java.contract.debug"/>
|
||||
<compilerarg value="-Acom.google.java.contract.dump=dump/"/>
|
||||
-->
|
||||
</javac>
|
||||
</target>
|
||||
|
||||
|
|
@ -851,6 +840,8 @@
|
|||
<pathelement location="${java.private.test.classes}" />
|
||||
<pathelement location="${scala.public.test.classes}" />
|
||||
<pathelement location="${scala.private.test.classes}" />
|
||||
<pathelement location="${R.tar.dir}" />
|
||||
<pathelement location="${R.public.scripts.dir}" />
|
||||
</path>
|
||||
|
||||
<path id="testng.gatk.releasetest.classpath">
|
||||
|
|
@ -1187,19 +1178,18 @@
|
|||
</target>
|
||||
|
||||
<target name="clean" description="clean up" depends="clean.javadoc,clean.scaladoc,clean.gatkdocs">
|
||||
<delete dir="out"/>
|
||||
<delete dir="${build.dir}"/>
|
||||
<delete dir="${lib.dir}"/>
|
||||
<delete dir="dump"/>
|
||||
<delete dir="${contract.dump.dir}"/>
|
||||
<delete dir="${staging.dir}"/>
|
||||
<delete dir="${dist.dir}"/>
|
||||
<delete dir="pipelinetests"/>
|
||||
<delete dir="${pipelinetest.dir}"/>
|
||||
</target>
|
||||
|
||||
<!-- Build gsalib R module -->
|
||||
<target name="gsalib">
|
||||
<exec executable="R" failonerror="true">
|
||||
<arg line="R CMD INSTALL -l public/R/ public/R/src/gsalib/" />
|
||||
<arg line="R CMD INSTALL -l ${R.library.dir} ${R.public.src.dir}/${R.package.path}/gsalib" />
|
||||
</exec>
|
||||
</target>
|
||||
</project>
|
||||
|
|
|
|||
|
Before Width: | Height: | Size: 49 KiB After Width: | Height: | Size: 49 KiB |
|
|
@ -25,35 +25,35 @@
|
|||
package org.broadinstitute.sting.utils.R;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.broadinstitute.sting.commandline.Advanced;
|
||||
import org.broadinstitute.sting.commandline.Argument;
|
||||
import org.broadinstitute.sting.commandline.ArgumentCollection;
|
||||
import org.broadinstitute.sting.gatk.walkers.recalibration.Covariate;
|
||||
import org.broadinstitute.sting.utils.PathUtils;
|
||||
import org.broadinstitute.sting.utils.Utils;
|
||||
import org.broadinstitute.sting.utils.exceptions.StingException;
|
||||
import org.broadinstitute.sting.utils.exceptions.UserException;
|
||||
import org.broadinstitute.sting.utils.io.IOUtils;
|
||||
import org.broadinstitute.sting.utils.io.Resource;
|
||||
import org.broadinstitute.sting.utils.runtime.ProcessController;
|
||||
import org.broadinstitute.sting.utils.runtime.ProcessSettings;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Generic service for executing RScripts in the GATK directory
|
||||
*
|
||||
* @author Your Name
|
||||
* @since Date created
|
||||
* Generic service for executing RScripts
|
||||
*/
|
||||
public class RScriptExecutor {
|
||||
/**
|
||||
* our log
|
||||
*/
|
||||
protected static Logger logger = Logger.getLogger(RScriptExecutor.class);
|
||||
private static Logger logger = Logger.getLogger(RScriptExecutor.class);
|
||||
|
||||
public static class RScriptArgumentCollection {
|
||||
@Advanced
|
||||
@Argument(fullName = "path_to_Rscript", shortName = "Rscript", doc = "The path to your implementation of Rscript. For Broad users this is maybe /broad/software/free/Linux/redhat_5_x86_64/pkgs/r_2.12.0/bin/Rscript", required = false)
|
||||
@Argument(fullName = "path_to_Rscript", shortName = "Rscript", doc = "The path to your implementation of Rscript. Defaults Rscript meaning to use the first available on the environment PATH. For Broad users should 'use R-2.12' or later.", required = false)
|
||||
public String PATH_TO_RSCRIPT = "Rscript";
|
||||
|
||||
@Advanced
|
||||
|
|
@ -62,40 +62,119 @@ public class RScriptExecutor {
|
|||
|
||||
public RScriptArgumentCollection() {}
|
||||
|
||||
/** For testing and convenience */
|
||||
/* For testing and convenience */
|
||||
public RScriptArgumentCollection(final String PATH_TO_RSCRIPT, final List<String> PATH_TO_RESOURCES) {
|
||||
this.PATH_TO_RSCRIPT = PATH_TO_RSCRIPT;
|
||||
this.PATH_TO_RESOURCES = PATH_TO_RESOURCES;
|
||||
}
|
||||
}
|
||||
|
||||
final RScriptArgumentCollection myArgs;
|
||||
final boolean exceptOnError;
|
||||
private final RScriptArgumentCollection myArgs;
|
||||
private final boolean exceptOnError;
|
||||
private final List<RScriptLibrary> libraries = new ArrayList<RScriptLibrary>();
|
||||
private final List<Resource> scriptResources = new ArrayList<Resource>();
|
||||
private final List<File> scriptFiles = new ArrayList<File>();
|
||||
private final List<String> args = new ArrayList<String>();
|
||||
|
||||
public RScriptExecutor(final RScriptArgumentCollection myArgs, final boolean exceptOnError) {
|
||||
this.myArgs = myArgs;
|
||||
this.exceptOnError = exceptOnError;
|
||||
}
|
||||
|
||||
public void callRScripts(String scriptName, Object... scriptArgs) {
|
||||
callRScripts(scriptName, Arrays.asList(scriptArgs));
|
||||
public void addLibrary(RScriptLibrary library) {
|
||||
this.libraries.add(library);
|
||||
}
|
||||
|
||||
public void callRScripts(String scriptName, List<Object> scriptArgs) {
|
||||
public void addScript(Resource script) {
|
||||
this.scriptResources.add(script);
|
||||
}
|
||||
|
||||
public void addScript(File script) {
|
||||
this.scriptFiles.add(script);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds args to the end of the Rscript command line.
|
||||
* @param args the args.
|
||||
* @throws NullPointerException if any of the args are null.
|
||||
*/
|
||||
public void addArgs(Object... args) {
|
||||
for (Object arg: args)
|
||||
this.args.add(arg.toString());
|
||||
}
|
||||
|
||||
public void exec() {
|
||||
List<File> tempFiles = new ArrayList<File>();
|
||||
try {
|
||||
final File pathToScript = findScript(scriptName);
|
||||
if ( pathToScript == null ) return; // we failed but shouldn't exception out
|
||||
final String argString = Utils.join(" ", scriptArgs);
|
||||
final String cmdLine = Utils.join(" ", Arrays.asList(myArgs.PATH_TO_RSCRIPT, pathToScript, argString));
|
||||
logger.info("Executing RScript: " + cmdLine);
|
||||
Runtime.getRuntime().exec(cmdLine).waitFor();
|
||||
} catch (InterruptedException e) {
|
||||
File tempLibDir = IOUtils.tempDir("R.", ".lib");
|
||||
tempFiles.add(tempLibDir);
|
||||
|
||||
StringBuilder expression = new StringBuilder("tempLibDir = '").append(tempLibDir).append("';");
|
||||
|
||||
if (this.libraries.size() > 0) {
|
||||
List<String> tempLibraryPaths = new ArrayList<String>();
|
||||
for (RScriptLibrary library: this.libraries) {
|
||||
File tempLibrary = library.writeTemp();
|
||||
tempFiles.add(tempLibrary);
|
||||
tempLibraryPaths.add(tempLibrary.getAbsolutePath());
|
||||
}
|
||||
|
||||
expression.append("install.packages(");
|
||||
expression.append("pkgs=c('").append(StringUtils.join(tempLibraryPaths, "', '")).append("'), lib=tempLibDir, repos=NULL, type='source', ");
|
||||
// Install faster by eliminating cruft.
|
||||
expression.append("INSTALL_opts=c('--no-libs', '--no-data', '--no-help', '--no-demo', '--no-exec')");
|
||||
expression.append(");");
|
||||
|
||||
for (RScriptLibrary library: this.libraries) {
|
||||
expression.append("require('").append(library.getLibraryName()).append("', lib.loc=tempLibDir);");
|
||||
}
|
||||
}
|
||||
|
||||
for (Resource script: this.scriptResources) {
|
||||
File tempScript = IOUtils.writeTempResource(script);
|
||||
tempFiles.add(tempScript);
|
||||
expression.append("source('").append(tempScript.getAbsolutePath()).append("');");
|
||||
}
|
||||
|
||||
for (File script: this.scriptFiles) {
|
||||
expression.append("source('").append(script.getAbsolutePath()).append("');");
|
||||
}
|
||||
|
||||
String[] cmd = new String[this.args.size() + 3];
|
||||
int i = 0;
|
||||
cmd[i++] = myArgs.PATH_TO_RSCRIPT;
|
||||
cmd[i++] = "-e";
|
||||
cmd[i++] = expression.toString();
|
||||
for (String arg: this.args)
|
||||
cmd[i++] = arg;
|
||||
|
||||
ProcessSettings processSettings = new ProcessSettings(cmd);
|
||||
if (logger.isDebugEnabled()) {
|
||||
processSettings.getStdoutSettings().printStandard(true);
|
||||
processSettings.getStderrSettings().printStandard(true);
|
||||
}
|
||||
|
||||
ProcessController controller = ProcessController.getThreadLocal();
|
||||
|
||||
logger.debug("Executing: " + Utils.join(" ", cmd));
|
||||
logger.debug("Result: " + controller.exec(processSettings).getExitValue());
|
||||
|
||||
} catch (StingException e) {
|
||||
generateException(e);
|
||||
} catch (IOException e) {
|
||||
generateException("Fatal Exception: Perhaps RScript jobs are being spawned too quickly?", e);
|
||||
} finally {
|
||||
for (File temp: tempFiles)
|
||||
FileUtils.deleteQuietly(temp);
|
||||
}
|
||||
}
|
||||
|
||||
public void callRScripts(String scriptName, Object... scriptArgs) {
|
||||
final File pathToScript = findScript(scriptName);
|
||||
if (pathToScript == null) return; // we failed but shouldn't exception out
|
||||
addScript(pathToScript);
|
||||
addArgs(scriptArgs);
|
||||
exec();
|
||||
}
|
||||
|
||||
public File findScript(final String scriptName) {
|
||||
for ( String pathToResource : myArgs.PATH_TO_RESOURCES ) {
|
||||
final File f = new File(pathToResource + "/" + scriptName);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,59 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.utils.R;
|
||||
|
||||
import org.broadinstitute.sting.utils.io.IOUtils;
|
||||
import org.broadinstitute.sting.utils.io.Resource;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
/**
|
||||
* Libraries embedded in the StingUtils package.
|
||||
*/
|
||||
public enum RScriptLibrary {
|
||||
GSALIB("gsalib");
|
||||
|
||||
private final String name;
|
||||
|
||||
private RScriptLibrary(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getLibraryName() {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
public String getResourcePath() {
|
||||
return name + ".tar.gz";
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the library source code to a temporary tar.gz file and returns the path.
|
||||
* @return The path to the library source code. The caller must delete the code when done.
|
||||
*/
|
||||
public File writeTemp() {
|
||||
return IOUtils.writeTempResource(new Resource(getResourcePath(), RScriptLibrary.class));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.utils.io;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
public interface FileExtension {
|
||||
/**
|
||||
* Returns a clone of the FileExtension with a new path.
|
||||
* @param path New path.
|
||||
* @return New FileExtension
|
||||
*/
|
||||
public File withPath(String path);
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package org.broadinstitute.sting.utils.io;
|
||||
|
||||
import org.apache.commons.io.output.ThresholdingOutputStream;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* An output stream which stops at the threshold
|
||||
* instead of potentially triggering early.
|
||||
*/
|
||||
public abstract class HardThresholdingOutputStream extends ThresholdingOutputStream {
|
||||
protected HardThresholdingOutputStream(int threshold) {
|
||||
super(threshold);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(byte[] b) throws IOException {
|
||||
write(b, 0, b.length);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(byte[] b, int off, int len) throws IOException {
|
||||
int remaining = this.getThreshold() - (int)this.getByteCount();
|
||||
if (!isThresholdExceeded() && len > remaining) {
|
||||
super.write(b, off, remaining);
|
||||
super.write(b, off + remaining, len - remaining);
|
||||
} else {
|
||||
super.write(b, off, len);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,353 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.utils.io;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.io.FilenameUtils;
|
||||
import org.apache.commons.io.LineIterator;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.broadinstitute.sting.utils.exceptions.StingException;
|
||||
import org.broadinstitute.sting.utils.exceptions.UserException;
|
||||
|
||||
import java.io.*;
|
||||
import java.util.*;
|
||||
|
||||
public class IOUtils {
|
||||
private static Logger logger = Logger.getLogger(IOUtils.class);
|
||||
|
||||
/**
|
||||
* Checks if the temp directory has been setup and throws an exception if they user hasn't set it correctly.
|
||||
*
|
||||
* @param tempDir Temporary directory.
|
||||
*/
|
||||
public static void checkTempDir(File tempDir) {
|
||||
String tempDirPath = tempDir.getAbsolutePath();
|
||||
// Keeps the user from leaving the temp directory as the default, and on Macs from having pluses
|
||||
// in the path which can cause problems with the Google Reflections library.
|
||||
// see also: http://benjchristensen.com/2009/09/22/mac-osx-10-6-java-java-io-tmpdir/
|
||||
if (tempDirPath.startsWith("/var/folders/") || (tempDirPath.equals("/tmp")) || (tempDirPath.equals("/tmp/")))
|
||||
throw new UserException.BadTmpDir("java.io.tmpdir must be explicitly set");
|
||||
if (!tempDir.exists() && !tempDir.mkdirs())
|
||||
throw new UserException.BadTmpDir("Could not create directory: " + tempDir.getAbsolutePath());
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a temp directory with the prefix and optional suffix.
|
||||
*
|
||||
* @param prefix Prefix for the directory name.
|
||||
* @param suffix Optional suffix for the directory name.
|
||||
* @return The created temporary directory.
|
||||
*/
|
||||
public static File tempDir(String prefix, String suffix) {
|
||||
return tempDir(prefix, suffix, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a temp directory with the prefix and optional suffix.
|
||||
*
|
||||
* @param prefix Prefix for the directory name.
|
||||
* @param suffix Optional suffix for the directory name.
|
||||
* @param tempDirParent Parent directory for the temp directory.
|
||||
* @return The created temporary directory.
|
||||
*/
|
||||
public static File tempDir(String prefix, String suffix, File tempDirParent) {
|
||||
try {
|
||||
if (tempDirParent == null)
|
||||
tempDirParent = FileUtils.getTempDirectory();
|
||||
if (!tempDirParent.exists() && !tempDirParent.mkdirs())
|
||||
throw new UserException.BadTmpDir("Could not create temp directory: " + tempDirParent);
|
||||
File temp = File.createTempFile(prefix + "-", suffix, tempDirParent);
|
||||
if (!temp.delete())
|
||||
throw new UserException.BadTmpDir("Could not delete sub file: " + temp.getAbsolutePath());
|
||||
if (!temp.mkdir())
|
||||
throw new UserException.BadTmpDir("Could not create sub directory: " + temp.getAbsolutePath());
|
||||
return absolute(temp);
|
||||
} catch (IOException e) {
|
||||
throw new UserException.BadTmpDir(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes content to a temp file and returns the path to the temporary file.
|
||||
*
|
||||
* @param content to write.
|
||||
* @param prefix Prefix for the temp file.
|
||||
* @param suffix Suffix for the temp file.
|
||||
* @param directory Directory for the temp file.
|
||||
* @return the path to the temp file.
|
||||
*/
|
||||
public static File writeTempFile(String content, String prefix, String suffix, File directory) {
|
||||
try {
|
||||
File tempFile = absolute(File.createTempFile(prefix, suffix, directory));
|
||||
FileUtils.writeStringToFile(tempFile, content);
|
||||
return tempFile;
|
||||
} catch (IOException e) {
|
||||
throw new UserException.BadTmpDir(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits for NFS to propagate a file creation, imposing a timeout.
|
||||
*
|
||||
* Based on Apache Commons IO FileUtils.waitFor()
|
||||
*
|
||||
* @param file The file to wait for.
|
||||
* @param seconds The maximum time in seconds to wait.
|
||||
* @return true if the file exists
|
||||
*/
|
||||
public static boolean waitFor(File file, int seconds) {
|
||||
return waitFor(Collections.singletonList(file), seconds).isEmpty();
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits for NFS to propagate a file creation, imposing a timeout.
|
||||
*
|
||||
* Based on Apache Commons IO FileUtils.waitFor()
|
||||
*
|
||||
* @param files The list of files to wait for.
|
||||
* @param seconds The maximum time in seconds to wait.
|
||||
* @return Files that still do not exists at the end of the timeout, or a empty list if all files exists.
|
||||
*/
|
||||
public static List<File> waitFor(Collection<File> files, int seconds) {
|
||||
long timeout = 0;
|
||||
long tick = 0;
|
||||
List<File> missingFiles = new ArrayList<File>();
|
||||
for (File file : files)
|
||||
if (!file.exists())
|
||||
missingFiles.add(file);
|
||||
|
||||
while (!missingFiles.isEmpty() && timeout <= seconds) {
|
||||
if (tick >= 10) {
|
||||
tick = 0;
|
||||
timeout++;
|
||||
}
|
||||
tick++;
|
||||
try {
|
||||
Thread.sleep(100);
|
||||
} catch (InterruptedException ignore) {
|
||||
}
|
||||
List<File> newMissingFiles = new ArrayList<File>();
|
||||
for (File file : missingFiles)
|
||||
if (!file.exists())
|
||||
newMissingFiles.add(file);
|
||||
missingFiles = newMissingFiles;
|
||||
}
|
||||
return missingFiles;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the directory at the number of levels deep.
|
||||
* For example 2 levels of /path/to/dir will return /path/to
|
||||
*
|
||||
* @param dir Directory path.
|
||||
* @param level how many levels deep from the root.
|
||||
* @return The path to the parent directory that is level-levels deep.
|
||||
*/
|
||||
public static File dirLevel(File dir, int level) {
|
||||
List<File> directories = new ArrayList<File>();
|
||||
File parentDir = absolute(dir);
|
||||
while (parentDir != null) {
|
||||
directories.add(0, parentDir);
|
||||
parentDir = parentDir.getParentFile();
|
||||
}
|
||||
if (directories.size() <= level)
|
||||
return directories.get(directories.size() - 1);
|
||||
else
|
||||
return directories.get(level);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the sub path rooted at the parent.
|
||||
*
|
||||
* @param parent The parent directory.
|
||||
* @param path The sub path to append to the parent, if the path is not absolute.
|
||||
* @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path.
|
||||
*/
|
||||
public static File absolute(File parent, String path) {
|
||||
return absolute(parent, new File(path));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the sub path rooted at the parent.
|
||||
*
|
||||
* @param parent The parent directory.
|
||||
* @param file The sub path to append to the parent, if the path is not absolute.
|
||||
* @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path.
|
||||
*/
|
||||
public static File absolute(File parent, File file) {
|
||||
String newPath;
|
||||
if (file.isAbsolute())
|
||||
newPath = absolutePath(file);
|
||||
else
|
||||
newPath = absolutePath(new File(parent, file.getPath()));
|
||||
return replacePath(file, newPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* A mix of getCanonicalFile and getAbsoluteFile that returns the
|
||||
* absolute path to the file without deferencing symbolic links.
|
||||
*
|
||||
* @param file the file.
|
||||
* @return the absolute path to the file.
|
||||
*/
|
||||
public static File absolute(File file) {
|
||||
return replacePath(file, absolutePath(file));
|
||||
}
|
||||
|
||||
private static String absolutePath(File file) {
|
||||
File fileAbs = file.getAbsoluteFile();
|
||||
LinkedList<String> names = new LinkedList<String>();
|
||||
while (fileAbs != null) {
|
||||
String name = fileAbs.getName();
|
||||
fileAbs = fileAbs.getParentFile();
|
||||
|
||||
if (".".equals(name)) {
|
||||
/* skip */
|
||||
|
||||
/* TODO: What do we do for ".."?
|
||||
} else if (name == "..") {
|
||||
|
||||
CentOS tcsh says use getCanonicalFile:
|
||||
~ $ mkdir -p test1/test2
|
||||
~ $ ln -s test1/test2 test3
|
||||
~ $ cd test3/..
|
||||
~/test1 $
|
||||
|
||||
Mac bash says keep going with getAbsoluteFile:
|
||||
~ $ mkdir -p test1/test2
|
||||
~ $ ln -s test1/test2 test3
|
||||
~ $ cd test3/..
|
||||
~ $
|
||||
|
||||
For now, leave it and let the shell figure it out.
|
||||
*/
|
||||
} else {
|
||||
names.add(0, name);
|
||||
}
|
||||
}
|
||||
|
||||
return ("/" + StringUtils.join(names, "/"));
|
||||
}
|
||||
|
||||
private static File replacePath(File file, String path) {
|
||||
if (file instanceof FileExtension)
|
||||
return ((FileExtension)file).withPath(path);
|
||||
if (!File.class.equals(file.getClass()))
|
||||
throw new StingException("Sub classes of java.io.File must also implement FileExtension");
|
||||
return new File(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the last lines of the file.
|
||||
* NOTE: This is only safe to run on smaller files!
|
||||
*
|
||||
* @param file File to read.
|
||||
* @param count Maximum number of lines to return.
|
||||
* @return The last count lines from file.
|
||||
* @throws IOException When unable to read the file.
|
||||
*/
|
||||
public static List<String> tail(File file, int count) throws IOException {
|
||||
LinkedList<String> tailLines = new LinkedList<String>();
|
||||
FileReader reader = new FileReader(file);
|
||||
try {
|
||||
LineIterator iterator = org.apache.commons.io.IOUtils.lineIterator(reader);
|
||||
int lineCount = 0;
|
||||
while (iterator.hasNext()) {
|
||||
String line = iterator.nextLine();
|
||||
lineCount++;
|
||||
if (lineCount > count)
|
||||
tailLines.removeFirst();
|
||||
tailLines.offer(line);
|
||||
}
|
||||
} finally {
|
||||
org.apache.commons.io.IOUtils.closeQuietly(reader);
|
||||
}
|
||||
return tailLines;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tries to delete a file. Emits a warning if the file was unable to be deleted.
|
||||
*
|
||||
* @param file File to delete.
|
||||
* @return true if the file was deleted.
|
||||
*/
|
||||
public static boolean tryDelete(File file) {
|
||||
boolean deleted = FileUtils.deleteQuietly(file);
|
||||
if (deleted)
|
||||
logger.debug("Deleted " + file);
|
||||
else if (file.exists())
|
||||
logger.warn("Unable to delete " + file);
|
||||
return deleted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the an embedded resource to a temp file.
|
||||
* File is not scheduled for deletion and must be cleaned up by the caller.
|
||||
* @param resource Embedded resource.
|
||||
* @return Path to the temp file with the contents of the resource.
|
||||
*/
|
||||
public static File writeTempResource(Resource resource) {
|
||||
File temp;
|
||||
try {
|
||||
temp = File.createTempFile(FilenameUtils.getBaseName(resource.getPath()) + ".", "." + FilenameUtils.getExtension(resource.getPath()));
|
||||
} catch (IOException e) {
|
||||
throw new UserException.BadTmpDir(e.getMessage());
|
||||
}
|
||||
writeResource(resource, temp);
|
||||
return temp;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the an embedded resource to a file.
|
||||
* File is not scheduled for deletion and must be cleaned up by the caller.
|
||||
* @param resource Embedded resource.
|
||||
* @param file File path to write.
|
||||
*/
|
||||
public static void writeResource(Resource resource, File file) {
|
||||
String path = resource.getPath();
|
||||
Class<?> clazz = resource.getRelativeClass();
|
||||
InputStream inputStream = null;
|
||||
OutputStream outputStream = null;
|
||||
try {
|
||||
if (clazz == null) {
|
||||
inputStream = ClassLoader.getSystemResourceAsStream(path);
|
||||
if (inputStream == null)
|
||||
throw new IllegalArgumentException("Resource not found: " + path);
|
||||
} else {
|
||||
inputStream = clazz.getResourceAsStream(path);
|
||||
if (inputStream == null)
|
||||
throw new IllegalArgumentException("Resource not found relative to " + clazz + ": " + path);
|
||||
}
|
||||
outputStream = FileUtils.openOutputStream(file);
|
||||
org.apache.commons.io.IOUtils.copy(inputStream, outputStream);
|
||||
} catch (IOException e) {
|
||||
throw new StingException(String.format("Unable to copy resource '%s' to '%s'", path, file), e);
|
||||
} finally {
|
||||
org.apache.commons.io.IOUtils.closeQuietly(inputStream);
|
||||
org.apache.commons.io.IOUtils.closeQuietly(outputStream);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.utils.io;
|
||||
|
||||
/**
|
||||
* Stores a resource by path and a relative class.
|
||||
*/
|
||||
public class Resource {
|
||||
private final String path;
|
||||
private final Class<?> relativeClass;
|
||||
|
||||
/**
|
||||
* Create a resource with a path and a relative class.
|
||||
* @param path Relative or absolute path to the class.
|
||||
* @param relativeClass Relative class to use as a class loader and for a relative package.
|
||||
*
|
||||
* If the relative class is null then the system classloader will be used and the path must be absolute.
|
||||
*/
|
||||
public Resource(String path, Class<?> relativeClass) {
|
||||
this.path = path;
|
||||
this.relativeClass = relativeClass;
|
||||
}
|
||||
|
||||
public Class<?> getRelativeClass() {
|
||||
return relativeClass;
|
||||
}
|
||||
|
||||
public String getPath() {
|
||||
return path;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,133 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.utils.runtime;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.io.output.NullOutputStream;
|
||||
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
|
||||
import org.broadinstitute.sting.utils.exceptions.UserException;
|
||||
import org.broadinstitute.sting.utils.io.HardThresholdingOutputStream;
|
||||
|
||||
import java.io.*;
|
||||
import java.util.EnumMap;
|
||||
|
||||
/**
|
||||
* Stream output captured from a stream.
|
||||
*/
|
||||
public class CapturedStreamOutput extends StreamOutput {
|
||||
private final InputStream processStream;
|
||||
private final EnumMap<StreamLocation, OutputStream> outputStreams = new EnumMap<StreamLocation, OutputStream>(StreamLocation.class);
|
||||
|
||||
/**
|
||||
* The byte stream to capture content or null if no output string content was requested.
|
||||
*/
|
||||
private final ByteArrayOutputStream bufferStream;
|
||||
|
||||
/**
|
||||
* True if the buffer is truncated.
|
||||
*/
|
||||
private boolean bufferTruncated = false;
|
||||
|
||||
/**
|
||||
* @param settings Settings that define what to capture.
|
||||
* @param processStream Stream to capture output.
|
||||
* @param standardStream Stream to write debug output.
|
||||
*/
|
||||
public CapturedStreamOutput(OutputStreamSettings settings, InputStream processStream, PrintStream standardStream) {
|
||||
this.processStream = processStream;
|
||||
int bufferSize = settings.getBufferSize();
|
||||
this.bufferStream = (bufferSize < 0) ? new ByteArrayOutputStream() : new ByteArrayOutputStream(bufferSize);
|
||||
|
||||
for (StreamLocation location : settings.getStreamLocations()) {
|
||||
OutputStream outputStream;
|
||||
switch (location) {
|
||||
case Buffer:
|
||||
if (bufferSize < 0) {
|
||||
outputStream = this.bufferStream;
|
||||
} else {
|
||||
outputStream = new HardThresholdingOutputStream(bufferSize) {
|
||||
@Override
|
||||
protected OutputStream getStream() throws IOException {
|
||||
return bufferTruncated ? NullOutputStream.NULL_OUTPUT_STREAM : bufferStream;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void thresholdReached() throws IOException {
|
||||
bufferTruncated = true;
|
||||
}
|
||||
};
|
||||
}
|
||||
break;
|
||||
case File:
|
||||
try {
|
||||
outputStream = new FileOutputStream(settings.getOutputFile(), settings.isAppendFile());
|
||||
} catch (IOException e) {
|
||||
throw new UserException.BadInput(e.getMessage());
|
||||
}
|
||||
break;
|
||||
case Standard:
|
||||
outputStream = standardStream;
|
||||
break;
|
||||
default:
|
||||
throw new ReviewedStingException("Unexpected stream location: " + location);
|
||||
}
|
||||
this.outputStreams.put(location, outputStream);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getBufferBytes() {
|
||||
return bufferStream.toByteArray();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isBufferTruncated() {
|
||||
return bufferTruncated;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drain the input stream to keep the process from backing up until it's empty.
|
||||
* File streams will be closed automatically when this method returns.
|
||||
*
|
||||
* @throws java.io.IOException When unable to read or write.
|
||||
*/
|
||||
public void readAndClose() throws IOException {
|
||||
try {
|
||||
byte[] buf = new byte[4096];
|
||||
int readCount;
|
||||
while ((readCount = processStream.read(buf)) >= 0)
|
||||
for (OutputStream outputStream : this.outputStreams.values()) {
|
||||
outputStream.write(buf, 0, readCount);
|
||||
}
|
||||
} finally {
|
||||
for (StreamLocation location : this.outputStreams.keySet()) {
|
||||
OutputStream outputStream = this.outputStreams.get(location);
|
||||
outputStream.flush();
|
||||
if (location != StreamLocation.Standard)
|
||||
IOUtils.closeQuietly(outputStream);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,115 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.utils.runtime;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Collections;
|
||||
import java.util.EnumSet;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Settings that define text to write to the process stdin.
|
||||
*/
|
||||
public class InputStreamSettings {
|
||||
private final EnumSet<StreamLocation> streamLocations = EnumSet.noneOf(StreamLocation.class);
|
||||
private byte[] inputBuffer;
|
||||
private File inputFile;
|
||||
|
||||
public InputStreamSettings() {
|
||||
}
|
||||
|
||||
/**
|
||||
* @param inputBuffer String to write to stdin.
|
||||
*/
|
||||
public InputStreamSettings(String inputBuffer) {
|
||||
setInputBuffer(inputBuffer);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param inputFile File to write to stdin.
|
||||
*/
|
||||
public InputStreamSettings(File inputFile) {
|
||||
setInputFile(inputFile);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param inputBuffer String to write to stdin.
|
||||
* @param inputFile File to write to stdin.
|
||||
*/
|
||||
public InputStreamSettings(byte[] inputBuffer, File inputFile) {
|
||||
setInputBuffer(inputBuffer);
|
||||
setInputFile(inputFile);
|
||||
}
|
||||
|
||||
public Set<StreamLocation> getStreamLocations() {
|
||||
return Collections.unmodifiableSet(streamLocations);
|
||||
}
|
||||
|
||||
public byte[] getInputBuffer() {
|
||||
return inputBuffer;
|
||||
}
|
||||
|
||||
public void setInputBuffer(String inputBuffer) {
|
||||
if (inputBuffer == null)
|
||||
throw new IllegalArgumentException("inputBuffer cannot be null");
|
||||
this.streamLocations.add(StreamLocation.Buffer);
|
||||
this.inputBuffer = inputBuffer.getBytes();
|
||||
}
|
||||
|
||||
public void setInputBuffer(byte[] inputBuffer) {
|
||||
if (inputBuffer == null)
|
||||
throw new IllegalArgumentException("inputBuffer cannot be null");
|
||||
this.streamLocations.add(StreamLocation.Buffer);
|
||||
this.inputBuffer = inputBuffer;
|
||||
}
|
||||
|
||||
public void clearInputBuffer() {
|
||||
this.streamLocations.remove(StreamLocation.Buffer);
|
||||
this.inputBuffer = null;
|
||||
}
|
||||
|
||||
public File getInputFile() {
|
||||
return inputFile;
|
||||
}
|
||||
|
||||
public void setInputFile(File inputFile) {
|
||||
if (inputFile == null)
|
||||
throw new IllegalArgumentException("inputFile cannot be null");
|
||||
this.streamLocations.add(StreamLocation.File);
|
||||
this.inputFile = inputFile;
|
||||
}
|
||||
|
||||
public void clearInputFile() {
|
||||
this.streamLocations.remove(StreamLocation.File);
|
||||
this.inputFile = null;
|
||||
}
|
||||
|
||||
public void setInputStandard(boolean inputStandard) {
|
||||
if (inputStandard)
|
||||
this.streamLocations.add(StreamLocation.Standard);
|
||||
else
|
||||
this.streamLocations.remove(StreamLocation.Standard);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.utils.runtime;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Collections;
|
||||
import java.util.EnumSet;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Settings that define text to capture from a process stream.
|
||||
*/
|
||||
public class OutputStreamSettings {
|
||||
private final EnumSet<StreamLocation> streamLocations = EnumSet.noneOf(StreamLocation.class);
|
||||
private int bufferSize;
|
||||
private File outputFile;
|
||||
private boolean appendFile;
|
||||
|
||||
public OutputStreamSettings() {
|
||||
}
|
||||
|
||||
/**
|
||||
* @param bufferSize The number of bytes to capture, or -1 for unlimited.
|
||||
*/
|
||||
public OutputStreamSettings(int bufferSize) {
|
||||
setBufferSize(bufferSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param outputFile The file to write output to.
|
||||
*/
|
||||
public OutputStreamSettings(File outputFile) {
|
||||
setOutputFile(outputFile);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param outputFile The file to write output to.
|
||||
* @param append true if the output file should be appended to.
|
||||
*/
|
||||
public OutputStreamSettings(File outputFile, boolean append) {
|
||||
setOutputFile(outputFile, append);
|
||||
}
|
||||
|
||||
public OutputStreamSettings(int bufferSize, File outputFile, boolean appendFile) {
|
||||
setBufferSize(bufferSize);
|
||||
setOutputFile(outputFile, appendFile);
|
||||
}
|
||||
|
||||
public Set<StreamLocation> getStreamLocations() {
|
||||
return Collections.unmodifiableSet(streamLocations);
|
||||
}
|
||||
|
||||
public int getBufferSize() {
|
||||
return bufferSize;
|
||||
}
|
||||
|
||||
public void setBufferSize(int bufferSize) {
|
||||
this.streamLocations.add(StreamLocation.Buffer);
|
||||
this.bufferSize = bufferSize;
|
||||
}
|
||||
|
||||
public void clearBufferSize() {
|
||||
this.streamLocations.remove(StreamLocation.Buffer);
|
||||
this.bufferSize = 0;
|
||||
}
|
||||
|
||||
public File getOutputFile() {
|
||||
return outputFile;
|
||||
}
|
||||
|
||||
public boolean isAppendFile() {
|
||||
return appendFile;
|
||||
}
|
||||
|
||||
/**
|
||||
* Overwrites the outputFile with the process output.
|
||||
*
|
||||
* @param outputFile File to overwrite.
|
||||
*/
|
||||
public void setOutputFile(File outputFile) {
|
||||
setOutputFile(outputFile, false);
|
||||
}
|
||||
|
||||
public void setOutputFile(File outputFile, boolean append) {
|
||||
if (outputFile == null)
|
||||
throw new IllegalArgumentException("outputFile cannot be null");
|
||||
streamLocations.add(StreamLocation.File);
|
||||
this.outputFile = outputFile;
|
||||
this.appendFile = append;
|
||||
}
|
||||
|
||||
public void clearOutputFile() {
|
||||
streamLocations.remove(StreamLocation.File);
|
||||
this.outputFile = null;
|
||||
this.appendFile = false;
|
||||
}
|
||||
|
||||
public void printStandard(boolean print) {
|
||||
if (print)
|
||||
this.streamLocations.add(StreamLocation.Standard);
|
||||
else
|
||||
this.streamLocations.remove(StreamLocation.Standard);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,363 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.utils.runtime;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
|
||||
import org.broadinstitute.sting.utils.exceptions.UserException;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* Facade to Runtime.exec() and java.lang.Process. Handles
|
||||
* running a process to completion and returns stdout and stderr
|
||||
* as strings. Creates separate threads for reading stdout and stderr,
|
||||
* then reuses those threads for each process most efficient use is
|
||||
* to create one of these and use it repeatedly. Instances are not
|
||||
* thread-safe, however.
|
||||
*
|
||||
* TODO: java.io sometimes zombies the backround threads locking up on read().
|
||||
* Supposedly NIO has better ways of interrupting a blocked stream but will
|
||||
* require a little bit of refactoring.
|
||||
*
|
||||
* @author Michael Koehrsen
|
||||
* @author Khalid Shakir
|
||||
*/
|
||||
public class ProcessController {
|
||||
private static Logger logger = Logger.getLogger(ProcessController.class);
|
||||
|
||||
private static enum ProcessStream {Stdout, Stderr}
|
||||
|
||||
// Tracks running processes.
|
||||
private static final Set<ProcessController> running = Collections.synchronizedSet(new HashSet<ProcessController>());
|
||||
|
||||
// Tracks this running process.
|
||||
private Process process;
|
||||
|
||||
// Threads that capture stdout and stderr
|
||||
private final OutputCapture stdoutCapture;
|
||||
private final OutputCapture stderrCapture;
|
||||
|
||||
// When a caller destroyes a controller a new thread local version will be created
|
||||
private boolean destroyed = false;
|
||||
|
||||
// Communication channels with output capture threads
|
||||
|
||||
// Holds the stdout and stderr sent to the background capture threads
|
||||
private final Map<ProcessStream, CapturedStreamOutput> toCapture =
|
||||
new EnumMap<ProcessStream, CapturedStreamOutput>(ProcessStream.class);
|
||||
|
||||
// Holds the results of the capture from the background capture threads.
|
||||
// May be the content via toCapture or an StreamOutput.EMPTY if the capture was interrupted.
|
||||
private final Map<ProcessStream, StreamOutput> fromCapture =
|
||||
new EnumMap<ProcessStream, StreamOutput>(ProcessStream.class);
|
||||
|
||||
// Useful for debugging if background threads have shut down correctly
|
||||
private static int nextControllerId = 0;
|
||||
private final int controllerId;
|
||||
|
||||
public ProcessController() {
|
||||
// Start the background threads for this controller.
|
||||
synchronized (running) {
|
||||
controllerId = nextControllerId++;
|
||||
}
|
||||
stdoutCapture = new OutputCapture(ProcessStream.Stdout, controllerId);
|
||||
stderrCapture = new OutputCapture(ProcessStream.Stderr, controllerId);
|
||||
stdoutCapture.start();
|
||||
stderrCapture.start();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a thread local ProcessController.
|
||||
* Should NOT be closed when finished so it can be reused by the thread.
|
||||
*
|
||||
* @return a thread local ProcessController.
|
||||
*/
|
||||
public static ProcessController getThreadLocal() {
|
||||
// If the local controller was destroyed get a fresh instance.
|
||||
if (threadProcessController.get().destroyed)
|
||||
threadProcessController.remove();
|
||||
return threadProcessController.get();
|
||||
}
|
||||
|
||||
/**
|
||||
* Thread local process controller container.
|
||||
*/
|
||||
private static final ThreadLocal<ProcessController> threadProcessController =
|
||||
new ThreadLocal<ProcessController>() {
|
||||
@Override
|
||||
protected ProcessController initialValue() {
|
||||
return new ProcessController();
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Similar to Runtime.exec() but drains the output and error streams.
|
||||
*
|
||||
* @param command Command to run.
|
||||
* @return The result code.
|
||||
*/
|
||||
public static int exec(String[] command) {
|
||||
ProcessController controller = ProcessController.getThreadLocal();
|
||||
return controller.exec(new ProcessSettings(command)).getExitValue();
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes a command line program with the settings and waits for it to return,
|
||||
* processing the output on a background thread.
|
||||
*
|
||||
* @param settings Settings to be run.
|
||||
* @return The output of the command.
|
||||
*/
|
||||
public ProcessOutput exec(ProcessSettings settings) {
|
||||
if (destroyed)
|
||||
throw new IllegalStateException("This controller was destroyed");
|
||||
|
||||
ProcessBuilder builder = new ProcessBuilder(settings.getCommand());
|
||||
builder.directory(settings.getDirectory());
|
||||
|
||||
Map<String, String> settingsEnvironment = settings.getEnvironment();
|
||||
if (settingsEnvironment != null) {
|
||||
Map<String, String> builderEnvironment = builder.environment();
|
||||
builderEnvironment.clear();
|
||||
builderEnvironment.putAll(settingsEnvironment);
|
||||
}
|
||||
|
||||
builder.redirectErrorStream(settings.isRedirectErrorStream());
|
||||
|
||||
StreamOutput stdout = null;
|
||||
StreamOutput stderr = null;
|
||||
|
||||
// Start the process running.
|
||||
|
||||
try {
|
||||
synchronized (toCapture) {
|
||||
process = builder.start();
|
||||
}
|
||||
running.add(this);
|
||||
} catch (IOException e) {
|
||||
throw new ReviewedStingException("Unable to start command: " + StringUtils.join(builder.command(), " "));
|
||||
}
|
||||
|
||||
int exitCode;
|
||||
|
||||
try {
|
||||
// Notify the background threads to start capturing.
|
||||
synchronized (toCapture) {
|
||||
toCapture.put(ProcessStream.Stdout,
|
||||
new CapturedStreamOutput(settings.getStdoutSettings(), process.getInputStream(), System.out));
|
||||
toCapture.put(ProcessStream.Stderr,
|
||||
new CapturedStreamOutput(settings.getStderrSettings(), process.getErrorStream(), System.err));
|
||||
toCapture.notifyAll();
|
||||
}
|
||||
|
||||
// Write stdin content
|
||||
InputStreamSettings stdinSettings = settings.getStdinSettings();
|
||||
Set<StreamLocation> streamLocations = stdinSettings.getStreamLocations();
|
||||
if (!streamLocations.isEmpty()) {
|
||||
try {
|
||||
OutputStream stdinStream = process.getOutputStream();
|
||||
for (StreamLocation location : streamLocations) {
|
||||
InputStream inputStream;
|
||||
switch (location) {
|
||||
case Buffer:
|
||||
inputStream = new ByteArrayInputStream(stdinSettings.getInputBuffer());
|
||||
break;
|
||||
case File:
|
||||
try {
|
||||
inputStream = FileUtils.openInputStream(stdinSettings.getInputFile());
|
||||
} catch (IOException e) {
|
||||
throw new UserException.BadInput(e.getMessage());
|
||||
}
|
||||
break;
|
||||
case Standard:
|
||||
inputStream = System.in;
|
||||
break;
|
||||
default:
|
||||
throw new ReviewedStingException("Unexpected stream location: " + location);
|
||||
}
|
||||
try {
|
||||
IOUtils.copy(inputStream, stdinStream);
|
||||
} finally {
|
||||
if (location != StreamLocation.Standard)
|
||||
IOUtils.closeQuietly(inputStream);
|
||||
}
|
||||
}
|
||||
stdinStream.flush();
|
||||
} catch (IOException e) {
|
||||
throw new ReviewedStingException("Error writing to stdin on command: " + StringUtils.join(builder.command(), " "), e);
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for the process to complete.
|
||||
try {
|
||||
process.getOutputStream().close();
|
||||
process.waitFor();
|
||||
} catch (IOException e) {
|
||||
throw new ReviewedStingException("Unable to close stdin on command: " + StringUtils.join(builder.command(), " "), e);
|
||||
} catch (InterruptedException e) {
|
||||
throw new ReviewedStingException("Process interrupted", e);
|
||||
} finally {
|
||||
while (!destroyed && stdout == null || stderr == null) {
|
||||
synchronized (fromCapture) {
|
||||
if (fromCapture.containsKey(ProcessStream.Stdout))
|
||||
stdout = fromCapture.remove(ProcessStream.Stdout);
|
||||
if (fromCapture.containsKey(ProcessStream.Stderr))
|
||||
stderr = fromCapture.remove(ProcessStream.Stderr);
|
||||
try {
|
||||
if (stdout == null || stderr == null)
|
||||
fromCapture.wait();
|
||||
} catch (InterruptedException e) {
|
||||
// Log the error, ignore the interrupt and wait patiently
|
||||
// for the OutputCaptures to (via finally) return their
|
||||
// stdout and stderr.
|
||||
logger.error(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (destroyed) {
|
||||
if (stdout == null)
|
||||
stdout = StreamOutput.EMPTY;
|
||||
if (stderr == null)
|
||||
stderr = StreamOutput.EMPTY;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
synchronized (toCapture) {
|
||||
exitCode = process.exitValue();
|
||||
process = null;
|
||||
}
|
||||
running.remove(this);
|
||||
}
|
||||
|
||||
return new ProcessOutput(exitCode, stdout, stderr);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The set of still running processes.
|
||||
*/
|
||||
public static Set<ProcessController> getRunning() {
|
||||
synchronized (running) {
|
||||
return new HashSet<ProcessController>(running);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops the process from running and tries to ensure process is cleaned up properly.
|
||||
* NOTE: sub-processes started by process may be zombied with their parents set to pid 1.
|
||||
* NOTE: capture threads may block on read.
|
||||
* TODO: Try to use NIO to interrupt streams.
|
||||
*/
|
||||
public void tryDestroy() {
|
||||
destroyed = true;
|
||||
synchronized (toCapture) {
|
||||
if (process != null) {
|
||||
process.destroy();
|
||||
IOUtils.closeQuietly(process.getInputStream());
|
||||
IOUtils.closeQuietly(process.getErrorStream());
|
||||
}
|
||||
stdoutCapture.interrupt();
|
||||
stderrCapture.interrupt();
|
||||
toCapture.notifyAll();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
try {
|
||||
tryDestroy();
|
||||
} catch (Exception e) {
|
||||
logger.error(e);
|
||||
}
|
||||
super.finalize();
|
||||
}
|
||||
|
||||
private class OutputCapture extends Thread {
|
||||
private final int controllerId;
|
||||
private final ProcessStream key;
|
||||
|
||||
/**
|
||||
* Reads in the output of a stream on a background thread to keep the output pipe from backing up and freezing the called process.
|
||||
*
|
||||
* @param key The stdout or stderr key for this output capture.
|
||||
* @param controllerId Unique id of the controller.
|
||||
*/
|
||||
public OutputCapture(ProcessStream key, int controllerId) {
|
||||
super(String.format("OutputCapture-%d-%s-%s-%d", controllerId, key.name().toLowerCase(),
|
||||
Thread.currentThread().getName(), Thread.currentThread().getId()));
|
||||
this.controllerId = controllerId;
|
||||
this.key = key;
|
||||
setDaemon(true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the capture.
|
||||
*/
|
||||
@Override
|
||||
public void run() {
|
||||
while (!destroyed) {
|
||||
StreamOutput processStream = StreamOutput.EMPTY;
|
||||
try {
|
||||
// Wait for a new input stream to be passed from this process controller.
|
||||
CapturedStreamOutput capturedProcessStream = null;
|
||||
while (!destroyed && capturedProcessStream == null) {
|
||||
synchronized (toCapture) {
|
||||
if (toCapture.containsKey(key)) {
|
||||
capturedProcessStream = toCapture.remove(key);
|
||||
} else {
|
||||
toCapture.wait();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!destroyed) {
|
||||
// Read in the input stream
|
||||
processStream = capturedProcessStream;
|
||||
capturedProcessStream.readAndClose();
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
logger.info("OutputCapture interrupted, exiting");
|
||||
break;
|
||||
} catch (IOException e) {
|
||||
logger.error("Error reading process output", e);
|
||||
} finally {
|
||||
// Send the string back to the process controller.
|
||||
synchronized (fromCapture) {
|
||||
fromCapture.put(key, processStream);
|
||||
fromCapture.notify();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.utils.runtime;
|
||||
|
||||
public class ProcessOutput {
|
||||
private final int exitValue;
|
||||
private final StreamOutput stdout;
|
||||
private final StreamOutput stderr;
|
||||
|
||||
/**
|
||||
* The output of a process.
|
||||
*
|
||||
* @param exitValue The exit value.
|
||||
* @param stdout The capture of stdout as defined by the stdout OutputStreamSettings.
|
||||
* @param stderr The capture of stderr as defined by the stderr OutputStreamSettings.
|
||||
*/
|
||||
public ProcessOutput(int exitValue, StreamOutput stdout, StreamOutput stderr) {
|
||||
this.exitValue = exitValue;
|
||||
this.stdout = stdout;
|
||||
this.stderr = stderr;
|
||||
}
|
||||
|
||||
public int getExitValue() {
|
||||
return exitValue;
|
||||
}
|
||||
|
||||
public StreamOutput getStdout() {
|
||||
return stdout;
|
||||
}
|
||||
|
||||
public StreamOutput getStderr() {
|
||||
return stderr;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,140 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.utils.runtime;
|
||||
|
||||
import com.sun.corba.se.spi.orbutil.fsm.Input;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Map;
|
||||
|
||||
public class ProcessSettings {
|
||||
private String[] command;
|
||||
private Map<String, String> environment;
|
||||
private File directory;
|
||||
private boolean redirectErrorStream;
|
||||
private InputStreamSettings stdinSettings;
|
||||
private OutputStreamSettings stdoutSettings;
|
||||
private OutputStreamSettings stderrSettings;
|
||||
|
||||
/**
|
||||
* @param command Command line to run.
|
||||
*/
|
||||
public ProcessSettings(String[] command) {
|
||||
this(command, false, null, null, null, null, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param command Command line to run.
|
||||
* @param redirectErrorStream true if stderr should be sent to stdout.
|
||||
* @param environment Environment settings to override System.getEnv, or null to use System.getEnv.
|
||||
* @param directory The directory to run the command in, or null to run in the current directory.
|
||||
* @param stdinSettings Settings for writing to the process stdin.
|
||||
* @param stdoutSettings Settings for capturing the process stdout.
|
||||
* @param stderrSettings Setting for capturing the process stderr.
|
||||
*/
|
||||
public ProcessSettings(String[] command, boolean redirectErrorStream, File directory, Map<String, String> environment,
|
||||
InputStreamSettings stdinSettings, OutputStreamSettings stdoutSettings, OutputStreamSettings stderrSettings) {
|
||||
this.command = checkCommand(command);
|
||||
this.redirectErrorStream = redirectErrorStream;
|
||||
this.directory = directory;
|
||||
this.environment = environment;
|
||||
this.stdinSettings = checkSettings(stdinSettings);
|
||||
this.stdoutSettings = checkSettings(stdoutSettings);
|
||||
this.stderrSettings = checkSettings(stderrSettings);
|
||||
}
|
||||
|
||||
public String[] getCommand() {
|
||||
return command;
|
||||
}
|
||||
|
||||
public void setCommand(String[] command) {
|
||||
this.command = checkCommand(command);
|
||||
}
|
||||
|
||||
public boolean isRedirectErrorStream() {
|
||||
return redirectErrorStream;
|
||||
}
|
||||
|
||||
public void setRedirectErrorStream(boolean redirectErrorStream) {
|
||||
this.redirectErrorStream = redirectErrorStream;
|
||||
}
|
||||
|
||||
public File getDirectory() {
|
||||
return directory;
|
||||
}
|
||||
|
||||
public void setDirectory(File directory) {
|
||||
this.directory = directory;
|
||||
}
|
||||
|
||||
public Map<String, String> getEnvironment() {
|
||||
return environment;
|
||||
}
|
||||
|
||||
public void setEnvironment(Map<String, String> environment) {
|
||||
this.environment = environment;
|
||||
}
|
||||
|
||||
public InputStreamSettings getStdinSettings() {
|
||||
return stdinSettings;
|
||||
}
|
||||
|
||||
public void setStdinSettings(InputStreamSettings stdinSettings) {
|
||||
this.stdinSettings = checkSettings(stdinSettings);
|
||||
}
|
||||
|
||||
public OutputStreamSettings getStdoutSettings() {
|
||||
return stdoutSettings;
|
||||
}
|
||||
|
||||
public void setStdoutSettings(OutputStreamSettings stdoutSettings) {
|
||||
this.stdoutSettings = checkSettings(stdoutSettings);
|
||||
}
|
||||
|
||||
public OutputStreamSettings getStderrSettings() {
|
||||
return stderrSettings;
|
||||
}
|
||||
|
||||
public void setStderrSettings(OutputStreamSettings stderrSettings) {
|
||||
this.stderrSettings = checkSettings(stderrSettings);
|
||||
}
|
||||
|
||||
protected String[] checkCommand(String[] command) {
|
||||
if (command == null)
|
||||
throw new IllegalArgumentException("Command is not allowed to be null");
|
||||
for (String s: command)
|
||||
if (s == null)
|
||||
throw new IllegalArgumentException("Command is not allowed to contain nulls");
|
||||
return command;
|
||||
}
|
||||
|
||||
protected InputStreamSettings checkSettings(InputStreamSettings settings) {
|
||||
return settings == null ? new InputStreamSettings() : settings;
|
||||
}
|
||||
|
||||
protected OutputStreamSettings checkSettings(OutputStreamSettings settings) {
|
||||
return settings == null ? new OutputStreamSettings() : settings;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.utils.runtime;
|
||||
|
||||
/**
|
||||
* Where to read/write a stream
|
||||
*/
|
||||
public enum StreamLocation {
|
||||
Buffer, File, Standard
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.utils.runtime;
|
||||
|
||||
/**
|
||||
* The content of stdout or stderr.
|
||||
*/
|
||||
public abstract class StreamOutput {
|
||||
/**
|
||||
* Empty stream output when no output is captured due to an error.
|
||||
*/
|
||||
public static final StreamOutput EMPTY = new StreamOutput() {
|
||||
@Override
|
||||
public byte[] getBufferBytes() {
|
||||
return new byte[0];
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isBufferTruncated() {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns the content as a string.
|
||||
*
|
||||
* @return The content as a string.
|
||||
*/
|
||||
public String getBufferString() {
|
||||
return new String(getBufferBytes());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the content as a string.
|
||||
*
|
||||
* @return The content as a string.
|
||||
*/
|
||||
public abstract byte[] getBufferBytes();
|
||||
|
||||
/**
|
||||
* Returns true if the buffer was truncated.
|
||||
*
|
||||
* @return true if the buffer was truncated.
|
||||
*/
|
||||
public abstract boolean isBufferTruncated();
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.utils.R;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
public class RScriptLibraryUnitTest {
|
||||
@Test
|
||||
public void testProperties() {
|
||||
Assert.assertEquals(RScriptLibrary.GSALIB.getLibraryName(), "gsalib");
|
||||
Assert.assertEquals(RScriptLibrary.GSALIB.getResourcePath(), "gsalib.tar.gz");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWriteTemp() {
|
||||
File file = RScriptLibrary.GSALIB.writeTemp();
|
||||
Assert.assertTrue(file.exists(), "R library was not written to temp file: " + file);
|
||||
FileUtils.deleteQuietly(file);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,197 @@
|
|||
package org.broadinstitute.sting.utils.io;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.broadinstitute.sting.BaseTest;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import org.broadinstitute.sting.utils.exceptions.UserException;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
public class IOUtilsUnitTest extends BaseTest {
|
||||
@Test
|
||||
public void testGoodTempDir() {
|
||||
IOUtils.checkTempDir(new File("/tmp/queue"));
|
||||
}
|
||||
|
||||
@Test(expectedExceptions=UserException.BadTmpDir.class)
|
||||
public void testBadTempDir() {
|
||||
IOUtils.checkTempDir(new File("/tmp"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAbsoluteSubDir() {
|
||||
File subDir = IOUtils.absolute(new File("."), new File("/path/to/file"));
|
||||
Assert.assertEquals(subDir, new File("/path/to/file"));
|
||||
|
||||
subDir = IOUtils.absolute(new File("/different/path"), new File("/path/to/file"));
|
||||
Assert.assertEquals(subDir, new File("/path/to/file"));
|
||||
|
||||
subDir = IOUtils.absolute(new File("/different/path"), new File("."));
|
||||
Assert.assertEquals(subDir, new File("/different/path"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRelativeSubDir() throws IOException {
|
||||
File subDir = IOUtils.absolute(new File("."), new File("path/to/file"));
|
||||
Assert.assertEquals(subDir.getCanonicalFile(), new File("path/to/file").getCanonicalFile());
|
||||
|
||||
subDir = IOUtils.absolute(new File("/different/path"), new File("path/to/file"));
|
||||
Assert.assertEquals(subDir, new File("/different/path/path/to/file"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDottedSubDir() throws IOException {
|
||||
File subDir = IOUtils.absolute(new File("."), new File("path/../to/file"));
|
||||
Assert.assertEquals(subDir.getCanonicalFile(), new File("path/../to/./file").getCanonicalFile());
|
||||
|
||||
subDir = IOUtils.absolute(new File("."), new File("/path/../to/file"));
|
||||
Assert.assertEquals(subDir, new File("/path/../to/file"));
|
||||
|
||||
subDir = IOUtils.absolute(new File("/different/../path"), new File("path/to/file"));
|
||||
Assert.assertEquals(subDir, new File("/different/../path/path/to/file"));
|
||||
|
||||
subDir = IOUtils.absolute(new File("/different/./path"), new File("/path/../to/file"));
|
||||
Assert.assertEquals(subDir, new File("/path/../to/file"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTempDir() {
|
||||
File tempDir = IOUtils.tempDir("Q-Unit-Test", "", new File("queueTempDirToDelete"));
|
||||
Assert.assertTrue(tempDir.exists());
|
||||
Assert.assertFalse(tempDir.isFile());
|
||||
Assert.assertTrue(tempDir.isDirectory());
|
||||
boolean deleted = IOUtils.tryDelete(tempDir);
|
||||
Assert.assertTrue(deleted);
|
||||
Assert.assertFalse(tempDir.exists());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDirLevel() {
|
||||
File dir = IOUtils.dirLevel(new File("/path/to/directory"), 1);
|
||||
Assert.assertEquals(dir, new File("/path"));
|
||||
|
||||
dir = IOUtils.dirLevel(new File("/path/to/directory"), 2);
|
||||
Assert.assertEquals(dir, new File("/path/to"));
|
||||
|
||||
dir = IOUtils.dirLevel(new File("/path/to/directory"), 3);
|
||||
Assert.assertEquals(dir, new File("/path/to/directory"));
|
||||
|
||||
dir = IOUtils.dirLevel(new File("/path/to/directory"), 4);
|
||||
Assert.assertEquals(dir, new File("/path/to/directory"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAbsolute() {
|
||||
File dir = IOUtils.absolute(new File("/path/./to/./directory/."));
|
||||
Assert.assertEquals(dir, new File("/path/to/directory"));
|
||||
|
||||
dir = IOUtils.absolute(new File("/"));
|
||||
Assert.assertEquals(dir, new File("/"));
|
||||
|
||||
dir = IOUtils.absolute(new File("/."));
|
||||
Assert.assertEquals(dir, new File("/"));
|
||||
|
||||
dir = IOUtils.absolute(new File("/././."));
|
||||
Assert.assertEquals(dir, new File("/"));
|
||||
|
||||
dir = IOUtils.absolute(new File("/./directory/."));
|
||||
Assert.assertEquals(dir, new File("/directory"));
|
||||
|
||||
dir = IOUtils.absolute(new File("/./directory/./"));
|
||||
Assert.assertEquals(dir, new File("/directory"));
|
||||
|
||||
dir = IOUtils.absolute(new File("/./directory./"));
|
||||
Assert.assertEquals(dir, new File("/directory."));
|
||||
|
||||
dir = IOUtils.absolute(new File("/./.directory/"));
|
||||
Assert.assertEquals(dir, new File("/.directory"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTail() throws IOException {
|
||||
List<String> lines = Arrays.asList(
|
||||
"chr18_random 4262 3154410390 50 51",
|
||||
"chr19_random 301858 3154414752 50 51",
|
||||
"chr21_random 1679693 3154722662 50 51",
|
||||
"chr22_random 257318 3156435963 50 51",
|
||||
"chrX_random 1719168 3156698441 50 51");
|
||||
List<String> tail = IOUtils.tail(new File(BaseTest.hg18Reference + ".fai"), 5);
|
||||
Assert.assertEquals(tail.size(), 5);
|
||||
for (int i = 0; i < 5; i++)
|
||||
Assert.assertEquals(tail.get(i), lines.get(i));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWriteSystemFile() throws IOException {
|
||||
File temp = createTempFile("temp.", ".properties");
|
||||
try {
|
||||
IOUtils.writeResource(new Resource("StingText.properties", null), temp);
|
||||
} finally {
|
||||
FileUtils.deleteQuietly(temp);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWriteSystemTempFile() throws IOException {
|
||||
File temp = IOUtils.writeTempResource(new Resource("StingText.properties", null));
|
||||
try {
|
||||
Assert.assertTrue(temp.getName().startsWith("StingText"), "File does not start with 'StingText.': " + temp);
|
||||
Assert.assertTrue(temp.getName().endsWith(".properties"), "File does not end with '.properties': " + temp);
|
||||
} finally {
|
||||
FileUtils.deleteQuietly(temp);
|
||||
}
|
||||
}
|
||||
|
||||
@Test(expectedExceptions = IllegalArgumentException.class)
|
||||
public void testMissingSystemFile() throws IOException {
|
||||
File temp = createTempFile("temp.", ".properties");
|
||||
try {
|
||||
IOUtils.writeResource(new Resource("MissingStingText.properties", null), temp);
|
||||
} finally {
|
||||
FileUtils.deleteQuietly(temp);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWriteRelativeFile() throws IOException {
|
||||
File temp = createTempFile("temp.", ".properties");
|
||||
try {
|
||||
IOUtils.writeResource(new Resource("/StingText.properties", IOUtils.class), temp);
|
||||
} finally {
|
||||
FileUtils.deleteQuietly(temp);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWriteRelativeTempFile() throws IOException {
|
||||
File temp = IOUtils.writeTempResource(new Resource("/StingText.properties", IOUtils.class));
|
||||
try {
|
||||
Assert.assertTrue(temp.getName().startsWith("StingText"), "File does not start with 'StingText.': " + temp);
|
||||
Assert.assertTrue(temp.getName().endsWith(".properties"), "File does not end with '.properties': " + temp);
|
||||
} finally {
|
||||
FileUtils.deleteQuietly(temp);
|
||||
}
|
||||
}
|
||||
|
||||
@Test(expectedExceptions = IllegalArgumentException.class)
|
||||
public void testMissingRelativeFile() throws IOException {
|
||||
File temp = createTempFile("temp.", ".properties");
|
||||
try {
|
||||
// Looking for /org/broadinstitute/sting/utils/file/StingText.properties
|
||||
IOUtils.writeResource(new Resource("StingText.properties", IOUtils.class), temp);
|
||||
} finally {
|
||||
FileUtils.deleteQuietly(temp);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testResourceProperties() {
|
||||
Resource resource = new Resource("foo", Resource.class);
|
||||
Assert.assertEquals(resource.getPath(), "foo");
|
||||
Assert.assertEquals(resource.getRelativeClass(), Resource.class);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,517 @@
|
|||
/*
|
||||
* Copyright (c) 2011, The Broad Institute
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package org.broadinstitute.sting.utils.runtime;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.broadinstitute.sting.BaseTest;
|
||||
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
|
||||
import org.broadinstitute.sting.utils.exceptions.UserException;
|
||||
import org.broadinstitute.sting.utils.io.IOUtils;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.DataProvider;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class ProcessControllerUnitTest extends BaseTest {
|
||||
private static final String NL = String.format("%n");
|
||||
|
||||
@Test(timeOut = 60 * 1000)
|
||||
public void testDestroyThreadLocal() throws InterruptedException {
|
||||
for (int i = 0; i < 3; i++) {
|
||||
final ProcessController controller = ProcessController.getThreadLocal();
|
||||
final ProcessSettings job = new ProcessSettings(
|
||||
new String[] {"sh", "-c", "echo Hello World && sleep 600 && echo Goodbye"});
|
||||
job.getStdoutSettings().setBufferSize(-1);
|
||||
|
||||
Thread t = new Thread(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
System.out.println("BACK: Starting on background thread");
|
||||
ProcessOutput result = controller.exec(job);
|
||||
// Assert in background thread doesn't make it to main thread but does print a trace.
|
||||
Assert.assertTrue(result.getExitValue() != 0, "Destroy-attempted job returned zero exit status");
|
||||
System.out.println("BACK: Background thread exiting");
|
||||
}
|
||||
});
|
||||
|
||||
System.out.println("MAIN: Starting background thread");
|
||||
t.start();
|
||||
System.out.println("MAIN: Sleeping main thread 3s");
|
||||
Thread.sleep(3000);
|
||||
System.out.println("MAIN: Destroying job");
|
||||
controller.tryDestroy();
|
||||
System.out.println("MAIN: Not waiting on background thread to exit");
|
||||
// Using standard java.io this was blocking on linux.
|
||||
// TODO: try again with NIO.
|
||||
//t.join();
|
||||
//System.out.println("MAIN: Background thread exited");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReuseAfterError() {
|
||||
ProcessController controller = new ProcessController();
|
||||
|
||||
ProcessSettings job;
|
||||
|
||||
for (int i = 0; i < 3; i++) {
|
||||
// Test bad command
|
||||
job = new ProcessSettings(new String[] {"no_such_command"});
|
||||
try {
|
||||
controller.exec(job);
|
||||
} catch (ReviewedStingException e) {
|
||||
/* Was supposed to throw an exception */
|
||||
}
|
||||
|
||||
// Test exit != 0
|
||||
job = new ProcessSettings(new String[] {"cat", "non_existent_file"});
|
||||
int exitValue = controller.exec(job).getExitValue();
|
||||
Assert.assertTrue(exitValue != 0, "'cat' non existent file returned 0");
|
||||
|
||||
// Text success
|
||||
job = new ProcessSettings(new String[] {"echo", "Hello World"});
|
||||
exitValue = controller.exec(job).getExitValue();
|
||||
Assert.assertEquals(exitValue, 0, "Echo failed");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEnvironment() {
|
||||
String key = "MY_NEW_VAR";
|
||||
String value = "value is here";
|
||||
|
||||
ProcessSettings job = new ProcessSettings(new String[] {"sh", "-c", "echo $"+key});
|
||||
job.getStdoutSettings().setBufferSize(-1);
|
||||
job.setRedirectErrorStream(true);
|
||||
|
||||
Map<String, String> env = new HashMap<String, String>(System.getenv());
|
||||
env.put(key, value);
|
||||
job.setEnvironment(env);
|
||||
|
||||
ProcessController controller = new ProcessController();
|
||||
ProcessOutput result = controller.exec(job);
|
||||
int exitValue = result.getExitValue();
|
||||
|
||||
Assert.assertEquals(exitValue, 0, "Echo environment variable failed");
|
||||
Assert.assertEquals(result.getStdout().getBufferString(), value + NL, "Echo environment returned unexpected output");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDirectory() throws IOException {
|
||||
File dir = null;
|
||||
try {
|
||||
dir = IOUtils.tempDir("temp.", "").getCanonicalFile();
|
||||
|
||||
ProcessSettings job = new ProcessSettings(new String[] {"pwd"});
|
||||
job.getStdoutSettings().setBufferSize(-1);
|
||||
job.setRedirectErrorStream(true);
|
||||
job.setDirectory(dir);
|
||||
|
||||
ProcessController controller = new ProcessController();
|
||||
ProcessOutput result = controller.exec(job);
|
||||
int exitValue = result.getExitValue();
|
||||
|
||||
Assert.assertEquals(exitValue, 0, "Getting working directory failed");
|
||||
|
||||
Assert.assertEquals(result.getStdout().getBufferString(), dir.getAbsolutePath() + NL,
|
||||
"Setting/getting working directory returned unexpected output");
|
||||
} finally {
|
||||
FileUtils.deleteQuietly(dir);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReadStdInBuffer() {
|
||||
String bufferText = "Hello from buffer";
|
||||
ProcessSettings job = new ProcessSettings(new String[] {"cat"});
|
||||
job.getStdoutSettings().setBufferSize(-1);
|
||||
job.setRedirectErrorStream(true);
|
||||
job.getStdinSettings().setInputBuffer(bufferText);
|
||||
|
||||
ProcessController controller = new ProcessController();
|
||||
ProcessOutput output = controller.exec(job);
|
||||
|
||||
Assert.assertEquals(output.getStdout().getBufferString(), bufferText,
|
||||
"Unexpected output from cat stdin buffer");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReadStdInFile() {
|
||||
File input = null;
|
||||
try {
|
||||
String fileText = "Hello from file";
|
||||
input = IOUtils.writeTempFile(fileText, "stdin.", ".txt", null);
|
||||
|
||||
ProcessSettings job = new ProcessSettings(new String[] {"cat"});
|
||||
job.getStdoutSettings().setBufferSize(-1);
|
||||
job.setRedirectErrorStream(true);
|
||||
job.getStdinSettings().setInputFile(input);
|
||||
|
||||
ProcessController controller = new ProcessController();
|
||||
ProcessOutput output = controller.exec(job);
|
||||
|
||||
Assert.assertEquals(output.getStdout().getBufferString(), fileText,
|
||||
"Unexpected output from cat stdin file");
|
||||
} finally {
|
||||
FileUtils.deleteQuietly(input);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWriteStdOut() {
|
||||
ProcessSettings job = new ProcessSettings(new String[] {"echo", "Testing to stdout"});
|
||||
// Not going to call the System.setOut() for now. Just running a basic visual test.
|
||||
job.getStdoutSettings().printStandard(true);
|
||||
job.setRedirectErrorStream(true);
|
||||
|
||||
System.out.println("testWriteStdOut: Writing two lines to std out...");
|
||||
ProcessController controller = new ProcessController();
|
||||
controller.exec(job);
|
||||
job.setCommand(new String[]{"cat", "non_existent_file"});
|
||||
controller.exec(job);
|
||||
System.out.println("testWriteStdOut: ...two lines should have been printed to std out");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testErrorToOut() throws IOException {
|
||||
File outFile = null;
|
||||
File errFile = null;
|
||||
try {
|
||||
outFile = BaseTest.createTempFile("temp", "");
|
||||
errFile = BaseTest.createTempFile("temp", "");
|
||||
|
||||
ProcessSettings job = new ProcessSettings(new String[]{"cat", "non_existent_file"});
|
||||
job.getStdoutSettings().setOutputFile(outFile);
|
||||
job.getStdoutSettings().setBufferSize(-1);
|
||||
job.getStderrSettings().setOutputFile(errFile);
|
||||
job.getStderrSettings().setBufferSize(-1);
|
||||
job.setRedirectErrorStream(true);
|
||||
|
||||
ProcessOutput result = new ProcessController().exec(job);
|
||||
int exitValue = result.getExitValue();
|
||||
|
||||
Assert.assertTrue(exitValue != 0, "'cat' non existent file returned 0");
|
||||
|
||||
String fileString, bufferString;
|
||||
|
||||
fileString = FileUtils.readFileToString(outFile);
|
||||
Assert.assertTrue(fileString.length() > 0, "Out file was length 0");
|
||||
|
||||
bufferString = result.getStdout().getBufferString();
|
||||
Assert.assertTrue(bufferString.length() > 0, "Out buffer was length 0");
|
||||
|
||||
Assert.assertFalse(result.getStdout().isBufferTruncated(), "Out buffer was truncated");
|
||||
Assert.assertEquals(bufferString.length(), fileString.length(), "Out buffer length did not match file length");
|
||||
|
||||
fileString = FileUtils.readFileToString(errFile);
|
||||
Assert.assertEquals(fileString, "", "Unexpected output to err file");
|
||||
|
||||
bufferString = result.getStderr().getBufferString();
|
||||
Assert.assertEquals(bufferString, "", "Unexepected output to err buffer");
|
||||
} finally {
|
||||
FileUtils.deleteQuietly(outFile);
|
||||
FileUtils.deleteQuietly(errFile);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testErrorToErr() throws IOException {
|
||||
File outFile = null;
|
||||
File errFile = null;
|
||||
try {
|
||||
outFile = BaseTest.createTempFile("temp", "");
|
||||
errFile = BaseTest.createTempFile("temp", "");
|
||||
|
||||
ProcessSettings job = new ProcessSettings(new String[]{"cat", "non_existent_file"});
|
||||
job.getStdoutSettings().setOutputFile(outFile);
|
||||
job.getStdoutSettings().setBufferSize(-1);
|
||||
job.getStderrSettings().setOutputFile(errFile);
|
||||
job.getStderrSettings().setBufferSize(-1);
|
||||
job.setRedirectErrorStream(false);
|
||||
|
||||
ProcessOutput result = new ProcessController().exec(job);
|
||||
int exitValue = result.getExitValue();
|
||||
|
||||
Assert.assertTrue(exitValue != 0, "'cat' non existent file returned 0");
|
||||
|
||||
String fileString, bufferString;
|
||||
|
||||
fileString = FileUtils.readFileToString(errFile);
|
||||
Assert.assertTrue(fileString.length() > 0, "Err file was length 0");
|
||||
|
||||
bufferString = result.getStderr().getBufferString();
|
||||
Assert.assertTrue(bufferString.length() > 0, "Err buffer was length 0");
|
||||
|
||||
Assert.assertFalse(result.getStderr().isBufferTruncated(), "Err buffer was truncated");
|
||||
Assert.assertEquals(bufferString.length(), fileString.length(), "Err buffer length did not match file length");
|
||||
|
||||
fileString = FileUtils.readFileToString(outFile);
|
||||
Assert.assertEquals(fileString, "", "Unexpected output to out file");
|
||||
|
||||
bufferString = result.getStdout().getBufferString();
|
||||
Assert.assertEquals(bufferString, "", "Unexepected output to out buffer");
|
||||
} finally {
|
||||
FileUtils.deleteQuietly(outFile);
|
||||
FileUtils.deleteQuietly(errFile);
|
||||
}
|
||||
}
|
||||
|
||||
private static final String TRUNCATE_TEXT = "Hello World";
|
||||
private static final byte[] TRUNCATE_OUTPUT_BYTES = (TRUNCATE_TEXT + NL).getBytes();
|
||||
|
||||
/**
|
||||
* @return Test truncating content vs. not truncating (run at -1/+1 size)
|
||||
*/
|
||||
@DataProvider(name = "truncateSizes")
|
||||
public Object[][] getTruncateBufferSizes() {
|
||||
int l = TRUNCATE_OUTPUT_BYTES.length;
|
||||
return new Object[][]{
|
||||
new Object[]{0, 0},
|
||||
new Object[]{l, l},
|
||||
new Object[]{l + 1, l},
|
||||
new Object[]{l - 1, l - 1}
|
||||
};
|
||||
}
|
||||
|
||||
@Test(dataProvider = "truncateSizes")
|
||||
public void testTruncateBuffer(int truncateLen, int expectedLen) {
|
||||
byte[] expected = Arrays.copyOf(TRUNCATE_OUTPUT_BYTES, expectedLen);
|
||||
|
||||
String[] command = {"echo", TRUNCATE_TEXT};
|
||||
ProcessController controller = new ProcessController();
|
||||
|
||||
ProcessSettings job = new ProcessSettings(command);
|
||||
job.getStdoutSettings().setBufferSize(truncateLen);
|
||||
ProcessOutput result = controller.exec(job);
|
||||
|
||||
int exitValue = result.getExitValue();
|
||||
|
||||
Assert.assertEquals(exitValue, 0,
|
||||
String.format("Echo returned %d: %s", exitValue, TRUNCATE_TEXT));
|
||||
|
||||
byte[] bufferBytes = result.getStdout().getBufferBytes();
|
||||
|
||||
Assert.assertEquals(bufferBytes, expected,
|
||||
String.format("Output buffer didn't match (%d vs %d)", expected.length, bufferBytes.length));
|
||||
|
||||
boolean truncated = result.getStdout().isBufferTruncated();
|
||||
|
||||
Assert.assertEquals(truncated, TRUNCATE_OUTPUT_BYTES.length > truncateLen,
|
||||
"Unexpected buffer truncation result");
|
||||
}
|
||||
|
||||
private static final String[] LONG_COMMAND = getLongCommand();
|
||||
private static final String LONG_COMMAND_STRING = StringUtils.join(LONG_COMMAND, " ");
|
||||
private static final String LONG_COMMAND_DESCRIPTION = "<long command>";
|
||||
|
||||
@DataProvider(name = "echoCommands")
|
||||
public Object[][] getEchoCommands() {
|
||||
|
||||
new EchoCommand(new String[]{"echo", "Hello", "World"}, "Hello World" + NL);
|
||||
new EchoCommand(new String[]{"echo", "'Hello", "World"}, "'Hello World" + NL);
|
||||
new EchoCommand(new String[]{"echo", "Hello", "World'"}, "Hello World'" + NL);
|
||||
new EchoCommand(new String[]{"echo", "'Hello", "World'"}, "'Hello World'" + NL);
|
||||
|
||||
String[] longCommand = new String[LONG_COMMAND.length + 1];
|
||||
longCommand[0] = "echo";
|
||||
System.arraycopy(LONG_COMMAND, 0, longCommand, 1, LONG_COMMAND.length);
|
||||
new EchoCommand(longCommand, LONG_COMMAND_STRING + NL) {
|
||||
@Override
|
||||
public String toString() {
|
||||
return LONG_COMMAND_DESCRIPTION;
|
||||
}
|
||||
};
|
||||
|
||||
return TestDataProvider.getTests(EchoCommand.class);
|
||||
}
|
||||
|
||||
@Test(dataProvider = "echoCommands")
|
||||
public void testEcho(EchoCommand script) throws IOException {
|
||||
File outputFile = null;
|
||||
try {
|
||||
outputFile = BaseTest.createTempFile("temp", "");
|
||||
|
||||
ProcessSettings job = new ProcessSettings(script.command);
|
||||
if (script.output != null) {
|
||||
job.getStdoutSettings().setOutputFile(outputFile);
|
||||
job.getStdoutSettings().setBufferSize(script.output.getBytes().length);
|
||||
}
|
||||
|
||||
ProcessOutput result = new ProcessController().exec(job);
|
||||
int exitValue = result.getExitValue();
|
||||
|
||||
Assert.assertEquals(exitValue, 0,
|
||||
String.format("Echo returned %d: %s", exitValue, script));
|
||||
|
||||
if (script.output != null) {
|
||||
|
||||
String fileString = FileUtils.readFileToString(outputFile);
|
||||
Assert.assertEquals(fileString, script.output,
|
||||
String.format("Output file didn't match (%d vs %d): %s",
|
||||
fileString.length(), script.output.length(), script));
|
||||
|
||||
String bufferString = result.getStdout().getBufferString();
|
||||
Assert.assertEquals(bufferString, script.output,
|
||||
String.format("Output content didn't match (%d vs %d): %s",
|
||||
bufferString.length(), script.output.length(), script));
|
||||
|
||||
Assert.assertFalse(result.getStdout().isBufferTruncated(),
|
||||
"Output content was truncated: " + script);
|
||||
}
|
||||
} finally {
|
||||
FileUtils.deleteQuietly(outputFile);
|
||||
}
|
||||
}
|
||||
|
||||
@Test(expectedExceptions = ReviewedStingException.class)
|
||||
public void testUnableToStart() {
|
||||
ProcessSettings job = new ProcessSettings(new String[]{"no_such_command"});
|
||||
new ProcessController().exec(job);
|
||||
}
|
||||
|
||||
@DataProvider(name = "scriptCommands")
|
||||
public Object[][] getScriptCommands() {
|
||||
new ScriptCommand(true, "echo Hello World", "Hello World" + NL);
|
||||
new ScriptCommand(false, "echo 'Hello World", null);
|
||||
new ScriptCommand(false, "echo Hello World'", null);
|
||||
new ScriptCommand(true, "echo 'Hello World'", "Hello World" + NL);
|
||||
new ScriptCommand(true, "echo \"Hello World\"", "Hello World" + NL);
|
||||
new ScriptCommand(false, "no_such_echo Hello World", null);
|
||||
new ScriptCommand(true, "echo #", NL);
|
||||
new ScriptCommand(true, "echo \\#", "#" + NL);
|
||||
new ScriptCommand(true, "echo \\\\#", "\\#" + NL);
|
||||
|
||||
new ScriptCommand(true, "echo " + LONG_COMMAND_STRING, LONG_COMMAND_STRING + NL) {
|
||||
@Override
|
||||
public String toString() {
|
||||
return LONG_COMMAND_DESCRIPTION;
|
||||
}
|
||||
};
|
||||
|
||||
return TestDataProvider.getTests(ScriptCommand.class);
|
||||
}
|
||||
|
||||
@Test(dataProvider = "scriptCommands")
|
||||
public void testScript(ScriptCommand script) throws IOException {
|
||||
File scriptFile = null;
|
||||
File outputFile = null;
|
||||
try {
|
||||
scriptFile = writeScript(script.content);
|
||||
outputFile = BaseTest.createTempFile("temp", "");
|
||||
|
||||
ProcessSettings job = new ProcessSettings(new String[]{"sh", scriptFile.getAbsolutePath()});
|
||||
if (script.output != null) {
|
||||
job.getStdoutSettings().setOutputFile(outputFile);
|
||||
job.getStdoutSettings().setBufferSize(script.output.getBytes().length);
|
||||
}
|
||||
|
||||
ProcessOutput result = new ProcessController().exec(job);
|
||||
int exitValue = result.getExitValue();
|
||||
|
||||
Assert.assertEquals(exitValue == 0, script.succeed,
|
||||
String.format("Script returned %d: %s", exitValue, script));
|
||||
|
||||
if (script.output != null) {
|
||||
|
||||
String fileString = FileUtils.readFileToString(outputFile);
|
||||
Assert.assertEquals(fileString, script.output,
|
||||
String.format("Output file didn't match (%d vs %d): %s",
|
||||
fileString.length(), script.output.length(), script));
|
||||
|
||||
String bufferString = result.getStdout().getBufferString();
|
||||
Assert.assertEquals(bufferString, script.output,
|
||||
String.format("Output content didn't match (%d vs %d): %s",
|
||||
bufferString.length(), script.output.length(), script));
|
||||
|
||||
Assert.assertFalse(result.getStdout().isBufferTruncated(),
|
||||
"Output content was truncated: " + script);
|
||||
}
|
||||
} finally {
|
||||
FileUtils.deleteQuietly(scriptFile);
|
||||
FileUtils.deleteQuietly(outputFile);
|
||||
}
|
||||
}
|
||||
|
||||
private static String[] getLongCommand() {
|
||||
// This command fails on some systems with a 4096 character limit when run via the old sh -c "echo ...",
|
||||
// but works on the same systems when run via sh <script>
|
||||
int cnt = 500;
|
||||
String[] command = new String[cnt];
|
||||
for (int i = 1; i <= cnt; i++) {
|
||||
command[i - 1] = String.format("%03d______", i);
|
||||
}
|
||||
return command;
|
||||
}
|
||||
|
||||
private static File writeScript(String contents) {
|
||||
try {
|
||||
File file = BaseTest.createTempFile("temp", "");
|
||||
FileUtils.writeStringToFile(file, contents);
|
||||
return file;
|
||||
} catch (IOException e) {
|
||||
throw new UserException.BadTmpDir(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static class EchoCommand extends TestDataProvider {
|
||||
public final String[] command;
|
||||
public final String output;
|
||||
|
||||
public EchoCommand(String[] command, String output) {
|
||||
super(EchoCommand.class);
|
||||
this.command = command;
|
||||
this.output = output;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return StringUtils.join(command, " ");
|
||||
}
|
||||
}
|
||||
|
||||
public static class ScriptCommand extends TestDataProvider {
|
||||
public final boolean succeed;
|
||||
public final String content;
|
||||
public final String output;
|
||||
|
||||
public ScriptCommand(boolean succeed, String content, String output) {
|
||||
super(ScriptCommand.class);
|
||||
this.succeed = succeed;
|
||||
this.content = content;
|
||||
this.output = output;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return content;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -179,13 +179,19 @@
|
|||
</xsl:when>
|
||||
</xsl:choose>
|
||||
</xsl:for-each>
|
||||
<xsl:for-each select="dir">
|
||||
<fileset dir="{$staging.dir}">
|
||||
<xsl:attribute name="includes">
|
||||
<xsl:value-of select="concat(@name,'/**')"/>
|
||||
</xsl:attribute>
|
||||
</fileset>
|
||||
</xsl:for-each>
|
||||
<xsl:for-each select="dir">
|
||||
<xsl:variable name="includes">
|
||||
<xsl:choose>
|
||||
<xsl:when test="@includes = ''">
|
||||
<xsl:value-of select="concat(@name,'/**')"/>
|
||||
</xsl:when>
|
||||
<xsl:otherwise>
|
||||
<xsl:value-of select="concat(@name,'/',@includes)"/>
|
||||
</xsl:otherwise>
|
||||
</xsl:choose>
|
||||
</xsl:variable>
|
||||
<fileset dir="{$staging.dir}" includes="{$includes}"/>
|
||||
</xsl:for-each>
|
||||
</xsl:template>
|
||||
|
||||
<!-- Determine the short name (filename w/o directory structure of the given filename -->
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@
|
|||
<package name="org.broad.tribble.**" />
|
||||
<!-- Workaround - depend on the logger impl required by JEXL -->
|
||||
<package name="org.apache.commons.logging.impl" />
|
||||
<!-- R packages -->
|
||||
<dir name="org/broadinstitute/sting/utils/R" includes="*.tar.gz" />
|
||||
</dependencies>
|
||||
</executable>
|
||||
<resources>
|
||||
|
|
|
|||
|
|
@ -65,6 +65,10 @@
|
|||
<package name="ca.mcgill.mcb.pcingola.**" />
|
||||
<file path="snpEff_genes.ftl" />
|
||||
<file path="snpEff_summary.ftl" />
|
||||
|
||||
<!-- R scripts -->
|
||||
<dir name="org/broadinstitute/sting/queue" includes="**/*.R" />
|
||||
|
||||
</dependencies>
|
||||
<modules>
|
||||
<module file="GATKEngine.xml"/>
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ import org.broadinstitute.sting.queue.engine.{QGraphSettings, QGraph}
|
|||
import collection.JavaConversions._
|
||||
import org.broadinstitute.sting.utils.classloader.PluginManager
|
||||
import org.broadinstitute.sting.utils.exceptions.UserException
|
||||
import org.broadinstitute.sting.utils.io.IOUtils
|
||||
|
||||
/**
|
||||
* Entry point of Queue. Compiles and runs QScripts passed in to the command line.
|
||||
|
|
@ -48,7 +49,6 @@ object QCommandLine extends Logging {
|
|||
val shutdownHook = new Thread {
|
||||
override def run() {
|
||||
logger.info("Shutting down jobs. Please wait...")
|
||||
ProcessController.shutdown()
|
||||
qCommandLine.shutdown()
|
||||
}
|
||||
}
|
||||
|
|
@ -56,8 +56,12 @@ object QCommandLine extends Logging {
|
|||
Runtime.getRuntime.addShutdownHook(shutdownHook)
|
||||
|
||||
try {
|
||||
CommandLineProgram.start(qCommandLine, argv);
|
||||
Runtime.getRuntime.removeShutdownHook(shutdownHook)
|
||||
CommandLineProgram.start(qCommandLine, argv)
|
||||
try {
|
||||
Runtime.getRuntime.removeShutdownHook(shutdownHook)
|
||||
} catch {
|
||||
case _ => /* ignore, example 'java.lang.IllegalStateException: Shutdown in progress' */
|
||||
}
|
||||
if (CommandLineProgram.result != 0)
|
||||
System.exit(CommandLineProgram.result);
|
||||
} catch {
|
||||
|
|
@ -80,6 +84,7 @@ class QCommandLine extends CommandLineProgram with Logging {
|
|||
private val qScriptManager = new QScriptManager
|
||||
private val qGraph = new QGraph
|
||||
private var qScriptClasses: File = _
|
||||
private var shuttingDown = false
|
||||
|
||||
private lazy val pluginManager = {
|
||||
qScriptClasses = IOUtils.tempDir("Q-Classes", "", settings.qSettings.tempDirectory)
|
||||
|
|
@ -118,10 +123,16 @@ class QCommandLine extends CommandLineProgram with Logging {
|
|||
script.onExecutionDone(qGraph.getFunctionsAndStatus(script.functions), qGraph.success)
|
||||
if ( ! settings.disableJobReport ) {
|
||||
val jobStringName = (QScriptUtils.?(settings.jobReportFile)).getOrElse(settings.qSettings.jobNamePrefix + ".jobreport.txt")
|
||||
val jobReportFile = new File(jobStringName)
|
||||
logger.info("Writing JobLogging GATKReport to file " + jobReportFile)
|
||||
QJobReport.printReport(qGraph.getFunctionsAndStatus(script.functions), jobReportFile)
|
||||
QJobReport.plotReport(settings.rScriptArgs, jobReportFile)
|
||||
|
||||
if (!shuttingDown) {
|
||||
val reportFile = new File(jobStringName)
|
||||
logger.info("Writing JobLogging GATKReport to file " + reportFile)
|
||||
QJobReport.printReport(qGraph.getFunctionsAndStatus(script.functions), reportFile)
|
||||
|
||||
val pdfFile = new File(jobStringName + ".pdf")
|
||||
logger.info("Plotting JobLogging GATKReport to file " + pdfFile)
|
||||
QJobReport.plotReport(settings.rScriptArgs, reportFile, pdfFile)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -163,6 +174,7 @@ class QCommandLine extends CommandLineProgram with Logging {
|
|||
Arrays.asList(new ScalaCompoundArgumentTypeDescriptor)
|
||||
|
||||
def shutdown() = {
|
||||
shuttingDown = true
|
||||
qGraph.shutdown()
|
||||
if (qScriptClasses != null) IOUtils.tryDelete(qScriptClasses)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ package org.broadinstitute.sting.queue
|
|||
|
||||
import scala.tools.nsc.{Global, Settings}
|
||||
import scala.tools.nsc.io.PlainFile
|
||||
import org.broadinstitute.sting.queue.util.{Logging, IOUtils}
|
||||
import org.broadinstitute.sting.queue.util.Logging
|
||||
import collection.JavaConversions._
|
||||
import java.io.File
|
||||
import scala.tools.nsc.reporters.AbstractReporter
|
||||
|
|
@ -20,7 +20,7 @@ class QScriptManager() extends Logging {
|
|||
* Compiles and loads the scripts in the files into the current classloader.
|
||||
* Heavily based on scala/src/compiler/scala/tools/ant/Scalac.scala
|
||||
*/
|
||||
def loadScripts(scripts: List[File], tempDir: File) = {
|
||||
def loadScripts(scripts: List[File], tempDir: File) {
|
||||
if (scripts.size > 0) {
|
||||
val settings = new Settings((error: String) => logger.error(error))
|
||||
settings.deprecation.value = true
|
||||
|
|
@ -63,7 +63,7 @@ object QScriptManager extends Logging {
|
|||
* Heavily based on scala/src/compiler/scala/tools/nsc/reporters/ConsoleReporter.scala
|
||||
*/
|
||||
private class Log4JReporter(val settings: Settings) extends AbstractReporter {
|
||||
def displayPrompt = throw new UnsupportedOperationException("Unable to prompt the user. Prompting should be off.")
|
||||
def displayPrompt { throw new UnsupportedOperationException("Unable to prompt the user. Prompting should be off.") }
|
||||
|
||||
/**
|
||||
* Displays the message at position with severity.
|
||||
|
|
@ -71,7 +71,7 @@ object QScriptManager extends Logging {
|
|||
* @param msg Message to display.
|
||||
* @param severity Severity of the event.
|
||||
*/
|
||||
def display(posIn: Position, msg: String, severity: Severity) = {
|
||||
def display(posIn: Position, msg: String, severity: Severity) {
|
||||
severity.count += 1
|
||||
val level = severity match {
|
||||
case INFO => Level.INFO
|
||||
|
|
@ -87,7 +87,6 @@ object QScriptManager extends Logging {
|
|||
case NoPosition =>
|
||||
printMessage(level, msg)
|
||||
case _ =>
|
||||
val buf = new StringBuilder(msg)
|
||||
val file = pos.source.file
|
||||
printMessage(level, file.name+":"+pos.line+": "+msg)
|
||||
printSourceLine(level, pos)
|
||||
|
|
@ -97,7 +96,7 @@ object QScriptManager extends Logging {
|
|||
/**
|
||||
* Prints a summary count of warnings and errors.
|
||||
*/
|
||||
def printSummary() = {
|
||||
def printSummary() {
|
||||
if (WARNING.count > 0)
|
||||
printMessage(Level.WARN, countElementsAsString(WARNING.count, "warning") + " found")
|
||||
if (ERROR.count > 0)
|
||||
|
|
@ -119,15 +118,16 @@ object QScriptManager extends Logging {
|
|||
* @param level Severity level.
|
||||
* @param pos Position in the file of the event.
|
||||
*/
|
||||
private def printColumnMarker(level: Level, pos: Position) =
|
||||
private def printColumnMarker(level: Level, pos: Position) {
|
||||
if (pos.isDefined) { printMessage(level, " " * (pos.column - 1) + "^") }
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints the message at the severity level.
|
||||
* @param level Severity level.
|
||||
* @param message Message content.
|
||||
*/
|
||||
private def printMessage(level: Level, message: String) = {
|
||||
private def printMessage(level: Level, message: String) {
|
||||
logger.log(level, message)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,8 @@ package org.broadinstitute.sting.queue.engine
|
|||
|
||||
import org.broadinstitute.sting.queue.function.CommandLineFunction
|
||||
import java.io.File
|
||||
import org.broadinstitute.sting.queue.util.{Logging, IOUtils}
|
||||
import org.broadinstitute.sting.queue.util.Logging
|
||||
import org.broadinstitute.sting.utils.io.IOUtils
|
||||
|
||||
/**
|
||||
* Runs a command line function.
|
||||
|
|
@ -69,7 +70,7 @@ trait CommandLineJobRunner extends JobRunner[CommandLineFunction] with Logging {
|
|||
|
||||
override def init() {
|
||||
super.init()
|
||||
var exec = new StringBuilder
|
||||
val exec = new StringBuilder
|
||||
|
||||
var dirs = Set.empty[File]
|
||||
for (dir <- function.jobDirectories)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,10 @@ package org.broadinstitute.sting.queue.engine
|
|||
|
||||
import org.broadinstitute.sting.queue.function.QFunction
|
||||
import java.io.{StringWriter, PrintWriter}
|
||||
import org.broadinstitute.sting.queue.util.{Logging, IOUtils}
|
||||
import org.broadinstitute.sting.queue.util.Logging
|
||||
import org.broadinstitute.sting.utils.io.IOUtils
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
/**
|
||||
* An edge in the QGraph that runs a QFunction.
|
||||
|
|
@ -150,26 +153,19 @@ class FunctionEdge(val function: QFunction, val inputs: QNode, val outputs: QNod
|
|||
/**
|
||||
* Outputs the last lines of the error logs.
|
||||
*/
|
||||
private def tailError() = {
|
||||
private def tailError() {
|
||||
val errorFile = functionErrorFile
|
||||
if (IOUtils.waitFor(errorFile, 120)) {
|
||||
val maxLines = 100
|
||||
val tailLines = IOUtils.tail(errorFile, maxLines)
|
||||
val nl = "%n".format()
|
||||
val summary = if (tailLines.size > maxLines) "Last %d lines".format(maxLines) else "Contents"
|
||||
logger.error("%s of %s:%n%s".format(summary, errorFile, tailLines.mkString(nl)))
|
||||
logger.error("%s of %s:%n%s".format(summary, errorFile, StringUtils.join(tailLines, nl)))
|
||||
} else {
|
||||
logger.error("Unable to access log file: %s".format(errorFile))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the contents of the error to the error file.
|
||||
*/
|
||||
private def writeError(content: String) {
|
||||
IOUtils.writeContents(functionErrorFile, content)
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the stack trace to the error file.
|
||||
*/
|
||||
|
|
@ -178,8 +174,8 @@ class FunctionEdge(val function: QFunction, val inputs: QNode, val outputs: QNod
|
|||
val printWriter = new PrintWriter(stackTrace)
|
||||
printWriter.println(function.description)
|
||||
e.printStackTrace(printWriter)
|
||||
printWriter.close
|
||||
IOUtils.writeContents(functionErrorFile, stackTrace.toString)
|
||||
printWriter.close()
|
||||
FileUtils.writeStringToFile(functionErrorFile, stackTrace.toString)
|
||||
}
|
||||
|
||||
def getRunInfo = {
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ package org.broadinstitute.sting.queue.engine
|
|||
|
||||
import org.broadinstitute.sting.queue.function.InProcessFunction
|
||||
import java.util.Date
|
||||
import org.broadinstitute.sting.queue.util.{Logging, IOUtils}
|
||||
import org.broadinstitute.sting.utils.Utils
|
||||
import org.apache.commons.io.FileUtils
|
||||
|
||||
/**
|
||||
* Runs a function that executes in process and does not fork out an external process.
|
||||
|
|
@ -11,7 +11,7 @@ import org.broadinstitute.sting.utils.Utils
|
|||
class InProcessRunner(val function: InProcessFunction) extends JobRunner[InProcessFunction] {
|
||||
private var runStatus: RunnerStatus.Value = _
|
||||
|
||||
def start() = {
|
||||
def start() {
|
||||
getRunInfo.startTime = new Date()
|
||||
getRunInfo.exechosts = Utils.resolveHostname()
|
||||
runStatus = RunnerStatus.RUNNING
|
||||
|
|
@ -20,7 +20,7 @@ class InProcessRunner(val function: InProcessFunction) extends JobRunner[InProce
|
|||
|
||||
getRunInfo.doneTime = new Date()
|
||||
val content = "%s%nDone.".format(function.description)
|
||||
IOUtils.writeContents(function.jobOutputFile, content)
|
||||
FileUtils.writeStringToFile(function.jobOutputFile, content)
|
||||
runStatus = RunnerStatus.DONE
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ import collection.immutable.{TreeSet, TreeMap}
|
|||
import org.broadinstitute.sting.queue.function.scattergather.{ScatterFunction, CloneFunction, GatherFunction, ScatterGatherableFunction}
|
||||
import java.util.Date
|
||||
import org.broadinstitute.sting.utils.Utils
|
||||
import org.broadinstitute.sting.utils.io.IOUtils
|
||||
|
||||
/**
|
||||
* The internal dependency tracker between sets of function input and output files.
|
||||
|
|
@ -416,8 +417,12 @@ class QGraph extends Logging {
|
|||
startedJobsToEmail = Set.empty[FunctionEdge]
|
||||
}
|
||||
|
||||
if (readyJobs.size == 0 && runningJobs.size > 0)
|
||||
Thread.sleep(nextRunningCheck(lastRunningCheck))
|
||||
if (readyJobs.size == 0 && runningJobs.size > 0) {
|
||||
runningLock.synchronized {
|
||||
if (running)
|
||||
runningLock.wait(nextRunningCheck(lastRunningCheck))
|
||||
}
|
||||
}
|
||||
|
||||
lastRunningCheck = System.currentTimeMillis
|
||||
updateStatus()
|
||||
|
|
@ -1002,7 +1007,12 @@ class QGraph extends Logging {
|
|||
true
|
||||
} else {
|
||||
!this.jobGraph.edgeSet.exists(edge => {
|
||||
edge.isInstanceOf[FunctionEdge] && edge.asInstanceOf[FunctionEdge].status == RunnerStatus.FAILED
|
||||
if (edge.isInstanceOf[FunctionEdge]) {
|
||||
val status = edge.asInstanceOf[FunctionEdge].status
|
||||
(status == RunnerStatus.PENDING || status == RunnerStatus.RUNNING || status == RunnerStatus.FAILED)
|
||||
} else {
|
||||
false
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1051,7 +1061,13 @@ class QGraph extends Logging {
|
|||
def shutdown() {
|
||||
// Signal the main thread to shutdown.
|
||||
running = false
|
||||
// Wait for the thread to finish and exit normally.
|
||||
|
||||
// Try and wait for the thread to finish and exit normally.
|
||||
runningLock.synchronized {
|
||||
runningLock.notify()
|
||||
}
|
||||
|
||||
// Start killing jobs.
|
||||
runningLock.synchronized {
|
||||
val runners = runningJobs.map(_.runner)
|
||||
runningJobs = Set.empty[FunctionEdge]
|
||||
|
|
|
|||
|
|
@ -56,6 +56,6 @@ class DrmaaJobManager extends CommandLineJobManager[DrmaaJobRunner] {
|
|||
updatedRunners
|
||||
}
|
||||
override def tryStop(runners: Set[DrmaaJobRunner]) {
|
||||
runners.filterNot(_.jobId == null).foreach(_.tryStop())
|
||||
runners.foreach(_.tryStop())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -135,14 +135,18 @@ class DrmaaJobRunner(val session: Session, val function: CommandLineFunction) ex
|
|||
|
||||
def tryStop() {
|
||||
session.synchronized {
|
||||
try {
|
||||
// Stop runners. SIGTERM(15) is preferred to SIGKILL(9).
|
||||
// Only way to send SIGTERM is for the Sys Admin set the terminate_method
|
||||
// resource of the designated queue to SIGTERM
|
||||
session.control(jobId, Session.TERMINATE)
|
||||
} catch {
|
||||
case e =>
|
||||
logger.error("Unable to kill job " + jobId, e)
|
||||
// Assumes that after being set the job may be
|
||||
// reassigned but will not be reset back to null
|
||||
if (jobId != null) {
|
||||
try {
|
||||
// Stop runners. SIGTERM(15) is preferred to SIGKILL(9).
|
||||
// Only way to send SIGTERM is for the Sys Admin set the terminate_method
|
||||
// resource of the designated queue to SIGTERM
|
||||
session.control(jobId, Session.TERMINATE)
|
||||
} catch {
|
||||
case e =>
|
||||
logger.error("Unable to kill job " + jobId, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,4 +30,5 @@ import org.broadinstitute.sting.queue.engine.CommandLineJobManager
|
|||
class ShellJobManager extends CommandLineJobManager[ShellJobRunner] {
|
||||
def runnerType = classOf[ShellJobRunner]
|
||||
def create(function: CommandLineFunction) = new ShellJobRunner(function)
|
||||
override def tryStop(runners: Set[ShellJobRunner]) { runners.foreach(_.tryStop()) }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,41 +25,66 @@
|
|||
package org.broadinstitute.sting.queue.engine.shell
|
||||
|
||||
import org.broadinstitute.sting.queue.function.CommandLineFunction
|
||||
import org.broadinstitute.sting.queue.util.ShellJob
|
||||
import org.broadinstitute.sting.queue.engine.{RunnerStatus, CommandLineJobRunner}
|
||||
import java.util.Date
|
||||
import org.broadinstitute.sting.gatk.phonehome.GATKRunReport
|
||||
import org.broadinstitute.sting.utils.Utils
|
||||
import org.broadinstitute.sting.utils.runtime.{ProcessSettings, OutputStreamSettings, ProcessController}
|
||||
|
||||
/**
|
||||
* Runs jobs one at a time locally
|
||||
*/
|
||||
class ShellJobRunner(val function: CommandLineFunction) extends CommandLineJobRunner {
|
||||
private var runStatus: RunnerStatus.Value = _
|
||||
// Controller on the thread that started the job
|
||||
private var controller: ProcessController = null
|
||||
|
||||
/**
|
||||
* Runs the function on the local shell.
|
||||
* @param function Command to run.
|
||||
*/
|
||||
def start() {
|
||||
val job = new ShellJob
|
||||
val commandLine = Array("sh", jobScript.getAbsolutePath)
|
||||
val stdoutSettings = new OutputStreamSettings
|
||||
val stderrSettings = new OutputStreamSettings
|
||||
val mergeError = (function.jobErrorFile != null)
|
||||
|
||||
job.workingDir = function.commandDirectory
|
||||
job.outputFile = function.jobOutputFile
|
||||
job.errorFile = function.jobErrorFile
|
||||
stdoutSettings.setOutputFile(function.jobOutputFile, true)
|
||||
if (function.jobErrorFile != null)
|
||||
stderrSettings.setOutputFile(function.jobErrorFile, true)
|
||||
|
||||
job.shellScript = jobScript
|
||||
if (logger.isDebugEnabled) {
|
||||
stdoutSettings.printStandard(true)
|
||||
stderrSettings.printStandard(true)
|
||||
}
|
||||
|
||||
// Allow advanced users to update the job.
|
||||
updateJobRun(job)
|
||||
val processSettings = new ProcessSettings(
|
||||
commandLine, mergeError, function.commandDirectory, null,
|
||||
null, stdoutSettings, stderrSettings)
|
||||
|
||||
updateJobRun(processSettings)
|
||||
|
||||
getRunInfo.startTime = new Date()
|
||||
getRunInfo.exechosts = Utils.resolveHostname()
|
||||
updateStatus(RunnerStatus.RUNNING)
|
||||
job.run()
|
||||
controller = ProcessController.getThreadLocal
|
||||
val exitStatus = controller.exec(processSettings).getExitValue
|
||||
getRunInfo.doneTime = new Date()
|
||||
updateStatus(RunnerStatus.DONE)
|
||||
updateStatus(if (exitStatus == 0) RunnerStatus.DONE else RunnerStatus.FAILED)
|
||||
}
|
||||
|
||||
override def checkUnknownStatus() {}
|
||||
/**
|
||||
* Possibly invoked from a shutdown thread, find and
|
||||
* stop the controller from the originating thread
|
||||
*/
|
||||
def tryStop() {
|
||||
// Assumes that after being set the job may be
|
||||
// reassigned but will not be reset back to null
|
||||
if (controller != null) {
|
||||
try {
|
||||
controller.tryDestroy()
|
||||
} catch {
|
||||
case e =>
|
||||
logger.error("Unable to kill shell job: " + function.description)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ package org.broadinstitute.sting.queue.extensions.gatk
|
|||
import org.broadinstitute.sting.utils.interval.IntervalUtils
|
||||
import java.io.File
|
||||
import collection.JavaConversions._
|
||||
import org.broadinstitute.sting.queue.util.IOUtils
|
||||
import org.broadinstitute.sting.utils.io.IOUtils
|
||||
import org.broadinstitute.sting.queue.function.scattergather.{CloneFunction, ScatterFunction}
|
||||
import org.broadinstitute.sting.commandline.Output
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
package org.broadinstitute.sting.queue.extensions.gatk
|
||||
|
||||
import java.io.File
|
||||
import org.broadinstitute.sting.queue.util.FileExtension
|
||||
import org.broadinstitute.sting.utils.io.FileExtension
|
||||
import java.lang.String
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
package org.broadinstitute.sting.queue.extensions.gatk
|
||||
|
||||
import java.io.File
|
||||
import org.broadinstitute.sting.queue.util.FileExtension
|
||||
import org.broadinstitute.sting.utils.io.FileExtension
|
||||
|
||||
/**
|
||||
* Used to provide tagged -I input_file arguments to the GATK.
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@
|
|||
package org.broadinstitute.sting.queue.function
|
||||
|
||||
import org.broadinstitute.sting.commandline.Argument
|
||||
import org.broadinstitute.sting.queue.util.IOUtils
|
||||
import org.broadinstitute.sting.utils.io.IOUtils
|
||||
import java.io.File
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import org.broadinstitute.sting.queue.{QException, QSettings}
|
|||
import collection.JavaConversions._
|
||||
import org.broadinstitute.sting.queue.function.scattergather.SimpleTextGatherFunction
|
||||
import org.broadinstitute.sting.queue.util._
|
||||
import org.broadinstitute.sting.utils.io.IOUtils
|
||||
|
||||
/**
|
||||
* The base interface for all functions in Queue.
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ import java.io.File
|
|||
import org.broadinstitute.sting.commandline.{Input, Output}
|
||||
import org.broadinstitute.sting.queue.function.QFunction
|
||||
import org.broadinstitute.sting.queue.QException
|
||||
import org.broadinstitute.sting.queue.util.IOUtils
|
||||
import org.broadinstitute.sting.utils.io.IOUtils
|
||||
import collection.JavaConversions._
|
||||
|
||||
/**
|
||||
* Base class for Gather command line functions.
|
||||
|
|
@ -29,7 +30,7 @@ trait GatherFunction extends QFunction {
|
|||
/**
|
||||
* Waits for gather parts to propagate over NFS or throws an exception.
|
||||
*/
|
||||
protected def waitForGatherParts = {
|
||||
protected def waitForGatherParts() {
|
||||
val missing = IOUtils.waitFor(gatherParts, 120)
|
||||
if (!missing.isEmpty)
|
||||
throw new QException("Unable to find gather inputs: " + missing.mkString(", "))
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import org.broadinstitute.sting.queue.util._
|
|||
import org.broadinstitute.sting.commandline.{Gatherer, Gather, ArgumentSource}
|
||||
import org.broadinstitute.sting.queue.function.{QFunction, CommandLineFunction}
|
||||
import org.broadinstitute.sting.queue.QException
|
||||
import org.broadinstitute.sting.utils.io.IOUtils
|
||||
|
||||
/**
|
||||
* A function that can be run faster by splitting it up into pieces and then joining together the results.
|
||||
|
|
@ -82,8 +83,8 @@ trait ScatterGatherableFunction extends CommandLineFunction {
|
|||
/**
|
||||
* Sets the scatter gather directory to the command directory if it is not already set.
|
||||
*/
|
||||
override def freezeFieldValues = {
|
||||
super.freezeFieldValues
|
||||
override def freezeFieldValues() {
|
||||
super.freezeFieldValues()
|
||||
|
||||
if (this.scatterGatherDirectory == null) {
|
||||
if (qSettings.jobScatterGatherDirectory != null) {
|
||||
|
|
@ -98,10 +99,14 @@ trait ScatterGatherableFunction extends CommandLineFunction {
|
|||
* The scatter function.
|
||||
*/
|
||||
private lazy val scatterFunction = {
|
||||
// Only depend on input fields that have a value
|
||||
val inputFieldsWithValues = this.inputFields.filter(hasFieldValue(_))
|
||||
val inputFiles = inputFieldsWithValues.flatMap(getFieldFiles(_)).toSet
|
||||
|
||||
val scatterFunction = newScatterFunction()
|
||||
this.copySettingsTo(scatterFunction)
|
||||
scatterFunction.originalFunction = this
|
||||
scatterFunction.originalInputs = this.inputs
|
||||
scatterFunction.originalInputs = inputFiles
|
||||
scatterFunction.commandDirectory = this.scatterGatherTempDir("scatter")
|
||||
scatterFunction.isIntermediate = true
|
||||
scatterFunction.addOrder = this.addOrder :+ 1
|
||||
|
|
@ -121,8 +126,6 @@ trait ScatterGatherableFunction extends CommandLineFunction {
|
|||
def generateFunctions() = {
|
||||
var functions = List.empty[QFunction]
|
||||
|
||||
// Only depend on input fields that have a value
|
||||
val inputFieldsWithValues = this.inputFields.filter(hasFieldValue(_))
|
||||
// Only gather up fields that will have a value
|
||||
val outputFieldsWithValues = this.outputFields.filter(hasFieldValue(_))
|
||||
|
||||
|
|
@ -228,7 +231,7 @@ trait ScatterGatherableFunction extends CommandLineFunction {
|
|||
* Calls setupScatterFunction with scatterFunction.
|
||||
* @param scatterFunction The function that will create the scatter pieces in the temporary directories.
|
||||
*/
|
||||
protected def initScatterFunction(scatterFunction: ScatterFunction) = {
|
||||
protected def initScatterFunction(scatterFunction: ScatterFunction) {
|
||||
if (this.setupScatterFunction != null)
|
||||
if (this.setupScatterFunction.isDefinedAt(scatterFunction))
|
||||
this.setupScatterFunction(scatterFunction)
|
||||
|
|
@ -272,7 +275,7 @@ trait ScatterGatherableFunction extends CommandLineFunction {
|
|||
* @param gatherFunction The function that will merge the gather pieces from the temporary directories.
|
||||
* @param gatherField The output field being gathered.
|
||||
*/
|
||||
protected def initGatherFunction(gatherFunction: GatherFunction, gatherField: ArgumentSource) = {
|
||||
protected def initGatherFunction(gatherFunction: GatherFunction, gatherField: ArgumentSource) {
|
||||
if (this.setupGatherFunction != null)
|
||||
if (this.setupGatherFunction.isDefinedAt(gatherFunction, gatherField))
|
||||
this.setupGatherFunction(gatherFunction, gatherField)
|
||||
|
|
@ -289,7 +292,7 @@ trait ScatterGatherableFunction extends CommandLineFunction {
|
|||
* @param cloneFunction The clone of this ScatterGatherableFunction
|
||||
* @param index The one based index (from 1..scatterCount inclusive) of the scatter piece.
|
||||
*/
|
||||
protected def initCloneFunction(cloneFunction: CloneFunction, index: Int) = {
|
||||
protected def initCloneFunction(cloneFunction: CloneFunction, index: Int) {
|
||||
if (this.setupCloneFunction != null)
|
||||
if (this.setupCloneFunction.isDefinedAt(cloneFunction, index))
|
||||
this.setupCloneFunction(cloneFunction, index)
|
||||
|
|
|
|||
|
|
@ -1,51 +0,0 @@
|
|||
package org.broadinstitute.sting.queue.util
|
||||
|
||||
import java.io.File
|
||||
|
||||
/**
|
||||
* Base utility class for a command line job.
|
||||
*/
|
||||
abstract class CommandLineJob {
|
||||
var shellScript: File = _
|
||||
var workingDir: File = _
|
||||
var inputFile: File = _
|
||||
var outputFile: File = _
|
||||
var errorFile: File = _
|
||||
|
||||
/**
|
||||
* Runs the command, either immediately or dispatching it to a compute farm.
|
||||
* If it is dispatched to a compute farm it should not start until jobs it depends on are finished.
|
||||
*/
|
||||
def run()
|
||||
|
||||
/**
|
||||
* Returns the content of a command output.
|
||||
* @param streamOutput The output of the command.
|
||||
* @return The content of the command, along with a message if it was truncated.
|
||||
*/
|
||||
protected def content(streamOutput: ProcessController.StreamOutput) = {
|
||||
var content = streamOutput.content
|
||||
if (streamOutput.contentTruncated)
|
||||
content += "%n%n<truncated>".format()
|
||||
content
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the ProcessController for this thread.
|
||||
* @return The ProcessController for this thread.
|
||||
*/
|
||||
protected def processController = CommandLineJob.threadProcessController.get
|
||||
|
||||
/** A five mb limit of characters for display. */
|
||||
protected val FIVE_MB = 1024 * 512 * 5;
|
||||
}
|
||||
|
||||
/**
|
||||
* Base class for a command line job.
|
||||
*/
|
||||
object CommandLineJob {
|
||||
/** Thread local process controller container. */
|
||||
private val threadProcessController = new ThreadLocal[ProcessController] {
|
||||
override def initialValue = new ProcessController
|
||||
}
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
package org.broadinstitute.sting.queue.util
|
||||
|
||||
import java.io.File
|
||||
|
||||
/**
|
||||
* An trait for @Input or @Output CommandLineFunction fields that are extensions of files.
|
||||
*/
|
||||
trait FileExtension extends File {
|
||||
/**
|
||||
* Returns a clone of the FileExtension with the new path.
|
||||
* @param newPath new path for the clone of this FileExtension
|
||||
* @return a clone of the FileExtension with the new path.
|
||||
*/
|
||||
def withPath(newPath: String): File
|
||||
}
|
||||
|
|
@ -1,253 +0,0 @@
|
|||
package org.broadinstitute.sting.queue.util
|
||||
|
||||
import org.apache.commons.io.FileUtils
|
||||
import java.io.{FileReader, File}
|
||||
import org.broadinstitute.sting.utils.exceptions.UserException
|
||||
import org.broadinstitute.sting.queue.QException
|
||||
|
||||
/**
|
||||
* A collection of utilities for modifying java.io.
|
||||
*/
|
||||
object IOUtils extends Logging {
|
||||
/**
|
||||
* Checks if the temp directory has been setup and throws an exception if they user hasn't set it correctly.
|
||||
* @param tempDir Temporary directory.
|
||||
*/
|
||||
def checkTempDir(tempDir: File) {
|
||||
val tempDirPath = tempDir.getAbsolutePath
|
||||
// Keeps the user from leaving the temp directory as the default, and on Macs from having pluses
|
||||
// in the path which can cause problems with the Google Reflections library.
|
||||
// see also: http://benjchristensen.com/2009/09/22/mac-osx-10-6-java-java-io-tmpdir/
|
||||
if (tempDirPath.startsWith("/var/folders/") || (tempDirPath == "/tmp") || (tempDirPath == "/tmp/"))
|
||||
throw new UserException.BadTmpDir("java.io.tmpdir must be explicitly set")
|
||||
if (!tempDir.exists && !tempDir.mkdirs)
|
||||
throw new UserException.BadTmpDir("Could not create directory: " + tempDir.getAbsolutePath)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a temp directory with the prefix and optional suffix.
|
||||
* @param prefix Prefix for the directory name.
|
||||
* @param suffix Optional suffix for the directory name.
|
||||
* @param tempDirParent Parent directory for the temp directory.
|
||||
* @return The created temporary directory.
|
||||
*/
|
||||
def tempDir(prefix: String, suffix: String = "", tempDirParent: File) = {
|
||||
if (!tempDirParent.exists && !tempDirParent.mkdirs)
|
||||
throw new UserException.BadTmpDir("Could not create temp directory: " + tempDirParent)
|
||||
val temp = File.createTempFile(prefix + "-", suffix, tempDirParent)
|
||||
if (!temp.delete)
|
||||
throw new UserException.BadTmpDir("Could not delete sub file: " + temp.getAbsolutePath)
|
||||
if (!temp.mkdir)
|
||||
throw new UserException.BadTmpDir("Could not create sub directory: " + temp.getAbsolutePath)
|
||||
absolute(temp)
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes content into a file.
|
||||
* @param file File to write to.
|
||||
* @param content Content to write.
|
||||
*/
|
||||
def writeContents(file: File, content: String) { FileUtils.writeStringToFile(file, content) }
|
||||
|
||||
/**
|
||||
* Reads content of a file into a string.
|
||||
* Only for use on really small files!
|
||||
* @param file File to read to.
|
||||
* @return content Content of the file.
|
||||
*/
|
||||
def readContents(file: File) = FileUtils.readFileToString(file)
|
||||
|
||||
/**
|
||||
* Writes content to a temp file and returns the path to the temporary file.
|
||||
* @param content to write.
|
||||
* @param prefix Prefix for the temp file.
|
||||
* @param suffix Suffix for the temp file.
|
||||
* @param directory Directory for the temp file.
|
||||
* @return the path to the temp file.
|
||||
*/
|
||||
def writeTempFile(content: String, prefix: String, suffix: String, directory: File) = {
|
||||
val tempFile = absolute(File.createTempFile(prefix, suffix, directory))
|
||||
writeContents(tempFile, content)
|
||||
tempFile
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits for NFS to propagate a file creation, imposing a timeout.
|
||||
*
|
||||
* Based on Apache Commons IO FileUtils.waitFor()
|
||||
*
|
||||
* @param file The file to wait for.
|
||||
* @param seconds The maximum time in seconds to wait.
|
||||
* @return true if the file exists
|
||||
*/
|
||||
def waitFor(file: File, seconds: Int): Boolean = waitFor(List(file), seconds).isEmpty
|
||||
|
||||
/**
|
||||
* Waits for NFS to propagate a file creation, imposing a timeout.
|
||||
*
|
||||
* Based on Apache Commons IO FileUtils.waitFor()
|
||||
*
|
||||
* @param files The list of files to wait for.
|
||||
* @param seconds The maximum time in seconds to wait.
|
||||
* @return Files that still do not exists at the end of the timeout, or a empty list if all files exists.
|
||||
*/
|
||||
def waitFor[T <: Traversable[File]](files: T, seconds: Int): Traversable[File] = {
|
||||
var timeout = 0;
|
||||
var tick = 0;
|
||||
var missingFiles = files.filterNot(_.exists)
|
||||
while (!missingFiles.isEmpty && timeout <= seconds) {
|
||||
if (tick >= 10) {
|
||||
tick = 0;
|
||||
timeout += 1
|
||||
}
|
||||
tick += 1
|
||||
try {
|
||||
Thread.sleep(100)
|
||||
} catch {
|
||||
case ignore: InterruptedException =>
|
||||
}
|
||||
missingFiles = missingFiles.filterNot(_.exists)
|
||||
}
|
||||
missingFiles
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the directory at the number of levels deep.
|
||||
* For example 2 levels of /path/to/dir will return /path/to
|
||||
* @param dir Directory path.
|
||||
* @param level how many levels deep from the root.
|
||||
* @return The path to the parent directory that is level-levels deep.
|
||||
*/
|
||||
def dirLevel(dir: File, level: Int): File = {
|
||||
var directories = List.empty[File]
|
||||
var parentDir = absolute(dir)
|
||||
while (parentDir != null) {
|
||||
directories +:= parentDir
|
||||
parentDir = parentDir.getParentFile
|
||||
}
|
||||
if (directories.size <= level)
|
||||
directories.last
|
||||
else
|
||||
directories(level)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the sub path rooted at the parent.
|
||||
* @param parent The parent directory.
|
||||
* @param path The sub path to append to the parent, if the path is not absolute.
|
||||
* @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path.
|
||||
*/
|
||||
def absolute(parent: File, path: String): File =
|
||||
absolute(parent, new File(path))
|
||||
|
||||
/**
|
||||
* Returns the sub path rooted at the parent.
|
||||
* @param parent The parent directory.
|
||||
* @param file The sub path to append to the parent, if the path is not absolute.
|
||||
* @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path.
|
||||
*/
|
||||
def absolute(parent: File, file: File): File = {
|
||||
val newPath =
|
||||
if (file.isAbsolute)
|
||||
absolutePath(file)
|
||||
else
|
||||
absolutePath(new File(parent, file.getPath))
|
||||
replacePath(file, newPath)
|
||||
}
|
||||
|
||||
/**
|
||||
* A mix of getCanonicalFile and getAbsoluteFile that returns the
|
||||
* absolute path to the file without deferencing symbolic links.
|
||||
* @param file the file.
|
||||
* @return the absolute path to the file.
|
||||
*/
|
||||
def absolute(file: File) = {
|
||||
replacePath(file, absolutePath(file))
|
||||
}
|
||||
|
||||
private def absolutePath(file: File) = {
|
||||
var fileAbs = file.getAbsoluteFile
|
||||
var names = List.empty[String]
|
||||
while (fileAbs != null) {
|
||||
val name = fileAbs.getName
|
||||
fileAbs = fileAbs.getParentFile
|
||||
|
||||
if (name == ".") {
|
||||
/* skip */
|
||||
|
||||
/* TODO: What do we do for ".."?
|
||||
} else if (name == "..") {
|
||||
|
||||
CentOS tcsh says use getCanonicalFile:
|
||||
~ $ mkdir -p test1/test2
|
||||
~ $ ln -s test1/test2 test3
|
||||
~ $ cd test3/..
|
||||
~/test1 $
|
||||
|
||||
Mac bash says keep going with getAbsoluteFile:
|
||||
~ $ mkdir -p test1/test2
|
||||
~ $ ln -s test1/test2 test3
|
||||
~ $ cd test3/..
|
||||
~ $
|
||||
|
||||
For now, leave it and let the shell figure it out.
|
||||
*/
|
||||
} else {
|
||||
names +:= name
|
||||
}
|
||||
}
|
||||
|
||||
names.mkString("/", "/", "")
|
||||
}
|
||||
|
||||
private def replacePath(file: File, path: String) = {
|
||||
file match {
|
||||
case fileExtension: FileExtension =>
|
||||
fileExtension.withPath(path)
|
||||
case file: File =>
|
||||
if (file.getClass != classOf[File])
|
||||
throw new QException("Sub classes of java.io.File must also implement FileExtension so that the path can be modified.")
|
||||
new File(path)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the last lines of the file.
|
||||
* NOTE: This is only safe to run on smaller files!
|
||||
* @param file File to read.
|
||||
* @param count Maximum number of lines to return.
|
||||
* @return The last count lines from file.
|
||||
*/
|
||||
def tail(file: File, count: Int) = {
|
||||
var tailLines = List.empty[String]
|
||||
var reader = new FileReader(file)
|
||||
try {
|
||||
val iterator = org.apache.commons.io.IOUtils.lineIterator(reader)
|
||||
var lineCount = 0
|
||||
while (iterator.hasNext) {
|
||||
val line = iterator.nextLine
|
||||
lineCount += 1
|
||||
if (lineCount > count)
|
||||
tailLines = tailLines.tail
|
||||
tailLines :+= line
|
||||
}
|
||||
} finally {
|
||||
org.apache.commons.io.IOUtils.closeQuietly(reader)
|
||||
}
|
||||
tailLines
|
||||
}
|
||||
|
||||
/**
|
||||
* Tries to delete a file. Emits a warning if the file was unable to be deleted.
|
||||
* @param file File to delete.
|
||||
* @return true if the file was deleted.
|
||||
*/
|
||||
def tryDelete(file: File) = {
|
||||
val deleted = FileUtils.deleteQuietly(file)
|
||||
if (deleted)
|
||||
logger.debug("Deleted " + file)
|
||||
else if (file.exists)
|
||||
logger.warn("Unable to delete " + file)
|
||||
deleted
|
||||
}
|
||||
}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
package org.broadinstitute.sting.queue.util
|
||||
|
||||
import org.broadinstitute.sting.queue.QException
|
||||
|
||||
/**
|
||||
* Captures the exit code and error text from a failed process.
|
||||
*/
|
||||
class JobExitException(val exitText: String, val commandLine: Array[String], val exitCode: Int, val stdErr: String)
|
||||
extends QException("%s%nCommand line:%n%s%nExit code: %s%nStandard error contained: %n%s"
|
||||
.format(exitText, commandLine.mkString(" "), exitCode, stdErr)) {
|
||||
}
|
||||
|
|
@ -1,369 +0,0 @@
|
|||
package org.broadinstitute.sting.queue.util
|
||||
|
||||
import java.io._
|
||||
import scala.collection.mutable.{HashSet, ListMap}
|
||||
|
||||
/**
|
||||
* Facade to Runtime.exec() and java.lang.Process. Handles
|
||||
* running a process to completion and returns stdout and stderr
|
||||
* as strings. Creates separate threads for reading stdout and stderr,
|
||||
* then reuses those threads for each process most efficient use is
|
||||
* to create one of these and use it repeatedly. Instances are not
|
||||
* thread-safe, however.
|
||||
*
|
||||
* @author originally by Michael Koehrsen ported to scala and enhanced by Khalid Shakir
|
||||
*/
|
||||
class ProcessController extends Logging {
|
||||
|
||||
// Threads that capture stdout and stderr
|
||||
private val stdoutCapture = new OutputCapture(ProcessController.STDOUT_KEY)
|
||||
private val stderrCapture = new OutputCapture(ProcessController.STDERR_KEY)
|
||||
|
||||
// Communication channels with output capture threads
|
||||
/** Holds the stdout and stderr sent to the background capture threads */
|
||||
private val toCapture = new ListMap[String, ProcessController.CapturedStreamOutput]
|
||||
|
||||
/** Holds the results of the capture from the background capture threads.
|
||||
* May be the content via toCapture or an EmptyStreamOutput if the capture was interrupted. */
|
||||
private val fromCapture = new ListMap[String, ProcessController.StreamOutput]
|
||||
|
||||
// Start the background threads for this controller.
|
||||
stdoutCapture.start()
|
||||
stderrCapture.start()
|
||||
|
||||
/**
|
||||
* Executes a command line program with the settings and waits for it to return, processing the output on a background thread.
|
||||
* @param settings Settings to be run.
|
||||
* @return The output of the command.
|
||||
*/
|
||||
def exec(settings: ProcessController.ProcessSettings): ProcessController.ProcessOutput = {
|
||||
var builder = new ProcessBuilder(settings.cmdarray:_*)
|
||||
builder.directory(settings.directory)
|
||||
|
||||
if (settings.environment != null) {
|
||||
val builderEnvironment = builder.environment
|
||||
builderEnvironment.clear()
|
||||
settings.environment.foreach{case (name, value) => builderEnvironment.put(name, value)}
|
||||
}
|
||||
|
||||
builder.redirectErrorStream(settings.redirectErrorStream)
|
||||
|
||||
var stdout: ProcessController.StreamOutput = null
|
||||
var stderr: ProcessController.StreamOutput = null
|
||||
val process = builder.start
|
||||
|
||||
ProcessController.running.add(process)
|
||||
try {
|
||||
val stdoutSettings = if (settings.stdoutSettings == null) ProcessController.EmptyStreamSettings else settings.stdoutSettings
|
||||
val stderrSettings = if (settings.stderrSettings == null) ProcessController.EmptyStreamSettings else settings.stderrSettings
|
||||
|
||||
toCapture.synchronized {
|
||||
toCapture.put(ProcessController.STDOUT_KEY, new ProcessController.CapturedStreamOutput(process.getInputStream, stdoutSettings, scala.Console.out))
|
||||
toCapture.put(ProcessController.STDERR_KEY, new ProcessController.CapturedStreamOutput(process.getErrorStream, stderrSettings, scala.Console.err))
|
||||
toCapture.notifyAll()
|
||||
}
|
||||
|
||||
if (settings.stdinSettings.input != null) {
|
||||
val writer = new OutputStreamWriter(process.getOutputStream)
|
||||
writer.write(settings.stdinSettings.input)
|
||||
writer.flush()
|
||||
}
|
||||
if (settings.stdinSettings.inputFile != null) {
|
||||
val reader = new FileReader(settings.stdinSettings.inputFile)
|
||||
val writer = new OutputStreamWriter(process.getOutputStream)
|
||||
val buf = new Array[Char](4096)
|
||||
var readCount = 0
|
||||
while ({readCount = reader.read(buf); readCount} >= 0)
|
||||
writer.write(buf, 0, readCount)
|
||||
writer.flush()
|
||||
reader.close()
|
||||
}
|
||||
|
||||
try {
|
||||
process.getOutputStream.close()
|
||||
process.waitFor()
|
||||
} finally {
|
||||
while (stdout == null || stderr == null) {
|
||||
fromCapture.synchronized {
|
||||
fromCapture.remove(ProcessController.STDOUT_KEY) match {
|
||||
case Some(stream) => stdout = stream
|
||||
case None => /* ignore */
|
||||
}
|
||||
fromCapture.remove(ProcessController.STDERR_KEY) match {
|
||||
case Some(stream) => stderr = stream
|
||||
case None => /* ignore */
|
||||
}
|
||||
|
||||
try {
|
||||
if (stdout == null || stderr == null)
|
||||
fromCapture.wait()
|
||||
} catch {
|
||||
case e: InterruptedException =>
|
||||
logger.error(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
ProcessController.running.remove(process)
|
||||
}
|
||||
|
||||
new ProcessController.ProcessOutput(process.exitValue, stdout, stderr)
|
||||
}
|
||||
|
||||
/** Ensures that the threads used to manipulate the IO for the process are cleaned up properly. */
|
||||
def close() = {
|
||||
try {
|
||||
stdoutCapture.interrupt()
|
||||
stderrCapture.interrupt()
|
||||
} catch {
|
||||
case e =>
|
||||
logger.error(e)
|
||||
}
|
||||
}
|
||||
|
||||
/** calls close() */
|
||||
override def finalize = close()
|
||||
|
||||
/**
|
||||
* Reads in the output of a stream on a background thread to keep the output pipe from backing up and freezing the called process.
|
||||
* @param key The stdout or stderr key for this output capture.
|
||||
*/
|
||||
private class OutputCapture(private val key: String)
|
||||
extends Thread("OutputCapture-" + key + "-" + Thread.currentThread.getName) {
|
||||
|
||||
setDaemon(true)
|
||||
|
||||
/** Runs the capture. */
|
||||
override def run = {
|
||||
var break = false
|
||||
while (!break) {
|
||||
var processStream: ProcessController.StreamOutput = ProcessController.EmptyStreamOutput
|
||||
try {
|
||||
// Wait for a new input stream to be passed from this process controller.
|
||||
var capturedProcessStream: ProcessController.CapturedStreamOutput = null
|
||||
while (capturedProcessStream == null) {
|
||||
toCapture.synchronized {
|
||||
toCapture.remove(key) match {
|
||||
case Some(stream) => capturedProcessStream = stream
|
||||
case None => toCapture.wait()
|
||||
}
|
||||
}
|
||||
}
|
||||
// Read in the input stream
|
||||
processStream = capturedProcessStream
|
||||
capturedProcessStream.read
|
||||
} catch {
|
||||
case e: InterruptedException => {
|
||||
logger.info("OutputReader interrupted, exiting")
|
||||
break = true
|
||||
}
|
||||
case e: IOException => {
|
||||
logger.error("Error reading process output", e)
|
||||
}
|
||||
} finally {
|
||||
// Send the string back to the process controller.
|
||||
fromCapture.synchronized {
|
||||
fromCapture.put(key, processStream)
|
||||
fromCapture.notify()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Facade to Runtime.exec() and java.lang.Process. Handles
|
||||
* running a process to completion and returns stdout and stderr
|
||||
* as strings. Creates separate threads for reading stdout and stderr,
|
||||
* then reuses those threads for each process most efficient use is
|
||||
* to create one of these and use it repeatedly. Instances are not
|
||||
* thread-safe, however.
|
||||
*
|
||||
* @author originally by Michael Koehrsen ported to scala and enhanced by Khalid Shakir
|
||||
*/
|
||||
object ProcessController extends Logging {
|
||||
|
||||
/**
|
||||
* Settings that define how to run a process.
|
||||
* @param cmdarray Command line to run.
|
||||
* @param environment Environment settings to override System.getEnv, or null to use System.getEnv.
|
||||
* @param directory The directory to run the command in, or null to run in the current directory.
|
||||
* @param stdinSettings Settings for writing to the process stdin.
|
||||
* @param stdoutSettings Settings for capturing the process stdout.
|
||||
* @param stderrSettings Setting for capturing the process stderr.
|
||||
* @param redirectErrorStream true if stderr should be sent to stdout.
|
||||
*/
|
||||
class ProcessSettings(val cmdarray: Array[String], val environment: Map[String, String], val directory: File,
|
||||
val stdinSettings: InputStreamSettings, val stdoutSettings: OutputStreamSettings,
|
||||
val stderrSettings: OutputStreamSettings, val redirectErrorStream: Boolean)
|
||||
|
||||
/**
|
||||
* Settings that define text to write to the process stdin.
|
||||
* @param input String to write to stdin.
|
||||
* @param inputFile File to write to stdin.
|
||||
*/
|
||||
class InputStreamSettings(val input: String, val inputFile: File)
|
||||
|
||||
/**
|
||||
* Settings that define text to capture from a process stream.
|
||||
* @param stringSize The number of characters to capture, or -1 for unlimited.
|
||||
* @param outputFile The file to write output to, or null to skip output.
|
||||
* @param outputFileAppend true if the output file should be appended to.
|
||||
*/
|
||||
class OutputStreamSettings(val stringSize: Int, val outputFile: File, val outputFileAppend: Boolean)
|
||||
|
||||
/**
|
||||
* The output of a process.
|
||||
* @param exitValue The exit value.
|
||||
* @param stdout The capture of stdout as defined by the stdout OutputStreamSettings.
|
||||
* @param stderr The capture of stderr as defined by the stderr OutputStreamSettings.
|
||||
*/
|
||||
class ProcessOutput(val exitValue: Int, val stdout: StreamOutput, val stderr: StreamOutput)
|
||||
|
||||
/**
|
||||
* The base class of stream output.
|
||||
*/
|
||||
abstract class StreamOutput {
|
||||
/**
|
||||
* Returns the content as a string.
|
||||
* @return The content as a string.
|
||||
*/
|
||||
def content: String
|
||||
|
||||
/**
|
||||
* Returns true if the content was truncated.
|
||||
* @return true if the content was truncated.
|
||||
*/
|
||||
def contentTruncated: Boolean
|
||||
}
|
||||
|
||||
private var currentCaptureId = 0
|
||||
/**
|
||||
* Returns the next output capture id.
|
||||
* @return The next output capture id.
|
||||
*/
|
||||
private def NEXT_OUTPUT_CAPTURE_ID = {
|
||||
currentCaptureId += 1
|
||||
currentCaptureId
|
||||
}
|
||||
private val STDOUT_KEY = "stdout"
|
||||
private val STDERR_KEY = "stderr"
|
||||
|
||||
/** Tracks running processes so that they can be killed as the JVM shuts down. */
|
||||
private val running = new HashSet[Process]
|
||||
|
||||
def shutdown() = {
|
||||
for (process <- running.clone) {
|
||||
logger.warn("Killing: " + process)
|
||||
try {
|
||||
process.destroy
|
||||
} catch {
|
||||
case _ => /* ignore */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Empty stream settings used when no output is requested. */
|
||||
private object EmptyStreamSettings extends OutputStreamSettings(0, null, false)
|
||||
|
||||
/** Empty stream output when no output is captured due to an error. */
|
||||
private object EmptyStreamOutput extends StreamOutput {
|
||||
def content = ""
|
||||
def contentTruncated = false
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream output captured from a stream.
|
||||
* @param stream Stream to capture output.
|
||||
* @param settings Settings that define what to capture.
|
||||
*/
|
||||
private class CapturedStreamOutput(val stream: InputStream, val settings: OutputStreamSettings, val debugStream: PrintStream) extends StreamOutput {
|
||||
/**
|
||||
* Returns the captured content as a string.
|
||||
* @return The captured content as a string.
|
||||
*/
|
||||
def content = stringWriter.toString()
|
||||
|
||||
/**
|
||||
* Returns true if the captured content was truncated.
|
||||
* @return true if the captured content was truncated.
|
||||
*/
|
||||
def contentTruncated = stringTruncated
|
||||
|
||||
/**
|
||||
* Drain the input stream to keep the process from backing up until it's empty.
|
||||
*/
|
||||
def read() = {
|
||||
val reader = new InputStreamReader(stream)
|
||||
val buf = new Array[Char](4096)
|
||||
var readCount = 0
|
||||
while ({readCount = reader.read(buf); readCount} >= 0) {
|
||||
writeString(buf, readCount)
|
||||
writeFile(buf, readCount)
|
||||
}
|
||||
closeFile()
|
||||
stream.close()
|
||||
}
|
||||
|
||||
/** The string to write capture content. */
|
||||
private lazy val stringWriter = if (settings.stringSize < 0) new StringWriter else new StringWriter(settings.stringSize)
|
||||
|
||||
/** True if the content is truncated. */
|
||||
private var stringTruncated = false
|
||||
|
||||
/** The number of characters left until the buffer is full. */
|
||||
private var stringRemaining = settings.stringSize
|
||||
|
||||
/**
|
||||
* Writes the buffer to the stringWriter up to stringRemaining characters.
|
||||
* @param chars Character buffer to write.
|
||||
* @param len Number of characters in the buffer.
|
||||
*/
|
||||
private def writeString(chars: Array[Char], len: Int) = {
|
||||
// If debug is enabled bypass the logger and dump directly to the screen
|
||||
if (logger.isDebugEnabled)
|
||||
debugStream.print(new String(chars, 0, len))
|
||||
if (settings.stringSize < 0) {
|
||||
stringWriter.write(chars, 0, len)
|
||||
} else {
|
||||
if (!stringTruncated) {
|
||||
stringWriter.write(chars, 0, if (len > stringRemaining) stringRemaining else len)
|
||||
stringRemaining -= len
|
||||
if (stringRemaining < 0)
|
||||
stringTruncated = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The file writer to capture content or null if no output file was requested. */
|
||||
private lazy val fileWriter = {
|
||||
if (settings.outputFile == null) {
|
||||
null
|
||||
} else {
|
||||
new FileWriter(settings.outputFile, settings.outputFileAppend)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the buffer to the fileWriter if it is not null.
|
||||
* @param chars Character buffer to write.
|
||||
* @param len Number of characters in the buffer.
|
||||
*/
|
||||
private def writeFile(chars: Array[Char], len: Int) = {
|
||||
if (fileWriter != null) {
|
||||
fileWriter.write(chars, 0, len)
|
||||
fileWriter.flush()
|
||||
}
|
||||
}
|
||||
|
||||
/** Closes the fileWriter if it is not null. */
|
||||
private def closeFile() = {
|
||||
if (fileWriter != null) {
|
||||
fileWriter.flush
|
||||
fileWriter.close
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -23,15 +23,15 @@
|
|||
*/
|
||||
|
||||
package org.broadinstitute.sting.queue.util
|
||||
|
||||
import org.broadinstitute.sting.queue.function.QFunction
|
||||
import org.broadinstitute.sting.gatk.report.{GATKReportTable, GATKReport}
|
||||
import org.broadinstitute.sting.utils.exceptions.UserException
|
||||
import org.broadinstitute.sting.queue.engine.JobRunInfo
|
||||
import java.io.{FileOutputStream, PrintStream, File}
|
||||
import org.broadinstitute.sting.queue.function.scattergather.{GathererFunction, ScatterFunction}
|
||||
import org.broadinstitute.sting.utils.R.RScriptExecutor.RScriptArgumentCollection
|
||||
import org.broadinstitute.sting.utils.R.RScriptExecutor
|
||||
import org.broadinstitute.sting.queue.QScript
|
||||
import org.broadinstitute.sting.utils.R.{RScriptLibrary, RScriptExecutor}
|
||||
import org.broadinstitute.sting.utils.io.Resource
|
||||
|
||||
/**
|
||||
* A mixin to add Job info to the class
|
||||
|
|
@ -104,10 +104,12 @@ object QJobReport {
|
|||
stream.close()
|
||||
}
|
||||
|
||||
def plotReport(args: RScriptArgumentCollection, jobReportFile: File) {
|
||||
def plotReport(args: RScriptArgumentCollection, reportFile: File, pdfFile: File) {
|
||||
val executor = new RScriptExecutor(args, false) // don't except on error
|
||||
val pdf = jobReportFile.getAbsolutePath + ".pdf"
|
||||
executor.callRScripts(JOB_REPORT_QUEUE_SCRIPT, jobReportFile.getAbsolutePath, pdf)
|
||||
executor.addLibrary(RScriptLibrary.GSALIB)
|
||||
executor.addScript(new Resource(JOB_REPORT_QUEUE_SCRIPT, classOf[QJobReport]))
|
||||
executor.addArgs(reportFile.getAbsolutePath, pdfFile.getAbsolutePath)
|
||||
executor.exec()
|
||||
}
|
||||
|
||||
def workAroundSameJobNames(func: QFunction):String = {
|
||||
|
|
|
|||
|
|
@ -1,27 +0,0 @@
|
|||
package org.broadinstitute.sting.queue.util
|
||||
|
||||
/**
|
||||
* Runs a job on the command line by invoking "sh -c <command>"
|
||||
*/
|
||||
class ShellJob extends CommandLineJob with Logging {
|
||||
/**
|
||||
* Runs the command and waits for the output.
|
||||
*/
|
||||
def run() = {
|
||||
val (redirectError, errorFile) = if (this.errorFile == null) (true, null) else (false, this.errorFile)
|
||||
val bufferSize = if (redirectError || logger.isDebugEnabled) FIVE_MB else 0
|
||||
val stdinSettings = new ProcessController.InputStreamSettings(null, this.inputFile)
|
||||
val stdoutSettings = new ProcessController.OutputStreamSettings(bufferSize, this.outputFile, true)
|
||||
val stderrSettings = new ProcessController.OutputStreamSettings(FIVE_MB, errorFile, true)
|
||||
val commandLine = Array("sh", shellScript.toString)
|
||||
val processSettings = new ProcessController.ProcessSettings(
|
||||
commandLine, null, this.workingDir, stdinSettings, stdoutSettings, stderrSettings, redirectError)
|
||||
|
||||
val output = processController.exec(processSettings)
|
||||
|
||||
if (output.exitValue != 0) {
|
||||
val streamOutput = if (redirectError) output.stdout else output.stderr
|
||||
throw new JobExitException("Failed to run job.", commandLine, output.exitValue, content(streamOutput))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -33,7 +33,7 @@ import java.text.SimpleDateFormat
|
|||
import org.broadinstitute.sting.BaseTest
|
||||
import org.broadinstitute.sting.MD5DB
|
||||
import org.broadinstitute.sting.queue.QCommandLine
|
||||
import org.broadinstitute.sting.queue.util.{Logging, ProcessController}
|
||||
import org.broadinstitute.sting.queue.util.Logging
|
||||
import java.io.File
|
||||
import org.broadinstitute.sting.gatk.report.GATKReport
|
||||
import org.apache.commons.io.FileUtils
|
||||
|
|
@ -217,11 +217,6 @@ object PipelineTest extends BaseTest with Logging {
|
|||
Runtime.getRuntime.addShutdownHook(new Thread {
|
||||
/** Cleanup as the JVM shuts down. */
|
||||
override def run() {
|
||||
try {
|
||||
ProcessController.shutdown()
|
||||
} catch {
|
||||
case _ => /*ignore */
|
||||
}
|
||||
runningCommandLines.foreach(commandLine =>
|
||||
try {
|
||||
commandLine.shutdown()
|
||||
|
|
|
|||
|
|
@ -1,122 +0,0 @@
|
|||
package org.broadinstitute.sting.queue.util
|
||||
|
||||
import org.broadinstitute.sting.BaseTest
|
||||
import java.io.File
|
||||
import org.broadinstitute.sting.utils.exceptions.UserException
|
||||
import org.testng.Assert
|
||||
import org.testng.annotations.Test
|
||||
|
||||
class IOUtilsUnitTest extends BaseTest {
|
||||
@Test
|
||||
def testGoodTempDir = {
|
||||
IOUtils.checkTempDir(new File("/tmp/queue"))
|
||||
}
|
||||
|
||||
@Test(expectedExceptions=Array(classOf[UserException.BadTmpDir]))
|
||||
def testBadTempDir = {
|
||||
IOUtils.checkTempDir(new File("/tmp"))
|
||||
}
|
||||
|
||||
@Test
|
||||
def testAbsoluteSubDir = {
|
||||
var subDir = IOUtils.absolute(new File("."), new File("/path/to/file"))
|
||||
Assert.assertEquals(subDir, new File("/path/to/file"))
|
||||
|
||||
subDir = IOUtils.absolute(new File("/different/path"), new File("/path/to/file"))
|
||||
Assert.assertEquals(subDir, new File("/path/to/file"))
|
||||
|
||||
subDir = IOUtils.absolute(new File("/different/path"), new File("."))
|
||||
Assert.assertEquals(subDir, new File("/different/path"))
|
||||
}
|
||||
|
||||
@Test
|
||||
def testRelativeSubDir = {
|
||||
var subDir = IOUtils.absolute(new File("."), new File("path/to/file"))
|
||||
Assert.assertEquals(subDir.getCanonicalFile, new File("path/to/file").getCanonicalFile)
|
||||
|
||||
subDir = IOUtils.absolute(new File("/different/path"), new File("path/to/file"))
|
||||
Assert.assertEquals(subDir, new File("/different/path/path/to/file"))
|
||||
}
|
||||
|
||||
@Test
|
||||
def testDottedSubDir = {
|
||||
var subDir = IOUtils.absolute(new File("."), new File("path/../to/file"))
|
||||
Assert.assertEquals(subDir.getCanonicalFile, new File("path/../to/./file").getCanonicalFile)
|
||||
|
||||
subDir = IOUtils.absolute(new File("."), new File("/path/../to/file"))
|
||||
Assert.assertEquals(subDir, new File("/path/../to/file"))
|
||||
|
||||
subDir = IOUtils.absolute(new File("/different/../path"), new File("path/to/file"))
|
||||
Assert.assertEquals(subDir, new File("/different/../path/path/to/file"))
|
||||
|
||||
subDir = IOUtils.absolute(new File("/different/./path"), new File("/path/../to/file"))
|
||||
Assert.assertEquals(subDir, new File("/path/../to/file"))
|
||||
}
|
||||
|
||||
@Test
|
||||
def testTempDir = {
|
||||
val tempDir = IOUtils.tempDir("Q-Unit-Test", "", new File("queueTempDirToDelete"))
|
||||
Assert.assertTrue(tempDir.exists)
|
||||
Assert.assertFalse(tempDir.isFile)
|
||||
Assert.assertTrue(tempDir.isDirectory)
|
||||
val deleted = IOUtils.tryDelete(tempDir)
|
||||
Assert.assertTrue(deleted)
|
||||
Assert.assertFalse(tempDir.exists)
|
||||
}
|
||||
|
||||
@Test
|
||||
def testDirLevel = {
|
||||
var dir = IOUtils.dirLevel(new File("/path/to/directory"), 1)
|
||||
Assert.assertEquals(dir, new File("/path"))
|
||||
|
||||
dir = IOUtils.dirLevel(new File("/path/to/directory"), 2)
|
||||
Assert.assertEquals(dir, new File("/path/to"))
|
||||
|
||||
dir = IOUtils.dirLevel(new File("/path/to/directory"), 3)
|
||||
Assert.assertEquals(dir, new File("/path/to/directory"))
|
||||
|
||||
dir = IOUtils.dirLevel(new File("/path/to/directory"), 4)
|
||||
Assert.assertEquals(dir, new File("/path/to/directory"))
|
||||
}
|
||||
|
||||
@Test
|
||||
def testAbsolute = {
|
||||
var dir = IOUtils.absolute(new File("/path/./to/./directory/."))
|
||||
Assert.assertEquals(dir, new File("/path/to/directory"))
|
||||
|
||||
dir = IOUtils.absolute(new File("/"))
|
||||
Assert.assertEquals(dir, new File("/"))
|
||||
|
||||
dir = IOUtils.absolute(new File("/."))
|
||||
Assert.assertEquals(dir, new File("/"))
|
||||
|
||||
dir = IOUtils.absolute(new File("/././."))
|
||||
Assert.assertEquals(dir, new File("/"))
|
||||
|
||||
dir = IOUtils.absolute(new File("/./directory/."))
|
||||
Assert.assertEquals(dir, new File("/directory"))
|
||||
|
||||
dir = IOUtils.absolute(new File("/./directory/./"))
|
||||
Assert.assertEquals(dir, new File("/directory"))
|
||||
|
||||
dir = IOUtils.absolute(new File("/./directory./"))
|
||||
Assert.assertEquals(dir, new File("/directory."))
|
||||
|
||||
dir = IOUtils.absolute(new File("/./.directory/"))
|
||||
Assert.assertEquals(dir, new File("/.directory"))
|
||||
}
|
||||
|
||||
@Test
|
||||
def testTail = {
|
||||
val lines = List(
|
||||
"chr18_random 4262 3154410390 50 51",
|
||||
"chr19_random 301858 3154414752 50 51",
|
||||
"chr21_random 1679693 3154722662 50 51",
|
||||
"chr22_random 257318 3156435963 50 51",
|
||||
"chrX_random 1719168 3156698441 50 51")
|
||||
val tail = IOUtils.tail(new File(BaseTest.hg18Reference + ".fai"), 5)
|
||||
Assert.assertEquals(tail.size, 5)
|
||||
for (i <- 0 until 5)
|
||||
Assert.assertEquals(tail(i), lines(i))
|
||||
}
|
||||
}
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
package org.broadinstitute.sting.queue.util
|
||||
|
||||
import org.broadinstitute.sting.BaseTest
|
||||
import org.testng.annotations.Test
|
||||
import org.testng.Assert
|
||||
|
||||
class ShellJobUnitTest {
|
||||
@Test
|
||||
def testEcho {
|
||||
val job = new ShellJob
|
||||
job.shellScript = writeScript("echo Hello World")
|
||||
job.run()
|
||||
}
|
||||
|
||||
@Test(expectedExceptions=Array(classOf[JobExitException]))
|
||||
def testBadQuotes {
|
||||
val job = new ShellJob
|
||||
job.shellScript = writeScript("echo 'Hello World")
|
||||
job.run()
|
||||
}
|
||||
|
||||
@Test
|
||||
def testGoodQuotes {
|
||||
val job = new ShellJob
|
||||
job.shellScript = writeScript("echo 'Hello World'")
|
||||
job.run()
|
||||
}
|
||||
|
||||
@Test
|
||||
def testEscapeCharacters {
|
||||
var job: ShellJob = null
|
||||
|
||||
job = new ShellJob
|
||||
job.shellScript = writeScript("echo #")
|
||||
job.outputFile = BaseTest.createTempFile("temp", "")
|
||||
job.run()
|
||||
Assert.assertEquals(IOUtils.readContents(job.outputFile).trim, "")
|
||||
|
||||
job = new ShellJob
|
||||
job.shellScript = writeScript("""echo \#""")
|
||||
job.outputFile = BaseTest.createTempFile("temp", "")
|
||||
job.run()
|
||||
Assert.assertEquals(IOUtils.readContents(job.outputFile).trim, "#")
|
||||
|
||||
job = new ShellJob
|
||||
job.shellScript = writeScript("""echo \\#""")
|
||||
job.outputFile = BaseTest.createTempFile("temp", "")
|
||||
job.run()
|
||||
Assert.assertEquals(IOUtils.readContents(job.outputFile).trim, """\#""")
|
||||
}
|
||||
|
||||
@Test
|
||||
def testLongCommand {
|
||||
// This command fails on some systems with a 4096 character limit when run via the old sh -c "echo ...",
|
||||
// but works on the same systems when run via sh <script>
|
||||
val builder = new StringBuilder
|
||||
builder.append("echo ")
|
||||
for (i <- 1 to 500) {
|
||||
val s = i.toString
|
||||
builder.append("000".take(3-s.length)).append(s).append("______ ")
|
||||
}
|
||||
|
||||
val job = new ShellJob
|
||||
job.shellScript = writeScript(builder.toString)
|
||||
job.run()
|
||||
}
|
||||
|
||||
private def writeScript(contents: String) = {
|
||||
val file = BaseTest.createTempFile("temp", "")
|
||||
IOUtils.writeContents(file, contents)
|
||||
file
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue