diff --git a/build.xml b/build.xml index 446982a44..6ca959c38 100644 --- a/build.xml +++ b/build.xml @@ -28,6 +28,8 @@ + + @@ -35,18 +37,25 @@ + + + + + + - + - + @@ -60,7 +69,7 @@ - + @@ -82,7 +91,7 @@ - + @@ -113,7 +122,7 @@ - + @@ -154,7 +163,7 @@ - + @@ -211,11 +220,11 @@ - + - + @@ -224,11 +233,11 @@ - + - + @@ -266,7 +275,7 @@ - + @@ -312,13 +321,13 @@ - + - + @@ -327,11 +336,11 @@ - + - @@ -341,9 +350,9 @@ - + - + @@ -362,14 +371,14 @@ - + - + - - + @@ -413,9 +422,9 @@ - + - + @@ -424,12 +433,12 @@ - + - + @@ -532,6 +541,11 @@ + + + + + @@ -539,7 +553,7 @@ - + @@ -551,6 +565,12 @@ + + + + + + @@ -579,6 +599,10 @@ + + + + @@ -593,6 +617,10 @@ + + + + @@ -605,28 +633,7 @@ - @@ -643,6 +650,9 @@ + + + @@ -682,20 +692,7 @@ - + @@ -780,10 +777,6 @@ - @@ -800,10 +793,6 @@ - @@ -851,6 +840,8 @@ + + @@ -1187,19 +1178,18 @@ - - + - + - + diff --git a/public/R/queueJobReport.R b/public/R/scripts/org/broadinstitute/sting/queue/util/queueJobReport.R similarity index 100% rename from public/R/queueJobReport.R rename to public/R/scripts/org/broadinstitute/sting/queue/util/queueJobReport.R diff --git a/public/R/src/gsalib/DESCRIPTION b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/DESCRIPTION similarity index 100% rename from public/R/src/gsalib/DESCRIPTION rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/DESCRIPTION diff --git a/public/R/src/gsalib/R/gsa.error.R b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.error.R similarity index 100% rename from public/R/src/gsalib/R/gsa.error.R rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.error.R diff --git a/public/R/src/gsalib/R/gsa.getargs.R b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.getargs.R similarity index 100% rename from public/R/src/gsalib/R/gsa.getargs.R rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.getargs.R diff --git a/public/R/src/gsalib/R/gsa.message.R b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.message.R similarity index 100% rename from public/R/src/gsalib/R/gsa.message.R rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.message.R diff --git a/public/R/src/gsalib/R/gsa.plot.venn.R b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.plot.venn.R similarity index 100% rename from public/R/src/gsalib/R/gsa.plot.venn.R rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.plot.venn.R diff --git a/public/R/src/gsalib/R/gsa.read.eval.R b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.read.eval.R similarity index 100% rename from public/R/src/gsalib/R/gsa.read.eval.R rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.read.eval.R diff --git a/public/R/src/gsalib/R/gsa.read.gatkreport.R b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.read.gatkreport.R similarity index 100% rename from public/R/src/gsalib/R/gsa.read.gatkreport.R rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.read.gatkreport.R diff --git a/public/R/src/gsalib/R/gsa.read.squidmetrics.R b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.read.squidmetrics.R similarity index 100% rename from public/R/src/gsalib/R/gsa.read.squidmetrics.R rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.read.squidmetrics.R diff --git a/public/R/src/gsalib/R/gsa.read.vcf.R b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.read.vcf.R similarity index 100% rename from public/R/src/gsalib/R/gsa.read.vcf.R rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.read.vcf.R diff --git a/public/R/src/gsalib/R/gsa.warn.R b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.warn.R similarity index 100% rename from public/R/src/gsalib/R/gsa.warn.R rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.warn.R diff --git a/public/R/src/gsalib/Read-and-delete-me b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/Read-and-delete-me similarity index 100% rename from public/R/src/gsalib/Read-and-delete-me rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/Read-and-delete-me diff --git a/public/R/src/gsalib/data/tearsheetdrop.jpg b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/data/tearsheetdrop.jpg similarity index 100% rename from public/R/src/gsalib/data/tearsheetdrop.jpg rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/data/tearsheetdrop.jpg diff --git a/public/R/src/gsalib/man/gsa.error.Rd b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsa.error.Rd similarity index 100% rename from public/R/src/gsalib/man/gsa.error.Rd rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsa.error.Rd diff --git a/public/R/src/gsalib/man/gsa.getargs.Rd b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsa.getargs.Rd similarity index 100% rename from public/R/src/gsalib/man/gsa.getargs.Rd rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsa.getargs.Rd diff --git a/public/R/src/gsalib/man/gsa.message.Rd b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsa.message.Rd similarity index 100% rename from public/R/src/gsalib/man/gsa.message.Rd rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsa.message.Rd diff --git a/public/R/src/gsalib/man/gsa.plot.venn.Rd b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsa.plot.venn.Rd similarity index 100% rename from public/R/src/gsalib/man/gsa.plot.venn.Rd rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsa.plot.venn.Rd diff --git a/public/R/src/gsalib/man/gsa.read.eval.Rd b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsa.read.eval.Rd similarity index 100% rename from public/R/src/gsalib/man/gsa.read.eval.Rd rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsa.read.eval.Rd diff --git a/public/R/src/gsalib/man/gsa.read.gatkreport.Rd b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsa.read.gatkreport.Rd similarity index 100% rename from public/R/src/gsalib/man/gsa.read.gatkreport.Rd rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsa.read.gatkreport.Rd diff --git a/public/R/src/gsalib/man/gsa.read.squidmetrics.Rd b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsa.read.squidmetrics.Rd similarity index 100% rename from public/R/src/gsalib/man/gsa.read.squidmetrics.Rd rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsa.read.squidmetrics.Rd diff --git a/public/R/src/gsalib/man/gsa.read.vcf.Rd b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsa.read.vcf.Rd similarity index 100% rename from public/R/src/gsalib/man/gsa.read.vcf.Rd rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsa.read.vcf.Rd diff --git a/public/R/src/gsalib/man/gsa.warn.Rd b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsa.warn.Rd similarity index 100% rename from public/R/src/gsalib/man/gsa.warn.Rd rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsa.warn.Rd diff --git a/public/R/src/gsalib/man/gsalib-package.Rd b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsalib-package.Rd similarity index 100% rename from public/R/src/gsalib/man/gsalib-package.Rd rename to public/R/src/org/broadinstitute/sting/utils/R/gsalib/man/gsalib-package.Rd diff --git a/public/java/src/org/broadinstitute/sting/utils/R/RScriptExecutor.java b/public/java/src/org/broadinstitute/sting/utils/R/RScriptExecutor.java index 58f7942fe..9180447b9 100644 --- a/public/java/src/org/broadinstitute/sting/utils/R/RScriptExecutor.java +++ b/public/java/src/org/broadinstitute/sting/utils/R/RScriptExecutor.java @@ -25,35 +25,35 @@ package org.broadinstitute.sting.utils.R; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.broadinstitute.sting.commandline.Advanced; import org.broadinstitute.sting.commandline.Argument; -import org.broadinstitute.sting.commandline.ArgumentCollection; -import org.broadinstitute.sting.gatk.walkers.recalibration.Covariate; -import org.broadinstitute.sting.utils.PathUtils; import org.broadinstitute.sting.utils.Utils; +import org.broadinstitute.sting.utils.exceptions.StingException; import org.broadinstitute.sting.utils.exceptions.UserException; +import org.broadinstitute.sting.utils.io.IOUtils; +import org.broadinstitute.sting.utils.io.Resource; +import org.broadinstitute.sting.utils.runtime.ProcessController; +import org.broadinstitute.sting.utils.runtime.ProcessSettings; import java.io.File; -import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; /** - * Generic service for executing RScripts in the GATK directory - * - * @author Your Name - * @since Date created + * Generic service for executing RScripts */ public class RScriptExecutor { /** * our log */ - protected static Logger logger = Logger.getLogger(RScriptExecutor.class); + private static Logger logger = Logger.getLogger(RScriptExecutor.class); public static class RScriptArgumentCollection { @Advanced - @Argument(fullName = "path_to_Rscript", shortName = "Rscript", doc = "The path to your implementation of Rscript. For Broad users this is maybe /broad/software/free/Linux/redhat_5_x86_64/pkgs/r_2.12.0/bin/Rscript", required = false) + @Argument(fullName = "path_to_Rscript", shortName = "Rscript", doc = "The path to your implementation of Rscript. Defaults Rscript meaning to use the first available on the environment PATH. For Broad users should 'use R-2.12' or later.", required = false) public String PATH_TO_RSCRIPT = "Rscript"; @Advanced @@ -62,40 +62,119 @@ public class RScriptExecutor { public RScriptArgumentCollection() {} - /** For testing and convenience */ + /* For testing and convenience */ public RScriptArgumentCollection(final String PATH_TO_RSCRIPT, final List PATH_TO_RESOURCES) { this.PATH_TO_RSCRIPT = PATH_TO_RSCRIPT; this.PATH_TO_RESOURCES = PATH_TO_RESOURCES; } } - final RScriptArgumentCollection myArgs; - final boolean exceptOnError; + private final RScriptArgumentCollection myArgs; + private final boolean exceptOnError; + private final List libraries = new ArrayList(); + private final List scriptResources = new ArrayList(); + private final List scriptFiles = new ArrayList(); + private final List args = new ArrayList(); public RScriptExecutor(final RScriptArgumentCollection myArgs, final boolean exceptOnError) { this.myArgs = myArgs; this.exceptOnError = exceptOnError; } - public void callRScripts(String scriptName, Object... scriptArgs) { - callRScripts(scriptName, Arrays.asList(scriptArgs)); + public void addLibrary(RScriptLibrary library) { + this.libraries.add(library); } - public void callRScripts(String scriptName, List scriptArgs) { + public void addScript(Resource script) { + this.scriptResources.add(script); + } + + public void addScript(File script) { + this.scriptFiles.add(script); + } + + /** + * Adds args to the end of the Rscript command line. + * @param args the args. + * @throws NullPointerException if any of the args are null. + */ + public void addArgs(Object... args) { + for (Object arg: args) + this.args.add(arg.toString()); + } + + public void exec() { + List tempFiles = new ArrayList(); try { - final File pathToScript = findScript(scriptName); - if ( pathToScript == null ) return; // we failed but shouldn't exception out - final String argString = Utils.join(" ", scriptArgs); - final String cmdLine = Utils.join(" ", Arrays.asList(myArgs.PATH_TO_RSCRIPT, pathToScript, argString)); - logger.info("Executing RScript: " + cmdLine); - Runtime.getRuntime().exec(cmdLine).waitFor(); - } catch (InterruptedException e) { + File tempLibDir = IOUtils.tempDir("R.", ".lib"); + tempFiles.add(tempLibDir); + + StringBuilder expression = new StringBuilder("tempLibDir = '").append(tempLibDir).append("';"); + + if (this.libraries.size() > 0) { + List tempLibraryPaths = new ArrayList(); + for (RScriptLibrary library: this.libraries) { + File tempLibrary = library.writeTemp(); + tempFiles.add(tempLibrary); + tempLibraryPaths.add(tempLibrary.getAbsolutePath()); + } + + expression.append("install.packages("); + expression.append("pkgs=c('").append(StringUtils.join(tempLibraryPaths, "', '")).append("'), lib=tempLibDir, repos=NULL, type='source', "); + // Install faster by eliminating cruft. + expression.append("INSTALL_opts=c('--no-libs', '--no-data', '--no-help', '--no-demo', '--no-exec')"); + expression.append(");"); + + for (RScriptLibrary library: this.libraries) { + expression.append("require('").append(library.getLibraryName()).append("', lib.loc=tempLibDir);"); + } + } + + for (Resource script: this.scriptResources) { + File tempScript = IOUtils.writeTempResource(script); + tempFiles.add(tempScript); + expression.append("source('").append(tempScript.getAbsolutePath()).append("');"); + } + + for (File script: this.scriptFiles) { + expression.append("source('").append(script.getAbsolutePath()).append("');"); + } + + String[] cmd = new String[this.args.size() + 3]; + int i = 0; + cmd[i++] = myArgs.PATH_TO_RSCRIPT; + cmd[i++] = "-e"; + cmd[i++] = expression.toString(); + for (String arg: this.args) + cmd[i++] = arg; + + ProcessSettings processSettings = new ProcessSettings(cmd); + if (logger.isDebugEnabled()) { + processSettings.getStdoutSettings().printStandard(true); + processSettings.getStderrSettings().printStandard(true); + } + + ProcessController controller = ProcessController.getThreadLocal(); + + logger.debug("Executing: " + Utils.join(" ", cmd)); + logger.debug("Result: " + controller.exec(processSettings).getExitValue()); + + } catch (StingException e) { generateException(e); - } catch (IOException e) { - generateException("Fatal Exception: Perhaps RScript jobs are being spawned too quickly?", e); + } finally { + for (File temp: tempFiles) + FileUtils.deleteQuietly(temp); } } + public void callRScripts(String scriptName, Object... scriptArgs) { + final File pathToScript = findScript(scriptName); + if (pathToScript == null) return; // we failed but shouldn't exception out + addScript(pathToScript); + addArgs(scriptArgs); + exec(); + } + public File findScript(final String scriptName) { for ( String pathToResource : myArgs.PATH_TO_RESOURCES ) { final File f = new File(pathToResource + "/" + scriptName); diff --git a/public/java/src/org/broadinstitute/sting/utils/R/RScriptLibrary.java b/public/java/src/org/broadinstitute/sting/utils/R/RScriptLibrary.java new file mode 100644 index 000000000..60cd7504b --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/R/RScriptLibrary.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.R; + +import org.broadinstitute.sting.utils.io.IOUtils; +import org.broadinstitute.sting.utils.io.Resource; + +import java.io.File; + +/** + * Libraries embedded in the StingUtils package. + */ +public enum RScriptLibrary { + GSALIB("gsalib"); + + private final String name; + + private RScriptLibrary(String name) { + this.name = name; + } + + public String getLibraryName() { + return this.name; + } + + public String getResourcePath() { + return name + ".tar.gz"; + } + + /** + * Writes the library source code to a temporary tar.gz file and returns the path. + * @return The path to the library source code. The caller must delete the code when done. + */ + public File writeTemp() { + return IOUtils.writeTempResource(new Resource(getResourcePath(), RScriptLibrary.class)); + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/io/FileExtension.java b/public/java/src/org/broadinstitute/sting/utils/io/FileExtension.java new file mode 100644 index 000000000..cd69ee126 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/io/FileExtension.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.io; + +import java.io.File; + +public interface FileExtension { + /** + * Returns a clone of the FileExtension with a new path. + * @param path New path. + * @return New FileExtension + */ + public File withPath(String path); +} diff --git a/public/java/src/org/broadinstitute/sting/utils/io/HardThresholdingOutputStream.java b/public/java/src/org/broadinstitute/sting/utils/io/HardThresholdingOutputStream.java new file mode 100755 index 000000000..26b5ae6fd --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/io/HardThresholdingOutputStream.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +package org.broadinstitute.sting.utils.io; + +import org.apache.commons.io.output.ThresholdingOutputStream; + +import java.io.IOException; + +/** + * An output stream which stops at the threshold + * instead of potentially triggering early. + */ +public abstract class HardThresholdingOutputStream extends ThresholdingOutputStream { + protected HardThresholdingOutputStream(int threshold) { + super(threshold); + } + + @Override + public void write(byte[] b) throws IOException { + write(b, 0, b.length); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + int remaining = this.getThreshold() - (int)this.getByteCount(); + if (!isThresholdExceeded() && len > remaining) { + super.write(b, off, remaining); + super.write(b, off + remaining, len - remaining); + } else { + super.write(b, off, len); + } + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java b/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java new file mode 100644 index 000000000..7bfaa0194 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java @@ -0,0 +1,353 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.io; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.LineIterator; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import org.broadinstitute.sting.utils.exceptions.StingException; +import org.broadinstitute.sting.utils.exceptions.UserException; + +import java.io.*; +import java.util.*; + +public class IOUtils { + private static Logger logger = Logger.getLogger(IOUtils.class); + + /** + * Checks if the temp directory has been setup and throws an exception if they user hasn't set it correctly. + * + * @param tempDir Temporary directory. + */ + public static void checkTempDir(File tempDir) { + String tempDirPath = tempDir.getAbsolutePath(); + // Keeps the user from leaving the temp directory as the default, and on Macs from having pluses + // in the path which can cause problems with the Google Reflections library. + // see also: http://benjchristensen.com/2009/09/22/mac-osx-10-6-java-java-io-tmpdir/ + if (tempDirPath.startsWith("/var/folders/") || (tempDirPath.equals("/tmp")) || (tempDirPath.equals("/tmp/"))) + throw new UserException.BadTmpDir("java.io.tmpdir must be explicitly set"); + if (!tempDir.exists() && !tempDir.mkdirs()) + throw new UserException.BadTmpDir("Could not create directory: " + tempDir.getAbsolutePath()); + } + + /** + * Creates a temp directory with the prefix and optional suffix. + * + * @param prefix Prefix for the directory name. + * @param suffix Optional suffix for the directory name. + * @return The created temporary directory. + */ + public static File tempDir(String prefix, String suffix) { + return tempDir(prefix, suffix, null); + } + + /** + * Creates a temp directory with the prefix and optional suffix. + * + * @param prefix Prefix for the directory name. + * @param suffix Optional suffix for the directory name. + * @param tempDirParent Parent directory for the temp directory. + * @return The created temporary directory. + */ + public static File tempDir(String prefix, String suffix, File tempDirParent) { + try { + if (tempDirParent == null) + tempDirParent = FileUtils.getTempDirectory(); + if (!tempDirParent.exists() && !tempDirParent.mkdirs()) + throw new UserException.BadTmpDir("Could not create temp directory: " + tempDirParent); + File temp = File.createTempFile(prefix + "-", suffix, tempDirParent); + if (!temp.delete()) + throw new UserException.BadTmpDir("Could not delete sub file: " + temp.getAbsolutePath()); + if (!temp.mkdir()) + throw new UserException.BadTmpDir("Could not create sub directory: " + temp.getAbsolutePath()); + return absolute(temp); + } catch (IOException e) { + throw new UserException.BadTmpDir(e.getMessage()); + } + } + + /** + * Writes content to a temp file and returns the path to the temporary file. + * + * @param content to write. + * @param prefix Prefix for the temp file. + * @param suffix Suffix for the temp file. + * @param directory Directory for the temp file. + * @return the path to the temp file. + */ + public static File writeTempFile(String content, String prefix, String suffix, File directory) { + try { + File tempFile = absolute(File.createTempFile(prefix, suffix, directory)); + FileUtils.writeStringToFile(tempFile, content); + return tempFile; + } catch (IOException e) { + throw new UserException.BadTmpDir(e.getMessage()); + } + } + + /** + * Waits for NFS to propagate a file creation, imposing a timeout. + * + * Based on Apache Commons IO FileUtils.waitFor() + * + * @param file The file to wait for. + * @param seconds The maximum time in seconds to wait. + * @return true if the file exists + */ + public static boolean waitFor(File file, int seconds) { + return waitFor(Collections.singletonList(file), seconds).isEmpty(); + } + + /** + * Waits for NFS to propagate a file creation, imposing a timeout. + * + * Based on Apache Commons IO FileUtils.waitFor() + * + * @param files The list of files to wait for. + * @param seconds The maximum time in seconds to wait. + * @return Files that still do not exists at the end of the timeout, or a empty list if all files exists. + */ + public static List waitFor(Collection files, int seconds) { + long timeout = 0; + long tick = 0; + List missingFiles = new ArrayList(); + for (File file : files) + if (!file.exists()) + missingFiles.add(file); + + while (!missingFiles.isEmpty() && timeout <= seconds) { + if (tick >= 10) { + tick = 0; + timeout++; + } + tick++; + try { + Thread.sleep(100); + } catch (InterruptedException ignore) { + } + List newMissingFiles = new ArrayList(); + for (File file : missingFiles) + if (!file.exists()) + newMissingFiles.add(file); + missingFiles = newMissingFiles; + } + return missingFiles; + } + + /** + * Returns the directory at the number of levels deep. + * For example 2 levels of /path/to/dir will return /path/to + * + * @param dir Directory path. + * @param level how many levels deep from the root. + * @return The path to the parent directory that is level-levels deep. + */ + public static File dirLevel(File dir, int level) { + List directories = new ArrayList(); + File parentDir = absolute(dir); + while (parentDir != null) { + directories.add(0, parentDir); + parentDir = parentDir.getParentFile(); + } + if (directories.size() <= level) + return directories.get(directories.size() - 1); + else + return directories.get(level); + } + + /** + * Returns the sub path rooted at the parent. + * + * @param parent The parent directory. + * @param path The sub path to append to the parent, if the path is not absolute. + * @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path. + */ + public static File absolute(File parent, String path) { + return absolute(parent, new File(path)); + } + + /** + * Returns the sub path rooted at the parent. + * + * @param parent The parent directory. + * @param file The sub path to append to the parent, if the path is not absolute. + * @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path. + */ + public static File absolute(File parent, File file) { + String newPath; + if (file.isAbsolute()) + newPath = absolutePath(file); + else + newPath = absolutePath(new File(parent, file.getPath())); + return replacePath(file, newPath); + } + + /** + * A mix of getCanonicalFile and getAbsoluteFile that returns the + * absolute path to the file without deferencing symbolic links. + * + * @param file the file. + * @return the absolute path to the file. + */ + public static File absolute(File file) { + return replacePath(file, absolutePath(file)); + } + + private static String absolutePath(File file) { + File fileAbs = file.getAbsoluteFile(); + LinkedList names = new LinkedList(); + while (fileAbs != null) { + String name = fileAbs.getName(); + fileAbs = fileAbs.getParentFile(); + + if (".".equals(name)) { + /* skip */ + + /* TODO: What do we do for ".."? + } else if (name == "..") { + + CentOS tcsh says use getCanonicalFile: + ~ $ mkdir -p test1/test2 + ~ $ ln -s test1/test2 test3 + ~ $ cd test3/.. + ~/test1 $ + + Mac bash says keep going with getAbsoluteFile: + ~ $ mkdir -p test1/test2 + ~ $ ln -s test1/test2 test3 + ~ $ cd test3/.. + ~ $ + + For now, leave it and let the shell figure it out. + */ + } else { + names.add(0, name); + } + } + + return ("/" + StringUtils.join(names, "/")); + } + + private static File replacePath(File file, String path) { + if (file instanceof FileExtension) + return ((FileExtension)file).withPath(path); + if (!File.class.equals(file.getClass())) + throw new StingException("Sub classes of java.io.File must also implement FileExtension"); + return new File(path); + } + + /** + * Returns the last lines of the file. + * NOTE: This is only safe to run on smaller files! + * + * @param file File to read. + * @param count Maximum number of lines to return. + * @return The last count lines from file. + * @throws IOException When unable to read the file. + */ + public static List tail(File file, int count) throws IOException { + LinkedList tailLines = new LinkedList(); + FileReader reader = new FileReader(file); + try { + LineIterator iterator = org.apache.commons.io.IOUtils.lineIterator(reader); + int lineCount = 0; + while (iterator.hasNext()) { + String line = iterator.nextLine(); + lineCount++; + if (lineCount > count) + tailLines.removeFirst(); + tailLines.offer(line); + } + } finally { + org.apache.commons.io.IOUtils.closeQuietly(reader); + } + return tailLines; + } + + /** + * Tries to delete a file. Emits a warning if the file was unable to be deleted. + * + * @param file File to delete. + * @return true if the file was deleted. + */ + public static boolean tryDelete(File file) { + boolean deleted = FileUtils.deleteQuietly(file); + if (deleted) + logger.debug("Deleted " + file); + else if (file.exists()) + logger.warn("Unable to delete " + file); + return deleted; + } + + /** + * Writes the an embedded resource to a temp file. + * File is not scheduled for deletion and must be cleaned up by the caller. + * @param resource Embedded resource. + * @return Path to the temp file with the contents of the resource. + */ + public static File writeTempResource(Resource resource) { + File temp; + try { + temp = File.createTempFile(FilenameUtils.getBaseName(resource.getPath()) + ".", "." + FilenameUtils.getExtension(resource.getPath())); + } catch (IOException e) { + throw new UserException.BadTmpDir(e.getMessage()); + } + writeResource(resource, temp); + return temp; + } + + /** + * Writes the an embedded resource to a file. + * File is not scheduled for deletion and must be cleaned up by the caller. + * @param resource Embedded resource. + * @param file File path to write. + */ + public static void writeResource(Resource resource, File file) { + String path = resource.getPath(); + Class clazz = resource.getRelativeClass(); + InputStream inputStream = null; + OutputStream outputStream = null; + try { + if (clazz == null) { + inputStream = ClassLoader.getSystemResourceAsStream(path); + if (inputStream == null) + throw new IllegalArgumentException("Resource not found: " + path); + } else { + inputStream = clazz.getResourceAsStream(path); + if (inputStream == null) + throw new IllegalArgumentException("Resource not found relative to " + clazz + ": " + path); + } + outputStream = FileUtils.openOutputStream(file); + org.apache.commons.io.IOUtils.copy(inputStream, outputStream); + } catch (IOException e) { + throw new StingException(String.format("Unable to copy resource '%s' to '%s'", path, file), e); + } finally { + org.apache.commons.io.IOUtils.closeQuietly(inputStream); + org.apache.commons.io.IOUtils.closeQuietly(outputStream); + } + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/io/Resource.java b/public/java/src/org/broadinstitute/sting/utils/io/Resource.java new file mode 100644 index 000000000..5473511b4 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/io/Resource.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.io; + +/** + * Stores a resource by path and a relative class. + */ +public class Resource { + private final String path; + private final Class relativeClass; + + /** + * Create a resource with a path and a relative class. + * @param path Relative or absolute path to the class. + * @param relativeClass Relative class to use as a class loader and for a relative package. + * + * If the relative class is null then the system classloader will be used and the path must be absolute. + */ + public Resource(String path, Class relativeClass) { + this.path = path; + this.relativeClass = relativeClass; + } + + public Class getRelativeClass() { + return relativeClass; + } + + public String getPath() { + return path; + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/runtime/CapturedStreamOutput.java b/public/java/src/org/broadinstitute/sting/utils/runtime/CapturedStreamOutput.java new file mode 100755 index 000000000..50622cef1 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/runtime/CapturedStreamOutput.java @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.runtime; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.output.NullOutputStream; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import org.broadinstitute.sting.utils.exceptions.UserException; +import org.broadinstitute.sting.utils.io.HardThresholdingOutputStream; + +import java.io.*; +import java.util.EnumMap; + +/** + * Stream output captured from a stream. + */ +public class CapturedStreamOutput extends StreamOutput { + private final InputStream processStream; + private final EnumMap outputStreams = new EnumMap(StreamLocation.class); + + /** + * The byte stream to capture content or null if no output string content was requested. + */ + private final ByteArrayOutputStream bufferStream; + + /** + * True if the buffer is truncated. + */ + private boolean bufferTruncated = false; + + /** + * @param settings Settings that define what to capture. + * @param processStream Stream to capture output. + * @param standardStream Stream to write debug output. + */ + public CapturedStreamOutput(OutputStreamSettings settings, InputStream processStream, PrintStream standardStream) { + this.processStream = processStream; + int bufferSize = settings.getBufferSize(); + this.bufferStream = (bufferSize < 0) ? new ByteArrayOutputStream() : new ByteArrayOutputStream(bufferSize); + + for (StreamLocation location : settings.getStreamLocations()) { + OutputStream outputStream; + switch (location) { + case Buffer: + if (bufferSize < 0) { + outputStream = this.bufferStream; + } else { + outputStream = new HardThresholdingOutputStream(bufferSize) { + @Override + protected OutputStream getStream() throws IOException { + return bufferTruncated ? NullOutputStream.NULL_OUTPUT_STREAM : bufferStream; + } + + @Override + protected void thresholdReached() throws IOException { + bufferTruncated = true; + } + }; + } + break; + case File: + try { + outputStream = new FileOutputStream(settings.getOutputFile(), settings.isAppendFile()); + } catch (IOException e) { + throw new UserException.BadInput(e.getMessage()); + } + break; + case Standard: + outputStream = standardStream; + break; + default: + throw new ReviewedStingException("Unexpected stream location: " + location); + } + this.outputStreams.put(location, outputStream); + } + } + + @Override + public byte[] getBufferBytes() { + return bufferStream.toByteArray(); + } + + @Override + public boolean isBufferTruncated() { + return bufferTruncated; + } + + /** + * Drain the input stream to keep the process from backing up until it's empty. + * File streams will be closed automatically when this method returns. + * + * @throws java.io.IOException When unable to read or write. + */ + public void readAndClose() throws IOException { + try { + byte[] buf = new byte[4096]; + int readCount; + while ((readCount = processStream.read(buf)) >= 0) + for (OutputStream outputStream : this.outputStreams.values()) { + outputStream.write(buf, 0, readCount); + } + } finally { + for (StreamLocation location : this.outputStreams.keySet()) { + OutputStream outputStream = this.outputStreams.get(location); + outputStream.flush(); + if (location != StreamLocation.Standard) + IOUtils.closeQuietly(outputStream); + } + } + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/runtime/InputStreamSettings.java b/public/java/src/org/broadinstitute/sting/utils/runtime/InputStreamSettings.java new file mode 100755 index 000000000..dfa380a68 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/runtime/InputStreamSettings.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.runtime; + +import java.io.File; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; + +/** + * Settings that define text to write to the process stdin. + */ +public class InputStreamSettings { + private final EnumSet streamLocations = EnumSet.noneOf(StreamLocation.class); + private byte[] inputBuffer; + private File inputFile; + + public InputStreamSettings() { + } + + /** + * @param inputBuffer String to write to stdin. + */ + public InputStreamSettings(String inputBuffer) { + setInputBuffer(inputBuffer); + } + + /** + * @param inputFile File to write to stdin. + */ + public InputStreamSettings(File inputFile) { + setInputFile(inputFile); + } + + /** + * @param inputBuffer String to write to stdin. + * @param inputFile File to write to stdin. + */ + public InputStreamSettings(byte[] inputBuffer, File inputFile) { + setInputBuffer(inputBuffer); + setInputFile(inputFile); + } + + public Set getStreamLocations() { + return Collections.unmodifiableSet(streamLocations); + } + + public byte[] getInputBuffer() { + return inputBuffer; + } + + public void setInputBuffer(String inputBuffer) { + if (inputBuffer == null) + throw new IllegalArgumentException("inputBuffer cannot be null"); + this.streamLocations.add(StreamLocation.Buffer); + this.inputBuffer = inputBuffer.getBytes(); + } + + public void setInputBuffer(byte[] inputBuffer) { + if (inputBuffer == null) + throw new IllegalArgumentException("inputBuffer cannot be null"); + this.streamLocations.add(StreamLocation.Buffer); + this.inputBuffer = inputBuffer; + } + + public void clearInputBuffer() { + this.streamLocations.remove(StreamLocation.Buffer); + this.inputBuffer = null; + } + + public File getInputFile() { + return inputFile; + } + + public void setInputFile(File inputFile) { + if (inputFile == null) + throw new IllegalArgumentException("inputFile cannot be null"); + this.streamLocations.add(StreamLocation.File); + this.inputFile = inputFile; + } + + public void clearInputFile() { + this.streamLocations.remove(StreamLocation.File); + this.inputFile = null; + } + + public void setInputStandard(boolean inputStandard) { + if (inputStandard) + this.streamLocations.add(StreamLocation.Standard); + else + this.streamLocations.remove(StreamLocation.Standard); + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/runtime/OutputStreamSettings.java b/public/java/src/org/broadinstitute/sting/utils/runtime/OutputStreamSettings.java new file mode 100755 index 000000000..468ece178 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/runtime/OutputStreamSettings.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.runtime; + +import java.io.File; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; + +/** + * Settings that define text to capture from a process stream. + */ +public class OutputStreamSettings { + private final EnumSet streamLocations = EnumSet.noneOf(StreamLocation.class); + private int bufferSize; + private File outputFile; + private boolean appendFile; + + public OutputStreamSettings() { + } + + /** + * @param bufferSize The number of bytes to capture, or -1 for unlimited. + */ + public OutputStreamSettings(int bufferSize) { + setBufferSize(bufferSize); + } + + /** + * @param outputFile The file to write output to. + */ + public OutputStreamSettings(File outputFile) { + setOutputFile(outputFile); + } + + /** + * @param outputFile The file to write output to. + * @param append true if the output file should be appended to. + */ + public OutputStreamSettings(File outputFile, boolean append) { + setOutputFile(outputFile, append); + } + + public OutputStreamSettings(int bufferSize, File outputFile, boolean appendFile) { + setBufferSize(bufferSize); + setOutputFile(outputFile, appendFile); + } + + public Set getStreamLocations() { + return Collections.unmodifiableSet(streamLocations); + } + + public int getBufferSize() { + return bufferSize; + } + + public void setBufferSize(int bufferSize) { + this.streamLocations.add(StreamLocation.Buffer); + this.bufferSize = bufferSize; + } + + public void clearBufferSize() { + this.streamLocations.remove(StreamLocation.Buffer); + this.bufferSize = 0; + } + + public File getOutputFile() { + return outputFile; + } + + public boolean isAppendFile() { + return appendFile; + } + + /** + * Overwrites the outputFile with the process output. + * + * @param outputFile File to overwrite. + */ + public void setOutputFile(File outputFile) { + setOutputFile(outputFile, false); + } + + public void setOutputFile(File outputFile, boolean append) { + if (outputFile == null) + throw new IllegalArgumentException("outputFile cannot be null"); + streamLocations.add(StreamLocation.File); + this.outputFile = outputFile; + this.appendFile = append; + } + + public void clearOutputFile() { + streamLocations.remove(StreamLocation.File); + this.outputFile = null; + this.appendFile = false; + } + + public void printStandard(boolean print) { + if (print) + this.streamLocations.add(StreamLocation.Standard); + else + this.streamLocations.remove(StreamLocation.Standard); + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/runtime/ProcessController.java b/public/java/src/org/broadinstitute/sting/utils/runtime/ProcessController.java new file mode 100755 index 000000000..6a3f9c753 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/runtime/ProcessController.java @@ -0,0 +1,363 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.runtime; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import org.broadinstitute.sting.utils.exceptions.UserException; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.*; + +/** + * Facade to Runtime.exec() and java.lang.Process. Handles + * running a process to completion and returns stdout and stderr + * as strings. Creates separate threads for reading stdout and stderr, + * then reuses those threads for each process most efficient use is + * to create one of these and use it repeatedly. Instances are not + * thread-safe, however. + * + * TODO: java.io sometimes zombies the backround threads locking up on read(). + * Supposedly NIO has better ways of interrupting a blocked stream but will + * require a little bit of refactoring. + * + * @author Michael Koehrsen + * @author Khalid Shakir + */ +public class ProcessController { + private static Logger logger = Logger.getLogger(ProcessController.class); + + private static enum ProcessStream {Stdout, Stderr} + + // Tracks running processes. + private static final Set running = Collections.synchronizedSet(new HashSet()); + + // Tracks this running process. + private Process process; + + // Threads that capture stdout and stderr + private final OutputCapture stdoutCapture; + private final OutputCapture stderrCapture; + + // When a caller destroyes a controller a new thread local version will be created + private boolean destroyed = false; + + // Communication channels with output capture threads + + // Holds the stdout and stderr sent to the background capture threads + private final Map toCapture = + new EnumMap(ProcessStream.class); + + // Holds the results of the capture from the background capture threads. + // May be the content via toCapture or an StreamOutput.EMPTY if the capture was interrupted. + private final Map fromCapture = + new EnumMap(ProcessStream.class); + + // Useful for debugging if background threads have shut down correctly + private static int nextControllerId = 0; + private final int controllerId; + + public ProcessController() { + // Start the background threads for this controller. + synchronized (running) { + controllerId = nextControllerId++; + } + stdoutCapture = new OutputCapture(ProcessStream.Stdout, controllerId); + stderrCapture = new OutputCapture(ProcessStream.Stderr, controllerId); + stdoutCapture.start(); + stderrCapture.start(); + } + + /** + * Returns a thread local ProcessController. + * Should NOT be closed when finished so it can be reused by the thread. + * + * @return a thread local ProcessController. + */ + public static ProcessController getThreadLocal() { + // If the local controller was destroyed get a fresh instance. + if (threadProcessController.get().destroyed) + threadProcessController.remove(); + return threadProcessController.get(); + } + + /** + * Thread local process controller container. + */ + private static final ThreadLocal threadProcessController = + new ThreadLocal() { + @Override + protected ProcessController initialValue() { + return new ProcessController(); + } + }; + + /** + * Similar to Runtime.exec() but drains the output and error streams. + * + * @param command Command to run. + * @return The result code. + */ + public static int exec(String[] command) { + ProcessController controller = ProcessController.getThreadLocal(); + return controller.exec(new ProcessSettings(command)).getExitValue(); + } + + /** + * Executes a command line program with the settings and waits for it to return, + * processing the output on a background thread. + * + * @param settings Settings to be run. + * @return The output of the command. + */ + public ProcessOutput exec(ProcessSettings settings) { + if (destroyed) + throw new IllegalStateException("This controller was destroyed"); + + ProcessBuilder builder = new ProcessBuilder(settings.getCommand()); + builder.directory(settings.getDirectory()); + + Map settingsEnvironment = settings.getEnvironment(); + if (settingsEnvironment != null) { + Map builderEnvironment = builder.environment(); + builderEnvironment.clear(); + builderEnvironment.putAll(settingsEnvironment); + } + + builder.redirectErrorStream(settings.isRedirectErrorStream()); + + StreamOutput stdout = null; + StreamOutput stderr = null; + + // Start the process running. + + try { + synchronized (toCapture) { + process = builder.start(); + } + running.add(this); + } catch (IOException e) { + throw new ReviewedStingException("Unable to start command: " + StringUtils.join(builder.command(), " ")); + } + + int exitCode; + + try { + // Notify the background threads to start capturing. + synchronized (toCapture) { + toCapture.put(ProcessStream.Stdout, + new CapturedStreamOutput(settings.getStdoutSettings(), process.getInputStream(), System.out)); + toCapture.put(ProcessStream.Stderr, + new CapturedStreamOutput(settings.getStderrSettings(), process.getErrorStream(), System.err)); + toCapture.notifyAll(); + } + + // Write stdin content + InputStreamSettings stdinSettings = settings.getStdinSettings(); + Set streamLocations = stdinSettings.getStreamLocations(); + if (!streamLocations.isEmpty()) { + try { + OutputStream stdinStream = process.getOutputStream(); + for (StreamLocation location : streamLocations) { + InputStream inputStream; + switch (location) { + case Buffer: + inputStream = new ByteArrayInputStream(stdinSettings.getInputBuffer()); + break; + case File: + try { + inputStream = FileUtils.openInputStream(stdinSettings.getInputFile()); + } catch (IOException e) { + throw new UserException.BadInput(e.getMessage()); + } + break; + case Standard: + inputStream = System.in; + break; + default: + throw new ReviewedStingException("Unexpected stream location: " + location); + } + try { + IOUtils.copy(inputStream, stdinStream); + } finally { + if (location != StreamLocation.Standard) + IOUtils.closeQuietly(inputStream); + } + } + stdinStream.flush(); + } catch (IOException e) { + throw new ReviewedStingException("Error writing to stdin on command: " + StringUtils.join(builder.command(), " "), e); + } + } + + // Wait for the process to complete. + try { + process.getOutputStream().close(); + process.waitFor(); + } catch (IOException e) { + throw new ReviewedStingException("Unable to close stdin on command: " + StringUtils.join(builder.command(), " "), e); + } catch (InterruptedException e) { + throw new ReviewedStingException("Process interrupted", e); + } finally { + while (!destroyed && stdout == null || stderr == null) { + synchronized (fromCapture) { + if (fromCapture.containsKey(ProcessStream.Stdout)) + stdout = fromCapture.remove(ProcessStream.Stdout); + if (fromCapture.containsKey(ProcessStream.Stderr)) + stderr = fromCapture.remove(ProcessStream.Stderr); + try { + if (stdout == null || stderr == null) + fromCapture.wait(); + } catch (InterruptedException e) { + // Log the error, ignore the interrupt and wait patiently + // for the OutputCaptures to (via finally) return their + // stdout and stderr. + logger.error(e); + } + } + } + + if (destroyed) { + if (stdout == null) + stdout = StreamOutput.EMPTY; + if (stderr == null) + stderr = StreamOutput.EMPTY; + } + } + } finally { + synchronized (toCapture) { + exitCode = process.exitValue(); + process = null; + } + running.remove(this); + } + + return new ProcessOutput(exitCode, stdout, stderr); + } + + /** + * @return The set of still running processes. + */ + public static Set getRunning() { + synchronized (running) { + return new HashSet(running); + } + } + + /** + * Stops the process from running and tries to ensure process is cleaned up properly. + * NOTE: sub-processes started by process may be zombied with their parents set to pid 1. + * NOTE: capture threads may block on read. + * TODO: Try to use NIO to interrupt streams. + */ + public void tryDestroy() { + destroyed = true; + synchronized (toCapture) { + if (process != null) { + process.destroy(); + IOUtils.closeQuietly(process.getInputStream()); + IOUtils.closeQuietly(process.getErrorStream()); + } + stdoutCapture.interrupt(); + stderrCapture.interrupt(); + toCapture.notifyAll(); + } + } + + @Override + protected void finalize() throws Throwable { + try { + tryDestroy(); + } catch (Exception e) { + logger.error(e); + } + super.finalize(); + } + + private class OutputCapture extends Thread { + private final int controllerId; + private final ProcessStream key; + + /** + * Reads in the output of a stream on a background thread to keep the output pipe from backing up and freezing the called process. + * + * @param key The stdout or stderr key for this output capture. + * @param controllerId Unique id of the controller. + */ + public OutputCapture(ProcessStream key, int controllerId) { + super(String.format("OutputCapture-%d-%s-%s-%d", controllerId, key.name().toLowerCase(), + Thread.currentThread().getName(), Thread.currentThread().getId())); + this.controllerId = controllerId; + this.key = key; + setDaemon(true); + } + + /** + * Runs the capture. + */ + @Override + public void run() { + while (!destroyed) { + StreamOutput processStream = StreamOutput.EMPTY; + try { + // Wait for a new input stream to be passed from this process controller. + CapturedStreamOutput capturedProcessStream = null; + while (!destroyed && capturedProcessStream == null) { + synchronized (toCapture) { + if (toCapture.containsKey(key)) { + capturedProcessStream = toCapture.remove(key); + } else { + toCapture.wait(); + } + } + } + + if (!destroyed) { + // Read in the input stream + processStream = capturedProcessStream; + capturedProcessStream.readAndClose(); + } + } catch (InterruptedException e) { + logger.info("OutputCapture interrupted, exiting"); + break; + } catch (IOException e) { + logger.error("Error reading process output", e); + } finally { + // Send the string back to the process controller. + synchronized (fromCapture) { + fromCapture.put(key, processStream); + fromCapture.notify(); + } + } + } + } + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/runtime/ProcessOutput.java b/public/java/src/org/broadinstitute/sting/utils/runtime/ProcessOutput.java new file mode 100755 index 000000000..211008950 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/runtime/ProcessOutput.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.runtime; + +public class ProcessOutput { + private final int exitValue; + private final StreamOutput stdout; + private final StreamOutput stderr; + + /** + * The output of a process. + * + * @param exitValue The exit value. + * @param stdout The capture of stdout as defined by the stdout OutputStreamSettings. + * @param stderr The capture of stderr as defined by the stderr OutputStreamSettings. + */ + public ProcessOutput(int exitValue, StreamOutput stdout, StreamOutput stderr) { + this.exitValue = exitValue; + this.stdout = stdout; + this.stderr = stderr; + } + + public int getExitValue() { + return exitValue; + } + + public StreamOutput getStdout() { + return stdout; + } + + public StreamOutput getStderr() { + return stderr; + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/runtime/ProcessSettings.java b/public/java/src/org/broadinstitute/sting/utils/runtime/ProcessSettings.java new file mode 100755 index 000000000..b9f67f3a4 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/runtime/ProcessSettings.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.runtime; + +import com.sun.corba.se.spi.orbutil.fsm.Input; + +import java.io.File; +import java.util.Map; + +public class ProcessSettings { + private String[] command; + private Map environment; + private File directory; + private boolean redirectErrorStream; + private InputStreamSettings stdinSettings; + private OutputStreamSettings stdoutSettings; + private OutputStreamSettings stderrSettings; + + /** + * @param command Command line to run. + */ + public ProcessSettings(String[] command) { + this(command, false, null, null, null, null, null); + } + + /** + * @param command Command line to run. + * @param redirectErrorStream true if stderr should be sent to stdout. + * @param environment Environment settings to override System.getEnv, or null to use System.getEnv. + * @param directory The directory to run the command in, or null to run in the current directory. + * @param stdinSettings Settings for writing to the process stdin. + * @param stdoutSettings Settings for capturing the process stdout. + * @param stderrSettings Setting for capturing the process stderr. + */ + public ProcessSettings(String[] command, boolean redirectErrorStream, File directory, Map environment, + InputStreamSettings stdinSettings, OutputStreamSettings stdoutSettings, OutputStreamSettings stderrSettings) { + this.command = checkCommand(command); + this.redirectErrorStream = redirectErrorStream; + this.directory = directory; + this.environment = environment; + this.stdinSettings = checkSettings(stdinSettings); + this.stdoutSettings = checkSettings(stdoutSettings); + this.stderrSettings = checkSettings(stderrSettings); + } + + public String[] getCommand() { + return command; + } + + public void setCommand(String[] command) { + this.command = checkCommand(command); + } + + public boolean isRedirectErrorStream() { + return redirectErrorStream; + } + + public void setRedirectErrorStream(boolean redirectErrorStream) { + this.redirectErrorStream = redirectErrorStream; + } + + public File getDirectory() { + return directory; + } + + public void setDirectory(File directory) { + this.directory = directory; + } + + public Map getEnvironment() { + return environment; + } + + public void setEnvironment(Map environment) { + this.environment = environment; + } + + public InputStreamSettings getStdinSettings() { + return stdinSettings; + } + + public void setStdinSettings(InputStreamSettings stdinSettings) { + this.stdinSettings = checkSettings(stdinSettings); + } + + public OutputStreamSettings getStdoutSettings() { + return stdoutSettings; + } + + public void setStdoutSettings(OutputStreamSettings stdoutSettings) { + this.stdoutSettings = checkSettings(stdoutSettings); + } + + public OutputStreamSettings getStderrSettings() { + return stderrSettings; + } + + public void setStderrSettings(OutputStreamSettings stderrSettings) { + this.stderrSettings = checkSettings(stderrSettings); + } + + protected String[] checkCommand(String[] command) { + if (command == null) + throw new IllegalArgumentException("Command is not allowed to be null"); + for (String s: command) + if (s == null) + throw new IllegalArgumentException("Command is not allowed to contain nulls"); + return command; + } + + protected InputStreamSettings checkSettings(InputStreamSettings settings) { + return settings == null ? new InputStreamSettings() : settings; + } + + protected OutputStreamSettings checkSettings(OutputStreamSettings settings) { + return settings == null ? new OutputStreamSettings() : settings; + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/runtime/StreamLocation.java b/public/java/src/org/broadinstitute/sting/utils/runtime/StreamLocation.java new file mode 100755 index 000000000..df72180f1 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/runtime/StreamLocation.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.runtime; + +/** + * Where to read/write a stream + */ +public enum StreamLocation { + Buffer, File, Standard +} diff --git a/public/java/src/org/broadinstitute/sting/utils/runtime/StreamOutput.java b/public/java/src/org/broadinstitute/sting/utils/runtime/StreamOutput.java new file mode 100755 index 000000000..5dc94815f --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/runtime/StreamOutput.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.runtime; + +/** + * The content of stdout or stderr. + */ +public abstract class StreamOutput { + /** + * Empty stream output when no output is captured due to an error. + */ + public static final StreamOutput EMPTY = new StreamOutput() { + @Override + public byte[] getBufferBytes() { + return new byte[0]; + } + + @Override + public boolean isBufferTruncated() { + return false; + } + }; + + /** + * Returns the content as a string. + * + * @return The content as a string. + */ + public String getBufferString() { + return new String(getBufferBytes()); + } + + /** + * Returns the content as a string. + * + * @return The content as a string. + */ + public abstract byte[] getBufferBytes(); + + /** + * Returns true if the buffer was truncated. + * + * @return true if the buffer was truncated. + */ + public abstract boolean isBufferTruncated(); +} diff --git a/public/java/test/org/broadinstitute/sting/utils/R/RScriptLibraryUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/R/RScriptLibraryUnitTest.java new file mode 100644 index 000000000..19fd5b316 --- /dev/null +++ b/public/java/test/org/broadinstitute/sting/utils/R/RScriptLibraryUnitTest.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.R; + +import org.apache.commons.io.FileUtils; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.io.File; + +public class RScriptLibraryUnitTest { + @Test + public void testProperties() { + Assert.assertEquals(RScriptLibrary.GSALIB.getLibraryName(), "gsalib"); + Assert.assertEquals(RScriptLibrary.GSALIB.getResourcePath(), "gsalib.tar.gz"); + } + + @Test + public void testWriteTemp() { + File file = RScriptLibrary.GSALIB.writeTemp(); + Assert.assertTrue(file.exists(), "R library was not written to temp file: " + file); + FileUtils.deleteQuietly(file); + } +} diff --git a/public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java new file mode 100644 index 000000000..4caf7f485 --- /dev/null +++ b/public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java @@ -0,0 +1,197 @@ +package org.broadinstitute.sting.utils.io; + +import org.apache.commons.io.FileUtils; +import org.broadinstitute.sting.BaseTest; +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import org.broadinstitute.sting.utils.exceptions.UserException; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class IOUtilsUnitTest extends BaseTest { + @Test + public void testGoodTempDir() { + IOUtils.checkTempDir(new File("/tmp/queue")); + } + + @Test(expectedExceptions=UserException.BadTmpDir.class) + public void testBadTempDir() { + IOUtils.checkTempDir(new File("/tmp")); + } + + @Test + public void testAbsoluteSubDir() { + File subDir = IOUtils.absolute(new File("."), new File("/path/to/file")); + Assert.assertEquals(subDir, new File("/path/to/file")); + + subDir = IOUtils.absolute(new File("/different/path"), new File("/path/to/file")); + Assert.assertEquals(subDir, new File("/path/to/file")); + + subDir = IOUtils.absolute(new File("/different/path"), new File(".")); + Assert.assertEquals(subDir, new File("/different/path")); + } + + @Test + public void testRelativeSubDir() throws IOException { + File subDir = IOUtils.absolute(new File("."), new File("path/to/file")); + Assert.assertEquals(subDir.getCanonicalFile(), new File("path/to/file").getCanonicalFile()); + + subDir = IOUtils.absolute(new File("/different/path"), new File("path/to/file")); + Assert.assertEquals(subDir, new File("/different/path/path/to/file")); + } + + @Test + public void testDottedSubDir() throws IOException { + File subDir = IOUtils.absolute(new File("."), new File("path/../to/file")); + Assert.assertEquals(subDir.getCanonicalFile(), new File("path/../to/./file").getCanonicalFile()); + + subDir = IOUtils.absolute(new File("."), new File("/path/../to/file")); + Assert.assertEquals(subDir, new File("/path/../to/file")); + + subDir = IOUtils.absolute(new File("/different/../path"), new File("path/to/file")); + Assert.assertEquals(subDir, new File("/different/../path/path/to/file")); + + subDir = IOUtils.absolute(new File("/different/./path"), new File("/path/../to/file")); + Assert.assertEquals(subDir, new File("/path/../to/file")); + } + + @Test + public void testTempDir() { + File tempDir = IOUtils.tempDir("Q-Unit-Test", "", new File("queueTempDirToDelete")); + Assert.assertTrue(tempDir.exists()); + Assert.assertFalse(tempDir.isFile()); + Assert.assertTrue(tempDir.isDirectory()); + boolean deleted = IOUtils.tryDelete(tempDir); + Assert.assertTrue(deleted); + Assert.assertFalse(tempDir.exists()); + } + + @Test + public void testDirLevel() { + File dir = IOUtils.dirLevel(new File("/path/to/directory"), 1); + Assert.assertEquals(dir, new File("/path")); + + dir = IOUtils.dirLevel(new File("/path/to/directory"), 2); + Assert.assertEquals(dir, new File("/path/to")); + + dir = IOUtils.dirLevel(new File("/path/to/directory"), 3); + Assert.assertEquals(dir, new File("/path/to/directory")); + + dir = IOUtils.dirLevel(new File("/path/to/directory"), 4); + Assert.assertEquals(dir, new File("/path/to/directory")); + } + + @Test + public void testAbsolute() { + File dir = IOUtils.absolute(new File("/path/./to/./directory/.")); + Assert.assertEquals(dir, new File("/path/to/directory")); + + dir = IOUtils.absolute(new File("/")); + Assert.assertEquals(dir, new File("/")); + + dir = IOUtils.absolute(new File("/.")); + Assert.assertEquals(dir, new File("/")); + + dir = IOUtils.absolute(new File("/././.")); + Assert.assertEquals(dir, new File("/")); + + dir = IOUtils.absolute(new File("/./directory/.")); + Assert.assertEquals(dir, new File("/directory")); + + dir = IOUtils.absolute(new File("/./directory/./")); + Assert.assertEquals(dir, new File("/directory")); + + dir = IOUtils.absolute(new File("/./directory./")); + Assert.assertEquals(dir, new File("/directory.")); + + dir = IOUtils.absolute(new File("/./.directory/")); + Assert.assertEquals(dir, new File("/.directory")); + } + + @Test + public void testTail() throws IOException { + List lines = Arrays.asList( + "chr18_random 4262 3154410390 50 51", + "chr19_random 301858 3154414752 50 51", + "chr21_random 1679693 3154722662 50 51", + "chr22_random 257318 3156435963 50 51", + "chrX_random 1719168 3156698441 50 51"); + List tail = IOUtils.tail(new File(BaseTest.hg18Reference + ".fai"), 5); + Assert.assertEquals(tail.size(), 5); + for (int i = 0; i < 5; i++) + Assert.assertEquals(tail.get(i), lines.get(i)); + } + + @Test + public void testWriteSystemFile() throws IOException { + File temp = createTempFile("temp.", ".properties"); + try { + IOUtils.writeResource(new Resource("StingText.properties", null), temp); + } finally { + FileUtils.deleteQuietly(temp); + } + } + + @Test + public void testWriteSystemTempFile() throws IOException { + File temp = IOUtils.writeTempResource(new Resource("StingText.properties", null)); + try { + Assert.assertTrue(temp.getName().startsWith("StingText"), "File does not start with 'StingText.': " + temp); + Assert.assertTrue(temp.getName().endsWith(".properties"), "File does not end with '.properties': " + temp); + } finally { + FileUtils.deleteQuietly(temp); + } + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testMissingSystemFile() throws IOException { + File temp = createTempFile("temp.", ".properties"); + try { + IOUtils.writeResource(new Resource("MissingStingText.properties", null), temp); + } finally { + FileUtils.deleteQuietly(temp); + } + } + + @Test + public void testWriteRelativeFile() throws IOException { + File temp = createTempFile("temp.", ".properties"); + try { + IOUtils.writeResource(new Resource("/StingText.properties", IOUtils.class), temp); + } finally { + FileUtils.deleteQuietly(temp); + } + } + + @Test + public void testWriteRelativeTempFile() throws IOException { + File temp = IOUtils.writeTempResource(new Resource("/StingText.properties", IOUtils.class)); + try { + Assert.assertTrue(temp.getName().startsWith("StingText"), "File does not start with 'StingText.': " + temp); + Assert.assertTrue(temp.getName().endsWith(".properties"), "File does not end with '.properties': " + temp); + } finally { + FileUtils.deleteQuietly(temp); + } + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testMissingRelativeFile() throws IOException { + File temp = createTempFile("temp.", ".properties"); + try { + // Looking for /org/broadinstitute/sting/utils/file/StingText.properties + IOUtils.writeResource(new Resource("StingText.properties", IOUtils.class), temp); + } finally { + FileUtils.deleteQuietly(temp); + } + } + + @Test + public void testResourceProperties() { + Resource resource = new Resource("foo", Resource.class); + Assert.assertEquals(resource.getPath(), "foo"); + Assert.assertEquals(resource.getRelativeClass(), Resource.class); + } +} diff --git a/public/java/test/org/broadinstitute/sting/utils/runtime/ProcessControllerUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/runtime/ProcessControllerUnitTest.java new file mode 100644 index 000000000..7a31ceee0 --- /dev/null +++ b/public/java/test/org/broadinstitute/sting/utils/runtime/ProcessControllerUnitTest.java @@ -0,0 +1,517 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.runtime; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.broadinstitute.sting.BaseTest; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import org.broadinstitute.sting.utils.exceptions.UserException; +import org.broadinstitute.sting.utils.io.IOUtils; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +public class ProcessControllerUnitTest extends BaseTest { + private static final String NL = String.format("%n"); + + @Test(timeOut = 60 * 1000) + public void testDestroyThreadLocal() throws InterruptedException { + for (int i = 0; i < 3; i++) { + final ProcessController controller = ProcessController.getThreadLocal(); + final ProcessSettings job = new ProcessSettings( + new String[] {"sh", "-c", "echo Hello World && sleep 600 && echo Goodbye"}); + job.getStdoutSettings().setBufferSize(-1); + + Thread t = new Thread(new Runnable() { + @Override + public void run() { + System.out.println("BACK: Starting on background thread"); + ProcessOutput result = controller.exec(job); + // Assert in background thread doesn't make it to main thread but does print a trace. + Assert.assertTrue(result.getExitValue() != 0, "Destroy-attempted job returned zero exit status"); + System.out.println("BACK: Background thread exiting"); + } + }); + + System.out.println("MAIN: Starting background thread"); + t.start(); + System.out.println("MAIN: Sleeping main thread 3s"); + Thread.sleep(3000); + System.out.println("MAIN: Destroying job"); + controller.tryDestroy(); + System.out.println("MAIN: Not waiting on background thread to exit"); + // Using standard java.io this was blocking on linux. + // TODO: try again with NIO. + //t.join(); + //System.out.println("MAIN: Background thread exited"); + } + } + + @Test + public void testReuseAfterError() { + ProcessController controller = new ProcessController(); + + ProcessSettings job; + + for (int i = 0; i < 3; i++) { + // Test bad command + job = new ProcessSettings(new String[] {"no_such_command"}); + try { + controller.exec(job); + } catch (ReviewedStingException e) { + /* Was supposed to throw an exception */ + } + + // Test exit != 0 + job = new ProcessSettings(new String[] {"cat", "non_existent_file"}); + int exitValue = controller.exec(job).getExitValue(); + Assert.assertTrue(exitValue != 0, "'cat' non existent file returned 0"); + + // Text success + job = new ProcessSettings(new String[] {"echo", "Hello World"}); + exitValue = controller.exec(job).getExitValue(); + Assert.assertEquals(exitValue, 0, "Echo failed"); + } + } + + @Test + public void testEnvironment() { + String key = "MY_NEW_VAR"; + String value = "value is here"; + + ProcessSettings job = new ProcessSettings(new String[] {"sh", "-c", "echo $"+key}); + job.getStdoutSettings().setBufferSize(-1); + job.setRedirectErrorStream(true); + + Map env = new HashMap(System.getenv()); + env.put(key, value); + job.setEnvironment(env); + + ProcessController controller = new ProcessController(); + ProcessOutput result = controller.exec(job); + int exitValue = result.getExitValue(); + + Assert.assertEquals(exitValue, 0, "Echo environment variable failed"); + Assert.assertEquals(result.getStdout().getBufferString(), value + NL, "Echo environment returned unexpected output"); + } + + @Test + public void testDirectory() throws IOException { + File dir = null; + try { + dir = IOUtils.tempDir("temp.", "").getCanonicalFile(); + + ProcessSettings job = new ProcessSettings(new String[] {"pwd"}); + job.getStdoutSettings().setBufferSize(-1); + job.setRedirectErrorStream(true); + job.setDirectory(dir); + + ProcessController controller = new ProcessController(); + ProcessOutput result = controller.exec(job); + int exitValue = result.getExitValue(); + + Assert.assertEquals(exitValue, 0, "Getting working directory failed"); + + Assert.assertEquals(result.getStdout().getBufferString(), dir.getAbsolutePath() + NL, + "Setting/getting working directory returned unexpected output"); + } finally { + FileUtils.deleteQuietly(dir); + } + } + + @Test + public void testReadStdInBuffer() { + String bufferText = "Hello from buffer"; + ProcessSettings job = new ProcessSettings(new String[] {"cat"}); + job.getStdoutSettings().setBufferSize(-1); + job.setRedirectErrorStream(true); + job.getStdinSettings().setInputBuffer(bufferText); + + ProcessController controller = new ProcessController(); + ProcessOutput output = controller.exec(job); + + Assert.assertEquals(output.getStdout().getBufferString(), bufferText, + "Unexpected output from cat stdin buffer"); + } + + @Test + public void testReadStdInFile() { + File input = null; + try { + String fileText = "Hello from file"; + input = IOUtils.writeTempFile(fileText, "stdin.", ".txt", null); + + ProcessSettings job = new ProcessSettings(new String[] {"cat"}); + job.getStdoutSettings().setBufferSize(-1); + job.setRedirectErrorStream(true); + job.getStdinSettings().setInputFile(input); + + ProcessController controller = new ProcessController(); + ProcessOutput output = controller.exec(job); + + Assert.assertEquals(output.getStdout().getBufferString(), fileText, + "Unexpected output from cat stdin file"); + } finally { + FileUtils.deleteQuietly(input); + } + } + + @Test + public void testWriteStdOut() { + ProcessSettings job = new ProcessSettings(new String[] {"echo", "Testing to stdout"}); + // Not going to call the System.setOut() for now. Just running a basic visual test. + job.getStdoutSettings().printStandard(true); + job.setRedirectErrorStream(true); + + System.out.println("testWriteStdOut: Writing two lines to std out..."); + ProcessController controller = new ProcessController(); + controller.exec(job); + job.setCommand(new String[]{"cat", "non_existent_file"}); + controller.exec(job); + System.out.println("testWriteStdOut: ...two lines should have been printed to std out"); + } + + @Test + public void testErrorToOut() throws IOException { + File outFile = null; + File errFile = null; + try { + outFile = BaseTest.createTempFile("temp", ""); + errFile = BaseTest.createTempFile("temp", ""); + + ProcessSettings job = new ProcessSettings(new String[]{"cat", "non_existent_file"}); + job.getStdoutSettings().setOutputFile(outFile); + job.getStdoutSettings().setBufferSize(-1); + job.getStderrSettings().setOutputFile(errFile); + job.getStderrSettings().setBufferSize(-1); + job.setRedirectErrorStream(true); + + ProcessOutput result = new ProcessController().exec(job); + int exitValue = result.getExitValue(); + + Assert.assertTrue(exitValue != 0, "'cat' non existent file returned 0"); + + String fileString, bufferString; + + fileString = FileUtils.readFileToString(outFile); + Assert.assertTrue(fileString.length() > 0, "Out file was length 0"); + + bufferString = result.getStdout().getBufferString(); + Assert.assertTrue(bufferString.length() > 0, "Out buffer was length 0"); + + Assert.assertFalse(result.getStdout().isBufferTruncated(), "Out buffer was truncated"); + Assert.assertEquals(bufferString.length(), fileString.length(), "Out buffer length did not match file length"); + + fileString = FileUtils.readFileToString(errFile); + Assert.assertEquals(fileString, "", "Unexpected output to err file"); + + bufferString = result.getStderr().getBufferString(); + Assert.assertEquals(bufferString, "", "Unexepected output to err buffer"); + } finally { + FileUtils.deleteQuietly(outFile); + FileUtils.deleteQuietly(errFile); + } + } + + @Test + public void testErrorToErr() throws IOException { + File outFile = null; + File errFile = null; + try { + outFile = BaseTest.createTempFile("temp", ""); + errFile = BaseTest.createTempFile("temp", ""); + + ProcessSettings job = new ProcessSettings(new String[]{"cat", "non_existent_file"}); + job.getStdoutSettings().setOutputFile(outFile); + job.getStdoutSettings().setBufferSize(-1); + job.getStderrSettings().setOutputFile(errFile); + job.getStderrSettings().setBufferSize(-1); + job.setRedirectErrorStream(false); + + ProcessOutput result = new ProcessController().exec(job); + int exitValue = result.getExitValue(); + + Assert.assertTrue(exitValue != 0, "'cat' non existent file returned 0"); + + String fileString, bufferString; + + fileString = FileUtils.readFileToString(errFile); + Assert.assertTrue(fileString.length() > 0, "Err file was length 0"); + + bufferString = result.getStderr().getBufferString(); + Assert.assertTrue(bufferString.length() > 0, "Err buffer was length 0"); + + Assert.assertFalse(result.getStderr().isBufferTruncated(), "Err buffer was truncated"); + Assert.assertEquals(bufferString.length(), fileString.length(), "Err buffer length did not match file length"); + + fileString = FileUtils.readFileToString(outFile); + Assert.assertEquals(fileString, "", "Unexpected output to out file"); + + bufferString = result.getStdout().getBufferString(); + Assert.assertEquals(bufferString, "", "Unexepected output to out buffer"); + } finally { + FileUtils.deleteQuietly(outFile); + FileUtils.deleteQuietly(errFile); + } + } + + private static final String TRUNCATE_TEXT = "Hello World"; + private static final byte[] TRUNCATE_OUTPUT_BYTES = (TRUNCATE_TEXT + NL).getBytes(); + + /** + * @return Test truncating content vs. not truncating (run at -1/+1 size) + */ + @DataProvider(name = "truncateSizes") + public Object[][] getTruncateBufferSizes() { + int l = TRUNCATE_OUTPUT_BYTES.length; + return new Object[][]{ + new Object[]{0, 0}, + new Object[]{l, l}, + new Object[]{l + 1, l}, + new Object[]{l - 1, l - 1} + }; + } + + @Test(dataProvider = "truncateSizes") + public void testTruncateBuffer(int truncateLen, int expectedLen) { + byte[] expected = Arrays.copyOf(TRUNCATE_OUTPUT_BYTES, expectedLen); + + String[] command = {"echo", TRUNCATE_TEXT}; + ProcessController controller = new ProcessController(); + + ProcessSettings job = new ProcessSettings(command); + job.getStdoutSettings().setBufferSize(truncateLen); + ProcessOutput result = controller.exec(job); + + int exitValue = result.getExitValue(); + + Assert.assertEquals(exitValue, 0, + String.format("Echo returned %d: %s", exitValue, TRUNCATE_TEXT)); + + byte[] bufferBytes = result.getStdout().getBufferBytes(); + + Assert.assertEquals(bufferBytes, expected, + String.format("Output buffer didn't match (%d vs %d)", expected.length, bufferBytes.length)); + + boolean truncated = result.getStdout().isBufferTruncated(); + + Assert.assertEquals(truncated, TRUNCATE_OUTPUT_BYTES.length > truncateLen, + "Unexpected buffer truncation result"); + } + + private static final String[] LONG_COMMAND = getLongCommand(); + private static final String LONG_COMMAND_STRING = StringUtils.join(LONG_COMMAND, " "); + private static final String LONG_COMMAND_DESCRIPTION = ""; + + @DataProvider(name = "echoCommands") + public Object[][] getEchoCommands() { + + new EchoCommand(new String[]{"echo", "Hello", "World"}, "Hello World" + NL); + new EchoCommand(new String[]{"echo", "'Hello", "World"}, "'Hello World" + NL); + new EchoCommand(new String[]{"echo", "Hello", "World'"}, "Hello World'" + NL); + new EchoCommand(new String[]{"echo", "'Hello", "World'"}, "'Hello World'" + NL); + + String[] longCommand = new String[LONG_COMMAND.length + 1]; + longCommand[0] = "echo"; + System.arraycopy(LONG_COMMAND, 0, longCommand, 1, LONG_COMMAND.length); + new EchoCommand(longCommand, LONG_COMMAND_STRING + NL) { + @Override + public String toString() { + return LONG_COMMAND_DESCRIPTION; + } + }; + + return TestDataProvider.getTests(EchoCommand.class); + } + + @Test(dataProvider = "echoCommands") + public void testEcho(EchoCommand script) throws IOException { + File outputFile = null; + try { + outputFile = BaseTest.createTempFile("temp", ""); + + ProcessSettings job = new ProcessSettings(script.command); + if (script.output != null) { + job.getStdoutSettings().setOutputFile(outputFile); + job.getStdoutSettings().setBufferSize(script.output.getBytes().length); + } + + ProcessOutput result = new ProcessController().exec(job); + int exitValue = result.getExitValue(); + + Assert.assertEquals(exitValue, 0, + String.format("Echo returned %d: %s", exitValue, script)); + + if (script.output != null) { + + String fileString = FileUtils.readFileToString(outputFile); + Assert.assertEquals(fileString, script.output, + String.format("Output file didn't match (%d vs %d): %s", + fileString.length(), script.output.length(), script)); + + String bufferString = result.getStdout().getBufferString(); + Assert.assertEquals(bufferString, script.output, + String.format("Output content didn't match (%d vs %d): %s", + bufferString.length(), script.output.length(), script)); + + Assert.assertFalse(result.getStdout().isBufferTruncated(), + "Output content was truncated: " + script); + } + } finally { + FileUtils.deleteQuietly(outputFile); + } + } + + @Test(expectedExceptions = ReviewedStingException.class) + public void testUnableToStart() { + ProcessSettings job = new ProcessSettings(new String[]{"no_such_command"}); + new ProcessController().exec(job); + } + + @DataProvider(name = "scriptCommands") + public Object[][] getScriptCommands() { + new ScriptCommand(true, "echo Hello World", "Hello World" + NL); + new ScriptCommand(false, "echo 'Hello World", null); + new ScriptCommand(false, "echo Hello World'", null); + new ScriptCommand(true, "echo 'Hello World'", "Hello World" + NL); + new ScriptCommand(true, "echo \"Hello World\"", "Hello World" + NL); + new ScriptCommand(false, "no_such_echo Hello World", null); + new ScriptCommand(true, "echo #", NL); + new ScriptCommand(true, "echo \\#", "#" + NL); + new ScriptCommand(true, "echo \\\\#", "\\#" + NL); + + new ScriptCommand(true, "echo " + LONG_COMMAND_STRING, LONG_COMMAND_STRING + NL) { + @Override + public String toString() { + return LONG_COMMAND_DESCRIPTION; + } + }; + + return TestDataProvider.getTests(ScriptCommand.class); + } + + @Test(dataProvider = "scriptCommands") + public void testScript(ScriptCommand script) throws IOException { + File scriptFile = null; + File outputFile = null; + try { + scriptFile = writeScript(script.content); + outputFile = BaseTest.createTempFile("temp", ""); + + ProcessSettings job = new ProcessSettings(new String[]{"sh", scriptFile.getAbsolutePath()}); + if (script.output != null) { + job.getStdoutSettings().setOutputFile(outputFile); + job.getStdoutSettings().setBufferSize(script.output.getBytes().length); + } + + ProcessOutput result = new ProcessController().exec(job); + int exitValue = result.getExitValue(); + + Assert.assertEquals(exitValue == 0, script.succeed, + String.format("Script returned %d: %s", exitValue, script)); + + if (script.output != null) { + + String fileString = FileUtils.readFileToString(outputFile); + Assert.assertEquals(fileString, script.output, + String.format("Output file didn't match (%d vs %d): %s", + fileString.length(), script.output.length(), script)); + + String bufferString = result.getStdout().getBufferString(); + Assert.assertEquals(bufferString, script.output, + String.format("Output content didn't match (%d vs %d): %s", + bufferString.length(), script.output.length(), script)); + + Assert.assertFalse(result.getStdout().isBufferTruncated(), + "Output content was truncated: " + script); + } + } finally { + FileUtils.deleteQuietly(scriptFile); + FileUtils.deleteQuietly(outputFile); + } + } + + private static String[] getLongCommand() { + // This command fails on some systems with a 4096 character limit when run via the old sh -c "echo ...", + // but works on the same systems when run via sh