Merge branch 'master' of ssh://gsa1/humgen/gsa-scr1/gsa-engineering/git/unstable

build.xml

@@ -28,6 +28,8 @@
 
     <property name="build.dir" value="build" />
     <property name="dist.dir" value="dist" />
+    <property name="contract.dump.dir" value="dump" />
+    <property name="pipelinetest.dir" value="pipelinetests" />
     <property name="lib.dir" value="lib" />
     <property name="external.dir" value="external" />
     <property name="public.dir" value="public" />
@@ -35,6 +37,13 @@
     <property name="java.public.source.dir" value="${public.dir}/java/src" />
     <property name="java.private.source.dir" value="${private.dir}/java/src" />
     <property name="java.classes" value="${build.dir}/java/classes" />
+    <property name="R.public.scripts.dir" value="${public.dir}/R/scripts" />
+    <property name="R.public.src.dir" value="${public.dir}/R/src" />
+    <!-- Legacy: Installing libraries back into the source directory
+    instead of the build or dist directory... intentionally avoids ant clean?? -->
+    <property name="R.library.dir" value="${public.dir}/R" />
+    <property name="R.tar.dir" value="${build.dir}/R/src" />
+    <property name="R.package.path" value="org/broadinstitute/sting/utils/R" />
     <property name="resource.file" value="StingText.properties" />
     <property name="resource.path" value="${java.classes}/StingText.properties" />
 
@@ -341,7 +350,7 @@
                 <pathelement path="${java.classes}" />
             </classpath>
             <compilerarg value="-Acom.google.java.contract.debug"/>
-            <compilerarg value="-Acom.google.java.contract.dump=dump/"/>
+            <compilerarg value="-Acom.google.java.contract.dump=${contract.dump.dir}"/>
             <compilerarg value="-proc:only"/>
         </javac>
    </target>
@@ -362,7 +371,7 @@
                 <pathelement path="${java.classes}" />
             </classpath>
             <compilerarg value="-Acom.google.java.contract.debug"/>
-            <compilerarg value="-Acom.google.java.contract.dump=dump/"/>
+            <compilerarg value="-Acom.google.java.contract.dump=${contract.dump.dir}"/>
             <compilerarg value="-proc:only"/>
         </javac>
    </target>
@@ -532,6 +541,11 @@
 
     <target name="sting.compile" depends="gatk.compile, scala.compile" />
 
+    <target name="R.public.tar">
+        <mkdir dir="${R.tar.dir}/${R.package.path}" />
+        <tar compression="gzip" basedir="${R.public.src.dir}/${R.package.path}" includes="gsalib/**" destfile="${R.tar.dir}/${R.package.path}/gsalib.tar.gz" />
+    </target>
+
     <target name="init.jar" depends="sting.compile,extracthelp">
         <mkdir dir="${dist.dir}"/>
         <copy todir="${dist.dir}">
@@ -539,7 +553,7 @@
         </copy>
     </target>
 
-    <target name="sting-utils.jar" depends="gatk.compile, init.jar">
+    <target name="sting-utils.jar" depends="gatk.compile, init.jar, R.public.tar">
         <jar jarfile="${dist.dir}/StingUtils.jar">
             <fileset dir="${java.classes}">
               <include name="**/utils/**/*.class"/>
@@ -551,6 +565,12 @@
             <fileset dir="${java.classes}" includes="**/sting/jna/**/*.class"/>
             <fileset dir="${java.classes}" includes="net/sf/picard/**/*.class"/>
             <fileset dir="${java.classes}" includes="net/sf/samtools/**/*.class"/>
+            <fileset dir="${R.tar.dir}">
+                <include name="**/${R.package.path}/**/*.tar.gz"/>
+            </fileset>
+            <fileset dir="${R.public.scripts.dir}">
+              <include name="**/utils/**/*.R"/>
+            </fileset>
             <manifest>
                 <attribute name="Premain-Class" value="org.broadinstitute.sting.utils.instrumentation.Sizeof" />
             </manifest>
@@ -579,6 +599,10 @@
               <include name="**/gatk/**/*.class" />
               <include name="**/alignment/**/*.class"/>
             </fileset>
+            <fileset dir="${R.public.scripts.dir}">
+                <include name="**/gatk/**/*.R"/>
+                <include name="**/alignment/**/*.R"/>
+            </fileset>
             <manifest>
                 <attribute name="Main-Class" value="org.broadinstitute.sting.gatk.CommandLineGATK" />
             </manifest>
@@ -593,6 +617,10 @@
             <include name="**/analyzecovariates/**/*.class" />
             <include name="**/gatk/walkers/recalibration/*.class" />
           </fileset>
+            <fileset dir="${R.public.scripts.dir}">
+                <include name="**/analyzecovariates/**/*.R"/>
+                <include name="**/gatk/walkers/recalibration/**/*.R"/>
+            </fileset>
           <manifest>
             <attribute name="Main-Class" value="org.broadinstitute.sting.analyzecovariates.AnalyzeCovariates" />
           </manifest>
@@ -605,28 +633,7 @@
             <fileset dir="${external.dir}" includes="*/build.xml" erroronmissingdir="false" />
         </subant>
     </target>
-<!--
-    <target name="gatk.oneoffs.jar" depends="gatk.compile, init.jar"
-            description="generate the GATK oneoffs distribution" if="include.oneoffs">
-        <jar jarfile="${dist.dir}/CompareBAMAlignments.jar" whenmanifestonly="skip">
-          <fileset dir="${java.classes}">
-            <include name="**/tools/**/*.class" />
-          </fileset>
-          <manifest>
-            <attribute name="Main-Class" value="org.broadinstitute.sting.oneoffprojects.tools.CompareBAMAlignments" />
-          </manifest>
-        </jar>
 
-        <jar jarfile="${dist.dir}/SliceBams.jar" whenmanifestonly="skip">
-          <fileset dir="${java.classes}">
-            <include name="**/tools/**/*.class" />
-          </fileset>
-          <manifest>
-            <attribute name="Main-Class" value="org.broadinstitute.sting.playground.tools.SliceBams" />
-          </manifest>
-        </jar>
-    </target>
--->
     <target name="scala.jar" depends="scala.compile, init.jar" if="scala.include">
         <jar jarfile="${dist.dir}/GATKScala.jar">
           <fileset dir="${scala.classes}">
@@ -643,6 +650,9 @@
             <fileset dir="${java.classes}">
               <include name="org/broadinstitute/sting/queue/**/*.class" />
             </fileset>
+            <fileset dir="${R.public.scripts.dir}">
+                <include name="org/broadinstitute/sting/queue/**/*.R"/>
+            </fileset>
             <manifest>
                 <attribute name="Main-Class" value="org.broadinstitute.sting.queue.QCommandLine" />
             </manifest>
@@ -682,20 +692,7 @@
         </jar>
 
     </target>
-<!--
+
-    <target name="gatk.oneoffs.manifests" depends="gatk.oneoffs.jar, init.manifests" if="include.oneoffs">
-        <jar jarfile="${dist.dir}/CompareBAMAlignments.jar" update="true" whenmanifestonly="skip">
-            <manifest>
-                <attribute name="Class-Path" value="${jar.classpath}" />
-            </manifest>
-        </jar>
-        <jar jarfile="${dist.dir}/SliceBams.jar" update="true" whenmanifestonly="skip">
-            <manifest>
-                <attribute name="Class-Path" value="${jar.classpath}" />
-            </manifest>
-        </jar>
-    </target>
--->
     <target name="queue.manifests" depends="queue.jar, init.manifests" if="scala.include">
         <jar jarfile="${dist.dir}/Queue.jar" update="true" >
             <manifest>
@@ -780,10 +777,6 @@
                 <pathelement location="${testng.jar}"/>
             </classpath>
             <compilerarg value="-proc:none"/>
-<!--
-            <compilerarg value="-Acom.google.java.contract.debug"/>
-            <compilerarg value="-Acom.google.java.contract.dump=dump/"/>
- -->
         </javac>
     </target>
 
@@ -800,10 +793,6 @@
                 <pathelement location="${testng.jar}"/>
             </classpath>
             <compilerarg value="-proc:none"/>
-<!--
-            <compilerarg value="-Acom.google.java.contract.debug"/>
-            <compilerarg value="-Acom.google.java.contract.dump=dump/"/>
- -->
         </javac>
     </target>
 
@@ -851,6 +840,8 @@
         <pathelement location="${java.private.test.classes}" />
         <pathelement location="${scala.public.test.classes}" />
         <pathelement location="${scala.private.test.classes}" />
+        <pathelement location="${R.tar.dir}" />
+        <pathelement location="${R.public.scripts.dir}" />
     </path>
 
     <path id="testng.gatk.releasetest.classpath">
@@ -1187,19 +1178,18 @@
     </target>
 
     <target name="clean" description="clean up" depends="clean.javadoc,clean.scaladoc,clean.gatkdocs">
-        <delete dir="out"/>
         <delete dir="${build.dir}"/>
         <delete dir="${lib.dir}"/>
-        <delete dir="dump"/>
+        <delete dir="${contract.dump.dir}"/>
         <delete dir="${staging.dir}"/>
         <delete dir="${dist.dir}"/>
-        <delete dir="pipelinetests"/>
+        <delete dir="${pipelinetest.dir}"/>
     </target>
 
     <!-- Build gsalib R module -->
     <target name="gsalib">
       <exec executable="R" failonerror="true">
-        <arg line="R CMD INSTALL -l public/R/ public/R/src/gsalib/" />
+        <arg line="R CMD INSTALL -l ${R.library.dir} ${R.public.src.dir}/${R.package.path}/gsalib" />
       </exec>
     </target>
 </project>

public/java/src/org/broadinstitute/sting/commandline/ArgumentMatch.java

@@ -46,7 +46,7 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
     /**
      * Maps indices of command line arguments to values paired with that argument.
      */
-    public final SortedMap<Integer,List<String>> indices = new TreeMap<Integer,List<String>>();
+    public final SortedMap<ArgumentMatchSite,List<String>> sites = new TreeMap<ArgumentMatchSite,List<String>>();
 
     /**
      * An ordered, freeform collection of tags.
@@ -72,32 +72,32 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
     }
 
     /**
-     * A simple way of indicating that an argument with the given label and definition exists at this index.
+     * A simple way of indicating that an argument with the given label and definition exists at this site.
      * @param label Label of the argument match.  Must not be null.
      * @param definition The associated definition, if one exists.  May be null.
-     * @param index Position of the argument.  Must not be null.
+     * @param site Position of the argument.  Must not be null.
      * @param tags ordered freeform text tags associated with this argument.
      */
-    public ArgumentMatch(final String label, final ArgumentDefinition definition, final int index, final Tags tags) {
+    public ArgumentMatch(final String label, final ArgumentDefinition definition, final ArgumentMatchSite site, final Tags tags) {
-        this( label, definition, index, null, tags );
+        this( label, definition, site, null, tags );
     }
 
     /**
-     * A simple way of indicating that an argument with the given label and definition exists at this index.
+     * A simple way of indicating that an argument with the given label and definition exists at this site.
      * @param label Label of the argument match.  Must not be null.
      * @param definition The associated definition, if one exists.  May be null.
-     * @param index Position of the argument.  Must not be null.
+     * @param site Position of the argument.  Must not be null.
      * @param value Value for the argument at this position.
      * @param tags ordered freeform text tags associated with this argument.
      */
-    private ArgumentMatch(final String label, final ArgumentDefinition definition, final int index, final String value, final Tags tags) {
+    private ArgumentMatch(final String label, final ArgumentDefinition definition, final ArgumentMatchSite site, final String value, final Tags tags) {
         this.label = label;
         this.definition = definition;
 
         ArrayList<String> values = new ArrayList<String>();
         if( value != null )
             values.add(value);
-        indices.put(index,values );
+        sites.put(site,values );
 
         this.tags = tags;
     }
@@ -117,7 +117,7 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
         ArgumentMatch otherArgumentMatch = (ArgumentMatch)other;
         return this.definition.equals(otherArgumentMatch.definition) &&
                 this.label.equals(otherArgumentMatch.label) &&
-                this.indices.equals(otherArgumentMatch.indices) &&
+                this.sites.equals(otherArgumentMatch.sites) &&
                 this.tags.equals(otherArgumentMatch.tags);
     }
 
@@ -129,16 +129,17 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
      * @param key Key which specifies the transform.
      * @return A variant of this ArgumentMatch with all keys transformed.
      */
+    @SuppressWarnings("unchecked")
     ArgumentMatch transform(Multiplexer multiplexer, Object key) {
-        SortedMap<Integer,List<String>> newIndices = new TreeMap<Integer,List<String>>();
+        SortedMap<ArgumentMatchSite,List<String>> newIndices = new TreeMap<ArgumentMatchSite,List<String>>();
-        for(Map.Entry<Integer,List<String>> index: indices.entrySet()) {
+        for(Map.Entry<ArgumentMatchSite,List<String>> site: sites.entrySet()) {
             List<String> newEntries = new ArrayList<String>();
-            for(String entry: index.getValue())
+            for(String entry: site.getValue())
                 newEntries.add(multiplexer.transformArgument(key,entry));
-            newIndices.put(index.getKey(),newEntries);
+            newIndices.put(site.getKey(),newEntries);
         }
         ArgumentMatch newArgumentMatch = new ArgumentMatch(label,definition);
-        newArgumentMatch.indices.putAll(newIndices);
+        newArgumentMatch.sites.putAll(newIndices);
         return newArgumentMatch;
     }
 
@@ -157,9 +158,9 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
     public Iterator<ArgumentMatch> iterator() {
         return new Iterator<ArgumentMatch>() {
             /**
-             * Iterate over each the available index.
+             * Iterate over each the available site.
              */
-            private Iterator<Integer> indexIterator = null;
+            private Iterator<ArgumentMatchSite> siteIterator = null;
 
             /**
              * Iterate over each available token.
@@ -167,9 +168,9 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
             private Iterator<String> tokenIterator = null;
 
             /**
-             * The next index to return.  Null if none remain.
+             * The next site to return.  Null if none remain.
              */
-            Integer nextIndex = null;
+            ArgumentMatchSite nextSite = null;
 
             /**
              * The next token to return.  Null if none remain.
@@ -177,7 +178,7 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
             String nextToken = null;
 
             {
-                indexIterator = indices.keySet().iterator();
+                siteIterator = sites.keySet().iterator();
                 prepareNext();
             }
 
@@ -194,32 +195,32 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
              * @return The next ArgumentMatch in the series.  Should never be null.
              */
             public ArgumentMatch next() {
-                if( nextIndex == null || nextToken == null )
+                if( nextSite == null || nextToken == null )
                     throw new IllegalStateException( "No more ArgumentMatches are available" );
 
-                ArgumentMatch match = new ArgumentMatch( label, definition, nextIndex, nextToken, tags );
+                ArgumentMatch match = new ArgumentMatch( label, definition, nextSite, nextToken, tags );
                 prepareNext();
                 return match;
             }
 
             /**
              * Initialize the next ArgumentMatch to return.  If no ArgumentMatches are available,
-             * initialize nextIndex / nextToken to null.
+             * initialize nextSite / nextToken to null.
              */
             private void prepareNext() {
                 if( tokenIterator != null && tokenIterator.hasNext() ) {
                     nextToken = tokenIterator.next();
                 }
                 else {
-                    nextIndex = null;
+                    nextSite = null;
                     nextToken = null;
 
                     // Do a nested loop.  While more data is present in the inner loop, grab that data.
                     // Otherwise, troll the outer iterator looking for more data.
-                    while( indexIterator.hasNext() ) {
+                    while( siteIterator.hasNext() ) {
-                        nextIndex = indexIterator.next();
+                        nextSite = siteIterator.next();
-                        if( indices.get(nextIndex) != null ) {
+                        if( sites.get(nextSite) != null ) {
-                            tokenIterator = indices.get(nextIndex).iterator();
+                            tokenIterator = sites.get(nextSite).iterator();
                             if( tokenIterator.hasNext() ) {
                                 nextToken = tokenIterator.next();
                                 break;
@@ -245,29 +246,29 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
      * @param other The other match to merge into.
      */
     public void mergeInto( ArgumentMatch other ) {
-        indices.putAll(other.indices);
+        sites.putAll(other.sites);
     }
 
     /**
      * Associate a value with this merge maapping.
-     * @param index index of the command-line argument to which this value is mated.
+     * @param site site of the command-line argument to which this value is mated.
      * @param value Text representation of value to add.
      */
-    public void addValue( int index, String value ) {
+    public void addValue( ArgumentMatchSite site, String value ) {
-        if( !indices.containsKey(index) || indices.get(index) == null )
+        if( !sites.containsKey(site) || sites.get(site) == null )
-            indices.put(index, new ArrayList<String>() );
+            sites.put(site, new ArrayList<String>() );
-        indices.get(index).add(value);
+        sites.get(site).add(value);
     }
 
     /**
      * Does this argument already have a value at the given site?
      * Arguments are only allowed to be single-valued per site, and
      * flags aren't allowed a value at all.
-     * @param index Index at which to check for values.
+     * @param site Site at which to check for values.
      * @return True if the argument has a value at the given site.  False otherwise.
      */
-    public boolean hasValueAtSite( int index ) {
+    public boolean hasValueAtSite( ArgumentMatchSite site ) {
-        return (indices.get(index) != null && indices.get(index).size() >= 1) || isArgumentFlag();
+        return (sites.get(site) != null && sites.get(site).size() >= 1) || isArgumentFlag();
     }
 
     /**
@@ -276,9 +277,9 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
      */
     public List<String> values() {
         List<String> values = new ArrayList<String>();
-        for( int index: indices.keySet() ) {
+        for( ArgumentMatchSite site: sites.keySet() ) {
-            if( indices.get(index) != null )
+            if( sites.get(site) != null )
-                values.addAll(indices.get(index));
+                values.addAll(sites.get(site));
         }
         return values;
     }

public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchSite.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.commandline;
+
+/**
+ * Which source and the index within the source where an argument match was found.
+ */
+public class ArgumentMatchSite implements Comparable<ArgumentMatchSite> {
+    private final ArgumentMatchSource source;
+    private final int index;
+
+    public ArgumentMatchSite(ArgumentMatchSource source, int index) {
+        this.source = source;
+        this.index = index;
+    }
+
+    public ArgumentMatchSource getSource() {
+        return source;
+    }
+
+    public int getIndex() {
+        return index;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        ArgumentMatchSite that = (ArgumentMatchSite) o;
+
+        return (index == that.index) && (source == null ? that.source == null : source.equals(that.source));
+    }
+
+    @Override
+    public int hashCode() {
+        int result = source != null ? source.hashCode() : 0;
+        // Generated by intellij. No other special reason to this implementation. -ks
+        result = 31 * result + index;
+        return result;
+    }
+
+    @Override
+    public int compareTo(ArgumentMatchSite that) {
+        int comp = this.source.compareTo(that.source);
+        if (comp != 0)
+            return comp;
+
+        // Both files are the same.
+        if (this.index == that.index)
+            return 0;
+        return this.index < that.index ? -1 : 1;
+    }
+}

public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchSource.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.commandline;
+
+import java.io.File;
+
+/**
+ * Where an argument match originated, via the commandline or a file.
+ */
+public class ArgumentMatchSource implements Comparable<ArgumentMatchSource> {
+    public static final ArgumentMatchSource COMMAND_LINE = new ArgumentMatchSource(ArgumentMatchSourceType.CommandLine, null);
+
+    private final ArgumentMatchSourceType type;
+    private final File file;
+
+    /**
+     * Creates an argument match source from the specified file.
+     * @param file File specifying the arguments. Must not be null.
+     */
+    public ArgumentMatchSource(File file) {
+        this(ArgumentMatchSourceType.File, file);
+    }
+
+    private ArgumentMatchSource(ArgumentMatchSourceType type, File file) {
+        if (type == ArgumentMatchSourceType.File && file == null)
+            throw new IllegalArgumentException("An argument match source of type File cannot have a null file.");
+        this.type = type;
+        this.file = file;
+    }
+
+    public ArgumentMatchSourceType getType() {
+        return type;
+    }
+
+    public File getFile() {
+        return file;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        ArgumentMatchSource that = (ArgumentMatchSource) o;
+
+        return (type == that.type) && (file == null ? that.file == null : file.equals(that.file));
+    }
+
+    @Override
+    public int hashCode() {
+        int result = type != null ? type.hashCode() : 0;
+        result = 31 * result + (file != null ? file.hashCode() : 0);
+        return result;
+    }
+
+    /**
+     * Compares two sources, putting the command line first, then files.
+     */
+    @Override
+    public int compareTo(ArgumentMatchSource that) {
+        int comp = this.type.compareTo(that.type);
+        if (comp != 0)
+            return comp;
+
+        File f1 = this.file;
+        File f2 = that.file;
+
+        if ((f1 == null) ^ (f2 == null)) {
+            // If one of the files is null and the other is not
+            // put the null file first
+            return f1 == null ? -1 : 1;
+        }
+
+        return f1 == null ? 0 : f1.compareTo(f2);
+    }
+}

public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchSourceType.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.commandline;
+
+/**
+ * Type of where an argument match originated, via the commandline or a file.
+ */
+public enum ArgumentMatchSourceType {
+    CommandLine, File
+}

public/java/src/org/broadinstitute/sting/commandline/ArgumentMatches.java

@@ -37,7 +37,7 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
      * Collection matches from argument definition to argument value.
      * Package protected access is deliberate.
      */
-    Map<Integer,ArgumentMatch> argumentMatches = new TreeMap<Integer,ArgumentMatch>();
+    Map<ArgumentMatchSite,ArgumentMatch> argumentMatches = new TreeMap<ArgumentMatchSite,ArgumentMatch>();
 
     /**
      * Provide a place to put command-line argument values that don't seem to belong to
@@ -80,7 +80,7 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
      * @param site Site at which to check.
      * @return True if the site has a match.  False otherwise.
      */
-    boolean hasMatch( int site ) {
+    boolean hasMatch( ArgumentMatchSite site ) {
         return argumentMatches.containsKey( site );
     }
 
@@ -90,7 +90,7 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
      * @return The match present at the given site.
      * @throws IllegalArgumentException if site does not contain a match.
      */
-    ArgumentMatch getMatch( int site ) {
+    ArgumentMatch getMatch( ArgumentMatchSite site ) {
         if( !argumentMatches.containsKey(site) )
             throw new IllegalArgumentException( "Site does not contain an argument: " + site );
         return argumentMatches.get(site);
@@ -107,6 +107,7 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
 
     /**
      * Return all argument matches of this source.
+     * @param parsingEngine Parsing engine.
      * @param argumentSource Argument source to match.
      * @return List of all matches.
      */
@@ -167,6 +168,7 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
      * TODO: Generify this.
      * @param multiplexer Multiplexer that controls the transformation process.
      * @param key Key which specifies the transform.
+     * @return new argument matches.
      */
     ArgumentMatches transform(Multiplexer multiplexer, Object key) {
         ArgumentMatches newArgumentMatches = new ArgumentMatches();
@@ -187,15 +189,15 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
         for( ArgumentMatch argumentMatch: getUniqueMatches() ) {
             if( argumentMatch.definition == match.definition && argumentMatch.tags.equals(match.tags) ) {
                 argumentMatch.mergeInto( match );
-                for( int index: match.indices.keySet() )
+                for( ArgumentMatchSite site: match.sites.keySet() )
-                    argumentMatches.put( index, argumentMatch );
+                    argumentMatches.put( site, argumentMatch );
                 definitionExists = true;
             }
         }
 
         if( !definitionExists ) {
-            for( int index: match.indices.keySet() )
+            for( ArgumentMatchSite site: match.sites.keySet() )
-                argumentMatches.put( index, match );
+                argumentMatches.put( site, match );
         }
     }    
 

public/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java

@@ -35,10 +35,7 @@ import org.broadinstitute.sting.utils.help.ApplicationDetails;
 import org.broadinstitute.sting.utils.help.HelpFormatter;
 
 import java.io.IOException;
-import java.util.Collection;
+import java.util.*;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.Locale;
 
 public abstract class CommandLineProgram {
 
@@ -155,6 +152,7 @@ public abstract class CommandLineProgram {
      *
      * @param clp  the command line program to execute
      * @param args the command line arguments passed in
+     * @param dryRun dry run
      * @throws Exception when an exception occurs
      */
     @SuppressWarnings("unchecked")
@@ -176,6 +174,8 @@ public abstract class CommandLineProgram {
             ParsingEngine parser = clp.parser = new ParsingEngine(clp);
             parser.addArgumentSource(clp.getClass());
 
+            Map<ArgumentMatchSource, List<String>> parsedArgs;
+
             // process the args
             if (clp.canAddArgumentsDynamically()) {
                 // if the command-line program can toss in extra args, fetch them and reparse the arguments.
@@ -196,14 +196,14 @@ public abstract class CommandLineProgram {
                 Class[] argumentSources = clp.getArgumentSources();
                 for (Class argumentSource : argumentSources)
                     parser.addArgumentSource(clp.getArgumentSourceName(argumentSource), argumentSource);
-                parser.parse(args);
+                parsedArgs = parser.parse(args);
 
                 if (isHelpPresent(parser))
                     printHelpAndExit(clp, parser);
 
                 if ( ! dryRun ) parser.validate();
             } else {
-                parser.parse(args);
+                parsedArgs = parser.parse(args);
 
                 if ( ! dryRun ) {
                     if (isHelpPresent(parser))
@@ -230,7 +230,7 @@ public abstract class CommandLineProgram {
                 }
 
                 // regardless of what happens next, generate the header information
-                HelpFormatter.generateHeaderInformation(clp.getApplicationDetails(), args);
+                HelpFormatter.generateHeaderInformation(clp.getApplicationDetails(), parsedArgs);
 
                 // call the execute
                 CommandLineProgram.result = clp.execute();

public/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java

@@ -26,6 +26,7 @@
 package org.broadinstitute.sting.commandline;
 
 import com.google.java.contract.Requires;
+import org.apache.commons.io.FileUtils;
 import org.apache.log4j.Logger;
 import org.broadinstitute.sting.utils.Utils;
 import org.broadinstitute.sting.utils.classloader.JVMUtils;
@@ -35,6 +36,8 @@ import org.broadinstitute.sting.utils.exceptions.UserException;
 import org.broadinstitute.sting.utils.help.ApplicationDetails;
 import org.broadinstitute.sting.utils.help.HelpFormatter;
 
+import java.io.File;
+import java.io.IOException;
 import java.lang.reflect.Field;
 import java.util.*;
 
@@ -100,6 +103,8 @@ public class ParsingEngine {
         if(clp != null)
             argumentTypeDescriptors.addAll(clp.getArgumentTypeDescriptors());
         argumentTypeDescriptors.addAll(STANDARD_ARGUMENT_TYPE_DESCRIPTORS);
+
+        addArgumentSource(ParsingEngineArgumentFiles.class);
     }
 
     /**
@@ -148,21 +153,43 @@ public class ParsingEngine {
      * command-line arguments to the arguments that are actually
      * required.
      * @param tokens Tokens passed on the command line.
+     * @return The parsed arguments by file.
      */
-    public void parse( String[] tokens ) {
+    public SortedMap<ArgumentMatchSource, List<String>> parse( String[] tokens ) {
         argumentMatches = new ArgumentMatches();
+        SortedMap<ArgumentMatchSource, List<String>> parsedArgs = new TreeMap<ArgumentMatchSource, List<String>>();
 
-        int lastArgumentMatchSite = -1;
+        List<String> cmdLineTokens = Arrays.asList(tokens);
+        parse(ArgumentMatchSource.COMMAND_LINE, cmdLineTokens, argumentMatches, parsedArgs);
 
-        for( int i = 0; i < tokens.length; i++ ) {
+        ParsingEngineArgumentFiles argumentFiles = new ParsingEngineArgumentFiles();
-            String token = tokens[i];
+
+        // Load the arguments ONLY into the argument files.
+        // Validation may optionally run on the rest of the arguments.
+        loadArgumentsIntoObject(argumentFiles);
+
+        for (File file: argumentFiles.files) {
+            List<String> fileTokens = getArguments(file);
+            parse(new ArgumentMatchSource(file), fileTokens, argumentMatches, parsedArgs);
+        }
+
+        return parsedArgs;
+    }
+
+    private void parse(ArgumentMatchSource matchSource, List<String> tokens,
+                       ArgumentMatches argumentMatches, SortedMap<ArgumentMatchSource, List<String>> parsedArgs) {
+        ArgumentMatchSite lastArgumentMatchSite = new ArgumentMatchSite(matchSource, -1);
+
+        int i = 0;
+        for (String token: tokens) {
             // If the token is of argument form, parse it into its own argument match.
             // Otherwise, pair it with the most recently used argument discovered.
+            ArgumentMatchSite site = new ArgumentMatchSite(matchSource, i);
             if( isArgumentForm(token) ) {
-                ArgumentMatch argumentMatch = parseArgument( token, i );
+                ArgumentMatch argumentMatch = parseArgument( token, site );
                 if( argumentMatch != null ) {
                     argumentMatches.mergeInto( argumentMatch );
-                    lastArgumentMatchSite = i;
+                    lastArgumentMatchSite = site;
                 }
             }
             else {
@@ -170,10 +197,31 @@ public class ParsingEngine {
                     !argumentMatches.getMatch(lastArgumentMatchSite).hasValueAtSite(lastArgumentMatchSite))
                     argumentMatches.getMatch(lastArgumentMatchSite).addValue( lastArgumentMatchSite, token );
                 else
-                    argumentMatches.MissingArgument.addValue( i, token );
+                    argumentMatches.MissingArgument.addValue( site, token );
 
             }
+            i++;
         }
+
+        parsedArgs.put(matchSource, tokens);
+    }
+
+    private List<String> getArguments(File file) {
+        try {
+            if (file.getAbsolutePath().endsWith(".list")) {
+                return getListArguments(file);
+            }
+        } catch (IOException e) {
+            throw new UserException.CouldNotReadInputFile(file, e);
+        }
+        throw new UserException.CouldNotReadInputFile(file, "file extension is not .list");
+    }
+
+    private List<String> getListArguments(File file) throws IOException {
+        ArrayList<String> argsList = new ArrayList<String>();
+        for (String line: FileUtils.readLines(file))
+            argsList.addAll(Arrays.asList(Utils.escapeExpressions(line)));
+        return argsList;
     }
 
     public enum ValidationType { MissingRequiredArgument,
@@ -494,7 +542,7 @@ public class ParsingEngine {
      * @param position The position of the token in question.
      * @return ArgumentMatch associated with this token, or null if no match exists.
      */    
-    private ArgumentMatch parseArgument( String token, int position ) {
+    private ArgumentMatch parseArgument( String token, ArgumentMatchSite position ) {
         if( !isArgumentForm(token) )
             throw new IllegalArgumentException( "Token is not recognizable as an argument: " + token );
 
@@ -579,9 +627,21 @@ class UnmatchedArgumentException extends ArgumentException {
 
     private static String formatArguments( ArgumentMatch invalidValues ) {
         StringBuilder sb = new StringBuilder();
-        for( int index: invalidValues.indices.keySet() )
+        for( ArgumentMatchSite site: invalidValues.sites.keySet() )
-            for( String value: invalidValues.indices.get(index) ) {
+            for( String value: invalidValues.sites.get(site) ) {
-                sb.append( String.format("%nInvalid argument value '%s' at position %d.", value, index) );
+                switch (site.getSource().getType()) {
+                    case CommandLine:
+                        sb.append( String.format("%nInvalid argument value '%s' at position %d.",
+                                value, site.getIndex()) );
+                        break;
+                    case File:
+                        sb.append( String.format("%nInvalid argument value '%s' in file %s at position %d.",
+                                value, site.getSource().getFile().getAbsolutePath(), site.getIndex()) );
+                        break;
+                    default:
+                        throw new RuntimeException( String.format("Unexpected argument match source type: %s",
+                                site.getSource().getType()));
+                }
                 if(value != null && Utils.dupString(' ',value.length()).equals(value))
                     sb.append("  Please make sure any line continuation backslashes on your command line are not followed by whitespace.");
             }
@@ -635,3 +695,12 @@ class UnknownEnumeratedValueException extends ArgumentException {
         return String.format("Invalid value %s specified for argument %s; valid options are (%s).", argumentPassed, definition.fullName, Utils.join(",",definition.validOptions));
     }
 }
+
+/**
+ * Container class to store the list of argument files.
+ * The files will be parsed after the command line arguments.
+ */
+class ParsingEngineArgumentFiles {
+    @Argument(fullName = "arg_file", shortName = "args", doc = "Reads arguments from the specified file", required = false)
+    public List<File> files = new ArrayList<File>();
+}

public/java/src/org/broadinstitute/sting/commandline/ParsingMethod.java

@@ -68,7 +68,7 @@ public abstract class ParsingMethod {
      * @return An argument match.  Definition field will be populated if a match was found or
      *         empty if no appropriate definition could be found. 
      */
-    public ArgumentMatch match( ArgumentDefinitions definitions, String token, int position ) {
+    public ArgumentMatch match( ArgumentDefinitions definitions, String token, ArgumentMatchSite position ) {
         // If the argument is valid, parse out the argument.
         Matcher matcher = pattern.matcher(token);
 
@@ -102,9 +102,7 @@ public abstract class ParsingMethod {
 
         // Try to find a matching argument.  If found, label that as the match.  If not found, add the argument
         // with a null definition.
-        ArgumentMatch argumentMatch = new ArgumentMatch(argument,argumentDefinition,position,tags);
+        return new ArgumentMatch(argument,argumentDefinition,position,tags);
-
-        return argumentMatch;
     }
 
     /**

public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java

@@ -220,12 +220,12 @@ public class GenomeAnalysisEngine {
         ShardStrategy shardStrategy = getShardStrategy(readsDataSource,microScheduler.getReference(),intervals);
 
         // execute the microscheduler, storing the results
-        Object result =  microScheduler.execute(this.walker, shardStrategy);
+        return microScheduler.execute(this.walker, shardStrategy);
 
         //monitor.stop();
         //logger.info(String.format("Maximum heap size consumed: %d",monitor.getMaxMemoryUsed()));
 
-        return result;
+        //return result;
     }
 
     /**
@@ -296,10 +296,14 @@ public class GenomeAnalysisEngine {
         else if(WalkerManager.getDownsamplingMethod(walker) != null)
             method = WalkerManager.getDownsamplingMethod(walker);
         else
-            method = argCollection.getDefaultDownsamplingMethod();
+            method = GATKArgumentCollection.getDefaultDownsamplingMethod();
         return method;
     }
 
+    protected void setDownsamplingMethod(DownsamplingMethod method) {
+        argCollection.setDownsamplingMethod(method);
+    }
+
     public BAQ.QualityMode getWalkerBAQQualityMode()         { return WalkerManager.getBAQQualityMode(walker); }
     public BAQ.ApplicationTime getWalkerBAQApplicationTime() { return WalkerManager.getBAQApplicationTime(walker); }    
 
@@ -389,7 +393,9 @@ public class GenomeAnalysisEngine {
     /**
      * Get the sharding strategy given a driving data source.
      *
+     * @param readsDataSource readsDataSource
      * @param drivingDataSource Data on which to shard.
+     * @param intervals intervals
      * @return the sharding strategy
      */
     protected ShardStrategy getShardStrategy(SAMDataSource readsDataSource, ReferenceSequenceFile drivingDataSource, GenomeLocSortedSet intervals) {
@@ -426,7 +432,7 @@ public class GenomeAnalysisEngine {
             return new MonolithicShardStrategy(getGenomeLocParser(), readsDataSource,shardType,region);
         }
 
-        ShardStrategy shardStrategy = null;
+        ShardStrategy shardStrategy;
         ShardStrategyFactory.SHATTER_STRATEGY shardType;
 
         long SHARD_SIZE = 100000L;
@@ -435,6 +441,8 @@ public class GenomeAnalysisEngine {
             if (walker instanceof RodWalker) SHARD_SIZE *= 1000;
 
             if (intervals != null && !intervals.isEmpty()) {
+                if (readsDataSource == null)
+                    throw new IllegalArgumentException("readsDataSource is null");
                 if(!readsDataSource.isEmpty() && readsDataSource.getSortOrder() != SAMFileHeader.SortOrder.coordinate)
                     throw new UserException.MissortedBAM(SAMFileHeader.SortOrder.coordinate, "Locus walkers can only traverse coordinate-sorted data.  Please resort your input BAM file(s) or set the Sort Order tag in the header appropriately.");
 
@@ -498,7 +506,8 @@ public class GenomeAnalysisEngine {
      */
     private void initializeTempDirectory() {
         File tempDir = new File(System.getProperty("java.io.tmpdir"));
-        tempDir.mkdirs();
+        if (!tempDir.exists() && !tempDir.mkdirs())
+            throw new UserException.BadTmpDir("Unable to create directory");
     }
 
     /**
@@ -729,6 +738,7 @@ public class GenomeAnalysisEngine {
      * @param reads     Reads data source.
      * @param reference Reference data source.
      * @param rods    a collection of the reference ordered data tracks
+     * @param manager manager
      */
     private void validateSourcesAgainstReference(SAMDataSource reads, ReferenceSequenceFile reference, Collection<ReferenceOrderedDataSource> rods, RMDTrackBuilder manager) {
         if ((reads.isEmpty() && (rods == null || rods.isEmpty())) || reference == null )
@@ -757,15 +767,22 @@ public class GenomeAnalysisEngine {
     /**
      * Gets a data source for the given set of reads.
      *
+     * @param argCollection arguments
+     * @param genomeLocParser parser
+     * @param refReader reader
      * @return A data source for the given set of reads.
      */
     private SAMDataSource createReadsDataSource(GATKArgumentCollection argCollection, GenomeLocParser genomeLocParser, IndexedFastaSequenceFile refReader) {
         DownsamplingMethod method = getDownsamplingMethod();
 
+        // Synchronize the method back into the collection so that it shows up when
+        // interrogating for the downsample method during command line recreation.
+        setDownsamplingMethod(method);
+
         if ( getWalkerBAQApplicationTime() == BAQ.ApplicationTime.FORBIDDEN && argCollection.BAQMode != BAQ.CalculationMode.OFF)
             throw new UserException.BadArgumentValue("baq", "Walker cannot accept BAQ'd base qualities, and yet BAQ mode " + argCollection.BAQMode + " was requested.");
 
-        SAMDataSource dataSource = new SAMDataSource(
+        return new SAMDataSource(
                 samReaderIDs,
                 genomeLocParser,
                 argCollection.useOriginalBaseQualities,
@@ -781,14 +798,12 @@ public class GenomeAnalysisEngine {
                 refReader,
                 argCollection.defaultBaseQualities,
                 !argCollection.disableLowMemorySharding);
-        return dataSource;
     }
 
     /**
      * Opens a reference sequence file paired with an index.  Only public for testing purposes
      *
      * @param refFile Handle to a reference sequence file.  Non-null.
-     * @return A thread-safe file wrapper.
      */
     public void setReferenceDataSource(File refFile) {
         this.referenceDataSource = new ReferenceDataSource(refFile);

public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java

@@ -135,8 +135,8 @@ public class GATKArgumentCollection {
 
     /**
      * Gets the downsampling method explicitly specified by the user.  If the user didn't specify
-     * a default downsampling mechanism, return null.
+     * a default downsampling mechanism, return the default.
-     * @return The explicitly specified downsampling mechanism, or null if none exists.
+     * @return The explicitly specified downsampling mechanism, or the default if none exists.
      */
     public DownsamplingMethod getDownsamplingMethod() {
         if(downsamplingType == null && downsampleFraction == null && downsampleCoverage == null)
@@ -146,6 +146,18 @@ public class GATKArgumentCollection {
         return new DownsamplingMethod(downsamplingType,downsampleCoverage,downsampleFraction);
     }
 
+    /**
+     * Set the downsampling method stored in the argument collection so that it is read back out when interrogating the command line arguments.
+     * @param method The downsampling mechanism.
+     */
+    public void setDownsamplingMethod(DownsamplingMethod method) {
+        if (method == null)
+            throw new IllegalArgumentException("method is null");
+        downsamplingType = method.type;
+        downsampleCoverage = method.toCoverage;
+        downsampleFraction = method.toFraction;
+    }
+
     // --------------------------------------------------------------------------------------------------------------
     //
     // BAQ arguments

public/java/src/org/broadinstitute/sting/utils/R/RScriptExecutor.java

@@ -25,35 +25,35 @@
 package org.broadinstitute.sting.utils.R;
 
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
 import org.broadinstitute.sting.commandline.Advanced;
 import org.broadinstitute.sting.commandline.Argument;
-import org.broadinstitute.sting.commandline.ArgumentCollection;
-import org.broadinstitute.sting.gatk.walkers.recalibration.Covariate;
-import org.broadinstitute.sting.utils.PathUtils;
 import org.broadinstitute.sting.utils.Utils;
+import org.broadinstitute.sting.utils.exceptions.StingException;
 import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.broadinstitute.sting.utils.io.IOUtils;
+import org.broadinstitute.sting.utils.io.Resource;
+import org.broadinstitute.sting.utils.runtime.ProcessController;
+import org.broadinstitute.sting.utils.runtime.ProcessSettings;
 
 import java.io.File;
-import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
 /**
- * Generic service for executing RScripts in the GATK directory
+ * Generic service for executing RScripts
- *
- * @author Your Name
- * @since Date created
  */
 public class RScriptExecutor {
     /**
      * our log
      */
-    protected static Logger logger = Logger.getLogger(RScriptExecutor.class);
+    private static Logger logger = Logger.getLogger(RScriptExecutor.class);
 
     public static class RScriptArgumentCollection {
         @Advanced
-        @Argument(fullName = "path_to_Rscript", shortName = "Rscript", doc = "The path to your implementation of Rscript. For Broad users this is maybe /broad/software/free/Linux/redhat_5_x86_64/pkgs/r_2.12.0/bin/Rscript", required = false)
+        @Argument(fullName = "path_to_Rscript", shortName = "Rscript", doc = "The path to your implementation of Rscript. Defaults Rscript meaning to use the first available on the environment PATH. For Broad users should 'use R-2.12' or later.", required = false)
         public String PATH_TO_RSCRIPT = "Rscript";
 
         @Advanced
@@ -62,40 +62,119 @@ public class RScriptExecutor {
 
         public RScriptArgumentCollection() {}
 
-        /** For testing and convenience */
+        /* For testing and convenience */
         public RScriptArgumentCollection(final String PATH_TO_RSCRIPT, final List<String> PATH_TO_RESOURCES) {
             this.PATH_TO_RSCRIPT = PATH_TO_RSCRIPT;
             this.PATH_TO_RESOURCES = PATH_TO_RESOURCES;
         }
     }
 
-    final RScriptArgumentCollection myArgs;
+    private final RScriptArgumentCollection myArgs;
-    final boolean exceptOnError;
+    private final boolean exceptOnError;
+    private final List<RScriptLibrary> libraries = new ArrayList<RScriptLibrary>();
+    private final List<Resource> scriptResources = new ArrayList<Resource>();
+    private final List<File> scriptFiles = new ArrayList<File>();
+    private final List<String> args = new ArrayList<String>();
 
     public RScriptExecutor(final RScriptArgumentCollection myArgs, final boolean exceptOnError) {
         this.myArgs = myArgs;
         this.exceptOnError = exceptOnError;
     }
 
-    public void callRScripts(String scriptName, Object... scriptArgs) {
+    public void addLibrary(RScriptLibrary library) {
-        callRScripts(scriptName, Arrays.asList(scriptArgs));
+        this.libraries.add(library);
     }
 
-    public void callRScripts(String scriptName, List<Object> scriptArgs) {
+    public void addScript(Resource script) {
+        this.scriptResources.add(script);
+    }
+
+    public void addScript(File script) {
+        this.scriptFiles.add(script);
+    }
+
+    /**
+     * Adds args to the end of the Rscript command line.
+     * @param args the args.
+     * @throws NullPointerException if any of the args are null.
+     */
+    public void addArgs(Object... args) {
+        for (Object arg: args)
+            this.args.add(arg.toString());
+    }
+
+    public void exec() {
+        List<File> tempFiles = new ArrayList<File>();
         try {
-            final File pathToScript = findScript(scriptName);
+            File tempLibDir = IOUtils.tempDir("R.", ".lib");
-            if ( pathToScript == null ) return; // we failed but shouldn't exception out
+            tempFiles.add(tempLibDir);
-            final String argString = Utils.join(" ", scriptArgs);
+
-            final String cmdLine = Utils.join(" ", Arrays.asList(myArgs.PATH_TO_RSCRIPT, pathToScript, argString));
+            StringBuilder expression = new StringBuilder("tempLibDir = '").append(tempLibDir).append("';");
-            logger.info("Executing RScript: " + cmdLine);
+
-            Runtime.getRuntime().exec(cmdLine).waitFor();
+            if (this.libraries.size() > 0) {
-        } catch (InterruptedException e) {
+                List<String> tempLibraryPaths = new ArrayList<String>();
+                for (RScriptLibrary library: this.libraries) {
+                    File tempLibrary = library.writeTemp();
+                    tempFiles.add(tempLibrary);
+                    tempLibraryPaths.add(tempLibrary.getAbsolutePath());
+                }
+
+                expression.append("install.packages(");
+                expression.append("pkgs=c('").append(StringUtils.join(tempLibraryPaths, "', '")).append("'), lib=tempLibDir, repos=NULL, type='source', ");
+                // Install faster by eliminating cruft.
+                expression.append("INSTALL_opts=c('--no-libs', '--no-data', '--no-help', '--no-demo', '--no-exec')");
+                expression.append(");");
+
+                for (RScriptLibrary library: this.libraries) {
+                    expression.append("require('").append(library.getLibraryName()).append("', lib.loc=tempLibDir);");
+                }
+            }
+
+            for (Resource script: this.scriptResources) {
+                File tempScript = IOUtils.writeTempResource(script);
+                tempFiles.add(tempScript);
+                expression.append("source('").append(tempScript.getAbsolutePath()).append("');");
+            }
+
+            for (File script: this.scriptFiles) {
+                expression.append("source('").append(script.getAbsolutePath()).append("');");
+            }
+
+            String[] cmd = new String[this.args.size() + 3];
+            int i = 0;
+            cmd[i++] = myArgs.PATH_TO_RSCRIPT;
+            cmd[i++] = "-e";
+            cmd[i++] = expression.toString();
+            for (String arg: this.args)
+                cmd[i++] = arg;
+
+            ProcessSettings processSettings = new ProcessSettings(cmd);
+            if (logger.isDebugEnabled()) {
+                processSettings.getStdoutSettings().printStandard(true);
+                processSettings.getStderrSettings().printStandard(true);
+            }
+
+            ProcessController controller = ProcessController.getThreadLocal();
+
+            logger.debug("Executing: " + Utils.join(" ", cmd));
+            logger.debug("Result: " + controller.exec(processSettings).getExitValue());
+
+        } catch (StingException e) {
             generateException(e);
-        } catch (IOException e) {
+        } finally {
-            generateException("Fatal Exception: Perhaps RScript jobs are being spawned too quickly?", e);
+            for (File temp: tempFiles)
+                FileUtils.deleteQuietly(temp);
         }
     }
 
+    public void callRScripts(String scriptName, Object... scriptArgs) {
+        final File pathToScript = findScript(scriptName);
+        if (pathToScript == null) return; // we failed but shouldn't exception out
+        addScript(pathToScript);
+        addArgs(scriptArgs);
+        exec();
+    }
+
     public File findScript(final String scriptName) {
         for ( String pathToResource : myArgs.PATH_TO_RESOURCES ) {
             final File f = new File(pathToResource + "/" + scriptName);

public/java/src/org/broadinstitute/sting/utils/R/RScriptLibrary.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.R;
+
+import org.broadinstitute.sting.utils.io.IOUtils;
+import org.broadinstitute.sting.utils.io.Resource;
+
+import java.io.File;
+
+/**
+ * Libraries embedded in the StingUtils package.
+ */
+public enum RScriptLibrary {
+    GSALIB("gsalib");
+
+    private final String name;
+
+    private RScriptLibrary(String name) {
+        this.name = name;
+    }
+
+    public String getLibraryName() {
+        return this.name;
+    }
+
+    public String getResourcePath() {
+        return name + ".tar.gz";
+    }
+
+    /**
+     * Writes the library source code to a temporary tar.gz file and returns the path.
+     * @return The path to the library source code. The caller must delete the code when done.
+     */
+    public File writeTemp() {
+        return IOUtils.writeTempResource(new Resource(getResourcePath(), RScriptLibrary.class));
+    }
+}

public/java/src/org/broadinstitute/sting/utils/help/HelpFormatter.java

@@ -29,6 +29,7 @@ import org.apache.log4j.Logger;
 import org.broadinstitute.sting.commandline.ArgumentDefinition;
 import org.broadinstitute.sting.commandline.ArgumentDefinitionGroup;
 import org.broadinstitute.sting.commandline.ArgumentDefinitions;
+import org.broadinstitute.sting.commandline.ArgumentMatchSource;
 import org.broadinstitute.sting.utils.Utils;
 import org.broadinstitute.sting.utils.text.TextFormattingUtils;
 
@@ -47,6 +48,7 @@ public class HelpFormatter {
 
     /**
      * Prints the help, given a collection of argument definitions.
+     * @param applicationDetails Application details
      * @param argumentDefinitions Argument definitions for which help should be printed.
      */
     public void printHelp( ApplicationDetails applicationDetails, ArgumentDefinitions argumentDefinitions ) {
@@ -233,7 +235,7 @@ public class HelpFormatter {
     private List<ArgumentDefinitionGroup> prepareArgumentGroups( ArgumentDefinitions argumentDefinitions ) {
         // Sort the list of argument definitions according to how they should be shown.
         // Put the sorted results into a new cloned data structure.
-        Comparator definitionComparator = new Comparator<ArgumentDefinition>() {
+        Comparator<ArgumentDefinition> definitionComparator = new Comparator<ArgumentDefinition>() {
             public int compare( ArgumentDefinition lhs, ArgumentDefinition rhs ) {
                 if( lhs.required && rhs.required ) return 0;
                 if( lhs.required ) return -1;
@@ -242,15 +244,15 @@ public class HelpFormatter {
             }
         };
 
-        List<ArgumentDefinitionGroup> argumentGroups = new ArrayList();        
+        List<ArgumentDefinitionGroup> argumentGroups = new ArrayList<ArgumentDefinitionGroup>();
         for( ArgumentDefinitionGroup argumentGroup: argumentDefinitions.getArgumentDefinitionGroups() ) {
-            List<ArgumentDefinition> sortedDefinitions = new ArrayList( argumentGroup.argumentDefinitions );
+            List<ArgumentDefinition> sortedDefinitions = new ArrayList<ArgumentDefinition>( argumentGroup.argumentDefinitions );
             Collections.sort( sortedDefinitions, definitionComparator );
             argumentGroups.add( new ArgumentDefinitionGroup(argumentGroup.groupName,sortedDefinitions) );
         }
 
         // Sort the argument groups themselves with main arguments first, followed by plugins sorted in name order.
-        Comparator groupComparator = new Comparator<ArgumentDefinitionGroup>() {
+        Comparator<ArgumentDefinitionGroup> groupComparator = new Comparator<ArgumentDefinitionGroup>() {
             public int compare( ArgumentDefinitionGroup lhs, ArgumentDefinitionGroup rhs ) {
                 if( lhs.groupName == null && rhs.groupName == null ) return 0;
                 if( lhs.groupName == null ) return -1;
@@ -271,9 +273,9 @@ public class HelpFormatter {
      * Generate a standard header for the logger
      *
      * @param applicationDetails details of the application to run.
-     * @param args the command line arguments passed in
+     * @param parsedArgs the command line arguments passed in
      */
-    public static void generateHeaderInformation(ApplicationDetails applicationDetails, String[] args) {
+    public static void generateHeaderInformation(ApplicationDetails applicationDetails, Map<ArgumentMatchSource, List<String>> parsedArgs) {
 
         DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
         java.util.Date date = new java.util.Date();
@@ -283,11 +285,22 @@ public class HelpFormatter {
         logger.info(barrier);
         for (String headerLine : applicationDetails.applicationHeader)
             logger.info(headerLine);
-        String output = "";
+        logger.debug("Current directory: " + System.getProperty("user.dir"));
-        for (String str : args) {
+        for (Map.Entry<ArgumentMatchSource, List<String>> entry: parsedArgs.entrySet()) {
-            output = output + str + " ";
+            ArgumentMatchSource matchSource = entry.getKey();
+            final String sourceName;
+            switch (matchSource.getType()) {
+                case CommandLine: sourceName = "Program"; break;
+                case File: sourceName = matchSource.getFile().getPath(); break;
+                default: throw new RuntimeException("Unexpected argument match source type: " + matchSource.getType());
+            }
+
+            String output = sourceName + " Args:";
+            for (String str : entry.getValue()) {
+                output = output + " " + str;
+            }
+            logger.info(output);
         }
-        logger.info("Program Args: " + output);
         logger.info("Date/Time: " + dateFormat.format(date));
         logger.info(barrier);
 

public/java/src/org/broadinstitute/sting/utils/io/FileExtension.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person 
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.io;
+
+import java.io.File;
+
+public interface FileExtension {
+    /**
+     * Returns a clone of the FileExtension with a new path.
+     * @param path New path.
+     * @return New FileExtension
+     */
+    public File withPath(String path);
+}

public/java/src/org/broadinstitute/sting/utils/io/HardThresholdingOutputStream.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+package org.broadinstitute.sting.utils.io;
+
+import org.apache.commons.io.output.ThresholdingOutputStream;
+
+import java.io.IOException;
+
+/**
+ * An output stream which stops at the threshold
+ * instead of potentially triggering early.
+ */
+public abstract class HardThresholdingOutputStream extends ThresholdingOutputStream {
+    protected HardThresholdingOutputStream(int threshold) {
+        super(threshold);
+    }
+
+    @Override
+    public void write(byte[] b) throws IOException {
+        write(b, 0, b.length);
+    }
+
+    @Override
+    public void write(byte[] b, int off, int len) throws IOException {
+        int remaining = this.getThreshold() - (int)this.getByteCount();
+        if (!isThresholdExceeded() && len > remaining) {
+            super.write(b, off, remaining);
+            super.write(b, off + remaining, len - remaining);
+        } else {
+            super.write(b, off, len);
+        }
+    }
+}

public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java

@@ -0,0 +1,353 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.io;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.io.LineIterator;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+import org.broadinstitute.sting.utils.exceptions.StingException;
+import org.broadinstitute.sting.utils.exceptions.UserException;
+
+import java.io.*;
+import java.util.*;
+
+public class IOUtils {
+    private static Logger logger = Logger.getLogger(IOUtils.class);
+
+    /**
+     * Checks if the temp directory has been setup and throws an exception if they user hasn't set it correctly.
+     *
+     * @param tempDir Temporary directory.
+     */
+    public static void checkTempDir(File tempDir) {
+        String tempDirPath = tempDir.getAbsolutePath();
+        // Keeps the user from leaving the temp directory as the default, and on Macs from having pluses
+        // in the path which can cause problems with the Google Reflections library.
+        // see also: http://benjchristensen.com/2009/09/22/mac-osx-10-6-java-java-io-tmpdir/
+        if (tempDirPath.startsWith("/var/folders/") || (tempDirPath.equals("/tmp")) || (tempDirPath.equals("/tmp/")))
+            throw new UserException.BadTmpDir("java.io.tmpdir must be explicitly set");
+        if (!tempDir.exists() && !tempDir.mkdirs())
+            throw new UserException.BadTmpDir("Could not create directory: " + tempDir.getAbsolutePath());
+    }
+
+    /**
+     * Creates a temp directory with the prefix and optional suffix.
+     *
+     * @param prefix       Prefix for the directory name.
+     * @param suffix       Optional suffix for the directory name.
+     * @return The created temporary directory.
+     */
+    public static File tempDir(String prefix, String suffix) {
+        return tempDir(prefix, suffix, null);
+    }
+
+    /**
+     * Creates a temp directory with the prefix and optional suffix.
+     *
+     * @param prefix        Prefix for the directory name.
+     * @param suffix        Optional suffix for the directory name.
+     * @param tempDirParent Parent directory for the temp directory.
+     * @return The created temporary directory.
+     */
+    public static File tempDir(String prefix, String suffix, File tempDirParent) {
+        try {
+            if (tempDirParent == null)
+                tempDirParent = FileUtils.getTempDirectory();
+            if (!tempDirParent.exists() && !tempDirParent.mkdirs())
+                throw new UserException.BadTmpDir("Could not create temp directory: " + tempDirParent);
+            File temp = File.createTempFile(prefix + "-", suffix, tempDirParent);
+            if (!temp.delete())
+                throw new UserException.BadTmpDir("Could not delete sub file: " + temp.getAbsolutePath());
+            if (!temp.mkdir())
+                throw new UserException.BadTmpDir("Could not create sub directory: " + temp.getAbsolutePath());
+            return absolute(temp);
+        } catch (IOException e) {
+            throw new UserException.BadTmpDir(e.getMessage());
+        }
+    }
+
+    /**
+     * Writes content to a temp file and returns the path to the temporary file.
+     *
+     * @param content   to write.
+     * @param prefix    Prefix for the temp file.
+     * @param suffix    Suffix for the temp file.
+     * @param directory Directory for the temp file.
+     * @return the path to the temp file.
+     */
+    public static File writeTempFile(String content, String prefix, String suffix, File directory) {
+        try {
+            File tempFile = absolute(File.createTempFile(prefix, suffix, directory));
+            FileUtils.writeStringToFile(tempFile, content);
+            return tempFile;
+        } catch (IOException e) {
+            throw new UserException.BadTmpDir(e.getMessage());
+        }
+    }
+
+    /**
+     * Waits for NFS to propagate a file creation, imposing a timeout.
+     *
+     * Based on Apache Commons IO FileUtils.waitFor()
+     *
+     * @param file    The file to wait for.
+     * @param seconds The maximum time in seconds to wait.
+     * @return true if the file exists
+     */
+    public static boolean waitFor(File file, int seconds) {
+        return waitFor(Collections.singletonList(file), seconds).isEmpty();
+    }
+
+    /**
+     * Waits for NFS to propagate a file creation, imposing a timeout.
+     *
+     * Based on Apache Commons IO FileUtils.waitFor()
+     *
+     * @param files   The list of files to wait for.
+     * @param seconds The maximum time in seconds to wait.
+     * @return Files that still do not exists at the end of the timeout, or a empty list if all files exists.
+     */
+    public static List<File> waitFor(Collection<File> files, int seconds) {
+        long timeout = 0;
+        long tick = 0;
+        List<File> missingFiles = new ArrayList<File>();
+        for (File file : files)
+            if (!file.exists())
+                missingFiles.add(file);
+
+        while (!missingFiles.isEmpty() && timeout <= seconds) {
+            if (tick >= 10) {
+                tick = 0;
+                timeout++;
+            }
+            tick++;
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException ignore) {
+            }
+            List<File> newMissingFiles = new ArrayList<File>();
+            for (File file : missingFiles)
+                if (!file.exists())
+                    newMissingFiles.add(file);
+            missingFiles = newMissingFiles;
+        }
+        return missingFiles;
+    }
+
+    /**
+     * Returns the directory at the number of levels deep.
+     * For example 2 levels of /path/to/dir will return /path/to
+     *
+     * @param dir   Directory path.
+     * @param level how many levels deep from the root.
+     * @return The path to the parent directory that is level-levels deep.
+     */
+    public static File dirLevel(File dir, int level) {
+        List<File> directories = new ArrayList<File>();
+        File parentDir = absolute(dir);
+        while (parentDir != null) {
+            directories.add(0, parentDir);
+            parentDir = parentDir.getParentFile();
+        }
+        if (directories.size() <= level)
+            return directories.get(directories.size() - 1);
+        else
+            return directories.get(level);
+    }
+
+    /**
+     * Returns the sub path rooted at the parent.
+     *
+     * @param parent The parent directory.
+     * @param path   The sub path to append to the parent, if the path is not absolute.
+     * @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path.
+     */
+    public static File absolute(File parent, String path) {
+        return absolute(parent, new File(path));
+    }
+
+    /**
+     * Returns the sub path rooted at the parent.
+     *
+     * @param parent The parent directory.
+     * @param file   The sub path to append to the parent, if the path is not absolute.
+     * @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path.
+     */
+    public static File absolute(File parent, File file) {
+        String newPath;
+        if (file.isAbsolute())
+            newPath = absolutePath(file);
+        else
+            newPath = absolutePath(new File(parent, file.getPath()));
+        return replacePath(file, newPath);
+    }
+
+    /**
+     * A mix of getCanonicalFile and getAbsoluteFile that returns the
+     * absolute path to the file without deferencing symbolic links.
+     *
+     * @param file the file.
+     * @return the absolute path to the file.
+     */
+    public static File absolute(File file) {
+        return replacePath(file, absolutePath(file));
+    }
+
+    private static String absolutePath(File file) {
+        File fileAbs = file.getAbsoluteFile();
+        LinkedList<String> names = new LinkedList<String>();
+        while (fileAbs != null) {
+            String name = fileAbs.getName();
+            fileAbs = fileAbs.getParentFile();
+
+            if (".".equals(name)) {
+                /* skip */
+
+                /* TODO: What do we do for ".."?
+              } else if (name == "..") {
+
+                CentOS tcsh says use getCanonicalFile:
+                ~ $ mkdir -p test1/test2
+                ~ $ ln -s test1/test2 test3
+                ~ $ cd test3/..
+                ~/test1 $
+
+                Mac bash says keep going with getAbsoluteFile:
+                ~ $ mkdir -p test1/test2
+                ~ $ ln -s test1/test2 test3
+                ~ $ cd test3/..
+                ~ $
+
+                For now, leave it and let the shell figure it out.
+                */
+            } else {
+                names.add(0, name);
+            }
+        }
+
+        return ("/" + StringUtils.join(names, "/"));
+    }
+
+    private static File replacePath(File file, String path) {
+        if (file instanceof FileExtension)
+            return ((FileExtension)file).withPath(path);
+        if (!File.class.equals(file.getClass()))
+            throw new StingException("Sub classes of java.io.File must also implement FileExtension");
+        return new File(path);
+    }
+
+    /**
+     * Returns the last lines of the file.
+     * NOTE: This is only safe to run on smaller files!
+     *
+     * @param file  File to read.
+     * @param count Maximum number of lines to return.
+     * @return The last count lines from file.
+     * @throws IOException When unable to read the file.
+     */
+    public static List<String> tail(File file, int count) throws IOException {
+        LinkedList<String> tailLines = new LinkedList<String>();
+        FileReader reader = new FileReader(file);
+        try {
+            LineIterator iterator = org.apache.commons.io.IOUtils.lineIterator(reader);
+            int lineCount = 0;
+            while (iterator.hasNext()) {
+                String line = iterator.nextLine();
+                lineCount++;
+                if (lineCount > count)
+                    tailLines.removeFirst();
+                tailLines.offer(line);
+            }
+        } finally {
+            org.apache.commons.io.IOUtils.closeQuietly(reader);
+        }
+        return tailLines;
+    }
+
+    /**
+     * Tries to delete a file. Emits a warning if the file was unable to be deleted.
+     *
+     * @param file File to delete.
+     * @return true if the file was deleted.
+     */
+    public static boolean tryDelete(File file) {
+        boolean deleted = FileUtils.deleteQuietly(file);
+        if (deleted)
+            logger.debug("Deleted " + file);
+        else if (file.exists())
+            logger.warn("Unable to delete " + file);
+        return deleted;
+    }
+
+    /**
+     * Writes the an embedded resource to a temp file.
+     * File is not scheduled for deletion and must be cleaned up by the caller.
+     * @param resource Embedded resource.
+     * @return Path to the temp file with the contents of the resource.
+     */
+    public static File writeTempResource(Resource resource) {
+        File temp;
+        try {
+            temp = File.createTempFile(FilenameUtils.getBaseName(resource.getPath()) + ".", "." + FilenameUtils.getExtension(resource.getPath()));
+        } catch (IOException e) {
+            throw new UserException.BadTmpDir(e.getMessage());
+        }
+        writeResource(resource, temp);
+        return temp;
+    }
+
+    /**
+     * Writes the an embedded resource to a file.
+     * File is not scheduled for deletion and must be cleaned up by the caller.
+     * @param resource Embedded resource.
+     * @param file File path to write.
+     */
+    public static void writeResource(Resource resource, File file) {
+        String path = resource.getPath();
+        Class<?> clazz = resource.getRelativeClass();
+        InputStream inputStream = null;
+        OutputStream outputStream = null;
+        try {
+            if (clazz == null) {
+                inputStream = ClassLoader.getSystemResourceAsStream(path);
+                if (inputStream == null)
+                    throw new IllegalArgumentException("Resource not found: " + path);
+            } else {
+                inputStream = clazz.getResourceAsStream(path);
+                if (inputStream == null)
+                    throw new IllegalArgumentException("Resource not found relative to " + clazz + ": " + path);
+            }
+            outputStream = FileUtils.openOutputStream(file);
+            org.apache.commons.io.IOUtils.copy(inputStream, outputStream);
+        } catch (IOException e) {
+            throw new StingException(String.format("Unable to copy resource '%s' to '%s'", path, file), e);
+        } finally {
+            org.apache.commons.io.IOUtils.closeQuietly(inputStream);
+            org.apache.commons.io.IOUtils.closeQuietly(outputStream);
+        }
+    }
+}

public/java/src/org/broadinstitute/sting/utils/io/Resource.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.io;
+
+/**
+ * Stores a resource by path and a relative class.
+ */
+public class Resource {
+    private final String path;
+    private final Class<?> relativeClass;
+
+    /**
+     * Create a resource with a path and a relative class.
+     * @param path Relative or absolute path to the class.
+     * @param relativeClass Relative class to use as a class loader and for a relative package.
+     *
+     * If the relative class is null then the system classloader will be used and the path must be absolute.
+     */
+    public Resource(String path, Class<?> relativeClass) {
+        this.path = path;
+        this.relativeClass = relativeClass;
+    }
+
+    public Class<?> getRelativeClass() {
+        return relativeClass;
+    }
+
+    public String getPath() {
+        return path;
+    }
+}

public/java/src/org/broadinstitute/sting/utils/runtime/CapturedStreamOutput.java

@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.runtime;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.output.NullOutputStream;
+import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
+import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.broadinstitute.sting.utils.io.HardThresholdingOutputStream;
+
+import java.io.*;
+import java.util.EnumMap;
+
+/**
+ * Stream output captured from a stream.
+ */
+public class CapturedStreamOutput extends StreamOutput {
+    private final InputStream processStream;
+    private final EnumMap<StreamLocation, OutputStream> outputStreams = new EnumMap<StreamLocation, OutputStream>(StreamLocation.class);
+
+    /**
+     * The byte stream to capture content or null if no output string content was requested.
+     */
+    private final ByteArrayOutputStream bufferStream;
+
+    /**
+     * True if the buffer is truncated.
+     */
+    private boolean bufferTruncated = false;
+
+    /**
+     * @param settings       Settings that define what to capture.
+     * @param processStream  Stream to capture output.
+     * @param standardStream Stream to write debug output.
+     */
+    public CapturedStreamOutput(OutputStreamSettings settings, InputStream processStream, PrintStream standardStream) {
+        this.processStream = processStream;
+        int bufferSize = settings.getBufferSize();
+        this.bufferStream = (bufferSize < 0) ? new ByteArrayOutputStream() : new ByteArrayOutputStream(bufferSize);
+
+        for (StreamLocation location : settings.getStreamLocations()) {
+            OutputStream outputStream;
+            switch (location) {
+                case Buffer:
+                    if (bufferSize < 0) {
+                        outputStream = this.bufferStream;
+                    } else {
+                        outputStream = new HardThresholdingOutputStream(bufferSize) {
+                            @Override
+                            protected OutputStream getStream() throws IOException {
+                                return bufferTruncated ? NullOutputStream.NULL_OUTPUT_STREAM : bufferStream;
+                            }
+
+                            @Override
+                            protected void thresholdReached() throws IOException {
+                                bufferTruncated = true;
+                            }
+                        };
+                    }
+                    break;
+                case File:
+                    try {
+                        outputStream = new FileOutputStream(settings.getOutputFile(), settings.isAppendFile());
+                    } catch (IOException e) {
+                        throw new UserException.BadInput(e.getMessage());
+                    }
+                    break;
+                case Standard:
+                    outputStream = standardStream;
+                    break;
+                default:
+                    throw new ReviewedStingException("Unexpected stream location: " + location);
+            }
+            this.outputStreams.put(location, outputStream);
+        }
+    }
+
+    @Override
+    public byte[] getBufferBytes() {
+        return bufferStream.toByteArray();
+    }
+
+    @Override
+    public boolean isBufferTruncated() {
+        return bufferTruncated;
+    }
+
+    /**
+     * Drain the input stream to keep the process from backing up until it's empty.
+     * File streams will be closed automatically when this method returns.
+     *
+     * @throws java.io.IOException When unable to read or write.
+     */
+    public void readAndClose() throws IOException {
+        try {
+            byte[] buf = new byte[4096];
+            int readCount;
+            while ((readCount = processStream.read(buf)) >= 0)
+                for (OutputStream outputStream : this.outputStreams.values()) {
+                    outputStream.write(buf, 0, readCount);
+                }
+        } finally {
+            for (StreamLocation location : this.outputStreams.keySet()) {
+                OutputStream outputStream = this.outputStreams.get(location);
+                outputStream.flush();
+                if (location != StreamLocation.Standard)
+                    IOUtils.closeQuietly(outputStream);
+            }
+        }
+    }
+}

public/java/src/org/broadinstitute/sting/utils/runtime/InputStreamSettings.java

@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.runtime;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Set;
+
+/**
+ * Settings that define text to write to the process stdin.
+ */
+public class InputStreamSettings {
+    private final EnumSet<StreamLocation> streamLocations = EnumSet.noneOf(StreamLocation.class);
+    private byte[] inputBuffer;
+    private File inputFile;
+
+    public InputStreamSettings() {
+    }
+
+    /**
+     * @param inputBuffer String to write to stdin.
+     */
+    public InputStreamSettings(String inputBuffer) {
+        setInputBuffer(inputBuffer);
+    }
+
+    /**
+     * @param inputFile File to write to stdin.
+     */
+    public InputStreamSettings(File inputFile) {
+        setInputFile(inputFile);
+    }
+
+    /**
+     * @param inputBuffer String to write to stdin.
+     * @param inputFile   File to write to stdin.
+     */
+    public InputStreamSettings(byte[] inputBuffer, File inputFile) {
+        setInputBuffer(inputBuffer);
+        setInputFile(inputFile);
+    }
+
+    public Set<StreamLocation> getStreamLocations() {
+        return Collections.unmodifiableSet(streamLocations);
+    }
+
+    public byte[] getInputBuffer() {
+        return inputBuffer;
+    }
+
+    public void setInputBuffer(String inputBuffer) {
+        if (inputBuffer == null)
+            throw new IllegalArgumentException("inputBuffer cannot be null");
+        this.streamLocations.add(StreamLocation.Buffer);
+        this.inputBuffer = inputBuffer.getBytes();
+    }
+
+    public void setInputBuffer(byte[] inputBuffer) {
+        if (inputBuffer == null)
+            throw new IllegalArgumentException("inputBuffer cannot be null");
+        this.streamLocations.add(StreamLocation.Buffer);
+        this.inputBuffer = inputBuffer;
+    }
+
+    public void clearInputBuffer() {
+        this.streamLocations.remove(StreamLocation.Buffer);
+        this.inputBuffer = null;
+    }
+
+    public File getInputFile() {
+        return inputFile;
+    }
+
+    public void setInputFile(File inputFile) {
+        if (inputFile == null)
+            throw new IllegalArgumentException("inputFile cannot be null");
+        this.streamLocations.add(StreamLocation.File);
+        this.inputFile = inputFile;
+    }
+
+    public void clearInputFile() {
+        this.streamLocations.remove(StreamLocation.File);
+        this.inputFile = null;
+    }
+
+    public void setInputStandard(boolean inputStandard) {
+        if (inputStandard)
+            this.streamLocations.add(StreamLocation.Standard);
+        else
+            this.streamLocations.remove(StreamLocation.Standard);
+    }
+}

public/java/src/org/broadinstitute/sting/utils/runtime/OutputStreamSettings.java

@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.runtime;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Set;
+
+/**
+ * Settings that define text to capture from a process stream.
+ */
+public class OutputStreamSettings {
+    private final EnumSet<StreamLocation> streamLocations = EnumSet.noneOf(StreamLocation.class);
+    private int bufferSize;
+    private File outputFile;
+    private boolean appendFile;
+
+    public OutputStreamSettings() {
+    }
+
+    /**
+     * @param bufferSize The number of bytes to capture, or -1 for unlimited.
+     */
+    public OutputStreamSettings(int bufferSize) {
+        setBufferSize(bufferSize);
+    }
+
+    /**
+     * @param outputFile The file to write output to.
+     */
+    public OutputStreamSettings(File outputFile) {
+        setOutputFile(outputFile);
+    }
+
+    /**
+     * @param outputFile The file to write output to.
+     * @param append     true if the output file should be appended to.
+     */
+    public OutputStreamSettings(File outputFile, boolean append) {
+        setOutputFile(outputFile, append);
+    }
+
+    public OutputStreamSettings(int bufferSize, File outputFile, boolean appendFile) {
+        setBufferSize(bufferSize);
+        setOutputFile(outputFile, appendFile);
+    }
+
+    public Set<StreamLocation> getStreamLocations() {
+        return Collections.unmodifiableSet(streamLocations);
+    }
+
+    public int getBufferSize() {
+        return bufferSize;
+    }
+
+    public void setBufferSize(int bufferSize) {
+        this.streamLocations.add(StreamLocation.Buffer);
+        this.bufferSize = bufferSize;
+    }
+
+    public void clearBufferSize() {
+        this.streamLocations.remove(StreamLocation.Buffer);
+        this.bufferSize = 0;
+    }
+
+    public File getOutputFile() {
+        return outputFile;
+    }
+
+    public boolean isAppendFile() {
+        return appendFile;
+    }
+
+    /**
+     * Overwrites the outputFile with the process output.
+     *
+     * @param outputFile File to overwrite.
+     */
+    public void setOutputFile(File outputFile) {
+        setOutputFile(outputFile, false);
+    }
+
+    public void setOutputFile(File outputFile, boolean append) {
+        if (outputFile == null)
+            throw new IllegalArgumentException("outputFile cannot be null");
+        streamLocations.add(StreamLocation.File);
+        this.outputFile = outputFile;
+        this.appendFile = append;
+    }
+
+    public void clearOutputFile() {
+        streamLocations.remove(StreamLocation.File);
+        this.outputFile = null;
+        this.appendFile = false;
+    }
+
+    public void printStandard(boolean print) {
+        if (print)
+            this.streamLocations.add(StreamLocation.Standard);
+        else
+            this.streamLocations.remove(StreamLocation.Standard);
+    }
+}

public/java/src/org/broadinstitute/sting/utils/runtime/ProcessController.java

@@ -0,0 +1,363 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.runtime;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
+import org.broadinstitute.sting.utils.exceptions.UserException;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.*;
+
+/**
+ * Facade to Runtime.exec() and java.lang.Process.  Handles
+ * running a process to completion and returns stdout and stderr
+ * as strings.  Creates separate threads for reading stdout and stderr,
+ * then reuses those threads for each process most efficient use is
+ * to create one of these and use it repeatedly.  Instances are not
+ * thread-safe, however.
+ *
+ * TODO: java.io sometimes zombies the backround threads locking up on read().
+ * Supposedly NIO has better ways of interrupting a blocked stream but will
+ * require a little bit of refactoring.
+ *
+ * @author Michael Koehrsen
+ * @author Khalid Shakir
+ */
+public class ProcessController {
+    private static Logger logger = Logger.getLogger(ProcessController.class);
+
+    private static enum ProcessStream {Stdout, Stderr}
+
+    // Tracks running processes.
+    private static final Set<ProcessController> running = Collections.synchronizedSet(new HashSet<ProcessController>());
+
+    // Tracks this running process.
+    private Process process;
+
+    // Threads that capture stdout and stderr
+    private final OutputCapture stdoutCapture;
+    private final OutputCapture stderrCapture;
+
+    // When a caller destroyes a controller a new thread local version will be created
+    private boolean destroyed = false;
+
+    // Communication channels with output capture threads
+
+    // Holds the stdout and stderr sent to the background capture threads
+    private final Map<ProcessStream, CapturedStreamOutput> toCapture =
+            new EnumMap<ProcessStream, CapturedStreamOutput>(ProcessStream.class);
+
+    // Holds the results of the capture from the background capture threads.
+    // May be the content via toCapture or an StreamOutput.EMPTY if the capture was interrupted.
+    private final Map<ProcessStream, StreamOutput> fromCapture =
+            new EnumMap<ProcessStream, StreamOutput>(ProcessStream.class);
+
+    // Useful for debugging if background threads have shut down correctly
+    private static int nextControllerId = 0;
+    private final int controllerId;
+
+    public ProcessController() {
+        // Start the background threads for this controller.
+        synchronized (running) {
+            controllerId = nextControllerId++;
+        }
+        stdoutCapture = new OutputCapture(ProcessStream.Stdout, controllerId);
+        stderrCapture = new OutputCapture(ProcessStream.Stderr, controllerId);
+        stdoutCapture.start();
+        stderrCapture.start();
+    }
+
+    /**
+     * Returns a thread local ProcessController.
+     * Should NOT be closed when finished so it can be reused by the thread.
+     *
+     * @return a thread local ProcessController.
+     */
+    public static ProcessController getThreadLocal() {
+        // If the local controller was destroyed get a fresh instance.
+        if (threadProcessController.get().destroyed)
+            threadProcessController.remove();
+        return threadProcessController.get();
+    }
+
+    /**
+     * Thread local process controller container.
+     */
+    private static final ThreadLocal<ProcessController> threadProcessController =
+            new ThreadLocal<ProcessController>() {
+                @Override
+                protected ProcessController initialValue() {
+                    return new ProcessController();
+                }
+            };
+
+    /**
+     * Similar to Runtime.exec() but drains the output and error streams.
+     *
+     * @param command Command to run.
+     * @return The result code.
+     */
+    public static int exec(String[] command) {
+        ProcessController controller = ProcessController.getThreadLocal();
+        return controller.exec(new ProcessSettings(command)).getExitValue();
+    }
+
+    /**
+     * Executes a command line program with the settings and waits for it to return,
+     * processing the output on a background thread.
+     *
+     * @param settings Settings to be run.
+     * @return The output of the command.
+     */
+    public ProcessOutput exec(ProcessSettings settings) {
+        if (destroyed)
+            throw new IllegalStateException("This controller was destroyed");
+
+        ProcessBuilder builder = new ProcessBuilder(settings.getCommand());
+        builder.directory(settings.getDirectory());
+
+        Map<String, String> settingsEnvironment = settings.getEnvironment();
+        if (settingsEnvironment != null) {
+            Map<String, String> builderEnvironment = builder.environment();
+            builderEnvironment.clear();
+            builderEnvironment.putAll(settingsEnvironment);
+        }
+
+        builder.redirectErrorStream(settings.isRedirectErrorStream());
+
+        StreamOutput stdout = null;
+        StreamOutput stderr = null;
+
+        // Start the process running.
+
+        try {
+            synchronized (toCapture) {
+                process = builder.start();
+            }
+            running.add(this);
+        } catch (IOException e) {
+            throw new ReviewedStingException("Unable to start command: " + StringUtils.join(builder.command(), " "));
+        }
+
+        int exitCode;
+
+        try {
+            // Notify the background threads to start capturing.
+            synchronized (toCapture) {
+                toCapture.put(ProcessStream.Stdout,
+                        new CapturedStreamOutput(settings.getStdoutSettings(), process.getInputStream(), System.out));
+                toCapture.put(ProcessStream.Stderr,
+                        new CapturedStreamOutput(settings.getStderrSettings(), process.getErrorStream(), System.err));
+                toCapture.notifyAll();
+            }
+
+            // Write stdin content
+            InputStreamSettings stdinSettings = settings.getStdinSettings();
+            Set<StreamLocation> streamLocations = stdinSettings.getStreamLocations();
+            if (!streamLocations.isEmpty()) {
+                try {
+                    OutputStream stdinStream = process.getOutputStream();
+                    for (StreamLocation location : streamLocations) {
+                        InputStream inputStream;
+                        switch (location) {
+                            case Buffer:
+                                inputStream = new ByteArrayInputStream(stdinSettings.getInputBuffer());
+                                break;
+                            case File:
+                                try {
+                                    inputStream = FileUtils.openInputStream(stdinSettings.getInputFile());
+                                } catch (IOException e) {
+                                    throw new UserException.BadInput(e.getMessage());
+                                }
+                                break;
+                            case Standard:
+                                inputStream = System.in;
+                                break;
+                            default:
+                                throw new ReviewedStingException("Unexpected stream location: " + location);
+                        }
+                        try {
+                            IOUtils.copy(inputStream, stdinStream);
+                        } finally {
+                            if (location != StreamLocation.Standard)
+                                IOUtils.closeQuietly(inputStream);
+                        }
+                    }
+                    stdinStream.flush();
+                } catch (IOException e) {
+                    throw new ReviewedStingException("Error writing to stdin on command: " + StringUtils.join(builder.command(), " "), e);
+                }
+            }
+
+            // Wait for the process to complete.
+            try {
+                process.getOutputStream().close();
+                process.waitFor();
+            } catch (IOException e) {
+                throw new ReviewedStingException("Unable to close stdin on command: " + StringUtils.join(builder.command(), " "), e);
+            } catch (InterruptedException e) {
+                throw new ReviewedStingException("Process interrupted", e);
+            } finally {
+                while (!destroyed && stdout == null || stderr == null) {
+                    synchronized (fromCapture) {
+                        if (fromCapture.containsKey(ProcessStream.Stdout))
+                            stdout = fromCapture.remove(ProcessStream.Stdout);
+                        if (fromCapture.containsKey(ProcessStream.Stderr))
+                            stderr = fromCapture.remove(ProcessStream.Stderr);
+                        try {
+                            if (stdout == null || stderr == null)
+                                fromCapture.wait();
+                        } catch (InterruptedException e) {
+                            // Log the error, ignore the interrupt and wait patiently
+                            // for the OutputCaptures to (via finally) return their
+                            // stdout and stderr.
+                            logger.error(e);
+                        }
+                    }
+                }
+
+                if (destroyed) {
+                    if (stdout == null)
+                        stdout = StreamOutput.EMPTY;
+                    if (stderr == null)
+                        stderr = StreamOutput.EMPTY;
+                }
+            }
+        } finally {
+            synchronized (toCapture) {
+                exitCode = process.exitValue();
+                process = null;
+            }
+            running.remove(this);
+        }
+
+        return new ProcessOutput(exitCode, stdout, stderr);
+    }
+
+    /**
+     * @return The set of still running processes.
+     */
+    public static Set<ProcessController> getRunning() {
+        synchronized (running) {
+            return new HashSet<ProcessController>(running);
+        }
+    }
+
+    /**
+     * Stops the process from running and tries to ensure process is cleaned up properly.
+     * NOTE: sub-processes started by process may be zombied with their parents set to pid 1.
+     * NOTE: capture threads may block on read.
+     * TODO: Try to use NIO to interrupt streams.
+     */
+    public void tryDestroy() {
+        destroyed = true;
+        synchronized (toCapture) {
+            if (process != null) {
+                process.destroy();
+                IOUtils.closeQuietly(process.getInputStream());
+                IOUtils.closeQuietly(process.getErrorStream());
+            }
+            stdoutCapture.interrupt();
+            stderrCapture.interrupt();
+            toCapture.notifyAll();
+        }
+    }
+
+    @Override
+    protected void finalize() throws Throwable {
+        try {
+            tryDestroy();
+        } catch (Exception e) {
+            logger.error(e);
+        }
+        super.finalize();
+    }
+
+    private class OutputCapture extends Thread {
+        private final int controllerId;
+        private final ProcessStream key;
+
+        /**
+         * Reads in the output of a stream on a background thread to keep the output pipe from backing up and freezing the called process.
+         *
+         * @param key The stdout or stderr key for this output capture.
+         * @param controllerId Unique id of the controller.
+         */
+        public OutputCapture(ProcessStream key, int controllerId) {
+            super(String.format("OutputCapture-%d-%s-%s-%d", controllerId, key.name().toLowerCase(),
+                    Thread.currentThread().getName(), Thread.currentThread().getId()));
+            this.controllerId = controllerId;
+            this.key = key;
+            setDaemon(true);
+        }
+
+        /**
+         * Runs the capture.
+         */
+        @Override
+        public void run() {
+            while (!destroyed) {
+                StreamOutput processStream = StreamOutput.EMPTY;
+                try {
+                    // Wait for a new input stream to be passed from this process controller.
+                    CapturedStreamOutput capturedProcessStream = null;
+                    while (!destroyed && capturedProcessStream == null) {
+                        synchronized (toCapture) {
+                            if (toCapture.containsKey(key)) {
+                                capturedProcessStream = toCapture.remove(key);
+                            } else {
+                                toCapture.wait();
+                            }
+                        }
+                    }
+
+                    if (!destroyed) {
+                        // Read in the input stream
+                        processStream = capturedProcessStream;
+                        capturedProcessStream.readAndClose();
+                    }
+                } catch (InterruptedException e) {
+                    logger.info("OutputCapture interrupted, exiting");
+                    break;
+                } catch (IOException e) {
+                    logger.error("Error reading process output", e);
+                } finally {
+                    // Send the string back to the process controller.
+                    synchronized (fromCapture) {
+                        fromCapture.put(key, processStream);
+                        fromCapture.notify();
+                    }
+                }
+            }
+        }
+    }
+}

public/java/src/org/broadinstitute/sting/utils/runtime/ProcessOutput.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.runtime;
+
+public class ProcessOutput {
+    private final int exitValue;
+    private final StreamOutput stdout;
+    private final StreamOutput stderr;
+
+    /**
+     * The output of a process.
+     *
+     * @param exitValue The exit value.
+     * @param stdout    The capture of stdout as defined by the stdout OutputStreamSettings.
+     * @param stderr    The capture of stderr as defined by the stderr OutputStreamSettings.
+     */
+    public ProcessOutput(int exitValue, StreamOutput stdout, StreamOutput stderr) {
+        this.exitValue = exitValue;
+        this.stdout = stdout;
+        this.stderr = stderr;
+    }
+
+    public int getExitValue() {
+        return exitValue;
+    }
+
+    public StreamOutput getStdout() {
+        return stdout;
+    }
+
+    public StreamOutput getStderr() {
+        return stderr;
+    }
+}

public/java/src/org/broadinstitute/sting/utils/runtime/ProcessSettings.java

@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.runtime;
+
+import com.sun.corba.se.spi.orbutil.fsm.Input;
+
+import java.io.File;
+import java.util.Map;
+
+public class ProcessSettings {
+    private String[] command;
+    private Map<String, String> environment;
+    private File directory;
+    private boolean redirectErrorStream;
+    private InputStreamSettings stdinSettings;
+    private OutputStreamSettings stdoutSettings;
+    private OutputStreamSettings stderrSettings;
+
+    /**
+     * @param command Command line to run.
+     */
+    public ProcessSettings(String[] command) {
+        this(command, false, null, null, null, null, null);
+    }
+
+    /**
+     * @param command             Command line to run.
+     * @param redirectErrorStream true if stderr should be sent to stdout.
+     * @param environment         Environment settings to override System.getEnv, or null to use System.getEnv.
+     * @param directory           The directory to run the command in, or null to run in the current directory.
+     * @param stdinSettings       Settings for writing to the process stdin.
+     * @param stdoutSettings      Settings for capturing the process stdout.
+     * @param stderrSettings      Setting for capturing the process stderr.
+     */
+    public ProcessSettings(String[] command, boolean redirectErrorStream, File directory, Map<String, String> environment,
+                           InputStreamSettings stdinSettings, OutputStreamSettings stdoutSettings, OutputStreamSettings stderrSettings) {
+        this.command = checkCommand(command);
+        this.redirectErrorStream = redirectErrorStream;
+        this.directory = directory;
+        this.environment = environment;
+        this.stdinSettings = checkSettings(stdinSettings);
+        this.stdoutSettings = checkSettings(stdoutSettings);
+        this.stderrSettings = checkSettings(stderrSettings);
+    }
+
+    public String[] getCommand() {
+        return command;
+    }
+
+    public void setCommand(String[] command) {
+        this.command = checkCommand(command);
+    }
+
+    public boolean isRedirectErrorStream() {
+        return redirectErrorStream;
+    }
+
+    public void setRedirectErrorStream(boolean redirectErrorStream) {
+        this.redirectErrorStream = redirectErrorStream;
+    }
+
+    public File getDirectory() {
+        return directory;
+    }
+
+    public void setDirectory(File directory) {
+        this.directory = directory;
+    }
+
+    public Map<String, String> getEnvironment() {
+        return environment;
+    }
+
+    public void setEnvironment(Map<String, String> environment) {
+        this.environment = environment;
+    }
+
+    public InputStreamSettings getStdinSettings() {
+        return stdinSettings;
+    }
+
+    public void setStdinSettings(InputStreamSettings stdinSettings) {
+        this.stdinSettings = checkSettings(stdinSettings);
+    }
+
+    public OutputStreamSettings getStdoutSettings() {
+        return stdoutSettings;
+    }
+
+    public void setStdoutSettings(OutputStreamSettings stdoutSettings) {
+        this.stdoutSettings = checkSettings(stdoutSettings);
+    }
+
+    public OutputStreamSettings getStderrSettings() {
+        return stderrSettings;
+    }
+
+    public void setStderrSettings(OutputStreamSettings stderrSettings) {
+        this.stderrSettings = checkSettings(stderrSettings);
+    }
+
+    protected String[] checkCommand(String[] command) {
+        if (command == null)
+            throw new IllegalArgumentException("Command is not allowed to be null");
+        for (String s: command)
+            if (s == null)
+                throw new IllegalArgumentException("Command is not allowed to contain nulls");
+        return command;
+    }
+
+    protected InputStreamSettings checkSettings(InputStreamSettings settings) {
+        return settings == null ? new InputStreamSettings() : settings;
+    }
+
+    protected OutputStreamSettings checkSettings(OutputStreamSettings settings) {
+        return settings == null ? new OutputStreamSettings() : settings;
+    }
+}

public/java/src/org/broadinstitute/sting/utils/runtime/StreamLocation.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.runtime;
+
+/**
+ * Where to read/write a stream
+ */
+public enum StreamLocation {
+    Buffer, File, Standard
+}

public/java/src/org/broadinstitute/sting/utils/runtime/StreamOutput.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.runtime;
+
+/**
+ * The content of stdout or stderr.
+ */
+public abstract class StreamOutput {
+    /**
+     * Empty stream output when no output is captured due to an error.
+     */
+    public static final StreamOutput EMPTY = new StreamOutput() {
+        @Override
+        public byte[] getBufferBytes() {
+            return new byte[0];
+        }
+
+        @Override
+        public boolean isBufferTruncated() {
+            return false;
+        }
+    };
+
+    /**
+     * Returns the content as a string.
+     *
+     * @return The content as a string.
+     */
+    public String getBufferString() {
+        return new String(getBufferBytes());
+    }
+
+    /**
+     * Returns the content as a string.
+     *
+     * @return The content as a string.
+     */
+    public abstract byte[] getBufferBytes();
+
+    /**
+     * Returns true if the buffer was truncated.
+     *
+     * @return true if the buffer was truncated.
+     */
+    public abstract boolean isBufferTruncated();
+}

public/java/test/org/broadinstitute/sting/commandline/ArgumentMatchSiteUnitTest.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.commandline;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.File;
+
+public class ArgumentMatchSiteUnitTest {
+    @Test
+    public void testCommandLine() {
+        ArgumentMatchSite site = new ArgumentMatchSite(ArgumentMatchSource.COMMAND_LINE, 1);
+        Assert.assertEquals(site.getSource(), ArgumentMatchSource.COMMAND_LINE);
+        Assert.assertEquals(site.getIndex(), 1);
+    }
+
+    @Test
+    public void testFile() {
+        ArgumentMatchSource source = new ArgumentMatchSource(new File("test"));
+        ArgumentMatchSite site = new ArgumentMatchSite(source, 1);
+        Assert.assertEquals(site.getSource(), source);
+        Assert.assertEquals(site.getIndex(), 1);
+    }
+
+    @Test
+    public void testEquals() {
+        ArgumentMatchSource cmdLine = ArgumentMatchSource.COMMAND_LINE;
+        ArgumentMatchSite site1 = new ArgumentMatchSite(cmdLine, 1);
+        ArgumentMatchSite site2 = new ArgumentMatchSite(cmdLine, 2);
+
+        Assert.assertFalse(site1.equals(null));
+
+        Assert.assertTrue(site1.equals(site1));
+        Assert.assertFalse(site1.equals(site2));
+
+        Assert.assertFalse(site2.equals(site1));
+        Assert.assertTrue(site2.equals(site2));
+    }
+
+    @Test
+    public void testCompareTo() {
+        ArgumentMatchSource cmdLine = ArgumentMatchSource.COMMAND_LINE;
+        ArgumentMatchSite site1 = new ArgumentMatchSite(cmdLine, 1);
+        ArgumentMatchSite site2 = new ArgumentMatchSite(cmdLine, 2);
+
+        Assert.assertTrue(site1.compareTo(site1) == 0);
+        Assert.assertTrue(site1.compareTo(site2) < 0);
+        Assert.assertTrue(site2.compareTo(site1) > 0);
+        Assert.assertTrue(site2.compareTo(site2) == 0);
+    }
+
+    @Test(expectedExceptions = NullPointerException.class)
+    public void testCompareToNull() {
+        new ArgumentMatchSite(ArgumentMatchSource.COMMAND_LINE, 0).compareTo(null);
+    }
+}

public/java/test/org/broadinstitute/sting/commandline/ArgumentMatchSourceUnitTest.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.commandline;
+
+import org.broadinstitute.sting.BaseTest;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.File;
+
+public class ArgumentMatchSourceUnitTest extends BaseTest {
+    @Test
+    public void testCommandLine() {
+        ArgumentMatchSource source = ArgumentMatchSource.COMMAND_LINE;
+        Assert.assertEquals(source.getType(), ArgumentMatchSourceType.CommandLine);
+        Assert.assertNull(source.getFile());
+    }
+
+    @Test
+    public void testFile() {
+        File f = new File("test");
+        ArgumentMatchSource source = new ArgumentMatchSource(f);
+        Assert.assertEquals(source.getType(), ArgumentMatchSourceType.File);
+        Assert.assertEquals(source.getFile(), f);
+    }
+
+    @Test(expectedExceptions = IllegalArgumentException.class)
+    public void testNullFile() {
+        new ArgumentMatchSource(null);
+    }
+
+    @Test
+    public void testEquals() {
+        ArgumentMatchSource cmdLine = ArgumentMatchSource.COMMAND_LINE;
+        ArgumentMatchSource fileA = new ArgumentMatchSource(new File("a"));
+        ArgumentMatchSource fileB = new ArgumentMatchSource(new File("b"));
+
+        Assert.assertFalse(cmdLine.equals(null));
+
+        Assert.assertTrue(cmdLine.equals(cmdLine));
+        Assert.assertFalse(cmdLine.equals(fileA));
+        Assert.assertFalse(cmdLine.equals(fileB));
+
+        Assert.assertFalse(fileA.equals(cmdLine));
+        Assert.assertTrue(fileA.equals(fileA));
+        Assert.assertFalse(fileA.equals(fileB));
+
+        Assert.assertFalse(fileB.equals(cmdLine));
+        Assert.assertFalse(fileB.equals(fileA));
+        Assert.assertTrue(fileB.equals(fileB));
+    }
+
+    @Test
+    public void testCompareTo() {
+        ArgumentMatchSource cmdLine = ArgumentMatchSource.COMMAND_LINE;
+        ArgumentMatchSource fileA = new ArgumentMatchSource(new File("a"));
+        ArgumentMatchSource fileB = new ArgumentMatchSource(new File("b"));
+
+        Assert.assertTrue(cmdLine.compareTo(cmdLine) == 0);
+        Assert.assertTrue(cmdLine.compareTo(fileA) < 0);
+        Assert.assertTrue(cmdLine.compareTo(fileB) < 0);
+
+        Assert.assertTrue(fileA.compareTo(cmdLine) > 0);
+        Assert.assertTrue(fileA.compareTo(fileA) == 0);
+        Assert.assertTrue(fileA.compareTo(fileB) < 0);
+
+        Assert.assertTrue(fileB.compareTo(cmdLine) > 0);
+        Assert.assertTrue(fileB.compareTo(fileA) > 0);
+        Assert.assertTrue(fileB.compareTo(fileB) == 0);
+    }
+
+    @Test(expectedExceptions = NullPointerException.class)
+    public void testCompareToNull() {
+        ArgumentMatchSource.COMMAND_LINE.compareTo(null);
+    }
+}

public/java/test/org/broadinstitute/sting/commandline/ParsingEngineUnitTest.java

@@ -25,6 +25,7 @@
 
 package org.broadinstitute.sting.commandline;
 
+import org.apache.commons.io.FileUtils;
 import org.broad.tribble.Feature;
 import org.broadinstitute.sting.utils.exceptions.UserException;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
@@ -34,6 +35,8 @@ import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
+import java.io.File;
+import java.io.IOException;
 import java.util.List;
 import java.util.EnumSet;
 /**
@@ -493,6 +496,7 @@ public class ParsingEngineUnitTest extends BaseTest {
         Assert.assertNotNull(definition, "Invalid default argument name assigned");
     }
 
+    @SuppressWarnings("unused")
     private class CamelCaseArgProvider {
         @Argument(doc="my arg")
         Integer myArg;
@@ -507,6 +511,7 @@ public class ParsingEngineUnitTest extends BaseTest {
         parsingEngine.validate();
     }
 
+    @SuppressWarnings("unused")
     private class BooleanArgProvider {
         @Argument(doc="my bool")
         boolean myBool;
@@ -561,6 +566,7 @@ public class ParsingEngineUnitTest extends BaseTest {
         parsingEngine.validate();
     }
 
+    @SuppressWarnings("unused")
     private class MutuallyExclusiveArgProvider {
         @Argument(doc="foo",exclusiveOf="bar")
         Integer foo;
@@ -618,6 +624,7 @@ public class ParsingEngineUnitTest extends BaseTest {
         parsingEngine.addArgumentSource( MultipleArgumentCollectionProvider.class );
     }
 
+    @SuppressWarnings("unused")
     private class MultipleArgumentCollectionProvider {
         @ArgumentCollection
         RequiredArgProvider rap1 = new RequiredArgProvider();
@@ -937,4 +944,23 @@ public class ParsingEngineUnitTest extends BaseTest {
         VariantContextRodBindingArgProvider argProvider = new VariantContextRodBindingArgProvider();
         parsingEngine.loadArgumentsIntoObject( argProvider );
     }
+
+    @Test
+    public void argumentListTest() throws IOException {
+        File argsFile = BaseTest.createTempFile("args.", ".list");
+        try {
+            FileUtils.write(argsFile, "-I na12878.bam");
+            final String[] commandLine = new String[] {"-args", argsFile.getPath()};
+            parsingEngine.addArgumentSource(InputFileArgProvider.class);
+            parsingEngine.parse(commandLine);
+            parsingEngine.validate();
+
+            InputFileArgProvider argProvider = new InputFileArgProvider();
+            parsingEngine.loadArgumentsIntoObject(argProvider);
+
+            Assert.assertEquals(argProvider.inputFile, "na12878.bam", "Argument is not correctly initialized");
+        } finally {
+            FileUtils.deleteQuietly(argsFile);
+        }
+    }
 }

public/java/test/org/broadinstitute/sting/utils/R/RScriptLibraryUnitTest.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.R;
+
+import org.apache.commons.io.FileUtils;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.File;
+
+public class RScriptLibraryUnitTest {
+    @Test
+    public void testProperties() {
+        Assert.assertEquals(RScriptLibrary.GSALIB.getLibraryName(), "gsalib");
+        Assert.assertEquals(RScriptLibrary.GSALIB.getResourcePath(), "gsalib.tar.gz");
+    }
+
+    @Test
+    public void testWriteTemp() {
+        File file = RScriptLibrary.GSALIB.writeTemp();
+        Assert.assertTrue(file.exists(), "R library was not written to temp file: " + file);
+        FileUtils.deleteQuietly(file);
+    }
+}

public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java

@@ -0,0 +1,197 @@
+package org.broadinstitute.sting.utils.io;
+
+import org.apache.commons.io.FileUtils;
+import org.broadinstitute.sting.BaseTest;
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class IOUtilsUnitTest extends BaseTest {
+  @Test
+  public void testGoodTempDir() {
+    IOUtils.checkTempDir(new File("/tmp/queue"));
+  }
+
+  @Test(expectedExceptions=UserException.BadTmpDir.class)
+  public void testBadTempDir() {
+    IOUtils.checkTempDir(new File("/tmp"));
+  }
+
+  @Test
+  public void testAbsoluteSubDir() {
+    File subDir = IOUtils.absolute(new File("."), new File("/path/to/file"));
+    Assert.assertEquals(subDir, new File("/path/to/file"));
+
+    subDir = IOUtils.absolute(new File("/different/path"), new File("/path/to/file"));
+    Assert.assertEquals(subDir, new File("/path/to/file"));
+
+    subDir = IOUtils.absolute(new File("/different/path"), new File("."));
+    Assert.assertEquals(subDir, new File("/different/path"));
+  }
+
+  @Test
+  public void testRelativeSubDir() throws IOException {
+    File subDir = IOUtils.absolute(new File("."), new File("path/to/file"));
+    Assert.assertEquals(subDir.getCanonicalFile(), new File("path/to/file").getCanonicalFile());
+
+    subDir = IOUtils.absolute(new File("/different/path"), new File("path/to/file"));
+    Assert.assertEquals(subDir, new File("/different/path/path/to/file"));
+  }
+
+  @Test
+  public void testDottedSubDir() throws IOException {
+    File subDir = IOUtils.absolute(new File("."), new File("path/../to/file"));
+    Assert.assertEquals(subDir.getCanonicalFile(), new File("path/../to/./file").getCanonicalFile());
+
+    subDir = IOUtils.absolute(new File("."), new File("/path/../to/file"));
+    Assert.assertEquals(subDir, new File("/path/../to/file"));
+
+    subDir = IOUtils.absolute(new File("/different/../path"), new File("path/to/file"));
+    Assert.assertEquals(subDir, new File("/different/../path/path/to/file"));
+
+    subDir = IOUtils.absolute(new File("/different/./path"), new File("/path/../to/file"));
+    Assert.assertEquals(subDir, new File("/path/../to/file"));
+  }
+
+  @Test
+  public void testTempDir() {
+    File tempDir = IOUtils.tempDir("Q-Unit-Test", "", new File("queueTempDirToDelete"));
+    Assert.assertTrue(tempDir.exists());
+    Assert.assertFalse(tempDir.isFile());
+    Assert.assertTrue(tempDir.isDirectory());
+    boolean deleted = IOUtils.tryDelete(tempDir);
+    Assert.assertTrue(deleted);
+    Assert.assertFalse(tempDir.exists());
+  }
+
+  @Test
+  public void testDirLevel() {
+    File dir = IOUtils.dirLevel(new File("/path/to/directory"), 1);
+    Assert.assertEquals(dir, new File("/path"));
+
+    dir = IOUtils.dirLevel(new File("/path/to/directory"), 2);
+    Assert.assertEquals(dir, new File("/path/to"));
+
+    dir = IOUtils.dirLevel(new File("/path/to/directory"), 3);
+    Assert.assertEquals(dir, new File("/path/to/directory"));
+
+    dir = IOUtils.dirLevel(new File("/path/to/directory"), 4);
+    Assert.assertEquals(dir, new File("/path/to/directory"));
+  }
+
+  @Test
+  public void testAbsolute() {
+    File dir = IOUtils.absolute(new File("/path/./to/./directory/."));
+    Assert.assertEquals(dir, new File("/path/to/directory"));
+
+    dir = IOUtils.absolute(new File("/"));
+    Assert.assertEquals(dir, new File("/"));
+
+    dir = IOUtils.absolute(new File("/."));
+    Assert.assertEquals(dir, new File("/"));
+
+    dir = IOUtils.absolute(new File("/././."));
+    Assert.assertEquals(dir, new File("/"));
+
+    dir = IOUtils.absolute(new File("/./directory/."));
+    Assert.assertEquals(dir, new File("/directory"));
+
+    dir = IOUtils.absolute(new File("/./directory/./"));
+    Assert.assertEquals(dir, new File("/directory"));
+
+    dir = IOUtils.absolute(new File("/./directory./"));
+    Assert.assertEquals(dir, new File("/directory."));
+
+    dir = IOUtils.absolute(new File("/./.directory/"));
+    Assert.assertEquals(dir, new File("/.directory"));
+  }
+
+  @Test
+  public void testTail() throws IOException {
+    List<String> lines = Arrays.asList(
+            "chr18_random	4262	3154410390	50	51",
+            "chr19_random	301858	3154414752	50	51",
+            "chr21_random	1679693	3154722662	50	51",
+            "chr22_random	257318	3156435963	50	51",
+            "chrX_random	1719168	3156698441	50	51");
+    List<String> tail = IOUtils.tail(new File(BaseTest.hg18Reference + ".fai"), 5);
+    Assert.assertEquals(tail.size(), 5);
+    for (int i = 0; i < 5; i++)
+      Assert.assertEquals(tail.get(i), lines.get(i));
+  }
+
+    @Test
+    public void testWriteSystemFile() throws IOException {
+        File temp = createTempFile("temp.", ".properties");
+        try {
+            IOUtils.writeResource(new Resource("StingText.properties", null), temp);
+        } finally {
+            FileUtils.deleteQuietly(temp);
+        }
+    }
+
+    @Test
+    public void testWriteSystemTempFile() throws IOException {
+        File temp = IOUtils.writeTempResource(new Resource("StingText.properties", null));
+        try {
+            Assert.assertTrue(temp.getName().startsWith("StingText"), "File does not start with 'StingText.': " + temp);
+            Assert.assertTrue(temp.getName().endsWith(".properties"), "File does not end with '.properties': " + temp);
+        } finally {
+            FileUtils.deleteQuietly(temp);
+        }
+    }
+
+    @Test(expectedExceptions = IllegalArgumentException.class)
+    public void testMissingSystemFile() throws IOException {
+        File temp = createTempFile("temp.", ".properties");
+        try {
+            IOUtils.writeResource(new Resource("MissingStingText.properties", null), temp);
+        } finally {
+            FileUtils.deleteQuietly(temp);
+        }
+    }
+
+    @Test
+    public void testWriteRelativeFile() throws IOException {
+        File temp = createTempFile("temp.", ".properties");
+        try {
+            IOUtils.writeResource(new Resource("/StingText.properties", IOUtils.class), temp);
+        } finally {
+            FileUtils.deleteQuietly(temp);
+        }
+    }
+
+    @Test
+    public void testWriteRelativeTempFile() throws IOException {
+        File temp = IOUtils.writeTempResource(new Resource("/StingText.properties", IOUtils.class));
+        try {
+            Assert.assertTrue(temp.getName().startsWith("StingText"), "File does not start with 'StingText.': " + temp);
+            Assert.assertTrue(temp.getName().endsWith(".properties"), "File does not end with '.properties': " + temp);
+        } finally {
+            FileUtils.deleteQuietly(temp);
+        }
+    }
+
+    @Test(expectedExceptions = IllegalArgumentException.class)
+    public void testMissingRelativeFile() throws IOException {
+        File temp = createTempFile("temp.", ".properties");
+        try {
+            // Looking for /org/broadinstitute/sting/utils/file/StingText.properties
+            IOUtils.writeResource(new Resource("StingText.properties", IOUtils.class), temp);
+        } finally {
+            FileUtils.deleteQuietly(temp);
+        }
+    }
+
+    @Test
+    public void testResourceProperties() {
+        Resource resource = new Resource("foo", Resource.class);
+        Assert.assertEquals(resource.getPath(), "foo");
+        Assert.assertEquals(resource.getRelativeClass(), Resource.class);
+    }
+}

public/java/test/org/broadinstitute/sting/utils/runtime/ProcessControllerUnitTest.java

@@ -0,0 +1,517 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.runtime;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.broadinstitute.sting.BaseTest;
+import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
+import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.broadinstitute.sting.utils.io.IOUtils;
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+public class ProcessControllerUnitTest extends BaseTest {
+    private static final String NL = String.format("%n");
+
+    @Test(timeOut = 60 * 1000)
+    public void testDestroyThreadLocal() throws InterruptedException {
+        for (int i = 0; i < 3; i++) {
+            final ProcessController controller = ProcessController.getThreadLocal();
+            final ProcessSettings job = new ProcessSettings(
+                    new String[] {"sh", "-c", "echo Hello World && sleep 600 && echo Goodbye"});
+            job.getStdoutSettings().setBufferSize(-1);
+
+            Thread t = new Thread(new Runnable() {
+                @Override
+                public void run() {
+                    System.out.println("BACK: Starting on background thread");
+                    ProcessOutput result = controller.exec(job);
+                    // Assert in background thread doesn't make it to main thread but does print a trace.
+                    Assert.assertTrue(result.getExitValue() != 0, "Destroy-attempted job returned zero exit status");
+                    System.out.println("BACK: Background thread exiting");
+                }
+            });
+
+            System.out.println("MAIN: Starting background thread");
+            t.start();
+            System.out.println("MAIN: Sleeping main thread 3s");
+            Thread.sleep(3000);
+            System.out.println("MAIN: Destroying job");
+            controller.tryDestroy();
+            System.out.println("MAIN: Not waiting on background thread to exit");
+            // Using standard java.io this was blocking on linux.
+            // TODO: try again with NIO.
+            //t.join();
+            //System.out.println("MAIN: Background thread exited");
+        }
+    }
+
+    @Test
+    public void testReuseAfterError() {
+        ProcessController controller = new ProcessController();
+
+        ProcessSettings job;
+
+        for (int i = 0; i < 3; i++) {
+            // Test bad command
+            job = new ProcessSettings(new String[] {"no_such_command"});
+            try {
+                controller.exec(job);
+            } catch (ReviewedStingException e) {
+                /* Was supposed to throw an exception */
+            }
+
+            // Test exit != 0
+            job = new ProcessSettings(new String[] {"cat", "non_existent_file"});
+            int exitValue = controller.exec(job).getExitValue();
+            Assert.assertTrue(exitValue != 0, "'cat' non existent file returned 0");
+
+            // Text success
+            job = new ProcessSettings(new String[] {"echo", "Hello World"});
+            exitValue = controller.exec(job).getExitValue();
+            Assert.assertEquals(exitValue, 0, "Echo failed");
+        }
+    }
+
+    @Test
+    public void testEnvironment() {
+        String key = "MY_NEW_VAR";
+        String value = "value is here";
+
+        ProcessSettings job = new ProcessSettings(new String[] {"sh", "-c", "echo $"+key});
+        job.getStdoutSettings().setBufferSize(-1);
+        job.setRedirectErrorStream(true);
+
+        Map<String, String> env = new HashMap<String, String>(System.getenv());
+        env.put(key, value);
+        job.setEnvironment(env);
+
+        ProcessController controller = new ProcessController();
+        ProcessOutput result = controller.exec(job);
+        int exitValue = result.getExitValue();
+
+        Assert.assertEquals(exitValue, 0, "Echo environment variable failed");
+        Assert.assertEquals(result.getStdout().getBufferString(), value + NL, "Echo environment returned unexpected output");
+    }
+
+    @Test
+    public void testDirectory() throws IOException {
+        File dir = null;
+        try {
+            dir = IOUtils.tempDir("temp.", "").getCanonicalFile();
+
+            ProcessSettings job = new ProcessSettings(new String[] {"pwd"});
+            job.getStdoutSettings().setBufferSize(-1);
+            job.setRedirectErrorStream(true);
+            job.setDirectory(dir);
+
+            ProcessController controller = new ProcessController();
+            ProcessOutput result = controller.exec(job);
+            int exitValue = result.getExitValue();
+
+            Assert.assertEquals(exitValue, 0, "Getting working directory failed");
+
+            Assert.assertEquals(result.getStdout().getBufferString(), dir.getAbsolutePath() + NL,
+                    "Setting/getting working directory returned unexpected output");
+        } finally {
+            FileUtils.deleteQuietly(dir);
+        }
+    }
+
+    @Test
+    public void testReadStdInBuffer() {
+        String bufferText = "Hello from buffer";
+        ProcessSettings job = new ProcessSettings(new String[] {"cat"});
+        job.getStdoutSettings().setBufferSize(-1);
+        job.setRedirectErrorStream(true);
+        job.getStdinSettings().setInputBuffer(bufferText);
+
+        ProcessController controller = new ProcessController();
+        ProcessOutput output = controller.exec(job);
+
+        Assert.assertEquals(output.getStdout().getBufferString(), bufferText,
+                "Unexpected output from cat stdin buffer");
+    }
+
+    @Test
+    public void testReadStdInFile() {
+        File input = null;
+        try {
+            String fileText = "Hello from file";
+            input = IOUtils.writeTempFile(fileText, "stdin.", ".txt", null);
+
+            ProcessSettings job = new ProcessSettings(new String[] {"cat"});
+            job.getStdoutSettings().setBufferSize(-1);
+            job.setRedirectErrorStream(true);
+            job.getStdinSettings().setInputFile(input);
+
+            ProcessController controller = new ProcessController();
+            ProcessOutput output = controller.exec(job);
+
+            Assert.assertEquals(output.getStdout().getBufferString(), fileText,
+                    "Unexpected output from cat stdin file");
+        } finally {
+            FileUtils.deleteQuietly(input);
+        }
+    }
+
+    @Test
+    public void testWriteStdOut() {
+        ProcessSettings job = new ProcessSettings(new String[] {"echo", "Testing to stdout"});
+        // Not going to call the System.setOut() for now. Just running a basic visual test.
+        job.getStdoutSettings().printStandard(true);
+        job.setRedirectErrorStream(true);
+
+        System.out.println("testWriteStdOut: Writing two lines to std out...");
+        ProcessController controller = new ProcessController();
+        controller.exec(job);
+        job.setCommand(new String[]{"cat", "non_existent_file"});
+        controller.exec(job);
+        System.out.println("testWriteStdOut: ...two lines should have been printed to std out");
+    }
+
+    @Test
+    public void testErrorToOut() throws IOException {
+        File outFile = null;
+        File errFile = null;
+        try {
+            outFile = BaseTest.createTempFile("temp", "");
+            errFile = BaseTest.createTempFile("temp", "");
+
+            ProcessSettings job = new ProcessSettings(new String[]{"cat", "non_existent_file"});
+            job.getStdoutSettings().setOutputFile(outFile);
+            job.getStdoutSettings().setBufferSize(-1);
+            job.getStderrSettings().setOutputFile(errFile);
+            job.getStderrSettings().setBufferSize(-1);
+            job.setRedirectErrorStream(true);
+
+            ProcessOutput result = new ProcessController().exec(job);
+            int exitValue = result.getExitValue();
+
+            Assert.assertTrue(exitValue != 0, "'cat' non existent file returned 0");
+
+            String fileString, bufferString;
+
+            fileString = FileUtils.readFileToString(outFile);
+            Assert.assertTrue(fileString.length() > 0, "Out file was length 0");
+
+            bufferString = result.getStdout().getBufferString();
+            Assert.assertTrue(bufferString.length() > 0, "Out buffer was length 0");
+
+            Assert.assertFalse(result.getStdout().isBufferTruncated(), "Out buffer was truncated");
+            Assert.assertEquals(bufferString.length(), fileString.length(), "Out buffer length did not match file length");
+
+            fileString = FileUtils.readFileToString(errFile);
+            Assert.assertEquals(fileString, "", "Unexpected output to err file");
+
+            bufferString = result.getStderr().getBufferString();
+            Assert.assertEquals(bufferString, "", "Unexepected output to err buffer");
+        } finally {
+            FileUtils.deleteQuietly(outFile);
+            FileUtils.deleteQuietly(errFile);
+        }
+    }
+
+    @Test
+    public void testErrorToErr() throws IOException {
+        File outFile = null;
+        File errFile = null;
+        try {
+            outFile = BaseTest.createTempFile("temp", "");
+            errFile = BaseTest.createTempFile("temp", "");
+
+            ProcessSettings job = new ProcessSettings(new String[]{"cat", "non_existent_file"});
+            job.getStdoutSettings().setOutputFile(outFile);
+            job.getStdoutSettings().setBufferSize(-1);
+            job.getStderrSettings().setOutputFile(errFile);
+            job.getStderrSettings().setBufferSize(-1);
+            job.setRedirectErrorStream(false);
+
+            ProcessOutput result = new ProcessController().exec(job);
+            int exitValue = result.getExitValue();
+
+            Assert.assertTrue(exitValue != 0, "'cat' non existent file returned 0");
+
+            String fileString, bufferString;
+
+            fileString = FileUtils.readFileToString(errFile);
+            Assert.assertTrue(fileString.length() > 0, "Err file was length 0");
+
+            bufferString = result.getStderr().getBufferString();
+            Assert.assertTrue(bufferString.length() > 0, "Err buffer was length 0");
+
+            Assert.assertFalse(result.getStderr().isBufferTruncated(), "Err buffer was truncated");
+            Assert.assertEquals(bufferString.length(), fileString.length(), "Err buffer length did not match file length");
+
+            fileString = FileUtils.readFileToString(outFile);
+            Assert.assertEquals(fileString, "", "Unexpected output to out file");
+
+            bufferString = result.getStdout().getBufferString();
+            Assert.assertEquals(bufferString, "", "Unexepected output to out buffer");
+        } finally {
+            FileUtils.deleteQuietly(outFile);
+            FileUtils.deleteQuietly(errFile);
+        }
+    }
+
+    private static final String TRUNCATE_TEXT = "Hello World";
+    private static final byte[] TRUNCATE_OUTPUT_BYTES = (TRUNCATE_TEXT + NL).getBytes();
+
+    /**
+     * @return Test truncating content vs. not truncating (run at -1/+1 size)
+     */
+    @DataProvider(name = "truncateSizes")
+    public Object[][] getTruncateBufferSizes() {
+        int l = TRUNCATE_OUTPUT_BYTES.length;
+        return new Object[][]{
+                new Object[]{0, 0},
+                new Object[]{l, l},
+                new Object[]{l + 1, l},
+                new Object[]{l - 1, l - 1}
+        };
+    }
+
+    @Test(dataProvider = "truncateSizes")
+    public void testTruncateBuffer(int truncateLen, int expectedLen) {
+        byte[] expected = Arrays.copyOf(TRUNCATE_OUTPUT_BYTES, expectedLen);
+
+        String[] command = {"echo", TRUNCATE_TEXT};
+        ProcessController controller = new ProcessController();
+
+        ProcessSettings job = new ProcessSettings(command);
+        job.getStdoutSettings().setBufferSize(truncateLen);
+        ProcessOutput result = controller.exec(job);
+
+        int exitValue = result.getExitValue();
+
+        Assert.assertEquals(exitValue, 0,
+                String.format("Echo returned %d: %s", exitValue, TRUNCATE_TEXT));
+
+        byte[] bufferBytes = result.getStdout().getBufferBytes();
+
+        Assert.assertEquals(bufferBytes, expected,
+                String.format("Output buffer didn't match (%d vs %d)", expected.length, bufferBytes.length));
+
+        boolean truncated = result.getStdout().isBufferTruncated();
+
+        Assert.assertEquals(truncated, TRUNCATE_OUTPUT_BYTES.length > truncateLen,
+                "Unexpected buffer truncation result");
+    }
+
+    private static final String[] LONG_COMMAND = getLongCommand();
+    private static final String LONG_COMMAND_STRING = StringUtils.join(LONG_COMMAND, " ");
+    private static final String LONG_COMMAND_DESCRIPTION = "<long command>";
+
+    @DataProvider(name = "echoCommands")
+    public Object[][] getEchoCommands() {
+
+        new EchoCommand(new String[]{"echo", "Hello", "World"}, "Hello World" + NL);
+        new EchoCommand(new String[]{"echo", "'Hello", "World"}, "'Hello World" + NL);
+        new EchoCommand(new String[]{"echo", "Hello", "World'"}, "Hello World'" + NL);
+        new EchoCommand(new String[]{"echo", "'Hello", "World'"}, "'Hello World'" + NL);
+
+        String[] longCommand = new String[LONG_COMMAND.length + 1];
+        longCommand[0] = "echo";
+        System.arraycopy(LONG_COMMAND, 0, longCommand, 1, LONG_COMMAND.length);
+        new EchoCommand(longCommand, LONG_COMMAND_STRING + NL) {
+            @Override
+            public String toString() {
+                return LONG_COMMAND_DESCRIPTION;
+            }
+        };
+
+        return TestDataProvider.getTests(EchoCommand.class);
+    }
+
+    @Test(dataProvider = "echoCommands")
+    public void testEcho(EchoCommand script) throws IOException {
+        File outputFile = null;
+        try {
+            outputFile = BaseTest.createTempFile("temp", "");
+
+            ProcessSettings job = new ProcessSettings(script.command);
+            if (script.output != null) {
+                job.getStdoutSettings().setOutputFile(outputFile);
+                job.getStdoutSettings().setBufferSize(script.output.getBytes().length);
+            }
+
+            ProcessOutput result = new ProcessController().exec(job);
+            int exitValue = result.getExitValue();
+
+            Assert.assertEquals(exitValue, 0,
+                    String.format("Echo returned %d: %s", exitValue, script));
+
+            if (script.output != null) {
+
+                String fileString = FileUtils.readFileToString(outputFile);
+                Assert.assertEquals(fileString, script.output,
+                        String.format("Output file didn't match (%d vs %d): %s",
+                                fileString.length(), script.output.length(), script));
+
+                String bufferString = result.getStdout().getBufferString();
+                Assert.assertEquals(bufferString, script.output,
+                        String.format("Output content didn't match (%d vs %d): %s",
+                                bufferString.length(), script.output.length(), script));
+
+                Assert.assertFalse(result.getStdout().isBufferTruncated(),
+                        "Output content was truncated: " + script);
+            }
+        } finally {
+            FileUtils.deleteQuietly(outputFile);
+        }
+    }
+
+    @Test(expectedExceptions = ReviewedStingException.class)
+    public void testUnableToStart() {
+        ProcessSettings job = new ProcessSettings(new String[]{"no_such_command"});
+        new ProcessController().exec(job);
+    }
+
+    @DataProvider(name = "scriptCommands")
+    public Object[][] getScriptCommands() {
+        new ScriptCommand(true, "echo Hello World", "Hello World" + NL);
+        new ScriptCommand(false, "echo 'Hello World", null);
+        new ScriptCommand(false, "echo Hello World'", null);
+        new ScriptCommand(true, "echo 'Hello World'", "Hello World" + NL);
+        new ScriptCommand(true, "echo \"Hello World\"", "Hello World" + NL);
+        new ScriptCommand(false, "no_such_echo Hello World", null);
+        new ScriptCommand(true, "echo #", NL);
+        new ScriptCommand(true, "echo \\#", "#" + NL);
+        new ScriptCommand(true, "echo \\\\#", "\\#" + NL);
+
+        new ScriptCommand(true, "echo " + LONG_COMMAND_STRING, LONG_COMMAND_STRING + NL) {
+            @Override
+            public String toString() {
+                return LONG_COMMAND_DESCRIPTION;
+            }
+        };
+
+        return TestDataProvider.getTests(ScriptCommand.class);
+    }
+
+    @Test(dataProvider = "scriptCommands")
+    public void testScript(ScriptCommand script) throws IOException {
+        File scriptFile = null;
+        File outputFile = null;
+        try {
+            scriptFile = writeScript(script.content);
+            outputFile = BaseTest.createTempFile("temp", "");
+
+            ProcessSettings job = new ProcessSettings(new String[]{"sh", scriptFile.getAbsolutePath()});
+            if (script.output != null) {
+                job.getStdoutSettings().setOutputFile(outputFile);
+                job.getStdoutSettings().setBufferSize(script.output.getBytes().length);
+            }
+
+            ProcessOutput result = new ProcessController().exec(job);
+            int exitValue = result.getExitValue();
+
+            Assert.assertEquals(exitValue == 0, script.succeed,
+                    String.format("Script returned %d: %s", exitValue, script));
+
+            if (script.output != null) {
+
+                String fileString = FileUtils.readFileToString(outputFile);
+                Assert.assertEquals(fileString, script.output,
+                        String.format("Output file didn't match (%d vs %d): %s",
+                                fileString.length(), script.output.length(), script));
+
+                String bufferString = result.getStdout().getBufferString();
+                Assert.assertEquals(bufferString, script.output,
+                        String.format("Output content didn't match (%d vs %d): %s",
+                                bufferString.length(), script.output.length(), script));
+
+                Assert.assertFalse(result.getStdout().isBufferTruncated(),
+                        "Output content was truncated: " + script);
+            }
+        } finally {
+            FileUtils.deleteQuietly(scriptFile);
+            FileUtils.deleteQuietly(outputFile);
+        }
+    }
+
+    private static String[] getLongCommand() {
+        // This command fails on some systems with a 4096 character limit when run via the old sh -c "echo ...",
+        // but works on the same systems when run via sh <script>
+        int cnt = 500;
+        String[] command = new String[cnt];
+        for (int i = 1; i <= cnt; i++) {
+            command[i - 1] = String.format("%03d______", i);
+        }
+        return command;
+    }
+
+    private static File writeScript(String contents) {
+        try {
+            File file = BaseTest.createTempFile("temp", "");
+            FileUtils.writeStringToFile(file, contents);
+            return file;
+        } catch (IOException e) {
+            throw new UserException.BadTmpDir(e.getMessage());
+        }
+    }
+
+    private static class EchoCommand extends TestDataProvider {
+        public final String[] command;
+        public final String output;
+
+        public EchoCommand(String[] command, String output) {
+            super(EchoCommand.class);
+            this.command = command;
+            this.output = output;
+        }
+
+        @Override
+        public String toString() {
+            return StringUtils.join(command, " ");
+        }
+    }
+
+    public static class ScriptCommand extends TestDataProvider {
+        public final boolean succeed;
+        public final String content;
+        public final String output;
+
+        public ScriptCommand(boolean succeed, String content, String output) {
+            super(ScriptCommand.class);
+            this.succeed = succeed;
+            this.content = content;
+            this.output = output;
+        }
+
+        @Override
+        public String toString() {
+            return content;
+        }
+    }
+}

public/packages/CreatePackager.xsl

@@ -179,13 +179,19 @@
       </xsl:when>
     </xsl:choose>
   </xsl:for-each>
-    <xsl:for-each select="dir">
+  <xsl:for-each select="dir">
-        <fileset dir="{$staging.dir}">
+    <xsl:variable name="includes">
-            <xsl:attribute name="includes">
+      <xsl:choose>
-                <xsl:value-of select="concat(@name,'/**')"/>
+        <xsl:when test="@includes = ''">
-            </xsl:attribute>
+          <xsl:value-of select="concat(@name,'/**')"/>
-        </fileset>
+        </xsl:when>
-    </xsl:for-each>
+        <xsl:otherwise>
+          <xsl:value-of select="concat(@name,'/',@includes)"/>
+        </xsl:otherwise>
+      </xsl:choose>
+    </xsl:variable>
+    <fileset dir="{$staging.dir}" includes="{$includes}"/>
+  </xsl:for-each>
 </xsl:template>
 
 <!-- Determine the short name (filename w/o directory structure of the given filename -->

public/packages/GATKEngine.xml

@@ -32,6 +32,8 @@
       <package name="org.broad.tribble.**" />
       <!-- Workaround - depend on the logger impl required by JEXL -->
       <package name="org.apache.commons.logging.impl" />
+      <!-- R packages -->
+      <dir name="org/broadinstitute/sting/utils/R" includes="*.tar.gz" />
     </dependencies>
   </executable>
   <resources>

public/packages/Queue.xml

@@ -65,6 +65,10 @@
             <package name="ca.mcgill.mcb.pcingola.**" />
             <file path="snpEff_genes.ftl" />
             <file path="snpEff_summary.ftl" />
+            
+            <!-- R scripts -->
+            <dir name="org/broadinstitute/sting/queue" includes="**/*.R" />
+
         </dependencies>
         <modules>
             <module file="GATKEngine.xml"/>

public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala

@@ -33,6 +33,7 @@ import org.broadinstitute.sting.queue.engine.{QGraphSettings, QGraph}
 import collection.JavaConversions._
 import org.broadinstitute.sting.utils.classloader.PluginManager
 import org.broadinstitute.sting.utils.exceptions.UserException
+import org.broadinstitute.sting.utils.io.IOUtils
 
 /**
  * Entry point of Queue.  Compiles and runs QScripts passed in to the command line.
@@ -48,7 +49,6 @@ object QCommandLine extends Logging {
     val shutdownHook = new Thread {
       override def run() {
         logger.info("Shutting down jobs. Please wait...")
-        ProcessController.shutdown()
         qCommandLine.shutdown()
       }
     }
@@ -56,8 +56,12 @@ object QCommandLine extends Logging {
     Runtime.getRuntime.addShutdownHook(shutdownHook)
 
     try {
-      CommandLineProgram.start(qCommandLine, argv);
+      CommandLineProgram.start(qCommandLine, argv)
-      Runtime.getRuntime.removeShutdownHook(shutdownHook)
+      try {
+        Runtime.getRuntime.removeShutdownHook(shutdownHook)
+      } catch {
+        case _ => /* ignore, example 'java.lang.IllegalStateException: Shutdown in progress' */
+      }
       if (CommandLineProgram.result != 0)
         System.exit(CommandLineProgram.result);
     } catch {
@@ -80,6 +84,7 @@ class QCommandLine extends CommandLineProgram with Logging {
   private val qScriptManager = new QScriptManager
   private val qGraph = new QGraph
   private var qScriptClasses: File = _
+  private var shuttingDown = false
 
   private lazy val pluginManager = {
     qScriptClasses = IOUtils.tempDir("Q-Classes", "", settings.qSettings.tempDirectory)
@@ -118,10 +123,16 @@ class QCommandLine extends CommandLineProgram with Logging {
       script.onExecutionDone(qGraph.getFunctionsAndStatus(script.functions), qGraph.success)
       if ( ! settings.disableJobReport ) {
         val jobStringName = (QScriptUtils.?(settings.jobReportFile)).getOrElse(settings.qSettings.jobNamePrefix + ".jobreport.txt")
-        val jobReportFile = new File(jobStringName)
+
-        logger.info("Writing JobLogging GATKReport to file " + jobReportFile)
+        if (!shuttingDown) {
-        QJobReport.printReport(qGraph.getFunctionsAndStatus(script.functions), jobReportFile)
+          val reportFile = new File(jobStringName)
-        QJobReport.plotReport(settings.rScriptArgs, jobReportFile)
+          logger.info("Writing JobLogging GATKReport to file " + reportFile)
+          QJobReport.printReport(qGraph.getFunctionsAndStatus(script.functions), reportFile)
+
+          val pdfFile = new File(jobStringName + ".pdf")
+          logger.info("Plotting JobLogging GATKReport to file " + pdfFile)
+          QJobReport.plotReport(settings.rScriptArgs, reportFile, pdfFile)
+        }
       }
     }
 
@@ -163,6 +174,7 @@ class QCommandLine extends CommandLineProgram with Logging {
     Arrays.asList(new ScalaCompoundArgumentTypeDescriptor)
 
   def shutdown() = {
+    shuttingDown = true
     qGraph.shutdown()
     if (qScriptClasses != null) IOUtils.tryDelete(qScriptClasses)
   }

public/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala

@@ -2,7 +2,7 @@ package org.broadinstitute.sting.queue
 
 import scala.tools.nsc.{Global, Settings}
 import scala.tools.nsc.io.PlainFile
-import org.broadinstitute.sting.queue.util.{Logging, IOUtils}
+import org.broadinstitute.sting.queue.util.Logging
 import collection.JavaConversions._
 import java.io.File
 import scala.tools.nsc.reporters.AbstractReporter
@@ -20,7 +20,7 @@ class QScriptManager() extends Logging {
    * Compiles and loads the scripts in the files into the current classloader.
    * Heavily based on scala/src/compiler/scala/tools/ant/Scalac.scala
    */
-  def loadScripts(scripts: List[File], tempDir: File) = {
+  def loadScripts(scripts: List[File], tempDir: File) {
     if (scripts.size > 0) {
       val settings = new Settings((error: String) => logger.error(error))
       settings.deprecation.value = true
@@ -63,7 +63,7 @@ object QScriptManager extends Logging {
    * Heavily based on scala/src/compiler/scala/tools/nsc/reporters/ConsoleReporter.scala
    */
   private class Log4JReporter(val settings: Settings) extends AbstractReporter {
-    def displayPrompt = throw new UnsupportedOperationException("Unable to prompt the user.  Prompting should be off.")
+    def displayPrompt { throw new UnsupportedOperationException("Unable to prompt the user.  Prompting should be off.") }
 
     /**
      * Displays the message at position with severity.
@@ -71,7 +71,7 @@ object QScriptManager extends Logging {
      * @param msg Message to display.
      * @param severity Severity of the event.
      */
-    def display(posIn: Position, msg: String, severity: Severity) = {
+    def display(posIn: Position, msg: String, severity: Severity) {
       severity.count += 1
       val level = severity match {
         case INFO => Level.INFO
@@ -87,7 +87,6 @@ object QScriptManager extends Logging {
         case NoPosition =>
           printMessage(level, msg)
         case _ =>
-          val buf = new StringBuilder(msg)
           val file = pos.source.file
           printMessage(level, file.name+":"+pos.line+": "+msg)
           printSourceLine(level, pos)
@@ -97,7 +96,7 @@ object QScriptManager extends Logging {
     /**
      * Prints a summary count of warnings and errors.
      */
-    def printSummary() = {
+    def printSummary() {
       if (WARNING.count > 0)
         printMessage(Level.WARN, countElementsAsString(WARNING.count, "warning") + " found")
       if (ERROR.count > 0)
@@ -119,15 +118,16 @@ object QScriptManager extends Logging {
      * @param level Severity level.
      * @param pos Position in the file of the event.
      */
-    private def printColumnMarker(level: Level, pos: Position) =
+    private def printColumnMarker(level: Level, pos: Position) {
       if (pos.isDefined) { printMessage(level, " " * (pos.column - 1) + "^") }
+    }
 
     /**
      * Prints the message at the severity level.
      * @param level Severity level.
      * @param message Message content.
      */
-    private def printMessage(level: Level, message: String) = {
+    private def printMessage(level: Level, message: String) {
       logger.log(level, message)
     }
   }

public/scala/src/org/broadinstitute/sting/queue/engine/CommandLineJobRunner.scala

@@ -26,7 +26,8 @@ package org.broadinstitute.sting.queue.engine
 
 import org.broadinstitute.sting.queue.function.CommandLineFunction
 import java.io.File
-import org.broadinstitute.sting.queue.util.{Logging, IOUtils}
+import org.broadinstitute.sting.queue.util.Logging
+import org.broadinstitute.sting.utils.io.IOUtils
 
 /**
  * Runs a command line function.
@@ -69,7 +70,7 @@ trait CommandLineJobRunner extends JobRunner[CommandLineFunction] with Logging {
 
   override def init() {
     super.init()
-    var exec = new StringBuilder
+    val exec = new StringBuilder
     
     var dirs = Set.empty[File]
     for (dir <- function.jobDirectories)

public/scala/src/org/broadinstitute/sting/queue/engine/FunctionEdge.scala

@@ -2,7 +2,10 @@ package org.broadinstitute.sting.queue.engine
 
 import org.broadinstitute.sting.queue.function.QFunction
 import java.io.{StringWriter, PrintWriter}
-import org.broadinstitute.sting.queue.util.{Logging, IOUtils}
+import org.broadinstitute.sting.queue.util.Logging
+import org.broadinstitute.sting.utils.io.IOUtils
+import org.apache.commons.io.FileUtils
+import org.apache.commons.lang.StringUtils
 
 /**
  * An edge in the QGraph that runs a QFunction.
@@ -150,26 +153,19 @@ class FunctionEdge(val function: QFunction, val inputs: QNode, val outputs: QNod
   /**
    * Outputs the last lines of the error logs.
    */
-  private def tailError() = {
+  private def tailError() {
     val errorFile = functionErrorFile
     if (IOUtils.waitFor(errorFile, 120)) {
       val maxLines = 100
       val tailLines = IOUtils.tail(errorFile, maxLines)
       val nl = "%n".format()
       val summary = if (tailLines.size > maxLines) "Last %d lines".format(maxLines) else "Contents"
-      logger.error("%s of %s:%n%s".format(summary, errorFile, tailLines.mkString(nl)))
+      logger.error("%s of %s:%n%s".format(summary, errorFile, StringUtils.join(tailLines, nl)))
     } else {
       logger.error("Unable to access log file: %s".format(errorFile))
     }
   }
 
-  /**
-   * Writes the contents of the error to the error file.
-   */
-  private def writeError(content: String) {
-    IOUtils.writeContents(functionErrorFile, content)
-  }
-
   /**
    * Writes the stack trace to the error file.
    */
@@ -178,8 +174,8 @@ class FunctionEdge(val function: QFunction, val inputs: QNode, val outputs: QNod
     val printWriter = new PrintWriter(stackTrace)
     printWriter.println(function.description)
     e.printStackTrace(printWriter)
-    printWriter.close
+    printWriter.close()
-    IOUtils.writeContents(functionErrorFile, stackTrace.toString)
+    FileUtils.writeStringToFile(functionErrorFile, stackTrace.toString)
   }
 
   def getRunInfo = {

public/scala/src/org/broadinstitute/sting/queue/engine/InProcessRunner.scala

@@ -2,8 +2,8 @@ package org.broadinstitute.sting.queue.engine
 
 import org.broadinstitute.sting.queue.function.InProcessFunction
 import java.util.Date
-import org.broadinstitute.sting.queue.util.{Logging, IOUtils}
 import org.broadinstitute.sting.utils.Utils
+import org.apache.commons.io.FileUtils
 
 /**
  * Runs a function that executes in process and does not fork out an external process.
@@ -11,7 +11,7 @@ import org.broadinstitute.sting.utils.Utils
 class InProcessRunner(val function: InProcessFunction) extends JobRunner[InProcessFunction] {
   private var runStatus: RunnerStatus.Value = _
 
-  def start() = {
+  def start() {
     getRunInfo.startTime = new Date()
     getRunInfo.exechosts = Utils.resolveHostname()
     runStatus = RunnerStatus.RUNNING
@@ -20,7 +20,7 @@ class InProcessRunner(val function: InProcessFunction) extends JobRunner[InProce
 
     getRunInfo.doneTime = new Date()
     val content = "%s%nDone.".format(function.description)
-    IOUtils.writeContents(function.jobOutputFile, content)
+    FileUtils.writeStringToFile(function.jobOutputFile, content)
     runStatus = RunnerStatus.DONE
   }
 

public/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala

@@ -40,6 +40,7 @@ import collection.immutable.{TreeSet, TreeMap}
 import org.broadinstitute.sting.queue.function.scattergather.{ScatterFunction, CloneFunction, GatherFunction, ScatterGatherableFunction}
 import java.util.Date
 import org.broadinstitute.sting.utils.Utils
+import org.broadinstitute.sting.utils.io.IOUtils
 
 /**
  * The internal dependency tracker between sets of function input and output files.
@@ -416,8 +417,12 @@ class QGraph extends Logging {
           startedJobsToEmail = Set.empty[FunctionEdge]
         }
 
-        if (readyJobs.size == 0 && runningJobs.size > 0)
+        if (readyJobs.size == 0 && runningJobs.size > 0) {
-          Thread.sleep(nextRunningCheck(lastRunningCheck))
+          runningLock.synchronized {
+            if (running)
+              runningLock.wait(nextRunningCheck(lastRunningCheck))
+          }
+        }
 
         lastRunningCheck = System.currentTimeMillis
         updateStatus()
@@ -1002,7 +1007,12 @@ class QGraph extends Logging {
       true
     } else {
       !this.jobGraph.edgeSet.exists(edge => {
-        edge.isInstanceOf[FunctionEdge] && edge.asInstanceOf[FunctionEdge].status == RunnerStatus.FAILED
+        if (edge.isInstanceOf[FunctionEdge]) {
+          val status = edge.asInstanceOf[FunctionEdge].status
+          (status == RunnerStatus.PENDING || status == RunnerStatus.RUNNING || status == RunnerStatus.FAILED)
+        } else {
+          false
+        }
       })
     }
   }
@@ -1051,7 +1061,13 @@ class QGraph extends Logging {
   def shutdown() {
     // Signal the main thread to shutdown.
     running = false
-    // Wait for the thread to finish and exit normally.
+
+    // Try and wait for the thread to finish and exit normally.
+    runningLock.synchronized {
+      runningLock.notify()
+    }
+
+    // Start killing jobs.
     runningLock.synchronized {
       val runners = runningJobs.map(_.runner)
       runningJobs = Set.empty[FunctionEdge]

public/scala/src/org/broadinstitute/sting/queue/engine/drmaa/DrmaaJobManager.scala

@@ -56,6 +56,6 @@ class DrmaaJobManager extends CommandLineJobManager[DrmaaJobRunner] {
     updatedRunners
   }
   override def tryStop(runners: Set[DrmaaJobRunner]) {
-    runners.filterNot(_.jobId == null).foreach(_.tryStop())
+    runners.foreach(_.tryStop())
   }
 }

public/scala/src/org/broadinstitute/sting/queue/engine/drmaa/DrmaaJobRunner.scala

@@ -135,14 +135,18 @@ class DrmaaJobRunner(val session: Session, val function: CommandLineFunction) ex
 
   def tryStop() {
     session.synchronized {
-      try {
+      // Assumes that after being set the job may be
-        // Stop runners. SIGTERM(15) is preferred to SIGKILL(9).
+      // reassigned but will not be reset back to null
-        // Only way to send SIGTERM is for the Sys Admin set the terminate_method
+      if (jobId != null) {
-        // resource of the designated queue to SIGTERM
+        try {
-        session.control(jobId, Session.TERMINATE)
+          // Stop runners. SIGTERM(15) is preferred to SIGKILL(9).
-      } catch {
+          // Only way to send SIGTERM is for the Sys Admin set the terminate_method
-        case e =>
+          // resource of the designated queue to SIGTERM
-          logger.error("Unable to kill job " + jobId, e)
+          session.control(jobId, Session.TERMINATE)
+        } catch {
+          case e =>
+            logger.error("Unable to kill job " + jobId, e)
+        }
       }
     }
   }

public/scala/src/org/broadinstitute/sting/queue/engine/shell/ShellJobManager.scala

@@ -30,4 +30,5 @@ import org.broadinstitute.sting.queue.engine.CommandLineJobManager
 class ShellJobManager extends CommandLineJobManager[ShellJobRunner] {
   def runnerType = classOf[ShellJobRunner]
   def create(function: CommandLineFunction) = new ShellJobRunner(function)
+  override def tryStop(runners: Set[ShellJobRunner]) { runners.foreach(_.tryStop()) }
 }

public/scala/src/org/broadinstitute/sting/queue/engine/shell/ShellJobRunner.scala

@@ -25,41 +25,66 @@
 package org.broadinstitute.sting.queue.engine.shell
 
 import org.broadinstitute.sting.queue.function.CommandLineFunction
-import org.broadinstitute.sting.queue.util.ShellJob
 import org.broadinstitute.sting.queue.engine.{RunnerStatus, CommandLineJobRunner}
 import java.util.Date
-import org.broadinstitute.sting.gatk.phonehome.GATKRunReport
 import org.broadinstitute.sting.utils.Utils
+import org.broadinstitute.sting.utils.runtime.{ProcessSettings, OutputStreamSettings, ProcessController}
 
 /**
  * Runs jobs one at a time locally
  */
 class ShellJobRunner(val function: CommandLineFunction) extends CommandLineJobRunner {
-  private var runStatus: RunnerStatus.Value = _
+  // Controller on the thread that started the job
+  private var controller: ProcessController = null
 
   /**
    * Runs the function on the local shell.
    * @param function Command to run.
    */
   def start() {
-    val job = new ShellJob
+    val commandLine = Array("sh", jobScript.getAbsolutePath)
+    val stdoutSettings = new OutputStreamSettings
+    val stderrSettings = new OutputStreamSettings
+    val mergeError = (function.jobErrorFile != null)
 
-    job.workingDir = function.commandDirectory
+    stdoutSettings.setOutputFile(function.jobOutputFile, true)
-    job.outputFile = function.jobOutputFile
+    if (function.jobErrorFile != null)
-    job.errorFile = function.jobErrorFile
+      stderrSettings.setOutputFile(function.jobErrorFile, true)
 
-    job.shellScript = jobScript
+    if (logger.isDebugEnabled) {
+      stdoutSettings.printStandard(true)
+      stderrSettings.printStandard(true)
+    }
 
-    // Allow advanced users to update the job.
+    val processSettings = new ProcessSettings(
-    updateJobRun(job)
+      commandLine, mergeError, function.commandDirectory, null,
+      null, stdoutSettings, stderrSettings)
+
+    updateJobRun(processSettings)
 
     getRunInfo.startTime = new Date()
     getRunInfo.exechosts = Utils.resolveHostname()
     updateStatus(RunnerStatus.RUNNING)
-    job.run()
+    controller = ProcessController.getThreadLocal
+    val exitStatus = controller.exec(processSettings).getExitValue
     getRunInfo.doneTime = new Date()
-    updateStatus(RunnerStatus.DONE)
+    updateStatus(if (exitStatus == 0) RunnerStatus.DONE else RunnerStatus.FAILED)
   }
 
-  override def checkUnknownStatus() {}
+  /**
+   * Possibly invoked from a shutdown thread, find and
+   * stop the controller from the originating thread
+   */
+  def tryStop() {
+    // Assumes that after being set the job may be
+    // reassigned but will not be reset back to null
+    if (controller != null) {
+      try {
+        controller.tryDestroy()
+      } catch {
+        case e =>
+          logger.error("Unable to kill shell job: " + function.description)
+      }
+    }
+  }
 }

public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKScatterFunction.scala

@@ -27,7 +27,7 @@ package org.broadinstitute.sting.queue.extensions.gatk
 import org.broadinstitute.sting.utils.interval.IntervalUtils
 import java.io.File
 import collection.JavaConversions._
-import org.broadinstitute.sting.queue.util.IOUtils
+import org.broadinstitute.sting.utils.io.IOUtils
 import org.broadinstitute.sting.queue.function.scattergather.{CloneFunction, ScatterFunction}
 import org.broadinstitute.sting.commandline.Output
 

public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/RodBind.scala

@@ -1,7 +1,7 @@
 package org.broadinstitute.sting.queue.extensions.gatk
 
 import java.io.File
-import org.broadinstitute.sting.queue.util.FileExtension
+import org.broadinstitute.sting.utils.io.FileExtension
 import java.lang.String
 
 /**

public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/TaggedFile.scala

@@ -1,7 +1,7 @@
 package org.broadinstitute.sting.queue.extensions.gatk
 
 import java.io.File
-import org.broadinstitute.sting.queue.util.FileExtension
+import org.broadinstitute.sting.utils.io.FileExtension
 
 /**
  * Used to provide tagged -I input_file arguments to the GATK.

public/scala/src/org/broadinstitute/sting/queue/function/JavaCommandLineFunction.scala

@@ -25,7 +25,7 @@
 package org.broadinstitute.sting.queue.function
 
 import org.broadinstitute.sting.commandline.Argument
-import org.broadinstitute.sting.queue.util.IOUtils
+import org.broadinstitute.sting.utils.io.IOUtils
 import java.io.File
 
 /**

public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala

@@ -31,6 +31,7 @@ import org.broadinstitute.sting.queue.{QException, QSettings}
 import collection.JavaConversions._
 import org.broadinstitute.sting.queue.function.scattergather.SimpleTextGatherFunction
 import org.broadinstitute.sting.queue.util._
+import org.broadinstitute.sting.utils.io.IOUtils
 
 /**
  * The base interface for all functions in Queue.

public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GatherFunction.scala

@@ -4,7 +4,8 @@ import java.io.File
 import org.broadinstitute.sting.commandline.{Input, Output}
 import org.broadinstitute.sting.queue.function.QFunction
 import org.broadinstitute.sting.queue.QException
-import org.broadinstitute.sting.queue.util.IOUtils
+import org.broadinstitute.sting.utils.io.IOUtils
+import collection.JavaConversions._
 
 /**
  * Base class for Gather command line functions.
@@ -29,7 +30,7 @@ trait GatherFunction extends QFunction {
   /**
    * Waits for gather parts to propagate over NFS or throws an exception.
    */
-  protected def waitForGatherParts = {
+  protected def waitForGatherParts() {
     val missing = IOUtils.waitFor(gatherParts, 120)
     if (!missing.isEmpty)
       throw new QException("Unable to find gather inputs: " + missing.mkString(", "))

public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala

@@ -29,6 +29,7 @@ import org.broadinstitute.sting.queue.util._
 import org.broadinstitute.sting.commandline.{Gatherer, Gather, ArgumentSource}
 import org.broadinstitute.sting.queue.function.{QFunction, CommandLineFunction}
 import org.broadinstitute.sting.queue.QException
+import org.broadinstitute.sting.utils.io.IOUtils
 
 /**
  * A function that can be run faster by splitting it up into pieces and then joining together the results.
@@ -82,8 +83,8 @@ trait ScatterGatherableFunction extends CommandLineFunction {
   /**
    * Sets the scatter gather directory to the command directory if it is not already set.
    */
-  override def freezeFieldValues = {
+  override def freezeFieldValues() {
-    super.freezeFieldValues
+    super.freezeFieldValues()
 
     if (this.scatterGatherDirectory == null) {
       if (qSettings.jobScatterGatherDirectory != null) {
@@ -98,10 +99,14 @@ trait ScatterGatherableFunction extends CommandLineFunction {
    * The scatter function.
    */
   private lazy val scatterFunction = {
+    // Only depend on input fields that have a value
+    val inputFieldsWithValues = this.inputFields.filter(hasFieldValue(_))
+    val inputFiles = inputFieldsWithValues.flatMap(getFieldFiles(_)).toSet
+
     val scatterFunction = newScatterFunction()
     this.copySettingsTo(scatterFunction)
     scatterFunction.originalFunction = this
-    scatterFunction.originalInputs = this.inputs
+    scatterFunction.originalInputs = inputFiles
     scatterFunction.commandDirectory = this.scatterGatherTempDir("scatter")
     scatterFunction.isIntermediate = true
     scatterFunction.addOrder = this.addOrder :+ 1
@@ -121,8 +126,6 @@ trait ScatterGatherableFunction extends CommandLineFunction {
   def generateFunctions() = {
     var functions = List.empty[QFunction]
 
-    // Only depend on input fields that have a value
-    val inputFieldsWithValues = this.inputFields.filter(hasFieldValue(_))
     // Only gather up fields that will have a value
     val outputFieldsWithValues = this.outputFields.filter(hasFieldValue(_))
 
@@ -228,7 +231,7 @@ trait ScatterGatherableFunction extends CommandLineFunction {
    * Calls setupScatterFunction with scatterFunction.
    * @param scatterFunction The function that will create the scatter pieces in the temporary directories.
    */
-  protected def initScatterFunction(scatterFunction: ScatterFunction) = {
+  protected def initScatterFunction(scatterFunction: ScatterFunction) {
     if (this.setupScatterFunction != null)
       if (this.setupScatterFunction.isDefinedAt(scatterFunction))
         this.setupScatterFunction(scatterFunction)
@@ -272,7 +275,7 @@ trait ScatterGatherableFunction extends CommandLineFunction {
    * @param gatherFunction The function that will merge the gather pieces from the temporary directories.
    * @param gatherField The output field being gathered.
    */
-  protected def initGatherFunction(gatherFunction: GatherFunction, gatherField: ArgumentSource) = {
+  protected def initGatherFunction(gatherFunction: GatherFunction, gatherField: ArgumentSource) {
     if (this.setupGatherFunction != null)
       if (this.setupGatherFunction.isDefinedAt(gatherFunction, gatherField))
         this.setupGatherFunction(gatherFunction, gatherField)
@@ -289,7 +292,7 @@ trait ScatterGatherableFunction extends CommandLineFunction {
    * @param cloneFunction The clone of this ScatterGatherableFunction
    * @param index The one based index (from 1..scatterCount inclusive) of the scatter piece.
    */
-  protected def initCloneFunction(cloneFunction: CloneFunction, index: Int) = {
+  protected def initCloneFunction(cloneFunction: CloneFunction, index: Int) {
     if (this.setupCloneFunction != null)
       if (this.setupCloneFunction.isDefinedAt(cloneFunction, index))
         this.setupCloneFunction(cloneFunction, index)

public/scala/src/org/broadinstitute/sting/queue/util/CommandLineJob.scala

@@ -1,51 +0,0 @@
-package org.broadinstitute.sting.queue.util
-
-import java.io.File
-
-/**
- * Base utility class for a command line job.
- */
-abstract class CommandLineJob {
-  var shellScript: File = _
-  var workingDir: File = _
-  var inputFile: File = _
-  var outputFile: File = _
-  var errorFile: File = _
-
-  /**
-   * Runs the command, either immediately or dispatching it to a compute farm.
-   * If it is dispatched to a compute farm it should not start until jobs it depends on are finished.
-   */
-  def run()
-
-  /**
-   * Returns the content of a command output.
-   * @param streamOutput The output of the command.
-   * @return The content of the command, along with a message if it was truncated.
-   */
-  protected def content(streamOutput: ProcessController.StreamOutput) = {
-    var content = streamOutput.content
-    if (streamOutput.contentTruncated)
-      content += "%n%n<truncated>".format()
-    content
-  }
-
-  /**
-   * Returns the ProcessController for this thread.
-   * @return The ProcessController for this thread.
-   */
-  protected def processController = CommandLineJob.threadProcessController.get
-
-  /** A five mb limit of characters for display. */
-  protected val FIVE_MB = 1024 * 512 * 5;
-}
-
-/**
- * Base class for a command line job.
- */
-object CommandLineJob {
-  /** Thread local process controller container. */
-  private val threadProcessController = new ThreadLocal[ProcessController] {
-    override def initialValue = new ProcessController
-  }
-}

public/scala/src/org/broadinstitute/sting/queue/util/FileExtension.scala

@@ -1,15 +0,0 @@
-package org.broadinstitute.sting.queue.util
-
-import java.io.File
-
-/**
- * An trait for @Input or @Output CommandLineFunction fields that are extensions of files.
- */
-trait FileExtension extends File {
-  /**
-   * Returns a clone of the FileExtension with the new path.
-   * @param newPath new path for the clone of this FileExtension
-   * @return a clone of the FileExtension with the new path.
-   */
-  def withPath(newPath: String): File
-}

public/scala/src/org/broadinstitute/sting/queue/util/IOUtils.scala

@@ -1,253 +0,0 @@
-package org.broadinstitute.sting.queue.util
-
-import org.apache.commons.io.FileUtils
-import java.io.{FileReader, File}
-import org.broadinstitute.sting.utils.exceptions.UserException
-import org.broadinstitute.sting.queue.QException
-
-/**
- * A collection of utilities for modifying java.io.
- */
-object IOUtils extends Logging {
-  /**
-   * Checks if the temp directory has been setup and throws an exception if they user hasn't set it correctly.
-   * @param tempDir Temporary directory.
-   */
-  def checkTempDir(tempDir: File) {
-    val tempDirPath = tempDir.getAbsolutePath
-    // Keeps the user from leaving the temp directory as the default, and on Macs from having pluses
-    // in the path which can cause problems with the Google Reflections library.
-    // see also: http://benjchristensen.com/2009/09/22/mac-osx-10-6-java-java-io-tmpdir/
-    if (tempDirPath.startsWith("/var/folders/") || (tempDirPath == "/tmp") || (tempDirPath == "/tmp/"))
-      throw new UserException.BadTmpDir("java.io.tmpdir must be explicitly set")
-    if (!tempDir.exists && !tempDir.mkdirs)
-      throw new UserException.BadTmpDir("Could not create directory: " + tempDir.getAbsolutePath)
-  }
-
-  /**
-   * Creates a temp directory with the prefix and optional suffix.
-   * @param prefix Prefix for the directory name.
-   * @param suffix Optional suffix for the directory name.
-   * @param tempDirParent Parent directory for the temp directory.
-   * @return The created temporary directory.
-   */
-  def tempDir(prefix: String, suffix: String = "", tempDirParent: File) = {
-    if (!tempDirParent.exists && !tempDirParent.mkdirs)
-       throw new UserException.BadTmpDir("Could not create temp directory: " + tempDirParent)
-    val temp = File.createTempFile(prefix + "-", suffix, tempDirParent)
-    if (!temp.delete)
-      throw new UserException.BadTmpDir("Could not delete sub file: " + temp.getAbsolutePath)
-    if (!temp.mkdir)
-      throw new UserException.BadTmpDir("Could not create sub directory: " + temp.getAbsolutePath)
-    absolute(temp)
-  }
-
-  /**
-   * Writes content into a file.
-   * @param file File to write to.
-   * @param content Content to write.
-   */
-  def writeContents(file: File, content: String) { FileUtils.writeStringToFile(file, content) }
-
-  /**
-   * Reads content of a file into a string.
-   * Only for use on really small files!
-   * @param file File to read to.
-   * @return content Content of the file.
-   */
-  def readContents(file: File) = FileUtils.readFileToString(file)
-
-  /**
-   * Writes content to a temp file and returns the path to the temporary file.
-   * @param content to write.
-   * @param prefix Prefix for the temp file.
-   * @param suffix Suffix for the temp file.
-   * @param directory Directory for the temp file.
-   * @return the path to the temp file.
-   */
-  def writeTempFile(content: String, prefix: String, suffix: String, directory: File) = {
-    val tempFile = absolute(File.createTempFile(prefix, suffix, directory))
-    writeContents(tempFile, content)
-    tempFile
-  }
-
-  /**
-   * Waits for NFS to propagate a file creation, imposing a timeout.
-   *
-   * Based on Apache Commons IO FileUtils.waitFor()
-   *
-   * @param file The file to wait for.
-   * @param seconds The maximum time in seconds to wait.
-   * @return true if the file exists
-   */
-  def waitFor(file: File, seconds: Int): Boolean = waitFor(List(file), seconds).isEmpty
-
-  /**
-   * Waits for NFS to propagate a file creation, imposing a timeout.
-   *
-   * Based on Apache Commons IO FileUtils.waitFor()
-   *
-   * @param files The list of files to wait for.
-   * @param seconds The maximum time in seconds to wait.
-   * @return Files that still do not exists at the end of the timeout, or a empty list if all files exists.
-   */
-  def waitFor[T <: Traversable[File]](files: T, seconds: Int): Traversable[File] = {
-      var timeout = 0;
-      var tick = 0;
-      var missingFiles = files.filterNot(_.exists)
-      while (!missingFiles.isEmpty && timeout <= seconds) {
-        if (tick >= 10) {
-          tick = 0;
-          timeout += 1
-        }
-        tick += 1
-        try {
-            Thread.sleep(100)
-        } catch {
-          case ignore: InterruptedException =>
-        }
-        missingFiles = missingFiles.filterNot(_.exists)
-      }
-      missingFiles
-  }
-
-  /**
-   * Returns the directory at the number of levels deep.
-   * For example 2 levels of /path/to/dir will return /path/to
-   * @param dir Directory path.
-   * @param level how many levels deep from the root.
-   * @return The path to the parent directory that is level-levels deep.
-   */
-  def dirLevel(dir: File, level: Int): File = {
-    var directories = List.empty[File]
-    var parentDir = absolute(dir)
-    while (parentDir != null) {
-      directories +:= parentDir
-      parentDir = parentDir.getParentFile
-    }
-    if (directories.size <= level)
-      directories.last
-    else
-      directories(level)
-  }
-
-  /**
-   * Returns the sub path rooted at the parent.
-   * @param parent The parent directory.
-   * @param path The sub path to append to the parent, if the path is not absolute.
-   * @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path.
-   */
-  def absolute(parent: File, path: String): File =
-    absolute(parent, new File(path))
-
-  /**
-   * Returns the sub path rooted at the parent.
-   * @param parent The parent directory.
-   * @param file The sub path to append to the parent, if the path is not absolute.
-   * @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path.
-   */
-  def absolute(parent: File, file: File): File = {
-    val newPath =
-      if (file.isAbsolute)
-        absolutePath(file)
-      else
-        absolutePath(new File(parent, file.getPath))
-    replacePath(file, newPath)
-  }
-
-  /**
-   * A mix of getCanonicalFile and getAbsoluteFile that returns the
-   * absolute path to the file without deferencing symbolic links.
-   * @param file the file.
-   * @return the absolute path to the file.
-   */
-  def absolute(file: File) = {
-    replacePath(file, absolutePath(file))
-  }
-
-  private def absolutePath(file: File) = {
-    var fileAbs = file.getAbsoluteFile
-    var names = List.empty[String]
-    while (fileAbs != null) {
-      val name = fileAbs.getName
-      fileAbs = fileAbs.getParentFile
-
-      if (name == ".") {
-        /* skip */
-
-        /* TODO: What do we do for ".."?
-      } else if (name == "..") {
-
-        CentOS tcsh says use getCanonicalFile:
-        ~ $ mkdir -p test1/test2
-        ~ $ ln -s test1/test2 test3
-        ~ $ cd test3/..
-        ~/test1 $
-
-        Mac bash says keep going with getAbsoluteFile:
-        ~ $ mkdir -p test1/test2
-        ~ $ ln -s test1/test2 test3
-        ~ $ cd test3/..
-        ~ $
-
-        For now, leave it and let the shell figure it out.
-        */
-      } else {
-        names +:= name
-      }
-    }
-
-    names.mkString("/", "/", "")
-  }
-
-  private def replacePath(file: File, path: String) = {
-    file match {
-      case fileExtension: FileExtension =>
-        fileExtension.withPath(path)
-      case file: File =>
-        if (file.getClass != classOf[File])
-          throw new QException("Sub classes of java.io.File must also implement FileExtension so that the path can be modified.")
-        new File(path)
-    }
-  }
-
-  /**
-   * Returns the last lines of the file.
-   * NOTE: This is only safe to run on smaller files!
-   * @param file File to read.
-   * @param count Maximum number of lines to return.
-   * @return The last count lines from file.
-   */
-  def tail(file: File, count: Int) = {
-    var tailLines = List.empty[String]
-    var reader = new FileReader(file)
-    try {
-      val iterator = org.apache.commons.io.IOUtils.lineIterator(reader)
-      var lineCount = 0
-      while (iterator.hasNext) {
-        val line = iterator.nextLine
-        lineCount += 1
-        if (lineCount > count)
-          tailLines = tailLines.tail
-        tailLines :+= line
-      }
-    } finally {
-      org.apache.commons.io.IOUtils.closeQuietly(reader)
-    }
-    tailLines
-  }
-
-  /**
-   * Tries to delete a file.  Emits a warning if the file was unable to be deleted.
-   * @param file File to delete.
-   * @return true if the file was deleted.
-   */
-  def tryDelete(file: File) = {
-    val deleted = FileUtils.deleteQuietly(file)
-    if (deleted)
-      logger.debug("Deleted " + file)
-    else if (file.exists)
-      logger.warn("Unable to delete " + file)
-    deleted
-  }
-}

public/scala/src/org/broadinstitute/sting/queue/util/JobExitException.scala

@@ -1,11 +0,0 @@
-package org.broadinstitute.sting.queue.util
-
-import org.broadinstitute.sting.queue.QException
-
-/**
- * Captures the exit code and error text from a failed process.
- */
-class JobExitException(val exitText: String, val commandLine: Array[String], val exitCode: Int, val stdErr: String)
-        extends QException("%s%nCommand line:%n%s%nExit code: %s%nStandard error contained: %n%s"
-                .format(exitText, commandLine.mkString(" "), exitCode, stdErr)) {
-}

public/scala/src/org/broadinstitute/sting/queue/util/ProcessController.scala

@@ -1,369 +0,0 @@
-package org.broadinstitute.sting.queue.util
-
-import java.io._
-import scala.collection.mutable.{HashSet, ListMap}
-
-/**
- * Facade to Runtime.exec() and java.lang.Process.  Handles
- * running a process to completion and returns stdout and stderr
- * as strings.  Creates separate threads for reading stdout and stderr,
- * then reuses those threads for each process most efficient use is
- * to create one of these and use it repeatedly.  Instances are not
- * thread-safe, however.
- *
- * @author originally by Michael Koehrsen ported to scala and enhanced by Khalid Shakir
- */
-class ProcessController extends Logging {
-
-  // Threads that capture stdout and stderr
-  private val stdoutCapture = new OutputCapture(ProcessController.STDOUT_KEY)
-  private val stderrCapture = new OutputCapture(ProcessController.STDERR_KEY)
-
-  // Communication channels with output capture threads
-  /** Holds the stdout and stderr sent to the background capture threads */
-  private val toCapture = new ListMap[String, ProcessController.CapturedStreamOutput]
-
-  /** Holds the results of the capture from the background capture threads.
-   * May be the content via toCapture or an EmptyStreamOutput if the capture was interrupted. */
-  private val fromCapture = new ListMap[String, ProcessController.StreamOutput]
-
-  // Start the background threads for this controller.
-  stdoutCapture.start()
-  stderrCapture.start()
-
-  /**
-   * Executes a command line program with the settings and waits for it to return, processing the output on a background thread.
-   * @param settings Settings to be run.
-   * @return The output of the command.
-   */
-  def exec(settings: ProcessController.ProcessSettings): ProcessController.ProcessOutput = {
-    var builder = new ProcessBuilder(settings.cmdarray:_*)
-    builder.directory(settings.directory)
-
-    if (settings.environment != null) {
-      val builderEnvironment = builder.environment
-      builderEnvironment.clear()
-      settings.environment.foreach{case (name, value) => builderEnvironment.put(name, value)}
-    }
-
-    builder.redirectErrorStream(settings.redirectErrorStream)
-
-    var stdout: ProcessController.StreamOutput = null
-    var stderr: ProcessController.StreamOutput = null
-    val process = builder.start
-
-    ProcessController.running.add(process)
-    try {
-      val stdoutSettings = if (settings.stdoutSettings == null) ProcessController.EmptyStreamSettings else settings.stdoutSettings
-      val stderrSettings = if (settings.stderrSettings == null) ProcessController.EmptyStreamSettings else settings.stderrSettings
-
-      toCapture.synchronized {
-        toCapture.put(ProcessController.STDOUT_KEY, new ProcessController.CapturedStreamOutput(process.getInputStream, stdoutSettings, scala.Console.out))
-        toCapture.put(ProcessController.STDERR_KEY, new ProcessController.CapturedStreamOutput(process.getErrorStream, stderrSettings, scala.Console.err))
-        toCapture.notifyAll()
-      }
-
-      if (settings.stdinSettings.input != null) {
-        val writer = new OutputStreamWriter(process.getOutputStream)
-        writer.write(settings.stdinSettings.input)
-        writer.flush()
-      }
-      if (settings.stdinSettings.inputFile != null) {
-        val reader = new FileReader(settings.stdinSettings.inputFile)
-        val writer = new OutputStreamWriter(process.getOutputStream)
-        val buf = new Array[Char](4096)
-        var readCount = 0
-        while ({readCount = reader.read(buf); readCount} >= 0)
-          writer.write(buf, 0, readCount)
-        writer.flush()
-        reader.close()
-      }
-      
-      try {
-        process.getOutputStream.close()
-        process.waitFor()
-      } finally {
-        while (stdout == null || stderr == null) {
-          fromCapture.synchronized {
-            fromCapture.remove(ProcessController.STDOUT_KEY) match {
-              case Some(stream) => stdout = stream
-              case None => /* ignore */
-            }
-            fromCapture.remove(ProcessController.STDERR_KEY) match {
-              case Some(stream) => stderr = stream
-              case None => /* ignore */
-            }
-
-            try {
-              if (stdout == null || stderr == null)
-                fromCapture.wait()
-            } catch {
-              case e: InterruptedException =>
-                logger.error(e)
-            }
-          }
-        }
-      }
-    } finally {
-      ProcessController.running.remove(process)
-    }
-
-    new ProcessController.ProcessOutput(process.exitValue, stdout, stderr)
-  }
-
-  /** Ensures that the threads used to manipulate the IO for the process are cleaned up properly. */
-  def close() = {
-    try {
-      stdoutCapture.interrupt()
-      stderrCapture.interrupt()
-    } catch {
-      case e =>
-        logger.error(e)
-    }
-  }
-
-  /** calls close() */
-  override def finalize = close()
-
-  /**
-   * Reads in the output of a stream on a background thread to keep the output pipe from backing up and freezing the called process.
-   * @param key The stdout or stderr key for this output capture.
-   */
-  private class OutputCapture(private val key: String)
-          extends Thread("OutputCapture-" + key + "-" + Thread.currentThread.getName) {
-
-    setDaemon(true)
-
-    /** Runs the capture. */
-    override def run = {
-      var break = false
-      while (!break) {
-        var processStream: ProcessController.StreamOutput = ProcessController.EmptyStreamOutput
-        try {
-          // Wait for a new input stream to be passed from this process controller.
-          var capturedProcessStream: ProcessController.CapturedStreamOutput = null
-          while (capturedProcessStream == null) {
-            toCapture.synchronized {
-              toCapture.remove(key) match {
-                case Some(stream) => capturedProcessStream = stream
-                case None => toCapture.wait()
-              }
-            }
-          }
-          // Read in the input stream
-          processStream = capturedProcessStream
-          capturedProcessStream.read
-        } catch {
-          case e: InterruptedException => {
-            logger.info("OutputReader interrupted, exiting")
-            break = true
-          }
-          case e: IOException => {
-            logger.error("Error reading process output", e)
-          }
-        } finally {
-          // Send the string back to the process controller.
-          fromCapture.synchronized {
-            fromCapture.put(key, processStream)
-            fromCapture.notify()
-          }
-        }
-      }
-    }
-  }
-}
-
-/**
- * Facade to Runtime.exec() and java.lang.Process.  Handles
- * running a process to completion and returns stdout and stderr
- * as strings.  Creates separate threads for reading stdout and stderr,
- * then reuses those threads for each process most efficient use is
- * to create one of these and use it repeatedly.  Instances are not
- * thread-safe, however.
- *
- * @author originally by Michael Koehrsen ported to scala and enhanced by Khalid Shakir
- */
-object ProcessController extends Logging {
-
-  /**
-   * Settings that define how to run a process.
-   * @param cmdarray Command line to run.
-   * @param environment Environment settings to override System.getEnv, or null to use System.getEnv.
-   * @param directory The directory to run the command in, or null to run in the current directory.
-   * @param stdinSettings Settings for writing to the process stdin.
-   * @param stdoutSettings Settings for capturing the process stdout.
-   * @param stderrSettings Setting for capturing the process stderr.
-   * @param redirectErrorStream true if stderr should be sent to stdout.
-   */
-  class ProcessSettings(val cmdarray: Array[String], val environment: Map[String, String], val directory: File,
-                        val stdinSettings: InputStreamSettings, val stdoutSettings: OutputStreamSettings,
-                        val stderrSettings: OutputStreamSettings, val redirectErrorStream: Boolean)
-
-  /**
-   * Settings that define text to write to the process stdin.
-   * @param input String to write to stdin.
-   * @param inputFile File to write to stdin.
-   */
-  class InputStreamSettings(val input: String, val inputFile: File)
-
-  /**
-   * Settings that define text to capture from a process stream.
-   * @param stringSize The number of characters to capture, or -1 for unlimited.
-   * @param outputFile The file to write output to, or null to skip output.
-   * @param outputFileAppend true if the output file should be appended to.
-   */
-  class OutputStreamSettings(val stringSize: Int, val outputFile: File, val outputFileAppend: Boolean)
-
-  /**
-   * The output of a process.
-   * @param exitValue The exit value.
-   * @param stdout The capture of stdout as defined by the stdout OutputStreamSettings.
-   * @param stderr The capture of stderr as defined by the stderr OutputStreamSettings.
-   */
-  class ProcessOutput(val exitValue: Int, val stdout: StreamOutput, val stderr: StreamOutput)
-
-  /**
-   * The base class of stream output.
-   */
-  abstract class StreamOutput {
-    /**
-     * Returns the content as a string.
-     * @return The content as a string.
-     */
-    def content: String
-
-    /**
-     * Returns true if the content was truncated.
-     * @return true if the content was truncated.
-     */
-    def contentTruncated: Boolean
-  }
-
-  private var currentCaptureId = 0
-  /**
-   * Returns the next output capture id.
-   * @return The next output capture id.
-   */
-  private def NEXT_OUTPUT_CAPTURE_ID = {
-    currentCaptureId += 1
-    currentCaptureId
-  }
-  private val STDOUT_KEY = "stdout"
-  private val STDERR_KEY = "stderr"
-
-  /** Tracks running processes so that they can be killed as the JVM shuts down. */
-  private val running = new HashSet[Process]
-
-  def shutdown() = {
-    for (process <- running.clone) {
-      logger.warn("Killing: " + process)
-      try {
-        process.destroy
-      } catch {
-        case _ => /* ignore */
-      }
-    }
-  }
-
-  /** Empty stream settings used when no output is requested. */
-  private object EmptyStreamSettings extends OutputStreamSettings(0, null, false)
-
-  /** Empty stream output when no output is captured due to an error. */
-  private object EmptyStreamOutput extends StreamOutput {
-    def content = ""
-    def contentTruncated = false
-  }
-
-  /**
-   * Stream output captured from a stream.
-   * @param stream Stream to capture output.
-   * @param settings Settings that define what to capture.
-   */
-  private class CapturedStreamOutput(val stream: InputStream, val settings: OutputStreamSettings, val debugStream: PrintStream) extends StreamOutput {
-    /**
-     * Returns the captured content as a string.
-     * @return The captured content as a string.
-     */
-    def content = stringWriter.toString()
-
-    /**
-     * Returns true if the captured content was truncated.
-     * @return true if the captured content was truncated.
-     */
-    def contentTruncated = stringTruncated
-
-    /**
-     * Drain the input stream to keep the process from backing up until it's empty.
-     */
-    def read() = {
-      val reader = new InputStreamReader(stream)
-      val buf = new Array[Char](4096)
-      var readCount = 0
-      while ({readCount = reader.read(buf); readCount} >= 0) {
-        writeString(buf, readCount)
-        writeFile(buf, readCount)
-      }
-      closeFile()
-      stream.close()
-    }
-
-    /** The string to write capture content. */
-    private lazy val stringWriter = if (settings.stringSize < 0) new StringWriter else new StringWriter(settings.stringSize)
-
-    /** True if the content is truncated. */
-    private var stringTruncated = false
-
-    /** The number of characters left until the buffer is full. */
-    private var stringRemaining = settings.stringSize
-
-    /**
-     * Writes the buffer to the stringWriter up to stringRemaining characters.
-     * @param chars Character buffer to write.
-     * @param len Number of characters in the buffer.
-     */
-    private def writeString(chars: Array[Char], len: Int) = {
-      // If debug is enabled bypass the logger and dump directly to the screen
-      if (logger.isDebugEnabled)
-        debugStream.print(new String(chars, 0, len))
-      if (settings.stringSize < 0) {
-        stringWriter.write(chars, 0, len)
-      } else {
-        if (!stringTruncated) {
-          stringWriter.write(chars, 0, if (len > stringRemaining) stringRemaining else len)
-          stringRemaining -= len
-          if (stringRemaining < 0)
-            stringTruncated = true
-        }
-      }
-    }
-
-    /** The file writer to capture content or null if no output file was requested. */
-    private lazy val fileWriter = {
-      if (settings.outputFile == null) {
-        null
-      } else {
-        new FileWriter(settings.outputFile, settings.outputFileAppend)
-      }
-    }
-
-    /**
-     * Writes the buffer to the fileWriter if it is not null.
-     * @param chars Character buffer to write.
-     * @param len Number of characters in the buffer.
-     */
-    private def writeFile(chars: Array[Char], len: Int) = {
-      if (fileWriter != null) {
-        fileWriter.write(chars, 0, len)
-        fileWriter.flush()
-      }
-    }
-
-    /** Closes the fileWriter if it is not null. */
-    private def closeFile() = {
-      if (fileWriter != null) {
-        fileWriter.flush
-        fileWriter.close
-      }
-    }
-  }
-}

public/scala/src/org/broadinstitute/sting/queue/util/QJobReport.scala

@@ -23,15 +23,15 @@
  */
 
 package org.broadinstitute.sting.queue.util
+
 import org.broadinstitute.sting.queue.function.QFunction
 import org.broadinstitute.sting.gatk.report.{GATKReportTable, GATKReport}
 import org.broadinstitute.sting.utils.exceptions.UserException
 import org.broadinstitute.sting.queue.engine.JobRunInfo
 import java.io.{FileOutputStream, PrintStream, File}
-import org.broadinstitute.sting.queue.function.scattergather.{GathererFunction, ScatterFunction}
 import org.broadinstitute.sting.utils.R.RScriptExecutor.RScriptArgumentCollection
-import org.broadinstitute.sting.utils.R.RScriptExecutor
+import org.broadinstitute.sting.utils.R.{RScriptLibrary, RScriptExecutor}
-import org.broadinstitute.sting.queue.QScript
+import org.broadinstitute.sting.utils.io.Resource
 
 /**
  * A mixin to add Job info to the class
@@ -104,10 +104,12 @@ object QJobReport {
     stream.close()
   }
 
-  def plotReport(args: RScriptArgumentCollection, jobReportFile: File) {
+  def plotReport(args: RScriptArgumentCollection, reportFile: File, pdfFile: File) {
     val executor = new RScriptExecutor(args, false) // don't except on error
-    val pdf = jobReportFile.getAbsolutePath + ".pdf"
+    executor.addLibrary(RScriptLibrary.GSALIB)
-    executor.callRScripts(JOB_REPORT_QUEUE_SCRIPT, jobReportFile.getAbsolutePath, pdf)
+    executor.addScript(new Resource(JOB_REPORT_QUEUE_SCRIPT, classOf[QJobReport]))
+    executor.addArgs(reportFile.getAbsolutePath, pdfFile.getAbsolutePath)
+    executor.exec()
   }
 
   def workAroundSameJobNames(func: QFunction):String = {

public/scala/src/org/broadinstitute/sting/queue/util/ShellJob.scala

@@ -1,27 +0,0 @@
-package org.broadinstitute.sting.queue.util
-
-/**
- * Runs a job on the command line by invoking "sh -c <command>"
- */
-class ShellJob extends CommandLineJob with Logging {
-  /**
-   * Runs the command and waits for the output.
-   */
-  def run() = {
-    val (redirectError, errorFile) = if (this.errorFile == null) (true, null) else (false, this.errorFile)
-    val bufferSize = if (redirectError || logger.isDebugEnabled) FIVE_MB else 0
-    val stdinSettings = new ProcessController.InputStreamSettings(null, this.inputFile)
-    val stdoutSettings = new ProcessController.OutputStreamSettings(bufferSize, this.outputFile, true)
-    val stderrSettings = new ProcessController.OutputStreamSettings(FIVE_MB, errorFile, true)
-    val commandLine = Array("sh", shellScript.toString)
-    val processSettings = new ProcessController.ProcessSettings(
-      commandLine, null, this.workingDir, stdinSettings, stdoutSettings, stderrSettings, redirectError)
-
-    val output = processController.exec(processSettings)
-
-    if (output.exitValue != 0) {
-      val streamOutput = if (redirectError) output.stdout else output.stderr
-      throw new JobExitException("Failed to run job.", commandLine, output.exitValue, content(streamOutput))
-    }
-  }
-}

public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala

@@ -33,7 +33,7 @@ import java.text.SimpleDateFormat
 import org.broadinstitute.sting.BaseTest
 import org.broadinstitute.sting.MD5DB
 import org.broadinstitute.sting.queue.QCommandLine
-import org.broadinstitute.sting.queue.util.{Logging, ProcessController}
+import org.broadinstitute.sting.queue.util.Logging
 import java.io.File
 import org.broadinstitute.sting.gatk.report.GATKReport
 import org.apache.commons.io.FileUtils
@@ -217,11 +217,6 @@ object PipelineTest extends BaseTest with Logging {
   Runtime.getRuntime.addShutdownHook(new Thread {
     /** Cleanup as the JVM shuts down. */
     override def run() {
-      try {
-        ProcessController.shutdown()
-      } catch {
-        case _ => /*ignore */
-      }
       runningCommandLines.foreach(commandLine =>
         try {
           commandLine.shutdown()

public/scala/test/org/broadinstitute/sting/queue/util/IOUtilsUnitTest.scala

@@ -1,122 +0,0 @@
-package org.broadinstitute.sting.queue.util
-
-import org.broadinstitute.sting.BaseTest
-import java.io.File
-import org.broadinstitute.sting.utils.exceptions.UserException
-import org.testng.Assert
-import org.testng.annotations.Test
-
-class IOUtilsUnitTest extends BaseTest {
-  @Test
-  def testGoodTempDir = {
-    IOUtils.checkTempDir(new File("/tmp/queue"))
-  }
-
-  @Test(expectedExceptions=Array(classOf[UserException.BadTmpDir]))
-  def testBadTempDir = {
-    IOUtils.checkTempDir(new File("/tmp"))
-  }
-
-  @Test
-  def testAbsoluteSubDir = {
-    var subDir = IOUtils.absolute(new File("."), new File("/path/to/file"))
-    Assert.assertEquals(subDir, new File("/path/to/file"))
-
-    subDir = IOUtils.absolute(new File("/different/path"), new File("/path/to/file"))
-    Assert.assertEquals(subDir, new File("/path/to/file"))
-
-    subDir = IOUtils.absolute(new File("/different/path"), new File("."))
-    Assert.assertEquals(subDir, new File("/different/path"))
-  }
-
-  @Test
-  def testRelativeSubDir = {
-    var subDir = IOUtils.absolute(new File("."), new File("path/to/file"))
-    Assert.assertEquals(subDir.getCanonicalFile, new File("path/to/file").getCanonicalFile)
-
-    subDir = IOUtils.absolute(new File("/different/path"), new File("path/to/file"))
-    Assert.assertEquals(subDir, new File("/different/path/path/to/file"))
-  }
-
-  @Test
-  def testDottedSubDir = {
-    var subDir = IOUtils.absolute(new File("."), new File("path/../to/file"))
-    Assert.assertEquals(subDir.getCanonicalFile, new File("path/../to/./file").getCanonicalFile)
-
-    subDir = IOUtils.absolute(new File("."), new File("/path/../to/file"))
-    Assert.assertEquals(subDir, new File("/path/../to/file"))
-
-    subDir = IOUtils.absolute(new File("/different/../path"), new File("path/to/file"))
-    Assert.assertEquals(subDir, new File("/different/../path/path/to/file"))
-
-    subDir = IOUtils.absolute(new File("/different/./path"), new File("/path/../to/file"))
-    Assert.assertEquals(subDir, new File("/path/../to/file"))
-  }
-
-  @Test
-  def testTempDir = {
-    val tempDir = IOUtils.tempDir("Q-Unit-Test", "", new File("queueTempDirToDelete"))
-    Assert.assertTrue(tempDir.exists)
-    Assert.assertFalse(tempDir.isFile)
-    Assert.assertTrue(tempDir.isDirectory)
-    val deleted = IOUtils.tryDelete(tempDir)
-    Assert.assertTrue(deleted)
-    Assert.assertFalse(tempDir.exists)
-  }
-
-  @Test
-  def testDirLevel = {
-    var dir = IOUtils.dirLevel(new File("/path/to/directory"), 1)
-    Assert.assertEquals(dir, new File("/path"))
-
-    dir = IOUtils.dirLevel(new File("/path/to/directory"), 2)
-    Assert.assertEquals(dir, new File("/path/to"))
-
-    dir = IOUtils.dirLevel(new File("/path/to/directory"), 3)
-    Assert.assertEquals(dir, new File("/path/to/directory"))
-
-    dir = IOUtils.dirLevel(new File("/path/to/directory"), 4)
-    Assert.assertEquals(dir, new File("/path/to/directory"))
-  }
-
-  @Test
-  def testAbsolute = {
-    var dir = IOUtils.absolute(new File("/path/./to/./directory/."))
-    Assert.assertEquals(dir, new File("/path/to/directory"))
-
-    dir = IOUtils.absolute(new File("/"))
-    Assert.assertEquals(dir, new File("/"))
-
-    dir = IOUtils.absolute(new File("/."))
-    Assert.assertEquals(dir, new File("/"))
-
-    dir = IOUtils.absolute(new File("/././."))
-    Assert.assertEquals(dir, new File("/"))
-
-    dir = IOUtils.absolute(new File("/./directory/."))
-    Assert.assertEquals(dir, new File("/directory"))
-
-    dir = IOUtils.absolute(new File("/./directory/./"))
-    Assert.assertEquals(dir, new File("/directory"))
-
-    dir = IOUtils.absolute(new File("/./directory./"))
-    Assert.assertEquals(dir, new File("/directory."))
-
-    dir = IOUtils.absolute(new File("/./.directory/"))
-    Assert.assertEquals(dir, new File("/.directory"))
-  }
-
-  @Test
-  def testTail = {
-    val lines = List(
-      "chr18_random	4262	3154410390	50	51",
-      "chr19_random	301858	3154414752	50	51",
-      "chr21_random	1679693	3154722662	50	51",
-      "chr22_random	257318	3156435963	50	51",
-      "chrX_random	1719168	3156698441	50	51")
-    val tail = IOUtils.tail(new File(BaseTest.hg18Reference + ".fai"), 5)
-    Assert.assertEquals(tail.size, 5)
-    for (i <- 0 until 5)
-      Assert.assertEquals(tail(i), lines(i))
-  }
-}

public/scala/test/org/broadinstitute/sting/queue/util/ShellJobUnitTest.scala

@@ -1,73 +0,0 @@
-package org.broadinstitute.sting.queue.util
-
-import org.broadinstitute.sting.BaseTest
-import org.testng.annotations.Test
-import org.testng.Assert
-
-class ShellJobUnitTest {
-  @Test
-  def testEcho {
-    val job = new ShellJob
-    job.shellScript = writeScript("echo Hello World")
-    job.run()
-  }
-
-  @Test(expectedExceptions=Array(classOf[JobExitException]))
-  def testBadQuotes {
-    val job = new ShellJob
-    job.shellScript = writeScript("echo 'Hello World")
-    job.run()
-  }
-
-  @Test
-  def testGoodQuotes {
-    val job = new ShellJob
-    job.shellScript = writeScript("echo 'Hello World'")
-    job.run()
-  }
-
-  @Test
-  def testEscapeCharacters {
-    var job: ShellJob = null
-
-    job = new ShellJob
-    job.shellScript = writeScript("echo #")
-    job.outputFile = BaseTest.createTempFile("temp", "")
-    job.run()
-    Assert.assertEquals(IOUtils.readContents(job.outputFile).trim, "")
-
-    job = new ShellJob
-    job.shellScript = writeScript("""echo \#""")
-    job.outputFile = BaseTest.createTempFile("temp", "")
-    job.run()
-    Assert.assertEquals(IOUtils.readContents(job.outputFile).trim, "#")
-
-    job = new ShellJob
-    job.shellScript = writeScript("""echo \\#""")
-    job.outputFile = BaseTest.createTempFile("temp", "")
-    job.run()
-    Assert.assertEquals(IOUtils.readContents(job.outputFile).trim, """\#""")
-  }
-
-  @Test
-  def testLongCommand {
-    // This command fails on some systems with a 4096 character limit when run via the old sh -c "echo ...",
-    // but works on the same systems when run via sh <script>
-    val builder = new StringBuilder
-    builder.append("echo ")
-    for (i <- 1 to 500) {
-      val s = i.toString
-      builder.append("000".take(3-s.length)).append(s).append("______ ")
-    }
-
-    val job = new ShellJob
-    job.shellScript = writeScript(builder.toString)
-    job.run()
-  }
-
-  private def writeScript(contents: String) = {
-    val file = BaseTest.createTempFile("temp", "")
-    IOUtils.writeContents(file, contents)
-    file
-  }
-}