Merge branch 'master' of ssh://nickel.broadinstitute.org/humgen/gsa-scr1/gsa-engineering/git/unstable

.gitignore

@@ -0,0 +1,20 @@
+/*.bam
+/*.bai
+/*.bed
+*.idx
+*~
+/*.vcf
+/*.txt
+/*.csh
+/.*
+/*.pdf
+/*.eval
+*.ipr
+*.iws
+*.iml
+.DS_Store
+queueScatterGather
+/foo*
+/bar*
+integrationtests/
+public/testdata/onTheFlyOutputTest.vcf

build.xml

@@ -28,6 +28,8 @@
 
     <property name="build.dir" value="build" />
     <property name="dist.dir" value="dist" />
+    <property name="contract.dump.dir" value="dump" />
+    <property name="pipelinetest.dir" value="pipelinetests" />
     <property name="lib.dir" value="lib" />
     <property name="external.dir" value="external" />
     <property name="public.dir" value="public" />
@@ -35,18 +37,25 @@
     <property name="java.public.source.dir" value="${public.dir}/java/src" />
     <property name="java.private.source.dir" value="${private.dir}/java/src" />
     <property name="java.classes" value="${build.dir}/java/classes" />
+    <property name="R.public.scripts.dir" value="${public.dir}/R/scripts" />
+    <property name="R.public.src.dir" value="${public.dir}/R/src" />
+    <!-- Legacy: Installing libraries back into the source directory
+    instead of the build or dist directory... intentionally avoids ant clean?? -->
+    <property name="R.library.dir" value="${public.dir}/R" />
+    <property name="R.tar.dir" value="${build.dir}/R/src" />
+    <property name="R.package.path" value="org/broadinstitute/sting/utils/R" />
     <property name="resource.file" value="StingText.properties" />
     <property name="resource.path" value="${java.classes}/StingText.properties" />
 
     <property name="scala.public.source.dir" value="${public.dir}/scala/src" />
-    <property name="scala.private.source.dir" value="${private.dir}/scala/src" />    
+    <property name="scala.private.source.dir" value="${private.dir}/scala/src" />
     <property name="scala.classes" value="${build.dir}/scala/classes" />
 
     <property name="queue-extensions.source.dir" value="${build.dir}/queue-extensions/src" />
 
     <property name="javadoc.dir" value="javadoc" />
     <property name="scaladoc.dir" value="scaladoc" />
-    
+
     <!-- Contracts for Java -->
     <!-- By default, enabled only for test targets -->
     <!-- To disable for test targets, run with -Duse.contracts=false -->
@@ -60,7 +69,7 @@
 
     <!-- do we want to halt on failure of a unit test? default to yes (Bamboo uses 'no') -->
     <property name="halt" value="yes" />
-    
+
     <!-- should our unit test output go to a file or the screen?
     false means it goes to the screen (default) true to file -->
     <property name="usefile" value="false" />
@@ -82,7 +91,7 @@
         <patternset refid="java.source.pattern" />
     </fileset>
 
-    <!-- terrible hack to get gatkdocs to see all files --> 
+    <!-- terrible hack to get gatkdocs to see all files -->
     <patternset id="all.java.source.pattern">
         <include name="${java.public.source.dir}/**/*.java" />
         <include name="${java.private.source.dir}/**/*.java" />
@@ -113,7 +122,7 @@
       <exclude name="testng*.jar" />
       <exclude name="bcel*.jar" />
     </patternset>
-    
+
     <path id="external.dependencies">
       <fileset dir="${lib.dir}">
         <patternset refid="dependency.mask" />
@@ -154,7 +163,7 @@
         <property name="ivy.jar.file" value="ivy-${ivy.install.version}.jar"/>
         <property name="ivy.settings.dir" value="settings"/>
         <property file="${ivy.settings.dir}/ivysettings.properties"/>
-  
+
       <mkdir dir="${lib.dir}"/>
       <mkdir dir="${ivy.jar.dir}"/>
 
@@ -211,11 +220,11 @@
         <equals arg1="${git.describe.exit.value}" arg2="0" />
       </condition>
     </target>
-    
+
     <target name="tagged.build.version" depends="git.describe" if="git.describe.succeeded">
         <property name="build.version" value="${git.describe.output}" />
     </target>
-    
+
     <target name="git.rev-parse" depends="git.describe" unless="git.describe.succeeded">
       <exec executable="git" outputproperty="git.rev-parse.output" resultproperty="git.rev-parse.exit.value" failonerror="false">
         <arg line="rev-parse HEAD" />
@@ -224,11 +233,11 @@
         <equals arg1="${git.rev-parse.exit.value}" arg2="0" />
       </condition>
     </target>
-    
+
     <target name="untagged.build.version" depends="git.rev-parse" if="git.rev-parse.succeeded">
         <property name="build.version" value="${git.rev-parse.output}" />
     </target>
-    
+
     <target name="generate.build.version" depends="tagged.build.version, untagged.build.version">
         <!-- Set build.version to exported if no other value has been set -->
         <property name="build.version" value="exported" />
@@ -266,7 +275,7 @@
         <echo message="Scala build     : ${scala.target}"/>
       <echo message="source revision : ${build.version}"/>
       <echo message="build time      : ${build.timestamp}" />
-      
+
       <condition property="include.private">
         <equals arg1="${gatk.target}" arg2="private" casesensitive="false" />
       </condition>
@@ -312,13 +321,13 @@
     <target name="gatk.compile.public.source" depends="init,resolve">
         <javac fork="true" srcdir="${java.public.source.dir}" memoryMaximumSize="512m" destdir="${java.classes}" debug="true" debuglevel="lines,vars,source" classpathref="external.dependencies" tempdir="${java.io.tmpdir}">
           <compilerarg value="-proc:none"/>
-        </javac>    
+        </javac>
     </target>
 
     <target name="gatk.compile.private.source" depends="gatk.compile.public.source" if="include.private">
         <javac fork="true" srcdir="${java.private.source.dir}" memoryMaximumSize="512m" destdir="${java.classes}" debug="true" debuglevel="lines,vars,source" classpathref="external.dependencies" tempdir="${java.io.tmpdir}">
           <compilerarg value="-proc:none"/>
-        </javac>    
+        </javac>
     </target>
 
     <target name="gatk.compile.external.source" depends="gatk.compile.public.source,gatk.compile.private.source">
@@ -327,11 +336,11 @@
             <property name="dist.dir" value="${external.dist.dir}" />
             <property name="gatk.classpath" value="${external.gatk.classpath}" />
             <fileset dir="${external.dir}" includes="*/build.xml" erroronmissingdir="false" />
-        </subant>    
+        </subant>
     </target>
 
-    <target name="gatk.compile.source" 
+    <target name="gatk.compile.source"
-            depends="gatk.compile.public.source,gatk.compile.private.source,gatk.compile.external.source" 
+            depends="gatk.compile.public.source,gatk.compile.private.source,gatk.compile.external.source"
             description="compile the GATK source" />
 
    <target name="gatk.contracts.public" depends="gatk.compile.source" if="include.contracts">
@@ -341,9 +350,9 @@
                 <pathelement path="${java.classes}" />
             </classpath>
             <compilerarg value="-Acom.google.java.contract.debug"/>
-            <compilerarg value="-Acom.google.java.contract.dump=dump/"/>
+            <compilerarg value="-Acom.google.java.contract.dump=${contract.dump.dir}"/>
             <compilerarg value="-proc:only"/>
-        </javac>   
+        </javac>
    </target>
 
     <target name="check.contracts.private" depends="gatk.contracts.public">
@@ -362,14 +371,14 @@
                 <pathelement path="${java.classes}" />
             </classpath>
             <compilerarg value="-Acom.google.java.contract.debug"/>
-            <compilerarg value="-Acom.google.java.contract.dump=dump/"/>
+            <compilerarg value="-Acom.google.java.contract.dump=${contract.dump.dir}"/>
             <compilerarg value="-proc:only"/>
-        </javac>   
+        </javac>
    </target>
 
-    <target name="gatk.contracts" depends="gatk.contracts.public,gatk.contracts.private" 
+    <target name="gatk.contracts" depends="gatk.contracts.public,gatk.contracts.private"
             description="create GATK contracts" if="include.contracts" />
-    
+
     <target name="gatk.compile" depends="init,resolve,gatk.compile.source,gatk.contracts" />
 
     <target name="init.queue-extensions.generate" depends="gatk.compile">
@@ -413,9 +422,9 @@
             <src path="${scala.public.source.dir}" />
             <src path="${queue-extensions.source.dir}" />
             <include name="**/*.scala"/>
-        </scalac>    
+        </scalac>
     </target>
-    
+
     <target name="check.scala.private" depends="scala.compile.public">
         <condition property="include.scala.private">
           <and>
@@ -424,12 +433,12 @@
           </and>
         </condition>
     </target>
-    
+
     <target name="scala.compile.private" depends="check.scala.private" if="include.scala.private">
         <scalac fork="true" jvmargs="-Xmx512m" destdir="${scala.classes}" classpathref="scala.dependencies" deprecation="yes" unchecked="yes">
             <src path="${scala.private.source.dir}" />
             <include name="**/*.scala"/>
-        </scalac>       
+        </scalac>
     </target>
 
     <target name="scala.compile" depends="scala.compile.public,scala.compile.private" if="scala.include" description="compile Scala" />
@@ -532,6 +541,11 @@
 
     <target name="sting.compile" depends="gatk.compile, scala.compile" />
 
+    <target name="R.public.tar">
+        <mkdir dir="${R.tar.dir}/${R.package.path}" />
+        <tar compression="gzip" basedir="${R.public.src.dir}/${R.package.path}" includes="gsalib/**" destfile="${R.tar.dir}/${R.package.path}/gsalib.tar.gz" />
+    </target>
+
     <target name="init.jar" depends="sting.compile,extracthelp">
         <mkdir dir="${dist.dir}"/>
         <copy todir="${dist.dir}">
@@ -539,7 +553,7 @@
         </copy>
     </target>
 
-    <target name="sting-utils.jar" depends="gatk.compile, init.jar">
+    <target name="sting-utils.jar" depends="gatk.compile, init.jar, R.public.tar">
         <jar jarfile="${dist.dir}/StingUtils.jar">
             <fileset dir="${java.classes}">
               <include name="**/utils/**/*.class"/>
@@ -551,6 +565,12 @@
             <fileset dir="${java.classes}" includes="**/sting/jna/**/*.class"/>
             <fileset dir="${java.classes}" includes="net/sf/picard/**/*.class"/>
             <fileset dir="${java.classes}" includes="net/sf/samtools/**/*.class"/>
+            <fileset dir="${R.tar.dir}">
+                <include name="**/${R.package.path}/**/*.tar.gz"/>
+            </fileset>
+            <fileset dir="${R.public.scripts.dir}">
+              <include name="**/utils/**/*.R"/>
+            </fileset>
             <manifest>
                 <attribute name="Premain-Class" value="org.broadinstitute.sting.utils.instrumentation.Sizeof" />
             </manifest>
@@ -579,6 +599,10 @@
               <include name="**/gatk/**/*.class" />
               <include name="**/alignment/**/*.class"/>
             </fileset>
+            <fileset dir="${R.public.scripts.dir}">
+                <include name="**/gatk/**/*.R"/>
+                <include name="**/alignment/**/*.R"/>
+            </fileset>
             <manifest>
                 <attribute name="Main-Class" value="org.broadinstitute.sting.gatk.CommandLineGATK" />
             </manifest>
@@ -593,6 +617,10 @@
             <include name="**/analyzecovariates/**/*.class" />
             <include name="**/gatk/walkers/recalibration/*.class" />
           </fileset>
+            <fileset dir="${R.public.scripts.dir}">
+                <include name="**/analyzecovariates/**/*.R"/>
+                <include name="**/gatk/walkers/recalibration/**/*.R"/>
+            </fileset>
           <manifest>
             <attribute name="Main-Class" value="org.broadinstitute.sting.analyzecovariates.AnalyzeCovariates" />
           </manifest>
@@ -605,28 +633,7 @@
             <fileset dir="${external.dir}" includes="*/build.xml" erroronmissingdir="false" />
         </subant>
     </target>
-<!--
-    <target name="gatk.oneoffs.jar" depends="gatk.compile, init.jar"
-            description="generate the GATK oneoffs distribution" if="include.oneoffs">
-        <jar jarfile="${dist.dir}/CompareBAMAlignments.jar" whenmanifestonly="skip">
-          <fileset dir="${java.classes}">
-            <include name="**/tools/**/*.class" />
-          </fileset>
-          <manifest>
-            <attribute name="Main-Class" value="org.broadinstitute.sting.oneoffprojects.tools.CompareBAMAlignments" />
-          </manifest>
-        </jar>
 
-        <jar jarfile="${dist.dir}/SliceBams.jar" whenmanifestonly="skip">
-          <fileset dir="${java.classes}">
-            <include name="**/tools/**/*.class" />
-          </fileset>
-          <manifest>
-            <attribute name="Main-Class" value="org.broadinstitute.sting.playground.tools.SliceBams" />
-          </manifest>
-        </jar>
-    </target>
--->
     <target name="scala.jar" depends="scala.compile, init.jar" if="scala.include">
         <jar jarfile="${dist.dir}/GATKScala.jar">
           <fileset dir="${scala.classes}">
@@ -643,6 +650,9 @@
             <fileset dir="${java.classes}">
               <include name="org/broadinstitute/sting/queue/**/*.class" />
             </fileset>
+            <fileset dir="${R.public.scripts.dir}">
+                <include name="org/broadinstitute/sting/queue/**/*.R"/>
+            </fileset>
             <manifest>
                 <attribute name="Main-Class" value="org.broadinstitute.sting.queue.QCommandLine" />
             </manifest>
@@ -682,20 +692,7 @@
         </jar>
 
     </target>
-<!--
+
-    <target name="gatk.oneoffs.manifests" depends="gatk.oneoffs.jar, init.manifests" if="include.oneoffs">
-        <jar jarfile="${dist.dir}/CompareBAMAlignments.jar" update="true" whenmanifestonly="skip">
-            <manifest>
-                <attribute name="Class-Path" value="${jar.classpath}" />
-            </manifest>
-        </jar>
-        <jar jarfile="${dist.dir}/SliceBams.jar" update="true" whenmanifestonly="skip">
-            <manifest>
-                <attribute name="Class-Path" value="${jar.classpath}" />
-            </manifest>
-        </jar>
-    </target>
--->
     <target name="queue.manifests" depends="queue.jar, init.manifests" if="scala.include">
         <jar jarfile="${dist.dir}/Queue.jar" update="true" >
             <manifest>
@@ -780,10 +777,6 @@
                 <pathelement location="${testng.jar}"/>
             </classpath>
             <compilerarg value="-proc:none"/>
-<!--
-            <compilerarg value="-Acom.google.java.contract.debug"/>
-            <compilerarg value="-Acom.google.java.contract.dump=dump/"/>
- -->
         </javac>
     </target>
 
@@ -800,10 +793,6 @@
                 <pathelement location="${testng.jar}"/>
             </classpath>
             <compilerarg value="-proc:none"/>
-<!--
-            <compilerarg value="-Acom.google.java.contract.debug"/>
-            <compilerarg value="-Acom.google.java.contract.dump=dump/"/>
- -->
         </javac>
     </target>
 
@@ -851,6 +840,8 @@
         <pathelement location="${java.private.test.classes}" />
         <pathelement location="${scala.public.test.classes}" />
         <pathelement location="${scala.private.test.classes}" />
+        <pathelement location="${R.tar.dir}" />
+        <pathelement location="${R.public.scripts.dir}" />
     </path>
 
     <path id="testng.gatk.releasetest.classpath">
@@ -1187,19 +1178,18 @@
     </target>
 
     <target name="clean" description="clean up" depends="clean.javadoc,clean.scaladoc,clean.gatkdocs">
-        <delete dir="out"/>
         <delete dir="${build.dir}"/>
         <delete dir="${lib.dir}"/>
-        <delete dir="dump"/>
+        <delete dir="${contract.dump.dir}"/>
         <delete dir="${staging.dir}"/>
         <delete dir="${dist.dir}"/>
-        <delete dir="pipelinetests"/>
+        <delete dir="${pipelinetest.dir}"/>
     </target>
 
     <!-- Build gsalib R module -->
     <target name="gsalib">
       <exec executable="R" failonerror="true">
-        <arg line="R CMD INSTALL -l public/R/ public/R/src/gsalib/" />
+        <arg line="R CMD INSTALL -l ${R.library.dir} ${R.public.src.dir}/${R.package.path}/gsalib" />
       </exec>
     </target>
 </project>

public/java/src/org/broadinstitute/sting/commandline/ArgumentMatch.java

@@ -46,7 +46,7 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
     /**
      * Maps indices of command line arguments to values paired with that argument.
      */
-    public final SortedMap<Integer,List<String>> indices = new TreeMap<Integer,List<String>>();
+    public final SortedMap<ArgumentMatchSite,List<String>> sites = new TreeMap<ArgumentMatchSite,List<String>>();
 
     /**
      * An ordered, freeform collection of tags.
@@ -72,32 +72,32 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
     }
 
     /**
-     * A simple way of indicating that an argument with the given label and definition exists at this index.
+     * A simple way of indicating that an argument with the given label and definition exists at this site.
      * @param label Label of the argument match.  Must not be null.
      * @param definition The associated definition, if one exists.  May be null.
-     * @param index Position of the argument.  Must not be null.
+     * @param site Position of the argument.  Must not be null.
      * @param tags ordered freeform text tags associated with this argument.
      */
-    public ArgumentMatch(final String label, final ArgumentDefinition definition, final int index, final Tags tags) {
+    public ArgumentMatch(final String label, final ArgumentDefinition definition, final ArgumentMatchSite site, final Tags tags) {
-        this( label, definition, index, null, tags );
+        this( label, definition, site, null, tags );
     }
 
     /**
-     * A simple way of indicating that an argument with the given label and definition exists at this index.
+     * A simple way of indicating that an argument with the given label and definition exists at this site.
      * @param label Label of the argument match.  Must not be null.
      * @param definition The associated definition, if one exists.  May be null.
-     * @param index Position of the argument.  Must not be null.
+     * @param site Position of the argument.  Must not be null.
      * @param value Value for the argument at this position.
      * @param tags ordered freeform text tags associated with this argument.
      */
-    private ArgumentMatch(final String label, final ArgumentDefinition definition, final int index, final String value, final Tags tags) {
+    private ArgumentMatch(final String label, final ArgumentDefinition definition, final ArgumentMatchSite site, final String value, final Tags tags) {
         this.label = label;
         this.definition = definition;
 
         ArrayList<String> values = new ArrayList<String>();
         if( value != null )
             values.add(value);
-        indices.put(index,values );
+        sites.put(site,values );
 
         this.tags = tags;
     }
@@ -117,7 +117,7 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
         ArgumentMatch otherArgumentMatch = (ArgumentMatch)other;
         return this.definition.equals(otherArgumentMatch.definition) &&
                 this.label.equals(otherArgumentMatch.label) &&
-                this.indices.equals(otherArgumentMatch.indices) &&
+                this.sites.equals(otherArgumentMatch.sites) &&
                 this.tags.equals(otherArgumentMatch.tags);
     }
 
@@ -129,16 +129,17 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
      * @param key Key which specifies the transform.
      * @return A variant of this ArgumentMatch with all keys transformed.
      */
+    @SuppressWarnings("unchecked")
     ArgumentMatch transform(Multiplexer multiplexer, Object key) {
-        SortedMap<Integer,List<String>> newIndices = new TreeMap<Integer,List<String>>();
+        SortedMap<ArgumentMatchSite,List<String>> newIndices = new TreeMap<ArgumentMatchSite,List<String>>();
-        for(Map.Entry<Integer,List<String>> index: indices.entrySet()) {
+        for(Map.Entry<ArgumentMatchSite,List<String>> site: sites.entrySet()) {
             List<String> newEntries = new ArrayList<String>();
-            for(String entry: index.getValue())
+            for(String entry: site.getValue())
                 newEntries.add(multiplexer.transformArgument(key,entry));
-            newIndices.put(index.getKey(),newEntries);
+            newIndices.put(site.getKey(),newEntries);
         }
         ArgumentMatch newArgumentMatch = new ArgumentMatch(label,definition);
-        newArgumentMatch.indices.putAll(newIndices);
+        newArgumentMatch.sites.putAll(newIndices);
         return newArgumentMatch;
     }
 
@@ -157,9 +158,9 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
     public Iterator<ArgumentMatch> iterator() {
         return new Iterator<ArgumentMatch>() {
             /**
-             * Iterate over each the available index.
+             * Iterate over each the available site.
              */
-            private Iterator<Integer> indexIterator = null;
+            private Iterator<ArgumentMatchSite> siteIterator = null;
 
             /**
              * Iterate over each available token.
@@ -167,9 +168,9 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
             private Iterator<String> tokenIterator = null;
 
             /**
-             * The next index to return.  Null if none remain.
+             * The next site to return.  Null if none remain.
              */
-            Integer nextIndex = null;
+            ArgumentMatchSite nextSite = null;
 
             /**
              * The next token to return.  Null if none remain.
@@ -177,7 +178,7 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
             String nextToken = null;
 
             {
-                indexIterator = indices.keySet().iterator();
+                siteIterator = sites.keySet().iterator();
                 prepareNext();
             }
 
@@ -186,7 +187,7 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
              * @return True if there's another token waiting in the wings.  False otherwise.
              */
             public boolean hasNext() {
-                return nextToken != null;    
+                return nextToken != null;
             }
 
             /**
@@ -194,32 +195,32 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
              * @return The next ArgumentMatch in the series.  Should never be null.
              */
             public ArgumentMatch next() {
-                if( nextIndex == null || nextToken == null )
+                if( nextSite == null || nextToken == null )
                     throw new IllegalStateException( "No more ArgumentMatches are available" );
 
-                ArgumentMatch match = new ArgumentMatch( label, definition, nextIndex, nextToken, tags );
+                ArgumentMatch match = new ArgumentMatch( label, definition, nextSite, nextToken, tags );
                 prepareNext();
                 return match;
             }
 
             /**
              * Initialize the next ArgumentMatch to return.  If no ArgumentMatches are available,
-             * initialize nextIndex / nextToken to null.
+             * initialize nextSite / nextToken to null.
              */
             private void prepareNext() {
                 if( tokenIterator != null && tokenIterator.hasNext() ) {
                     nextToken = tokenIterator.next();
                 }
                 else {
-                    nextIndex = null;
+                    nextSite = null;
                     nextToken = null;
 
                     // Do a nested loop.  While more data is present in the inner loop, grab that data.
                     // Otherwise, troll the outer iterator looking for more data.
-                    while( indexIterator.hasNext() ) {
+                    while( siteIterator.hasNext() ) {
-                        nextIndex = indexIterator.next();
+                        nextSite = siteIterator.next();
-                        if( indices.get(nextIndex) != null ) {
+                        if( sites.get(nextSite) != null ) {
-                            tokenIterator = indices.get(nextIndex).iterator();
+                            tokenIterator = sites.get(nextSite).iterator();
                             if( tokenIterator.hasNext() ) {
                                 nextToken = tokenIterator.next();
                                 break;
@@ -245,29 +246,29 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
      * @param other The other match to merge into.
      */
     public void mergeInto( ArgumentMatch other ) {
-        indices.putAll(other.indices);
+        sites.putAll(other.sites);
     }
 
     /**
      * Associate a value with this merge maapping.
-     * @param index index of the command-line argument to which this value is mated.
+     * @param site site of the command-line argument to which this value is mated.
      * @param value Text representation of value to add.
      */
-    public void addValue( int index, String value ) {
+    public void addValue( ArgumentMatchSite site, String value ) {
-        if( !indices.containsKey(index) || indices.get(index) == null )
+        if( !sites.containsKey(site) || sites.get(site) == null )
-            indices.put(index, new ArrayList<String>() );
+            sites.put(site, new ArrayList<String>() );
-        indices.get(index).add(value);
+        sites.get(site).add(value);
     }
 
     /**
      * Does this argument already have a value at the given site?
      * Arguments are only allowed to be single-valued per site, and
      * flags aren't allowed a value at all.
-     * @param index Index at which to check for values.
+     * @param site Site at which to check for values.
      * @return True if the argument has a value at the given site.  False otherwise.
      */
-    public boolean hasValueAtSite( int index ) {
+    public boolean hasValueAtSite( ArgumentMatchSite site ) {
-        return (indices.get(index) != null && indices.get(index).size() >= 1) || isArgumentFlag();
+        return (sites.get(site) != null && sites.get(site).size() >= 1) || isArgumentFlag();
     }
 
     /**
@@ -276,9 +277,9 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
      */
     public List<String> values() {
         List<String> values = new ArrayList<String>();
-        for( int index: indices.keySet() ) {
+        for( ArgumentMatchSite site: sites.keySet() ) {
-            if( indices.get(index) != null )
+            if( sites.get(site) != null )
-                values.addAll(indices.get(index));
+                values.addAll(sites.get(site));
         }
         return values;
     }

public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchSite.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.commandline;
+
+/**
+ * Which source and the index within the source where an argument match was found.
+ */
+public class ArgumentMatchSite implements Comparable<ArgumentMatchSite> {
+    private final ArgumentMatchSource source;
+    private final int index;
+
+    public ArgumentMatchSite(ArgumentMatchSource source, int index) {
+        this.source = source;
+        this.index = index;
+    }
+
+    public ArgumentMatchSource getSource() {
+        return source;
+    }
+
+    public int getIndex() {
+        return index;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        ArgumentMatchSite that = (ArgumentMatchSite) o;
+
+        return (index == that.index) && (source == null ? that.source == null : source.equals(that.source));
+    }
+
+    @Override
+    public int hashCode() {
+        int result = source != null ? source.hashCode() : 0;
+        // Generated by intellij. No other special reason to this implementation. -ks
+        result = 31 * result + index;
+        return result;
+    }
+
+    @Override
+    public int compareTo(ArgumentMatchSite that) {
+        int comp = this.source.compareTo(that.source);
+        if (comp != 0)
+            return comp;
+
+        // Both files are the same.
+        if (this.index == that.index)
+            return 0;
+        return this.index < that.index ? -1 : 1;
+    }
+}

public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchSource.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.commandline;
+
+import java.io.File;
+
+/**
+ * Where an argument match originated, via the commandline or a file.
+ */
+public class ArgumentMatchSource implements Comparable<ArgumentMatchSource> {
+    public static final ArgumentMatchSource COMMAND_LINE = new ArgumentMatchSource(ArgumentMatchSourceType.CommandLine, null);
+
+    private final ArgumentMatchSourceType type;
+    private final File file;
+
+    /**
+     * Creates an argument match source from the specified file.
+     * @param file File specifying the arguments. Must not be null.
+     */
+    public ArgumentMatchSource(File file) {
+        this(ArgumentMatchSourceType.File, file);
+    }
+
+    private ArgumentMatchSource(ArgumentMatchSourceType type, File file) {
+        if (type == ArgumentMatchSourceType.File && file == null)
+            throw new IllegalArgumentException("An argument match source of type File cannot have a null file.");
+        this.type = type;
+        this.file = file;
+    }
+
+    public ArgumentMatchSourceType getType() {
+        return type;
+    }
+
+    public File getFile() {
+        return file;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        ArgumentMatchSource that = (ArgumentMatchSource) o;
+
+        return (type == that.type) && (file == null ? that.file == null : file.equals(that.file));
+    }
+
+    @Override
+    public int hashCode() {
+        int result = type != null ? type.hashCode() : 0;
+        result = 31 * result + (file != null ? file.hashCode() : 0);
+        return result;
+    }
+
+    /**
+     * Compares two sources, putting the command line first, then files.
+     */
+    @Override
+    public int compareTo(ArgumentMatchSource that) {
+        int comp = this.type.compareTo(that.type);
+        if (comp != 0)
+            return comp;
+
+        File f1 = this.file;
+        File f2 = that.file;
+
+        if ((f1 == null) ^ (f2 == null)) {
+            // If one of the files is null and the other is not
+            // put the null file first
+            return f1 == null ? -1 : 1;
+        }
+
+        return f1 == null ? 0 : f1.compareTo(f2);
+    }
+}

public/java/src/org/broadinstitute/sting/commandline/ArgumentMatchSourceType.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.commandline;
+
+/**
+ * Type of where an argument match originated, via the commandline or a file.
+ */
+public enum ArgumentMatchSourceType {
+    CommandLine, File
+}

public/java/src/org/broadinstitute/sting/commandline/ArgumentMatches.java

@@ -37,7 +37,7 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
      * Collection matches from argument definition to argument value.
      * Package protected access is deliberate.
      */
-    Map<Integer,ArgumentMatch> argumentMatches = new TreeMap<Integer,ArgumentMatch>();
+    Map<ArgumentMatchSite,ArgumentMatch> argumentMatches = new TreeMap<ArgumentMatchSite,ArgumentMatch>();
 
     /**
      * Provide a place to put command-line argument values that don't seem to belong to
@@ -80,7 +80,7 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
      * @param site Site at which to check.
      * @return True if the site has a match.  False otherwise.
      */
-    boolean hasMatch( int site ) {
+    boolean hasMatch( ArgumentMatchSite site ) {
         return argumentMatches.containsKey( site );
     }
 
@@ -90,7 +90,7 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
      * @return The match present at the given site.
      * @throws IllegalArgumentException if site does not contain a match.
      */
-    ArgumentMatch getMatch( int site ) {
+    ArgumentMatch getMatch( ArgumentMatchSite site ) {
         if( !argumentMatches.containsKey(site) )
             throw new IllegalArgumentException( "Site does not contain an argument: " + site );
         return argumentMatches.get(site);
@@ -107,6 +107,7 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
 
     /**
      * Return all argument matches of this source.
+     * @param parsingEngine Parsing engine.
      * @param argumentSource Argument source to match.
      * @return List of all matches.
      */
@@ -167,6 +168,7 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
      * TODO: Generify this.
      * @param multiplexer Multiplexer that controls the transformation process.
      * @param key Key which specifies the transform.
+     * @return new argument matches.
      */
     ArgumentMatches transform(Multiplexer multiplexer, Object key) {
         ArgumentMatches newArgumentMatches = new ArgumentMatches();
@@ -187,15 +189,15 @@ public class ArgumentMatches implements Iterable<ArgumentMatch> {
         for( ArgumentMatch argumentMatch: getUniqueMatches() ) {
             if( argumentMatch.definition == match.definition && argumentMatch.tags.equals(match.tags) ) {
                 argumentMatch.mergeInto( match );
-                for( int index: match.indices.keySet() )
+                for( ArgumentMatchSite site: match.sites.keySet() )
-                    argumentMatches.put( index, argumentMatch );
+                    argumentMatches.put( site, argumentMatch );
                 definitionExists = true;
             }
         }
 
         if( !definitionExists ) {
-            for( int index: match.indices.keySet() )
+            for( ArgumentMatchSite site: match.sites.keySet() )
-                argumentMatches.put( index, match );
+                argumentMatches.put( site, match );
         }
     }    
 

public/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java

@@ -35,10 +35,7 @@ import org.broadinstitute.sting.utils.help.ApplicationDetails;
 import org.broadinstitute.sting.utils.help.HelpFormatter;
 
 import java.io.IOException;
-import java.util.Collection;
+import java.util.*;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.Locale;
 
 public abstract class CommandLineProgram {
 
@@ -155,6 +152,7 @@ public abstract class CommandLineProgram {
      *
      * @param clp  the command line program to execute
      * @param args the command line arguments passed in
+     * @param dryRun dry run
      * @throws Exception when an exception occurs
      */
     @SuppressWarnings("unchecked")
@@ -176,6 +174,8 @@ public abstract class CommandLineProgram {
             ParsingEngine parser = clp.parser = new ParsingEngine(clp);
             parser.addArgumentSource(clp.getClass());
 
+            Map<ArgumentMatchSource, List<String>> parsedArgs;
+
             // process the args
             if (clp.canAddArgumentsDynamically()) {
                 // if the command-line program can toss in extra args, fetch them and reparse the arguments.
@@ -196,14 +196,14 @@ public abstract class CommandLineProgram {
                 Class[] argumentSources = clp.getArgumentSources();
                 for (Class argumentSource : argumentSources)
                     parser.addArgumentSource(clp.getArgumentSourceName(argumentSource), argumentSource);
-                parser.parse(args);
+                parsedArgs = parser.parse(args);
 
                 if (isHelpPresent(parser))
                     printHelpAndExit(clp, parser);
 
                 if ( ! dryRun ) parser.validate();
             } else {
-                parser.parse(args);
+                parsedArgs = parser.parse(args);
 
                 if ( ! dryRun ) {
                     if (isHelpPresent(parser))
@@ -230,7 +230,7 @@ public abstract class CommandLineProgram {
                 }
 
                 // regardless of what happens next, generate the header information
-                HelpFormatter.generateHeaderInformation(clp.getApplicationDetails(), args);
+                HelpFormatter.generateHeaderInformation(clp.getApplicationDetails(), parsedArgs);
 
                 // call the execute
                 CommandLineProgram.result = clp.execute();

public/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java

@@ -26,6 +26,7 @@
 package org.broadinstitute.sting.commandline;
 
 import com.google.java.contract.Requires;
+import org.apache.commons.io.FileUtils;
 import org.apache.log4j.Logger;
 import org.broadinstitute.sting.utils.Utils;
 import org.broadinstitute.sting.utils.classloader.JVMUtils;
@@ -35,6 +36,8 @@ import org.broadinstitute.sting.utils.exceptions.UserException;
 import org.broadinstitute.sting.utils.help.ApplicationDetails;
 import org.broadinstitute.sting.utils.help.HelpFormatter;
 
+import java.io.File;
+import java.io.IOException;
 import java.lang.reflect.Field;
 import java.util.*;
 
@@ -101,6 +104,8 @@ public class ParsingEngine {
         if(clp != null)
             argumentTypeDescriptors.addAll(clp.getArgumentTypeDescriptors());
         argumentTypeDescriptors.addAll(STANDARD_ARGUMENT_TYPE_DESCRIPTORS);
+
+        addArgumentSource(ParsingEngineArgumentFiles.class);
     }
 
     /**
@@ -149,21 +154,43 @@ public class ParsingEngine {
      * command-line arguments to the arguments that are actually
      * required.
      * @param tokens Tokens passed on the command line.
+     * @return The parsed arguments by file.
      */
-    public void parse( String[] tokens ) {
+    public SortedMap<ArgumentMatchSource, List<String>> parse( String[] tokens ) {
         argumentMatches = new ArgumentMatches();
+        SortedMap<ArgumentMatchSource, List<String>> parsedArgs = new TreeMap<ArgumentMatchSource, List<String>>();
 
-        int lastArgumentMatchSite = -1;
+        List<String> cmdLineTokens = Arrays.asList(tokens);
+        parse(ArgumentMatchSource.COMMAND_LINE, cmdLineTokens, argumentMatches, parsedArgs);
 
-        for( int i = 0; i < tokens.length; i++ ) {
+        ParsingEngineArgumentFiles argumentFiles = new ParsingEngineArgumentFiles();
-            String token = tokens[i];
+
+        // Load the arguments ONLY into the argument files.
+        // Validation may optionally run on the rest of the arguments.
+        loadArgumentsIntoObject(argumentFiles);
+
+        for (File file: argumentFiles.files) {
+            List<String> fileTokens = getArguments(file);
+            parse(new ArgumentMatchSource(file), fileTokens, argumentMatches, parsedArgs);
+        }
+
+        return parsedArgs;
+    }
+
+    private void parse(ArgumentMatchSource matchSource, List<String> tokens,
+                       ArgumentMatches argumentMatches, SortedMap<ArgumentMatchSource, List<String>> parsedArgs) {
+        ArgumentMatchSite lastArgumentMatchSite = new ArgumentMatchSite(matchSource, -1);
+
+        int i = 0;
+        for (String token: tokens) {
             // If the token is of argument form, parse it into its own argument match.
             // Otherwise, pair it with the most recently used argument discovered.
+            ArgumentMatchSite site = new ArgumentMatchSite(matchSource, i);
             if( isArgumentForm(token) ) {
-                ArgumentMatch argumentMatch = parseArgument( token, i );
+                ArgumentMatch argumentMatch = parseArgument( token, site );
                 if( argumentMatch != null ) {
                     argumentMatches.mergeInto( argumentMatch );
-                    lastArgumentMatchSite = i;
+                    lastArgumentMatchSite = site;
                 }
             }
             else {
@@ -171,10 +198,31 @@ public class ParsingEngine {
                     !argumentMatches.getMatch(lastArgumentMatchSite).hasValueAtSite(lastArgumentMatchSite))
                     argumentMatches.getMatch(lastArgumentMatchSite).addValue( lastArgumentMatchSite, token );
                 else
-                    argumentMatches.MissingArgument.addValue( i, token );
+                    argumentMatches.MissingArgument.addValue( site, token );
 
             }
+            i++;
         }
+
+        parsedArgs.put(matchSource, tokens);
+    }
+
+    private List<String> getArguments(File file) {
+        try {
+            if (file.getAbsolutePath().endsWith(".list")) {
+                return getListArguments(file);
+            }
+        } catch (IOException e) {
+            throw new UserException.CouldNotReadInputFile(file, e);
+        }
+        throw new UserException.CouldNotReadInputFile(file, "file extension is not .list");
+    }
+
+    private List<String> getListArguments(File file) throws IOException {
+        ArrayList<String> argsList = new ArrayList<String>();
+        for (String line: FileUtils.readLines(file))
+            argsList.addAll(Arrays.asList(Utils.escapeExpressions(line)));
+        return argsList;
     }
 
     public enum ValidationType { MissingRequiredArgument,
@@ -495,7 +543,7 @@ public class ParsingEngine {
      * @param position The position of the token in question.
      * @return ArgumentMatch associated with this token, or null if no match exists.
      */    
-    private ArgumentMatch parseArgument( String token, int position ) {
+    private ArgumentMatch parseArgument( String token, ArgumentMatchSite position ) {
         if( !isArgumentForm(token) )
             throw new IllegalArgumentException( "Token is not recognizable as an argument: " + token );
 
@@ -580,9 +628,21 @@ class UnmatchedArgumentException extends ArgumentException {
 
     private static String formatArguments( ArgumentMatch invalidValues ) {
         StringBuilder sb = new StringBuilder();
-        for( int index: invalidValues.indices.keySet() )
+        for( ArgumentMatchSite site: invalidValues.sites.keySet() )
-            for( String value: invalidValues.indices.get(index) ) {
+            for( String value: invalidValues.sites.get(site) ) {
-                sb.append( String.format("%nInvalid argument value '%s' at position %d.", value, index) );
+                switch (site.getSource().getType()) {
+                    case CommandLine:
+                        sb.append( String.format("%nInvalid argument value '%s' at position %d.",
+                                value, site.getIndex()) );
+                        break;
+                    case File:
+                        sb.append( String.format("%nInvalid argument value '%s' in file %s at position %d.",
+                                value, site.getSource().getFile().getAbsolutePath(), site.getIndex()) );
+                        break;
+                    default:
+                        throw new RuntimeException( String.format("Unexpected argument match source type: %s",
+                                site.getSource().getType()));
+                }
                 if(value != null && Utils.dupString(' ',value.length()).equals(value))
                     sb.append("  Please make sure any line continuation backslashes on your command line are not followed by whitespace.");
             }
@@ -635,4 +695,13 @@ class UnknownEnumeratedValueException extends ArgumentException {
     private static String formatArguments(ArgumentDefinition definition, String argumentPassed) {
         return String.format("Invalid value %s specified for argument %s; valid options are (%s).", argumentPassed, definition.fullName, Utils.join(",",definition.validOptions));
     }
-}
+}
+
+/**
+ * Container class to store the list of argument files.
+ * The files will be parsed after the command line arguments.
+ */
+class ParsingEngineArgumentFiles {
+    @Argument(fullName = "arg_file", shortName = "args", doc = "Reads arguments from the specified file", required = false)
+    public List<File> files = new ArrayList<File>();
+}

public/java/src/org/broadinstitute/sting/commandline/ParsingMethod.java

@@ -68,7 +68,7 @@ public abstract class ParsingMethod {
      * @return An argument match.  Definition field will be populated if a match was found or
      *         empty if no appropriate definition could be found. 
      */
-    public ArgumentMatch match( ArgumentDefinitions definitions, String token, int position ) {
+    public ArgumentMatch match( ArgumentDefinitions definitions, String token, ArgumentMatchSite position ) {
         // If the argument is valid, parse out the argument.
         Matcher matcher = pattern.matcher(token);
 
@@ -102,9 +102,7 @@ public abstract class ParsingMethod {
 
         // Try to find a matching argument.  If found, label that as the match.  If not found, add the argument
         // with a null definition.
-        ArgumentMatch argumentMatch = new ArgumentMatch(argument,argumentDefinition,position,tags);
+        return new ArgumentMatch(argument,argumentDefinition,position,tags);
-
-        return argumentMatch;
     }
 
     /**

public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java

@@ -221,12 +221,12 @@ public class GenomeAnalysisEngine {
         ShardStrategy shardStrategy = getShardStrategy(readsDataSource,microScheduler.getReference(),intervals);
 
         // execute the microscheduler, storing the results
-        Object result =  microScheduler.execute(this.walker, shardStrategy);
+        return microScheduler.execute(this.walker, shardStrategy);
 
         //monitor.stop();
         //logger.info(String.format("Maximum heap size consumed: %d",monitor.getMaxMemoryUsed()));
 
-        return result;
+        //return result;
     }
 
     /**
@@ -301,6 +301,10 @@ public class GenomeAnalysisEngine {
         return method;
     }
 
+    protected void setDownsamplingMethod(DownsamplingMethod method) {
+        argCollection.setDownsamplingMethod(method);
+    }
+
     public BAQ.QualityMode getWalkerBAQQualityMode()         { return WalkerManager.getBAQQualityMode(walker); }
     public BAQ.ApplicationTime getWalkerBAQApplicationTime() { return WalkerManager.getBAQApplicationTime(walker); }    
 
@@ -390,7 +394,9 @@ public class GenomeAnalysisEngine {
     /**
      * Get the sharding strategy given a driving data source.
      *
+     * @param readsDataSource readsDataSource
      * @param drivingDataSource Data on which to shard.
+     * @param intervals intervals
      * @return the sharding strategy
      */
     protected ShardStrategy getShardStrategy(SAMDataSource readsDataSource, ReferenceSequenceFile drivingDataSource, GenomeLocSortedSet intervals) {
@@ -427,7 +433,7 @@ public class GenomeAnalysisEngine {
             return new MonolithicShardStrategy(getGenomeLocParser(), readsDataSource,shardType,region);
         }
 
-        ShardStrategy shardStrategy = null;
+        ShardStrategy shardStrategy;
         ShardStrategyFactory.SHATTER_STRATEGY shardType;
 
         long SHARD_SIZE = 100000L;
@@ -436,6 +442,8 @@ public class GenomeAnalysisEngine {
             if (walker instanceof RodWalker) SHARD_SIZE *= 1000;
 
             if (intervals != null && !intervals.isEmpty()) {
+                if (readsDataSource == null)
+                    throw new IllegalArgumentException("readsDataSource is null");
                 if(!readsDataSource.isEmpty() && readsDataSource.getSortOrder() != SAMFileHeader.SortOrder.coordinate)
                     throw new UserException.MissortedBAM(SAMFileHeader.SortOrder.coordinate, "Locus walkers can only traverse coordinate-sorted data.  Please resort your input BAM file(s) or set the Sort Order tag in the header appropriately.");
 
@@ -499,7 +507,8 @@ public class GenomeAnalysisEngine {
      */
     private void initializeTempDirectory() {
         File tempDir = new File(System.getProperty("java.io.tmpdir"));
-        tempDir.mkdirs();
+        if (!tempDir.exists() && !tempDir.mkdirs())
+            throw new UserException.BadTmpDir("Unable to create directory");
     }
 
     /**
@@ -707,6 +716,7 @@ public class GenomeAnalysisEngine {
      * @param reads     Reads data source.
      * @param reference Reference data source.
      * @param rods    a collection of the reference ordered data tracks
+     * @param manager manager
      */
     private void validateSourcesAgainstReference(SAMDataSource reads, ReferenceSequenceFile reference, Collection<ReferenceOrderedDataSource> rods, RMDTrackBuilder manager) {
         if ((reads.isEmpty() && (rods == null || rods.isEmpty())) || reference == null )
@@ -735,15 +745,22 @@ public class GenomeAnalysisEngine {
     /**
      * Gets a data source for the given set of reads.
      *
+     * @param argCollection arguments
+     * @param genomeLocParser parser
+     * @param refReader reader
      * @return A data source for the given set of reads.
      */
     private SAMDataSource createReadsDataSource(GATKArgumentCollection argCollection, GenomeLocParser genomeLocParser, IndexedFastaSequenceFile refReader) {
         DownsamplingMethod method = getDownsamplingMethod();
 
+        // Synchronize the method back into the collection so that it shows up when
+        // interrogating for the downsample method during command line recreation.
+        setDownsamplingMethod(method);
+
         if ( getWalkerBAQApplicationTime() == BAQ.ApplicationTime.FORBIDDEN && argCollection.BAQMode != BAQ.CalculationMode.OFF)
             throw new UserException.BadArgumentValue("baq", "Walker cannot accept BAQ'd base qualities, and yet BAQ mode " + argCollection.BAQMode + " was requested.");
 
-        SAMDataSource dataSource = new SAMDataSource(
+        return new SAMDataSource(
                 samReaderIDs,
                 genomeLocParser,
                 argCollection.useOriginalBaseQualities,
@@ -759,14 +776,12 @@ public class GenomeAnalysisEngine {
                 refReader,
                 argCollection.defaultBaseQualities,
                 !argCollection.disableLowMemorySharding);
-        return dataSource;
     }
 
     /**
      * Opens a reference sequence file paired with an index.  Only public for testing purposes
      *
      * @param refFile Handle to a reference sequence file.  Non-null.
-     * @return A thread-safe file wrapper.
      */
     public void setReferenceDataSource(File refFile) {
         this.referenceDataSource = new ReferenceDataSource(refFile);

public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java

@@ -138,8 +138,8 @@ public class GATKArgumentCollection {
 
     /**
      * Gets the downsampling method explicitly specified by the user.  If the user didn't specify
-     * a default downsampling mechanism, return null.
+     * a default downsampling mechanism, return the default.
-     * @return The explicitly specified downsampling mechanism, or null if none exists.
+     * @return The explicitly specified downsampling mechanism, or the default if none exists.
      */
     public DownsamplingMethod getDownsamplingMethod() {
         if(downsamplingType == null && downsampleFraction == null && downsampleCoverage == null)
@@ -149,6 +149,18 @@ public class GATKArgumentCollection {
         return new DownsamplingMethod(downsamplingType,downsampleCoverage,downsampleFraction);
     }
 
+    /**
+     * Set the downsampling method stored in the argument collection so that it is read back out when interrogating the command line arguments.
+     * @param method The downsampling mechanism.
+     */
+    public void setDownsamplingMethod(DownsamplingMethod method) {
+        if (method == null)
+            throw new IllegalArgumentException("method is null");
+        downsamplingType = method.type;
+        downsampleCoverage = method.toCoverage;
+        downsampleFraction = method.toFraction;
+    }
+
     // --------------------------------------------------------------------------------------------------------------
     //
     // BAQ arguments

public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java

@@ -43,6 +43,7 @@ import org.broadinstitute.sting.utils.baq.BAQ;
 import org.broadinstitute.sting.utils.baq.BAQSamIterator;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.broadinstitute.sting.utils.sam.GATKSamRecordFactory;
 
 import java.io.File;
 import java.lang.reflect.InvocationTargetException;
@@ -57,6 +58,8 @@ import java.util.*;
  * Converts shards to SAM iterators over the specified region
  */
 public class SAMDataSource {
+    final private static GATKSamRecordFactory factory = new GATKSamRecordFactory();
+
     /** Backing support for reads. */
     protected final ReadProperties readProperties;
 
@@ -644,7 +647,9 @@ public class SAMDataSource {
                                                         BAQ.QualityMode qmode,
                                                         IndexedFastaSequenceFile refReader,
                                                         byte defaultBaseQualities) {
-        wrappedIterator = new ReadFormattingIterator(wrappedIterator, useOriginalBaseQualities, defaultBaseQualities);
+        if ( useOriginalBaseQualities || defaultBaseQualities >= 0 )
+            // only wrap if we are replacing the original qualitiies or using a default base quality
+            wrappedIterator = new ReadFormattingIterator(wrappedIterator, useOriginalBaseQualities, defaultBaseQualities);
 
         // NOTE: this (and other filtering) should be done before on-the-fly sorting
         //  as there is no reason to sort something that we will end of throwing away
@@ -756,6 +761,7 @@ public class SAMDataSource {
         public SAMReaders(Collection<SAMReaderID> readerIDs, SAMFileReader.ValidationStringency validationStringency) {
             for(SAMReaderID readerID: readerIDs) {
                 SAMFileReader reader = new SAMFileReader(readerID.samFile);
+                reader.setSAMRecordFactory(factory);
                 reader.enableFileSource(true);
                 reader.enableIndexMemoryMapping(false);
                 if(!enableLowMemorySharding)

public/java/src/org/broadinstitute/sting/gatk/iterators/ReadFormattingIterator.java

@@ -2,7 +2,6 @@ package org.broadinstitute.sting.gatk.iterators;
 
 import net.sf.samtools.SAMRecord;
 import org.apache.log4j.Logger;
-import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 
 /**
  * An iterator which does post-processing of a read, including potentially wrapping
@@ -78,7 +77,30 @@ public class ReadFormattingIterator implements StingSAMIterator {
      *         no next exists.
      */
     public SAMRecord next() {
-        return new GATKSAMRecord(wrappedIterator.next(), useOriginalBaseQualities, defaultBaseQualities);
+        SAMRecord rec = wrappedIterator.next();
+
+        // if we are using default quals, check if we need them, and add if necessary.
+        // 1. we need if reads are lacking or have incomplete quality scores
+        // 2. we add if defaultBaseQualities has a positive value
+        if (defaultBaseQualities >= 0) {
+            byte reads [] = rec.getReadBases();
+            byte quals [] = rec.getBaseQualities();
+            if (quals == null || quals.length < reads.length) {
+                byte new_quals [] = new byte [reads.length];
+                for (int i=0; i<reads.length; i++)
+                    new_quals[i] = defaultBaseQualities;
+                rec.setBaseQualities(new_quals);
+            }
+        }
+
+        // if we are using original quals, set them now if they are present in the record
+        if ( useOriginalBaseQualities ) {
+            byte[] originalQuals = rec.getOriginalBaseQualities();
+            if ( originalQuals != null )
+                rec.setBaseQualities(originalQuals);
+        }
+
+        return rec;
     }
 
     /**

public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/AlleleFrequencyCalculationModel.java

@@ -68,16 +68,12 @@ public abstract class AlleleFrequencyCalculationModel implements Cloneable {
 
     /**
      * Must be overridden by concrete subclasses
-     * @param tracker                         rod data
-     * @param ref                             reference context
      * @param GLs                             genotype likelihoods
      * @param Alleles                       Alleles corresponding to GLs
      * @param log10AlleleFrequencyPriors      priors
      * @param log10AlleleFrequencyPosteriors  array (pre-allocated) to store results
      */
-    protected abstract void getLog10PNonRef(RefMetaDataTracker tracker,
+    protected abstract void getLog10PNonRef(Map<String, Genotype> GLs,  List<Allele> Alleles,
-                                            ReferenceContext ref,
-                                            Map<String, Genotype> GLs,  List<Allele> Alleles,
                                             double[] log10AlleleFrequencyPriors,
                                             double[] log10AlleleFrequencyPosteriors);
 

public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java

@@ -51,9 +51,7 @@ public class ExactAFCalculationModel extends AlleleFrequencyCalculationModel {
         super(UAC, N, logger, verboseWriter);
     }
 
-    public void getLog10PNonRef(RefMetaDataTracker tracker,
+    public void getLog10PNonRef(Map<String, Genotype> GLs, List<Allele> alleles,
-                                ReferenceContext ref,
-                                Map<String, Genotype> GLs, List<Allele> alleles,
                                 double[] log10AlleleFrequencyPriors,
                                 double[] log10AlleleFrequencyPosteriors) {
         final int numAlleles = alleles.size();

public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GridSearchAFEstimation.java

@@ -52,9 +52,7 @@ public class GridSearchAFEstimation extends AlleleFrequencyCalculationModel {
         AFMatrix = new AlleleFrequencyMatrix(N);
     }
 
-    protected void getLog10PNonRef(RefMetaDataTracker tracker,
+    protected void getLog10PNonRef(Map<String, Genotype> GLs, List<Allele> alleles,
-                                   ReferenceContext ref,
-                                   Map<String, Genotype> GLs, List<Allele> alleles,
                                    double[] log10AlleleFrequencyPriors,
                                    double[] log10AlleleFrequencyPosteriors) {
         initializeAFMatrix(GLs);

public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java

@@ -30,12 +30,10 @@ import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
 import org.broadinstitute.sting.gatk.contexts.AlignmentContextUtils;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
-import org.broadinstitute.sting.gatk.walkers.indels.HaplotypeIndelErrorModel;
 import org.broadinstitute.sting.gatk.walkers.indels.PairHMMIndelErrorModel;
 import org.broadinstitute.sting.utils.BaseUtils;
 import org.broadinstitute.sting.utils.GenomeLoc;
 import org.broadinstitute.sting.utils.Haplotype;
-import org.broadinstitute.sting.utils.collections.Pair;
 import org.broadinstitute.sting.utils.exceptions.StingException;
 import org.broadinstitute.sting.utils.pileup.ExtendedEventPileupElement;
 import org.broadinstitute.sting.utils.pileup.PileupElement;

public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/MultiallelicGenotypeLikelihoods.java

@@ -4,6 +4,7 @@ import org.broadinstitute.sting.utils.exceptions.StingException;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
 import java.util.ArrayList;
+import java.util.List;
 
 /**
  * Created by IntelliJ IDEA.
@@ -15,11 +16,11 @@ import java.util.ArrayList;
 public class MultiallelicGenotypeLikelihoods {
     private String sample;
     private double[] GLs;
-    private ArrayList<Allele> alleleList;
+    private List<Allele> alleleList;
     private int depth;
 
     public MultiallelicGenotypeLikelihoods(String sample,
-                                         ArrayList<Allele> A,
+                                         List<Allele> A,
                                          double[] log10Likelihoods, int depth) {
         /* Check for consistency between likelihood vector and number of alleles */
         int numAlleles = A.size();
@@ -40,7 +41,7 @@ public class MultiallelicGenotypeLikelihoods {
          return GLs;
      }
 
-     public ArrayList<Allele> getAlleles() {
+     public List<Allele> getAlleles() {
          return alleleList;
      }
 

public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java

@@ -325,7 +325,7 @@ public class UnifiedGenotyperEngine {
 
         // 'zero' out the AFs (so that we don't have to worry if not all samples have reads at this position)
         clearAFarray(log10AlleleFrequencyPosteriors.get());
-        afcm.get().getLog10PNonRef(tracker, refContext, vc.getGenotypes(), vc.getAlleles(), getAlleleFrequencyPriors(model), log10AlleleFrequencyPosteriors.get());
+        afcm.get().getLog10PNonRef(vc.getGenotypes(), vc.getAlleles(), getAlleleFrequencyPriors(model), log10AlleleFrequencyPosteriors.get());
 
         // find the most likely frequency
         int bestAFguess = MathUtils.maxElementIndex(log10AlleleFrequencyPosteriors.get());
@@ -383,7 +383,7 @@ public class UnifiedGenotyperEngine {
             // the overall lod
             VariantContext vcOverall = calculateLikelihoods(tracker, refContext, stratifiedContexts, AlignmentContextUtils.ReadOrientation.COMPLETE, vc.getAlternateAllele(0), false, model);
             clearAFarray(log10AlleleFrequencyPosteriors.get());
-            afcm.get().getLog10PNonRef(tracker, refContext, vcOverall.getGenotypes(), vc.getAlleles(), getAlleleFrequencyPriors(model), log10AlleleFrequencyPosteriors.get());
+            afcm.get().getLog10PNonRef(vcOverall.getGenotypes(), vc.getAlleles(), getAlleleFrequencyPriors(model), log10AlleleFrequencyPosteriors.get());
             //double overallLog10PofNull = log10AlleleFrequencyPosteriors.get()[0];
             double overallLog10PofF = MathUtils.log10sumLog10(log10AlleleFrequencyPosteriors.get(), 1);
             //if ( DEBUG_SLOD ) System.out.println("overallLog10PofF=" + overallLog10PofF);
@@ -391,7 +391,7 @@ public class UnifiedGenotyperEngine {
             // the forward lod
             VariantContext vcForward = calculateLikelihoods(tracker, refContext, stratifiedContexts, AlignmentContextUtils.ReadOrientation.FORWARD, vc.getAlternateAllele(0), false, model);
             clearAFarray(log10AlleleFrequencyPosteriors.get());
-            afcm.get().getLog10PNonRef(tracker, refContext, vcForward.getGenotypes(), vc.getAlleles(), getAlleleFrequencyPriors(model), log10AlleleFrequencyPosteriors.get());
+            afcm.get().getLog10PNonRef(vcForward.getGenotypes(), vc.getAlleles(), getAlleleFrequencyPriors(model), log10AlleleFrequencyPosteriors.get());
             //double[] normalizedLog10Posteriors = MathUtils.normalizeFromLog10(log10AlleleFrequencyPosteriors.get(), true);
             double forwardLog10PofNull = log10AlleleFrequencyPosteriors.get()[0];
             double forwardLog10PofF = MathUtils.log10sumLog10(log10AlleleFrequencyPosteriors.get(), 1);
@@ -400,7 +400,7 @@ public class UnifiedGenotyperEngine {
             // the reverse lod
             VariantContext vcReverse = calculateLikelihoods(tracker, refContext, stratifiedContexts, AlignmentContextUtils.ReadOrientation.REVERSE, vc.getAlternateAllele(0), false, model);
             clearAFarray(log10AlleleFrequencyPosteriors.get());
-            afcm.get().getLog10PNonRef(tracker, refContext, vcReverse.getGenotypes(), vc.getAlleles(), getAlleleFrequencyPriors(model), log10AlleleFrequencyPosteriors.get());
+            afcm.get().getLog10PNonRef(vcReverse.getGenotypes(), vc.getAlleles(), getAlleleFrequencyPriors(model), log10AlleleFrequencyPosteriors.get());
             //normalizedLog10Posteriors = MathUtils.normalizeFromLog10(log10AlleleFrequencyPosteriors.get(), true);
             double reverseLog10PofNull = log10AlleleFrequencyPosteriors.get()[0];
             double reverseLog10PofF = MathUtils.log10sumLog10(log10AlleleFrequencyPosteriors.get(), 1);
@@ -447,6 +447,78 @@ public class UnifiedGenotyperEngine {
         return new VariantCallContext(vcCall, confidentlyCalled(phredScaledConfidence, PofF));
     }
 
+    // A barebones entry point to the exact model when there is no tracker or stratified contexts available -- only GLs
+    public VariantCallContext calculateGenotypes(final VariantContext vc, final GenomeLoc loc, final GenotypeLikelihoodsCalculationModel.Model model) {
+
+        // initialize the data for this thread if that hasn't been done yet
+        if ( afcm.get() == null ) {
+            log10AlleleFrequencyPosteriors.set(new double[N+1]);
+            afcm.set(getAlleleFrequencyCalculationObject(N, logger, verboseWriter, UAC));
+        }
+
+        // estimate our confidence in a reference call and return
+        if ( vc.getNSamples() == 0 )
+            return null;
+
+        // 'zero' out the AFs (so that we don't have to worry if not all samples have reads at this position)
+        clearAFarray(log10AlleleFrequencyPosteriors.get());
+        afcm.get().getLog10PNonRef(vc.getGenotypes(), vc.getAlleles(), getAlleleFrequencyPriors(model), log10AlleleFrequencyPosteriors.get());
+
+        // find the most likely frequency
+        int bestAFguess = MathUtils.maxElementIndex(log10AlleleFrequencyPosteriors.get());
+
+        // calculate p(f>0)
+        double[] normalizedPosteriors = MathUtils.normalizeFromLog10(log10AlleleFrequencyPosteriors.get());
+        double sum = 0.0;
+        for (int i = 1; i <= N; i++)
+            sum += normalizedPosteriors[i];
+        double PofF = Math.min(sum, 1.0); // deal with precision errors
+
+        double phredScaledConfidence;
+        if ( bestAFguess != 0 || UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES ) {
+            phredScaledConfidence = QualityUtils.phredScaleErrorRate(normalizedPosteriors[0]);
+            if ( Double.isInfinite(phredScaledConfidence) )
+                phredScaledConfidence = -10.0 * log10AlleleFrequencyPosteriors.get()[0];
+        } else {
+            phredScaledConfidence = QualityUtils.phredScaleErrorRate(PofF);
+            if ( Double.isInfinite(phredScaledConfidence) ) {
+                sum = 0.0;
+                for (int i = 1; i <= N; i++) {
+                    if ( log10AlleleFrequencyPosteriors.get()[i] == AlleleFrequencyCalculationModel.VALUE_NOT_CALCULATED )
+                        break;
+                    sum += log10AlleleFrequencyPosteriors.get()[i];
+                }
+                phredScaledConfidence = (MathUtils.compareDoubles(sum, 0.0) == 0 ? 0 : -10.0 * sum);
+            }
+        }
+
+        // return a null call if we don't pass the confidence cutoff or the most likely allele frequency is zero
+        if ( UAC.OutputMode != OUTPUT_MODE.EMIT_ALL_SITES && !passesEmitThreshold(phredScaledConfidence, bestAFguess) ) {
+            // technically, at this point our confidence in a reference call isn't accurately estimated
+            //  because it didn't take into account samples with no data, so let's get a better estimate
+            return null;
+        }
+
+        // create the genotypes
+        Map<String, Genotype> genotypes = afcm.get().assignGenotypes(vc, log10AlleleFrequencyPosteriors.get(), bestAFguess);
+
+        // *** note that calculating strand bias involves overwriting data structures, so we do that last
+        HashMap<String, Object> attributes = new HashMap<String, Object>();
+
+        int endLoc = calculateEndPos(vc.getAlleles(), vc.getReference(), loc);
+
+        Set<Allele> myAlleles = new HashSet<Allele>(vc.getAlleles());
+        // strip out the alternate allele if it's a ref call
+        if ( bestAFguess == 0 && UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.DISCOVERY ) {
+            myAlleles = new HashSet<Allele>(1);
+            myAlleles.add(vc.getReference());
+        }
+        VariantContext vcCall = new VariantContext("UG_call", loc.getContig(), loc.getStart(), endLoc,
+                myAlleles, genotypes, phredScaledConfidence/10.0, passesCallThreshold(phredScaledConfidence) ? null : filter, attributes, vc.getReferenceBaseForIndel());
+
+        return new VariantCallContext(vcCall, confidentlyCalled(phredScaledConfidence, PofF));
+    }
+
     private int calculateEndPos(Collection<Allele> alleles, Allele refAllele, GenomeLoc loc) {
         // TODO - temp fix until we can deal with extended events properly
         // for indels, stop location is one more than ref allele length

public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java

@@ -32,11 +32,17 @@ import net.sf.samtools.SAMRecord;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.utils.Haplotype;
 import org.broadinstitute.sting.utils.MathUtils;
+import org.broadinstitute.sting.utils.exceptions.UserException;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
+import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.sam.ReadUtils;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
+import java.io.File;
+import java.io.FileWriter;
+import java.io.PrintStream;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -45,9 +51,6 @@ import java.util.LinkedHashMap;
 public class PairHMMIndelErrorModel {
     public static final int BASE_QUAL_THRESHOLD = 20;
 
-    private final double logGapOpenProbability;
-    private final double logGapContinuationProbability;
-
     private boolean DEBUG = false;
     private boolean bandedLikelihoods = false;
 
@@ -89,8 +92,6 @@ public class PairHMMIndelErrorModel {
     }
 
     public PairHMMIndelErrorModel(double indelGOP, double indelGCP, boolean deb, boolean bandedLikelihoods) {
-        this.logGapOpenProbability = -indelGOP/10.0; // QUAL to log prob
-        this.logGapContinuationProbability = -indelGCP/10.0; // QUAL to log prob
         this.DEBUG = deb;
         this.bandedLikelihoods = bandedLikelihoods;
 
@@ -98,13 +99,14 @@ public class PairHMMIndelErrorModel {
         this.GAP_CONT_PROB_TABLE = new double[MAX_HRUN_GAP_IDX];
         this.GAP_OPEN_PROB_TABLE = new double[MAX_HRUN_GAP_IDX];
 
+        double gop = -indelGOP/10.0;
+        double gcp = -indelGCP/10.0;
+
         for (int i = 0; i < START_HRUN_GAP_IDX; i++) {
-            GAP_OPEN_PROB_TABLE[i] = logGapOpenProbability;
+            GAP_OPEN_PROB_TABLE[i] = gop;
-            GAP_CONT_PROB_TABLE[i] = logGapContinuationProbability;
+            GAP_CONT_PROB_TABLE[i] = gcp;
         }
 
-        double gop = logGapOpenProbability;
-        double gcp = logGapContinuationProbability;
         double step = GAP_PENALTY_HRUN_STEP/10.0;
 
         double maxGOP = -MIN_GAP_OPEN_PENALTY/10.0;  // phred to log prob
@@ -185,60 +187,57 @@ public class PairHMMIndelErrorModel {
     }
 
     private double computeReadLikelihoodGivenHaplotypeAffineGaps(byte[] haplotypeBases, byte[] readBases, byte[] readQuals,
-                                                                 double[] currentGOP, double[] currentGCP, int eventLength) {
+                                                                 double[] currentGOP, double[] currentGCP, int indToStart,
+                                                                 double[][] matchMetricArray, double[][] XMetricArray, double[][] YMetricArray) {
 
         final int X_METRIC_LENGTH = readBases.length+1;
         final int Y_METRIC_LENGTH = haplotypeBases.length+1;
 
-        // initialize path metric and traceback memories for likelihood computation
+        if (indToStart == 0) {
-        final double[][] matchMetricArray = new double[X_METRIC_LENGTH][Y_METRIC_LENGTH];
+            // default initialization for all arrays
-        final double[][] XMetricArray = new double[X_METRIC_LENGTH][Y_METRIC_LENGTH];
-        final double[][] YMetricArray = new double[X_METRIC_LENGTH][Y_METRIC_LENGTH];
 
-
+            for (int i=0; i < X_METRIC_LENGTH; i++) {
-        final double DIAG_TOL = 20; // means that max - min element in diags have to be > this number for banding to take effect.
+                Arrays.fill(matchMetricArray[i],Double.NEGATIVE_INFINITY);
-        // default initialization for all arrays
+                Arrays.fill(YMetricArray[i],Double.NEGATIVE_INFINITY);
-        for (int i=0; i < X_METRIC_LENGTH; i++) {
+                Arrays.fill(XMetricArray[i],Double.NEGATIVE_INFINITY);
-            Arrays.fill(matchMetricArray[i],Double.NEGATIVE_INFINITY);
-            Arrays.fill(YMetricArray[i],Double.NEGATIVE_INFINITY);
-            Arrays.fill(XMetricArray[i],Double.NEGATIVE_INFINITY);
-        }
-
-        for (int i=1; i < X_METRIC_LENGTH; i++) {
-            //initialize first column
-            XMetricArray[i][0]      = END_GAP_COST*(i);
-        }
-
-        for (int j=1; j < Y_METRIC_LENGTH; j++) {
-            // initialize first row
-            YMetricArray[0][j]      = END_GAP_COST*(j);
-        }
-        matchMetricArray[0][0]= END_GAP_COST;//Double.NEGATIVE_INFINITY;
-        XMetricArray[0][0]=  YMetricArray[0][0] = 0;
-
-
-        final int numDiags = X_METRIC_LENGTH +  Y_METRIC_LENGTH -1;
-        final int elemsInDiag = Math.min(X_METRIC_LENGTH, Y_METRIC_LENGTH);
-
-        int idxWithMaxElement = 0;
-
-        double maxElementInDiag = Double.NEGATIVE_INFINITY;
-
-        for (int  diag=0; diag <  numDiags; diag++) {
-            // compute default I and J start positions at edge of diagonals
-            int indI = 0;
-            int indJ = diag;
-            if (diag >= Y_METRIC_LENGTH ) {
-                indI = diag-(Y_METRIC_LENGTH-1);
-                indJ = Y_METRIC_LENGTH-1;
             }
 
-            // first pass: from max element to edge
+            for (int i=1; i < X_METRIC_LENGTH; i++) {
-            int idxLow = bandedLikelihoods? idxWithMaxElement : 0;
+                //initialize first column
+                XMetricArray[i][0]      = END_GAP_COST*(i);
+            }
 
-            // reset diag max value before starting
+            for (int j=1; j < Y_METRIC_LENGTH; j++) {
-            if (bandedLikelihoods) {
+                // initialize first row
-                maxElementInDiag = Double.NEGATIVE_INFINITY;
+                YMetricArray[0][j]      = END_GAP_COST*(j);
+            }
+            matchMetricArray[0][0]= END_GAP_COST;//Double.NEGATIVE_INFINITY;
+            XMetricArray[0][0]=  YMetricArray[0][0] = 0;
+        }
+
+
+        if (bandedLikelihoods) {
+            final double DIAG_TOL = 20; // means that max - min element in diags have to be > this number for banding to take effect.
+
+            final int numDiags = X_METRIC_LENGTH +  Y_METRIC_LENGTH -1;
+            final int elemsInDiag = Math.min(X_METRIC_LENGTH, Y_METRIC_LENGTH);
+
+            int idxWithMaxElement = 0;
+
+            for (int  diag=indToStart; diag <  numDiags; diag++) {
+                // compute default I and J start positions at edge of diagonals
+                int indI = 0;
+                int indJ = diag;
+                if (diag >= Y_METRIC_LENGTH ) {
+                    indI = diag-(Y_METRIC_LENGTH-1);
+                    indJ = Y_METRIC_LENGTH-1;
+                }
+
+                // first pass: from max element to edge
+                int idxLow =  idxWithMaxElement;
+
+                // reset diag max value before starting
+                double maxElementInDiag = Double.NEGATIVE_INFINITY;
                 // set indI, indJ to correct values
                 indI += idxLow;
                 indJ -= idxLow;
@@ -248,46 +247,10 @@ public class PairHMMIndelErrorModel {
                     indJ++;
                 }
 
-            }
-
-            for (int el = idxLow; el < elemsInDiag; el++) {
-                updateCell(indI, indJ, X_METRIC_LENGTH, Y_METRIC_LENGTH, readBases, readQuals, haplotypeBases,
-                            currentGOP, currentGCP,  matchMetricArray,  XMetricArray, YMetricArray);
-                // update max in diagonal
-                if (bandedLikelihoods) {
-                    final double bestMetric = MathUtils.max(matchMetricArray[indI][indJ], XMetricArray[indI][indJ], YMetricArray[indI][indJ]);
-
-                    // check if we've fallen off diagonal value by threshold
-                    if (bestMetric > maxElementInDiag) {
-                        maxElementInDiag = bestMetric;
-                        idxWithMaxElement = el;
-                    }
-                    else if (bestMetric < maxElementInDiag - DIAG_TOL)
-                        break; // done w/current diagonal
-                }
-
-                indI++;
-                if (indI >=X_METRIC_LENGTH )
-                    break;
-                indJ--;
-                if (indJ <= 0)
-                    break;
-            }
-            if (bandedLikelihoods && idxLow > 0) {
-                // now do second part in opposite direction
-                indI = 0;
-                indJ = diag;
-                if (diag >= Y_METRIC_LENGTH ) {
-                    indI = diag-(Y_METRIC_LENGTH-1);
-                    indJ = Y_METRIC_LENGTH-1;
-                }
-
-                indI += idxLow-1;
-                indJ -= idxLow-1;
-                for (int el = idxLow-1; el >= 0; el--) {
 
+                for (int el = idxLow; el < elemsInDiag; el++) {
                     updateCell(indI, indJ, X_METRIC_LENGTH, Y_METRIC_LENGTH, readBases, readQuals, haplotypeBases,
-                                currentGOP, currentGCP,  matchMetricArray,  XMetricArray, YMetricArray);
+                            currentGOP, currentGCP,  matchMetricArray,  XMetricArray, YMetricArray);
                     // update max in diagonal
                     final double bestMetric = MathUtils.max(matchMetricArray[indI][indJ], XMetricArray[indI][indJ], YMetricArray[indI][indJ]);
 
@@ -296,34 +259,81 @@ public class PairHMMIndelErrorModel {
                         maxElementInDiag = bestMetric;
                         idxWithMaxElement = el;
                     }
-                    else if (bestMetric < maxElementInDiag - DIAG_TOL)
+                    else if (bestMetric < maxElementInDiag - DIAG_TOL && idxWithMaxElement > 0)
                         break; // done w/current diagonal
 
-                    indJ++;
+                    indI++;
-                    if (indJ >= Y_METRIC_LENGTH )
+                    if (indI >=X_METRIC_LENGTH )
                         break;
-                    indI--;
+                    indJ--;
-                    if (indI <= 0)
+                    if (indJ <= 0)
                         break;
                 }
+                if (idxLow > 0) {
+                    // now do second part in opposite direction
+                    indI = 0;
+                    indJ = diag;
+                    if (diag >= Y_METRIC_LENGTH ) {
+                        indI = diag-(Y_METRIC_LENGTH-1);
+                        indJ = Y_METRIC_LENGTH-1;
+                    }
+
+                    indI += idxLow-1;
+                    indJ -= idxLow-1;
+                    for (int el = idxLow-1; el >= 0; el--) {
+
+                        updateCell(indI, indJ, X_METRIC_LENGTH, Y_METRIC_LENGTH, readBases, readQuals, haplotypeBases,
+                                currentGOP, currentGCP,  matchMetricArray,  XMetricArray, YMetricArray);
+                        // update max in diagonal
+                        final double bestMetric = MathUtils.max(matchMetricArray[indI][indJ], XMetricArray[indI][indJ], YMetricArray[indI][indJ]);
+
+                        // check if we've fallen off diagonal value by threshold
+                        if (bestMetric > maxElementInDiag) {
+                            maxElementInDiag = bestMetric;
+                            idxWithMaxElement = el;
+                        }
+                        else if (bestMetric < maxElementInDiag - DIAG_TOL)
+                            break; // done w/current diagonal
+
+                        indJ++;
+                        if (indJ >= Y_METRIC_LENGTH )
+                            break;
+                        indI--;
+                        if (indI <= 0)
+                            break;
+                    }
+                }
+                // if (DEBUG)
+                //     System.out.format("Max:%4.1f el:%d\n",maxElementInDiag,  idxWithMaxElement);
             }
-           // if (DEBUG)
-           //     System.out.format("Max:%4.1f el:%d\n",maxElementInDiag,  idxWithMaxElement);
         }
+        else {
+            // simplified rectangular version of update loop
+            for (int indI=1; indI < X_METRIC_LENGTH; indI++) {
+                for (int indJ=indToStart+1; indJ < Y_METRIC_LENGTH; indJ++) {
+                    updateCell(indI, indJ, X_METRIC_LENGTH, Y_METRIC_LENGTH, readBases, readQuals, haplotypeBases,
+                            currentGOP, currentGCP,  matchMetricArray,  XMetricArray, YMetricArray);
+
+                }
+            }
+        }
+
+
 
         final int bestI = X_METRIC_LENGTH - 1, bestJ = Y_METRIC_LENGTH - 1;
         final double bestMetric = MathUtils.softMax(matchMetricArray[bestI][bestJ],
                 XMetricArray[bestI][bestJ],
                 YMetricArray[bestI][bestJ]);
-     /*
+
+        /*
         if (DEBUG) {
             PrintStream outx, outy, outm, outs;
             double[][] sumMetrics = new double[X_METRIC_LENGTH][Y_METRIC_LENGTH];
             try {
-                outx = new PrintStream("../../UGOptim/datax.txt");
+                outx = new PrintStream("datax.txt");
-                outy = new PrintStream("../../UGOptim/datay.txt");
+                outy = new PrintStream("datay.txt");
-                outm = new PrintStream("../../UGOptim/datam.txt");
+                outm = new PrintStream("datam.txt");
-                outs = new PrintStream("../../UGOptim/datas.txt");
+                outs = new PrintStream("datas.txt");
                 double metrics[] = new double[3];
                 for (int indI=0; indI < X_METRIC_LENGTH; indI++) {
                     for (int indJ=0; indJ < Y_METRIC_LENGTH; indJ++) {
@@ -393,7 +403,7 @@ public class PairHMMIndelErrorModel {
 
         for (PileupElement p: pileup) {
             // > 1 when the read is a consensus read representing multiple independent observations
-            final boolean isReduced = ReadUtils.isReducedRead(p.getRead());
+            final boolean isReduced = p.isReducedRead();
             readCounts[readIdx] = isReduced ? p.getReducedCount() : 1;
 
             // check if we've already computed likelihoods for this pileup element (i.e. for this read at this location)
@@ -414,8 +424,6 @@ public class PairHMMIndelErrorModel {
                     continue;
                 }
 
-                double[] recalQuals = null;
-
                 // get bases of candidate haplotypes that overlap with reads
                 final int trailingBases = 3;
 
@@ -534,6 +542,12 @@ public class PairHMMIndelErrorModel {
                             unclippedReadBases.length-numEndClippedBases);
 
                     int j=0;
+
+                    // initialize path metric and traceback memories for likelihood computation
+                    double[][] matchMetricArray = null, XMetricArray = null, YMetricArray = null;
+                    byte[] previousHaplotypeSeen = null;
+                    double[] previousGOP = null;
+                    int startIdx;
                     for (Allele a: haplotypeMap.keySet()) {
 
 
@@ -551,11 +565,37 @@ public class PairHMMIndelErrorModel {
                         byte[] haplotypeBases = Arrays.copyOfRange(haplotype.getBasesAsBytes(),
                                 (int)indStart, (int)indStop);
 
+                        double readLikelihood;
+                        if (matchMetricArray == null) {
+                            final int X_METRIC_LENGTH = readBases.length+1;
+                            final int Y_METRIC_LENGTH = haplotypeBases.length+1;
+
+                            matchMetricArray = new double[X_METRIC_LENGTH][Y_METRIC_LENGTH];
+                            XMetricArray = new double[X_METRIC_LENGTH][Y_METRIC_LENGTH];
+                            YMetricArray = new double[X_METRIC_LENGTH][Y_METRIC_LENGTH];
+                        }
                         final double[] currentContextGOP = Arrays.copyOfRange(gapOpenProbabilityMap.get(a), (int)indStart, (int)indStop);
                         final double[] currentContextGCP = Arrays.copyOfRange(gapContProbabilityMap.get(a), (int)indStart, (int)indStop);
-                        final double readLikelihood = computeReadLikelihoodGivenHaplotypeAffineGaps(haplotypeBases, readBases, readQuals,
+                        if (previousHaplotypeSeen == null)
-                                currentContextGOP, currentContextGCP, eventLength);
+                            startIdx = 0;
+                        else {
+                            int s1 = computeFirstDifferingPosition(haplotypeBases, previousHaplotypeSeen);
+                            int s2 = computeFirstDifferingPosition(currentContextGOP, previousGOP);
+                            startIdx = Math.min(s1,s2);
+                        }
+                        previousHaplotypeSeen = haplotypeBases.clone();
+                        previousGOP = currentContextGOP.clone();
 
+
+
+                        readLikelihood = computeReadLikelihoodGivenHaplotypeAffineGaps(haplotypeBases, readBases, readQuals,
+                                currentContextGOP, currentContextGCP, startIdx, matchMetricArray, XMetricArray, YMetricArray);
+                        if (DEBUG) {
+                            System.out.println("H:"+new String(haplotypeBases));
+                            System.out.println("R:"+new String(readBases));
+                            System.out.format("L:%4.2f\n",readLikelihood);
+                            System.out.format("StPos:%d\n", startIdx);
+                        }
                         readEl.put(a,readLikelihood);
                         readLikelihoods[readIdx][j++] = readLikelihood;
                     }
@@ -579,6 +619,28 @@ public class PairHMMIndelErrorModel {
         return getHaplotypeLikelihoods(numHaplotypes, readCounts, readLikelihoods);
     }
 
+    private int computeFirstDifferingPosition(byte[] b1, byte[] b2) {
+        if (b1.length != b2.length)
+            return 0; // sanity check
+
+        for (int i=0; i < b1.length; i++ ){
+            if ( b1[i]!= b2[i])
+                return i;
+        }
+        return 0; // sanity check
+    }
+
+    private int computeFirstDifferingPosition(double[] b1, double[] b2) {
+        if (b1.length != b2.length)
+            return 0; // sanity check
+
+        for (int i=0; i < b1.length; i++ ){
+            if ( b1[i]!= b2[i])
+                return i;
+        }
+        return 0; // sanity check
+    }
+
     private final static double[] getHaplotypeLikelihoods(final int numHaplotypes, final int readCounts[], final double readLikelihoods[][]) {
         final double[][] haplotypeLikehoodMatrix = new double[numHaplotypes][numHaplotypes];
 

public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java

@@ -2,9 +2,12 @@ package org.broadinstitute.sting.gatk.walkers.recalibration;
 
 import net.sf.samtools.SAMRecord;
 import org.broadinstitute.sting.utils.BaseUtils;
+import org.broadinstitute.sting.utils.NGSPlatform;
 import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 
 import java.util.Arrays;
+import java.util.EnumSet;
 import java.util.List;
 
 /*
@@ -46,6 +49,9 @@ import java.util.List;
  */
 
 public class CycleCovariate implements StandardCovariate {
+    private final static EnumSet<NGSPlatform> DISCRETE_CYCLE_PLATFORMS = EnumSet.of(NGSPlatform.ILLUMINA, NGSPlatform.SOLID, NGSPlatform.PACBIO, NGSPlatform.COMPLETE_GENOMICS);
+    private final static EnumSet<NGSPlatform> FLOW_CYCLE_PLATFORMS = EnumSet.of(NGSPlatform.LS454,  NGSPlatform.ION_TORRENT);
+
     // Initialize any member variables using the command-line arguments passed to the walkers
     public void initialize( final RecalibrationArgumentCollection RAC ) {
         if( RAC.DEFAULT_PLATFORM != null ) {
@@ -58,122 +64,6 @@ public class CycleCovariate implements StandardCovariate {
         }
     }
 
-    /*
-    // Used to pick out the covariate's value from attributes of the read
-    public final Comparable getValue( final SAMRecord read, final int offset ) {
-
-        int cycle = 1;
-
-        //-----------------------------
-        // ILLUMINA and SOLID
-        //-----------------------------
-
-        if( read.getReadGroup().getPlatform().equalsIgnoreCase( "ILLUMINA" ) || read.getReadGroup().getPlatform().equalsIgnoreCase( "SLX" ) || // Some bams have "illumina" and others have "SLX"
-            read.getReadGroup().getPlatform().equalsIgnoreCase( "SOLID" ) || read.getReadGroup().getPlatform().equalsIgnoreCase( "ABI_SOLID" )) { // Some bams have "solid" and others have "ABI_SOLID"
-            cycle = offset + 1;
-            if( read.getReadNegativeStrandFlag() ) {
-                cycle = read.getReadLength() - offset;
-            }
-        }
-
-        //-----------------------------
-        // 454
-        //-----------------------------
-
-        else if( read.getReadGroup().getPlatform().contains( "454" ) ) { // Some bams have "LS454" and others have just "454"
-            final byte[] bases = read.getReadBases();
-
-            // BUGBUG: Consider looking at degradation of base quality scores in homopolymer runs to detect when the cycle incremented even though the nucleotide didn't change
-            // For example, AAAAAAA was probably read in two flow cycles but here we count it as one
-            if( !read.getReadNegativeStrandFlag() ) { // Forward direction
-                int iii = 0;
-                while( iii <= offset )
-                {
-                    while( iii <= offset && bases[iii] == (byte)'T' ) { iii++; }
-                    while( iii <= offset && bases[iii] == (byte)'A' ) { iii++; }
-                    while( iii <= offset && bases[iii] == (byte)'C' ) { iii++; }
-                    while( iii <= offset && bases[iii] == (byte)'G' ) { iii++; }
-                    if( iii <= offset ) { cycle++; }
-                    if( iii <= offset && !BaseUtils.isRegularBase(bases[iii]) ) { iii++; }
-
-                }
-            } else { // Negative direction
-                int iii = bases.length-1;
-                while( iii >= offset )
-                {
-                    while( iii >= offset && bases[iii] == (byte)'T' ) { iii--; }
-                    while( iii >= offset && bases[iii] == (byte)'A' ) { iii--; }
-                    while( iii >= offset && bases[iii] == (byte)'C' ) { iii--; }
-                    while( iii >= offset && bases[iii] == (byte)'G' ) { iii--; }
-                    if( iii >= offset ) { cycle++; }
-                    if( iii >= offset && !BaseUtils.isRegularBase(bases[iii]) ) { iii--; }
-                }
-            }
-        }
-
-        //-----------------------------
-        // SOLID (unused), only to be used in conjunction with PrimerRoundCovariate
-        //-----------------------------
-
-        //else if( read.getReadGroup().getPlatform().equalsIgnoreCase( "SOLID" ) ) {
-        //    // The ligation cycle according to http://www3.appliedbiosystems.com/cms/groups/mcb_marketing/documents/generaldocuments/cms_057511.pdf
-        //    int pos = offset + 1;
-        //    if( read.getReadNegativeStrandFlag() ) {
-        //        pos = read.getReadLength() - offset;
-        //    }
-        //    cycle = pos / 5; // integer division
-        //}
-
-        //-----------------------------
-        // UNRECOGNIZED PLATFORM
-        //-----------------------------
-
-        else { // Platform is unrecognized so revert to the default platform but warn the user first
-            if( defaultPlatform != null) { // The user set a default platform
-                if( !warnedUserBadPlatform ) {
-                    Utils.warnUser( "Platform string (" + read.getReadGroup().getPlatform() + ") unrecognized in CycleCovariate. " +
-                            "Defaulting to platform = " + defaultPlatform + "." );
-                }
-                warnedUserBadPlatform = true;
-
-                read.getReadGroup().setPlatform( defaultPlatform );
-                return getValue( read, offset ); // A recursive call
-            } else { // The user did not set a default platform
-                throw new StingException( "Platform string (" + read.getReadGroup().getPlatform() + ") unrecognized in CycleCovariate. " +
-                        "No default platform specified. Users must set the default platform using the --default_platform <String> argument." );
-            }
-        }
-
-        // Differentiate between first and second of pair.
-        // The sequencing machine cycle keeps incrementing for the second read in a pair. So it is possible for a read group
-        // to have an error affecting quality at a particular cycle on the first of pair which carries over to the second of pair.
-        // Therefore the cycle covariate must differentiate between first and second of pair reads.
-        // This effect can not be corrected by pulling out the first of pair and second of pair flags into a separate covariate because
-        //   the current sequential model would consider the effects independently instead of jointly.
-        if( read.getReadPairedFlag() && read.getSecondOfPairFlag() ) {
-            cycle *= -1;
-        }
-
-        return cycle;
-    }
-    */
-
-    // todo -- this should be put into a common place in the code base
-    private static List<String> ILLUMINA_NAMES = Arrays.asList("ILLUMINA", "SLX", "SOLEXA");
-    private static List<String> SOLID_NAMES = Arrays.asList("SOLID");
-    private static List<String> LS454_NAMES = Arrays.asList("454");
-    private static List<String> COMPLETE_GENOMICS_NAMES = Arrays.asList("COMPLETE");
-    private static List<String> PACBIO_NAMES = Arrays.asList("PACBIO");
-    private static List<String> ION_TORRENT_NAMES = Arrays.asList("IONTORRENT");
-
-    private static boolean isPlatform(SAMRecord read, List<String> names) {
-        String pl = read.getReadGroup().getPlatform().toUpperCase();
-        for ( String name : names )
-            if ( pl.contains( name ) )
-                return true;
-        return false;
-    }
-
     // Used to pick out the covariate's value from attributes of the read
     public void getValues(SAMRecord read, Comparable[] comparable) {
 
@@ -181,7 +71,8 @@ public class CycleCovariate implements StandardCovariate {
         // Illumina, Solid, PacBio, and Complete Genomics
         //-----------------------------
 
-        if( isPlatform(read, ILLUMINA_NAMES) || isPlatform(read, SOLID_NAMES) || isPlatform(read, PACBIO_NAMES) || isPlatform(read, COMPLETE_GENOMICS_NAMES) ) {
+        final NGSPlatform ngsPlatform = ((GATKSAMRecord)read).getNGSPlatform();
+        if( DISCRETE_CYCLE_PLATFORMS.contains(ngsPlatform) ) {
             final int init;
             final int increment;
             if( !read.getReadNegativeStrandFlag() ) {
@@ -227,8 +118,7 @@ public class CycleCovariate implements StandardCovariate {
         //-----------------------------
         // 454 and Ion Torrent
         //-----------------------------
-
+        else if( FLOW_CYCLE_PLATFORMS.contains(ngsPlatform) ) {
-        else if ( isPlatform(read, LS454_NAMES) || isPlatform(read, ION_TORRENT_NAMES)) { // Some bams have "LS454" and others have just "454"
 
             final int readLength = read.getReadLength();
             final byte[] bases = read.getReadBases();
@@ -273,8 +163,6 @@ public class CycleCovariate implements StandardCovariate {
         else  {
             throw new IllegalStateException("This method hasn't been implemented yet for " + read.getReadGroup().getPlatform());
         }
-
-
     }
 
     // Used to get the covariate's value from input csv file in TableRecalibrationWalker

public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java

@@ -35,6 +35,7 @@ import org.broadinstitute.sting.utils.collections.NestedHashMap;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.broadinstitute.sting.utils.exceptions.UserException;
 import org.broadinstitute.sting.utils.sam.AlignmentUtils;
+import org.broadinstitute.sting.utils.sam.GATKSAMReadGroupRecord;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 
 import java.util.ArrayList;
@@ -228,8 +229,7 @@ public class RecalDataManager {
      * @param RAC The list of shared command line arguments
      */
     public static void parseSAMRecord( final SAMRecord read, final RecalibrationArgumentCollection RAC ) {
-
+        GATKSAMReadGroupRecord readGroup = ((GATKSAMRecord)read).getReadGroup();
-        SAMReadGroupRecord readGroup = read.getReadGroup();
 
         // If there are no read groups we have to default to something, and that something could be specified by the user using command line arguments
         if( readGroup == null ) {
@@ -241,7 +241,7 @@ public class RecalDataManager {
                     warnUserNullReadGroup = true;
                 }
                 // There is no readGroup so defaulting to these values
-                readGroup = new SAMReadGroupRecord( RAC.DEFAULT_READ_GROUP );
+                readGroup = new GATKSAMReadGroupRecord( RAC.DEFAULT_READ_GROUP );
                 readGroup.setPlatform( RAC.DEFAULT_PLATFORM );
                 ((GATKSAMRecord)read).setReadGroup( readGroup );
             } else {
@@ -251,7 +251,7 @@ public class RecalDataManager {
 
         if( RAC.FORCE_READ_GROUP != null && !readGroup.getReadGroupId().equals(RAC.FORCE_READ_GROUP) ) { // Collapse all the read groups into a single common String provided by the user
             final String oldPlatform = readGroup.getPlatform();
-            readGroup = new SAMReadGroupRecord( RAC.FORCE_READ_GROUP );
+            readGroup = new GATKSAMReadGroupRecord( RAC.FORCE_READ_GROUP );
             readGroup.setPlatform( oldPlatform );
             ((GATKSAMRecord)read).setReadGroup( readGroup );
         }

public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/IndelSize.java

@@ -0,0 +1,52 @@
+package org.broadinstitute.sting.gatk.walkers.varianteval.stratifications;
+
+import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
+import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
+import org.broadinstitute.sting.utils.MathUtils;
+import org.broadinstitute.sting.utils.variantcontext.VariantContext;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Stratifies the eval RODs by the indel size
+ *
+ * Indel sizes are stratified from sizes -100 to +100. Sizes greater than this are lumped in the +/- 100 bin
+ * This stratification ignores multi-allelic indels (whose size is not defined uniquely)
+ */
+public class IndelSize extends VariantStratifier {
+    static final int MAX_INDEL_SIZE = 100;
+    @Override
+    public void initialize() {
+        states = new ArrayList<String>();
+        for( int a=-MAX_INDEL_SIZE; a <=MAX_INDEL_SIZE; a++ ) {
+            states.add(String.format("%d", a));
+        }
+    }
+
+    public List<String> getRelevantStates(ReferenceContext ref, RefMetaDataTracker tracker, VariantContext comp, String compName, VariantContext eval, String evalName, String sampleName) {
+        ArrayList<String> relevantStates = new ArrayList<String>();
+
+        if (eval != null && eval.isIndel() && eval.isBiallelic()) {
+            try {
+                int eventLength = 0;
+                if ( eval.isSimpleInsertion() ) {
+                    eventLength = eval.getAlternateAllele(0).length();
+                } else if ( eval.isSimpleDeletion() ) {
+                    eventLength = -eval.getReference().length();
+                }
+
+                if (eventLength > MAX_INDEL_SIZE)
+                    eventLength = MAX_INDEL_SIZE;
+                else if (eventLength < -MAX_INDEL_SIZE)
+                    eventLength = -MAX_INDEL_SIZE;
+
+                relevantStates.add(String.format("%d",eventLength));
+            } catch (Exception e) {
+                return relevantStates;
+            }
+        }
+
+        return relevantStates;
+    }
+}

public/java/src/org/broadinstitute/sting/utils/NGSPlatform.java

@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils;
+
+import net.sf.samtools.SAMReadGroupRecord;
+import net.sf.samtools.SAMRecord;
+
+/**
+ * A canonical, master list of the standard NGS platforms.  These values
+ * can be obtained (efficiently) from a GATKSAMRecord object with the
+ * getNGSPlatform method.
+ *
+ * @author Mark DePristo
+ * @since 2011
+ */
+public enum NGSPlatform {
+    ILLUMINA("ILLUMINA", "SLX", "SOLEXA"),
+    SOLID("SOLID"),
+    LS454("454"),
+    COMPLETE_GENOMICS("COMPLETE"),
+    PACBIO("PACBIO"),
+    ION_TORRENT("IONTORRENT"),
+    UNKNOWN("UNKNOWN");
+
+    /**
+     * Array of the prefix names in a BAM file for each of the platforms.
+     */
+    private final String[] BAM_PL_NAMES;
+
+    NGSPlatform(final String... BAM_PL_NAMES) {
+        for ( int i = 0; i < BAM_PL_NAMES.length; i++ )
+            BAM_PL_NAMES[i] = BAM_PL_NAMES[i].toUpperCase();
+        this.BAM_PL_NAMES = BAM_PL_NAMES;
+    }
+
+    /**
+     * Returns a representative PL string for this platform
+     * @return
+     */
+    public final String getDefaultPlatform() {
+        return BAM_PL_NAMES[0];
+    }
+
+    /**
+     * Convenience constructor -- calculates the NGSPlatfrom from a SAMRecord.
+     * Note you should not use this function if you have a GATKSAMRecord -- use the
+     * accessor method instead.
+     *
+     * @param read
+     * @return an NGSPlatform object matching the PL field of the header, of UNKNOWN if there was no match
+     */
+    public static final NGSPlatform fromRead(SAMRecord read) {
+        return fromReadGroup(read.getReadGroup());
+    }
+
+    /**
+     * Returns the NGSPlatform corresponding to the PL tag in the read group
+     * @param rg
+     * @return an NGSPlatform object matching the PL field of the header, of UNKNOWN if there was no match
+     */
+    public static final NGSPlatform fromReadGroup(SAMReadGroupRecord rg) {
+        return fromReadGroupPL(rg.getPlatform());
+    }
+
+    /**
+     * Returns the NGSPlatform corresponding to the PL tag in the read group
+     * @param plFromRG -- the PL field (or equivalent) in a ReadGroup object
+     * @return an NGSPlatform object matching the PL field of the header, of UNKNOWN if there was no match
+     */
+    public static final NGSPlatform fromReadGroupPL(final String plFromRG) {
+        if ( plFromRG == null ) return UNKNOWN;
+
+        // todo -- algorithm could be implemented more efficiently, as the list of all
+        // todo -- names is known upfront, so a decision tree could be used to identify
+        // todo -- a prefix common to PL
+        final String pl = plFromRG.toUpperCase();
+        for ( final NGSPlatform ngsPlatform : NGSPlatform.values() ) {
+            for ( final String bamPLName : ngsPlatform.BAM_PL_NAMES ) {
+                if ( pl.contains(bamPLName) )
+                    return ngsPlatform;
+            }
+        }
+
+        return UNKNOWN;
+    }
+}

public/java/src/org/broadinstitute/sting/utils/R/RScriptExecutor.java

@@ -25,35 +25,35 @@
 package org.broadinstitute.sting.utils.R;
 
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
 import org.broadinstitute.sting.commandline.Advanced;
 import org.broadinstitute.sting.commandline.Argument;
-import org.broadinstitute.sting.commandline.ArgumentCollection;
-import org.broadinstitute.sting.gatk.walkers.recalibration.Covariate;
-import org.broadinstitute.sting.utils.PathUtils;
 import org.broadinstitute.sting.utils.Utils;
+import org.broadinstitute.sting.utils.exceptions.StingException;
 import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.broadinstitute.sting.utils.io.IOUtils;
+import org.broadinstitute.sting.utils.io.Resource;
+import org.broadinstitute.sting.utils.runtime.ProcessController;
+import org.broadinstitute.sting.utils.runtime.ProcessSettings;
 
 import java.io.File;
-import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
 /**
- * Generic service for executing RScripts in the GATK directory
+ * Generic service for executing RScripts
- *
- * @author Your Name
- * @since Date created
  */
 public class RScriptExecutor {
     /**
      * our log
      */
-    protected static Logger logger = Logger.getLogger(RScriptExecutor.class);
+    private static Logger logger = Logger.getLogger(RScriptExecutor.class);
 
     public static class RScriptArgumentCollection {
         @Advanced
-        @Argument(fullName = "path_to_Rscript", shortName = "Rscript", doc = "The path to your implementation of Rscript. For Broad users this is maybe /broad/software/free/Linux/redhat_5_x86_64/pkgs/r_2.12.0/bin/Rscript", required = false)
+        @Argument(fullName = "path_to_Rscript", shortName = "Rscript", doc = "The path to your implementation of Rscript. Defaults Rscript meaning to use the first available on the environment PATH. For Broad users should 'use R-2.12' or later.", required = false)
         public String PATH_TO_RSCRIPT = "Rscript";
 
         @Advanced
@@ -62,40 +62,119 @@ public class RScriptExecutor {
 
         public RScriptArgumentCollection() {}
 
-        /** For testing and convenience */
+        /* For testing and convenience */
         public RScriptArgumentCollection(final String PATH_TO_RSCRIPT, final List<String> PATH_TO_RESOURCES) {
             this.PATH_TO_RSCRIPT = PATH_TO_RSCRIPT;
             this.PATH_TO_RESOURCES = PATH_TO_RESOURCES;
         }
     }
 
-    final RScriptArgumentCollection myArgs;
+    private final RScriptArgumentCollection myArgs;
-    final boolean exceptOnError;
+    private final boolean exceptOnError;
+    private final List<RScriptLibrary> libraries = new ArrayList<RScriptLibrary>();
+    private final List<Resource> scriptResources = new ArrayList<Resource>();
+    private final List<File> scriptFiles = new ArrayList<File>();
+    private final List<String> args = new ArrayList<String>();
 
     public RScriptExecutor(final RScriptArgumentCollection myArgs, final boolean exceptOnError) {
         this.myArgs = myArgs;
         this.exceptOnError = exceptOnError;
     }
 
-    public void callRScripts(String scriptName, Object... scriptArgs) {
+    public void addLibrary(RScriptLibrary library) {
-        callRScripts(scriptName, Arrays.asList(scriptArgs));
+        this.libraries.add(library);
     }
 
-    public void callRScripts(String scriptName, List<Object> scriptArgs) {
+    public void addScript(Resource script) {
+        this.scriptResources.add(script);
+    }
+
+    public void addScript(File script) {
+        this.scriptFiles.add(script);
+    }
+
+    /**
+     * Adds args to the end of the Rscript command line.
+     * @param args the args.
+     * @throws NullPointerException if any of the args are null.
+     */
+    public void addArgs(Object... args) {
+        for (Object arg: args)
+            this.args.add(arg.toString());
+    }
+
+    public void exec() {
+        List<File> tempFiles = new ArrayList<File>();
         try {
-            final File pathToScript = findScript(scriptName);
+            File tempLibDir = IOUtils.tempDir("R.", ".lib");
-            if ( pathToScript == null ) return; // we failed but shouldn't exception out
+            tempFiles.add(tempLibDir);
-            final String argString = Utils.join(" ", scriptArgs);
+
-            final String cmdLine = Utils.join(" ", Arrays.asList(myArgs.PATH_TO_RSCRIPT, pathToScript, argString));
+            StringBuilder expression = new StringBuilder("tempLibDir = '").append(tempLibDir).append("';");
-            logger.info("Executing RScript: " + cmdLine);
+
-            Runtime.getRuntime().exec(cmdLine).waitFor();
+            if (this.libraries.size() > 0) {
-        } catch (InterruptedException e) {
+                List<String> tempLibraryPaths = new ArrayList<String>();
+                for (RScriptLibrary library: this.libraries) {
+                    File tempLibrary = library.writeTemp();
+                    tempFiles.add(tempLibrary);
+                    tempLibraryPaths.add(tempLibrary.getAbsolutePath());
+                }
+
+                expression.append("install.packages(");
+                expression.append("pkgs=c('").append(StringUtils.join(tempLibraryPaths, "', '")).append("'), lib=tempLibDir, repos=NULL, type='source', ");
+                // Install faster by eliminating cruft.
+                expression.append("INSTALL_opts=c('--no-libs', '--no-data', '--no-help', '--no-demo', '--no-exec')");
+                expression.append(");");
+
+                for (RScriptLibrary library: this.libraries) {
+                    expression.append("require('").append(library.getLibraryName()).append("', lib.loc=tempLibDir);");
+                }
+            }
+
+            for (Resource script: this.scriptResources) {
+                File tempScript = IOUtils.writeTempResource(script);
+                tempFiles.add(tempScript);
+                expression.append("source('").append(tempScript.getAbsolutePath()).append("');");
+            }
+
+            for (File script: this.scriptFiles) {
+                expression.append("source('").append(script.getAbsolutePath()).append("');");
+            }
+
+            String[] cmd = new String[this.args.size() + 3];
+            int i = 0;
+            cmd[i++] = myArgs.PATH_TO_RSCRIPT;
+            cmd[i++] = "-e";
+            cmd[i++] = expression.toString();
+            for (String arg: this.args)
+                cmd[i++] = arg;
+
+            ProcessSettings processSettings = new ProcessSettings(cmd);
+            if (logger.isDebugEnabled()) {
+                processSettings.getStdoutSettings().printStandard(true);
+                processSettings.getStderrSettings().printStandard(true);
+            }
+
+            ProcessController controller = ProcessController.getThreadLocal();
+
+            logger.debug("Executing: " + Utils.join(" ", cmd));
+            logger.debug("Result: " + controller.exec(processSettings).getExitValue());
+
+        } catch (StingException e) {
             generateException(e);
-        } catch (IOException e) {
+        } finally {
-            generateException("Fatal Exception: Perhaps RScript jobs are being spawned too quickly?", e);
+            for (File temp: tempFiles)
+                FileUtils.deleteQuietly(temp);
         }
     }
 
+    public void callRScripts(String scriptName, Object... scriptArgs) {
+        final File pathToScript = findScript(scriptName);
+        if (pathToScript == null) return; // we failed but shouldn't exception out
+        addScript(pathToScript);
+        addArgs(scriptArgs);
+        exec();
+    }
+
     public File findScript(final String scriptName) {
         for ( String pathToResource : myArgs.PATH_TO_RESOURCES ) {
             final File f = new File(pathToResource + "/" + scriptName);

public/java/src/org/broadinstitute/sting/utils/R/RScriptLibrary.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.R;
+
+import org.broadinstitute.sting.utils.io.IOUtils;
+import org.broadinstitute.sting.utils.io.Resource;
+
+import java.io.File;
+
+/**
+ * Libraries embedded in the StingUtils package.
+ */
+public enum RScriptLibrary {
+    GSALIB("gsalib");
+
+    private final String name;
+
+    private RScriptLibrary(String name) {
+        this.name = name;
+    }
+
+    public String getLibraryName() {
+        return this.name;
+    }
+
+    public String getResourcePath() {
+        return name + ".tar.gz";
+    }
+
+    /**
+     * Writes the library source code to a temporary tar.gz file and returns the path.
+     * @return The path to the library source code. The caller must delete the code when done.
+     */
+    public File writeTemp() {
+        return IOUtils.writeTempResource(new Resource(getResourcePath(), RScriptLibrary.class));
+    }
+}

public/java/src/org/broadinstitute/sting/utils/help/HelpFormatter.java

@@ -29,6 +29,7 @@ import org.apache.log4j.Logger;
 import org.broadinstitute.sting.commandline.ArgumentDefinition;
 import org.broadinstitute.sting.commandline.ArgumentDefinitionGroup;
 import org.broadinstitute.sting.commandline.ArgumentDefinitions;
+import org.broadinstitute.sting.commandline.ArgumentMatchSource;
 import org.broadinstitute.sting.utils.Utils;
 import org.broadinstitute.sting.utils.text.TextFormattingUtils;
 
@@ -47,6 +48,7 @@ public class HelpFormatter {
 
     /**
      * Prints the help, given a collection of argument definitions.
+     * @param applicationDetails Application details
      * @param argumentDefinitions Argument definitions for which help should be printed.
      */
     public void printHelp( ApplicationDetails applicationDetails, ArgumentDefinitions argumentDefinitions ) {
@@ -233,7 +235,7 @@ public class HelpFormatter {
     private List<ArgumentDefinitionGroup> prepareArgumentGroups( ArgumentDefinitions argumentDefinitions ) {
         // Sort the list of argument definitions according to how they should be shown.
         // Put the sorted results into a new cloned data structure.
-        Comparator definitionComparator = new Comparator<ArgumentDefinition>() {
+        Comparator<ArgumentDefinition> definitionComparator = new Comparator<ArgumentDefinition>() {
             public int compare( ArgumentDefinition lhs, ArgumentDefinition rhs ) {
                 if( lhs.required && rhs.required ) return 0;
                 if( lhs.required ) return -1;
@@ -242,15 +244,15 @@ public class HelpFormatter {
             }
         };
 
-        List<ArgumentDefinitionGroup> argumentGroups = new ArrayList();        
+        List<ArgumentDefinitionGroup> argumentGroups = new ArrayList<ArgumentDefinitionGroup>();
         for( ArgumentDefinitionGroup argumentGroup: argumentDefinitions.getArgumentDefinitionGroups() ) {
-            List<ArgumentDefinition> sortedDefinitions = new ArrayList( argumentGroup.argumentDefinitions );
+            List<ArgumentDefinition> sortedDefinitions = new ArrayList<ArgumentDefinition>( argumentGroup.argumentDefinitions );
             Collections.sort( sortedDefinitions, definitionComparator );
             argumentGroups.add( new ArgumentDefinitionGroup(argumentGroup.groupName,sortedDefinitions) );
         }
 
         // Sort the argument groups themselves with main arguments first, followed by plugins sorted in name order.
-        Comparator groupComparator = new Comparator<ArgumentDefinitionGroup>() {
+        Comparator<ArgumentDefinitionGroup> groupComparator = new Comparator<ArgumentDefinitionGroup>() {
             public int compare( ArgumentDefinitionGroup lhs, ArgumentDefinitionGroup rhs ) {
                 if( lhs.groupName == null && rhs.groupName == null ) return 0;
                 if( lhs.groupName == null ) return -1;
@@ -271,9 +273,9 @@ public class HelpFormatter {
      * Generate a standard header for the logger
      *
      * @param applicationDetails details of the application to run.
-     * @param args the command line arguments passed in
+     * @param parsedArgs the command line arguments passed in
      */
-    public static void generateHeaderInformation(ApplicationDetails applicationDetails, String[] args) {
+    public static void generateHeaderInformation(ApplicationDetails applicationDetails, Map<ArgumentMatchSource, List<String>> parsedArgs) {
 
         DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
         java.util.Date date = new java.util.Date();
@@ -283,11 +285,22 @@ public class HelpFormatter {
         logger.info(barrier);
         for (String headerLine : applicationDetails.applicationHeader)
             logger.info(headerLine);
-        String output = "";
+        logger.debug("Current directory: " + System.getProperty("user.dir"));
-        for (String str : args) {
+        for (Map.Entry<ArgumentMatchSource, List<String>> entry: parsedArgs.entrySet()) {
-            output = output + str + " ";
+            ArgumentMatchSource matchSource = entry.getKey();
+            final String sourceName;
+            switch (matchSource.getType()) {
+                case CommandLine: sourceName = "Program"; break;
+                case File: sourceName = matchSource.getFile().getPath(); break;
+                default: throw new RuntimeException("Unexpected argument match source type: " + matchSource.getType());
+            }
+
+            String output = sourceName + " Args:";
+            for (String str : entry.getValue()) {
+                output = output + " " + str;
+            }
+            logger.info(output);
         }
-        logger.info("Program Args: " + output);
         logger.info("Date/Time: " + dateFormat.format(date));
         logger.info(barrier);
 

public/java/src/org/broadinstitute/sting/utils/io/FileExtension.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person 
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.io;
+
+import java.io.File;
+
+public interface FileExtension {
+    /**
+     * Returns a clone of the FileExtension with a new path.
+     * @param path New path.
+     * @return New FileExtension
+     */
+    public File withPath(String path);
+}

public/java/src/org/broadinstitute/sting/utils/io/HardThresholdingOutputStream.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+package org.broadinstitute.sting.utils.io;
+
+import org.apache.commons.io.output.ThresholdingOutputStream;
+
+import java.io.IOException;
+
+/**
+ * An output stream which stops at the threshold
+ * instead of potentially triggering early.
+ */
+public abstract class HardThresholdingOutputStream extends ThresholdingOutputStream {
+    protected HardThresholdingOutputStream(int threshold) {
+        super(threshold);
+    }
+
+    @Override
+    public void write(byte[] b) throws IOException {
+        write(b, 0, b.length);
+    }
+
+    @Override
+    public void write(byte[] b, int off, int len) throws IOException {
+        int remaining = this.getThreshold() - (int)this.getByteCount();
+        if (!isThresholdExceeded() && len > remaining) {
+            super.write(b, off, remaining);
+            super.write(b, off + remaining, len - remaining);
+        } else {
+            super.write(b, off, len);
+        }
+    }
+}

public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java

@@ -0,0 +1,353 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.io;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.io.LineIterator;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+import org.broadinstitute.sting.utils.exceptions.StingException;
+import org.broadinstitute.sting.utils.exceptions.UserException;
+
+import java.io.*;
+import java.util.*;
+
+public class IOUtils {
+    private static Logger logger = Logger.getLogger(IOUtils.class);
+
+    /**
+     * Checks if the temp directory has been setup and throws an exception if they user hasn't set it correctly.
+     *
+     * @param tempDir Temporary directory.
+     */
+    public static void checkTempDir(File tempDir) {
+        String tempDirPath = tempDir.getAbsolutePath();
+        // Keeps the user from leaving the temp directory as the default, and on Macs from having pluses
+        // in the path which can cause problems with the Google Reflections library.
+        // see also: http://benjchristensen.com/2009/09/22/mac-osx-10-6-java-java-io-tmpdir/
+        if (tempDirPath.startsWith("/var/folders/") || (tempDirPath.equals("/tmp")) || (tempDirPath.equals("/tmp/")))
+            throw new UserException.BadTmpDir("java.io.tmpdir must be explicitly set");
+        if (!tempDir.exists() && !tempDir.mkdirs())
+            throw new UserException.BadTmpDir("Could not create directory: " + tempDir.getAbsolutePath());
+    }
+
+    /**
+     * Creates a temp directory with the prefix and optional suffix.
+     *
+     * @param prefix       Prefix for the directory name.
+     * @param suffix       Optional suffix for the directory name.
+     * @return The created temporary directory.
+     */
+    public static File tempDir(String prefix, String suffix) {
+        return tempDir(prefix, suffix, null);
+    }
+
+    /**
+     * Creates a temp directory with the prefix and optional suffix.
+     *
+     * @param prefix        Prefix for the directory name.
+     * @param suffix        Optional suffix for the directory name.
+     * @param tempDirParent Parent directory for the temp directory.
+     * @return The created temporary directory.
+     */
+    public static File tempDir(String prefix, String suffix, File tempDirParent) {
+        try {
+            if (tempDirParent == null)
+                tempDirParent = FileUtils.getTempDirectory();
+            if (!tempDirParent.exists() && !tempDirParent.mkdirs())
+                throw new UserException.BadTmpDir("Could not create temp directory: " + tempDirParent);
+            File temp = File.createTempFile(prefix + "-", suffix, tempDirParent);
+            if (!temp.delete())
+                throw new UserException.BadTmpDir("Could not delete sub file: " + temp.getAbsolutePath());
+            if (!temp.mkdir())
+                throw new UserException.BadTmpDir("Could not create sub directory: " + temp.getAbsolutePath());
+            return absolute(temp);
+        } catch (IOException e) {
+            throw new UserException.BadTmpDir(e.getMessage());
+        }
+    }
+
+    /**
+     * Writes content to a temp file and returns the path to the temporary file.
+     *
+     * @param content   to write.
+     * @param prefix    Prefix for the temp file.
+     * @param suffix    Suffix for the temp file.
+     * @param directory Directory for the temp file.
+     * @return the path to the temp file.
+     */
+    public static File writeTempFile(String content, String prefix, String suffix, File directory) {
+        try {
+            File tempFile = absolute(File.createTempFile(prefix, suffix, directory));
+            FileUtils.writeStringToFile(tempFile, content);
+            return tempFile;
+        } catch (IOException e) {
+            throw new UserException.BadTmpDir(e.getMessage());
+        }
+    }
+
+    /**
+     * Waits for NFS to propagate a file creation, imposing a timeout.
+     *
+     * Based on Apache Commons IO FileUtils.waitFor()
+     *
+     * @param file    The file to wait for.
+     * @param seconds The maximum time in seconds to wait.
+     * @return true if the file exists
+     */
+    public static boolean waitFor(File file, int seconds) {
+        return waitFor(Collections.singletonList(file), seconds).isEmpty();
+    }
+
+    /**
+     * Waits for NFS to propagate a file creation, imposing a timeout.
+     *
+     * Based on Apache Commons IO FileUtils.waitFor()
+     *
+     * @param files   The list of files to wait for.
+     * @param seconds The maximum time in seconds to wait.
+     * @return Files that still do not exists at the end of the timeout, or a empty list if all files exists.
+     */
+    public static List<File> waitFor(Collection<File> files, int seconds) {
+        long timeout = 0;
+        long tick = 0;
+        List<File> missingFiles = new ArrayList<File>();
+        for (File file : files)
+            if (!file.exists())
+                missingFiles.add(file);
+
+        while (!missingFiles.isEmpty() && timeout <= seconds) {
+            if (tick >= 10) {
+                tick = 0;
+                timeout++;
+            }
+            tick++;
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException ignore) {
+            }
+            List<File> newMissingFiles = new ArrayList<File>();
+            for (File file : missingFiles)
+                if (!file.exists())
+                    newMissingFiles.add(file);
+            missingFiles = newMissingFiles;
+        }
+        return missingFiles;
+    }
+
+    /**
+     * Returns the directory at the number of levels deep.
+     * For example 2 levels of /path/to/dir will return /path/to
+     *
+     * @param dir   Directory path.
+     * @param level how many levels deep from the root.
+     * @return The path to the parent directory that is level-levels deep.
+     */
+    public static File dirLevel(File dir, int level) {
+        List<File> directories = new ArrayList<File>();
+        File parentDir = absolute(dir);
+        while (parentDir != null) {
+            directories.add(0, parentDir);
+            parentDir = parentDir.getParentFile();
+        }
+        if (directories.size() <= level)
+            return directories.get(directories.size() - 1);
+        else
+            return directories.get(level);
+    }
+
+    /**
+     * Returns the sub path rooted at the parent.
+     *
+     * @param parent The parent directory.
+     * @param path   The sub path to append to the parent, if the path is not absolute.
+     * @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path.
+     */
+    public static File absolute(File parent, String path) {
+        return absolute(parent, new File(path));
+    }
+
+    /**
+     * Returns the sub path rooted at the parent.
+     *
+     * @param parent The parent directory.
+     * @param file   The sub path to append to the parent, if the path is not absolute.
+     * @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path.
+     */
+    public static File absolute(File parent, File file) {
+        String newPath;
+        if (file.isAbsolute())
+            newPath = absolutePath(file);
+        else
+            newPath = absolutePath(new File(parent, file.getPath()));
+        return replacePath(file, newPath);
+    }
+
+    /**
+     * A mix of getCanonicalFile and getAbsoluteFile that returns the
+     * absolute path to the file without deferencing symbolic links.
+     *
+     * @param file the file.
+     * @return the absolute path to the file.
+     */
+    public static File absolute(File file) {
+        return replacePath(file, absolutePath(file));
+    }
+
+    private static String absolutePath(File file) {
+        File fileAbs = file.getAbsoluteFile();
+        LinkedList<String> names = new LinkedList<String>();
+        while (fileAbs != null) {
+            String name = fileAbs.getName();
+            fileAbs = fileAbs.getParentFile();
+
+            if (".".equals(name)) {
+                /* skip */
+
+                /* TODO: What do we do for ".."?
+              } else if (name == "..") {
+
+                CentOS tcsh says use getCanonicalFile:
+                ~ $ mkdir -p test1/test2
+                ~ $ ln -s test1/test2 test3
+                ~ $ cd test3/..
+                ~/test1 $
+
+                Mac bash says keep going with getAbsoluteFile:
+                ~ $ mkdir -p test1/test2
+                ~ $ ln -s test1/test2 test3
+                ~ $ cd test3/..
+                ~ $
+
+                For now, leave it and let the shell figure it out.
+                */
+            } else {
+                names.add(0, name);
+            }
+        }
+
+        return ("/" + StringUtils.join(names, "/"));
+    }
+
+    private static File replacePath(File file, String path) {
+        if (file instanceof FileExtension)
+            return ((FileExtension)file).withPath(path);
+        if (!File.class.equals(file.getClass()))
+            throw new StingException("Sub classes of java.io.File must also implement FileExtension");
+        return new File(path);
+    }
+
+    /**
+     * Returns the last lines of the file.
+     * NOTE: This is only safe to run on smaller files!
+     *
+     * @param file  File to read.
+     * @param count Maximum number of lines to return.
+     * @return The last count lines from file.
+     * @throws IOException When unable to read the file.
+     */
+    public static List<String> tail(File file, int count) throws IOException {
+        LinkedList<String> tailLines = new LinkedList<String>();
+        FileReader reader = new FileReader(file);
+        try {
+            LineIterator iterator = org.apache.commons.io.IOUtils.lineIterator(reader);
+            int lineCount = 0;
+            while (iterator.hasNext()) {
+                String line = iterator.nextLine();
+                lineCount++;
+                if (lineCount > count)
+                    tailLines.removeFirst();
+                tailLines.offer(line);
+            }
+        } finally {
+            org.apache.commons.io.IOUtils.closeQuietly(reader);
+        }
+        return tailLines;
+    }
+
+    /**
+     * Tries to delete a file. Emits a warning if the file was unable to be deleted.
+     *
+     * @param file File to delete.
+     * @return true if the file was deleted.
+     */
+    public static boolean tryDelete(File file) {
+        boolean deleted = FileUtils.deleteQuietly(file);
+        if (deleted)
+            logger.debug("Deleted " + file);
+        else if (file.exists())
+            logger.warn("Unable to delete " + file);
+        return deleted;
+    }
+
+    /**
+     * Writes the an embedded resource to a temp file.
+     * File is not scheduled for deletion and must be cleaned up by the caller.
+     * @param resource Embedded resource.
+     * @return Path to the temp file with the contents of the resource.
+     */
+    public static File writeTempResource(Resource resource) {
+        File temp;
+        try {
+            temp = File.createTempFile(FilenameUtils.getBaseName(resource.getPath()) + ".", "." + FilenameUtils.getExtension(resource.getPath()));
+        } catch (IOException e) {
+            throw new UserException.BadTmpDir(e.getMessage());
+        }
+        writeResource(resource, temp);
+        return temp;
+    }
+
+    /**
+     * Writes the an embedded resource to a file.
+     * File is not scheduled for deletion and must be cleaned up by the caller.
+     * @param resource Embedded resource.
+     * @param file File path to write.
+     */
+    public static void writeResource(Resource resource, File file) {
+        String path = resource.getPath();
+        Class<?> clazz = resource.getRelativeClass();
+        InputStream inputStream = null;
+        OutputStream outputStream = null;
+        try {
+            if (clazz == null) {
+                inputStream = ClassLoader.getSystemResourceAsStream(path);
+                if (inputStream == null)
+                    throw new IllegalArgumentException("Resource not found: " + path);
+            } else {
+                inputStream = clazz.getResourceAsStream(path);
+                if (inputStream == null)
+                    throw new IllegalArgumentException("Resource not found relative to " + clazz + ": " + path);
+            }
+            outputStream = FileUtils.openOutputStream(file);
+            org.apache.commons.io.IOUtils.copy(inputStream, outputStream);
+        } catch (IOException e) {
+            throw new StingException(String.format("Unable to copy resource '%s' to '%s'", path, file), e);
+        } finally {
+            org.apache.commons.io.IOUtils.closeQuietly(inputStream);
+            org.apache.commons.io.IOUtils.closeQuietly(outputStream);
+        }
+    }
+}

public/java/src/org/broadinstitute/sting/utils/io/Resource.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.io;
+
+/**
+ * Stores a resource by path and a relative class.
+ */
+public class Resource {
+    private final String path;
+    private final Class<?> relativeClass;
+
+    /**
+     * Create a resource with a path and a relative class.
+     * @param path Relative or absolute path to the class.
+     * @param relativeClass Relative class to use as a class loader and for a relative package.
+     *
+     * If the relative class is null then the system classloader will be used and the path must be absolute.
+     */
+    public Resource(String path, Class<?> relativeClass) {
+        this.path = path;
+        this.relativeClass = relativeClass;
+    }
+
+    public Class<?> getRelativeClass() {
+        return relativeClass;
+    }
+
+    public String getPath() {
+        return path;
+    }
+}

public/java/src/org/broadinstitute/sting/utils/pileup/FragmentPileup.java

@@ -1,9 +1,10 @@
 package org.broadinstitute.sting.utils.pileup;
 
-import java.util.ArrayList;
+import net.sf.samtools.SAMRecord;
-import java.util.Collection;
+import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
-import java.util.HashMap;
+import org.broadinstitute.sting.utils.sam.ReadUtils;
-import java.util.Map;
+
+import java.util.*;
 
 /**
  * An easy to access fragment-based pileup, which contains two separate pileups.  The first
@@ -13,31 +14,51 @@ import java.util.Map;
  *
  * Based on the original code by E. Banks
  *
- * TODO -- technically we could generalize this code to support a pseudo-duplicate marking
+ * Oct 21: note that the order of the oneReadPileup and twoReadPileups are not
- * TODO -- algorithm that could collect all duplicates into a single super pileup element
+ * defined.  The algorithms that produce these lists are in fact producing
+ * lists of Pileup elements *NOT* sorted by alignment start position of the underlying
+ * reads.
  *
  * User: depristo
  * Date: 3/26/11
  * Time: 10:09 PM
  */
 public class FragmentPileup {
-    final Collection<PileupElement> oneReadPile;
+    Collection<PileupElement> oneReadPile = null;
-    final Collection<TwoReadPileupElement> twoReadPile = new ArrayList<TwoReadPileupElement>();
+    Collection<TwoReadPileupElement> twoReadPile = null;
+
+    protected enum FragmentMatchingAlgorithm {
+        ORIGINAL,
+        skipNonOverlapping,
+    }
 
     /**
      * Create a new Fragment-based pileup from the standard read-based pileup
      * @param pileup
      */
     public FragmentPileup(ReadBackedPileup pileup) {
-        Map<String, PileupElement> nameMap = new HashMap<String, PileupElement>();
+        skipNonOverlapping(pileup);
+    }
+
+    /** For performance testing only */
+    protected FragmentPileup(ReadBackedPileup pileup, FragmentMatchingAlgorithm algorithm) {
+        switch ( algorithm ) {
+            case ORIGINAL: oldSlowCalculation(pileup); break;
+            case skipNonOverlapping: skipNonOverlapping(pileup); break;
+        }
+    }
+
+    private final void oldSlowCalculation(final ReadBackedPileup pileup) {
+        final Map<String, PileupElement> nameMap = new HashMap<String, PileupElement>(pileup.size());
 
         // build an initial map, grabbing all of the multi-read fragments
-        for ( PileupElement p : pileup ) {
+        for ( final PileupElement p : pileup ) {
-            String readName = p.getRead().getReadName();
+            final String readName = p.getRead().getReadName();
 
-            PileupElement pe1 = nameMap.get(readName);
+            final PileupElement pe1 = nameMap.get(readName);
             if ( pe1 != null ) {
                 // assumes we have at most 2 reads per fragment
+                if ( twoReadPile == null ) twoReadPile = new ArrayList<TwoReadPileupElement>();
                 twoReadPile.add(new TwoReadPileupElement(pe1, p));
                 nameMap.remove(readName);
             } else {
@@ -45,17 +66,54 @@ public class FragmentPileup {
             }
         }
 
-        // now set the one Read pile to the values in the nameMap with only a single read
         oneReadPile = nameMap.values();
     }
 
+    private final void skipNonOverlapping(final ReadBackedPileup pileup) {
+        Map<String, PileupElement> nameMap = null;
+
+        // build an initial map, grabbing all of the multi-read fragments
+        for ( final PileupElement p : pileup ) {
+            final SAMRecord read = p.getRead();
+            final int mateStart = read.getMateAlignmentStart();
+
+            if ( mateStart == 0 || mateStart > read.getAlignmentEnd() ) {
+                // if we know that this read won't overlap its mate, or doesn't have one, jump out early
+                if ( oneReadPile == null ) oneReadPile = new ArrayList<PileupElement>(pileup.size()); // lazy init
+                oneReadPile.add(p);
+            } else {
+                // the read might overlap it's mate, or is the rightmost read of a pair
+                final String readName = p.getRead().getReadName();
+                final PileupElement pe1 = nameMap == null ? null : nameMap.get(readName);
+                if ( pe1 != null ) {
+                    // assumes we have at most 2 reads per fragment
+                    if ( twoReadPile == null ) twoReadPile = new ArrayList<TwoReadPileupElement>(); // lazy init
+                    twoReadPile.add(new TwoReadPileupElement(pe1, p));
+                    nameMap.remove(readName);
+                } else {
+                    if ( nameMap == null ) nameMap = new HashMap<String, PileupElement>(pileup.size()); // lazy init
+                    nameMap.put(readName, p);
+                }
+            }
+        }
+
+        // add all of the reads that are potentially overlapping but whose mate never showed
+        // up to the oneReadPile
+        if ( nameMap != null && ! nameMap.isEmpty() ) {
+            if ( oneReadPile == null )
+                oneReadPile = nameMap.values();
+            else
+                oneReadPile.addAll(nameMap.values());
+        }
+    }
+
     /**
      * Gets the pileup elements containing two reads, in no particular order
      *
      * @return
      */
     public Collection<TwoReadPileupElement> getTwoReadPileup() {
-        return twoReadPile;
+        return twoReadPile == null ? Collections.<TwoReadPileupElement>emptyList() : twoReadPile;
     }
 
     /**
@@ -64,7 +122,7 @@ public class FragmentPileup {
      * @return
      */
     public Collection<PileupElement> getOneReadPileup() {
-        return oneReadPile;
+        return oneReadPile == null ? Collections.<PileupElement>emptyList() : oneReadPile;
     }
 
     /**

public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java

@@ -4,6 +4,7 @@ import com.google.java.contract.Ensures;
 import com.google.java.contract.Requires;
 import net.sf.samtools.SAMRecord;
 import org.broadinstitute.sting.utils.BaseUtils;
+import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.sam.ReadUtils;
 
 /**
@@ -12,7 +13,7 @@ import org.broadinstitute.sting.utils.sam.ReadUtils;
  * Date: Apr 14, 2009
  * Time: 8:54:05 AM
  */
-public class PileupElement {
+public class PileupElement implements Comparable<PileupElement> {
     public static final byte DELETION_BASE = BaseUtils.D;
     public static final byte DELETION_QUAL = (byte) 16;
     public static final byte A_FOLLOWED_BY_INSERTION_BASE = (byte) 87;
@@ -75,6 +76,20 @@ public class PileupElement {
         return isDeletion() ? DELETION_QUAL : read.getBaseQualities()[offset];
     }
 
+    @Override
+    public int compareTo(final PileupElement pileupElement) {
+        if ( offset < pileupElement.offset )
+            return -1;
+        else if ( offset > pileupElement.offset )
+            return 1;
+        else if ( read.getAlignmentStart() < pileupElement.read.getAlignmentStart() )
+            return -1;
+        else if ( read.getAlignmentStart() > pileupElement.read.getAlignmentStart() )
+            return 1;
+        else
+            return 0;
+    }
+
     // --------------------------------------------------------------------------
     //
     // Reduced read accessors
@@ -82,16 +97,16 @@ public class PileupElement {
     // --------------------------------------------------------------------------
 
     public boolean isReducedRead() {
-        return ReadUtils.isReducedRead(getRead());
+        return ((GATKSAMRecord)read).isReducedRead();
     }
 
     public int getReducedCount() {
         if ( ! isReducedRead() ) throw new IllegalArgumentException("Cannot get reduced count for non-reduced read " + getRead().getReadName());
-        return ReadUtils.getReducedCount(getRead(), offset);
+        return ((GATKSAMRecord)read).getReducedCount(offset);
     }
 
     public byte getReducedQual() {
         if ( ! isReducedRead() ) throw new IllegalArgumentException("Cannot get reduced qual for non-reduced read " + getRead().getReadName());
-        return ReadUtils.getReducedQual(getRead(), offset);
+        return getQual();
     }
 }

public/java/src/org/broadinstitute/sting/utils/runtime/CapturedStreamOutput.java

@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.runtime;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.output.NullOutputStream;
+import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
+import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.broadinstitute.sting.utils.io.HardThresholdingOutputStream;
+
+import java.io.*;
+import java.util.EnumMap;
+
+/**
+ * Stream output captured from a stream.
+ */
+public class CapturedStreamOutput extends StreamOutput {
+    private final InputStream processStream;
+    private final EnumMap<StreamLocation, OutputStream> outputStreams = new EnumMap<StreamLocation, OutputStream>(StreamLocation.class);
+
+    /**
+     * The byte stream to capture content or null if no output string content was requested.
+     */
+    private final ByteArrayOutputStream bufferStream;
+
+    /**
+     * True if the buffer is truncated.
+     */
+    private boolean bufferTruncated = false;
+
+    /**
+     * @param settings       Settings that define what to capture.
+     * @param processStream  Stream to capture output.
+     * @param standardStream Stream to write debug output.
+     */
+    public CapturedStreamOutput(OutputStreamSettings settings, InputStream processStream, PrintStream standardStream) {
+        this.processStream = processStream;
+        int bufferSize = settings.getBufferSize();
+        this.bufferStream = (bufferSize < 0) ? new ByteArrayOutputStream() : new ByteArrayOutputStream(bufferSize);
+
+        for (StreamLocation location : settings.getStreamLocations()) {
+            OutputStream outputStream;
+            switch (location) {
+                case Buffer:
+                    if (bufferSize < 0) {
+                        outputStream = this.bufferStream;
+                    } else {
+                        outputStream = new HardThresholdingOutputStream(bufferSize) {
+                            @Override
+                            protected OutputStream getStream() throws IOException {
+                                return bufferTruncated ? NullOutputStream.NULL_OUTPUT_STREAM : bufferStream;
+                            }
+
+                            @Override
+                            protected void thresholdReached() throws IOException {
+                                bufferTruncated = true;
+                            }
+                        };
+                    }
+                    break;
+                case File:
+                    try {
+                        outputStream = new FileOutputStream(settings.getOutputFile(), settings.isAppendFile());
+                    } catch (IOException e) {
+                        throw new UserException.BadInput(e.getMessage());
+                    }
+                    break;
+                case Standard:
+                    outputStream = standardStream;
+                    break;
+                default:
+                    throw new ReviewedStingException("Unexpected stream location: " + location);
+            }
+            this.outputStreams.put(location, outputStream);
+        }
+    }
+
+    @Override
+    public byte[] getBufferBytes() {
+        return bufferStream.toByteArray();
+    }
+
+    @Override
+    public boolean isBufferTruncated() {
+        return bufferTruncated;
+    }
+
+    /**
+     * Drain the input stream to keep the process from backing up until it's empty.
+     * File streams will be closed automatically when this method returns.
+     *
+     * @throws java.io.IOException When unable to read or write.
+     */
+    public void readAndClose() throws IOException {
+        try {
+            byte[] buf = new byte[4096];
+            int readCount;
+            while ((readCount = processStream.read(buf)) >= 0)
+                for (OutputStream outputStream : this.outputStreams.values()) {
+                    outputStream.write(buf, 0, readCount);
+                }
+        } finally {
+            for (StreamLocation location : this.outputStreams.keySet()) {
+                OutputStream outputStream = this.outputStreams.get(location);
+                outputStream.flush();
+                if (location != StreamLocation.Standard)
+                    IOUtils.closeQuietly(outputStream);
+            }
+        }
+    }
+}

public/java/src/org/broadinstitute/sting/utils/runtime/InputStreamSettings.java

@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.runtime;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Set;
+
+/**
+ * Settings that define text to write to the process stdin.
+ */
+public class InputStreamSettings {
+    private final EnumSet<StreamLocation> streamLocations = EnumSet.noneOf(StreamLocation.class);
+    private byte[] inputBuffer;
+    private File inputFile;
+
+    public InputStreamSettings() {
+    }
+
+    /**
+     * @param inputBuffer String to write to stdin.
+     */
+    public InputStreamSettings(String inputBuffer) {
+        setInputBuffer(inputBuffer);
+    }
+
+    /**
+     * @param inputFile File to write to stdin.
+     */
+    public InputStreamSettings(File inputFile) {
+        setInputFile(inputFile);
+    }
+
+    /**
+     * @param inputBuffer String to write to stdin.
+     * @param inputFile   File to write to stdin.
+     */
+    public InputStreamSettings(byte[] inputBuffer, File inputFile) {
+        setInputBuffer(inputBuffer);
+        setInputFile(inputFile);
+    }
+
+    public Set<StreamLocation> getStreamLocations() {
+        return Collections.unmodifiableSet(streamLocations);
+    }
+
+    public byte[] getInputBuffer() {
+        return inputBuffer;
+    }
+
+    public void setInputBuffer(String inputBuffer) {
+        if (inputBuffer == null)
+            throw new IllegalArgumentException("inputBuffer cannot be null");
+        this.streamLocations.add(StreamLocation.Buffer);
+        this.inputBuffer = inputBuffer.getBytes();
+    }
+
+    public void setInputBuffer(byte[] inputBuffer) {
+        if (inputBuffer == null)
+            throw new IllegalArgumentException("inputBuffer cannot be null");
+        this.streamLocations.add(StreamLocation.Buffer);
+        this.inputBuffer = inputBuffer;
+    }
+
+    public void clearInputBuffer() {
+        this.streamLocations.remove(StreamLocation.Buffer);
+        this.inputBuffer = null;
+    }
+
+    public File getInputFile() {
+        return inputFile;
+    }
+
+    public void setInputFile(File inputFile) {
+        if (inputFile == null)
+            throw new IllegalArgumentException("inputFile cannot be null");
+        this.streamLocations.add(StreamLocation.File);
+        this.inputFile = inputFile;
+    }
+
+    public void clearInputFile() {
+        this.streamLocations.remove(StreamLocation.File);
+        this.inputFile = null;
+    }
+
+    public void setInputStandard(boolean inputStandard) {
+        if (inputStandard)
+            this.streamLocations.add(StreamLocation.Standard);
+        else
+            this.streamLocations.remove(StreamLocation.Standard);
+    }
+}

public/java/src/org/broadinstitute/sting/utils/runtime/OutputStreamSettings.java

@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.runtime;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Set;
+
+/**
+ * Settings that define text to capture from a process stream.
+ */
+public class OutputStreamSettings {
+    private final EnumSet<StreamLocation> streamLocations = EnumSet.noneOf(StreamLocation.class);
+    private int bufferSize;
+    private File outputFile;
+    private boolean appendFile;
+
+    public OutputStreamSettings() {
+    }
+
+    /**
+     * @param bufferSize The number of bytes to capture, or -1 for unlimited.
+     */
+    public OutputStreamSettings(int bufferSize) {
+        setBufferSize(bufferSize);
+    }
+
+    /**
+     * @param outputFile The file to write output to.
+     */
+    public OutputStreamSettings(File outputFile) {
+        setOutputFile(outputFile);
+    }
+
+    /**
+     * @param outputFile The file to write output to.
+     * @param append     true if the output file should be appended to.
+     */
+    public OutputStreamSettings(File outputFile, boolean append) {
+        setOutputFile(outputFile, append);
+    }
+
+    public OutputStreamSettings(int bufferSize, File outputFile, boolean appendFile) {
+        setBufferSize(bufferSize);
+        setOutputFile(outputFile, appendFile);
+    }
+
+    public Set<StreamLocation> getStreamLocations() {
+        return Collections.unmodifiableSet(streamLocations);
+    }
+
+    public int getBufferSize() {
+        return bufferSize;
+    }
+
+    public void setBufferSize(int bufferSize) {
+        this.streamLocations.add(StreamLocation.Buffer);
+        this.bufferSize = bufferSize;
+    }
+
+    public void clearBufferSize() {
+        this.streamLocations.remove(StreamLocation.Buffer);
+        this.bufferSize = 0;
+    }
+
+    public File getOutputFile() {
+        return outputFile;
+    }
+
+    public boolean isAppendFile() {
+        return appendFile;
+    }
+
+    /**
+     * Overwrites the outputFile with the process output.
+     *
+     * @param outputFile File to overwrite.
+     */
+    public void setOutputFile(File outputFile) {
+        setOutputFile(outputFile, false);
+    }
+
+    public void setOutputFile(File outputFile, boolean append) {
+        if (outputFile == null)
+            throw new IllegalArgumentException("outputFile cannot be null");
+        streamLocations.add(StreamLocation.File);
+        this.outputFile = outputFile;
+        this.appendFile = append;
+    }
+
+    public void clearOutputFile() {
+        streamLocations.remove(StreamLocation.File);
+        this.outputFile = null;
+        this.appendFile = false;
+    }
+
+    public void printStandard(boolean print) {
+        if (print)
+            this.streamLocations.add(StreamLocation.Standard);
+        else
+            this.streamLocations.remove(StreamLocation.Standard);
+    }
+}

public/java/src/org/broadinstitute/sting/utils/runtime/ProcessController.java

@@ -0,0 +1,363 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.runtime;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
+import org.broadinstitute.sting.utils.exceptions.UserException;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.*;
+
+/**
+ * Facade to Runtime.exec() and java.lang.Process.  Handles
+ * running a process to completion and returns stdout and stderr
+ * as strings.  Creates separate threads for reading stdout and stderr,
+ * then reuses those threads for each process most efficient use is
+ * to create one of these and use it repeatedly.  Instances are not
+ * thread-safe, however.
+ *
+ * TODO: java.io sometimes zombies the backround threads locking up on read().
+ * Supposedly NIO has better ways of interrupting a blocked stream but will
+ * require a little bit of refactoring.
+ *
+ * @author Michael Koehrsen
+ * @author Khalid Shakir
+ */
+public class ProcessController {
+    private static Logger logger = Logger.getLogger(ProcessController.class);
+
+    private static enum ProcessStream {Stdout, Stderr}
+
+    // Tracks running processes.
+    private static final Set<ProcessController> running = Collections.synchronizedSet(new HashSet<ProcessController>());
+
+    // Tracks this running process.
+    private Process process;
+
+    // Threads that capture stdout and stderr
+    private final OutputCapture stdoutCapture;
+    private final OutputCapture stderrCapture;
+
+    // When a caller destroyes a controller a new thread local version will be created
+    private boolean destroyed = false;
+
+    // Communication channels with output capture threads
+
+    // Holds the stdout and stderr sent to the background capture threads
+    private final Map<ProcessStream, CapturedStreamOutput> toCapture =
+            new EnumMap<ProcessStream, CapturedStreamOutput>(ProcessStream.class);
+
+    // Holds the results of the capture from the background capture threads.
+    // May be the content via toCapture or an StreamOutput.EMPTY if the capture was interrupted.
+    private final Map<ProcessStream, StreamOutput> fromCapture =
+            new EnumMap<ProcessStream, StreamOutput>(ProcessStream.class);
+
+    // Useful for debugging if background threads have shut down correctly
+    private static int nextControllerId = 0;
+    private final int controllerId;
+
+    public ProcessController() {
+        // Start the background threads for this controller.
+        synchronized (running) {
+            controllerId = nextControllerId++;
+        }
+        stdoutCapture = new OutputCapture(ProcessStream.Stdout, controllerId);
+        stderrCapture = new OutputCapture(ProcessStream.Stderr, controllerId);
+        stdoutCapture.start();
+        stderrCapture.start();
+    }
+
+    /**
+     * Returns a thread local ProcessController.
+     * Should NOT be closed when finished so it can be reused by the thread.
+     *
+     * @return a thread local ProcessController.
+     */
+    public static ProcessController getThreadLocal() {
+        // If the local controller was destroyed get a fresh instance.
+        if (threadProcessController.get().destroyed)
+            threadProcessController.remove();
+        return threadProcessController.get();
+    }
+
+    /**
+     * Thread local process controller container.
+     */
+    private static final ThreadLocal<ProcessController> threadProcessController =
+            new ThreadLocal<ProcessController>() {
+                @Override
+                protected ProcessController initialValue() {
+                    return new ProcessController();
+                }
+            };
+
+    /**
+     * Similar to Runtime.exec() but drains the output and error streams.
+     *
+     * @param command Command to run.
+     * @return The result code.
+     */
+    public static int exec(String[] command) {
+        ProcessController controller = ProcessController.getThreadLocal();
+        return controller.exec(new ProcessSettings(command)).getExitValue();
+    }
+
+    /**
+     * Executes a command line program with the settings and waits for it to return,
+     * processing the output on a background thread.
+     *
+     * @param settings Settings to be run.
+     * @return The output of the command.
+     */
+    public ProcessOutput exec(ProcessSettings settings) {
+        if (destroyed)
+            throw new IllegalStateException("This controller was destroyed");
+
+        ProcessBuilder builder = new ProcessBuilder(settings.getCommand());
+        builder.directory(settings.getDirectory());
+
+        Map<String, String> settingsEnvironment = settings.getEnvironment();
+        if (settingsEnvironment != null) {
+            Map<String, String> builderEnvironment = builder.environment();
+            builderEnvironment.clear();
+            builderEnvironment.putAll(settingsEnvironment);
+        }
+
+        builder.redirectErrorStream(settings.isRedirectErrorStream());
+
+        StreamOutput stdout = null;
+        StreamOutput stderr = null;
+
+        // Start the process running.
+
+        try {
+            synchronized (toCapture) {
+                process = builder.start();
+            }
+            running.add(this);
+        } catch (IOException e) {
+            throw new ReviewedStingException("Unable to start command: " + StringUtils.join(builder.command(), " "));
+        }
+
+        int exitCode;
+
+        try {
+            // Notify the background threads to start capturing.
+            synchronized (toCapture) {
+                toCapture.put(ProcessStream.Stdout,
+                        new CapturedStreamOutput(settings.getStdoutSettings(), process.getInputStream(), System.out));
+                toCapture.put(ProcessStream.Stderr,
+                        new CapturedStreamOutput(settings.getStderrSettings(), process.getErrorStream(), System.err));
+                toCapture.notifyAll();
+            }
+
+            // Write stdin content
+            InputStreamSettings stdinSettings = settings.getStdinSettings();
+            Set<StreamLocation> streamLocations = stdinSettings.getStreamLocations();
+            if (!streamLocations.isEmpty()) {
+                try {
+                    OutputStream stdinStream = process.getOutputStream();
+                    for (StreamLocation location : streamLocations) {
+                        InputStream inputStream;
+                        switch (location) {
+                            case Buffer:
+                                inputStream = new ByteArrayInputStream(stdinSettings.getInputBuffer());
+                                break;
+                            case File:
+                                try {
+                                    inputStream = FileUtils.openInputStream(stdinSettings.getInputFile());
+                                } catch (IOException e) {
+                                    throw new UserException.BadInput(e.getMessage());
+                                }
+                                break;
+                            case Standard:
+                                inputStream = System.in;
+                                break;
+                            default:
+                                throw new ReviewedStingException("Unexpected stream location: " + location);
+                        }
+                        try {
+                            IOUtils.copy(inputStream, stdinStream);
+                        } finally {
+                            if (location != StreamLocation.Standard)
+                                IOUtils.closeQuietly(inputStream);
+                        }
+                    }
+                    stdinStream.flush();
+                } catch (IOException e) {
+                    throw new ReviewedStingException("Error writing to stdin on command: " + StringUtils.join(builder.command(), " "), e);
+                }
+            }
+
+            // Wait for the process to complete.
+            try {
+                process.getOutputStream().close();
+                process.waitFor();
+            } catch (IOException e) {
+                throw new ReviewedStingException("Unable to close stdin on command: " + StringUtils.join(builder.command(), " "), e);
+            } catch (InterruptedException e) {
+                throw new ReviewedStingException("Process interrupted", e);
+            } finally {
+                while (!destroyed && stdout == null || stderr == null) {
+                    synchronized (fromCapture) {
+                        if (fromCapture.containsKey(ProcessStream.Stdout))
+                            stdout = fromCapture.remove(ProcessStream.Stdout);
+                        if (fromCapture.containsKey(ProcessStream.Stderr))
+                            stderr = fromCapture.remove(ProcessStream.Stderr);
+                        try {
+                            if (stdout == null || stderr == null)
+                                fromCapture.wait();
+                        } catch (InterruptedException e) {
+                            // Log the error, ignore the interrupt and wait patiently
+                            // for the OutputCaptures to (via finally) return their
+                            // stdout and stderr.
+                            logger.error(e);
+                        }
+                    }
+                }
+
+                if (destroyed) {
+                    if (stdout == null)
+                        stdout = StreamOutput.EMPTY;
+                    if (stderr == null)
+                        stderr = StreamOutput.EMPTY;
+                }
+            }
+        } finally {
+            synchronized (toCapture) {
+                exitCode = process.exitValue();
+                process = null;
+            }
+            running.remove(this);
+        }
+
+        return new ProcessOutput(exitCode, stdout, stderr);
+    }
+
+    /**
+     * @return The set of still running processes.
+     */
+    public static Set<ProcessController> getRunning() {
+        synchronized (running) {
+            return new HashSet<ProcessController>(running);
+        }
+    }
+
+    /**
+     * Stops the process from running and tries to ensure process is cleaned up properly.
+     * NOTE: sub-processes started by process may be zombied with their parents set to pid 1.
+     * NOTE: capture threads may block on read.
+     * TODO: Try to use NIO to interrupt streams.
+     */
+    public void tryDestroy() {
+        destroyed = true;
+        synchronized (toCapture) {
+            if (process != null) {
+                process.destroy();
+                IOUtils.closeQuietly(process.getInputStream());
+                IOUtils.closeQuietly(process.getErrorStream());
+            }
+            stdoutCapture.interrupt();
+            stderrCapture.interrupt();
+            toCapture.notifyAll();
+        }
+    }
+
+    @Override
+    protected void finalize() throws Throwable {
+        try {
+            tryDestroy();
+        } catch (Exception e) {
+            logger.error(e);
+        }
+        super.finalize();
+    }
+
+    private class OutputCapture extends Thread {
+        private final int controllerId;
+        private final ProcessStream key;
+
+        /**
+         * Reads in the output of a stream on a background thread to keep the output pipe from backing up and freezing the called process.
+         *
+         * @param key The stdout or stderr key for this output capture.
+         * @param controllerId Unique id of the controller.
+         */
+        public OutputCapture(ProcessStream key, int controllerId) {
+            super(String.format("OutputCapture-%d-%s-%s-%d", controllerId, key.name().toLowerCase(),
+                    Thread.currentThread().getName(), Thread.currentThread().getId()));
+            this.controllerId = controllerId;
+            this.key = key;
+            setDaemon(true);
+        }
+
+        /**
+         * Runs the capture.
+         */
+        @Override
+        public void run() {
+            while (!destroyed) {
+                StreamOutput processStream = StreamOutput.EMPTY;
+                try {
+                    // Wait for a new input stream to be passed from this process controller.
+                    CapturedStreamOutput capturedProcessStream = null;
+                    while (!destroyed && capturedProcessStream == null) {
+                        synchronized (toCapture) {
+                            if (toCapture.containsKey(key)) {
+                                capturedProcessStream = toCapture.remove(key);
+                            } else {
+                                toCapture.wait();
+                            }
+                        }
+                    }
+
+                    if (!destroyed) {
+                        // Read in the input stream
+                        processStream = capturedProcessStream;
+                        capturedProcessStream.readAndClose();
+                    }
+                } catch (InterruptedException e) {
+                    logger.info("OutputCapture interrupted, exiting");
+                    break;
+                } catch (IOException e) {
+                    logger.error("Error reading process output", e);
+                } finally {
+                    // Send the string back to the process controller.
+                    synchronized (fromCapture) {
+                        fromCapture.put(key, processStream);
+                        fromCapture.notify();
+                    }
+                }
+            }
+        }
+    }
+}

public/java/src/org/broadinstitute/sting/utils/runtime/ProcessOutput.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.runtime;
+
+public class ProcessOutput {
+    private final int exitValue;
+    private final StreamOutput stdout;
+    private final StreamOutput stderr;
+
+    /**
+     * The output of a process.
+     *
+     * @param exitValue The exit value.
+     * @param stdout    The capture of stdout as defined by the stdout OutputStreamSettings.
+     * @param stderr    The capture of stderr as defined by the stderr OutputStreamSettings.
+     */
+    public ProcessOutput(int exitValue, StreamOutput stdout, StreamOutput stderr) {
+        this.exitValue = exitValue;
+        this.stdout = stdout;
+        this.stderr = stderr;
+    }
+
+    public int getExitValue() {
+        return exitValue;
+    }
+
+    public StreamOutput getStdout() {
+        return stdout;
+    }
+
+    public StreamOutput getStderr() {
+        return stderr;
+    }
+}

public/java/src/org/broadinstitute/sting/utils/runtime/ProcessSettings.java

@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.runtime;
+
+import com.sun.corba.se.spi.orbutil.fsm.Input;
+
+import java.io.File;
+import java.util.Map;
+
+public class ProcessSettings {
+    private String[] command;
+    private Map<String, String> environment;
+    private File directory;
+    private boolean redirectErrorStream;
+    private InputStreamSettings stdinSettings;
+    private OutputStreamSettings stdoutSettings;
+    private OutputStreamSettings stderrSettings;
+
+    /**
+     * @param command Command line to run.
+     */
+    public ProcessSettings(String[] command) {
+        this(command, false, null, null, null, null, null);
+    }
+
+    /**
+     * @param command             Command line to run.
+     * @param redirectErrorStream true if stderr should be sent to stdout.
+     * @param environment         Environment settings to override System.getEnv, or null to use System.getEnv.
+     * @param directory           The directory to run the command in, or null to run in the current directory.
+     * @param stdinSettings       Settings for writing to the process stdin.
+     * @param stdoutSettings      Settings for capturing the process stdout.
+     * @param stderrSettings      Setting for capturing the process stderr.
+     */
+    public ProcessSettings(String[] command, boolean redirectErrorStream, File directory, Map<String, String> environment,
+                           InputStreamSettings stdinSettings, OutputStreamSettings stdoutSettings, OutputStreamSettings stderrSettings) {
+        this.command = checkCommand(command);
+        this.redirectErrorStream = redirectErrorStream;
+        this.directory = directory;
+        this.environment = environment;
+        this.stdinSettings = checkSettings(stdinSettings);
+        this.stdoutSettings = checkSettings(stdoutSettings);
+        this.stderrSettings = checkSettings(stderrSettings);
+    }
+
+    public String[] getCommand() {
+        return command;
+    }
+
+    public void setCommand(String[] command) {
+        this.command = checkCommand(command);
+    }
+
+    public boolean isRedirectErrorStream() {
+        return redirectErrorStream;
+    }
+
+    public void setRedirectErrorStream(boolean redirectErrorStream) {
+        this.redirectErrorStream = redirectErrorStream;
+    }
+
+    public File getDirectory() {
+        return directory;
+    }
+
+    public void setDirectory(File directory) {
+        this.directory = directory;
+    }
+
+    public Map<String, String> getEnvironment() {
+        return environment;
+    }
+
+    public void setEnvironment(Map<String, String> environment) {
+        this.environment = environment;
+    }
+
+    public InputStreamSettings getStdinSettings() {
+        return stdinSettings;
+    }
+
+    public void setStdinSettings(InputStreamSettings stdinSettings) {
+        this.stdinSettings = checkSettings(stdinSettings);
+    }
+
+    public OutputStreamSettings getStdoutSettings() {
+        return stdoutSettings;
+    }
+
+    public void setStdoutSettings(OutputStreamSettings stdoutSettings) {
+        this.stdoutSettings = checkSettings(stdoutSettings);
+    }
+
+    public OutputStreamSettings getStderrSettings() {
+        return stderrSettings;
+    }
+
+    public void setStderrSettings(OutputStreamSettings stderrSettings) {
+        this.stderrSettings = checkSettings(stderrSettings);
+    }
+
+    protected String[] checkCommand(String[] command) {
+        if (command == null)
+            throw new IllegalArgumentException("Command is not allowed to be null");
+        for (String s: command)
+            if (s == null)
+                throw new IllegalArgumentException("Command is not allowed to contain nulls");
+        return command;
+    }
+
+    protected InputStreamSettings checkSettings(InputStreamSettings settings) {
+        return settings == null ? new InputStreamSettings() : settings;
+    }
+
+    protected OutputStreamSettings checkSettings(OutputStreamSettings settings) {
+        return settings == null ? new OutputStreamSettings() : settings;
+    }
+}

public/java/src/org/broadinstitute/sting/utils/runtime/StreamLocation.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.runtime;
+
+/**
+ * Where to read/write a stream
+ */
+public enum StreamLocation {
+    Buffer, File, Standard
+}

public/java/src/org/broadinstitute/sting/utils/runtime/StreamOutput.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.runtime;
+
+/**
+ * The content of stdout or stderr.
+ */
+public abstract class StreamOutput {
+    /**
+     * Empty stream output when no output is captured due to an error.
+     */
+    public static final StreamOutput EMPTY = new StreamOutput() {
+        @Override
+        public byte[] getBufferBytes() {
+            return new byte[0];
+        }
+
+        @Override
+        public boolean isBufferTruncated() {
+            return false;
+        }
+    };
+
+    /**
+     * Returns the content as a string.
+     *
+     * @return The content as a string.
+     */
+    public String getBufferString() {
+        return new String(getBufferBytes());
+    }
+
+    /**
+     * Returns the content as a string.
+     *
+     * @return The content as a string.
+     */
+    public abstract byte[] getBufferBytes();
+
+    /**
+     * Returns true if the buffer was truncated.
+     *
+     * @return true if the buffer was truncated.
+     */
+    public abstract boolean isBufferTruncated();
+}

public/java/src/org/broadinstitute/sting/utils/sam/ArtificialSAMUtils.java

@@ -2,11 +2,15 @@ package org.broadinstitute.sting.utils.sam;
 
 import net.sf.samtools.*;
 import org.broadinstitute.sting.gatk.iterators.StingSAMIterator;
+import org.broadinstitute.sting.utils.GenomeLoc;
+import org.broadinstitute.sting.utils.GenomeLocParser;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
+import org.broadinstitute.sting.utils.pileup.PileupElement;
+import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
+import org.broadinstitute.sting.utils.pileup.ReadBackedPileupImpl;
 
 import java.io.File;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.List;
 
 /**
  * @author aaron
@@ -29,7 +33,7 @@ public class ArtificialSAMUtils {
         File outFile = new File(filename);
 
         SAMFileWriter out = new SAMFileWriterFactory().makeBAMWriter(header, true, outFile);
-        
+
         for (int x = startingChromosome; x < startingChromosome + numberOfChromosomes; x++) {
             for (int readNumber = 1; readNumber < readsPerChomosome; readNumber++) {
                 out.addAlignment(createArtificialRead(header, "Read_" + readNumber, x - startingChromosome, readNumber, DEFAULT_READ_LENGTH));
@@ -134,6 +138,7 @@ public class ArtificialSAMUtils {
     /**
      * Create an artificial read based on the parameters.  The cigar string will be *M, where * is the length of the read
      *
+     *
      * @param header         the SAM header to associate the read with
      * @param name           the name of the read
      * @param refIndex       the reference index, i.e. what chromosome to associate it with
@@ -142,11 +147,11 @@ public class ArtificialSAMUtils {
      *
      * @return the artificial read
      */
-    public static SAMRecord createArtificialRead( SAMFileHeader header, String name, int refIndex, int alignmentStart, int length ) {
+    public static GATKSAMRecord createArtificialRead(SAMFileHeader header, String name, int refIndex, int alignmentStart, int length) {
         if( (refIndex == SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX && alignmentStart != SAMRecord.NO_ALIGNMENT_START) ||
-            (refIndex != SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX && alignmentStart == SAMRecord.NO_ALIGNMENT_START) )
+                (refIndex != SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX && alignmentStart == SAMRecord.NO_ALIGNMENT_START) )
             throw new ReviewedStingException("Invalid alignment start for artificial read, start = " + alignmentStart);
-        SAMRecord record = new SAMRecord(header);
+        GATKSAMRecord record = new GATKSAMRecord(header);
         record.setReadName(name);
         record.setReferenceIndex(refIndex);
         record.setAlignmentStart(alignmentStart);
@@ -166,6 +171,7 @@ public class ArtificialSAMUtils {
         if (refIndex == SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX) {
             record.setReadUnmappedFlag(true);
         }
+
         return record;
     }
 
@@ -181,19 +187,51 @@ public class ArtificialSAMUtils {
      *
      * @return the artificial read
      */
-    public static SAMRecord createArtificialRead( SAMFileHeader header, String name, int refIndex, int alignmentStart, byte[] bases, byte[] qual ) {
+    public static GATKSAMRecord createArtificialRead( SAMFileHeader header, String name, int refIndex, int alignmentStart, byte[] bases, byte[] qual ) {
         if (bases.length != qual.length) {
             throw new ReviewedStingException("Passed in read string is different length then the quality array");
         }
-        SAMRecord rec = createArtificialRead(header, name, refIndex, alignmentStart, bases.length);
+        GATKSAMRecord rec = createArtificialRead(header, name, refIndex, alignmentStart, bases.length);
         rec.setReadBases(bases);
         rec.setBaseQualities(qual);
         if (refIndex == -1) {
             rec.setReadUnmappedFlag(true);
         }
+
         return rec;
     }
 
+    public final static List<SAMRecord> createPair(SAMFileHeader header, String name, int readLen, int leftStart, int rightStart, boolean leftIsFirst, boolean leftIsNegative) {
+        SAMRecord left = ArtificialSAMUtils.createArtificialRead(header, name, 0, leftStart, readLen);
+        SAMRecord right = ArtificialSAMUtils.createArtificialRead(header, name, 0, rightStart, readLen);
+
+        left.setReadPairedFlag(true);
+        right.setReadPairedFlag(true);
+
+        left.setProperPairFlag(true);
+        right.setProperPairFlag(true);
+
+        left.setFirstOfPairFlag(leftIsFirst);
+        right.setFirstOfPairFlag(! leftIsFirst);
+
+        left.setReadNegativeStrandFlag(leftIsNegative);
+        left.setMateNegativeStrandFlag(!leftIsNegative);
+        right.setReadNegativeStrandFlag(!leftIsNegative);
+        right.setMateNegativeStrandFlag(leftIsNegative);
+
+        left.setMateAlignmentStart(right.getAlignmentStart());
+        right.setMateAlignmentStart(left.getAlignmentStart());
+
+        left.setMateReferenceIndex(0);
+        right.setMateReferenceIndex(0);
+
+        int isize = rightStart + readLen - leftStart;
+        left.setInferredInsertSize(isize);
+        right.setInferredInsertSize(-isize);
+
+        return Arrays.asList(left, right);
+    }
+
     /**
      * create an iterator containing the specified read piles
      *
@@ -255,4 +293,52 @@ public class ArtificialSAMUtils {
 
         return new ArtificialSAMQueryIterator(startingChr, endingChr, readCount, unmappedReadCount, header);
     }
+
+    private final static int ranIntInclusive(Random ran, int start, int stop) {
+        final int range = stop - start;
+        return ran.nextInt(range) + start;
+    }
+
+    /**
+     * Creates a read backed pileup containing up to pileupSize reads at refID 0 from header at loc with
+     * reads created that have readLen bases.  Pairs are sampled from a gaussian distribution with mean insert
+     * size of insertSize and variation of insertSize / 10.  The first read will be in the pileup, and the second
+     * may be, depending on where this sampled insertSize puts it.
+     * @param header
+     * @param loc
+     * @param readLen
+     * @param insertSize
+     * @param pileupSize
+     * @return
+     */
+    public static ReadBackedPileup createReadBackedPileup(final SAMFileHeader header, final GenomeLoc loc, final int readLen, final int insertSize, final int pileupSize) {
+        final Random ran = new Random();
+        final boolean leftIsFirst = true;
+        final boolean leftIsNegative = false;
+        final int insertSizeVariation = insertSize / 10;
+        final int pos = loc.getStart();
+
+        final List<PileupElement> pileupElements = new ArrayList<PileupElement>();
+        for ( int i = 0; i < pileupSize / 2; i++ ) {
+            final String readName = "read" + i;
+            final int leftStart = ranIntInclusive(ran, 1, pos);
+            final int fragmentSize = (int)(ran.nextGaussian() * insertSizeVariation + insertSize);
+            final int rightStart = leftStart + fragmentSize - readLen;
+
+            if ( rightStart <= 0 ) continue;
+
+            List<SAMRecord> pair = createPair(header, readName, readLen, leftStart, rightStart, leftIsFirst, leftIsNegative);
+            final SAMRecord left = pair.get(0);
+            final SAMRecord right = pair.get(1);
+
+            pileupElements.add(new PileupElement(left, pos - leftStart));
+
+            if ( pos >= right.getAlignmentStart() && pos <= right.getAlignmentEnd() ) {
+                pileupElements.add(new PileupElement(right, pos - rightStart));
+            }
+        }
+
+        Collections.sort(pileupElements);
+        return new ReadBackedPileupImpl(loc, pileupElements);
+    }
 }

public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMReadGroupRecord.java

@@ -1,6 +1,7 @@
 package org.broadinstitute.sting.utils.sam;
 
 import net.sf.samtools.SAMReadGroupRecord;
+import org.broadinstitute.sting.utils.NGSPlatform;
 
 /**
  * @author ebanks
@@ -15,16 +16,28 @@ public class GATKSAMReadGroupRecord extends SAMReadGroupRecord {
     // the SAMReadGroupRecord data we're caching
     private String mSample = null;
     private String mPlatform = null;
+    private NGSPlatform mNGSPlatform = null;
 
     // because some values can be null, we don't want to duplicate effort
     private boolean retrievedSample = false;
     private boolean retrievedPlatform = false;
+    private boolean retrievedNGSPlatform = false;
 
+    public GATKSAMReadGroupRecord(final String id) {
+        super(id);
+    }
 
     public GATKSAMReadGroupRecord(SAMReadGroupRecord record) {
         super(record.getReadGroupId(), record);
     }
 
+    public GATKSAMReadGroupRecord(SAMReadGroupRecord record, NGSPlatform pl) {
+        super(record.getReadGroupId(), record);
+        setPlatform(pl.getDefaultPlatform());
+        mNGSPlatform = pl;
+        retrievedPlatform = retrievedNGSPlatform = true;
+    }
+
     ///////////////////////////////////////////////////////////////////////////////
     // *** The following methods are overloaded to cache the appropriate data ***//
     ///////////////////////////////////////////////////////////////////////////////
@@ -55,5 +68,15 @@ public class GATKSAMReadGroupRecord extends SAMReadGroupRecord {
         super.setPlatform(s);
         mPlatform = s;
         retrievedPlatform = true;
+        retrievedNGSPlatform = false;  // recalculate the NGSPlatform
+    }
+
+    public NGSPlatform getNGSPlatform() {
+        if ( ! retrievedNGSPlatform ) {
+            mNGSPlatform = NGSPlatform.fromReadGroupPL(getPlatform());
+            retrievedNGSPlatform = true;
+        }
+
+        return mNGSPlatform;
     }
 }

public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java

@@ -1,49 +1,56 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
 package org.broadinstitute.sting.utils.sam;
 
 import net.sf.samtools.*;
-import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
+import org.broadinstitute.sting.utils.NGSPlatform;
-import org.broadinstitute.sting.utils.exceptions.UserException;
 
-import java.lang.reflect.Method;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 /**
- * @author ebanks
+ * @author ebanks, depristo
  * GATKSAMRecord
  *
- * this class extends the samtools SAMRecord class and caches important
+ * this class extends the samtools BAMRecord class (and SAMRecord) and caches important
  * (and oft-accessed) data that's not already cached by the SAMRecord class
  *
  * IMPORTANT NOTE: Because ReadGroups are not set through the SAMRecord,
  *   if they are ever modified externally then one must also invoke the
  *   setReadGroup() method here to ensure that the cache is kept up-to-date.
  *
- * 13 Oct 2010 - mhanna - this class is fundamentally flawed: it uses a decorator
- *                        pattern to wrap a heavyweight object, which can lead
- *                        to heinous side effects if the wrapping is not carefully
- *                        done.  Hopefully SAMRecord will become an interface and
- *                        this will eventually be fixed.
  */
-public class GATKSAMRecord extends SAMRecord {
+public class GATKSAMRecord extends BAMRecord {
-
-    // the underlying SAMRecord which we are wrapping
-    private final SAMRecord mRecord;
-
     // the SAMRecord data we're caching
     private String mReadString = null;
-    private SAMReadGroupRecord mReadGroup = null;
+    private GATKSAMReadGroupRecord mReadGroup = null;
-    private boolean mNegativeStrandFlag;
+    private byte[] reducedReadCounts = null;
-    private boolean mUnmappedFlag;
-    private Boolean mSecondOfPairFlag = null;
 
     // because some values can be null, we don't want to duplicate effort
     private boolean retrievedReadGroup = false;
-
+    private boolean retrievedReduceReadCounts = false;
-    /** A private cache for the reduced read quality.  Null indicates the value hasn't be fetched yet or isn't available */
-    private boolean lookedUpReducedReadQuality = false;
-    private Integer reducedReadQuality;
 
     // These temporary attributes were added here to make life easier for
     // certain algorithms by providing a way to label or attach arbitrary data to
@@ -51,101 +58,112 @@ public class GATKSAMRecord extends SAMRecord {
     // These attributes exist in memory only, and are never written to disk.
     private Map<Object, Object> temporaryAttributes;
 
-    public GATKSAMRecord(SAMRecord record, boolean useOriginalBaseQualities, byte defaultBaseQualities) {
+    /**
-        super(null); // it doesn't matter - this isn't used
+     * HACK TO CREATE GATKSAMRECORD WITH ONLY A HEADER FOR TESTING PURPOSES ONLY
-        if ( record == null )
+     * @param header
-            throw new IllegalArgumentException("The SAMRecord argument cannot be null");
+     */
-        mRecord = record;
+    public GATKSAMRecord(final SAMFileHeader header) {
+        this(new SAMRecord(header));
+    }
 
-        mNegativeStrandFlag = mRecord.getReadNegativeStrandFlag();
+    /**
-        mUnmappedFlag = mRecord.getReadUnmappedFlag();
+     * HACK TO CREATE GATKSAMRECORD BASED ONLY A SAMRECORD FOR TESTING PURPOSES ONLY
+     * @param read
+     */
+    public GATKSAMRecord(final SAMRecord read) {
+        super(read.getHeader(), read.getMateReferenceIndex(),
+                read.getAlignmentStart(),
+                read.getReadName() != null ? (short)read.getReadNameLength() : 0,
+                (short)read.getMappingQuality(),
+                0,
+                read.getCigarLength(),
+                read.getFlags(),
+                read.getReadLength(),
+                read.getMateReferenceIndex(),
+                read.getMateAlignmentStart(),
+                read.getInferredInsertSize(),
+                new byte[]{});
+        super.clearAttributes();
+    }
 
-        // because attribute methods are declared to be final (and we can't overload them),
+    public GATKSAMRecord(final SAMFileHeader header,
-        // we need to actually set all of the attributes here
+                         final int referenceSequenceIndex,
-        List<SAMTagAndValue> attributes = record.getAttributes();
+                         final int alignmentStart,
-        for ( SAMTagAndValue attribute : attributes )
+                         final short readNameLength,
-            setAttribute(attribute.tag, attribute.value);
+                         final short mappingQuality,
-
+                         final int indexingBin,
-        // if we are using default quals, check if we need them, and add if necessary.
+                         final int cigarLen,
-        // 1. we need if reads are lacking or have incomplete quality scores
+                         final int flags,
-        // 2. we add if defaultBaseQualities has a positive value
+                         final int readLen,
-        if (defaultBaseQualities >= 0) {
+                         final int mateReferenceSequenceIndex,
-            byte reads [] = record.getReadBases();
+                         final int mateAlignmentStart,
-            byte quals [] = record.getBaseQualities();
+                         final int insertSize,
-            if (quals == null || quals.length < reads.length) {
+                         final byte[] variableLengthBlock) {
-                byte new_quals [] = new byte [reads.length];
+        super(header, referenceSequenceIndex, alignmentStart, readNameLength, mappingQuality, indexingBin, cigarLen,
-                for (int i=0; i<reads.length; i++)
+                flags, readLen, mateReferenceSequenceIndex, mateAlignmentStart, insertSize, variableLengthBlock);
-                    new_quals[i] = defaultBaseQualities;
-                record.setBaseQualities(new_quals);
-            }
-        }
-
-        // if we are using original quals, set them now if they are present in the record
-        if ( useOriginalBaseQualities ) {
-            byte[] originalQuals = mRecord.getOriginalBaseQualities();
-            if ( originalQuals != null )
-                mRecord.setBaseQualities(originalQuals);
-        }
     }
 
     ///////////////////////////////////////////////////////////////////////////////
     // *** The following methods are overloaded to cache the appropriate data ***//
     ///////////////////////////////////////////////////////////////////////////////
 
+    @Override
     public String getReadString() {
         if ( mReadString == null )
-            mReadString = mRecord.getReadString();
+            mReadString = super.getReadString();
         return mReadString;
     }
 
+    @Override
     public void setReadString(String s) {
-        mRecord.setReadString(s);
+        super.setReadString(s);
         mReadString = s;
     }
 
-    public SAMReadGroupRecord getReadGroup() {
+    @Override
+    public GATKSAMReadGroupRecord getReadGroup() {
         if ( !retrievedReadGroup ) {
-            SAMReadGroupRecord tempReadGroup = mRecord.getReadGroup();
+            SAMReadGroupRecord tempReadGroup = super.getReadGroup();
-            mReadGroup = (tempReadGroup == null ? tempReadGroup : new GATKSAMReadGroupRecord(tempReadGroup));
+            mReadGroup = (tempReadGroup == null ? null : new GATKSAMReadGroupRecord(tempReadGroup));
             retrievedReadGroup = true;
         }
         return mReadGroup;
     }
 
-    public void setReadGroup(SAMReadGroupRecord record) {
+    /**
-        mReadGroup = record;
+     * Efficient caching accessor that returns the GATK NGSPlatform of this read
+     * @return
+     */
+    public NGSPlatform getNGSPlatform() {
+        return getReadGroup().getNGSPlatform();
     }
 
-    public boolean getReadUnmappedFlag() {
+    public void setReadGroup( final GATKSAMReadGroupRecord readGroup ) {
-        return mUnmappedFlag;
+        mReadGroup = readGroup;
+        retrievedReadGroup = true;
     }
 
-    public void setReadUnmappedFlag(boolean b) {
+    //
-        mRecord.setReadUnmappedFlag(b);
+    //
-        mUnmappedFlag = b;
+    // Reduced read functions
-    }
+    //
+    //
 
-    public boolean getReadNegativeStrandFlag() {
+    public byte[] getReducedReadCounts() {
-        return mNegativeStrandFlag;
+        if ( ! retrievedReduceReadCounts ) {
-    }
+            reducedReadCounts = getByteArrayAttribute(ReadUtils.REDUCED_READ_QUALITY_TAG);
-
+            retrievedReduceReadCounts = true;
-    public void setReadNegativeStrandFlag(boolean b) {
-        mRecord.setReadNegativeStrandFlag(b);
-        mNegativeStrandFlag = b;
-    }
-
-    public boolean getSecondOfPairFlag() {
-        if( mSecondOfPairFlag == null ) {
-            //not done in constructor because this method can't be called for
-            //all SAMRecords.
-            mSecondOfPairFlag = mRecord.getSecondOfPairFlag();
         }
-        return mSecondOfPairFlag;
+
+        return reducedReadCounts;
     }
 
-    public void setSecondOfPairFlag(boolean b) {
+    public boolean isReducedRead() {
-        mRecord.setSecondOfPairFlag(b);
+        return getReducedReadCounts() != null;
-        mSecondOfPairFlag = b;
+    }
+
+    public final byte getReducedCount(final int i) {
+        return getReducedReadCounts()[i];
     }
 
     /**
@@ -201,269 +219,18 @@ public class GATKSAMRecord extends SAMRecord {
         return null;
     }
 
-    /**
+    @Override
-     * Removes the attribute that has the given key.
+    public int hashCode() {
-     *
+        return super.hashCode();
-     * Temporary attributes provide a way to label or attach arbitrary data to
-     * individual GATKSAMRecords. These attributes exist in memory only,
-     * and are never written to disk.
-     *
-     * @param key key
-     * @return The value that was associated with this key, or null.
-     */
-    public Object removeTemporaryAttribute(Object key) {
-         if(temporaryAttributes != null) {
-             return temporaryAttributes.remove(key);
-         }
-         return null;
     }
 
-    /////////////////////////////////////////////////////////////////////////////////
-    // *** The following methods just call the appropriate method in the record ***//
-    /////////////////////////////////////////////////////////////////////////////////
-
-    public String getReadName() { return mRecord.getReadName(); }
-
-    public int getReadNameLength() { return mRecord.getReadNameLength(); }
-
-    public void setReadName(String s) { mRecord.setReadName(s); }
-
-    public byte[] getReadBases() { return mRecord.getReadBases(); }
-
-    public void setReadBases(byte[] bytes) { mRecord.setReadBases(bytes); }
-
-    public int getReadLength() { return mRecord.getReadLength(); }
-
-    public byte[] getBaseQualities() { return mRecord.getBaseQualities(); }
-
-    public void setBaseQualities(byte[] bytes) { mRecord.setBaseQualities(bytes); }
-
-    public String getBaseQualityString() { return mRecord.getBaseQualityString(); }
-
-    public void setBaseQualityString(String s) { mRecord.setBaseQualityString(s); }
-
-    public byte[] getOriginalBaseQualities() { return mRecord.getOriginalBaseQualities(); }
-
-    public void setOriginalBaseQualities(byte[] bytes) { mRecord.setOriginalBaseQualities(bytes); }
-
-    public String getReferenceName() { return mRecord.getReferenceName(); }
-
-    public void setReferenceName(String s) { mRecord.setReferenceName(s); }
-
-    public Integer getReferenceIndex() { return mRecord.getReferenceIndex(); }
-
-    public void setReferenceIndex(int i) { mRecord.setReferenceIndex(i); }
-
-    public String getMateReferenceName() { return mRecord.getMateReferenceName(); }
-
-    public void setMateReferenceName(String s) { mRecord.setMateReferenceName(s); }
-
-    public Integer getMateReferenceIndex() { return mRecord.getMateReferenceIndex(); }
-
-    public void setMateReferenceIndex(int i) { mRecord.setMateReferenceIndex(i); }
-
-    public int getAlignmentStart() { return mRecord.getAlignmentStart(); }
-
-    public void setAlignmentStart(int i) { mRecord.setAlignmentStart(i); }
-
-    public int getAlignmentEnd() { return mRecord.getAlignmentEnd(); }
-
-    public int getUnclippedStart() { return mRecord.getUnclippedStart(); }
-
-    public int getUnclippedEnd() { return mRecord.getUnclippedEnd(); }
-
-    public void setAlignmentEnd(int i) { mRecord.setAlignmentEnd(i); }
-
-    public int getMateAlignmentStart() { return mRecord.getMateAlignmentStart(); }
-
-    public void setMateAlignmentStart(int i) { mRecord.setMateAlignmentStart(i); }
-
-    public int getInferredInsertSize() { return mRecord.getInferredInsertSize(); }
-
-    public void setInferredInsertSize(int i) { mRecord.setInferredInsertSize(i); }
-
-    public int getMappingQuality() { return mRecord.getMappingQuality(); }
-
-    public void setMappingQuality(int i) { mRecord.setMappingQuality(i); }
-
-    public String getCigarString() { return mRecord.getCigarString(); }
-
-    public void setCigarString(String s) { mRecord.setCigarString(s); }
-
-    public Cigar getCigar() { return mRecord.getCigar(); }
-
-    public int getCigarLength() { return mRecord.getCigarLength(); }
-
-    public void setCigar(Cigar cigar) { mRecord.setCigar(cigar); }
-
-    public int getFlags() { return mRecord.getFlags(); }
-
-    public void setFlags(int i) { mRecord.setFlags(i); }
-
-    public boolean getReadPairedFlag() { return mRecord.getReadPairedFlag(); }
-
-    public boolean getProperPairFlag() { return mRecord.getProperPairFlag(); }
-
-    public boolean getMateUnmappedFlag() { return mRecord.getMateUnmappedFlag(); }
-
-    public boolean getMateNegativeStrandFlag() { return mRecord.getMateNegativeStrandFlag(); }
-
-    public boolean getFirstOfPairFlag() { return mRecord.getFirstOfPairFlag(); }
-
-    public boolean getNotPrimaryAlignmentFlag() { return mRecord.getNotPrimaryAlignmentFlag(); }
-
-    public boolean getReadFailsVendorQualityCheckFlag() { return mRecord.getReadFailsVendorQualityCheckFlag(); }
-
-    public boolean getDuplicateReadFlag() { return mRecord.getDuplicateReadFlag(); }
-
-    public void setReadPairedFlag(boolean b) { mRecord.setReadPairedFlag(b); }
-
-    public void setProperPairFlag(boolean b) { mRecord.setProperPairFlag(b); }
-
-    public void setMateUnmappedFlag(boolean b) { mRecord.setMateUnmappedFlag(b); }
-
-    public void setMateNegativeStrandFlag(boolean b) { mRecord.setMateNegativeStrandFlag(b); }
-
-    public void setFirstOfPairFlag(boolean b) { mRecord.setFirstOfPairFlag(b); }
-
-    public void setNotPrimaryAlignmentFlag(boolean b) { mRecord.setNotPrimaryAlignmentFlag(b); }
-
-    public void setReadFailsVendorQualityCheckFlag(boolean b) { mRecord.setReadFailsVendorQualityCheckFlag(b); }
-
-    public void setDuplicateReadFlag(boolean b) { mRecord.setDuplicateReadFlag(b); }
-
-    public net.sf.samtools.SAMFileReader.ValidationStringency getValidationStringency() { return mRecord.getValidationStringency(); }
-
-    public void setValidationStringency(net.sf.samtools.SAMFileReader.ValidationStringency validationStringency) { mRecord.setValidationStringency(validationStringency); }
-
-    public Object getAttribute(final String tag) { return mRecord.getAttribute(tag); }
-
-    public Integer getIntegerAttribute(final String tag) {
-        if ( tag == ReadUtils.REDUCED_READ_QUALITY_TAG ) {
-            if ( ! lookedUpReducedReadQuality ) {
-                lookedUpReducedReadQuality = true;
-                reducedReadQuality = mRecord.getIntegerAttribute(tag);
-            }
-            return reducedReadQuality;
-        } else {
-            return mRecord.getIntegerAttribute(tag);
-        }
-    }
-
-    public Short getShortAttribute(final String tag) { return mRecord.getShortAttribute(tag); }
-
-    public Byte getByteAttribute(final String tag) { return mRecord.getByteAttribute(tag); }
-
-    public String getStringAttribute(final String tag) { return mRecord.getStringAttribute(tag); }
-
-    public Character getCharacterAttribute(final String tag) { return mRecord.getCharacterAttribute(tag); }
-
-    public Float getFloatAttribute(final String tag) { return mRecord.getFloatAttribute(tag); }
-
-    public byte[] getByteArrayAttribute(final String tag) { return mRecord.getByteArrayAttribute(tag); }
-
-    protected Object getAttribute(final short tag) {
-        Object attribute;
-        try {
-            Method method = mRecord.getClass().getDeclaredMethod("getAttribute",Short.TYPE);
-            method.setAccessible(true);
-            attribute = method.invoke(mRecord,tag);
-        }
-        catch(Exception ex) {
-            throw new ReviewedStingException("Unable to invoke getAttribute method",ex);
-        }
-        return attribute;
-    }
-
-    public void setAttribute(final String tag, final Object value) { mRecord.setAttribute(tag,value); }
-
-    protected void setAttribute(final short tag, final Object value) {
-        try {
-            Method method = mRecord.getClass().getDeclaredMethod("setAttribute",Short.TYPE,Object.class);
-            method.setAccessible(true);
-            method.invoke(mRecord,tag,value);
-        }
-        catch(Exception ex) {
-            throw new ReviewedStingException("Unable to invoke setAttribute method",ex);
-        }
-    }
-
-    public void clearAttributes() { mRecord.clearAttributes(); }
-
-    protected void setAttributes(final SAMBinaryTagAndValue attributes) {
-        try {
-            Method method = mRecord.getClass().getDeclaredMethod("setAttributes",SAMBinaryTagAndValue.class);
-            method.setAccessible(true);
-            method.invoke(mRecord,attributes);
-        }
-        catch(Exception ex) {
-            throw new ReviewedStingException("Unable to invoke setAttributes method",ex);
-        }
-   }
-
-    protected SAMBinaryTagAndValue getBinaryAttributes() {
-        SAMBinaryTagAndValue binaryAttributes;
-        try {
-            Method method = mRecord.getClass().getDeclaredMethod("getBinaryAttributes");
-            method.setAccessible(true);
-            binaryAttributes = (SAMBinaryTagAndValue)method.invoke(mRecord);
-        }
-        catch(Exception ex) {
-            throw new ReviewedStingException("Unable to invoke getBinaryAttributes method",ex);
-        }
-        return binaryAttributes;
-    }
-
-    public List<SAMTagAndValue> getAttributes() { return mRecord.getAttributes(); }
-
-    public SAMFileHeader getHeader() { return mRecord.getHeader(); }
-
-    public void setHeader(SAMFileHeader samFileHeader) { mRecord.setHeader(samFileHeader); }
-
-    public byte[] getVariableBinaryRepresentation() { return mRecord.getVariableBinaryRepresentation(); }
-
-    public int getAttributesBinarySize() { return mRecord.getAttributesBinarySize(); }
-
-    public String format() { return mRecord.format(); }
-
-    public List<AlignmentBlock> getAlignmentBlocks() { return mRecord.getAlignmentBlocks(); }
-
-    public List<SAMValidationError> validateCigar(long l) { return mRecord.validateCigar(l); }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
 
-        // note -- this forbids a GATKSAMRecord being equal to its underlying SAMRecord
         if (!(o instanceof GATKSAMRecord)) return false;
 
         // note that we do not consider the GATKSAMRecord internal state at all
-        return mRecord.equals(((GATKSAMRecord)o).mRecord);
+        return super.equals(o);
-    }
-
-    public int hashCode() { return mRecord.hashCode(); }
-
-    public List<SAMValidationError> isValid() { return mRecord.isValid(); }
-
-    public Object clone() throws CloneNotSupportedException { return mRecord.clone(); }
-
-    public String toString() { return mRecord.toString(); }
-
-    public SAMFileSource getFileSource() { return mRecord.getFileSource(); }
-
-    /**
-     * Sets a marker providing the source reader for this file and the position in the file from which the read originated.
-     * @param fileSource source of the given file.
-     */
-    @Override
-    protected void setFileSource(final SAMFileSource fileSource) {
-        try {
-            Method method = SAMRecord.class.getDeclaredMethod("setFileSource",SAMFileSource.class);
-            method.setAccessible(true);
-            method.invoke(mRecord,fileSource);
-        }
-        catch(Exception ex) {
-            throw new ReviewedStingException("Unable to invoke setFileSource method",ex);
-        }
     }
 }

public/java/src/org/broadinstitute/sting/utils/sam/GATKSamRecordFactory.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.sam;
+
+import net.sf.samtools.SAMFileHeader;
+import net.sf.samtools.SAMRecord;
+import net.sf.samtools.SAMRecordFactory;
+import net.sf.samtools.BAMRecord;
+import org.broadinstitute.sting.utils.exceptions.UserException;
+
+/**
+ * Factory interface implementation used to create GATKSamRecords
+ * from SAMFileReaders with SAM-JDK
+ *
+ * @author Mark DePristo
+ */
+public class GATKSamRecordFactory implements SAMRecordFactory {
+
+    /** Create a new SAMRecord to be filled in */
+    public SAMRecord createSAMRecord(SAMFileHeader header) {
+        throw new UserException.BadInput("The GATK now longer supports input SAM files");
+    }
+
+    /** Create a new BAM Record. */
+    public BAMRecord createBAMRecord(final SAMFileHeader header,
+                                     final int referenceSequenceIndex,
+                                     final int alignmentStart,
+                                     final short readNameLength,
+                                     final short mappingQuality,
+                                     final int indexingBin,
+                                     final int cigarLen,
+                                     final int flags,
+                                     final int readLen,
+                                     final int mateReferenceSequenceIndex,
+                                     final int mateAlignmentStart,
+                                     final int insertSize,
+                                     final byte[] variableLengthBlock) {
+        return new GATKSAMRecord(header,
+                referenceSequenceIndex,
+                alignmentStart,
+                readNameLength,
+                mappingQuality,
+                indexingBin,
+                cigarLen,
+                flags,
+                readLen,
+                mateReferenceSequenceIndex,
+                mateAlignmentStart,
+                insertSize,
+                variableLengthBlock);
+    }
+}

public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java

@@ -52,38 +52,6 @@ public class ReadUtils {
     // ----------------------------------------------------------------------------------------------------
 
     public static final String REDUCED_READ_QUALITY_TAG = "RQ";
-    public static final String REDUCED_READ_CONSENSUS_COUNTS_TAG = "CC";
-
-    public final static byte[] getReducedReadQualityTagValue(final SAMRecord read) {
-        return read.getByteArrayAttribute(ReadUtils.REDUCED_READ_QUALITY_TAG);
-    }
-
-    public final static boolean isReducedRead(final SAMRecord read) {
-        return getReducedReadQualityTagValue(read) != null;
-    }
-
-    public final static byte getReducedQual(final SAMRecord read, final int i) {
-        return read.getBaseQualities()[i];
-    }
-
-    public final static byte getReducedCount(final SAMRecord read, final int i) {
-        return getReducedReadQualityTagValue(read)[i];
-    }
-
-    public final static SAMRecord reducedReadWithReducedQuals(final SAMRecord read) {
-        if ( ! isReducedRead(read) ) throw new IllegalArgumentException("read must be a reduced read");
-        return read;
-//        try {
-//            SAMRecord newRead = (SAMRecord)read.clone();
-//            byte reducedQual = (byte)(int)getReducedReadQualityTagValue(read);
-//            byte[] newQuals = new byte[read.getBaseQualities().length];
-//            Arrays.fill(newQuals, reducedQual);
-//            newRead.setBaseQualities(newQuals);
-//            return newRead;
-//        } catch ( CloneNotSupportedException e ) {
-//            throw new ReviewedStingException("SAMRecord no longer supports clone", e);
-//        }
-    }
 
     // ----------------------------------------------------------------------------------------------------
     //

public/java/test/org/broadinstitute/sting/BaseTest.java

@@ -50,6 +50,7 @@ public abstract class BaseTest {
     public static final String hg18Reference = "/seq/references/Homo_sapiens_assembly18/v0/Homo_sapiens_assembly18.fasta";
     public static final String hg19Reference = "/seq/references/Homo_sapiens_assembly19/v1/Homo_sapiens_assembly19.fasta";
     public static final String b36KGReference = "/humgen/1kg/reference/human_b36_both.fasta";
+    //public static final String b37KGReference = "/Users/depristo/Desktop/broadLocal/localData/human_g1k_v37.fasta";
     public static final String b37KGReference = "/humgen/1kg/reference/human_g1k_v37.fasta";
     public static final String GATKDataLocation = "/humgen/gsa-hpprojects/GATK/data/";
     public static final String validationDataLocation = GATKDataLocation + "Validation_Data/";
@@ -99,10 +100,10 @@ public abstract class BaseTest {
         logger.setLevel(Level.WARN);
 
         // find our file sources
-        if (!fileExist(hg18Reference) || !fileExist(hg19Reference) || !fileExist(b36KGReference)) {
+//        if (!fileExist(hg18Reference) || !fileExist(hg19Reference) || !fileExist(b36KGReference)) {
-            logger.fatal("We can't locate the reference directories.  Aborting!");
+//            logger.fatal("We can't locate the reference directories.  Aborting!");
-            throw new RuntimeException("BaseTest setup failed: unable to locate the reference directories");
+//            throw new RuntimeException("BaseTest setup failed: unable to locate the reference directories");
-        }
+//        }
     }
 
     /**

public/java/test/org/broadinstitute/sting/commandline/ArgumentMatchSiteUnitTest.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.commandline;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.File;
+
+public class ArgumentMatchSiteUnitTest {
+    @Test
+    public void testCommandLine() {
+        ArgumentMatchSite site = new ArgumentMatchSite(ArgumentMatchSource.COMMAND_LINE, 1);
+        Assert.assertEquals(site.getSource(), ArgumentMatchSource.COMMAND_LINE);
+        Assert.assertEquals(site.getIndex(), 1);
+    }
+
+    @Test
+    public void testFile() {
+        ArgumentMatchSource source = new ArgumentMatchSource(new File("test"));
+        ArgumentMatchSite site = new ArgumentMatchSite(source, 1);
+        Assert.assertEquals(site.getSource(), source);
+        Assert.assertEquals(site.getIndex(), 1);
+    }
+
+    @Test
+    public void testEquals() {
+        ArgumentMatchSource cmdLine = ArgumentMatchSource.COMMAND_LINE;
+        ArgumentMatchSite site1 = new ArgumentMatchSite(cmdLine, 1);
+        ArgumentMatchSite site2 = new ArgumentMatchSite(cmdLine, 2);
+
+        Assert.assertFalse(site1.equals(null));
+
+        Assert.assertTrue(site1.equals(site1));
+        Assert.assertFalse(site1.equals(site2));
+
+        Assert.assertFalse(site2.equals(site1));
+        Assert.assertTrue(site2.equals(site2));
+    }
+
+    @Test
+    public void testCompareTo() {
+        ArgumentMatchSource cmdLine = ArgumentMatchSource.COMMAND_LINE;
+        ArgumentMatchSite site1 = new ArgumentMatchSite(cmdLine, 1);
+        ArgumentMatchSite site2 = new ArgumentMatchSite(cmdLine, 2);
+
+        Assert.assertTrue(site1.compareTo(site1) == 0);
+        Assert.assertTrue(site1.compareTo(site2) < 0);
+        Assert.assertTrue(site2.compareTo(site1) > 0);
+        Assert.assertTrue(site2.compareTo(site2) == 0);
+    }
+
+    @Test(expectedExceptions = NullPointerException.class)
+    public void testCompareToNull() {
+        new ArgumentMatchSite(ArgumentMatchSource.COMMAND_LINE, 0).compareTo(null);
+    }
+}

public/java/test/org/broadinstitute/sting/commandline/ArgumentMatchSourceUnitTest.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.commandline;
+
+import org.broadinstitute.sting.BaseTest;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.File;
+
+public class ArgumentMatchSourceUnitTest extends BaseTest {
+    @Test
+    public void testCommandLine() {
+        ArgumentMatchSource source = ArgumentMatchSource.COMMAND_LINE;
+        Assert.assertEquals(source.getType(), ArgumentMatchSourceType.CommandLine);
+        Assert.assertNull(source.getFile());
+    }
+
+    @Test
+    public void testFile() {
+        File f = new File("test");
+        ArgumentMatchSource source = new ArgumentMatchSource(f);
+        Assert.assertEquals(source.getType(), ArgumentMatchSourceType.File);
+        Assert.assertEquals(source.getFile(), f);
+    }
+
+    @Test(expectedExceptions = IllegalArgumentException.class)
+    public void testNullFile() {
+        new ArgumentMatchSource(null);
+    }
+
+    @Test
+    public void testEquals() {
+        ArgumentMatchSource cmdLine = ArgumentMatchSource.COMMAND_LINE;
+        ArgumentMatchSource fileA = new ArgumentMatchSource(new File("a"));
+        ArgumentMatchSource fileB = new ArgumentMatchSource(new File("b"));
+
+        Assert.assertFalse(cmdLine.equals(null));
+
+        Assert.assertTrue(cmdLine.equals(cmdLine));
+        Assert.assertFalse(cmdLine.equals(fileA));
+        Assert.assertFalse(cmdLine.equals(fileB));
+
+        Assert.assertFalse(fileA.equals(cmdLine));
+        Assert.assertTrue(fileA.equals(fileA));
+        Assert.assertFalse(fileA.equals(fileB));
+
+        Assert.assertFalse(fileB.equals(cmdLine));
+        Assert.assertFalse(fileB.equals(fileA));
+        Assert.assertTrue(fileB.equals(fileB));
+    }
+
+    @Test
+    public void testCompareTo() {
+        ArgumentMatchSource cmdLine = ArgumentMatchSource.COMMAND_LINE;
+        ArgumentMatchSource fileA = new ArgumentMatchSource(new File("a"));
+        ArgumentMatchSource fileB = new ArgumentMatchSource(new File("b"));
+
+        Assert.assertTrue(cmdLine.compareTo(cmdLine) == 0);
+        Assert.assertTrue(cmdLine.compareTo(fileA) < 0);
+        Assert.assertTrue(cmdLine.compareTo(fileB) < 0);
+
+        Assert.assertTrue(fileA.compareTo(cmdLine) > 0);
+        Assert.assertTrue(fileA.compareTo(fileA) == 0);
+        Assert.assertTrue(fileA.compareTo(fileB) < 0);
+
+        Assert.assertTrue(fileB.compareTo(cmdLine) > 0);
+        Assert.assertTrue(fileB.compareTo(fileA) > 0);
+        Assert.assertTrue(fileB.compareTo(fileB) == 0);
+    }
+
+    @Test(expectedExceptions = NullPointerException.class)
+    public void testCompareToNull() {
+        ArgumentMatchSource.COMMAND_LINE.compareTo(null);
+    }
+}

public/java/test/org/broadinstitute/sting/commandline/ParsingEngineUnitTest.java

@@ -25,6 +25,7 @@
 
 package org.broadinstitute.sting.commandline;
 
+import org.apache.commons.io.FileUtils;
 import org.broad.tribble.Feature;
 import org.broadinstitute.sting.utils.exceptions.UserException;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
@@ -34,6 +35,8 @@ import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
+import java.io.File;
+import java.io.IOException;
 import java.util.List;
 import java.util.EnumSet;
 /**
@@ -493,6 +496,7 @@ public class ParsingEngineUnitTest extends BaseTest {
         Assert.assertNotNull(definition, "Invalid default argument name assigned");
     }
 
+    @SuppressWarnings("unused")
     private class CamelCaseArgProvider {
         @Argument(doc="my arg")
         Integer myArg;
@@ -507,6 +511,7 @@ public class ParsingEngineUnitTest extends BaseTest {
         parsingEngine.validate();
     }
 
+    @SuppressWarnings("unused")
     private class BooleanArgProvider {
         @Argument(doc="my bool")
         boolean myBool;
@@ -561,6 +566,7 @@ public class ParsingEngineUnitTest extends BaseTest {
         parsingEngine.validate();
     }
 
+    @SuppressWarnings("unused")
     private class MutuallyExclusiveArgProvider {
         @Argument(doc="foo",exclusiveOf="bar")
         Integer foo;
@@ -618,6 +624,7 @@ public class ParsingEngineUnitTest extends BaseTest {
         parsingEngine.addArgumentSource( MultipleArgumentCollectionProvider.class );
     }
 
+    @SuppressWarnings("unused")
     private class MultipleArgumentCollectionProvider {
         @ArgumentCollection
         RequiredArgProvider rap1 = new RequiredArgProvider();
@@ -937,4 +944,23 @@ public class ParsingEngineUnitTest extends BaseTest {
         VariantContextRodBindingArgProvider argProvider = new VariantContextRodBindingArgProvider();
         parsingEngine.loadArgumentsIntoObject( argProvider );
     }
+
+    @Test
+    public void argumentListTest() throws IOException {
+        File argsFile = BaseTest.createTempFile("args.", ".list");
+        try {
+            FileUtils.write(argsFile, "-I na12878.bam");
+            final String[] commandLine = new String[] {"-args", argsFile.getPath()};
+            parsingEngine.addArgumentSource(InputFileArgProvider.class);
+            parsingEngine.parse(commandLine);
+            parsingEngine.validate();
+
+            InputFileArgProvider argProvider = new InputFileArgProvider();
+            parsingEngine.loadArgumentsIntoObject(argProvider);
+
+            Assert.assertEquals(argProvider.inputFile, "na12878.bam", "Argument is not correctly initialized");
+        } finally {
+            FileUtils.deleteQuietly(argsFile);
+        }
+    }
 }

public/java/test/org/broadinstitute/sting/gatk/walkers/BAQIntegrationTest.java

@@ -18,7 +18,7 @@ public class BAQIntegrationTest extends WalkerTest {
     // --------------------------------------------------------------------------------------------------------------
     @Test
     public void testPrintReadsNoBAQ() {
-        WalkerTestSpec spec = new WalkerTestSpec( baseCommand +" -baq OFF",  1, Arrays.asList("902197bf77ed5a828d50e08771685928"));
+        WalkerTestSpec spec = new WalkerTestSpec( baseCommand +" -baq OFF",  1, Arrays.asList("d97340a2bba2c6320d1ebeb86024a27c"));
         executeTest(String.format("testPrintReadsNoBAQ"), spec);
     }
 

public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java

@@ -224,7 +224,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                         " -o %s" +
                         " -L 1:10,000,000-10,500,000",
                 1,
-                Arrays.asList("0bece77ce6bc447438ef9b2921b2dc41"));
+                Arrays.asList("eeba568272f9b42d5450da75c7cc6d2d"));
 
         executeTest(String.format("test indel caller in SLX"), spec);
     }
@@ -252,7 +252,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                          " -o %s" +
                          " -L 1:10,000,000-10,500,000",
                  1,
-                 Arrays.asList("790b1a1d6ab79eee8c24812bb8ca6fae"));
+                 Arrays.asList("19ff9bd3139480bdf79dcbf117cf2b24"));
 
          executeTest(String.format("test indel calling, multiple technologies"), spec);
      }
@@ -262,7 +262,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "indelAllelesForUG.vcf -I " + validationDataLocation +
                         "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,100,000", 1,
-                Arrays.asList("408d3aba4d094c067fc00a43992c2292"));
+                Arrays.asList("118918f2e9e56a3cfc5ccb2856d529c8"));
         executeTest("test MultiSample Pilot2 indels with alleles passed in", spec1);
     }
 
@@ -272,7 +272,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                 baseCommandIndels + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles "
                         + validationDataLocation + "indelAllelesForUG.vcf -I " + validationDataLocation +
                         "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,100,000", 1,
-                Arrays.asList("5e4e09354410b76fc0d822050d84132a"));
+                Arrays.asList("a20799237accd52c1b8c2ac096309c8f"));
         executeTest("test MultiSample Pilot2 indels with alleles passed in and emitting all sites", spec2);
     }
 
@@ -282,7 +282,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         WalkerTest.WalkerTestSpec spec3 = new WalkerTest.WalkerTestSpec(
                 baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "ALL.wgs.union_v2.20101123.indels.sites.vcf -I " + validationDataLocation +
                         "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,080,000", 1,
-                Arrays.asList("c599eedbeb422713b8a28529e805e4ae"));
+                Arrays.asList("18ef8181157b4ac3eb8492f538467f92"));
         executeTest("test MultiSample Pilot2 indels with complicated records", spec3);
     }
 
@@ -291,7 +291,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         WalkerTest.WalkerTestSpec spec4 = new WalkerTest.WalkerTestSpec(
                 baseCommandIndelsb37 + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "ALL.wgs.union_v2_chr20_100_110K.20101123.indels.sites.vcf -I " + validationDataLocation +
                         "phase1_GBR_realigned.chr20.100K-110K.bam -o %s -L 20:100,000-110,000", 1,
-                Arrays.asList("37d908a682ac269f8f19dec939ff5b01"));
+                Arrays.asList("ad884e511a751b05e64db5314314365a"));
         executeTest("test MultiSample 1000G Phase1 indels with complicated records emitting all sites", spec4);
     }
 

public/java/test/org/broadinstitute/sting/utils/R/RScriptLibraryUnitTest.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.R;
+
+import org.apache.commons.io.FileUtils;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.File;
+
+public class RScriptLibraryUnitTest {
+    @Test
+    public void testProperties() {
+        Assert.assertEquals(RScriptLibrary.GSALIB.getLibraryName(), "gsalib");
+        Assert.assertEquals(RScriptLibrary.GSALIB.getResourcePath(), "gsalib.tar.gz");
+    }
+
+    @Test
+    public void testWriteTemp() {
+        File file = RScriptLibrary.GSALIB.writeTemp();
+        Assert.assertTrue(file.exists(), "R library was not written to temp file: " + file);
+        FileUtils.deleteQuietly(file);
+    }
+}

public/java/test/org/broadinstitute/sting/utils/ReadUtilsUnitTest.java

@@ -5,6 +5,7 @@ import net.sf.samtools.SAMRecord;
 import org.broadinstitute.sting.BaseTest;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.sam.ArtificialSAMUtils;
+import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.broadinstitute.sting.utils.sam.ReadUtils;
 import org.testng.Assert;
 import org.testng.annotations.BeforeTest;
@@ -12,7 +13,7 @@ import org.testng.annotations.Test;
 
 
 public class ReadUtilsUnitTest extends BaseTest {
-    SAMRecord read, reducedRead;
+    GATKSAMRecord read, reducedRead;
     final static String BASES = "ACTG";
     final static String QUALS = "!+5?";
     final private static byte[] REDUCED_READ_COUNTS = new byte[]{10, 20, 30, 40};
@@ -47,13 +48,12 @@ public class ReadUtilsUnitTest extends BaseTest {
 
     @Test
     public void testReducedReads() {
-        Assert.assertFalse(ReadUtils.isReducedRead(read), "isReducedRead is false for normal read");
+        Assert.assertFalse(read.isReducedRead(), "isReducedRead is false for normal read");
-        Assert.assertEquals(ReadUtils.getReducedReadQualityTagValue(read), null, "No reduced read tag in normal read");
+        Assert.assertEquals(read.getReducedReadCounts(), null, "No reduced read tag in normal read");
 
-        Assert.assertTrue(ReadUtils.isReducedRead(reducedRead), "isReducedRead is true for reduced read");
+        Assert.assertTrue(reducedRead.isReducedRead(), "isReducedRead is true for reduced read");
         for ( int i = 0; i < reducedRead.getReadLength(); i++) {
-            Assert.assertEquals(ReadUtils.getReducedQual(reducedRead, i), read.getBaseQualities()[i], "Reduced read quality not set to the expected value at " + i);
+            Assert.assertEquals(reducedRead.getReducedCount(i), REDUCED_READ_COUNTS[i], "Reduced read count not set to the expected value at " + i);
-            Assert.assertEquals(ReadUtils.getReducedCount(reducedRead, i), REDUCED_READ_COUNTS[i], "Reduced read count not set to the expected value at " + i);
         }
     }
 

public/java/test/org/broadinstitute/sting/utils/ReservoirDownsamplerUnitTest.java

@@ -1,5 +1,6 @@
 package org.broadinstitute.sting.utils;
 
+import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 import org.broadinstitute.sting.utils.sam.ArtificialSAMUtils;
@@ -28,7 +29,7 @@ public class ReservoirDownsamplerUnitTest {
 
     @Test
     public void testOneElementWithPoolSizeOne() {
-        List<SAMRecord> reads = Collections.singletonList(ArtificialSAMUtils.createArtificialRead(header,"read1",0,1,76));
+        List<GATKSAMRecord> reads = Collections.singletonList(ArtificialSAMUtils.createArtificialRead(header,"read1",0,1,76));
         ReservoirDownsampler<SAMRecord> downsampler = new ReservoirDownsampler<SAMRecord>(1);
         downsampler.addAll(reads);
 
@@ -40,7 +41,7 @@ public class ReservoirDownsamplerUnitTest {
 
     @Test
     public void testOneElementWithPoolSizeGreaterThanOne() {
-        List<SAMRecord> reads = Collections.singletonList(ArtificialSAMUtils.createArtificialRead(header,"read1",0,1,76));
+        List<GATKSAMRecord> reads = Collections.singletonList(ArtificialSAMUtils.createArtificialRead(header,"read1",0,1,76));
         ReservoirDownsampler<SAMRecord> downsampler = new ReservoirDownsampler<SAMRecord>(5);
         downsampler.addAll(reads);
 

public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java

@@ -0,0 +1,197 @@
+package org.broadinstitute.sting.utils.io;
+
+import org.apache.commons.io.FileUtils;
+import org.broadinstitute.sting.BaseTest;
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class IOUtilsUnitTest extends BaseTest {
+  @Test
+  public void testGoodTempDir() {
+    IOUtils.checkTempDir(new File("/tmp/queue"));
+  }
+
+  @Test(expectedExceptions=UserException.BadTmpDir.class)
+  public void testBadTempDir() {
+    IOUtils.checkTempDir(new File("/tmp"));
+  }
+
+  @Test
+  public void testAbsoluteSubDir() {
+    File subDir = IOUtils.absolute(new File("."), new File("/path/to/file"));
+    Assert.assertEquals(subDir, new File("/path/to/file"));
+
+    subDir = IOUtils.absolute(new File("/different/path"), new File("/path/to/file"));
+    Assert.assertEquals(subDir, new File("/path/to/file"));
+
+    subDir = IOUtils.absolute(new File("/different/path"), new File("."));
+    Assert.assertEquals(subDir, new File("/different/path"));
+  }
+
+  @Test
+  public void testRelativeSubDir() throws IOException {
+    File subDir = IOUtils.absolute(new File("."), new File("path/to/file"));
+    Assert.assertEquals(subDir.getCanonicalFile(), new File("path/to/file").getCanonicalFile());
+
+    subDir = IOUtils.absolute(new File("/different/path"), new File("path/to/file"));
+    Assert.assertEquals(subDir, new File("/different/path/path/to/file"));
+  }
+
+  @Test
+  public void testDottedSubDir() throws IOException {
+    File subDir = IOUtils.absolute(new File("."), new File("path/../to/file"));
+    Assert.assertEquals(subDir.getCanonicalFile(), new File("path/../to/./file").getCanonicalFile());
+
+    subDir = IOUtils.absolute(new File("."), new File("/path/../to/file"));
+    Assert.assertEquals(subDir, new File("/path/../to/file"));
+
+    subDir = IOUtils.absolute(new File("/different/../path"), new File("path/to/file"));
+    Assert.assertEquals(subDir, new File("/different/../path/path/to/file"));
+
+    subDir = IOUtils.absolute(new File("/different/./path"), new File("/path/../to/file"));
+    Assert.assertEquals(subDir, new File("/path/../to/file"));
+  }
+
+  @Test
+  public void testTempDir() {
+    File tempDir = IOUtils.tempDir("Q-Unit-Test", "", new File("queueTempDirToDelete"));
+    Assert.assertTrue(tempDir.exists());
+    Assert.assertFalse(tempDir.isFile());
+    Assert.assertTrue(tempDir.isDirectory());
+    boolean deleted = IOUtils.tryDelete(tempDir);
+    Assert.assertTrue(deleted);
+    Assert.assertFalse(tempDir.exists());
+  }
+
+  @Test
+  public void testDirLevel() {
+    File dir = IOUtils.dirLevel(new File("/path/to/directory"), 1);
+    Assert.assertEquals(dir, new File("/path"));
+
+    dir = IOUtils.dirLevel(new File("/path/to/directory"), 2);
+    Assert.assertEquals(dir, new File("/path/to"));
+
+    dir = IOUtils.dirLevel(new File("/path/to/directory"), 3);
+    Assert.assertEquals(dir, new File("/path/to/directory"));
+
+    dir = IOUtils.dirLevel(new File("/path/to/directory"), 4);
+    Assert.assertEquals(dir, new File("/path/to/directory"));
+  }
+
+  @Test
+  public void testAbsolute() {
+    File dir = IOUtils.absolute(new File("/path/./to/./directory/."));
+    Assert.assertEquals(dir, new File("/path/to/directory"));
+
+    dir = IOUtils.absolute(new File("/"));
+    Assert.assertEquals(dir, new File("/"));
+
+    dir = IOUtils.absolute(new File("/."));
+    Assert.assertEquals(dir, new File("/"));
+
+    dir = IOUtils.absolute(new File("/././."));
+    Assert.assertEquals(dir, new File("/"));
+
+    dir = IOUtils.absolute(new File("/./directory/."));
+    Assert.assertEquals(dir, new File("/directory"));
+
+    dir = IOUtils.absolute(new File("/./directory/./"));
+    Assert.assertEquals(dir, new File("/directory"));
+
+    dir = IOUtils.absolute(new File("/./directory./"));
+    Assert.assertEquals(dir, new File("/directory."));
+
+    dir = IOUtils.absolute(new File("/./.directory/"));
+    Assert.assertEquals(dir, new File("/.directory"));
+  }
+
+  @Test
+  public void testTail() throws IOException {
+    List<String> lines = Arrays.asList(
+            "chr18_random	4262	3154410390	50	51",
+            "chr19_random	301858	3154414752	50	51",
+            "chr21_random	1679693	3154722662	50	51",
+            "chr22_random	257318	3156435963	50	51",
+            "chrX_random	1719168	3156698441	50	51");
+    List<String> tail = IOUtils.tail(new File(BaseTest.hg18Reference + ".fai"), 5);
+    Assert.assertEquals(tail.size(), 5);
+    for (int i = 0; i < 5; i++)
+      Assert.assertEquals(tail.get(i), lines.get(i));
+  }
+
+    @Test
+    public void testWriteSystemFile() throws IOException {
+        File temp = createTempFile("temp.", ".properties");
+        try {
+            IOUtils.writeResource(new Resource("StingText.properties", null), temp);
+        } finally {
+            FileUtils.deleteQuietly(temp);
+        }
+    }
+
+    @Test
+    public void testWriteSystemTempFile() throws IOException {
+        File temp = IOUtils.writeTempResource(new Resource("StingText.properties", null));
+        try {
+            Assert.assertTrue(temp.getName().startsWith("StingText"), "File does not start with 'StingText.': " + temp);
+            Assert.assertTrue(temp.getName().endsWith(".properties"), "File does not end with '.properties': " + temp);
+        } finally {
+            FileUtils.deleteQuietly(temp);
+        }
+    }
+
+    @Test(expectedExceptions = IllegalArgumentException.class)
+    public void testMissingSystemFile() throws IOException {
+        File temp = createTempFile("temp.", ".properties");
+        try {
+            IOUtils.writeResource(new Resource("MissingStingText.properties", null), temp);
+        } finally {
+            FileUtils.deleteQuietly(temp);
+        }
+    }
+
+    @Test
+    public void testWriteRelativeFile() throws IOException {
+        File temp = createTempFile("temp.", ".properties");
+        try {
+            IOUtils.writeResource(new Resource("/StingText.properties", IOUtils.class), temp);
+        } finally {
+            FileUtils.deleteQuietly(temp);
+        }
+    }
+
+    @Test
+    public void testWriteRelativeTempFile() throws IOException {
+        File temp = IOUtils.writeTempResource(new Resource("/StingText.properties", IOUtils.class));
+        try {
+            Assert.assertTrue(temp.getName().startsWith("StingText"), "File does not start with 'StingText.': " + temp);
+            Assert.assertTrue(temp.getName().endsWith(".properties"), "File does not end with '.properties': " + temp);
+        } finally {
+            FileUtils.deleteQuietly(temp);
+        }
+    }
+
+    @Test(expectedExceptions = IllegalArgumentException.class)
+    public void testMissingRelativeFile() throws IOException {
+        File temp = createTempFile("temp.", ".properties");
+        try {
+            // Looking for /org/broadinstitute/sting/utils/file/StingText.properties
+            IOUtils.writeResource(new Resource("StingText.properties", IOUtils.class), temp);
+        } finally {
+            FileUtils.deleteQuietly(temp);
+        }
+    }
+
+    @Test
+    public void testResourceProperties() {
+        Resource resource = new Resource("foo", Resource.class);
+        Assert.assertEquals(resource.getPath(), "foo");
+        Assert.assertEquals(resource.getRelativeClass(), Resource.class);
+    }
+}

public/java/test/org/broadinstitute/sting/utils/pileup/FragmentPileupBenchmark.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.pileup;
+
+import com.google.caliper.Param;
+import com.google.caliper.SimpleBenchmark;
+import com.google.caliper.runner.CaliperMain;
+import net.sf.samtools.SAMFileHeader;
+import net.sf.samtools.SAMRecord;
+import org.broadinstitute.sting.utils.GenomeLoc;
+import org.broadinstitute.sting.utils.GenomeLocParser;
+import org.broadinstitute.sting.utils.sam.ArtificialSAMUtils;
+
+import java.util.*;
+
+/**
+ * Caliper microbenchmark of fragment pileup
+ */
+public class FragmentPileupBenchmark extends SimpleBenchmark {
+    List<ReadBackedPileup> pileups;
+
+    @Param({"0", "4", "30", "150", "1000"})
+    int pileupSize; // set automatically by framework
+
+    @Param({"200", "400"})
+    int insertSize; // set automatically by framework
+
+    @Override protected void setUp() {
+        final int nPileupsToGenerate = 100;
+        pileups = new ArrayList<ReadBackedPileup>(nPileupsToGenerate);
+        SAMFileHeader header = ArtificialSAMUtils.createArtificialSamHeader(1, 1, 1000);
+        GenomeLocParser genomeLocParser;
+        genomeLocParser = new GenomeLocParser(header.getSequenceDictionary());
+        GenomeLoc loc = genomeLocParser.createGenomeLoc("chr1", 50);
+        final int readLen = 100;
+
+        for ( int pileupN = 0; pileupN < nPileupsToGenerate; pileupN++ ) {
+            ReadBackedPileup rbp = ArtificialSAMUtils.createReadBackedPileup(header, loc, readLen, insertSize, pileupSize);
+            pileups.add(rbp);
+        }
+    }
+
+    private void run(int rep, FragmentPileup.FragmentMatchingAlgorithm algorithm) {
+        int nFrags = 0;
+        for ( int i = 0; i < rep; i++ ) {
+            for ( ReadBackedPileup rbp : pileups )
+                nFrags += new FragmentPileup(rbp, algorithm).getTwoReadPileup().size();
+        }
+    }
+
+    public void timeOriginal(int rep) {
+        run(rep, FragmentPileup.FragmentMatchingAlgorithm.ORIGINAL);
+    }
+
+    public void timeSkipNonOverlapping(int rep) {
+        run(rep, FragmentPileup.FragmentMatchingAlgorithm.skipNonOverlapping);
+    }
+
+    public static void main(String[] args) {
+        CaliperMain.main(FragmentPileupBenchmark.class, args);
+    }
+}

public/java/test/org/broadinstitute/sting/utils/pileup/FragmentPileupUnitTest.java

@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.pileup;
+
+import net.sf.samtools.SAMFileHeader;
+import net.sf.samtools.SAMReadGroupRecord;
+import net.sf.samtools.SAMRecord;
+import org.broadinstitute.sting.BaseTest;
+import org.broadinstitute.sting.utils.collections.Pair;
+import org.broadinstitute.sting.utils.sam.ArtificialSAMUtils;
+import org.testng.Assert;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import java.util.*;
+
+/**
+ * Test routines for read-backed pileup.
+ */
+public class FragmentPileupUnitTest extends BaseTest {
+    private static SAMFileHeader header;
+
+    private class FragmentPileupTest extends TestDataProvider {
+        List<TestState> states = new ArrayList<TestState>();
+
+        private FragmentPileupTest(String name, int readLen, int leftStart, int rightStart, boolean leftIsFirst, boolean leftIsNegative) {
+            super(FragmentPileupTest.class, String.format("%s-leftIsFirst:%b-leftIsNegative:%b", name, leftIsFirst, leftIsNegative));
+
+            List<SAMRecord> pair = ArtificialSAMUtils.createPair(header, "readpair", readLen, leftStart, rightStart, leftIsFirst, leftIsNegative);
+            SAMRecord left = pair.get(0);
+            SAMRecord right = pair.get(1);
+
+            for ( int pos = leftStart; pos < rightStart + readLen; pos++) {
+                boolean posCoveredByLeft = pos >= left.getAlignmentStart() && pos <= left.getAlignmentEnd();
+                boolean posCoveredByRight = pos >= right.getAlignmentStart() && pos <= right.getAlignmentEnd();
+
+                if ( posCoveredByLeft || posCoveredByRight ) {
+                    List<SAMRecord> reads = new ArrayList<SAMRecord>();
+                    List<Integer> offsets = new ArrayList<Integer>();
+
+                    if ( posCoveredByLeft ) {
+                        reads.add(left);
+                        offsets.add(pos - left.getAlignmentStart());
+                    }
+
+                    if ( posCoveredByRight ) {
+                        reads.add(right);
+                        offsets.add(pos - right.getAlignmentStart());
+                    }
+
+                    boolean shouldBeFragment = posCoveredByLeft && posCoveredByRight;
+                    ReadBackedPileup pileup = new ReadBackedPileupImpl(null, reads, offsets);
+                    TestState testState = new TestState(shouldBeFragment, pileup);
+                    states.add(testState);
+                }
+            }
+        }
+    }
+
+    private class TestState {
+        boolean shouldBeFragment;
+        ReadBackedPileup pileup;
+
+        private TestState(final boolean shouldBeFragment, final ReadBackedPileup pileup) {
+            this.shouldBeFragment = shouldBeFragment;
+            this.pileup = pileup;
+        }
+    }
+
+    @DataProvider(name = "fragmentPileupTest")
+    public Object[][] createTests() {
+        for ( boolean leftIsFirst : Arrays.asList(true, false) ) {
+            for ( boolean leftIsNegative : Arrays.asList(true, false) ) {
+                // Overlapping pair
+                // ---->        [first]
+                //   <---       [second]
+                new FragmentPileupTest("overlapping-pair", 10, 1, 5, leftIsFirst, leftIsNegative);
+
+                // Non-overlapping pair
+                // ---->
+                //          <----
+                new FragmentPileupTest("nonoverlapping-pair", 10, 1, 15, leftIsFirst, leftIsNegative);
+            }
+        }
+
+        return FragmentPileupTest.getTests(FragmentPileupTest.class);
+    }
+
+    @Test(enabled = true, dataProvider = "fragmentPileupTest")
+    public void testMe(FragmentPileupTest test) {
+        for ( TestState testState : test.states ) {
+            ReadBackedPileup rbp = testState.pileup;
+            FragmentPileup fp = new FragmentPileup(rbp);
+            Assert.assertEquals(fp.getTwoReadPileup().size(), testState.shouldBeFragment ? 1 : 0);
+            Assert.assertEquals(fp.getOneReadPileup().size(), testState.shouldBeFragment ? 0 : 1);
+        }
+    }
+
+    @BeforeTest
+    public void setup() {
+        header = ArtificialSAMUtils.createArtificialSamHeader(1,1,1000);
+    }
+}

public/java/test/org/broadinstitute/sting/utils/runtime/ProcessControllerUnitTest.java

@@ -0,0 +1,517 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.runtime;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.broadinstitute.sting.BaseTest;
+import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
+import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.broadinstitute.sting.utils.io.IOUtils;
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+public class ProcessControllerUnitTest extends BaseTest {
+    private static final String NL = String.format("%n");
+
+    @Test(timeOut = 60 * 1000)
+    public void testDestroyThreadLocal() throws InterruptedException {
+        for (int i = 0; i < 3; i++) {
+            final ProcessController controller = ProcessController.getThreadLocal();
+            final ProcessSettings job = new ProcessSettings(
+                    new String[] {"sh", "-c", "echo Hello World && sleep 600 && echo Goodbye"});
+            job.getStdoutSettings().setBufferSize(-1);
+
+            Thread t = new Thread(new Runnable() {
+                @Override
+                public void run() {
+                    System.out.println("BACK: Starting on background thread");
+                    ProcessOutput result = controller.exec(job);
+                    // Assert in background thread doesn't make it to main thread but does print a trace.
+                    Assert.assertTrue(result.getExitValue() != 0, "Destroy-attempted job returned zero exit status");
+                    System.out.println("BACK: Background thread exiting");
+                }
+            });
+
+            System.out.println("MAIN: Starting background thread");
+            t.start();
+            System.out.println("MAIN: Sleeping main thread 3s");
+            Thread.sleep(3000);
+            System.out.println("MAIN: Destroying job");
+            controller.tryDestroy();
+            System.out.println("MAIN: Not waiting on background thread to exit");
+            // Using standard java.io this was blocking on linux.
+            // TODO: try again with NIO.
+            //t.join();
+            //System.out.println("MAIN: Background thread exited");
+        }
+    }
+
+    @Test
+    public void testReuseAfterError() {
+        ProcessController controller = new ProcessController();
+
+        ProcessSettings job;
+
+        for (int i = 0; i < 3; i++) {
+            // Test bad command
+            job = new ProcessSettings(new String[] {"no_such_command"});
+            try {
+                controller.exec(job);
+            } catch (ReviewedStingException e) {
+                /* Was supposed to throw an exception */
+            }
+
+            // Test exit != 0
+            job = new ProcessSettings(new String[] {"cat", "non_existent_file"});
+            int exitValue = controller.exec(job).getExitValue();
+            Assert.assertTrue(exitValue != 0, "'cat' non existent file returned 0");
+
+            // Text success
+            job = new ProcessSettings(new String[] {"echo", "Hello World"});
+            exitValue = controller.exec(job).getExitValue();
+            Assert.assertEquals(exitValue, 0, "Echo failed");
+        }
+    }
+
+    @Test
+    public void testEnvironment() {
+        String key = "MY_NEW_VAR";
+        String value = "value is here";
+
+        ProcessSettings job = new ProcessSettings(new String[] {"sh", "-c", "echo $"+key});
+        job.getStdoutSettings().setBufferSize(-1);
+        job.setRedirectErrorStream(true);
+
+        Map<String, String> env = new HashMap<String, String>(System.getenv());
+        env.put(key, value);
+        job.setEnvironment(env);
+
+        ProcessController controller = new ProcessController();
+        ProcessOutput result = controller.exec(job);
+        int exitValue = result.getExitValue();
+
+        Assert.assertEquals(exitValue, 0, "Echo environment variable failed");
+        Assert.assertEquals(result.getStdout().getBufferString(), value + NL, "Echo environment returned unexpected output");
+    }
+
+    @Test
+    public void testDirectory() throws IOException {
+        File dir = null;
+        try {
+            dir = IOUtils.tempDir("temp.", "").getCanonicalFile();
+
+            ProcessSettings job = new ProcessSettings(new String[] {"pwd"});
+            job.getStdoutSettings().setBufferSize(-1);
+            job.setRedirectErrorStream(true);
+            job.setDirectory(dir);
+
+            ProcessController controller = new ProcessController();
+            ProcessOutput result = controller.exec(job);
+            int exitValue = result.getExitValue();
+
+            Assert.assertEquals(exitValue, 0, "Getting working directory failed");
+
+            Assert.assertEquals(result.getStdout().getBufferString(), dir.getAbsolutePath() + NL,
+                    "Setting/getting working directory returned unexpected output");
+        } finally {
+            FileUtils.deleteQuietly(dir);
+        }
+    }
+
+    @Test
+    public void testReadStdInBuffer() {
+        String bufferText = "Hello from buffer";
+        ProcessSettings job = new ProcessSettings(new String[] {"cat"});
+        job.getStdoutSettings().setBufferSize(-1);
+        job.setRedirectErrorStream(true);
+        job.getStdinSettings().setInputBuffer(bufferText);
+
+        ProcessController controller = new ProcessController();
+        ProcessOutput output = controller.exec(job);
+
+        Assert.assertEquals(output.getStdout().getBufferString(), bufferText,
+                "Unexpected output from cat stdin buffer");
+    }
+
+    @Test
+    public void testReadStdInFile() {
+        File input = null;
+        try {
+            String fileText = "Hello from file";
+            input = IOUtils.writeTempFile(fileText, "stdin.", ".txt", null);
+
+            ProcessSettings job = new ProcessSettings(new String[] {"cat"});
+            job.getStdoutSettings().setBufferSize(-1);
+            job.setRedirectErrorStream(true);
+            job.getStdinSettings().setInputFile(input);
+
+            ProcessController controller = new ProcessController();
+            ProcessOutput output = controller.exec(job);
+
+            Assert.assertEquals(output.getStdout().getBufferString(), fileText,
+                    "Unexpected output from cat stdin file");
+        } finally {
+            FileUtils.deleteQuietly(input);
+        }
+    }
+
+    @Test
+    public void testWriteStdOut() {
+        ProcessSettings job = new ProcessSettings(new String[] {"echo", "Testing to stdout"});
+        // Not going to call the System.setOut() for now. Just running a basic visual test.
+        job.getStdoutSettings().printStandard(true);
+        job.setRedirectErrorStream(true);
+
+        System.out.println("testWriteStdOut: Writing two lines to std out...");
+        ProcessController controller = new ProcessController();
+        controller.exec(job);
+        job.setCommand(new String[]{"cat", "non_existent_file"});
+        controller.exec(job);
+        System.out.println("testWriteStdOut: ...two lines should have been printed to std out");
+    }
+
+    @Test
+    public void testErrorToOut() throws IOException {
+        File outFile = null;
+        File errFile = null;
+        try {
+            outFile = BaseTest.createTempFile("temp", "");
+            errFile = BaseTest.createTempFile("temp", "");
+
+            ProcessSettings job = new ProcessSettings(new String[]{"cat", "non_existent_file"});
+            job.getStdoutSettings().setOutputFile(outFile);
+            job.getStdoutSettings().setBufferSize(-1);
+            job.getStderrSettings().setOutputFile(errFile);
+            job.getStderrSettings().setBufferSize(-1);
+            job.setRedirectErrorStream(true);
+
+            ProcessOutput result = new ProcessController().exec(job);
+            int exitValue = result.getExitValue();
+
+            Assert.assertTrue(exitValue != 0, "'cat' non existent file returned 0");
+
+            String fileString, bufferString;
+
+            fileString = FileUtils.readFileToString(outFile);
+            Assert.assertTrue(fileString.length() > 0, "Out file was length 0");
+
+            bufferString = result.getStdout().getBufferString();
+            Assert.assertTrue(bufferString.length() > 0, "Out buffer was length 0");
+
+            Assert.assertFalse(result.getStdout().isBufferTruncated(), "Out buffer was truncated");
+            Assert.assertEquals(bufferString.length(), fileString.length(), "Out buffer length did not match file length");
+
+            fileString = FileUtils.readFileToString(errFile);
+            Assert.assertEquals(fileString, "", "Unexpected output to err file");
+
+            bufferString = result.getStderr().getBufferString();
+            Assert.assertEquals(bufferString, "", "Unexepected output to err buffer");
+        } finally {
+            FileUtils.deleteQuietly(outFile);
+            FileUtils.deleteQuietly(errFile);
+        }
+    }
+
+    @Test
+    public void testErrorToErr() throws IOException {
+        File outFile = null;
+        File errFile = null;
+        try {
+            outFile = BaseTest.createTempFile("temp", "");
+            errFile = BaseTest.createTempFile("temp", "");
+
+            ProcessSettings job = new ProcessSettings(new String[]{"cat", "non_existent_file"});
+            job.getStdoutSettings().setOutputFile(outFile);
+            job.getStdoutSettings().setBufferSize(-1);
+            job.getStderrSettings().setOutputFile(errFile);
+            job.getStderrSettings().setBufferSize(-1);
+            job.setRedirectErrorStream(false);
+
+            ProcessOutput result = new ProcessController().exec(job);
+            int exitValue = result.getExitValue();
+
+            Assert.assertTrue(exitValue != 0, "'cat' non existent file returned 0");
+
+            String fileString, bufferString;
+
+            fileString = FileUtils.readFileToString(errFile);
+            Assert.assertTrue(fileString.length() > 0, "Err file was length 0");
+
+            bufferString = result.getStderr().getBufferString();
+            Assert.assertTrue(bufferString.length() > 0, "Err buffer was length 0");
+
+            Assert.assertFalse(result.getStderr().isBufferTruncated(), "Err buffer was truncated");
+            Assert.assertEquals(bufferString.length(), fileString.length(), "Err buffer length did not match file length");
+
+            fileString = FileUtils.readFileToString(outFile);
+            Assert.assertEquals(fileString, "", "Unexpected output to out file");
+
+            bufferString = result.getStdout().getBufferString();
+            Assert.assertEquals(bufferString, "", "Unexepected output to out buffer");
+        } finally {
+            FileUtils.deleteQuietly(outFile);
+            FileUtils.deleteQuietly(errFile);
+        }
+    }
+
+    private static final String TRUNCATE_TEXT = "Hello World";
+    private static final byte[] TRUNCATE_OUTPUT_BYTES = (TRUNCATE_TEXT + NL).getBytes();
+
+    /**
+     * @return Test truncating content vs. not truncating (run at -1/+1 size)
+     */
+    @DataProvider(name = "truncateSizes")
+    public Object[][] getTruncateBufferSizes() {
+        int l = TRUNCATE_OUTPUT_BYTES.length;
+        return new Object[][]{
+                new Object[]{0, 0},
+                new Object[]{l, l},
+                new Object[]{l + 1, l},
+                new Object[]{l - 1, l - 1}
+        };
+    }
+
+    @Test(dataProvider = "truncateSizes")
+    public void testTruncateBuffer(int truncateLen, int expectedLen) {
+        byte[] expected = Arrays.copyOf(TRUNCATE_OUTPUT_BYTES, expectedLen);
+
+        String[] command = {"echo", TRUNCATE_TEXT};
+        ProcessController controller = new ProcessController();
+
+        ProcessSettings job = new ProcessSettings(command);
+        job.getStdoutSettings().setBufferSize(truncateLen);
+        ProcessOutput result = controller.exec(job);
+
+        int exitValue = result.getExitValue();
+
+        Assert.assertEquals(exitValue, 0,
+                String.format("Echo returned %d: %s", exitValue, TRUNCATE_TEXT));
+
+        byte[] bufferBytes = result.getStdout().getBufferBytes();
+
+        Assert.assertEquals(bufferBytes, expected,
+                String.format("Output buffer didn't match (%d vs %d)", expected.length, bufferBytes.length));
+
+        boolean truncated = result.getStdout().isBufferTruncated();
+
+        Assert.assertEquals(truncated, TRUNCATE_OUTPUT_BYTES.length > truncateLen,
+                "Unexpected buffer truncation result");
+    }
+
+    private static final String[] LONG_COMMAND = getLongCommand();
+    private static final String LONG_COMMAND_STRING = StringUtils.join(LONG_COMMAND, " ");
+    private static final String LONG_COMMAND_DESCRIPTION = "<long command>";
+
+    @DataProvider(name = "echoCommands")
+    public Object[][] getEchoCommands() {
+
+        new EchoCommand(new String[]{"echo", "Hello", "World"}, "Hello World" + NL);
+        new EchoCommand(new String[]{"echo", "'Hello", "World"}, "'Hello World" + NL);
+        new EchoCommand(new String[]{"echo", "Hello", "World'"}, "Hello World'" + NL);
+        new EchoCommand(new String[]{"echo", "'Hello", "World'"}, "'Hello World'" + NL);
+
+        String[] longCommand = new String[LONG_COMMAND.length + 1];
+        longCommand[0] = "echo";
+        System.arraycopy(LONG_COMMAND, 0, longCommand, 1, LONG_COMMAND.length);
+        new EchoCommand(longCommand, LONG_COMMAND_STRING + NL) {
+            @Override
+            public String toString() {
+                return LONG_COMMAND_DESCRIPTION;
+            }
+        };
+
+        return TestDataProvider.getTests(EchoCommand.class);
+    }
+
+    @Test(dataProvider = "echoCommands")
+    public void testEcho(EchoCommand script) throws IOException {
+        File outputFile = null;
+        try {
+            outputFile = BaseTest.createTempFile("temp", "");
+
+            ProcessSettings job = new ProcessSettings(script.command);
+            if (script.output != null) {
+                job.getStdoutSettings().setOutputFile(outputFile);
+                job.getStdoutSettings().setBufferSize(script.output.getBytes().length);
+            }
+
+            ProcessOutput result = new ProcessController().exec(job);
+            int exitValue = result.getExitValue();
+
+            Assert.assertEquals(exitValue, 0,
+                    String.format("Echo returned %d: %s", exitValue, script));
+
+            if (script.output != null) {
+
+                String fileString = FileUtils.readFileToString(outputFile);
+                Assert.assertEquals(fileString, script.output,
+                        String.format("Output file didn't match (%d vs %d): %s",
+                                fileString.length(), script.output.length(), script));
+
+                String bufferString = result.getStdout().getBufferString();
+                Assert.assertEquals(bufferString, script.output,
+                        String.format("Output content didn't match (%d vs %d): %s",
+                                bufferString.length(), script.output.length(), script));
+
+                Assert.assertFalse(result.getStdout().isBufferTruncated(),
+                        "Output content was truncated: " + script);
+            }
+        } finally {
+            FileUtils.deleteQuietly(outputFile);
+        }
+    }
+
+    @Test(expectedExceptions = ReviewedStingException.class)
+    public void testUnableToStart() {
+        ProcessSettings job = new ProcessSettings(new String[]{"no_such_command"});
+        new ProcessController().exec(job);
+    }
+
+    @DataProvider(name = "scriptCommands")
+    public Object[][] getScriptCommands() {
+        new ScriptCommand(true, "echo Hello World", "Hello World" + NL);
+        new ScriptCommand(false, "echo 'Hello World", null);
+        new ScriptCommand(false, "echo Hello World'", null);
+        new ScriptCommand(true, "echo 'Hello World'", "Hello World" + NL);
+        new ScriptCommand(true, "echo \"Hello World\"", "Hello World" + NL);
+        new ScriptCommand(false, "no_such_echo Hello World", null);
+        new ScriptCommand(true, "echo #", NL);
+        new ScriptCommand(true, "echo \\#", "#" + NL);
+        new ScriptCommand(true, "echo \\\\#", "\\#" + NL);
+
+        new ScriptCommand(true, "echo " + LONG_COMMAND_STRING, LONG_COMMAND_STRING + NL) {
+            @Override
+            public String toString() {
+                return LONG_COMMAND_DESCRIPTION;
+            }
+        };
+
+        return TestDataProvider.getTests(ScriptCommand.class);
+    }
+
+    @Test(dataProvider = "scriptCommands")
+    public void testScript(ScriptCommand script) throws IOException {
+        File scriptFile = null;
+        File outputFile = null;
+        try {
+            scriptFile = writeScript(script.content);
+            outputFile = BaseTest.createTempFile("temp", "");
+
+            ProcessSettings job = new ProcessSettings(new String[]{"sh", scriptFile.getAbsolutePath()});
+            if (script.output != null) {
+                job.getStdoutSettings().setOutputFile(outputFile);
+                job.getStdoutSettings().setBufferSize(script.output.getBytes().length);
+            }
+
+            ProcessOutput result = new ProcessController().exec(job);
+            int exitValue = result.getExitValue();
+
+            Assert.assertEquals(exitValue == 0, script.succeed,
+                    String.format("Script returned %d: %s", exitValue, script));
+
+            if (script.output != null) {
+
+                String fileString = FileUtils.readFileToString(outputFile);
+                Assert.assertEquals(fileString, script.output,
+                        String.format("Output file didn't match (%d vs %d): %s",
+                                fileString.length(), script.output.length(), script));
+
+                String bufferString = result.getStdout().getBufferString();
+                Assert.assertEquals(bufferString, script.output,
+                        String.format("Output content didn't match (%d vs %d): %s",
+                                bufferString.length(), script.output.length(), script));
+
+                Assert.assertFalse(result.getStdout().isBufferTruncated(),
+                        "Output content was truncated: " + script);
+            }
+        } finally {
+            FileUtils.deleteQuietly(scriptFile);
+            FileUtils.deleteQuietly(outputFile);
+        }
+    }
+
+    private static String[] getLongCommand() {
+        // This command fails on some systems with a 4096 character limit when run via the old sh -c "echo ...",
+        // but works on the same systems when run via sh <script>
+        int cnt = 500;
+        String[] command = new String[cnt];
+        for (int i = 1; i <= cnt; i++) {
+            command[i - 1] = String.format("%03d______", i);
+        }
+        return command;
+    }
+
+    private static File writeScript(String contents) {
+        try {
+            File file = BaseTest.createTempFile("temp", "");
+            FileUtils.writeStringToFile(file, contents);
+            return file;
+        } catch (IOException e) {
+            throw new UserException.BadTmpDir(e.getMessage());
+        }
+    }
+
+    private static class EchoCommand extends TestDataProvider {
+        public final String[] command;
+        public final String output;
+
+        public EchoCommand(String[] command, String output) {
+            super(EchoCommand.class);
+            this.command = command;
+            this.output = output;
+        }
+
+        @Override
+        public String toString() {
+            return StringUtils.join(command, " ");
+        }
+    }
+
+    public static class ScriptCommand extends TestDataProvider {
+        public final boolean succeed;
+        public final String content;
+        public final String output;
+
+        public ScriptCommand(boolean succeed, String content, String output) {
+            super(ScriptCommand.class);
+            this.succeed = succeed;
+            this.content = content;
+            this.output = output;
+        }
+
+        @Override
+        public String toString() {
+            return content;
+        }
+    }
+}

public/java/test/org/broadinstitute/sting/utils/variantcontext/VCFJarClassLoadingUnitTest.java

@@ -28,8 +28,8 @@ import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.testng.annotations.Test;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.net.MalformedURLException;
-import java.net.URI;
 import java.net.URL;
 import java.net.URLClassLoader;
 
@@ -39,10 +39,16 @@ import java.net.URLClassLoader;
 public class VCFJarClassLoadingUnitTest {
     @Test
     public void testVCFJarClassLoading() throws ClassNotFoundException, MalformedURLException {
-        URI vcfURI = new File("dist/vcf.jar").toURI();
+        URL[] jarURLs;
-        URI tribbleURI = getTribbleJarFile().toURI();
 
-        ClassLoader classLoader = new URLClassLoader(new URL[] {vcfURI.toURL(),tribbleURI.toURL()}, null);
+        try {
+            jarURLs = new URL[] { getVCFJarFile().toURI().toURL(), getTribbleJarFile().toURI().toURL() };
+        }
+        catch ( FileNotFoundException e ) {
+            throw new ReviewedStingException("Could not find the VCF jar and/or its dependencies", e);
+        }
+
+        ClassLoader classLoader = new URLClassLoader(jarURLs, null);
         classLoader.loadClass("org.broadinstitute.sting.utils.variantcontext.VariantContext");
         classLoader.loadClass("org.broadinstitute.sting.utils.codecs.vcf.VCFCodec");
         classLoader.loadClass("org.broadinstitute.sting.utils.codecs.vcf.VCF3Codec");
@@ -51,19 +57,49 @@ public class VCFJarClassLoadingUnitTest {
     }
 
     /**
-     * A very unsafe way of determining the current location of the Tribble jar file.  Assumes that
+     * Locates the tribble jar within the dist directory.
-     * the tribble jar (as opposed to the constituent tribble classes) is on the classpath.
      *
-     * This method might or might not work when built via IntelliJ's debugger.
+     * Makes the horrible assumption that tests will always be run from the root of a Sting clone,
+     * but this is much less problematic than using the classpath to locate tribble, since
+     * the classpath won't explicitly contain tribble when we're testing the fully-packaged
+     * GATK jar.
      *
-     * @return The file representing the tribble jar.
+     * @return The tribble jar file, if found
+     * @throws FileNotFoundException If we couldn't locate a tribble jar within the dist directory
      */
-    private File getTribbleJarFile() {
+    private File getTribbleJarFile() throws FileNotFoundException {
-        String[] classPath = System.getProperty("java.class.path").split(File.pathSeparator);
+        File distDir = new File("dist");
-        for(String classPathEntry: classPath) {
+        if ( ! distDir.isDirectory() ) {
-            if(classPathEntry.contains("tribble"))
+            throw new FileNotFoundException("The dist directory does not exist");
-                return new File(classPathEntry);
         }
-        throw new ReviewedStingException("Unable to find Tribble jar file");
+
+        for ( File distDirEntry : distDir.listFiles() ) {
+            if ( distDirEntry.getName().startsWith("tribble") && distDirEntry.getName().endsWith(".jar") ) {
+                return distDirEntry;
+            }
+        }
+
+        throw new FileNotFoundException("Could not find a tribble jar file in the dist directory.");
+    }
+
+    /**
+     * Locates the vcf jar within the dist directory.
+     *
+     * Makes the horrible assumption that tests will always be run from the root of a Sting clone,
+     * but this is much less problematic than using the classpath to locate vcf.jar, since
+     * the classpath won't explicitly contain vcf.jar when we're testing the fully-packaged
+     * GATK jar.
+     *
+     * @return The vcf jar file, if found
+     * @throws FileNotFoundException If we couldn't locate a vcf jar within the dist directory
+     */
+    private File getVCFJarFile() throws FileNotFoundException {
+        File vcfJar = new File("dist/vcf.jar");
+
+        if ( ! vcfJar.exists() ) {
+            throw new FileNotFoundException("Could not find dist/vcf.jar");
+        }
+
+        return vcfJar;
     }
 }

public/packages/CreatePackager.xsl

@@ -179,13 +179,19 @@
       </xsl:when>
     </xsl:choose>
   </xsl:for-each>
-    <xsl:for-each select="dir">
+  <xsl:for-each select="dir">
-        <fileset dir="{$staging.dir}">
+    <xsl:variable name="includes">
-            <xsl:attribute name="includes">
+      <xsl:choose>
-                <xsl:value-of select="concat(@name,'/**')"/>
+        <xsl:when test="@includes = ''">
-            </xsl:attribute>
+          <xsl:value-of select="concat(@name,'/**')"/>
-        </fileset>
+        </xsl:when>
-    </xsl:for-each>
+        <xsl:otherwise>
+          <xsl:value-of select="concat(@name,'/',@includes)"/>
+        </xsl:otherwise>
+      </xsl:choose>
+    </xsl:variable>
+    <fileset dir="{$staging.dir}" includes="{$includes}"/>
+  </xsl:for-each>
 </xsl:template>
 
 <!-- Determine the short name (filename w/o directory structure of the given filename -->

public/packages/GATKEngine.xml

@@ -32,6 +32,8 @@
       <package name="org.broad.tribble.**" />
       <!-- Workaround - depend on the logger impl required by JEXL -->
       <package name="org.apache.commons.logging.impl" />
+      <!-- R packages -->
+      <dir name="org/broadinstitute/sting/utils/R" includes="*.tar.gz" />
     </dependencies>
   </executable>
   <resources>

public/packages/Queue.xml

@@ -65,6 +65,10 @@
             <package name="ca.mcgill.mcb.pcingola.**" />
             <file path="snpEff_genes.ftl" />
             <file path="snpEff_summary.ftl" />
+            
+            <!-- R scripts -->
+            <dir name="org/broadinstitute/sting/queue" includes="**/*.R" />
+
         </dependencies>
         <modules>
             <module file="GATKEngine.xml"/>

public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala

@@ -33,6 +33,7 @@ import org.broadinstitute.sting.queue.engine.{QGraphSettings, QGraph}
 import collection.JavaConversions._
 import org.broadinstitute.sting.utils.classloader.PluginManager
 import org.broadinstitute.sting.utils.exceptions.UserException
+import org.broadinstitute.sting.utils.io.IOUtils
 
 /**
  * Entry point of Queue.  Compiles and runs QScripts passed in to the command line.
@@ -48,7 +49,6 @@ object QCommandLine extends Logging {
     val shutdownHook = new Thread {
       override def run() {
         logger.info("Shutting down jobs. Please wait...")
-        ProcessController.shutdown()
         qCommandLine.shutdown()
       }
     }
@@ -56,8 +56,12 @@ object QCommandLine extends Logging {
     Runtime.getRuntime.addShutdownHook(shutdownHook)
 
     try {
-      CommandLineProgram.start(qCommandLine, argv);
+      CommandLineProgram.start(qCommandLine, argv)
-      Runtime.getRuntime.removeShutdownHook(shutdownHook)
+      try {
+        Runtime.getRuntime.removeShutdownHook(shutdownHook)
+      } catch {
+        case _ => /* ignore, example 'java.lang.IllegalStateException: Shutdown in progress' */
+      }
       if (CommandLineProgram.result != 0)
         System.exit(CommandLineProgram.result);
     } catch {
@@ -80,6 +84,7 @@ class QCommandLine extends CommandLineProgram with Logging {
   private val qScriptManager = new QScriptManager
   private val qGraph = new QGraph
   private var qScriptClasses: File = _
+  private var shuttingDown = false
 
   private lazy val pluginManager = {
     qScriptClasses = IOUtils.tempDir("Q-Classes", "", settings.qSettings.tempDirectory)
@@ -118,10 +123,16 @@ class QCommandLine extends CommandLineProgram with Logging {
       script.onExecutionDone(qGraph.getFunctionsAndStatus(script.functions), qGraph.success)
       if ( ! settings.disableJobReport ) {
         val jobStringName = (QScriptUtils.?(settings.jobReportFile)).getOrElse(settings.qSettings.jobNamePrefix + ".jobreport.txt")
-        val jobReportFile = new File(jobStringName)
+
-        logger.info("Writing JobLogging GATKReport to file " + jobReportFile)
+        if (!shuttingDown) {
-        QJobReport.printReport(qGraph.getFunctionsAndStatus(script.functions), jobReportFile)
+          val reportFile = new File(jobStringName)
-        QJobReport.plotReport(settings.rScriptArgs, jobReportFile)
+          logger.info("Writing JobLogging GATKReport to file " + reportFile)
+          QJobReport.printReport(qGraph.getFunctionsAndStatus(script.functions), reportFile)
+
+          val pdfFile = new File(jobStringName + ".pdf")
+          logger.info("Plotting JobLogging GATKReport to file " + pdfFile)
+          QJobReport.plotReport(settings.rScriptArgs, reportFile, pdfFile)
+        }
       }
     }
 
@@ -163,6 +174,7 @@ class QCommandLine extends CommandLineProgram with Logging {
     Arrays.asList(new ScalaCompoundArgumentTypeDescriptor)
 
   def shutdown() = {
+    shuttingDown = true
     qGraph.shutdown()
     if (qScriptClasses != null) IOUtils.tryDelete(qScriptClasses)
   }

public/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala

@@ -2,7 +2,7 @@ package org.broadinstitute.sting.queue
 
 import scala.tools.nsc.{Global, Settings}
 import scala.tools.nsc.io.PlainFile
-import org.broadinstitute.sting.queue.util.{Logging, IOUtils}
+import org.broadinstitute.sting.queue.util.Logging
 import collection.JavaConversions._
 import java.io.File
 import scala.tools.nsc.reporters.AbstractReporter
@@ -20,7 +20,7 @@ class QScriptManager() extends Logging {
    * Compiles and loads the scripts in the files into the current classloader.
    * Heavily based on scala/src/compiler/scala/tools/ant/Scalac.scala
    */
-  def loadScripts(scripts: List[File], tempDir: File) = {
+  def loadScripts(scripts: List[File], tempDir: File) {
     if (scripts.size > 0) {
       val settings = new Settings((error: String) => logger.error(error))
       settings.deprecation.value = true
@@ -63,7 +63,7 @@ object QScriptManager extends Logging {
    * Heavily based on scala/src/compiler/scala/tools/nsc/reporters/ConsoleReporter.scala
    */
   private class Log4JReporter(val settings: Settings) extends AbstractReporter {
-    def displayPrompt = throw new UnsupportedOperationException("Unable to prompt the user.  Prompting should be off.")
+    def displayPrompt { throw new UnsupportedOperationException("Unable to prompt the user.  Prompting should be off.") }
 
     /**
      * Displays the message at position with severity.
@@ -71,7 +71,7 @@ object QScriptManager extends Logging {
      * @param msg Message to display.
      * @param severity Severity of the event.
      */
-    def display(posIn: Position, msg: String, severity: Severity) = {
+    def display(posIn: Position, msg: String, severity: Severity) {
       severity.count += 1
       val level = severity match {
         case INFO => Level.INFO
@@ -87,7 +87,6 @@ object QScriptManager extends Logging {
         case NoPosition =>
           printMessage(level, msg)
         case _ =>
-          val buf = new StringBuilder(msg)
           val file = pos.source.file
           printMessage(level, file.name+":"+pos.line+": "+msg)
           printSourceLine(level, pos)
@@ -97,7 +96,7 @@ object QScriptManager extends Logging {
     /**
      * Prints a summary count of warnings and errors.
      */
-    def printSummary() = {
+    def printSummary() {
       if (WARNING.count > 0)
         printMessage(Level.WARN, countElementsAsString(WARNING.count, "warning") + " found")
       if (ERROR.count > 0)
@@ -119,15 +118,16 @@ object QScriptManager extends Logging {
      * @param level Severity level.
      * @param pos Position in the file of the event.
      */
-    private def printColumnMarker(level: Level, pos: Position) =
+    private def printColumnMarker(level: Level, pos: Position) {
       if (pos.isDefined) { printMessage(level, " " * (pos.column - 1) + "^") }
+    }
 
     /**
      * Prints the message at the severity level.
      * @param level Severity level.
      * @param message Message content.
      */
-    private def printMessage(level: Level, message: String) = {
+    private def printMessage(level: Level, message: String) {
       logger.log(level, message)
     }
   }

public/scala/src/org/broadinstitute/sting/queue/engine/CommandLineJobRunner.scala

@@ -26,7 +26,8 @@ package org.broadinstitute.sting.queue.engine
 
 import org.broadinstitute.sting.queue.function.CommandLineFunction
 import java.io.File
-import org.broadinstitute.sting.queue.util.{Logging, IOUtils}
+import org.broadinstitute.sting.queue.util.Logging
+import org.broadinstitute.sting.utils.io.IOUtils
 
 /**
  * Runs a command line function.
@@ -69,7 +70,7 @@ trait CommandLineJobRunner extends JobRunner[CommandLineFunction] with Logging {
 
   override def init() {
     super.init()
-    var exec = new StringBuilder
+    val exec = new StringBuilder
     
     var dirs = Set.empty[File]
     for (dir <- function.jobDirectories)

public/scala/src/org/broadinstitute/sting/queue/engine/FunctionEdge.scala

@@ -2,7 +2,10 @@ package org.broadinstitute.sting.queue.engine
 
 import org.broadinstitute.sting.queue.function.QFunction
 import java.io.{StringWriter, PrintWriter}
-import org.broadinstitute.sting.queue.util.{Logging, IOUtils}
+import org.broadinstitute.sting.queue.util.Logging
+import org.broadinstitute.sting.utils.io.IOUtils
+import org.apache.commons.io.FileUtils
+import org.apache.commons.lang.StringUtils
 
 /**
  * An edge in the QGraph that runs a QFunction.
@@ -150,26 +153,19 @@ class FunctionEdge(val function: QFunction, val inputs: QNode, val outputs: QNod
   /**
    * Outputs the last lines of the error logs.
    */
-  private def tailError() = {
+  private def tailError() {
     val errorFile = functionErrorFile
     if (IOUtils.waitFor(errorFile, 120)) {
       val maxLines = 100
       val tailLines = IOUtils.tail(errorFile, maxLines)
       val nl = "%n".format()
       val summary = if (tailLines.size > maxLines) "Last %d lines".format(maxLines) else "Contents"
-      logger.error("%s of %s:%n%s".format(summary, errorFile, tailLines.mkString(nl)))
+      logger.error("%s of %s:%n%s".format(summary, errorFile, StringUtils.join(tailLines, nl)))
     } else {
       logger.error("Unable to access log file: %s".format(errorFile))
     }
   }
 
-  /**
-   * Writes the contents of the error to the error file.
-   */
-  private def writeError(content: String) {
-    IOUtils.writeContents(functionErrorFile, content)
-  }
-
   /**
    * Writes the stack trace to the error file.
    */
@@ -178,8 +174,8 @@ class FunctionEdge(val function: QFunction, val inputs: QNode, val outputs: QNod
     val printWriter = new PrintWriter(stackTrace)
     printWriter.println(function.description)
     e.printStackTrace(printWriter)
-    printWriter.close
+    printWriter.close()
-    IOUtils.writeContents(functionErrorFile, stackTrace.toString)
+    FileUtils.writeStringToFile(functionErrorFile, stackTrace.toString)
   }
 
   def getRunInfo = {

public/scala/src/org/broadinstitute/sting/queue/engine/InProcessRunner.scala

@@ -2,8 +2,8 @@ package org.broadinstitute.sting.queue.engine
 
 import org.broadinstitute.sting.queue.function.InProcessFunction
 import java.util.Date
-import org.broadinstitute.sting.queue.util.{Logging, IOUtils}
 import org.broadinstitute.sting.utils.Utils
+import org.apache.commons.io.FileUtils
 
 /**
  * Runs a function that executes in process and does not fork out an external process.
@@ -11,7 +11,7 @@ import org.broadinstitute.sting.utils.Utils
 class InProcessRunner(val function: InProcessFunction) extends JobRunner[InProcessFunction] {
   private var runStatus: RunnerStatus.Value = _
 
-  def start() = {
+  def start() {
     getRunInfo.startTime = new Date()
     getRunInfo.exechosts = Utils.resolveHostname()
     runStatus = RunnerStatus.RUNNING
@@ -20,7 +20,7 @@ class InProcessRunner(val function: InProcessFunction) extends JobRunner[InProce
 
     getRunInfo.doneTime = new Date()
     val content = "%s%nDone.".format(function.description)
-    IOUtils.writeContents(function.jobOutputFile, content)
+    FileUtils.writeStringToFile(function.jobOutputFile, content)
     runStatus = RunnerStatus.DONE
   }
 

public/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala

@@ -40,6 +40,7 @@ import collection.immutable.{TreeSet, TreeMap}
 import org.broadinstitute.sting.queue.function.scattergather.{ScatterFunction, CloneFunction, GatherFunction, ScatterGatherableFunction}
 import java.util.Date
 import org.broadinstitute.sting.utils.Utils
+import org.broadinstitute.sting.utils.io.IOUtils
 
 /**
  * The internal dependency tracker between sets of function input and output files.
@@ -416,8 +417,12 @@ class QGraph extends Logging {
           startedJobsToEmail = Set.empty[FunctionEdge]
         }
 
-        if (readyJobs.size == 0 && runningJobs.size > 0)
+        if (readyJobs.size == 0 && runningJobs.size > 0) {
-          Thread.sleep(nextRunningCheck(lastRunningCheck))
+          runningLock.synchronized {
+            if (running)
+              runningLock.wait(nextRunningCheck(lastRunningCheck))
+          }
+        }
 
         lastRunningCheck = System.currentTimeMillis
         updateStatus()
@@ -1002,7 +1007,12 @@ class QGraph extends Logging {
       true
     } else {
       !this.jobGraph.edgeSet.exists(edge => {
-        edge.isInstanceOf[FunctionEdge] && edge.asInstanceOf[FunctionEdge].status == RunnerStatus.FAILED
+        if (edge.isInstanceOf[FunctionEdge]) {
+          val status = edge.asInstanceOf[FunctionEdge].status
+          (status == RunnerStatus.PENDING || status == RunnerStatus.RUNNING || status == RunnerStatus.FAILED)
+        } else {
+          false
+        }
       })
     }
   }
@@ -1051,7 +1061,13 @@ class QGraph extends Logging {
   def shutdown() {
     // Signal the main thread to shutdown.
     running = false
-    // Wait for the thread to finish and exit normally.
+
+    // Try and wait for the thread to finish and exit normally.
+    runningLock.synchronized {
+      runningLock.notify()
+    }
+
+    // Start killing jobs.
     runningLock.synchronized {
       val runners = runningJobs.map(_.runner)
       runningJobs = Set.empty[FunctionEdge]

public/scala/src/org/broadinstitute/sting/queue/engine/drmaa/DrmaaJobManager.scala

@@ -56,6 +56,6 @@ class DrmaaJobManager extends CommandLineJobManager[DrmaaJobRunner] {
     updatedRunners
   }
   override def tryStop(runners: Set[DrmaaJobRunner]) {
-    runners.filterNot(_.jobId == null).foreach(_.tryStop())
+    runners.foreach(_.tryStop())
   }
 }

public/scala/src/org/broadinstitute/sting/queue/engine/drmaa/DrmaaJobRunner.scala

@@ -135,14 +135,18 @@ class DrmaaJobRunner(val session: Session, val function: CommandLineFunction) ex
 
   def tryStop() {
     session.synchronized {
-      try {
+      // Assumes that after being set the job may be
-        // Stop runners. SIGTERM(15) is preferred to SIGKILL(9).
+      // reassigned but will not be reset back to null
-        // Only way to send SIGTERM is for the Sys Admin set the terminate_method
+      if (jobId != null) {
-        // resource of the designated queue to SIGTERM
+        try {
-        session.control(jobId, Session.TERMINATE)
+          // Stop runners. SIGTERM(15) is preferred to SIGKILL(9).
-      } catch {
+          // Only way to send SIGTERM is for the Sys Admin set the terminate_method
-        case e =>
+          // resource of the designated queue to SIGTERM
-          logger.error("Unable to kill job " + jobId, e)
+          session.control(jobId, Session.TERMINATE)
+        } catch {
+          case e =>
+            logger.error("Unable to kill job " + jobId, e)
+        }
       }
     }
   }

public/scala/src/org/broadinstitute/sting/queue/engine/shell/ShellJobManager.scala

@@ -30,4 +30,5 @@ import org.broadinstitute.sting.queue.engine.CommandLineJobManager
 class ShellJobManager extends CommandLineJobManager[ShellJobRunner] {
   def runnerType = classOf[ShellJobRunner]
   def create(function: CommandLineFunction) = new ShellJobRunner(function)
+  override def tryStop(runners: Set[ShellJobRunner]) { runners.foreach(_.tryStop()) }
 }

public/scala/src/org/broadinstitute/sting/queue/engine/shell/ShellJobRunner.scala

@@ -25,41 +25,66 @@
 package org.broadinstitute.sting.queue.engine.shell
 
 import org.broadinstitute.sting.queue.function.CommandLineFunction
-import org.broadinstitute.sting.queue.util.ShellJob
 import org.broadinstitute.sting.queue.engine.{RunnerStatus, CommandLineJobRunner}
 import java.util.Date
-import org.broadinstitute.sting.gatk.phonehome.GATKRunReport
 import org.broadinstitute.sting.utils.Utils
+import org.broadinstitute.sting.utils.runtime.{ProcessSettings, OutputStreamSettings, ProcessController}
 
 /**
  * Runs jobs one at a time locally
  */
 class ShellJobRunner(val function: CommandLineFunction) extends CommandLineJobRunner {
-  private var runStatus: RunnerStatus.Value = _
+  // Controller on the thread that started the job
+  private var controller: ProcessController = null
 
   /**
    * Runs the function on the local shell.
    * @param function Command to run.
    */
   def start() {
-    val job = new ShellJob
+    val commandLine = Array("sh", jobScript.getAbsolutePath)
+    val stdoutSettings = new OutputStreamSettings
+    val stderrSettings = new OutputStreamSettings
+    val mergeError = (function.jobErrorFile != null)
 
-    job.workingDir = function.commandDirectory
+    stdoutSettings.setOutputFile(function.jobOutputFile, true)
-    job.outputFile = function.jobOutputFile
+    if (function.jobErrorFile != null)
-    job.errorFile = function.jobErrorFile
+      stderrSettings.setOutputFile(function.jobErrorFile, true)
 
-    job.shellScript = jobScript
+    if (logger.isDebugEnabled) {
+      stdoutSettings.printStandard(true)
+      stderrSettings.printStandard(true)
+    }
 
-    // Allow advanced users to update the job.
+    val processSettings = new ProcessSettings(
-    updateJobRun(job)
+      commandLine, mergeError, function.commandDirectory, null,
+      null, stdoutSettings, stderrSettings)
+
+    updateJobRun(processSettings)
 
     getRunInfo.startTime = new Date()
     getRunInfo.exechosts = Utils.resolveHostname()
     updateStatus(RunnerStatus.RUNNING)
-    job.run()
+    controller = ProcessController.getThreadLocal
+    val exitStatus = controller.exec(processSettings).getExitValue
     getRunInfo.doneTime = new Date()
-    updateStatus(RunnerStatus.DONE)
+    updateStatus(if (exitStatus == 0) RunnerStatus.DONE else RunnerStatus.FAILED)
   }
 
-  override def checkUnknownStatus() {}
+  /**
+   * Possibly invoked from a shutdown thread, find and
+   * stop the controller from the originating thread
+   */
+  def tryStop() {
+    // Assumes that after being set the job may be
+    // reassigned but will not be reset back to null
+    if (controller != null) {
+      try {
+        controller.tryDestroy()
+      } catch {
+        case e =>
+          logger.error("Unable to kill shell job: " + function.description)
+      }
+    }
+  }
 }

public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKScatterFunction.scala

@@ -27,7 +27,7 @@ package org.broadinstitute.sting.queue.extensions.gatk
 import org.broadinstitute.sting.utils.interval.IntervalUtils
 import java.io.File
 import collection.JavaConversions._
-import org.broadinstitute.sting.queue.util.IOUtils
+import org.broadinstitute.sting.utils.io.IOUtils
 import org.broadinstitute.sting.queue.function.scattergather.{CloneFunction, ScatterFunction}
 import org.broadinstitute.sting.commandline.Output
 

public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/RodBind.scala

@@ -1,7 +1,7 @@
 package org.broadinstitute.sting.queue.extensions.gatk
 
 import java.io.File
-import org.broadinstitute.sting.queue.util.FileExtension
+import org.broadinstitute.sting.utils.io.FileExtension
 import java.lang.String
 
 /**

public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/TaggedFile.scala

@@ -1,7 +1,7 @@
 package org.broadinstitute.sting.queue.extensions.gatk
 
 import java.io.File
-import org.broadinstitute.sting.queue.util.FileExtension
+import org.broadinstitute.sting.utils.io.FileExtension
 
 /**
  * Used to provide tagged -I input_file arguments to the GATK.