Merge branch 'master' of ssh://nickel.broadinstitute.org/humgen/gsa-scr1/gsa-engineering/git/unstable

public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java

@@ -97,7 +97,6 @@ public class HierarchicalMicroScheduler extends MicroScheduler implements Hierar
         if (!( walker instanceof TreeReducible ))
             throw new IllegalArgumentException("The GATK can currently run in parallel only with TreeReducible walkers");
 
-        traversalEngine.startTimers();
         ReduceTree reduceTree = new ReduceTree(this);
         initializeWalker(walker);
 

public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java

@@ -44,7 +44,6 @@ public class LinearMicroScheduler extends MicroScheduler {
      * @param shardStrategy A strategy for sharding the data.
      */
     public Object execute(Walker walker, ShardStrategy shardStrategy) {
-        traversalEngine.startTimers();
         walker.initialize();
         Accumulator accumulator = Accumulator.create(engine,walker);
 
@@ -54,6 +53,7 @@ public class LinearMicroScheduler extends MicroScheduler {
             if ( done || shard == null ) // we ran out of shards that aren't owned
                 break;
 
+            traversalEngine.startTimersIfNecessary();
             if(shard.getShardType() == Shard.ShardType.LOCUS) {
                 LocusWalker lWalker = (LocusWalker)walker;
                 WindowMaker windowMaker = new WindowMaker(shard, engine.getGenomeLocParser(), getReadIterator(shard), shard.getGenomeLocs(), engine.getSampleMetadata());

public/java/src/org/broadinstitute/sting/gatk/executive/ShardTraverser.java

@@ -57,6 +57,7 @@ public class ShardTraverser implements Callable {
 
     public Object call() {
         try {
+            traversalEngine.startTimersIfNecessary();
             long startTime = System.currentTimeMillis();
 
             Object accumulator = walker.reduceInit();

public/java/src/org/broadinstitute/sting/gatk/traversals/TraversalEngine.java

@@ -115,12 +115,13 @@ public abstract class TraversalEngine<M,T,WalkerType extends Walker<M,T>,Provide
     LinkedList<ProcessingHistory> history = new LinkedList<ProcessingHistory>();
 
     /** We use the SimpleTimer to time our run */
-    private SimpleTimer timer = new SimpleTimer("Traversal");
+    private SimpleTimer timer = null;
 
     // How long can we go without printing some progress info?
     private static final int PRINT_PROGRESS_CHECK_FREQUENCY_IN_CYCLES = 1000;
     private int printProgressCheckCounter = 0;
     private long lastProgressPrintTime = -1;                       // When was the last time we printed progress log?
+    private long MIN_ELAPSED_TIME_BEFORE_FIRST_PROGRESS = 120 * 1000; // in milliseconds
     private long PROGRESS_PRINT_FREQUENCY = 10 * 1000;             // in milliseconds
     private final double TWO_HOURS_IN_SECONDS = 2.0 * 60.0 * 60.0;
     private final double TWELVE_HOURS_IN_SECONDS = 12.0 * 60.0 * 60.0;
@@ -209,11 +210,16 @@ public abstract class TraversalEngine<M,T,WalkerType extends Walker<M,T>,Provide
         }
     }
     /**
-     * Should be called to indicate that we're going to process records and the timer should start ticking
+     * Should be called to indicate that we're going to process records and the timer should start ticking.  This
+     * function should be called right before any traversal work is done, to avoid counting setup costs in the
+     * processing costs and inflating the estimated runtime.
      */
-    public void startTimers() {
+    public void startTimersIfNecessary() {
-        timer.start();
+        if ( timer == null ) {
-        lastProgressPrintTime = timer.currentTime();
+            timer = new SimpleTimer("Traversal");
+            timer.start();
+            lastProgressPrintTime = timer.currentTime();
+        }
     }
 
     /**
@@ -224,7 +230,8 @@ public abstract class TraversalEngine<M,T,WalkerType extends Walker<M,T>,Provide
      * @return true if the maximum interval (in millisecs) has passed since the last printing
      */
     private boolean maxElapsedIntervalForPrinting(final long curTime, long lastPrintTime, long printFreq) {
-        return (curTime - lastPrintTime) > printFreq;
+        long elapsed = curTime - lastPrintTime;
+        return elapsed > printFreq && elapsed > MIN_ELAPSED_TIME_BEFORE_FIRST_PROGRESS;
     }
 
     /**

public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java

@@ -15,6 +15,7 @@ import org.broadinstitute.sting.gatk.walkers.RodWalker;
 import org.broadinstitute.sting.gatk.walkers.TreeReducible;
 import org.broadinstitute.sting.gatk.walkers.Window;
 import org.broadinstitute.sting.gatk.walkers.varianteval.evaluators.VariantEvaluator;
+import org.broadinstitute.sting.gatk.walkers.varianteval.stratifications.JexlExpression;
 import org.broadinstitute.sting.gatk.walkers.varianteval.stratifications.VariantStratifier;
 import org.broadinstitute.sting.gatk.walkers.varianteval.util.*;
 import org.broadinstitute.sting.gatk.walkers.variantrecalibration.Tranche;
@@ -24,6 +25,7 @@ import org.broadinstitute.sting.utils.codecs.vcf.VCFHeader;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFUtils;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.broadinstitute.sting.utils.exceptions.StingException;
+import org.broadinstitute.sting.utils.exceptions.UserException;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 import org.broadinstitute.sting.utils.variantcontext.VariantContextUtils;
@@ -224,12 +226,6 @@ public class VariantEvalWalker extends RodWalker<Integer, Integer> implements Tr
         }
         sampleNamesForStratification.add(ALL_SAMPLE_NAME);
 
-        // Initialize select expressions
-        for (VariantContextUtils.JexlVCMatchExp jexl : VariantContextUtils.initializeMatchExps(SELECT_NAMES, SELECT_EXPS)) {
-            SortableJexlVCMatchExp sjexl = new SortableJexlVCMatchExp(jexl.name, jexl.exp);
-            jexlExpressions.add(sjexl);
-        }
-
         // Add select expressions for anything in the tranches file
         if ( TRANCHE_FILENAME != null ) {
             // we are going to build a few select names automatically from the tranches file
@@ -240,16 +236,27 @@ public class VariantEvalWalker extends RodWalker<Integer, Integer> implements Tr
             }
         }
 
+        // Initialize select expressions
+        for (VariantContextUtils.JexlVCMatchExp jexl : VariantContextUtils.initializeMatchExps(SELECT_NAMES, SELECT_EXPS)) {
+            SortableJexlVCMatchExp sjexl = new SortableJexlVCMatchExp(jexl.name, jexl.exp);
+            jexlExpressions.add(sjexl);
+        }
+
         // Initialize the set of stratifications and evaluations to use
         stratificationObjects = variantEvalUtils.initializeStratificationObjects(this, NO_STANDARD_STRATIFICATIONS, STRATIFICATIONS_TO_USE);
         Set<Class<? extends VariantEvaluator>> evaluationObjects = variantEvalUtils.initializeEvaluationObjects(NO_STANDARD_MODULES, MODULES_TO_USE);
+        boolean usingJEXL = false;
         for ( VariantStratifier vs : getStratificationObjects() ) {
             if ( vs.getClass().getSimpleName().equals("Filter") )
                 byFilterIsEnabled = true;
             else if ( vs.getClass().getSimpleName().equals("Sample") )
                 perSampleIsEnabled = true;
+            usingJEXL = usingJEXL || vs.getClass().equals(JexlExpression.class);
         }
 
+        if ( TRANCHE_FILENAME != null && ! usingJEXL )
+            throw new UserException.BadArgumentValue("tf", "Requires the JexlExpression ST to enabled");
+
         // Initialize the evaluation contexts
         evaluationContexts = variantEvalUtils.initializeEvaluationContexts(stratificationObjects, evaluationObjects, null, null);
 

public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/util/VariantEvalUtils.java

@@ -277,7 +277,7 @@ public class VariantEvalUtils {
      * @return a new VariantContext with just the requested samples
      */
     public VariantContext getSubsetOfVariantContext(VariantContext vc, Collection<String> sampleNames) {
-        VariantContext vcsub = vc.subContextFromGenotypes(vc.getGenotypes(sampleNames).values(), vc.getAlleles());
+        VariantContext vcsub = vc.subContextFromGenotypes(vc.getGenotypes(sampleNames).values());
 
         HashMap<String, Object> newAts = new HashMap<String, Object>(vcsub.getAttributes());
 

public/java/src/org/broadinstitute/sting/utils/clipreads/ReadClipper.java

@@ -63,6 +63,10 @@ public class ReadClipper {
         if (start < 0 || stop > read.getReadLength() - 1)
             throw new ReviewedStingException("Trying to clip before the start or after the end of a read");
 
+        // TODO add requires statement/check in the Hardclip function
+        if ( start > stop )
+            stop = ReadUtils.getReadCoordinateForReferenceCoordinate(read, ReadUtils.getRefCoordSoftUnclippedEnd(read));
+
         //System.out.println("Clipping start/stop: " + start + "/" + stop);
         this.addOp(new ClippingOp(start, stop));
         SAMRecord clippedRead = clipRead(ClippingRepresentation.HARDCLIP_BASES);