Infrastructure for combining VariantEvaluations

-- Not hooked up yet, so the output of VariantEval should be the same as before
-- Implemented a VariantEvalUnitTest that tests the low level strat / eval combinatorics and counting routines
-- Better docs throughout

public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalReportWriter.java

@@ -68,7 +68,7 @@ public class VariantEvalReportWriter {
      */
     public final void writeReport(final PrintStream out) {
         for ( int key = 0; key < stratManager.size(); key++ ) {
-            final String stratStateString = stratManager.getStratsAndStatesForKeyString(key);
+            final String stratStateString = stratManager.getStratsAndStatesStringForKey(key);
             final List<Pair<VariantStratifier, Object>> stratsAndStates = stratManager.getStratsAndStatesForKey(key);
             final EvaluationContext nec = stratManager.get(key);
 

public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java

@@ -17,6 +17,7 @@ import org.broadinstitute.sting.gatk.walkers.RodWalker;
 import org.broadinstitute.sting.gatk.walkers.TreeReducible;
 import org.broadinstitute.sting.gatk.walkers.Window;
 import org.broadinstitute.sting.gatk.walkers.varianteval.evaluators.VariantEvaluator;
+import org.broadinstitute.sting.gatk.walkers.varianteval.stratifications.DynamicStratification;
 import org.broadinstitute.sting.gatk.walkers.varianteval.stratifications.IntervalStratification;
 import org.broadinstitute.sting.gatk.walkers.varianteval.stratifications.VariantStratifier;
 import org.broadinstitute.sting.gatk.walkers.varianteval.stratifications.manager.StratificationManager;
@@ -221,6 +222,7 @@ public class VariantEvalWalker extends RodWalker<Integer, Integer> implements Tr
 
     // The set of all possible evaluation contexts
     StratificationManager<VariantStratifier, EvaluationContext> stratManager;
+    //Set<DynamicStratification> dynamicStratifications = Collections.emptySet();
 
     /**
      * Initialize the stratifications, evaluations, evaluation contexts, and reporting object
@@ -360,6 +362,14 @@ public class VariantEvalWalker extends RodWalker<Integer, Integer> implements Tr
         if (tracker != null) {
             String aastr = (ancestralAlignments == null) ? null : new String(ancestralAlignments.getSubsequenceAt(ref.getLocus().getContig(), ref.getLocus().getStart(), ref.getLocus().getStop()).getBases());
 
+//            // update the dynamic stratifications
+//            for (final VariantContext vc : tracker.getValues(evals, ref.getLocus())) {
+//                // don't worry -- DynamicStratification only work with one eval object
+//                for ( final DynamicStratification ds :  dynamicStratifications ) {
+//                    ds.update(vc);
+//                }
+//            }
+
             //      --------- track ---------           sample  - VariantContexts -
             HashMap<RodBinding<VariantContext>, HashMap<String, Collection<VariantContext>>> evalVCs = variantEvalUtils.bindVariantContexts(tracker, ref, evals, byFilterIsEnabled, true, perSampleIsEnabled, mergeEvals);
             HashMap<RodBinding<VariantContext>, HashMap<String, Collection<VariantContext>>> compVCs = variantEvalUtils.bindVariantContexts(tracker, ref, comps, byFilterIsEnabled, false, false, false);
@@ -456,13 +466,13 @@ public class VariantEvalWalker extends RodWalker<Integer, Integer> implements Tr
      * @param sampleName
      * @return
      */
-    private Collection<EvaluationContext> getEvaluationContexts(final RefMetaDataTracker tracker,
+    protected Collection<EvaluationContext> getEvaluationContexts(final RefMetaDataTracker tracker,
-                                                                   final ReferenceContext ref,
+                                                                  final ReferenceContext ref,
-                                                                   final VariantContext eval,
+                                                                  final VariantContext eval,
-                                                                   final String evalName,
+                                                                  final String evalName,
-                                                                   final VariantContext comp,
+                                                                  final VariantContext comp,
-                                                                   final String compName,
+                                                                  final String compName,
-                                                                   final String sampleName ) {
+                                                                  final String sampleName ) {
         final List<List<Object>> states = new LinkedList<List<Object>>();
         for ( final VariantStratifier vs : stratManager.getStratifiers() ) {
             states.add(vs.getRelevantStates(ref, tracker, comp, compName, eval, evalName, sampleName));

public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/VariantEvaluator.java

@@ -4,6 +4,7 @@ import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.varianteval.VariantEvalWalker;
+import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
 public abstract class VariantEvaluator implements Comparable<VariantEvaluator> {
@@ -67,4 +68,41 @@ public abstract class VariantEvaluator implements Comparable<VariantEvaluator> {
     public int compareTo(final VariantEvaluator variantEvaluator) {
         return getSimpleName().compareTo(variantEvaluator.getSimpleName());
     }
+
+    /**
+     * Evaluation modules that override this function to indicate that they support
+     * combining the results of two independent collections of eval data into
+     * a single meaningful result.  The purpose of this interface is to
+     * allow us to cut up the input data into many independent stratifications, and then
+     * at the end of the eval run decide which stratifications to combine.  This is
+     * important in the case of AC, where you may have thousands of distinct AC
+     * values that chop up the number of variants to too small a number of variants,
+     * and you'd like to combine the AC values into ranges containing some percent
+     * of the data.
+     *
+     * For example, suppose you have an eval that
+     * counts variants in a variable nVariants.  If you want to be able to combine
+     * multiple evaluations of this type, overload the combine function
+     * with a function that sets this.nVariants += other.nVariants.
+     *
+     * Add in the appropriate fields of the VariantEvaluator T
+     * (of the same type as this object) to the values of this object.
+     *
+     * The values in this and other are implicitly independent, so that
+     * the values can be added together.
+     *
+     * @param other a VariantEvaluator of the same type of this object
+     */
+    public void combine(final VariantEvaluator other) {
+        throw new ReviewedStingException(getSimpleName() + " doesn't support combining results, sorry");
+    }
+
+    /**
+     * Must be overloaded to return true for evaluation modules that support the combine operation
+     *
+     * @return
+     */
+    public boolean supportsCombine() {
+        return false;
+    }
 }

public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/DynamicStratification.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2012, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.gatk.walkers.varianteval.stratifications;
+
+import org.broadinstitute.sting.utils.variantcontext.VariantContext;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Tag this stratification as dynamically determining the final strat based on the input data
+ *
+ * The paradigm here is simple.  We upfront create a strat with N states that reflect the finest grained
+ * possible division of the data.  The data is processed, and statistics collected for each of the N states.
+ * An update call is made to the stratification for evaluation VariantContext during each map call,
+ * allowing the strat to collect data about the usage of each state.  A final call requests that
+ * the stratification map down the N states into M states (typically less than N, not necessarily
+ * a subset of N).  This is provided by returning a map from each of M state -> N states and
+ * the VariantEval walker will combine all of the evaluations for N into a single value for
+ * each M.
+ *
+ * For example, suppose I have a dynamic strat called AC, adopting 7 possible values 0,1,2,3,4,5,6.  This
+ * strats tracks the number of eval vcs for each state, with final counts 0=1, 1=100, 2=10, 3=5, 4=3, 5=2, 6=1.
+ * The stratification attempts to combine the strats down to so that each state has approximately the same
+ * fraction of the data in each bin.  Overall there is 1+100+10+5+3+2+1=124 observations and 7 bins so we really
+ * want ~ 18 observations in each bin.  So we merge 3-6 with 5+3+2+1 = 11 and keep 2, 1, and 0 as distinct bins.  We
+ * return a map from 0 -> 0, 1 -> 1, 2 -> 2, 3-6 -> {3,4,5,6}.
+ *
+ * TODO - some open implementation questions
+ * -- We should only create one stratifier overall.  How do we track this?  When we create the stratifiers
+ *    perhaps we can look at them and create a tracker?
+ * -- How do we create a new stratifier based on the finalStratifications() given the framework?  Conceptually
+ *    this new thing is itself a stratifier, just like before, but it's states are determined at the end.  We'd
+ *    then like to call not getRelevantStates but a different function that accepts an old state and returns
+ *    the new state.  Perhaps the process should look like:
+ *          finalizeStratification -> new Stratifier whose states are the final ones
+ *          getNewState(old state) -> new state (one of those in getFinalStratification)
+ *
+ * @author Mark DePristo
+ * @since 4/9/12
+ */
+public interface DynamicStratification {
+    public void update(final VariantContext eval);
+    public VariantStratifier finalizeStratification();
+    public Object getFinalState(final Object oldState);
+}

public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/manager/StratificationManager.java

@@ -26,6 +26,8 @@ package org.broadinstitute.sting.gatk.walkers.varianteval.stratifications.manage
 
 import com.google.java.contract.Ensures;
 import com.google.java.contract.Requires;
+import org.broadinstitute.sting.gatk.walkers.varianteval.util.EvaluationContext;
+import org.broadinstitute.sting.utils.Utils;
 import org.broadinstitute.sting.utils.collections.Pair;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 
@@ -226,7 +228,7 @@ public class StratificationManager<K extends Stratifier, V> implements Map<List<
         return states;
     }
 
-    public String getStratsAndStatesForKeyString(final int key) {
+    public String getStratsAndStatesStringForKey(final int key) {
         if ( keyStrings.get(key) == null ) {
             StringBuilder b = new StringBuilder();
             for ( int i = 0; i < stratifiers.size(); i++ ) {
@@ -348,4 +350,77 @@ public class StratificationManager<K extends Stratifier, V> implements Map<List<
         }
         return combined;
     }
-}
+
+    public interface Combiner<V> {
+        /** take two values of type V and return a combined value of type V */
+        public V combine(final V lhs, final V rhs);
+    }
+
+    /**
+     * Remaps the stratifications from one stratification set to another, combining
+     * the values in V according to the combiner function.
+     *
+     * stratifierToReplace defines a set of states S1, while newStratifier defines
+     * a new set S2.  remappedStates is a map from all of S1 into at least some of
+     * S2.  This function creates a new, fully initialized manager where all of the
+     * data in this new manager is derived from the original data in this object
+     * combined according to the mapping remappedStates.  When multiple
+     * elements of S1 can map to the same value in S2, these are sequentially
+     * combined by the function combiner.  Suppose for example at states s1, s2, and
+     * s3 all map to N1.  Eventually the value associated with state N1 would be
+     *
+     *   value(N1) = combine(value(s1), combine(value(s2), value(s3))
+     *
+     * in some order for s1, s2, and s3, which is not defined.  Note that this function
+     * only supports combining one stratification at a time, but in principle a loop over
+     * stratifications and this function could do the multi-dimensional collapse.
+     *
+     * @param stratifierToReplace
+     * @param newStratifier
+     * @param combiner
+     * @param remappedStates
+     * @return
+     */
+    public StratificationManager<K, V> combineStrats(final K stratifierToReplace,
+                                                     final K newStratifier,
+                                                     final Combiner<V> combiner,
+                                                     final Map<Object, Object> remappedStates) {
+        // make sure the mapping is reasonable
+        if ( ! newStratifier.getAllStates().containsAll(remappedStates.values()) )
+            throw new ReviewedStingException("combineStrats: remapped states contains states not found in newStratifer state set");
+
+        if ( ! remappedStates.keySet().containsAll(stratifierToReplace.getAllStates()) )
+            throw new ReviewedStingException("combineStrats: remapped states missing mapping for some states");
+
+        // the new strats are the old ones with the single replacement
+        final List<K> newStrats = new ArrayList<K>(getStratifiers());
+        final int stratOffset = newStrats.indexOf(stratifierToReplace);
+        if ( stratOffset == -1 )
+            throw new ReviewedStingException("Could not find strat to replace " + stratifierToReplace + " in existing strats " + newStrats);
+        newStrats.set(stratOffset, newStratifier);
+
+        // create an empty but fully initialized new manager
+        final StratificationManager<K, V> combined = new StratificationManager<K, V>(newStrats);
+
+        // for each key, get its state, update it according to the map, and update the combined manager
+        for ( int key = 0; key < size(); key++ ) {
+            // the new state is just the old one with the replacement
+            final List<Object> newStates = new ArrayList<Object>(getStatesForKey(key));
+            final Object oldState = newStates.get(stratOffset);
+            final Object newState = remappedStates.get(oldState);
+            newStates.set(stratOffset, newState);
+
+            // look up the new key given the new state
+            final int combinedKey = combined.getKey(newStates);
+            if ( combinedKey == -1 ) throw new ReviewedStingException("Couldn't find key for states: " + Utils.join(",", newStates));
+
+            // combine the old value with whatever new value is in combined already
+            final V combinedValue = combiner.combine(combined.get(combinedKey), get(key));
+
+            // update the value associated with combined key
+            combined.set(combinedKey, combinedValue);
+        }
+
+        return combined;
+    }
+}

public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/util/EvaluationContext.java

@@ -6,6 +6,7 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.varianteval.VariantEvalWalker;
 import org.broadinstitute.sting.gatk.walkers.varianteval.evaluators.VariantEvaluator;
 import org.broadinstitute.sting.gatk.walkers.varianteval.stratifications.VariantStratifier;
+import org.broadinstitute.sting.gatk.walkers.varianteval.stratifications.manager.StratificationManager;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.broadinstitute.sting.utils.exceptions.StingException;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
@@ -14,15 +15,23 @@ import java.util.*;
 
 public final class EvaluationContext {
     // NOTE: must be hashset to avoid O(log n) cost of iteration in the very frequently called apply function
-    private final HashSet<VariantEvaluator> evaluationInstances;
+    final VariantEvalWalker walker;
+    private final ArrayList<VariantEvaluator> evaluationInstances;
+    private final Set<Class<? extends VariantEvaluator>> evaluationClasses;
 
     public EvaluationContext(final VariantEvalWalker walker, final Set<Class<? extends VariantEvaluator>> evaluationClasses) {
-        evaluationInstances = new HashSet<VariantEvaluator>(evaluationClasses.size());
+        this(walker, evaluationClasses, true);
+    }
+
+    private EvaluationContext(final VariantEvalWalker walker, final Set<Class<? extends VariantEvaluator>> evaluationClasses, final boolean doInitialize) {
+        this.walker = walker;
+        this.evaluationClasses = evaluationClasses;
+        this.evaluationInstances = new ArrayList<VariantEvaluator>(evaluationClasses.size());
 
         for ( final Class<? extends VariantEvaluator> c : evaluationClasses ) {
             try {
                 final VariantEvaluator eval = c.newInstance();
-                eval.initialize(walker);
+                if ( doInitialize ) eval.initialize(walker);
                 evaluationInstances.add(eval);
             } catch (InstantiationException e) {
                 throw new ReviewedStingException("Unable to instantiate eval module '" + c.getSimpleName() + "'", e);
@@ -62,4 +71,20 @@ public final class EvaluationContext {
             }
         }
     }
+
+    public void combine(final EvaluationContext rhs) {
+        for ( int i = 0; i < evaluationInstances.size(); i++ )
+            evaluationInstances.get(i).combine(rhs.evaluationInstances.get(i));
+    }
+
+    public final static EvaluationContextCombiner COMBINER = new EvaluationContext.EvaluationContextCombiner();
+    private static class EvaluationContextCombiner implements StratificationManager.Combiner<EvaluationContext> {
+        @Override
+        public EvaluationContext combine(EvaluationContext lhs, final EvaluationContext rhs) {
+            if ( lhs == null )
+                lhs = new EvaluationContext(rhs.walker, rhs.evaluationClasses, false);
+            lhs.combine(rhs);
+            return lhs;
+        }
+    }
 }

public/java/src/org/broadinstitute/sting/utils/Utils.java

@@ -750,4 +750,18 @@ public class Utils {
     public static String formattedRatio(final long num, final long denom) {
         return denom == 0 ? "NA" : String.format("%.2f", num / (1.0 * denom));
     }
+
+    /**
+     * Create a constant map that maps each value in values to itself
+     * @param values
+     * @param <T>
+     * @return
+     */
+    public static <T> Map<T, T> makeIdentityFunctionMap(Collection<T> values) {
+        Map<T,T> map = new HashMap<T, T>(values.size());
+        for ( final T value : values )
+            map.put(value, value);
+        return Collections.unmodifiableMap(map);
+    }
+
 }

public/java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalUnitTest.java

@@ -0,0 +1,277 @@
+/*
+ * Copyright (c) 2012, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+// our package
+package org.broadinstitute.sting.gatk.walkers.varianteval;
+
+
+// the imports for unit testing.
+
+import org.broadinstitute.sting.BaseTest;
+import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
+import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
+import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
+import org.broadinstitute.sting.gatk.walkers.varianteval.evaluators.VariantEvaluator;
+import org.broadinstitute.sting.gatk.walkers.varianteval.stratifications.VariantStratifier;
+import org.broadinstitute.sting.gatk.walkers.varianteval.stratifications.manager.StratificationManager;
+import org.broadinstitute.sting.gatk.walkers.varianteval.util.EvaluationContext;
+import org.broadinstitute.sting.utils.Utils;
+import org.broadinstitute.sting.utils.variantcontext.Allele;
+import org.broadinstitute.sting.utils.variantcontext.VariantContext;
+import org.broadinstitute.sting.utils.variantcontext.VariantContextBuilder;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import java.util.*;
+
+
+public class VariantEvalUnitTest extends BaseTest {
+    VariantEvalWalker VEwalker;
+    VariantContext eval;
+
+
+    @BeforeMethod
+    public void init() {
+        VEwalker = new VariantEvalWalker();
+        eval = new VariantContextBuilder("x", "chr1", 1, 1, Collections.singleton(Allele.create("A", true))).make();
+    }
+
+    // --------------------------------------------------------------------------------
+    //
+    // Test stratifications / evaluations
+    //
+    // --------------------------------------------------------------------------------
+
+    private class StratifiedEvalTestProvider extends TestDataProvider {
+        final List<VariantStratifier> stratificationObjects = new ArrayList<VariantStratifier>();
+        final Set<Class<? extends VariantEvaluator>> evaluationObjects = new HashSet<Class<? extends VariantEvaluator>>();
+        final List<Integer> expectedCounts;
+        final int maxI;
+
+        /**
+         *
+         * @param maxI test integers from 1 ... maxI
+         * @param expectedCounts the expected number of integers from 1 ... maxI divisible by each combination, in order, of allStates
+         * @param allStates all stratification tests, in order
+         */
+        public StratifiedEvalTestProvider(int maxI,
+                                          final List<Integer> expectedCounts,
+                                          final List<Integer> ... allStates) {
+            super(StratifiedEvalTestProvider.class);
+
+            this.maxI = maxI;
+            this.expectedCounts = expectedCounts;
+            this.evaluationObjects.add(CounterEval.class);
+
+            String stateName = "";
+            for ( List<Integer> states : allStates ) {
+                stratificationObjects.add(new IntegerStratifier(states));
+                stateName = stateName + Utils.join(",", states) + " ";
+            }
+
+            setName(String.format("maxI=%d expectedCounts=%s states=%s", maxI, Utils.join(",", expectedCounts), stateName));
+        }
+    }
+
+    /**
+     * Test stratifier -> holds a list of integers, and the states are if the integer value of evalName is divisable
+     * by that number
+     */
+    public static class IntegerStratifier extends VariantStratifier {
+        final List<Integer> integers;
+
+        private IntegerStratifier(final List<Integer> integers) {
+            this.integers = integers;
+            initialize();
+        }
+
+        @Override
+        public void initialize() {
+            states.addAll(integers);
+        }
+
+        @Override
+        public List<Object> getRelevantStates(final ReferenceContext ref, final RefMetaDataTracker tracker, final VariantContext comp, final String compName, final VariantContext eval, final String evalName, final String sampleName) {
+            int i = Integer.valueOf(evalName); // a terrible hack, but we can now provide accessible states
+            List<Object> states = new ArrayList<Object>();
+            for ( int state : integers )
+                if ( i % state == 0 )
+                    states.add(state);
+            return states;
+        }
+    }
+
+    /**
+     * Test evaluator -> just counts the number of calls to update1
+     */
+    public static class CounterEval extends VariantEvaluator {
+        public int count = 0;
+
+        @Override public int getComparisonOrder() { return 1; }
+
+        @Override
+        public void update1(final VariantContext eval, final RefMetaDataTracker tracker, final ReferenceContext ref, final AlignmentContext context) {
+            count++;
+        }
+
+        @Override
+        public boolean supportsCombine() {
+            return true;
+        }
+
+        @Override
+        public void combine(final VariantEvaluator other) {
+            this.count += ((CounterEval)other).count;
+        }
+    }
+
+    private void initialize(StratifiedEvalTestProvider cfg) {
+        VEwalker.createStratificationStates(cfg.stratificationObjects, cfg.evaluationObjects);
+
+        final RefMetaDataTracker tracker = new RefMetaDataTracker();
+        final ReferenceContext ref = null;
+        final VariantContext comp = null;
+        final String compName = null, sampleName = null;
+
+        // increment eval counts for each stratification of divisors of i from from 1...maxI
+        for ( int i = 1; i <= cfg.maxI; i++ ) {
+            final String evalName = String.valueOf(i); // terrible hack to stratify by divisor
+            for ( EvaluationContext nec : VEwalker.getEvaluationContexts(tracker, ref, eval, evalName, comp, compName, sampleName) ) {
+                synchronized (nec) {
+                    nec.apply(tracker, ref, null, comp, eval);
+                }
+            }
+        }
+    }
+
+    @DataProvider(name = "StratifiedEvalTestProvider")
+    public Object[][] makeStratifiedEvalTestProvider() {
+
+        new StratifiedEvalTestProvider(4, // test 1, 2, 3, 4
+                Arrays.asList(4, 2), //  4 divisible by 1, 2 by 2
+                Arrays.asList(1, 2));
+
+        new StratifiedEvalTestProvider(6, // test 1, 2, 3, 4, 5, 6
+                Arrays.asList(6, 3, 2), //  6 divisible by 1, 3 by 2, 2 by 3
+                Arrays.asList(1, 2, 3));
+
+        // test that some states can be empty -- does this work in VE?
+        new StratifiedEvalTestProvider(6,
+                Arrays.asList(3, 2),
+                Arrays.asList(2, 3));
+
+        // test a single stratification
+        new StratifiedEvalTestProvider(6,
+                Arrays.asList(3),
+                Arrays.asList(2));
+
+        // test a meaningless state
+        new StratifiedEvalTestProvider(4, // test 1, 2, 3, 4
+                Arrays.asList(4, 2), //  4 divisible by 1, 2 by 2
+                Arrays.asList(1, 2), Arrays.asList(1));
+
+        // test a adding a state that divides space in half
+        new StratifiedEvalTestProvider(4,
+                Arrays.asList(2, 2),
+                Arrays.asList(1, 2), Arrays.asList(2));
+
+        // test pairs of strats
+        new StratifiedEvalTestProvider(12,
+                Arrays.asList(4, 3, 2, 3),
+                Arrays.asList(1, 2), Arrays.asList(3, 4));
+
+        return StratifiedEvalTestProvider.getTests(StratifiedEvalTestProvider.class);
+    }
+
+    /**
+     * Ensures that counting and stratifications all are working properly by iterating
+     * over integers 1...cfg.N and stratify according to cfg, and that the counts in
+     * each bin are as expected.
+     *
+     * @param cfg
+     */
+    @Test(dataProvider = "StratifiedEvalTestProvider")
+    public void testBasicOperation(StratifiedEvalTestProvider cfg) {
+        initialize(cfg);
+        checkStratificationCountsAreExpected(VEwalker.stratManager, cfg.expectedCounts);
+    }
+
+    private final void checkStratificationCountsAreExpected(final StratificationManager<VariantStratifier, EvaluationContext> manager,
+                                                            final List<Integer> expectedCounts) {
+        for ( int key = 0; key < manager.size(); key++ ) {
+            final String stratStateString = manager.getStratsAndStatesStringForKey(key);
+            final EvaluationContext nec = manager.get(key);
+
+            for ( final VariantEvaluator ve : nec.getVariantEvaluators() ) {
+                // test for count here
+                final CounterEval counterEval = (CounterEval)ve;
+                final int expected = expectedCounts.get(key);
+                Assert.assertEquals(counterEval.count, expected, "Count seen of " + counterEval.count + " not expected " + expected + " at " + stratStateString);
+            }
+        }
+    }
+
+    /**
+     * A derived test on testBasicOperation that checks that combining stratifications
+     * works as expected by ensuring the results are the same when the remapped
+     * strats are the identity map (A -> A, B -> B, etc)
+     */
+    @Test(dataProvider = "StratifiedEvalTestProvider", dependsOnMethods = {"testBasicOperation"})
+    public void testIdentityCombine(StratifiedEvalTestProvider cfg) {
+        for ( int i = 0; i < cfg.stratificationObjects.size(); i++ ) {
+            initialize(cfg);
+            final VariantStratifier toReplace = cfg.stratificationObjects.get(i);
+            final VariantStratifier newStrat = cfg.stratificationObjects.get(i);
+            final Map<Object, Object> remappedStates = Utils.makeIdentityFunctionMap(newStrat.getAllStates());
+            StratificationManager<VariantStratifier, EvaluationContext> combined =
+                    VEwalker.stratManager.combineStrats(toReplace, newStrat, EvaluationContext.COMBINER, remappedStates);
+            checkStratificationCountsAreExpected(combined, cfg.expectedCounts);
+        }
+    }
+
+//    /**
+//     * A derived test on testBasicOperation that checks that combining stratifications
+//     * works as expected. We look into cfg, and if there are multiple states we create
+//     * dynamically create a combinations of the stratifications, and ensure that the
+//     * combined results are as we expected.
+//     */
+//    @Test(dataProvider = "StratifiedEvalTestProvider", dependsOnMethods = {"testBasicOperation"})
+//    public void testCombinedEachStrat(StratifiedEvalTestProvider cfg) {
+//        for ( int i = 0; i < cfg.stratificationObjects.size(); i++ ) {
+//            initialize(cfg);
+//            final VariantStratifier toReplace = cfg.stratificationObjects.get(i);
+//
+//            // TODO -- replace this code with something that combines values in strat
+//            final VariantStratifier newStrat = cfg.stratificationObjects.get(i);
+//            final Map<Object, Object> remappedStates = Utils.makeIdentityFunctionMap(newStrat.getAllStates());
+//            final List<Integer> expected = cfg.expectedCounts;
+//
+//            StratificationManager<VariantStratifier, EvaluationContext> combined =
+//                    VEwalker.stratManager.combineStrats(toReplace, newStrat, EvaluationContext.COMBINER, remappedStates);
+//            checkStratificationCountsAreExpected(combined, expected);
+//        }
+//    }
+}