Merge branch 'master' of ssh://copper.broadinstitute.org/humgen/gsa-scr1/gsa-engineering/git/unstable

build.xml

@@ -520,6 +520,8 @@
             <fileset dir="${java.classes}">
               <include name="**/utils/codecs/**/*.class"/>
               <include name="**/utils/variantcontext/**/*.class"/>
+              <include name="org/broadinstitute/sting/utils/exceptions/**"/>
+	      <include name="org/broadinstitute/sting/utils/help/DocumentedGATKFeature.class"/>
             </fileset>
         </jar>
     </target>

public/java/src/org/broadinstitute/sting/commandline/ArgumentDefinitions.java

@@ -174,7 +174,8 @@ public class ArgumentDefinitions implements Iterable<ArgumentDefinition> {
 
     static DefinitionMatcher VerifiableDefinitionMatcher = new DefinitionMatcher() {
         public boolean matches( ArgumentDefinition definition, Object key ) {
-            return definition.validation != null;
+            // We can perform some sort of validation for anything that isn't a flag.
+            return !definition.isFlag;
         }        
     };
 }

public/java/src/org/broadinstitute/sting/commandline/ArgumentMatch.java

@@ -44,7 +44,7 @@ public class ArgumentMatch implements Iterable<ArgumentMatch> {
     public final String label;
 
     /**
-     * Maps indicies of command line arguments to values paired with that argument.
+     * Maps indices of command line arguments to values paired with that argument.
      */
     public final SortedMap<Integer,List<String>> indices = new TreeMap<Integer,List<String>>();
 

public/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java

@@ -41,6 +41,11 @@ import java.util.*;
  * A parser for Sting command-line arguments.
  */
 public class ParsingEngine {
+    /**
+     * The loaded argument sources along with their back definitions.
+     */
+    private Map<ArgumentDefinition,ArgumentSource> argumentSourcesByDefinition = new HashMap<ArgumentDefinition,ArgumentSource>();
+
     /**
      * A list of defined arguments against which command lines are matched.
      * Package protected for testing access.
@@ -107,8 +112,13 @@ public class ParsingEngine {
      */
     public void addArgumentSource( String sourceName, Class sourceClass ) {
         List<ArgumentDefinition> argumentsFromSource = new ArrayList<ArgumentDefinition>();
-        for( ArgumentSource argumentSource: extractArgumentSources(sourceClass) )
+        for( ArgumentSource argumentSource: extractArgumentSources(sourceClass) ) {
-            argumentsFromSource.addAll( argumentSource.createArgumentDefinitions() );
+            List<ArgumentDefinition> argumentDefinitions = argumentSource.createArgumentDefinitions();
+            for(ArgumentDefinition argumentDefinition: argumentDefinitions) {
+                argumentSourcesByDefinition.put(argumentDefinition,argumentSource);
+                argumentsFromSource.add( argumentDefinition );
+            }
+        }
         argumentDefinitions.add( new ArgumentDefinitionGroup(sourceName, argumentsFromSource) );
     }
 
@@ -199,16 +209,25 @@ public class ParsingEngine {
                 throw new InvalidArgumentException( invalidArguments );
         }
 
-        // Find invalid argument values (arguments that fail the regexp test.
+        // Find invalid argument values -- invalid arguments are either completely missing or fail the specified 'validation' regular expression.
         if( !skipValidationOf.contains(ValidationType.InvalidArgumentValue) ) {
             Collection<ArgumentDefinition> verifiableArguments = 
                     argumentDefinitions.findArgumentDefinitions( null, ArgumentDefinitions.VerifiableDefinitionMatcher );
             Collection<Pair<ArgumentDefinition,String>> invalidValues = new ArrayList<Pair<ArgumentDefinition,String>>();
             for( ArgumentDefinition verifiableArgument: verifiableArguments ) {
                 ArgumentMatches verifiableMatches = argumentMatches.findMatches( verifiableArgument );
+                // Check to see whether an argument value was specified.  Argument values must be provided
+                // when the argument name is specified and the argument is not a flag type.
+                for(ArgumentMatch verifiableMatch: verifiableMatches) {
+                    ArgumentSource argumentSource = argumentSourcesByDefinition.get(verifiableArgument);
+                    if(verifiableMatch.values().size() == 0 && !verifiableArgument.isFlag && argumentSource.createsTypeDefault())
+                        invalidValues.add(new Pair<ArgumentDefinition,String>(verifiableArgument,null));
+                }
+
+                // Ensure that the field contents meet the validation criteria specified by the regular expression.
                 for( ArgumentMatch verifiableMatch: verifiableMatches ) {
                     for( String value: verifiableMatch.values() ) {
-                        if( !value.matches(verifiableArgument.validation) )
+                        if( verifiableArgument.validation != null && !value.matches(verifiableArgument.validation) )
                             invalidValues.add( new Pair<ArgumentDefinition,String>(verifiableArgument, value) );
                     }
                 }
@@ -515,10 +534,14 @@ class InvalidArgumentValueException extends ArgumentException {
     private static String formatArguments( Collection<Pair<ArgumentDefinition,String>> invalidArgumentValues ) {
         StringBuilder sb = new StringBuilder();
         for( Pair<ArgumentDefinition,String> invalidValue: invalidArgumentValues ) {
-            sb.append( String.format("%nArgument '--%s' has value of incorrect format: %s (should match %s)",
+            if(invalidValue.getSecond() == null)
-                                     invalidValue.first.fullName,
+                sb.append( String.format("%nArgument '--%s' requires a value but none was provided",
-                                     invalidValue.second,
+                                         invalidValue.first.fullName) );
-                                     invalidValue.first.validation) );
+            else
+                sb.append( String.format("%nArgument '--%s' has value of incorrect format: %s (should match %s)",
+                        invalidValue.first.fullName,
+                        invalidValue.second,
+                        invalidValue.first.validation) );
         }
         return sb.toString();
     }

public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java

@@ -893,6 +893,7 @@ public class SAMDataSource {
          * Custom representation of interval bounds.
          * Makes it simpler to track current position. 
          */
+        private int[] intervalContigIndices;
         private int[] intervalStarts;
         private int[] intervalEnds;
 
@@ -917,12 +918,14 @@ public class SAMDataSource {
             if(foundMappedIntervals) {
                 if(keepOnlyUnmappedReads)
                     throw new ReviewedStingException("Tried to apply IntervalOverlapFilteringIterator to a mixed of mapped and unmapped intervals.  Please apply this filter to only mapped or only unmapped reads");
+                this.intervalContigIndices = new int[intervals.size()];
                 this.intervalStarts = new int[intervals.size()];
                 this.intervalEnds = new int[intervals.size()];
                 int i = 0;
                 for(GenomeLoc interval: intervals) {
-                    intervalStarts[i] = (int)interval.getStart();
+                    intervalContigIndices[i] = interval.getContigIndex();
-                    intervalEnds[i] = (int)interval.getStop();
+                    intervalStarts[i] = interval.getStart();
+                    intervalEnds[i] = interval.getStop();
                     i++;
                 }
             }
@@ -961,11 +964,10 @@ public class SAMDataSource {
             while(nextRead == null && (keepOnlyUnmappedReads || currentBound < intervalStarts.length)) {
                 if(!keepOnlyUnmappedReads) {
                     // Mapped read filter; check against GenomeLoc-derived bounds.
-                    if(candidateRead.getAlignmentEnd() >= intervalStarts[currentBound] ||
+                    if(readEndsOnOrAfterStartingBound(candidateRead)) {
-                            (candidateRead.getReadUnmappedFlag() && candidateRead.getAlignmentStart() >= intervalStarts[currentBound])) {
+                        // This read ends after the current interval begins.
-                        // This read ends after the current interval begins (or, if unmapped, starts within the bounds of the interval.
                         // Promising, but this read must be checked against the ending bound.
-                        if(candidateRead.getAlignmentStart() <= intervalEnds[currentBound]) {
+                        if(readStartsOnOrBeforeEndingBound(candidateRead)) {
                             // Yes, this read is within both bounds.  This must be our next read.
                             nextRead = candidateRead;
                             break;
@@ -993,6 +995,37 @@ public class SAMDataSource {
                 candidateRead = iterator.next();
             }
         }
+
+        /**
+         * Check whether the read lies after the start of the current bound.  If the read is unmapped but placed, its
+         * end will be distorted, so rely only on the alignment start.
+         * @param read The read to position-check.
+         * @return True if the read starts after the current bounds.  False otherwise.
+         */
+        private boolean readEndsOnOrAfterStartingBound(final SAMRecord read) {
+            return
+                    // Read ends on a later contig, or...
+                    read.getReferenceIndex() > intervalContigIndices[currentBound] ||
+                    // Read ends of this contig...
+                    (read.getReferenceIndex() == intervalContigIndices[currentBound] &&
+                            // either after this location, or...
+                            (read.getAlignmentEnd() >= intervalStarts[currentBound] ||
+                            // read is unmapped but positioned and alignment start is on or after this start point.
+                            (read.getReadUnmappedFlag() && read.getAlignmentStart() >= intervalStarts[currentBound])));
+        }
+
+        /**
+         * Check whether the read lies before the end of the current bound.
+         * @param read The read to position-check.
+         * @return True if the read starts after the current bounds.  False otherwise.
+         */
+        private boolean readStartsOnOrBeforeEndingBound(final SAMRecord read) {
+            return
+                    // Read starts on a prior contig, or...
+                    read.getReferenceIndex() < intervalContigIndices[currentBound] ||
+                    // Read starts on this contig and the alignment start is registered before this end point.
+                   (read.getReferenceIndex() == intervalContigIndices[currentBound] && read.getAlignmentStart() <= intervalEnds[currentBound]);
+        }
     }
 
     /**

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalance.java

@@ -42,7 +42,7 @@ import java.util.List;
 import java.util.Map;
 
 
-public class AlleleBalance implements InfoFieldAnnotation {
+public class AlleleBalance extends InfoFieldAnnotation {
 
     public Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref, Map<String, AlignmentContext> stratifiedContexts, VariantContext vc) {
         if ( stratifiedContexts.size() == 0 )

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalanceBySample.java

@@ -15,7 +15,7 @@ import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 import java.util.*;
 
 
-public class AlleleBalanceBySample implements GenotypeAnnotation, ExperimentalAnnotation {
+public class AlleleBalanceBySample extends GenotypeAnnotation implements ExperimentalAnnotation {
 
     public Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext stratifiedContext, VariantContext vc, Genotype g) {
         Double ratio = annotateSNP(stratifiedContext, vc, g);

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AnnotationByDepth.java

@@ -8,7 +8,7 @@ import java.util.Map;
 
 
 
-public abstract class AnnotationByDepth implements InfoFieldAnnotation {
+public abstract class AnnotationByDepth extends InfoFieldAnnotation {
 
 
     protected int annotationByVariantDepth(final Map<String, Genotype> genotypes, Map<String, AlignmentContext> stratifiedContexts) {

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseCounts.java

@@ -46,7 +46,7 @@ import java.util.List;
 import java.util.Map;
 
 
-public class BaseCounts implements InfoFieldAnnotation {
+public class BaseCounts extends InfoFieldAnnotation {
 
     public Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref, Map<String, AlignmentContext> stratifiedContexts, VariantContext vc) {
         if ( stratifiedContexts.size() == 0 )

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ChromosomeCounts.java

@@ -43,7 +43,7 @@ import java.util.List;
 import java.util.Map;
 
 
-public class ChromosomeCounts implements InfoFieldAnnotation, StandardAnnotation {
+public class ChromosomeCounts extends InfoFieldAnnotation implements StandardAnnotation {
 
     private String[] keyNames = { VCFConstants.ALLELE_NUMBER_KEY, VCFConstants.ALLELE_COUNT_KEY, VCFConstants.ALLELE_FREQUENCY_KEY };
     private VCFInfoHeaderLine[] descriptions = { new VCFInfoHeaderLine(VCFConstants.ALLELE_FREQUENCY_KEY, VCFHeaderLineCount.A, VCFHeaderLineType.Float, "Allele Frequency, for each ALT allele, in the same order as listed"),

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthOfCoverage.java

@@ -16,7 +16,7 @@ import java.util.List;
 import java.util.Map;
 
 
-public class DepthOfCoverage implements InfoFieldAnnotation, StandardAnnotation {
+public class DepthOfCoverage extends InfoFieldAnnotation implements StandardAnnotation {
 
     public Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref, Map<String, AlignmentContext> stratifiedContexts, VariantContext vc) {
         if ( stratifiedContexts.size() == 0 )

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthPerAlleleBySample.java

@@ -22,7 +22,7 @@ import java.util.List;
 import java.util.Map;
 
 
-public class DepthPerAlleleBySample implements GenotypeAnnotation, StandardAnnotation {
+public class DepthPerAlleleBySample extends GenotypeAnnotation implements StandardAnnotation {
 
     private static String REF_ALLELE = "REF";
 

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java

@@ -42,7 +42,7 @@ import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 import java.util.*;
 
 
-public class FisherStrand implements InfoFieldAnnotation, StandardAnnotation {
+public class FisherStrand extends InfoFieldAnnotation implements StandardAnnotation {
     private static final String FS = "FS";
     private static final double MIN_PVALUE = 1E-320;
 

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/GCContent.java

@@ -16,7 +16,7 @@ import java.util.List;
 import java.util.Map;
 
 
-public class GCContent implements InfoFieldAnnotation, ExperimentalAnnotation {
+public class GCContent extends InfoFieldAnnotation implements ExperimentalAnnotation {
 
     public Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref, Map<String, AlignmentContext> stratifiedContexts, VariantContext vc) {
         double content = computeGCContent(ref);

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/GLstats.java

@@ -23,7 +23,7 @@ import java.util.Map;
  */
 
 // A set of annotations calculated directly from the GLs
-public class GLstats implements InfoFieldAnnotation, StandardAnnotation {
+public class GLstats extends InfoFieldAnnotation implements StandardAnnotation {
 
     private static final int MIN_SAMPLES = 10;
 

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java

@@ -48,7 +48,7 @@ import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
 import java.util.*;
 
-public class HaplotypeScore implements InfoFieldAnnotation, StandardAnnotation {
+public class HaplotypeScore extends InfoFieldAnnotation implements StandardAnnotation {
     private final static boolean DEBUG = false;
     private final static int MIN_CONTEXT_WING_SIZE = 10;
     private final static int MAX_CONSENSUS_HAPLOTYPES_TO_CONSIDER = 50;

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HardyWeinberg.java

@@ -18,7 +18,7 @@ import java.util.List;
 import java.util.Map;
 
 
-public class HardyWeinberg implements InfoFieldAnnotation, WorkInProgressAnnotation {
+public class HardyWeinberg extends InfoFieldAnnotation implements WorkInProgressAnnotation {
 
     private static final int MIN_SAMPLES = 10;
     private static final int MIN_GENOTYPE_QUALITY = 10;

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HomopolymerRun.java

@@ -16,7 +16,7 @@ import java.util.List;
 import java.util.Map;
 
 
-public class HomopolymerRun implements InfoFieldAnnotation, StandardAnnotation {
+public class HomopolymerRun extends InfoFieldAnnotation implements StandardAnnotation {
 
     private boolean ANNOTATE_INDELS = true;
 

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/IndelType.java

@@ -19,7 +19,7 @@ import java.util.*;
  * Time: 11:47:33 AM
  * To change this template use File | Settings | File Templates.
  */
-public class IndelType implements InfoFieldAnnotation, ExperimentalAnnotation {
+public class IndelType extends InfoFieldAnnotation implements ExperimentalAnnotation {
 
     public Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref, Map<String, AlignmentContext> stratifiedContexts, VariantContext vc) {
 

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/LowMQ.java

@@ -16,7 +16,7 @@ import java.util.List;
 import java.util.Map;
 
 
-public class LowMQ implements InfoFieldAnnotation {
+public class LowMQ extends InfoFieldAnnotation {
 
     public Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref, Map<String, AlignmentContext> stratifiedContexts, VariantContext vc) {
         if ( stratifiedContexts.size() == 0 )

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZero.java

@@ -18,7 +18,7 @@ import java.util.List;
 import java.util.Map;
 
 
-public class MappingQualityZero implements InfoFieldAnnotation, StandardAnnotation {
+public class MappingQualityZero extends InfoFieldAnnotation implements StandardAnnotation {
 
     public Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref, Map<String, AlignmentContext> stratifiedContexts, VariantContext vc) {
         if ( stratifiedContexts.size() == 0 )

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroBySample.java

@@ -49,7 +49,7 @@ import java.util.Map;
  * Time: 6:46:25 PM
  * To change this template use File | Settings | File Templates.
  */
-public class MappingQualityZeroBySample implements GenotypeAnnotation {
+public class MappingQualityZeroBySample extends GenotypeAnnotation {
     public Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref,
                                         AlignmentContext context, VariantContext vc, Genotype g) {
         if ( g == null || !g.isCalled() )

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroFraction.java

@@ -18,7 +18,7 @@ import java.util.Map;
 
 
 
-public class MappingQualityZeroFraction implements InfoFieldAnnotation, ExperimentalAnnotation {
+public class MappingQualityZeroFraction extends InfoFieldAnnotation implements ExperimentalAnnotation {
 
     public Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref, Map<String, AlignmentContext> stratifiedContexts, VariantContext vc) {
         if ( stratifiedContexts.size() == 0 )

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/NBaseCount.java

@@ -21,7 +21,7 @@ import java.util.Map;
  * Date: 5/16/11
  */
 
-public class NBaseCount implements InfoFieldAnnotation {
+public class NBaseCount extends InfoFieldAnnotation {
     public Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref, Map<String, AlignmentContext> stratifiedContexts, VariantContext vc) {
         if( stratifiedContexts.size() == 0 )
             return null;

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/QualByDepth.java

@@ -16,7 +16,7 @@ import java.util.List;
 import java.util.Map;
 
 
-public class QualByDepth extends AnnotationByDepth implements InfoFieldAnnotation, StandardAnnotation {
+public class QualByDepth extends AnnotationByDepth implements StandardAnnotation {
 
     public Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref, Map<String, AlignmentContext> stratifiedContexts, VariantContext vc) {
         if ( stratifiedContexts.size() == 0 )

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RMSMappingQuality.java

@@ -20,7 +20,7 @@ import java.util.List;
 import java.util.Map;
 
 
-public class RMSMappingQuality implements InfoFieldAnnotation, StandardAnnotation {
+public class RMSMappingQuality extends InfoFieldAnnotation implements StandardAnnotation {
 
     public Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref, Map<String, AlignmentContext> stratifiedContexts, VariantContext vc) {
         if ( stratifiedContexts.size() == 0 )

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java

@@ -21,7 +21,7 @@ import java.util.Map;
 
 
 
-public abstract class RankSumTest implements InfoFieldAnnotation, StandardAnnotation {
+public abstract class RankSumTest extends InfoFieldAnnotation implements StandardAnnotation {
     static final double INDEL_LIKELIHOOD_THRESH = 0.1;
     static final boolean DEBUG = false;
 

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadDepthAndAllelicFractionBySample.java

@@ -52,7 +52,7 @@ import java.util.Map;
  * Time: 3:59:27 PM
  * To change this template use File | Settings | File Templates.
  */
-public class ReadDepthAndAllelicFractionBySample implements GenotypeAnnotation {
+public class ReadDepthAndAllelicFractionBySample extends GenotypeAnnotation {
 
         private static String REF_ALLELE = "REF";
 

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SampleList.java

@@ -41,7 +41,7 @@ import java.util.List;
 import java.util.Map;
 
 
-public class SampleList implements InfoFieldAnnotation {
+public class SampleList extends InfoFieldAnnotation {
 
     public Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref, Map<String, AlignmentContext> stratifiedContexts, VariantContext vc) {
         if ( vc.isMonomorphic() || !vc.hasGenotypes() )

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SpanningDeletions.java

@@ -16,7 +16,7 @@ import java.util.List;
 import java.util.Map;
 
 
-public class SpanningDeletions implements InfoFieldAnnotation, StandardAnnotation {
+public class SpanningDeletions extends InfoFieldAnnotation implements StandardAnnotation {
 
     public Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref, Map<String, AlignmentContext> stratifiedContexts, VariantContext vc) {
         if ( stratifiedContexts.size() == 0 )

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TechnologyComposition.java

@@ -24,7 +24,7 @@ import java.util.Map;
  * Time: 3:14 PM
  * To change this template use File | Settings | File Templates.
  */
-public class TechnologyComposition implements ExperimentalAnnotation,InfoFieldAnnotation {
+public class TechnologyComposition extends InfoFieldAnnotation implements ExperimentalAnnotation {
     private String nSLX = "NumSLX";
     private String n454 ="Num454";
     private String nSolid = "NumSOLiD";

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/genomicannotator/GenomicAnnotation.java

@@ -48,7 +48,7 @@ import java.util.Map.Entry;
  *
  * For details, see:  http://www.broadinstitute.org/gsa/wiki/index.php/GenomicAnnotator
  */
-public class GenomicAnnotation implements InfoFieldAnnotation {
+public class GenomicAnnotation extends InfoFieldAnnotation {
 
     public static final String CHR_COLUMN = "chr";
     public static final String START_COLUMN = "start";

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/GenotypeAnnotation.java

@@ -10,15 +10,12 @@ import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 import java.util.List;
 import java.util.Map;
 
-public interface GenotypeAnnotation {
+public abstract class GenotypeAnnotation extends VariantAnnotatorAnnotation {
 
     // return annotations for the given contexts/genotype split by sample
-    public Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext stratifiedContext, VariantContext vc, Genotype g);
+    public abstract Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext stratifiedContext, VariantContext vc, Genotype g);
-
-    // return the FORMAT keys
-    public List<String> getKeyNames();
 
     // return the descriptions used for the VCF FORMAT meta field
-    public List<VCFFormatHeaderLine> getDescriptions();
+    public abstract List<VCFFormatHeaderLine> getDescriptions();
 
 }

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/InfoFieldAnnotation.java

@@ -3,21 +3,18 @@ package org.broadinstitute.sting.gatk.walkers.annotator.interfaces;
 import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
+import org.broadinstitute.sting.gatk.walkers.annotator.VariantAnnotator;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
+import org.broadinstitute.sting.utils.help.DocumentedGATKFeature;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
 import java.util.List;
 import java.util.Map;
 
-public interface InfoFieldAnnotation {
+public abstract class InfoFieldAnnotation extends VariantAnnotatorAnnotation {
-
     // return annotations for the given contexts split by sample
-    public Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref, Map<String, AlignmentContext> stratifiedContexts, VariantContext vc);
+    public abstract Map<String, Object> annotate(RefMetaDataTracker tracker, ReferenceContext ref, Map<String, AlignmentContext> stratifiedContexts, VariantContext vc);
-
-    // return the INFO keys
-    public List<String> getKeyNames();
 
     // return the descriptions used for the VCF INFO meta field
-    public List<VCFInfoHeaderLine> getDescriptions();
+    public abstract List<VCFInfoHeaderLine> getDescriptions();
-
 }

public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/VariantAnnotatorAnnotation.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.gatk.walkers.annotator.interfaces;
+
+import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
+import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
+import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
+import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine;
+import org.broadinstitute.sting.utils.help.DocumentedGATKFeature;
+import org.broadinstitute.sting.utils.variantcontext.VariantContext;
+
+import java.util.List;
+import java.util.Map;
+
+@DocumentedGATKFeature(enable = true, groupName = "VariantAnnotator annotations", summary = "VariantAnnotator annotations")
+public abstract class VariantAnnotatorAnnotation {
+    // return the INFO keys
+    public abstract List<String> getKeyNames();
+}

public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffEngine.java

@@ -342,12 +342,12 @@ public class DiffEngine {
             return reader.readFromFile(file, maxElementsToRead);
     }
 
-    public static boolean simpleDiffFiles(File masterFile, File testFile, DiffEngine.SummaryReportParams params) {
+    public static boolean simpleDiffFiles(File masterFile, File testFile, int maxElementsToRead, DiffEngine.SummaryReportParams params) {
         DiffEngine diffEngine = new DiffEngine();
 
         if ( diffEngine.canRead(masterFile) && diffEngine.canRead(testFile) ) {
-            DiffElement master = diffEngine.createDiffableFromFile(masterFile);
+            DiffElement master = diffEngine.createDiffableFromFile(masterFile, maxElementsToRead);
-            DiffElement test = diffEngine.createDiffableFromFile(testFile);
+            DiffElement test = diffEngine.createDiffableFromFile(testFile, maxElementsToRead);
             List<Difference> diffs = diffEngine.diff(master, test);
             diffEngine.reportSummarizedDifferences(diffs, params);
             return true;

public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java

@@ -57,7 +57,7 @@ public class GATKDoclet {
      * @throws java.io.IOException if output can't be written.
      */
     public static boolean start(RootDoc rootDoc) throws IOException {
-        logger.setLevel(Level.INFO);
+        logger.setLevel(Level.DEBUG);
         // load arguments
         for(String[] options: rootDoc.options()) {
             if(options[0].equals("-build-timestamp"))
@@ -95,6 +95,10 @@ public class GATKDoclet {
         for ( ClassDoc doc : rootDoc.classes() ) {
             logger.debug("Considering " + doc);
             Class clazz = getClassForClassDoc(doc);
+
+            if ( clazz != null && clazz.getName().equals("org.broadinstitute.sting.gatk.walkers.annotator.AlleleBalance"))
+                logger.debug("foo");
+
             DocumentedGATKFeature feature = getFeatureForClassDoc(doc);
             DocumentedGATKFeatureHandler handler = createHandler(doc, feature);
             if ( handler != null && handler.shouldBeProcessed(doc) ) {

public/java/test/org/broadinstitute/sting/MD5DB.java

@@ -47,6 +47,7 @@ public class MD5DB {
     /**
      * Subdirectory under the ant build directory where we store integration test md5 results
      */
+    private static final int MAX_RECORDS_TO_READ = 10000;
     public static final String LOCAL_MD5_DB_DIR = "integrationtests";
     public static final String GLOBAL_MD5_DB_DIR = "/humgen/gsa-hpprojects/GATK/data/integrationtests";
 
@@ -78,8 +79,8 @@ public class MD5DB {
      * @return
      */
     public static String getMD5FilePath(final String md5, final String valueIfNotFound) {
-        // we prefer the local db to the global DB, so match it first
+        // we prefer the global db to the local DB, so match it first
-        for ( String dir : Arrays.asList(LOCAL_MD5_DB_DIR, GLOBAL_MD5_DB_DIR)) {
+        for ( String dir : Arrays.asList(GLOBAL_MD5_DB_DIR, LOCAL_MD5_DB_DIR)) {
             File f = getFileForMD5(md5, dir);
             if ( f.exists() && f.canRead() )
                 return f.getPath();
@@ -232,7 +233,7 @@ public class MD5DB {
 
                     // inline differences
                     DiffEngine.SummaryReportParams params = new DiffEngine.SummaryReportParams(System.out, 20, 10, 0);
-                    boolean success = DiffEngine.simpleDiffFiles(new File(pathToExpectedMD5File), new File(pathToFileMD5File), params);
+                    boolean success = DiffEngine.simpleDiffFiles(new File(pathToExpectedMD5File), new File(pathToFileMD5File), MAX_RECORDS_TO_READ, params);
                     if ( success )
                         System.out.printf("Note that the above list is not comprehensive.  At most 20 lines of output, and 10 specific differences will be listed.  Please use -T DiffObjects -R public/testdata/exampleFASTA.fasta -m %s -t %s to explore the differences more freely%n",
                                 pathToExpectedMD5File, pathToFileMD5File);