Call out ROD types by there record type, instead of the codec type (which was clumsy). So instead of:

@Requires(value={},referenceMetaData=@RMD(name="eval",type= VCFCodec.class))

you'd say:

@Requires(value={},referenceMetaData=@RMD(name="eval",type= VCFRecord.class))

Which is more in-line with what was done before.  All instances in the existing codebase should be switched over.



git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@3457 348d0f76-0448-11de-a6fe-93d51630548a

java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java

@@ -589,7 +589,7 @@ public class GenomeAnalysisEngine {
         for (RMD required : allRequired) {
             boolean found = false;
             for (RMDTrack rod : rods) {
-                if (rod.matches(required.name(), required.type()))
+                if (rod.matchesNameAndRecordType(required.name(), required.type()))
                     found = true;
             }
             if (!found)

java/src/org/broadinstitute/sting/gatk/WalkerManager.java

@@ -198,7 +198,7 @@ public class WalkerManager extends PluginManager<Walker> {
             return true;
 
         for( RMD allowed: allowsDataSource.referenceMetaData() ) {
-            if( rod.matches(allowed.name(),allowed.type()) )
+            if( rod.matchesNameAndRecordType(allowed.name(),allowed.type()) )
                 return true;
         }
         return false;

java/src/org/broadinstitute/sting/gatk/refdata/features/annotator/AnnotatorInputTableCodec.java

@@ -64,6 +64,11 @@ public class AnnotatorInputTableCodec implements FeatureCodec<AnnotatorInputTabl
         return lineCounter[0];
     }
 
+    @Override
+    public Class<AnnotatorInputTableFeature> getFeatureType() {
+        return AnnotatorInputTableFeature.class;
+    }
+
 
     /**
      * Parses the line into an AnnotatorInputTableFeature object.

java/src/org/broadinstitute/sting/gatk/refdata/features/sampileup/SAMPileupCodec.java

@@ -66,6 +66,11 @@ public class SAMPileupCodec implements FeatureCodec<SAMPileupFeature> {
         return 0;
     }
 
+    @Override
+    public Class<SAMPileupFeature> getFeatureType() {
+        return SAMPileupFeature.class;
+    }
+
     public SAMPileupFeature decode(String line) {
 //       0          1             2         3                  4         5            6         7
 //*     chrX     466           T         Y                170      170       88      32 ... (piles of read bases  and quals follow)

java/src/org/broadinstitute/sting/gatk/refdata/features/samread/SAMReadCodec.java

@@ -56,6 +56,11 @@ public class SAMReadCodec implements FeatureCodec<SAMReadFeature> {
         return 0;
     }
 
+    @Override
+    public Class<SAMReadFeature> getFeatureType() {
+        return SAMReadFeature.class;
+    }
+
     /**
      * Decode a single line in a SAM text file.
      * @param line line to decode.

java/src/org/broadinstitute/sting/gatk/refdata/tracks/FeatureReaderTrack.java

@@ -51,12 +51,13 @@ public class FeatureReaderTrack extends RMDTrack implements QueryableTrack {
      * Create a track
      *
      * @param type the type of track, used for track lookup
+     * @param recordType the type of record we produce
      * @param name the name of this specific track
      * @param file the associated file, for reference or recreating the reader
      * @param reader the feature reader to use as the underlying data source
      */
-    public FeatureReaderTrack(Class type, String name, File file, FeatureReader reader) {
+    public FeatureReaderTrack(Class type, Class recordType, String name, File file, FeatureReader reader) {
-        super(type, name, file);
+        super(type, recordType, name, file);
         this.reader = reader;
     }
 

java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrack.java

@@ -42,6 +42,7 @@ public abstract class RMDTrack {
 
     // the basics of a track:
     private final Class type;           // our type
+    private final Class recordType;     // the underlying records that are produced by this track
     private final String name;          // the name
     private final File file;            // the associated file we create the reader from
 
@@ -49,11 +50,13 @@ public abstract class RMDTrack {
      * Create a track
      *
      * @param type the type of track, used for track lookup
+     * @param recordType the type of record produced
      * @param name the name of this specific track
      * @param file the associated file, for reference or recreating the reader
      */
-    protected RMDTrack(Class type, String name, File file) {
+    protected RMDTrack(Class type, Class recordType, String name, File file) {
         this.type = type;
+        this.recordType = recordType;
         this.name = name;
         this.file = file;
     }
@@ -70,6 +73,10 @@ public abstract class RMDTrack {
         return file;
     }
 
+    public Class getRecordType() {
+        return recordType;
+    }
+
     /**
      * @return how to get an iterator of the underlying data.  This is all a track has to support,
      *         but other more advanced tracks support the query interface
@@ -77,14 +84,26 @@ public abstract class RMDTrack {
     public abstract CloseableIterator<GATKFeature> getIterator();
 
     /**
-     * helper function for determining if we are the same track
+     * helper function for determining if we are the same track based on name and codec type
      *
      * @param name the name to match
      * @param type the type to match
      *
      * @return true on a match, false if the name or type is different
      */
-    public boolean matches(String name, Type type) {
+    public boolean matchesNameAndType(String name, Type type) {
+        return (name.equals(this.name) && (type.getClass().isAssignableFrom(this.type.getClass())));
+    }
+
+    /**
+     * helper function for determining if we are the same track based on name and record type
+     *
+     * @param name the name to match
+     * @param type the type to match
+     *
+     * @return true on a match, false if the name or type is different
+     */
+    public boolean matchesNameAndRecordType(String name, Type type) {
         return (name.equals(this.name) && (type.getClass().isAssignableFrom(this.type.getClass())));
     }
 

java/src/org/broadinstitute/sting/gatk/refdata/tracks/RODRMDTrack.java

@@ -56,7 +56,7 @@ public class RODRMDTrack extends RMDTrack {
      * @param data the ROD to use as the underlying data source for this track
      */
     public RODRMDTrack(Class type, String name, File file, ReferenceOrderedData data) {
-        super(type, name, file);
+        super(type, type, name, file); // the decoder type and the record type are the same in this case
         this.data = data;
     }
 

java/src/org/broadinstitute/sting/gatk/refdata/tracks/builders/TribbleRMDTrackBuilder.java

@@ -94,7 +94,7 @@ public class TribbleRMDTrackBuilder extends PluginManager<FeatureCodec> implemen
     @Override
     public RMDTrack createInstanceOfTrack(Class targetClass, String name, File inputFile) throws RMDTrackCreationException {
         // return a feature reader track
-        return new FeatureReaderTrack(targetClass, name, inputFile, createFeatureReader(targetClass, inputFile));
+        return new FeatureReaderTrack(targetClass, this.createByType(targetClass).getFeatureType(), name, inputFile, createFeatureReader(targetClass, inputFile));
     }
 
     /**
@@ -221,19 +221,3 @@ public class TribbleRMDTrackBuilder extends PluginManager<FeatureCodec> implemen
         }
     }
 }
-
-/**
- * a fake Tribble track, used to test out the Tribble interface and feature codec detection
- */
-class FakeTribbleTrack implements FeatureCodec {
-
-    @Override
-    public Feature decode(String s) {
-        return null;
-    }
-
-    @Override
-    public int readHeader(LineReader reader) {
-        return 0; // the basics
-    }
-}

java/src/org/broadinstitute/sting/gatk/walkers/vcf/FilterLiftedVCF.java

@@ -38,7 +38,7 @@ import java.util.List;
 /**
  * Filters a lifted-over VCF file for ref bases that have been changed.
  */
-@Requires(value={},referenceMetaData=@RMD(name="vcf",type= VCFCodec.class))
+@Requires(value={},referenceMetaData=@RMD(name="vcf",type= VCFRecord.class))
 public class FilterLiftedVCF extends RodWalker<Integer, Integer> {
 
     private VCFWriter writer;

java/src/org/broadinstitute/sting/gatk/walkers/vcf/LiftoverVCF.java

@@ -45,7 +45,7 @@ import net.sf.samtools.SAMFileReader;
 /**
  * Lifts a VCF file over from one build to another.  Note that the resulting VCF could be mis-sorted.
  */
-@Requires(value={},referenceMetaData=@RMD(name="vcf",type= VCFCodec.class))
+@Requires(value={},referenceMetaData=@RMD(name="vcf",type= VCFRecord.class))
 public class LiftoverVCF extends RodWalker<Integer, Integer> {
 
     @Argument(fullName="chain", shortName="chain", doc="Chain file", required=true)

java/src/org/broadinstitute/sting/oneoffprojects/walkers/AlleleBalanceHistogramWalker.java

@@ -22,7 +22,7 @@ import java.util.*;
  * Time: 3:25:11 PM
  * To change this template use File | Settings | File Templates.
  */
-@Requires(value= DataSource.REFERENCE,referenceMetaData = {@RMD(name="variants",type= VCFCodec.class)})
+@Requires(value= DataSource.REFERENCE,referenceMetaData = {@RMD(name="variants",type=VCFRecord.class)})
 public class AlleleBalanceHistogramWalker extends LocusWalker<Map<String,Double>, Map<String,Set<Double>>> {
 
 

java/src/org/broadinstitute/sting/oneoffprojects/walkers/varianteval/multisample/MultiSampleConcordanceWalker.java

@@ -42,7 +42,7 @@ import org.broadinstitute.sting.commandline.Argument;
  * a VCF binding with the name 'variants'.
  * @Author: Chris Hartl
  */
-@Requires(value= DataSource.REFERENCE,referenceMetaData = {@RMD(name="truth",type= VCFCodec.class),@RMD(name="variants",type= VCFRecord.class)})
+@Requires(value= DataSource.REFERENCE,referenceMetaData = {@RMD(name="truth",type= VCFRecord.class),@RMD(name="variants",type= VCFRecord.class)})
 public class MultiSampleConcordanceWalker extends RodWalker< LocusConcordanceInfo, MultiSampleConcordanceSet > {
     @Argument(fullName="noLowDepthLoci", shortName="NLD", doc="Do not use loci in analysis where the variant depth (as specified in the VCF) is less than the given number; "+
             "DO NOT USE THIS IF YOUR VCF DOES NOT HAVE 'DP' IN THE FORMAT FIELD", required=false) private int minDepth = -1;

java/src/org/broadinstitute/sting/playground/gatk/walkers/diagnostics/SNPDensity.java

@@ -41,7 +41,7 @@ import org.broadinstitute.sting.commandline.Argument;
  * Computes the density of SNPs passing and failing filters in intervals on the genome and emits a table for display
  */
 @By(DataSource.REFERENCE)
-@Requires(value={},referenceMetaData=@RMD(name="eval",type= VCFCodec.class))
+@Requires(value={},referenceMetaData=@RMD(name="eval",type= VCFRecord.class))
 public class SNPDensity extends RefWalker<Pair<VariantContext, GenomeLoc>, SNPDensity.Counter> {
     @Argument(fullName="granularity", shortName="granularity", doc="", required=false)
     private int granularity = 1000000;

java/src/org/broadinstitute/sting/playground/gatk/walkers/validation/RodSystemValidationWalker.java

@@ -1,12 +1,17 @@
 package org.broadinstitute.sting.playground.gatk.walkers.validation;
 
+import org.broad.tribble.dbsnp.DbSNPFeature;
+import org.broad.tribble.vcf.VCFCodec;
+import org.broad.tribble.vcf.VCFRecord;
+import org.broadinstitute.sting.commandline.Argument;
 import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
 import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
+import org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContext;
 import org.broadinstitute.sting.gatk.datasources.simpleDataSources.ReferenceOrderedDataSource;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.refdata.utils.GATKFeature;
-import org.broadinstitute.sting.gatk.walkers.RodWalker;
+import org.broadinstitute.sting.gatk.walkers.*;
 import org.broadinstitute.sting.utils.StingException;
 
 import java.io.*;
@@ -19,14 +24,21 @@ import java.util.List;
 /**
  * a walker for validating (in the style of validating pile-up) the ROD system.
  */
+@Reference(window=@Window(start=-40,stop=40))
 public class RodSystemValidationWalker extends RodWalker<Integer,Integer> {
 
     // the divider to use in some of the text output
     private static final String DIVIDER = ",";
 
+    @Argument(fullName="PerLocusEqual",required=false,doc="Should we check that all records at the same site produce equivilent variant contexts")
+    public boolean allRecordsVariantContextEquivalent = false;
+    
     // used to calculate the MD5 of a file
     MessageDigest digest = null;
 
+    // we sometimes need to know what rods the engine's seen
+    List<ReferenceOrderedDataSource> rodList;
+
     /**
      * emit the md5 sums for each of the input ROD files (will save up a lot of time if and when the ROD files change
      * underneath us).
@@ -40,7 +52,7 @@ public class RodSystemValidationWalker extends RodWalker<Integer,Integer> {
         }
         out.println("Header:");
         // enumerate the list of ROD's we've loaded
-        List<ReferenceOrderedDataSource> rodList = GenomeAnalysisEngine.instance.getRodDataSources();
+        rodList = GenomeAnalysisEngine.instance.getRodDataSources();
         for (ReferenceOrderedDataSource rod : rodList) {
             out.println(rod.getName() + DIVIDER + rod.getReferenceOrderedData().getType());
             out.println(rod.getName() + DIVIDER + rod.getReferenceOrderedData().getFile());
@@ -54,19 +66,29 @@ public class RodSystemValidationWalker extends RodWalker<Integer,Integer> {
      * @param tracker the ref meta data tracker to get RODs
      * @param ref reference context
      * @param context the reads
-     * @return an 1 for each site with a rod, 0 otherwise
+     * @return an 1 for each site with a rod(s), 0 otherwise
      */
     @Override
     public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
+        int ret = 0;
         if (tracker != null && tracker.getAllRods().size() > 0) {
             out.print(context.getLocation() + DIVIDER);
             Collection<GATKFeature> features = tracker.getAllRods();
             for (GATKFeature feat : features)
                 out.print(feat.getName() + DIVIDER);
             out.println(";");
-            return 1;
+            ret++;
         }
-        return 0;
+
+        // if the argument was set, check for equivalence
+        if (allRecordsVariantContextEquivalent && tracker != null) {
+            Collection<VariantContext> col = tracker.getAllVariantContexts(ref);
+            VariantContext con = null;
+            for (VariantContext contextInList : col)
+                if (con == null) con = contextInList;
+                else if (!con.equals(col)) out.println("FAIL: context " + col + " doesn't match " + con);
+        }
+        return ret;
     }
 
     /**

java/src/org/broadinstitute/sting/playground/gatk/walkers/variantoptimizer/ApplyVariantClustersWalker.java

@@ -135,7 +135,7 @@ public class ApplyVariantClustersWalker extends RodWalker<ExpandingArrayList<Var
         final List<ReferenceOrderedDataSource> dataSources = this.getToolkit().getRodDataSources();
         for( final ReferenceOrderedDataSource source : dataSources ) {
             final RMDTrack rod = source.getReferenceOrderedData();
-            if( rod.getType().equals(VCFCodec.class) ) {
+            if( rod.getRecordType().equals(VCFRecord.class) ) {
                 final VCFReader reader = new VCFReader(rod.getFile());
                 final Set<String> vcfSamples = reader.getHeader().getGenotypeSamples();
                 samples.addAll(vcfSamples);

java/src/org/broadinstitute/sting/playground/gatk/walkers/vcftools/VCFSelectWalker.java

@@ -45,7 +45,7 @@ import java.util.*;
 /**
  * Selects variant calls for output from a user-supplied VCF file using a number of user-selectable, parameterizable criteria.
  */
-@Requires(value={},referenceMetaData=@RMD(name="variant",type= VCFCodec.class))
+@Requires(value={},referenceMetaData=@RMD(name="variant",type= VCFRecord.class))
 public class VCFSelectWalker extends RodWalker<Integer, Integer> {
     @Argument(fullName="match", shortName="match", doc="Expression used with INFO fields to select VCF records for inclusion in the output VCF(see wiki docs for more info)", required=false)
     protected String[] MATCH_STRINGS = new String[]{null};

java/src/org/broadinstitute/sting/utils/SampleUtils.java

@@ -80,7 +80,7 @@ public class SampleUtils {
         List<ReferenceOrderedDataSource> dataSources = toolkit.getRodDataSources();
         for ( ReferenceOrderedDataSource source : dataSources ) {
             RMDTrack rod = source.getReferenceOrderedData();
-            if ( rod.getType().equals(VCFCodec.class) ) {
+            if ( rod.getRecordType().equals(VCFRecord.class) ) {
                 VCFReader reader = new VCFReader(rod.getFile());
                 samples.addAll(reader.getHeader().getGenotypeSamples());
                 reader.close();
@@ -109,7 +109,7 @@ public class SampleUtils {
         List<ReferenceOrderedDataSource> dataSources = toolkit.getRodDataSources();
         for ( ReferenceOrderedDataSource source : dataSources ) {
             RMDTrack rod = source.getReferenceOrderedData();
-            if ( rod.getType().equals(VCFCodec.class) ) {
+            if ( rod.getRecordType().equals(VCFRecord.class) ) {
                 VCFReader reader = new VCFReader(rod.getFile());
                 Set<String> vcfSamples = reader.getHeader().getGenotypeSamples();
                 for ( String sample : vcfSamples )

java/src/org/broadinstitute/sting/utils/genotype/vcf/VCFUtils.java

@@ -49,7 +49,7 @@ public class VCFUtils {
 
         for ( ReferenceOrderedDataSource source : toolkit.getRodDataSources() ) {
             RMDTrack rod = source.getReferenceOrderedData();
-            if ( rod.getType().equals(VCFCodec.class) ) {
+            if ( rod.getRecordType().equals(VCFRecord.class) ) {
                 vcfs.add(rod);
             }
         }
@@ -73,7 +73,7 @@ public class VCFUtils {
         List<ReferenceOrderedDataSource> dataSources = toolkit.getRodDataSources();
         for ( ReferenceOrderedDataSource source : dataSources ) {
             RMDTrack rod = source.getReferenceOrderedData();
-            if ( rod.getType().equals(VCFCodec.class) ) {
+            if ( rod.getRecordType().equals(VCFRecord.class) ) {
                 VCFReader reader = new VCFReader(rod.getFile());
                 fields.addAll(reader.getHeader().getMetaData());
                 reader.close();

java/test/org/broadinstitute/sting/playground/gatk/walkers/validation/RodSystemValidationIntegrationTest.java

@@ -35,9 +35,9 @@ public class RodSystemValidationIntegrationTest extends WalkerTest {
     public void testComplexVCFPileup() {
         WalkerTestSpec spec = new WalkerTestSpec(
                 baseTestString1KG() + " -B eval,VCF," + validationDataLocation + "MultiSample.vcf" +
-                " -B eval,VCF," + validationDataLocation + "NA12878.chr1_10mb_11mb.slx.indels.vcf"
+                " -B eval2,VCF," + validationDataLocation + "NA12878.chr1_10mb_11mb.slx.indels.vcf"
                 , 1,
-                Arrays.asList("6dd0ed0a6fe7096ccb66beffb8d455da"));
+                Arrays.asList("0c8c2b705d23f8fe6e7827a3b474736a"));
         executeTest("testComplexVCFPileup", spec);
     }
 
@@ -45,10 +45,22 @@ public class RodSystemValidationIntegrationTest extends WalkerTest {
     public void testLargeComplexVCFPileup() {
         WalkerTestSpec spec = new WalkerTestSpec(
                 baseTestString1KG() + " -B eval,VCF," + validationDataLocation + "MultiSample.vcf" +
-                " -B eval,VCF," + validationDataLocation + "NA12878.chr1_10mb_11mb.slx.indels.vcf" +
+                " -B eval2,VCF," + validationDataLocation + "CEU_hapmap_nogt_23.vcf" +
-                " -B eval,VCF," + validationDataLocation + "CEU_hapmap_nogt_23.vcf" +
+                " -B eval3,VCF," + validationDataLocation + "CEU_hapmap_nogt_23.vcf" +
                 " -L 1 -L 2 -L 20"
                 , 1,
+                Arrays.asList("8805912af2c38ec8d1cbc8d82532725e"));
+        executeTest("testLargeComplexVCFPileup", spec);
+    }
+
+    //@Test
+    public void testBlockZippedVrsUnzippedVCF1() {
+        final String vcfName = validationDataLocation + "bgzipped_vcfs/vcfexample.vcf";
+        WalkerTestSpec spec = new WalkerTestSpec(
+                baseTestString1KG() + " -B eval,VCF," + vcfName +
+                " -B eval2,VCF," + vcfName + ".gz" +
+                " --PerLocusEqual"
+                , 1,
                 Arrays.asList("ab3da32eae65e8c15a9f4a787a190a37"));
         executeTest("testLargeComplexVCFPileup", spec);
     }